Skip to content

Commit a01e480

Browse files
muskan-crestmuskan-crest
committed
Resolving comment
1. Remove the -24h logic in data collection. 2. Add hide sensitive mask to few more fields.
1 parent ed00556 commit a01e480

11 files changed

Lines changed: 143 additions & 74 deletions

File tree

‎packages/spycloud/data_stream/breach_catalog/agent/stream/cel.yml.hbs‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ program: |
2626
?"since": (state.?want_more.orValue(false) || !has(state.?cursor.last_published_date)) ?
2727
optional.none()
2828
:
29-
optional.of([(timestamp(state.cursor.last_published_date) - duration("24h")).format("2006-01-02")]),
29+
optional.of([timestamp(state.cursor.last_published_date).format("2006-01-02")]),
3030
?"cursor": state.?want_more.orValue(false) ? optional.of([state.pagination_token]) : optional.none(),
3131
}.format_query()
3232
).with({
@@ -58,7 +58,12 @@ program: |
5858
"error": {
5959
"code": string(resp.StatusCode),
6060
"id": string(resp.Status),
61-
"message": string(resp.Body),
61+
"message": "GET:"+(
62+
size(resp.Body) != 0 ?
63+
string(resp.Body)
64+
:
65+
string(resp.Status) + ' (' + string(resp.StatusCode) + ')'
66+
),
6267
},
6368
},
6469
"want_more": false,

‎packages/spycloud/data_stream/breach_catalog/sample_event.json‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,30 @@
11
{
22
"@timestamp": "2022-11-24T00:00:00.000Z",
33
"agent": {
4-
"ephemeral_id": "90dc28d2-8a37-4b01-b145-4ee5f9207fcf",
5-
"id": "401b5614-f720-4272-afd1-fdd38e876355",
4+
"ephemeral_id": "d6e9d6f0-0baa-44b6-b60b-e4b811e50811",
5+
"id": "763d7558-f93b-440a-94ee-509804901acf",
66
"name": "docker-fleet-agent",
77
"type": "filebeat",
88
"version": "8.13.0"
99
},
1010
"data_stream": {
1111
"dataset": "spycloud.breach_catalog",
12-
"namespace": "88948",
12+
"namespace": "63685",
1313
"type": "logs"
1414
},
1515
"ecs": {
1616
"version": "8.11.0"
1717
},
1818
"elastic_agent": {
19-
"id": "401b5614-f720-4272-afd1-fdd38e876355",
19+
"id": "763d7558-f93b-440a-94ee-509804901acf",
2020
"snapshot": false,
2121
"version": "8.13.0"
2222
},
2323
"event": {
2424
"agent_id_status": "verified",
2525
"dataset": "spycloud.breach_catalog",
2626
"id": "39997",
27-
"ingested": "2024-08-09T09:31:23Z",
27+
"ingested": "2024-08-16T06:23:24Z",
2828
"kind": "asset",
2929
"original": "{\"acquisition_date\":\"2022-10-14T00:00:00Z\",\"assets\":{\"address_2\":363,\"age\":817,\"city\":1859,\"country\":177225,\"country_code\":177225,\"dob\":198,\"email\":177219,\"first_name\":177114,\"full_name\":177030,\"gender\":119505,\"industry\":162612,\"job_title\":160712,\"last_name\":177099,\"middle_name\":17749,\"phone\":511,\"postal_code\":1971,\"social_facebook\":51841,\"social_twitter\":57193},\"confidence\":3,\"description\":\"This source has been marked as sensitive due to one of the following reasons: Revealing the source may compromise an on-going investigation. The affected site is of a controversial nature but does not validate email addresses and could therefore be used to tarnish an employee's reputation.\",\"id\":39997,\"num_records\":177225,\"spycloud_publish_date\":\"2022-11-24T00:00:00Z\",\"title\":\"Sensitive Source\",\"type\":\"PRIVATE\",\"uuid\":\"9f5bf34b-092e-46f4-b87f-02c91b0adb3a\"}"
3030
},

‎packages/spycloud/data_stream/breach_record/_dev/test/system/test-default-config.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,5 +10,6 @@ data_stream:
1010
interval: 24h
1111
preserve_original_event: true
1212
preserve_duplicate_custom_fields: true
13+
hide_sensitive: true
1314
assert:
1415
hit_count: 1

‎packages/spycloud/data_stream/breach_record/agent/stream/cel.yml.hbs‎

Lines changed: 43 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -32,9 +32,9 @@ program: |
3232
:
3333
(state.?cursor.last_published_date).as(date,
3434
optional.of([(date.hasValue() ?
35-
timestamp(state.cursor.last_published_date) - duration("24h")
35+
timestamp(state.cursor.last_published_date)
3636
:
37-
now - (duration(state.initial_interval) + duration("24h"))
37+
now - duration(state.initial_interval)
3838
).format("2006-01-02")])
3939
)),
4040
?"cursor": state.?want_more.orValue(false) ? optional.of([state.pagination_token_since]) : optional.none(),
@@ -45,40 +45,44 @@ program: |
4545
"User-Agent": [useragent],
4646
}
4747
}).do_request().as(resp, resp.StatusCode == 200 ?
48-
bytes(resp.Body).decode_json().as(first_body,
49-
{
50-
"events": has(first_body.results) ?
51-
(
52-
first_body.results.size() > 0 ?
53-
first_body.results.map(e, {"message": e.encode_json()})
54-
:
55-
[{"message":"No hits for Breach Catalog using since filter"}]
56-
)
48+
bytes(resp.Body).decode_json().as(first_body, {
49+
"events": has(first_body.results) ?
50+
(
51+
first_body.results.size() > 0 ?
52+
first_body.results.map(e, {"message": e.encode_json()})
5753
:
58-
[{"message": first_body.encode_json()}],
59-
"cursor": {
60-
?"last_published_date": (has(first_body.results) && first_body.results.size() > 0) ?
61-
optional.of(string(([?state.?cursor.last_published_date] + first_body.results.map(e, e.spycloud_publish_date)).map(t, timestamp(t)).max()))
62-
:
63-
state.?cursor.last_published_date,
64-
?"last_modified_date": state.?cursor.last_modified_date,
65-
},
66-
?"pagination_token_since": first_body.?cursor,
67-
"since_data": first_body.?cursor.orValue("") != "",
68-
"want_more": first_body.?cursor.orValue("") != "" || state.?since_modified_data.orValue(false),
69-
"api_key": state.api_key,
70-
"since_modified_data": state.since_modified_data,
71-
?"pagination_token_since_modified": state.?pagination_token_since_modified,
72-
"initial_interval": state.initial_interval,
73-
"severity": state.severity,
74-
})
75-
:
76-
{
54+
[{"message":"No hits for Breach Catalog using since filter"}]
55+
)
56+
:
57+
[{"message": first_body.encode_json()}],
58+
"cursor": {
59+
?"last_published_date": (has(first_body.results) && first_body.results.size() > 0) ?
60+
optional.of(string(([?state.?cursor.last_published_date] + first_body.results.map(e, e.spycloud_publish_date)).map(t, timestamp(t)).max()))
61+
:
62+
state.?cursor.last_published_date,
63+
?"last_modified_date": state.?cursor.last_modified_date,
64+
},
65+
?"pagination_token_since": first_body.?cursor,
66+
"since_data": first_body.?cursor.orValue("") != "",
67+
"want_more": first_body.?cursor.orValue("") != "" || state.?since_modified_data.orValue(false),
68+
"api_key": state.api_key,
69+
"since_modified_data": state.since_modified_data,
70+
?"pagination_token_since_modified": state.?pagination_token_since_modified,
71+
"initial_interval": state.initial_interval,
72+
"severity": state.severity,
73+
})
74+
:
75+
{
7776
"events": {
7877
"error": {
7978
"code": string(resp.StatusCode),
8079
"id": string(resp.Status),
81-
"message": string(resp.Body),
80+
"message": "GET:"+(
81+
size(resp.Body) != 0 ?
82+
string(resp.Body)
83+
:
84+
string(resp.Status) + ' (' + string(resp.StatusCode) + ')'
85+
),
8286
}
8387
},
8488
"api_key": state.api_key,
@@ -97,9 +101,9 @@ program: |
97101
state.?pagination_token_since_modified.orValue("") != "" ? optional.none() :
98102
(state.?cursor.last_modified_date).as(date,
99103
optional.of([(date.hasValue() ?
100-
timestamp(state.cursor.last_modified_date) - duration("24h")
104+
timestamp(state.cursor.last_modified_date)
101105
:
102-
now - (duration(state.initial_interval) + duration("24h"))
106+
now - duration(state.initial_interval)
103107
).format("2006-01-02")])
104108
)
105109
),
@@ -143,7 +147,12 @@ program: |
143147
"error": {
144148
"code": string(response.StatusCode),
145149
"id": string(response.Status),
146-
"message": string(response.Body),
150+
"message": "GET:"+(
151+
size(response.Body) != 0 ?
152+
string(response.Body)
153+
:
154+
string(response.Status) + ' (' + string(response.StatusCode) + ')'
155+
),
147156
}
148157
},
149158
"api_key": state.api_key,

‎packages/spycloud/data_stream/breach_record/elasticsearch/ingest_pipeline/default.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -324,6 +324,12 @@ processors:
324324
tag: rename_password_plaintext
325325
target_field: spycloud.breach_record.password.plaintext
326326
ignore_missing: true
327+
- set:
328+
field: spycloud.breach_record.password.plaintext
329+
tag: mask_password_plaintext
330+
value: 'REDACTED'
331+
if: ctx.tags != null && ctx.tags.contains('hide_sensitive') && ctx.spycloud?.breach_record?.password?.plaintext != null
332+
ignore_empty_value: true
327333
- rename:
328334
field: json.password_type
329335
tag: rename_password_type

‎packages/spycloud/data_stream/breach_record/sample_event.json‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,15 @@
11
{
22
"@timestamp": "2023-11-29T00:00:00.000Z",
33
"agent": {
4-
"ephemeral_id": "cd4912d6-0935-4cbf-ae4b-8e9fbac4dbad",
5-
"id": "401b5614-f720-4272-afd1-fdd38e876355",
4+
"ephemeral_id": "ce02cb91-e24e-45fa-8591-70b683f1b86b",
5+
"id": "763d7558-f93b-440a-94ee-509804901acf",
66
"name": "docker-fleet-agent",
77
"type": "filebeat",
88
"version": "8.13.0"
99
},
1010
"data_stream": {
1111
"dataset": "spycloud.breach_record",
12-
"namespace": "30900",
12+
"namespace": "33260",
1313
"type": "logs"
1414
},
1515
"destination": {
@@ -20,15 +20,15 @@
2020
"version": "8.11.0"
2121
},
2222
"elastic_agent": {
23-
"id": "401b5614-f720-4272-afd1-fdd38e876355",
23+
"id": "763d7558-f93b-440a-94ee-509804901acf",
2424
"snapshot": false,
2525
"version": "8.13.0"
2626
},
2727
"event": {
2828
"agent_id_status": "verified",
2929
"dataset": "spycloud.breach_record",
3030
"id": "3350f1da-fa39-4415-b2cc-02057e2fbe99",
31-
"ingested": "2024-08-09T09:33:05Z",
31+
"ingested": "2024-08-16T06:24:19Z",
3232
"kind": "alert",
3333
"original": "{\"account_image_url\":\"https://www.chess.com/bundles/web/images/noavatar_l.84a92436.gif\",\"account_login_time\":\"2018-06-29T23:51:46Z\",\"account_modification_time\":\"2018-06-29T23:51:10Z\",\"account_signup_time\":\"2016-07-29T18:47:11Z\",\"av_softwares\":[\"McAfee\",\"Windows Defender\"],\"cc_bin\":\"489486\",\"cc_expiration\":\"06/2025\",\"cc_last_four\":\"1237\",\"cc_number\":\"3fdd0ce028ffaa147afdb6461f6ce95f8c07f484\",\"company_name\":\"ABC Corporation\",\"country\":\"United States\",\"country_code\":\"US\",\"display_resolution\":\"1920x1080\",\"document_id\":\"3350f1da-fa39-4415-b2cc-02057e2fbe99\",\"domain\":\"example1.com\",\"email\":\"john.doe@example.com\",\"email_domain\":\"example.com\",\"email_username\":\"john.doe\",\"first_name\":\"John\",\"full_name\":\"John Doe\",\"homepage\":\"https://www.chess.com/member/sarahjoh\",\"industry\":\"Technology\",\"infected_machine_id\":\"ABC123\",\"infected_path\":\"/documents/confidential\",\"infected_time\":\"2023-01-15T12:30:45Z\",\"ip_addresses\":[\"89.160.20.128\",\"89.160.20.112\"],\"job_title\":\"Software Engineer\",\"keyboard_languages\":[\"English\",\"Spanish\"],\"last_name\":\"Doe\",\"log_id\":\"76afa48107ec32f51a2aba4a314357c1e69d2267f1b04bf1afc948d0f77b1658\",\"password\":\"P@ssw0rd123\",\"password_plaintext\":\"******\",\"password_type\":\"alphanumeric\",\"record_addition_date\":\"2023-12-06T00:00:00Z\",\"record_cracked_date\":\"2023-11-29T00:00:00Z\",\"record_modification_date\":\"2023-11-29T00:00:00Z\",\"salt\":\"fbbdhd\",\"severity\":3,\"sighting\":17,\"social_linkedin\":[\"ildar-bazanov-961b14160\"],\"source_id\":50436,\"spycloud_publish_date\":\"2023-01-20T08:00:00Z\",\"target_domain\":\"example.com\",\"target_subdomain\":\"login.example.com\",\"target_url\":\"https://example.com/login\",\"user_browser\":\"Chrome\",\"user_hostname\":\"workstation-1\",\"user_os\":\"Windows 10\",\"user_sys_domain\":\"8ad8.default\",\"user_sys_registered_owner\":\"John Milk\",\"username\":\"john_doe\"}",
3434
"severity": 3
@@ -125,7 +125,7 @@
125125
"last_name": "Doe",
126126
"log_id": "76afa48107ec32f51a2aba4a314357c1e69d2267f1b04bf1afc948d0f77b1658",
127127
"password": {
128-
"plaintext": "******",
128+
"plaintext": "REDACTED",
129129
"type": "alphanumeric",
130130
"value": "REDACTED"
131131
},

‎packages/spycloud/data_stream/compass/_dev/test/system/test-default-config.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,5 +9,6 @@ data_stream:
99
interval: 24h
1010
preserve_original_event: true
1111
preserve_duplicate_custom_fields: true
12+
hide_sensitive: true
1213
assert:
1314
hit_count: 1

‎packages/spycloud/data_stream/compass/agent/stream/cel.yml.hbs‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,9 @@ program: |
2828
:
2929
(state.?cursor.last_published_date).as(date,
3030
optional.of([(date.hasValue() ?
31-
timestamp(state.cursor.last_published_date) - duration("24h")
31+
timestamp(state.cursor.last_published_date)
3232
:
33-
now - (duration(state.initial_interval) + duration("24h"))
33+
now - duration(state.initial_interval)
3434
).format("2006-01-02")]))
3535
),
3636
?"cursor": state.?want_more.orValue(false) ? optional.of([state.pagination_token]) : optional.none(),
@@ -65,7 +65,12 @@ program: |
6565
"error": {
6666
"code": string(resp.StatusCode),
6767
"id": string(resp.Status),
68-
"message": string(resp.Body)
68+
"message": "GET:"+(
69+
size(resp.Body) != 0 ?
70+
string(resp.Body)
71+
:
72+
string(resp.Status) + ' (' + string(resp.StatusCode) + ')'
73+
),
6974
}
7075
},
7176
"want_more": false,

‎packages/spycloud/data_stream/compass/elasticsearch/ingest_pipeline/default.yml‎

Lines changed: 43 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,12 @@ processors:
6464
tag: rename_bank_number
6565
target_field: spycloud.compass.bank_number
6666
ignore_missing: true
67+
- set:
68+
field: spycloud.compass.bank_number
69+
tag: mask_bank_number
70+
value: 'REDACTED'
71+
if: ctx.tags != null && ctx.tags.contains('hide_sensitive') && ctx.spycloud?.compass?.bank_number != null
72+
ignore_empty_value: true
6773
- rename:
6874
field: json.cc_bin
6975
tag: rename_cc_bin
@@ -164,11 +170,23 @@ processors:
164170
tag: rename_drivers_license
165171
target_field: spycloud.compass.drivers.license.number
166172
ignore_missing: true
173+
- set:
174+
field: spycloud.compass.license.number
175+
tag: mask_license_number
176+
value: 'REDACTED'
177+
if: ctx.tags != null && ctx.tags.contains('hide_sensitive') && ctx.spycloud?.compass?.license?.number != null
178+
ignore_empty_value: true
167179
- rename:
168180
field: json.drivers_license_state_code
169181
tag: rename_drivers_license_state_code
170182
target_field: spycloud.compass.drivers.license.state_code
171183
ignore_missing: true
184+
- set:
185+
field: spycloud.compass.license.state_code
186+
tag: mask_license_state_code
187+
value: 'REDACTED'
188+
if: ctx.tags != null && ctx.tags.contains('hide_sensitive') && ctx.spycloud?.compass?.license?.state_code != null
189+
ignore_empty_value: true
172190
- rename:
173191
field: json.email_domain
174192
tag: rename_email_domain
@@ -287,16 +305,34 @@ processors:
287305
tag: rename_national_id
288306
target_field: spycloud.compass.national_id
289307
ignore_missing: true
308+
- set:
309+
field: spycloud.compass.national_id
310+
tag: mask_national_id
311+
value: 'REDACTED'
312+
if: ctx.tags != null && ctx.tags.contains('hide_sensitive') && ctx.spycloud?.compass?.national_id != null
313+
ignore_empty_value: true
290314
- rename:
291315
field: json.passport_number
292316
tag: rename_passport_number
293317
target_field: spycloud.compass.passport_number
294318
ignore_missing: true
319+
- set:
320+
field: spycloud.compass.passport_number
321+
tag: mask_passport_number
322+
value: 'REDACTED'
323+
if: ctx.tags != null && ctx.tags.contains('hide_sensitive') && ctx.spycloud?.compass?.passport_number != null
324+
ignore_empty_value: true
295325
- rename:
296326
field: json.password_plaintext
297327
tag: rename_password_plaintext
298328
target_field: spycloud.compass.password.plaintext
299329
ignore_missing: true
330+
- set:
331+
field: spycloud.compass.password.plaintext
332+
tag: mask_password_plaintext
333+
value: 'REDACTED'
334+
if: ctx.tags != null && ctx.tags.contains('hide_sensitive') && ctx.spycloud?.compass?.password?.plaintext != null
335+
ignore_empty_value: true
300336
- rename:
301337
field: json.password_type
302338
tag: rename_password_type
@@ -311,13 +347,19 @@ processors:
311347
field: spycloud.compass.password.value
312348
tag: mask_password
313349
value: 'REDACTED'
314-
if: ctx.tags != null && ctx.tags.contains('hide_sensitive') && ctx.spycloud.compass.password?.value != null
350+
if: ctx.tags != null && ctx.tags.contains('hide_sensitive') && ctx.spycloud?.compass?.password?.value != null
315351
ignore_empty_value: true
316352
- rename:
317353
field: json.postal_code
318354
tag: rename_postal_code
319355
target_field: spycloud.compass.postal_code
320356
ignore_missing: true
357+
- set:
358+
field: spycloud.compass.postal_code
359+
tag: mask_postal_code
360+
value: 'REDACTED'
361+
if: ctx.tags != null && ctx.tags.contains('hide_sensitive') && ctx.spycloud?.compass?.postal_code != null
362+
ignore_empty_value: true
321363
- set:
322364
field: host.geo.postal_code
323365
tag: set_host_geo_postal_code_from_compass_postal_code

0 commit comments

Comments
 (0)