Skip to content

Commit 33c44c2

Browse files
muskan-crestmuskan-crest
committed
Resolved data-collection and pipeline comments
1. Syntactical change in cel code. 2. Added ecs mapping in breach record and compass. 3. Masked the user sensitive details.
1 parent e6ed02b commit 33c44c2

14 files changed

Lines changed: 209 additions & 59 deletions

File tree

‎packages/spycloud/_dev/build/docs/README.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,8 @@ There are some minimum requirements for running Elastic Agent and for more infor
4848

4949
### To collect logs through REST API, follow the below steps:
5050

51-
- Considering you already have a SpyCloud account, log in to your SpyCloud instance to obtain your API key. Navigate to Main > API, where you will find your API key under the Keys > API Key section.
52-
- To obtain the Base URL, navigate to Main > API and click on the "View Docs" link, your URL can be located within the API Reference section.
51+
- Considering you already have a SpyCloud account, log in to your SpyCloud instance to obtain your API key. Navigate to **Main > API**, where you will find your API key under the **Keys > API Key** section.
52+
- To obtain the Base URL, navigate to **Main > API** and click on the **View Docs** link, your URL can be located within the **API Reference** section.
5353

5454
**NOTE**: Your system's IP should be allowlisted by the SpyCloud team to be able to access the APIs and get the data.
5555

‎packages/spycloud/data_stream/breach_catalog/agent/stream/cel.yml.hbs‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -23,13 +23,10 @@ program: |
2323
request(
2424
"GET",
2525
state.url + "/breach/catalog?" + {
26-
?"since": (state.?want_more.orValue(false) ? optional.none()
26+
?"since": (state.?want_more.orValue(false) || !has(state.?cursor.last_published_date)) ?
27+
optional.none()
2728
:
28-
has(state.?cursor.last_published_date) ?
29-
optional.of([(timestamp(state.cursor.last_published_date) - duration("24h")).format("2006-01-02")])
30-
:
31-
optional.none()
32-
),
29+
optional.of([(timestamp(state.cursor.last_published_date) - duration("24h")).format("2006-01-02")]),
3330
?"cursor": state.?want_more.orValue(false) ? optional.of([state.pagination_token]) : optional.none(),
3431
}.format_query()
3532
).with({
@@ -46,7 +43,7 @@ program: |
4643
[{"message": body.encode_json()}],
4744
"cursor": {
4845
?"last_published_date": (has(body.results) && body.results.size() > 0) ?
49-
optional.of(([?state.?cursor.last_published_date] + body.results.map(e, e.spycloud_publish_date)).max())
46+
optional.of(string(([?state.?cursor.last_published_date] + body.results.map(e, e.spycloud_publish_date)).map(t, timestamp(t)).max()))
5047
:
5148
state.?cursor.last_published_date,
5249
},

‎packages/spycloud/data_stream/breach_catalog/sample_event.json‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,15 @@
11
{
22
"@timestamp": "2022-11-24T00:00:00.000Z",
33
"agent": {
4-
"ephemeral_id": "a505f3a6-f5c1-4ea0-a76c-14bc370fa922",
4+
"ephemeral_id": "f86c89b3-70df-4910-bc06-33ba173b5f94",
55
"id": "401b5614-f720-4272-afd1-fdd38e876355",
66
"name": "docker-fleet-agent",
77
"type": "filebeat",
88
"version": "8.13.0"
99
},
1010
"data_stream": {
1111
"dataset": "spycloud.breach_catalog",
12-
"namespace": "38468",
12+
"namespace": "65748",
1313
"type": "logs"
1414
},
1515
"ecs": {
@@ -24,7 +24,7 @@
2424
"agent_id_status": "verified",
2525
"dataset": "spycloud.breach_catalog",
2626
"id": "39997",
27-
"ingested": "2024-08-05T08:58:07Z",
27+
"ingested": "2024-08-08T12:08:56Z",
2828
"kind": "asset",
2929
"original": "{\"acquisition_date\":\"2022-10-14T00:00:00Z\",\"assets\":{\"address_2\":363,\"age\":817,\"city\":1859,\"country\":177225,\"country_code\":177225,\"dob\":198,\"email\":177219,\"first_name\":177114,\"full_name\":177030,\"gender\":119505,\"industry\":162612,\"job_title\":160712,\"last_name\":177099,\"middle_name\":17749,\"phone\":511,\"postal_code\":1971,\"social_facebook\":51841,\"social_twitter\":57193},\"confidence\":3,\"description\":\"This source has been marked as sensitive due to one of the following reasons: Revealing the source may compromise an on-going investigation. The affected site is of a controversial nature but does not validate email addresses and could therefore be used to tarnish an employee's reputation.\",\"id\":39997,\"num_records\":177225,\"spycloud_publish_date\":\"2022-11-24T00:00:00Z\",\"title\":\"Sensitive Source\",\"type\":\"PRIVATE\",\"uuid\":\"9f5bf34b-092e-46f4-b87f-02c91b0adb3a\"}"
3030
},

‎packages/spycloud/data_stream/breach_record/_dev/test/pipeline/test-breach-record.log-expected.json‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
},
1212
"event": {
1313
"id": "3350f1da-fa39-4415-b2cc-02057e2fbe99",
14-
"kind": "event",
14+
"kind": "alert",
1515
"original": "{ \"infected_time\": \"2023-01-15T12:30:45Z\", \"spycloud_publish_date\": \"2023-01-20T08:00:00Z\", \"severity\": 3, \"infected_path\": \"/documents/confidential\", \"target_url\": \"https://example.com/login\", \"document_id\": \"3350f1da-fa39-4415-b2cc-02057e2fbe99\", \"user_sys_domain\": \"8ad8.default\", \"country_code\": \"US\", \"country\": \"United States\", \"user_hostname\": \"workstation-1\", \"user_os\": \"Windows 10\", \"homepage\": \"https://www.chess.com/member/sarahjoh\", \"target_domain\": \"example.com\", \"email\": \"john.doe@example.com\", \"full_name\": \"John Doe\", \"email_domain\": \"example.com\", \"email_username\": \"john.doe\", \"username\": \"john_doe\", \"password\": \"P@ssw0rd123\", \"password_plaintext\": \"******\", \"password_type\": \"alphanumeric\", \"user_browser\": \"Chrome\", \"ip_addresses\": [\"89.160.20.128\", \"89.160.20.112\"], \"infected_machine_id\": \"ABC123\", \"av_softwares\": [\"McAfee\", \"Windows Defender\"], \"log_id\": \"76afa48107ec32f51a2aba4a314357c1e69d2267f1b04bf1afc948d0f77b1658\", \"user_sys_registered_owner\": \"John Milk\", \"source_id\": 50436, \"target_subdomain\": \"login.example.com\", \"domain\": \"example1.com\", \"sighting\": 17, \"keyboard_languages\": [\"English\", \"Spanish\"], \"display_resolution\": \"1920x1080\", \"account_image_url\": \"https://www.chess.com/bundles/web/images/noavatar_l.84a92436.gif\", \"salt\": \"fbbdhd\", \"cc_bin\": \"489486\", \"cc_expiration\": \"06/2025\", \"account_signup_time\": \"2016-07-29T18:47:11Z\", \"cc_last_four\": \"1237\", \"cc_number\": \"3fdd0ce028ffaa147afdb6461f6ce95f8c07f484\", \"company_name\": \"ABC Corporation\", \"first_name\": \"John\", \"industry\": \"Technology\", \"job_title\": \"Software Engineer\", \"last_name\": \"Doe\", \"account_login_time\": \"2018-06-29T23:51:46Z\", \"record_cracked_date\": \"2023-11-29T00:00:00Z\", \"account_modification_time\": \"2018-06-29T23:51:10Z\", \"record_addition_date\": \"2023-12-06T00:00:00Z\", \"social_linkedin\": [\"ildar-bazanov-961b14160\"] }",
1616
"severity": 3
1717
},
@@ -26,9 +26,13 @@
2626
"89.160.20.112"
2727
],
2828
"os": {
29-
"full": "Windows 10"
29+
"full": "Windows 10",
30+
"type": "windows"
3031
}
3132
},
33+
"organization": {
34+
"name": "ABC Corporation"
35+
},
3236
"related": {
3337
"hosts": [
3438
"workstation-1",
@@ -157,7 +161,7 @@
157161
},
158162
"event": {
159163
"id": "0d667a9e-51ab-4fc2-ac86-2f12aa2c4197",
160-
"kind": "event",
164+
"kind": "alert",
161165
"original": "{\"email\": \"zaman_oto_kaporta@example.com\",\"password\": \"2cfa8f9e50e0f510ede9d12338a5f564\",\"source_id\": 35862,\"spycloud_publish_date\": \"2020-12-17T00:00:00Z\",\"email_domain\": \"example.com\",\"email_username\": \"zaman_oto_kaporta\",\"domain\": \"example.com\",\"password_type\": \"md5\",\"severity\": 20,\"password_plaintext\": \"4863\",\"record_cracked_date\": \"2024-03-29T00:00:00Z\",\"record_modification_date\": \"2024-03-29T00:00:00Z\",\"document_id\": \"0d667a9e-51ab-4fc2-ac86-2f12aa2c4197\",\"sighting\": 1}",
162166
"severity": 20
163167
},

‎packages/spycloud/data_stream/breach_record/agent/stream/cel.yml.hbs‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -50,17 +50,15 @@ program: |
5050
"events": has(first_body.results) ?
5151
(
5252
first_body.results.size() > 0 ?
53-
(
5453
first_body.results.map(e, {"message": e.encode_json()})
55-
)
5654
:
5755
[{"message":"No hits for Breach Catalog using since filter"}]
5856
)
5957
:
6058
[{"message": first_body.encode_json()}],
6159
"cursor": {
6260
?"last_published_date": (has(first_body.results) && first_body.results.size() > 0) ?
63-
optional.of(([?state.?cursor.last_published_date] + first_body.results.map(e, e.spycloud_publish_date)).max())
61+
optional.of(string(([?state.?cursor.last_published_date] + first_body.results.map(e, e.spycloud_publish_date)).map(t, timestamp(t)).max()))
6462
:
6563
state.?cursor.last_published_date,
6664
"last_modified_date": state.?cursor.last_modified_date.orValue((now - duration(state.initial_interval)).format(time_layout.RFC3339)),
@@ -118,17 +116,15 @@ program: |
118116
"events": has(second_body.results) ?
119117
(
120118
second_body.size() > 0 ?
121-
(
122119
second_body.results.map(e, {"message": e.encode_json()})
123-
)
124120
:
125121
[{"message":"No hits for Breach Catalog using since_modification_date filter"}]
126122
)
127123
:
128124
[{"message": second_body.encode_json()}],
129125
"cursor": {
130126
?"last_modified_date": (has(second_body.results) && second_body.results.size() > 0) ?
131-
optional.of(([?state.?cursor.last_modified_date] + second_body.results.map(e, e.record_modification_date)).max())
127+
optional.of(string(([?state.?cursor.last_modified_date] + second_body.results.map(e, e.record_modification_date)).map(t, timestamp(t)).max()))
132128
:
133129
state.?cursor.last_modified_date,
134130
"last_published_date": state.?cursor.last_published_date.orValue(null),
@@ -165,6 +161,9 @@ tags:
165161
{{#if preserve_duplicate_custom_fields}}
166162
- preserve_duplicate_custom_fields
167163
{{/if}}
164+
{{#if hide_sensitive_true}}
165+
- hide_sensitive_true
166+
{{/if}}
168167
{{#each tags as |tag|}}
169168
- {{tag}}
170169
{{/each}}

‎packages/spycloud/data_stream/breach_record/elasticsearch/ingest_pipeline/default.yml‎

Lines changed: 56 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ processors:
1212
- set:
1313
field: event.kind
1414
tag: set_event_kind
15-
value: event
15+
value: alert
1616
- rename:
1717
field: message
1818
tag: rename_message_to_event_original
@@ -86,26 +86,55 @@ processors:
8686
tag: rename_cc_bin
8787
target_field: spycloud.breach_record.cc.bin
8888
ignore_missing: true
89+
- set:
90+
field: spycloud.breach_record.cc.bin
91+
tag: mask_cc_bin
92+
value: '******'
93+
if: ctx.tags != null && ctx.tags.contains('hide_sensitive_true') && ctx.spycloud?.breach_record?.cc?.bin != null
94+
ignore_empty_value: true
8995
- rename:
9096
field: json.cc_expiration
9197
tag: rename_cc_expiration
9298
target_field: spycloud.breach_record.cc.expiration
9399
ignore_missing: true
100+
- set:
101+
field: spycloud.breach_record.cc.expiration
102+
tag: mask_cc_expiration
103+
value: '******'
104+
if: ctx.tags != null && ctx.tags.contains('hide_sensitive_true') && ctx.spycloud?.breach_record?.cc?.expiration != null
105+
ignore_empty_value: true
94106
- rename:
95107
field: json.cc_last_four
96108
tag: rename_cc_last_four
97109
target_field: spycloud.breach_record.cc.last_four
98110
ignore_missing: true
111+
- set:
112+
field: spycloud.breach_record.cc.last_four
113+
tag: mask_cc_last_four
114+
value: '****'
115+
if: ctx.tags != null && ctx.tags.contains('hide_sensitive_true') && ctx.spycloud?.breach_record?.cc?.last_four != null
116+
ignore_empty_value: true
99117
- rename:
100118
field: json.cc_number
101119
tag: rename_cc_number
102120
target_field: spycloud.breach_record.cc.number
103121
ignore_missing: true
122+
- set:
123+
field: spycloud.breach_record.cc.number
124+
tag: mask_cc_number
125+
value: '******'
126+
if: ctx.tags != null && ctx.tags.contains('hide_sensitive_true') && ctx.spycloud?.breach_record?.cc?.number != null
127+
ignore_empty_value: true
104128
- rename:
105129
field: json.company_name
106130
tag: rename_company_name
107131
target_field: spycloud.breach_record.company_name
108132
ignore_missing: true
133+
- set:
134+
field: organization.name
135+
tag: set_organization_name_from_breach_record_company_name
136+
copy_from: spycloud.breach_record.company_name
137+
ignore_empty_value: true
109138
- rename:
110139
field: json.country_code
111140
tag: rename_country_code
@@ -305,6 +334,12 @@ processors:
305334
tag: rename_password
306335
target_field: spycloud.breach_record.password.value
307336
ignore_missing: true
337+
- set:
338+
field: spycloud.breach_record.password.value
339+
tag: mask_password
340+
value: '******'
341+
if: ctx.tags != null && ctx.tags.contains('hide_sensitive_true') && ctx.spycloud?.breach_record?.password?.value != null
342+
ignore_empty_value: true
308343
- date:
309344
field: json.record_addition_date
310345
tag: date_record_addition_date
@@ -492,6 +527,25 @@ processors:
492527
tag: set_host_os_full_from_breach_record_user_os
493528
copy_from: spycloud.breach_record.user.os
494529
ignore_empty_value: true
530+
- script:
531+
lang: painless
532+
tag: script_to_set_host_os_type
533+
description: Script to set host_os_type.
534+
if: ctx.spycloud?.breach_record?.user?.os != null
535+
source: |
536+
if (ctx.spycloud.breach_record.user.os.toLowerCase().contains('windows')) {
537+
ctx.host.os.type = 'windows';
538+
} else if (ctx.spycloud.breach_record.user.os.toLowerCase().contains('linux')) {
539+
ctx.host.os.type = 'linux';
540+
} else if (ctx.spycloud.breach_record.user.os.toLowerCase().contains('mac')) {
541+
ctx.host.os.type = 'macos';
542+
} else if (ctx.spycloud.breach_record.user.os.toLowerCase().contains('unix')) {
543+
ctx.host.os.type = 'unix';
544+
} else if (ctx.spycloud.breach_record.user.os.toLowerCase().contains('ios')) {
545+
ctx.host.os.type = 'ios';
546+
} else if (ctx.spycloud.breach_record.user.os.toLowerCase().contains('android')) {
547+
ctx.host.os.type = 'android';
548+
}
495549
- rename:
496550
field: json.user_sys_domain
497551
tag: rename_user_sys_domain
@@ -530,6 +584,7 @@ processors:
530584
- spycloud.breach_record.user.hostname
531585
- spycloud.breach_record.user.os
532586
- spycloud.breach_record.user.name
587+
- spycloud.breach_record.company_name
533588
tag: remove_custom_duplicate_fields
534589
ignore_missing: true
535590
if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields'))

‎packages/spycloud/data_stream/breach_record/manifest.yml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,14 @@ streams:
4242
text: "All"
4343
required: true
4444
show_user: true
45+
- name: hide_sensitive_true
46+
type: bool
47+
title: Hide Sensitive True
48+
description: Hide sensitive user details.
49+
multi: false
50+
required: false
51+
show_user: false
52+
default: true
4553
- name: http_client_timeout
4654
type: text
4755
title: HTTP Client Timeout

‎packages/spycloud/data_stream/breach_record/sample_event.json‎

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,15 @@
11
{
22
"@timestamp": "2023-11-29T00:00:00.000Z",
33
"agent": {
4-
"ephemeral_id": "0bc4a45b-5b39-4b85-8a16-1176a21b91cf",
4+
"ephemeral_id": "dde3a142-a8f9-4b3d-a78f-f623d3b6eea2",
55
"id": "401b5614-f720-4272-afd1-fdd38e876355",
66
"name": "docker-fleet-agent",
77
"type": "filebeat",
88
"version": "8.13.0"
99
},
1010
"data_stream": {
1111
"dataset": "spycloud.breach_record",
12-
"namespace": "61753",
12+
"namespace": "84538",
1313
"type": "logs"
1414
},
1515
"destination": {
@@ -28,8 +28,8 @@
2828
"agent_id_status": "verified",
2929
"dataset": "spycloud.breach_record",
3030
"id": "3350f1da-fa39-4415-b2cc-02057e2fbe99",
31-
"ingested": "2024-08-05T08:59:09Z",
32-
"kind": "event",
31+
"ingested": "2024-08-08T12:10:33Z",
32+
"kind": "alert",
3333
"original": "{\"account_image_url\":\"https://www.chess.com/bundles/web/images/noavatar_l.84a92436.gif\",\"account_login_time\":\"2018-06-29T23:51:46Z\",\"account_modification_time\":\"2018-06-29T23:51:10Z\",\"account_signup_time\":\"2016-07-29T18:47:11Z\",\"av_softwares\":[\"McAfee\",\"Windows Defender\"],\"cc_bin\":\"489486\",\"cc_expiration\":\"06/2025\",\"cc_last_four\":\"1237\",\"cc_number\":\"3fdd0ce028ffaa147afdb6461f6ce95f8c07f484\",\"company_name\":\"ABC Corporation\",\"country\":\"United States\",\"country_code\":\"US\",\"display_resolution\":\"1920x1080\",\"document_id\":\"3350f1da-fa39-4415-b2cc-02057e2fbe99\",\"domain\":\"example1.com\",\"email\":\"john.doe@example.com\",\"email_domain\":\"example.com\",\"email_username\":\"john.doe\",\"first_name\":\"John\",\"full_name\":\"John Doe\",\"homepage\":\"https://www.chess.com/member/sarahjoh\",\"industry\":\"Technology\",\"infected_machine_id\":\"ABC123\",\"infected_path\":\"/documents/confidential\",\"infected_time\":\"2023-01-15T12:30:45Z\",\"ip_addresses\":[\"89.160.20.128\",\"89.160.20.112\"],\"job_title\":\"Software Engineer\",\"keyboard_languages\":[\"English\",\"Spanish\"],\"last_name\":\"Doe\",\"log_id\":\"76afa48107ec32f51a2aba4a314357c1e69d2267f1b04bf1afc948d0f77b1658\",\"password\":\"P@ssw0rd123\",\"password_plaintext\":\"******\",\"password_type\":\"alphanumeric\",\"record_addition_date\":\"2023-12-06T00:00:00Z\",\"record_cracked_date\":\"2023-11-29T00:00:00Z\",\"record_modification_date\":\"2023-11-29T00:00:00Z\",\"salt\":\"fbbdhd\",\"severity\":3,\"sighting\":17,\"social_linkedin\":[\"ildar-bazanov-961b14160\"],\"source_id\":50436,\"spycloud_publish_date\":\"2023-01-20T08:00:00Z\",\"target_domain\":\"example.com\",\"target_subdomain\":\"login.example.com\",\"target_url\":\"https://example.com/login\",\"user_browser\":\"Chrome\",\"user_hostname\":\"workstation-1\",\"user_os\":\"Windows 10\",\"user_sys_domain\":\"8ad8.default\",\"user_sys_registered_owner\":\"John Milk\",\"username\":\"john_doe\"}",
3434
"severity": 3
3535
},
@@ -44,12 +44,16 @@
4444
"89.160.20.112"
4545
],
4646
"os": {
47-
"full": "Windows 10"
47+
"full": "Windows 10",
48+
"type": "windows"
4849
}
4950
},
5051
"input": {
5152
"type": "cel"
5253
},
54+
"organization": {
55+
"name": "ABC Corporation"
56+
},
5357
"related": {
5458
"hosts": [
5559
"workstation-1",
@@ -82,10 +86,10 @@
8286
"Windows Defender"
8387
],
8488
"cc": {
85-
"bin": "489486",
86-
"expiration": "06/2025",
87-
"last_four": "1237",
88-
"number": "3fdd0ce028ffaa147afdb6461f6ce95f8c07f484"
89+
"bin": "******",
90+
"expiration": "******",
91+
"last_four": "****",
92+
"number": "******"
8993
},
9094
"company_name": "ABC Corporation",
9195
"country": {
@@ -123,7 +127,7 @@
123127
"password": {
124128
"plaintext": "******",
125129
"type": "alphanumeric",
126-
"value": "P@ssw0rd123"
130+
"value": "******"
127131
},
128132
"record": {
129133
"addition_date": "2023-12-06T00:00:00.000Z",
@@ -158,6 +162,7 @@
158162
"tags": [
159163
"preserve_original_event",
160164
"preserve_duplicate_custom_fields",
165+
"hide_sensitive_true",
161166
"forwarded",
162167
"spycloud-breach_record"
163168
],

‎packages/spycloud/data_stream/compass/_dev/test/pipeline/test-compass.log-expected.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,8 @@
2727
"0.0.0.0"
2828
],
2929
"os": {
30-
"full": "Windows 10"
30+
"full": "Windows 10",
31+
"type": "windows"
3132
}
3233
},
3334
"related": {

‎packages/spycloud/data_stream/compass/agent/stream/cel.yml.hbs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ program: |
4949
[{"message": body.encode_json()}],
5050
"cursor": {
5151
?"last_published_date": (has(body.results) && body.results.size() > 0) ?
52-
optional.of(([?state.?cursor.last_published_date] + body.results.map(e, e.spycloud_publish_date)).max())
52+
optional.of(string(([?state.?cursor.last_published_date] + body.results.map(e, e.spycloud_publish_date)).map(t, timestamp(t)).max()))
5353
:
5454
state.?cursor.last_published_date,
5555
},
@@ -80,6 +80,9 @@ tags:
8080
{{#if preserve_duplicate_custom_fields}}
8181
- preserve_duplicate_custom_fields
8282
{{/if}}
83+
{{#if hide_sensitive_true}}
84+
- hide_sensitive_true
85+
{{/if}}
8386
{{#each tags as |tag|}}
8487
- {{tag}}
8588
{{/each}}

0 commit comments

Comments
 (0)