Resolving comment

1. Remove the -24h logic in data collection. 2. Add hide sensitive mask to few more fields.
kcreddy · Aug 16, 2024 · Jul 26, 2024 · Jul 26, 2024 · Jul 31, 2024 · Aug 5, 2024
commit a01e480efa178c99fd9db74820da7c7c4337e3f4
@@ -26,7 +26,7 @@ program: |
       ?"since": (state.?want_more.orValue(false) || !has(state.?cursor.last_published_date)) ?
         optional.none()
       :
-        optional.of([(timestamp(state.cursor.last_published_date) - duration("24h")).format("2006-01-02")]),
+        optional.of([timestamp(state.cursor.last_published_date).format("2006-01-02")]),
       ?"cursor": state.?want_more.orValue(false) ? optional.of([state.pagination_token]) : optional.none(),
     }.format_query()
   ).with({
@@ -58,7 +58,12 @@ program: |
         "error": {
           "code": string(resp.StatusCode),
           "id": string(resp.Status),
-          "message": string(resp.Body),
+          "message": "GET:"+(
+            size(resp.Body) != 0 ?
+              string(resp.Body)
+            :
+              string(resp.Status) + ' (' + string(resp.StatusCode) + ')'
+          ),
         },
       },
       "want_more": false,

@@ -1,30 +1,30 @@
 {
     "@timestamp": "2022-11-24T00:00:00.000Z",
     "agent": {
-        "ephemeral_id": "90dc28d2-8a37-4b01-b145-4ee5f9207fcf",
-        "id": "401b5614-f720-4272-afd1-fdd38e876355",
+        "ephemeral_id": "d6e9d6f0-0baa-44b6-b60b-e4b811e50811",
+        "id": "763d7558-f93b-440a-94ee-509804901acf",
         "name": "docker-fleet-agent",
         "type": "filebeat",
         "version": "8.13.0"
     },
     "data_stream": {
         "dataset": "spycloud.breach_catalog",
-        "namespace": "88948",
+        "namespace": "63685",
         "type": "logs"
     },
     "ecs": {
         "version": "8.11.0"
     },
     "elastic_agent": {
-        "id": "401b5614-f720-4272-afd1-fdd38e876355",
+        "id": "763d7558-f93b-440a-94ee-509804901acf",
         "snapshot": false,
         "version": "8.13.0"
     },
     "event": {
         "agent_id_status": "verified",
         "dataset": "spycloud.breach_catalog",
         "id": "39997",
-        "ingested": "2024-08-09T09:31:23Z",
+        "ingested": "2024-08-16T06:23:24Z",
         "kind": "asset",
         "original": "{\"acquisition_date\":\"2022-10-14T00:00:00Z\",\"assets\":{\"address_2\":363,\"age\":817,\"city\":1859,\"country\":177225,\"country_code\":177225,\"dob\":198,\"email\":177219,\"first_name\":177114,\"full_name\":177030,\"gender\":119505,\"industry\":162612,\"job_title\":160712,\"last_name\":177099,\"middle_name\":17749,\"phone\":511,\"postal_code\":1971,\"social_facebook\":51841,\"social_twitter\":57193},\"confidence\":3,\"description\":\"This source has been marked as sensitive due to one of the following reasons: Revealing the source may compromise an on-going investigation. The affected site is of a controversial nature but does not validate email addresses and could therefore be used to tarnish an employee's reputation.\",\"id\":39997,\"num_records\":177225,\"spycloud_publish_date\":\"2022-11-24T00:00:00Z\",\"title\":\"Sensitive Source\",\"type\":\"PRIVATE\",\"uuid\":\"9f5bf34b-092e-46f4-b87f-02c91b0adb3a\"}"
     },

@@ -10,5 +10,6 @@ data_stream:
     interval: 24h
     preserve_original_event: true
     preserve_duplicate_custom_fields: true
+    hide_sensitive: true
 assert:
   hit_count: 1
@@ -32,9 +32,9 @@ program: |
         :
           (state.?cursor.last_published_date).as(date,
             optional.of([(date.hasValue() ?
-              timestamp(state.cursor.last_published_date) - duration("24h")
+              timestamp(state.cursor.last_published_date)
             :
-              now - (duration(state.initial_interval) + duration("24h"))
+              now - duration(state.initial_interval)
           ).format("2006-01-02")])
         )),
         ?"cursor": state.?want_more.orValue(false) ? optional.of([state.pagination_token_since]) : optional.none(),
@@ -45,40 +45,44 @@ program: |
         "User-Agent": [useragent],
       }
     }).do_request().as(resp, resp.StatusCode == 200 ?
-      bytes(resp.Body).decode_json().as(first_body,
-        {
-          "events": has(first_body.results) ?
-          (
-            first_body.results.size() > 0 ?
-              first_body.results.map(e, {"message": e.encode_json()})
-            :
-              [{"message":"No hits for Breach Catalog using since filter"}]
-          )
+      bytes(resp.Body).decode_json().as(first_body, {
+        "events": has(first_body.results) ?
+        (
+          first_body.results.size() > 0 ?
+            first_body.results.map(e, {"message": e.encode_json()})
           :
-            [{"message": first_body.encode_json()}],
-          "cursor": {
-            ?"last_published_date": (has(first_body.results) && first_body.results.size() > 0) ?
-              optional.of(string(([?state.?cursor.last_published_date] + first_body.results.map(e, e.spycloud_publish_date)).map(t, timestamp(t)).max()))
-              :
-                state.?cursor.last_published_date,
-            ?"last_modified_date": state.?cursor.last_modified_date,
-          },
-          ?"pagination_token_since": first_body.?cursor,
-          "since_data": first_body.?cursor.orValue("") != "",
-          "want_more": first_body.?cursor.orValue("") != "" || state.?since_modified_data.orValue(false),
-          "api_key": state.api_key,
-          "since_modified_data": state.since_modified_data,
-          ?"pagination_token_since_modified": state.?pagination_token_since_modified,
-          "initial_interval": state.initial_interval,
-          "severity": state.severity,
-        })
-      :
-        {
+            [{"message":"No hits for Breach Catalog using since filter"}]
+        )
+        :
+          [{"message": first_body.encode_json()}],
+        "cursor": {
+          ?"last_published_date": (has(first_body.results) && first_body.results.size() > 0) ?
+            optional.of(string(([?state.?cursor.last_published_date] + first_body.results.map(e, e.spycloud_publish_date)).map(t, timestamp(t)).max()))
+            :
+              state.?cursor.last_published_date,
+          ?"last_modified_date": state.?cursor.last_modified_date,
+        },
+        ?"pagination_token_since": first_body.?cursor,
+        "since_data": first_body.?cursor.orValue("") != "",
+        "want_more": first_body.?cursor.orValue("") != "" || state.?since_modified_data.orValue(false),
+        "api_key": state.api_key,
+        "since_modified_data": state.since_modified_data,
+        ?"pagination_token_since_modified": state.?pagination_token_since_modified,
+        "initial_interval": state.initial_interval,
+        "severity": state.severity,
+      })
+    :
+      {
         "events": {
           "error": {
             "code": string(resp.StatusCode),
             "id": string(resp.Status),
-            "message": string(resp.Body),
+            "message": "GET:"+(
+              size(resp.Body) != 0 ?
+                string(resp.Body)
+              :
+                string(resp.Status) + ' (' + string(resp.StatusCode) + ')'
+            ),
           }
         },
         "api_key": state.api_key,
@@ -97,9 +101,9 @@ program: |
           state.?pagination_token_since_modified.orValue("") != "" ? optional.none() :
             (state.?cursor.last_modified_date).as(date,
               optional.of([(date.hasValue() ?
-                timestamp(state.cursor.last_modified_date) - duration("24h")
+                timestamp(state.cursor.last_modified_date)
               :
-                now - (duration(state.initial_interval) + duration("24h"))
+                now - duration(state.initial_interval)
             ).format("2006-01-02")])
           )
         ),
@@ -143,7 +147,12 @@ program: |
           "error": {
             "code": string(response.StatusCode),
             "id": string(response.Status),
-            "message": string(response.Body),
+            "message": "GET:"+(
+              size(response.Body) != 0 ?
+                string(response.Body)
+              :
+                string(response.Status) + ' (' + string(response.StatusCode) + ')'
+            ),
           }
         },
         "api_key": state.api_key,

@@ -324,6 +324,12 @@ processors:
       tag: rename_password_plaintext
       target_field: spycloud.breach_record.password.plaintext
       ignore_missing: true
+  - set:
+      field: spycloud.breach_record.password.plaintext
+      tag: mask_password_plaintext
+      value: 'REDACTED'
+      if: ctx.tags != null && ctx.tags.contains('hide_sensitive') && ctx.spycloud?.breach_record?.password?.plaintext != null
+      ignore_empty_value: true
   - rename:
       field: json.password_type
       tag: rename_password_type

@@ -1,15 +1,15 @@
 {
     "@timestamp": "2023-11-29T00:00:00.000Z",
     "agent": {
-        "ephemeral_id": "cd4912d6-0935-4cbf-ae4b-8e9fbac4dbad",
-        "id": "401b5614-f720-4272-afd1-fdd38e876355",
+        "ephemeral_id": "ce02cb91-e24e-45fa-8591-70b683f1b86b",
+        "id": "763d7558-f93b-440a-94ee-509804901acf",
         "name": "docker-fleet-agent",
         "type": "filebeat",
         "version": "8.13.0"
     },
     "data_stream": {
         "dataset": "spycloud.breach_record",
-        "namespace": "30900",
+        "namespace": "33260",
         "type": "logs"
     },
     "destination": {
@@ -20,15 +20,15 @@
         "version": "8.11.0"
     },
     "elastic_agent": {
-        "id": "401b5614-f720-4272-afd1-fdd38e876355",
+        "id": "763d7558-f93b-440a-94ee-509804901acf",
         "snapshot": false,
         "version": "8.13.0"
     },
     "event": {
         "agent_id_status": "verified",
         "dataset": "spycloud.breach_record",
         "id": "3350f1da-fa39-4415-b2cc-02057e2fbe99",
-        "ingested": "2024-08-09T09:33:05Z",
+        "ingested": "2024-08-16T06:24:19Z",
         "kind": "alert",
         "original": "{\"account_image_url\":\"https://www.chess.com/bundles/web/images/noavatar_l.84a92436.gif\",\"account_login_time\":\"2018-06-29T23:51:46Z\",\"account_modification_time\":\"2018-06-29T23:51:10Z\",\"account_signup_time\":\"2016-07-29T18:47:11Z\",\"av_softwares\":[\"McAfee\",\"Windows Defender\"],\"cc_bin\":\"489486\",\"cc_expiration\":\"06/2025\",\"cc_last_four\":\"1237\",\"cc_number\":\"3fdd0ce028ffaa147afdb6461f6ce95f8c07f484\",\"company_name\":\"ABC Corporation\",\"country\":\"United States\",\"country_code\":\"US\",\"display_resolution\":\"1920x1080\",\"document_id\":\"3350f1da-fa39-4415-b2cc-02057e2fbe99\",\"domain\":\"example1.com\",\"email\":\"john.doe@example.com\",\"email_domain\":\"example.com\",\"email_username\":\"john.doe\",\"first_name\":\"John\",\"full_name\":\"John Doe\",\"homepage\":\"https://www.chess.com/member/sarahjoh\",\"industry\":\"Technology\",\"infected_machine_id\":\"ABC123\",\"infected_path\":\"/documents/confidential\",\"infected_time\":\"2023-01-15T12:30:45Z\",\"ip_addresses\":[\"89.160.20.128\",\"89.160.20.112\"],\"job_title\":\"Software Engineer\",\"keyboard_languages\":[\"English\",\"Spanish\"],\"last_name\":\"Doe\",\"log_id\":\"76afa48107ec32f51a2aba4a314357c1e69d2267f1b04bf1afc948d0f77b1658\",\"password\":\"P@ssw0rd123\",\"password_plaintext\":\"******\",\"password_type\":\"alphanumeric\",\"record_addition_date\":\"2023-12-06T00:00:00Z\",\"record_cracked_date\":\"2023-11-29T00:00:00Z\",\"record_modification_date\":\"2023-11-29T00:00:00Z\",\"salt\":\"fbbdhd\",\"severity\":3,\"sighting\":17,\"social_linkedin\":[\"ildar-bazanov-961b14160\"],\"source_id\":50436,\"spycloud_publish_date\":\"2023-01-20T08:00:00Z\",\"target_domain\":\"example.com\",\"target_subdomain\":\"login.example.com\",\"target_url\":\"https://example.com/login\",\"user_browser\":\"Chrome\",\"user_hostname\":\"workstation-1\",\"user_os\":\"Windows 10\",\"user_sys_domain\":\"8ad8.default\",\"user_sys_registered_owner\":\"John Milk\",\"username\":\"john_doe\"}",
         "severity": 3
@@ -125,7 +125,7 @@
             "last_name": "Doe",
             "log_id": "76afa48107ec32f51a2aba4a314357c1e69d2267f1b04bf1afc948d0f77b1658",
             "password": {
-                "plaintext": "******",
+                "plaintext": "REDACTED",
                 "type": "alphanumeric",
                 "value": "REDACTED"
             },

@@ -9,5 +9,6 @@ data_stream:
     interval: 24h
     preserve_original_event: true
     preserve_duplicate_custom_fields: true
+    hide_sensitive: true
 assert:
   hit_count: 1
@@ -28,9 +28,9 @@ program: |
       :
         (state.?cursor.last_published_date).as(date,
           optional.of([(date.hasValue() ?
-            timestamp(state.cursor.last_published_date) - duration("24h")
+            timestamp(state.cursor.last_published_date)
           :
-            now - (duration(state.initial_interval) + duration("24h"))
+            now - duration(state.initial_interval)
         ).format("2006-01-02")]))
       ),
       ?"cursor": state.?want_more.orValue(false) ? optional.of([state.pagination_token]) : optional.none(),
@@ -65,7 +65,12 @@ program: |
         "error": {
           "code": string(resp.StatusCode),
           "id": string(resp.Status),
-          "message": string(resp.Body)
+          "message": "GET:"+(
+            size(resp.Body) != 0 ?
+              string(resp.Body)
+            :
+              string(resp.Status) + ' (' + string(resp.StatusCode) + ')'
+          ),
         }
       },
       "want_more": false,

@@ -64,6 +64,12 @@ processors:
       tag: rename_bank_number
       target_field: spycloud.compass.bank_number
       ignore_missing: true
+  - set:
+      field: spycloud.compass.bank_number
+      tag: mask_bank_number
+      value: 'REDACTED'
+      if: ctx.tags != null && ctx.tags.contains('hide_sensitive') && ctx.spycloud?.compass?.bank_number != null
+      ignore_empty_value: true
   - rename:
       field: json.cc_bin
       tag: rename_cc_bin
@@ -164,11 +170,23 @@ processors:
       tag: rename_drivers_license
       target_field: spycloud.compass.drivers.license.number
       ignore_missing: true
+  - set:
+      field: spycloud.compass.license.number
+      tag: mask_license_number
+      value: 'REDACTED'
+      if: ctx.tags != null && ctx.tags.contains('hide_sensitive') && ctx.spycloud?.compass?.license?.number != null
+      ignore_empty_value: true
   - rename:
       field: json.drivers_license_state_code
       tag: rename_drivers_license_state_code
       target_field: spycloud.compass.drivers.license.state_code
       ignore_missing: true
+  - set:
+      field: spycloud.compass.license.state_code
+      tag: mask_license_state_code
+      value: 'REDACTED'
+      if: ctx.tags != null && ctx.tags.contains('hide_sensitive') && ctx.spycloud?.compass?.license?.state_code != null
+      ignore_empty_value: true
   - rename:
       field: json.email_domain
       tag: rename_email_domain
@@ -287,16 +305,34 @@ processors:
       tag: rename_national_id
       target_field: spycloud.compass.national_id
       ignore_missing: true
+  - set:
+      field: spycloud.compass.national_id
+      tag: mask_national_id
+      value: 'REDACTED'
+      if: ctx.tags != null && ctx.tags.contains('hide_sensitive') && ctx.spycloud?.compass?.national_id != null
+      ignore_empty_value: true
   - rename:
       field: json.passport_number
       tag: rename_passport_number
       target_field: spycloud.compass.passport_number
       ignore_missing: true
+  - set:
+      field: spycloud.compass.passport_number
+      tag: mask_passport_number
+      value: 'REDACTED'
+      if: ctx.tags != null && ctx.tags.contains('hide_sensitive') && ctx.spycloud?.compass?.passport_number != null
+      ignore_empty_value: true
   - rename:
       field: json.password_plaintext
       tag: rename_password_plaintext
       target_field: spycloud.compass.password.plaintext
       ignore_missing: true
+  - set:
+      field: spycloud.compass.password.plaintext
+      tag: mask_password_plaintext
+      value: 'REDACTED'
+      if: ctx.tags != null && ctx.tags.contains('hide_sensitive') && ctx.spycloud?.compass?.password?.plaintext != null
+      ignore_empty_value: true
   - rename:
       field: json.password_type
       tag: rename_password_type
@@ -311,13 +347,19 @@ processors:
       field: spycloud.compass.password.value
       tag: mask_password
       value: 'REDACTED'
-      if: ctx.tags != null && ctx.tags.contains('hide_sensitive') && ctx.spycloud.compass.password?.value != null
+      if: ctx.tags != null && ctx.tags.contains('hide_sensitive') && ctx.spycloud?.compass?.password?.value != null
       ignore_empty_value: true
   - rename:
       field: json.postal_code
       tag: rename_postal_code
       target_field: spycloud.compass.postal_code
       ignore_missing: true
+  - set:
+      field: spycloud.compass.postal_code
+      tag: mask_postal_code
+      value: 'REDACTED'
+      if: ctx.tags != null && ctx.tags.contains('hide_sensitive') && ctx.spycloud?.compass?.postal_code != null
+      ignore_empty_value: true
   - set:
       field: host.geo.postal_code
       tag: set_host_geo_postal_code_from_compass_postal_code