Skip to content

Commit 48d46df

Browse files
kcreddykcreddy
committed
ssi: add processor tags and preserve_original_event on failure (1/4)
Tag every ingest pipeline processor across 51 SSI-owned packages (1password through entityanalytics_entra_id) with a unique, descriptive identifier so that failure telemetry can attribute errors to the specific step that failed rather than collapsing same-type processors into one bucket. Add preserve_original_event to pipeline-level on_failure handlers that were missing it, ensuring the raw payload is retained when a pipeline error document is indexed. Updates elastic#20558
1 parent fd18eb1 commit 48d46df

357 files changed

Lines changed: 9438 additions & 823 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎packages/1password/changelog.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
11
# newer versions go on top
2+
- version: "1.37.1"
3+
changes:
4+
- description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers.
5+
type: enhancement
6+
link: https://github.com/elastic/integrations/issues/20558
27
- version: "1.37.0"
38
changes:
49
- description: Set 'elastic' owner type.

‎packages/1password/data_stream/audit_events/elasticsearch/ingest_pipeline/default.yml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,78 +13,95 @@ processors:
1313
Removes the fields added by Agentless as metadata,
1414
as they can collide with ECS fields.
1515
- rename:
16+
tag: rename_message_to_event_original_56a77271
1617
field: message
1718
target_field: event.original
1819
ignore_missing: true
1920
if: ctx.event?.original == null
2021
- remove:
22+
tag: remove_message_2ee3a7d4
2123
field: message
2224
ignore_missing: true
2325
if: ctx.event?.original != null
2426
description: 'The `message` field is no longer required if the document has an `event.original` field.'
2527
- json:
28+
tag: json_event_original_to_onepassword_da036177
2629
field: event.original
2730
target_field: onepassword
2831
- drop:
32+
tag: drop_02acf80d
2933
description: Drop if no timestamp (invalid json)
3034
if: "ctx?.onepassword?.timestamp == null"
3135
#######################
3236
## ECS Event Mapping ##
3337
#######################
3438
- set:
39+
tag: set_ecs_version_0f7334bb
3540
field: ecs.version
3641
value: "8.11.0"
3742
# Sets event.created from the @timestamp field generated by filebeat before being overwritten further down
3843
- set:
44+
tag: set_event_created_e3f09e3b
3945
field: event.created
4046
copy_from: "@timestamp"
4147
- set:
48+
tag: set_event_kind_de80643c
4249
field: event.kind
4350
value: event
4451
- append:
52+
tag: append_event_category_c8ab9fa2
4553
field: event.category
4654
value: [configuration]
4755
- append:
56+
tag: append_event_type_8860cef4
4857
field: event.type
4958
value: [access]
5059
- rename:
60+
tag: rename_onepassword_action_to_event_action_24171185
5161
field: onepassword.action
5262
target_field: event.action
5363
ignore_missing: true
5464
#########################
5565
## ECS Related Mapping ##
5666
#########################
5767
- append:
68+
tag: append_related_user_572fd854
5869
field: related.user
5970
value: "{{{onepassword.actor_details.name}}}"
6071
allow_duplicates: false
6172
if: ctx.onepassword?.actor_details?.name != null
6273
- append:
74+
tag: append_related_user_6d2a829f
6375
field: related.user
6476
value: "{{{onepassword.actor_uuid}}}"
6577
allow_duplicates: false
6678
if: ctx.onepassword?.actor_uuid != null
6779
- append:
80+
tag: append_related_user_ec2b4068
6881
field: related.user
6982
value: "{{{onepassword.object_uuid}}}"
7083
allow_duplicates: false
7184
if: ctx.onepassword?.object_type == "user" && ctx?.onepassword?.object_uuid != null
7285
- append:
86+
tag: append_related_user_54a15ffd
7387
field: related.user
7488
value: "{{{onepassword.aux_uuid}}}"
7589
allow_duplicates: false
7690
if: ctx.onepassword?.object_type == "gm" && ctx?.onepassword?.aux_uuid != null
7791
- append:
92+
tag: append_related_user_b2338d93
7893
field: related.user
7994
value: "{{{onepassword.aux_uuid}}}"
8095
allow_duplicates: false
8196
if: ctx.onepassword?.object_type == "device" && ctx?.onepassword?.aux_uuid != null
8297
- append:
98+
tag: append_related_user_f39de967
8399
field: related.user
84100
value: "{{{onepassword.aux_uuid}}}"
85101
allow_duplicates: false
86102
if: ctx.onepassword?.object_type == "uva" && ctx?.onepassword?.aux_uuid != null
87103
- append:
104+
tag: append_related_ip_93983512
88105
field: related.ip
89106
value: "{{{onepassword.session.ip}}}"
90107
allow_duplicates: false
@@ -93,32 +110,39 @@ processors:
93110
## ECS User Mapping ##
94111
######################
95112
- rename:
113+
tag: rename_onepassword_actor_uuid_to_user_id_60afb126
96114
field: onepassword.actor_uuid
97115
target_field: user.id
98116
ignore_missing: true
99117
- rename:
118+
tag: rename_onepassword_actor_details_name_to_user_full_name_c880bb1c
100119
field: onepassword.actor_details.name
101120
target_field: user.full_name
102121
ignore_missing: true
103122
- set:
123+
tag: set_user_name_67fd4e49
104124
field: user.name
105125
copy_from: user.full_name
106126
ignore_empty_value: true
107127
- rename:
128+
tag: rename_onepassword_actor_details_email_to_user_email_1e9bf80a
108129
field: onepassword.actor_details.email
109130
target_field: user.email
110131
ignore_missing: true
111132
########################
112133
## ECS Source Mapping ##
113134
########################
114135
- rename:
136+
tag: rename_onepassword_session_ip_to_source_ip_23f74ca6
115137
field: onepassword.session.ip
116138
target_field: source.ip
117139
ignore_missing: true
118140
- geoip:
141+
tag: geoip_source_ip_to_source_geo_0494c6b1
119142
field: source.ip
120143
target_field: source.geo
121144
- geoip:
145+
tag: geoip_source_ip_to_source_as_28d69883
122146
database_file: GeoLite2-ASN.mmdb
123147
field: source.ip
124148
target_field: source.as
@@ -127,24 +151,28 @@ processors:
127151
- organization_name
128152
ignore_missing: true
129153
- rename:
154+
tag: rename_source_as_asn_to_source_as_number_a917047d
130155
field: source.as.asn
131156
target_field: source.as.number
132157
ignore_missing: true
133158
- rename:
159+
tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b
134160
field: source.as.organization_name
135161
target_field: source.as.organization.name
136162
ignore_missing: true
137163
######################
138164
## ECS Base Mapping ##
139165
######################
140166
- date:
167+
tag: date_onepassword_timestamp_bea6075a
141168
field: onepassword.timestamp
142169
formats:
143170
- ISO8601
144171
#############
145172
## Cleanup ##
146173
#############
147174
- remove:
175+
tag: remove_49b6d979
148176
field:
149177
- onepassword.timestamp
150178
- onepassword.user

‎packages/1password/data_stream/item_usages/elasticsearch/ingest_pipeline/default.yml‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,63 +13,77 @@ processors:
1313
Removes the fields added by Agentless as metadata,
1414
as they can collide with ECS fields.
1515
- rename:
16+
tag: rename_message_to_event_original_56a77271
1617
field: message
1718
target_field: event.original
1819
ignore_missing: true
1920
if: ctx.event?.original == null
2021
- remove:
22+
tag: remove_message_2ee3a7d4
2123
field: message
2224
ignore_missing: true
2325
if: ctx.event?.original != null
2426
description: 'The `message` field is no longer required if the document has an `event.original` field.'
2527
- json:
28+
tag: json_event_original_to_onepassword_da036177
2629
field: event.original
2730
target_field: onepassword
2831
- drop:
32+
tag: drop_02acf80d
2933
description: Drop if no timestamp (invalid json)
3034
if: "ctx?.onepassword?.timestamp == null"
3135
#######################
3236
## ECS Event Mapping ##
3337
#######################
3438
- set:
39+
tag: set_ecs_version_0f7334bb
3540
field: ecs.version
3641
value: '8.11.0'
3742
# Sets event.created from the @timestamp field generated by filebeat before being overwritten further down
3843
- set:
44+
tag: set_event_created_e3f09e3b
3945
field: event.created
4046
copy_from: "@timestamp"
4147
- set:
48+
tag: set_event_kind_de80643c
4249
field: event.kind
4350
value: event
4451
- append:
52+
tag: append_event_category_a495f7f8
4553
field: event.category
4654
value: [file]
4755
- append:
56+
tag: append_event_type_8860cef4
4857
field: event.type
4958
value: [access]
5059
- rename:
60+
tag: rename_onepassword_action_to_event_action_24171185
5161
field: onepassword.action
5262
target_field: event.action
5363
ignore_missing: true
5464
#########################
5565
## ECS Related Mapping ##
5666
#########################
5767
- append:
68+
tag: append_related_user_6bd9ebd3
5869
field: related.user
5970
value: "{{{onepassword.user.uuid}}}"
6071
allow_duplicates: false
6172
if: ctx?.onepassword?.user?.uuid != null
6273
- append:
74+
tag: append_related_user_59c6eae3
6375
field: related.user
6476
value: "{{{onepassword.user.email}}}"
6577
allow_duplicates: false
6678
if: ctx?.onepassword?.user?.email != null
6779
- append:
80+
tag: append_related_user_03367e37
6881
field: related.user
6982
value: "{{{onepassword.user.name}}}"
7083
allow_duplicates: false
7184
if: ctx?.onepassword?.user?.name != null
7285
- append:
86+
tag: append_related_ip_9c1d4a47
7387
field: related.ip
7488
value: "{{{onepassword.client.ip_address}}}"
7589
allow_duplicates: false
@@ -78,43 +92,52 @@ processors:
7892
## ECS User Mapping ##
7993
######################
8094
- rename:
95+
tag: rename_onepassword_user_uuid_to_user_id_e1ae97c7
8196
field: onepassword.user.uuid
8297
target_field: user.id
8398
ignore_missing: true
8499
- rename:
100+
tag: rename_onepassword_user_name_to_user_full_name_14ca796f
85101
field: onepassword.user.name
86102
target_field: user.full_name
87103
ignore_missing: true
88104
- set:
105+
tag: set_user_name_67fd4e49
89106
field: user.name
90107
copy_from: user.full_name
91108
ignore_empty_value: true
92109
- rename:
110+
tag: rename_onepassword_user_email_to_user_email_de5b00df
93111
field: onepassword.user.email
94112
target_field: user.email
95113
ignore_missing: true
96114
####################
97115
## ECS OS Mapping ##
98116
####################
99117
- rename:
118+
tag: rename_onepassword_client_os_name_to_host_os_name_700b63f5
100119
field: onepassword.client.os_name
101120
target_field: host.os.name
102121
ignore_missing: true
103122
- rename:
123+
tag: rename_onepassword_client_os_version_to_host_os_version_245989d7
104124
field: onepassword.client.os_version
105125
target_field: host.os.version
106126
ignore_missing: true
107127
########################
108128
## ECS Source Mapping ##
109129
########################
110130
- rename:
131+
tag: rename_onepassword_client_ip_address_to_source_ip_40e01f74
111132
field: onepassword.client.ip_address
112133
target_field: source.ip
113134
ignore_missing: true
114135
- geoip:
136+
tag: geoip_source_ip_to_source_geo_0494c6b1
115137
field: source.ip
116138
target_field: source.geo
117139
- geoip:
140+
tag: geoip_source_ip_to_source_as_28d69883
118141
database_file: GeoLite2-ASN.mmdb
119142
field: source.ip
120143
target_field: source.as
@@ -123,24 +146,28 @@ processors:
123146
- organization_name
124147
ignore_missing: true
125148
- rename:
149+
tag: rename_source_as_asn_to_source_as_number_a917047d
126150
field: source.as.asn
127151
target_field: source.as.number
128152
ignore_missing: true
129153
- rename:
154+
tag: rename_source_as_organization_name_to_source_as_organization_name_f1362d0b
130155
field: source.as.organization_name
131156
target_field: source.as.organization.name
132157
ignore_missing: true
133158
######################
134159
## ECS Base Mapping ##
135160
######################
136161
- date:
162+
tag: date_onepassword_timestamp_bea6075a
137163
field: onepassword.timestamp
138164
formats:
139165
- ISO8601
140166
#############
141167
## Cleanup ##
142168
#############
143169
- remove:
170+
tag: remove_49b6d979
144171
field:
145172
- onepassword.timestamp
146173
- onepassword.user

0 commit comments

Comments
 (0)