Removed event.original removal processors in azure and aws

andrewkroh · Sep 23, 2024 · Jul 8, 2024 · Jul 8, 2024 · Jul 8, 2024 · Aug 26, 2024
commit f75f2cf294d3e511aad4d665e5bc19e81049bfe1
@@ -81,11 +81,6 @@ processors:
       field: aws.apigateway.status
       type: long
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
   - date:
       field: aws.apigateway.request_time
       target_field: "aws.apigateway.request_time"

@@ -187,11 +187,6 @@ processors:
   - remove:
       field: _tmp
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: 'error.message'

@@ -779,11 +779,6 @@ processors:
   - remove:
       field: json
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: error.message

@@ -30,11 +30,6 @@ processors:
       field:
         - _tmp
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: 'error.message'

@@ -221,11 +221,6 @@ processors:
       field:
         - _tmp
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: 'error.message'

@@ -51,11 +51,6 @@ processors:
           return false;
         }
         drop(ctx);
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: 'error.message'

@@ -387,11 +387,6 @@ processors:
   - remove:
       field: json
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: 'ctx?.tags == null || !(ctx.tags.contains("preserve_original_event"))'
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: error.message

@@ -2451,10 +2451,6 @@ processors:
                 - _ingest._value.file.path
                 - _ingest._value.hash
               ignore_missing: true
-  - remove:
-      field: event.original
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
-      ignore_missing: true
   - script:
       description: Drops null/empty values recursively.
       lang: painless

@@ -701,10 +701,6 @@ processors:
             - _ingest._value.version
           if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields'))
           ignore_missing: true
-  - remove:
-      field: event.original
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
-      ignore_missing: true
   - script:
       description: Drops null/empty values recursively.
       lang: painless

@@ -119,11 +119,6 @@ processors:
       field:
         - _tmp
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: 'error.message'

@@ -285,11 +285,6 @@ processors:
         list.removeIf(v -> v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0));
       }
       handleMap(ctx);
-- remove:
-    field: event.original
-    if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-    ignore_failure: true
-    ignore_missing: true
 on_failure:
 - set:
     field: 'error.message'

@@ -215,11 +215,6 @@ processors:
           }
         }
         handleMap(ctx);
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: 'error.message'

@@ -2137,11 +2137,6 @@ processors:
           ignore_failure: true
           ignore_missing: true
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
-      ignore_failure: true
-      ignore_missing: true
   - script:
       description: Drops null/empty values recursively.
       lang: painless

@@ -987,11 +987,6 @@ processors:
       field:
         - json
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
-      ignore_failure: true
-      ignore_missing: true
   - script:
       description: Drops null/empty values recursively.
       lang: painless

@@ -304,11 +304,6 @@ processors:
         - aws.vpcflow.packets
         - aws.vpcflow.protocol
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: 'error.message'

@@ -197,11 +197,6 @@ processors:
             }
         }
         handleMap(ctx);
-- remove:
-    field: event.original
-    if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-    ignore_failure: true
-    ignore_missing: true
 on_failure:
   - set:
       field: 'error.message'

@@ -478,11 +478,6 @@ processors:
       tag: remove_duplicate_custom_fields
       ignore_missing: true
       if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields'))
-  - remove:
-      field: event.original
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
-      ignore_failure: true
-      ignore_missing: true
   - script:
       description: Remove massive event.original if it was wanted.
       if: ctx.event?.original instanceof String && ctx.event.original.length() > 32766

@@ -315,11 +315,6 @@ processors:
       value: event
   - pipeline:
       name: '{{ IngestPipeline "azure-shared-pipeline" }}'
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: error.message

@@ -282,11 +282,6 @@ processors:
       field: observer.name
       copy_from: azure.resource.name
       ignore_empty_value: true
-  - remove:
-      field: event.original
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
-      ignore_failure: true
-      ignore_missing: true
   - script:
       description: Drops null/empty values recursively.
       lang: painless

@@ -231,11 +231,6 @@ processors:
       value: event
   - pipeline:
       name: '{{ IngestPipeline "azure-shared-pipeline" }}'
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: error.message

@@ -42,11 +42,6 @@ processors:
   - pipeline:
       name: '{{ IngestPipeline "azure-shared-pipeline" }}'
       ignore_failure: true
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: error.message

@@ -243,11 +243,6 @@ processors:
       field: observer.name
       copy_from: azure.resource.name
       ignore_empty_value: true
-  - remove:
-      field: event.original
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
-      ignore_failure: true
-      ignore_missing: true
   - script:
       description: Drops null/empty values recursively.
       lang: painless

@@ -429,11 +429,6 @@ processors:
       field: client.as.organization_name
       target_field: client.as.organization.name
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_missing: true
-      tag: remove-event_original
   - script:
       tag: script_to_drop_null_values
       lang: painless

@@ -289,11 +289,6 @@ processors:
       value: event
   - pipeline:
       name: '{{ IngestPipeline "azure-shared-pipeline" }}'
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: error.message

@@ -275,11 +275,6 @@ processors:
       value: event
   - pipeline:
       name: '{{ IngestPipeline "azure-shared-pipeline" }}'
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
   - pipeline:
       if: "ctx?.azure_log_forwarder?.resource_type != null && (ctx?.azure_log_forwarder?.resource_type == 'Microsoft.AppPlatform/Spring'|| ctx?.azure_log_forwarder?.resource_type == 'MICROSOFT.APPPLATFORM/SPRING')"
       name: '{{ IngestPipeline "springcloudlogs-inner-pipeline" }}'

@@ -368,11 +368,6 @@ processors:
       value: event
   - pipeline:
       name: '{{ IngestPipeline "azure-shared-pipeline" }}'
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: error.message

@@ -286,11 +286,6 @@ processors:
 
   - pipeline:
       name: '{{ IngestPipeline "azure-shared-pipeline" }}'
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: error.message

@@ -301,11 +301,6 @@ processors:
       value: event
   - pipeline:
       name: '{{ IngestPipeline "azure-shared-pipeline" }}'
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: error.message

@@ -175,11 +175,6 @@ processors:
   - user_agent:
       field: user_agent.original
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 
 on_failure:
   - set:

@@ -115,11 +115,6 @@ processors:
       field: source.as.organization_name
       target_field: source.as.organization.name
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 
 on_failure:
   - set:

@@ -123,11 +123,6 @@ processors:
       ignore_missing: true
   - pipeline:
       name: '{{ IngestPipeline "azure-shared-pipeline" }}'
-  - remove:
-      field: event.original
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
-      ignore_failure: true
-      ignore_missing: true
   - script:
       description: Drops null/empty values recursively.
       lang: painless

@@ -712,11 +712,6 @@ processors:
       field: json
       tag: remove_json
       ignore_missing: true
-  - remove:
-      field: event.original
-      tag: remove_event_original
-      ignore_missing: true
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
   - pipeline:
       name: '{{ IngestPipeline "azure_shared_pipeline" }}'
       if: ctx.azure_network_watcher_nsg?.log?.resource_id != null

@@ -699,11 +699,6 @@ processors:
       field: json
       tag: remove_json
       ignore_missing: true
-  - remove:
-      field: event.original
-      tag: remove_event_original
-      ignore_missing: true
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
   - pipeline:
       name: '{{ IngestPipeline "azure_shared_pipeline" }}'
       if: ctx.azure_network_watcher_vnet?.log?.flow_log?.resource_id != null