Skip to content

[gcp]: Preserve gcp field data in flattened fields #3390

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
r00tu53r merged 12 commits into from
May 20, 2022
Merged

[gcp]: Preserve gcp field data in flattened fields #3390

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
9a7b097
Update fields
r00tu53r May 17, 2022
b5b7e2f
remove extraneous spaces
r00tu53r May 18, 2022
609b8d4
* Pipeline refactor and ECS updates
r00tu53r May 19, 2022
e64753f
add .labels as flattened field
r00tu53r May 19, 2022
26dbe3f
add severity to log.level
r00tu53r May 19, 2022
707f579
fix pipeline and system tests
r00tu53r May 19, 2022
3af2462
update readme
r00tu53r May 20, 2022
dbdb69e
Update version, changelog; client.user.id fix.
r00tu53r May 20, 2022
b169079
re-run system tests; regen readme
r00tu53r May 20, 2022
eea507d
fix principal_email
r00tu53r May 20, 2022
64ae9af
Add description to labels field
r00tu53r May 20, 2022
f3a3f27
fix bbuild error
r00tu53r May 20, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fix principal_email
  • Loading branch information
@r00tu53r
r00tu53r committed May 20, 2022
commit eea507da3ec3c068a03a48f1fec6dc7b7cbff2bf
40 changes: 40 additions & 0 deletions packages/gcp/data_stream/audit/_dev/test/pipeline/test-audit.log-expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,9 @@
},
"gcp": {
"audit": {
"authentication_info": {
"principal_email": "xxx@xxx.xxx"
},
"authorization_info": [
{
"granted": true,
Expand Down Expand Up @@ -99,6 +102,9 @@
},
"gcp": {
"audit": {
"authentication_info": {
"principal_email": "xxx@xxx.xxx"
},
"authorization_info": [
{
"granted": false,
Expand Down Expand Up @@ -189,6 +195,9 @@
},
"gcp": {
"audit": {
"authentication_info": {
"principal_email": "xxx@xxx.xxx"
},
"authorization_info": [
{
"granted": true,
Expand Down Expand Up @@ -287,6 +296,9 @@
},
"gcp": {
"audit": {
"authentication_info": {
"principal_email": "xxx@xxx.xxx"
},
"authorization_info": [
{
"permission": "compute.instances.list",
Expand Down Expand Up @@ -380,6 +392,9 @@
},
"gcp": {
"audit": {
"authentication_info": {
"principal_email": "system:serviceaccount:cert-manager:cert-manager-webhook"
},
"authorization_info": [
{
"granted": true,
Expand Down Expand Up @@ -515,6 +530,9 @@
},
"gcp": {
"audit": {
"authentication_info": {
"principal_email": "user@mycompany.com"
},
"authorization_info": [
{
"granted": true,
Expand Down Expand Up @@ -648,6 +666,9 @@
},
"gcp": {
"audit": {
"authentication_info": {
"principal_email": "user@mycompany.com"
},
"logentry_operation": {
"id": "operation-1596646123456-5ac2438b775f6-f8ca1382-e70b6831"
},
Expand Down Expand Up @@ -736,6 +757,9 @@
},
"gcp": {
"audit": {
"authentication_info": {
"principal_email": "xxx@xxx.xxx"
},
"authorization_info": [
{
"granted": true,
Expand Down Expand Up @@ -818,6 +842,9 @@
},
"gcp": {
"audit": {
"authentication_info": {
"principal_email": "xxx@xxx.xxx"
},
"authorization_info": [
{
"granted": true,
Expand Down Expand Up @@ -903,6 +930,9 @@
},
"gcp": {
"audit": {
"authentication_info": {
"principal_email": "system:anonymous"
},
"authorization_info": [
{
"granted": true,
Expand Down Expand Up @@ -985,6 +1015,9 @@
},
"gcp": {
"audit": {
"authentication_info": {
"principal_email": "system:serviceaccount:kube-system:generic-garbage-collector"
},
"authorization_info": [
{
"granted": true,
Expand Down Expand Up @@ -1070,6 +1103,7 @@
"gcp": {
"audit": {
"authentication_info": {
"principal_email": "xxx@xxx.xxx",
"principal_subject": "sub"
},
"authorization_info": [
Expand Down Expand Up @@ -1144,6 +1178,9 @@
},
"gcp": {
"audit": {
"authentication_info": {
"principal_email": "xxx@xxx.xxx"
},
"authorization_info": [
{
"granted": true,
Expand Down Expand Up @@ -1300,6 +1337,9 @@
},
"gcp": {
"audit": {
"authentication_info": {
"principal_email": "system:addon-manager"
},
"authorization_info": [
{
"granted": true,
Expand Down
2 changes: 1 addition & 1 deletion packages/gcp/data_stream/audit/elasticsearch/ingest_pipeline/default.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ processors:
- set:
field: gcp.audit.authentication_info.principal_email
copy_from: client.user.email
if: ctx.user?.email != null
if: ctx.client?.user?.email != null
- set:
field: gcp.audit.authentication_info.principal_subject
copy_from: client.user.id
Expand Down
13 changes: 8 additions & 5 deletions packages/gcp/data_stream/audit/sample_event.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
{
"@timestamp": "2019-12-19T00:44:25.051Z",
"agent": {
"ephemeral_id": "c80ea3eb-6a00-4668-ba41-ea9ebe4bc255",
"ephemeral_id": "15ffa48e-049a-4ead-9716-cea0236748c4",
"hostname": "docker-fleet-agent",
"id": "881eed9e-f1d0-4e58-9efd-819f27ff81e7",
"id": "df142714-8028-4ef0-a80c-4eb03051c084",
"name": "docker-fleet-agent",
"type": "filebeat",
"version": "7.17.0"
Expand All @@ -28,7 +28,7 @@
"version": "8.2.0"
},
"elastic_agent": {
"id": "881eed9e-f1d0-4e58-9efd-819f27ff81e7",
"id": "df142714-8028-4ef0-a80c-4eb03051c084",
"snapshot": false,
"version": "7.17.0"
},
Expand All @@ -39,10 +39,10 @@
"network",
"configuration"
],
"created": "2022-05-20T05:48:23.008Z",
"created": "2022-05-20T07:25:00.534Z",
"dataset": "gcp.audit",
"id": "yonau2dg2zi",
"ingested": "2022-05-20T05:48:26Z",
"ingested": "2022-05-20T07:25:01Z",
"kind": "event",
"outcome": "success",
"provider": "data_access",
Expand All @@ -53,6 +53,9 @@
},
"gcp": {
"audit": {
"authentication_info": {
"principal_email": "xxx@xxx.xxx"
},
"authorization_info": [
{
"granted": true,
Expand Down
10 changes: 5 additions & 5 deletions packages/gcp/data_stream/dns/sample_event.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
{
"@timestamp": "2022-01-23T09:16:05.341Z",
"agent": {
"ephemeral_id": "21a4eccb-1fd9-4d8b-beef-3c78c32f03f6",
"ephemeral_id": "0d2f83ac-67e6-454f-84eb-859aa503167a",
"hostname": "docker-fleet-agent",
"id": "881eed9e-f1d0-4e58-9efd-819f27ff81e7",
"id": "df142714-8028-4ef0-a80c-4eb03051c084",
"name": "docker-fleet-agent",
"type": "filebeat",
"version": "7.17.0"
Expand Down Expand Up @@ -50,16 +50,16 @@
"version": "8.2.0"
},
"elastic_agent": {
"id": "881eed9e-f1d0-4e58-9efd-819f27ff81e7",
"id": "df142714-8028-4ef0-a80c-4eb03051c084",
"snapshot": false,
"version": "7.17.0"
},
"event": {
"agent_id_status": "verified",
"created": "2022-05-20T05:49:10.978Z",
"created": "2022-05-20T07:25:43.755Z",
"dataset": "gcp.dns",
"id": "vwroyze8pg7y",
"ingested": "2022-05-20T05:49:11Z",
"ingested": "2022-05-20T07:25:44Z",
"kind": "event",
"outcome": "success"
},
Expand Down
10 changes: 5 additions & 5 deletions packages/gcp/data_stream/firewall/sample_event.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
{
"@timestamp": "2019-10-30T13:52:42.191Z",
"agent": {
"ephemeral_id": "142bc200-4e4b-4ea0-af2a-acad9400e523",
"ephemeral_id": "1f7633a7-3410-4684-bb55-14b0bd0e2bd4",
"hostname": "docker-fleet-agent",
"id": "881eed9e-f1d0-4e58-9efd-819f27ff81e7",
"id": "df142714-8028-4ef0-a80c-4eb03051c084",
"name": "docker-fleet-agent",
"type": "filebeat",
"version": "7.17.0"
Expand Down Expand Up @@ -31,18 +31,18 @@
"version": "8.2.0"
},
"elastic_agent": {
"id": "881eed9e-f1d0-4e58-9efd-819f27ff81e7",
"id": "df142714-8028-4ef0-a80c-4eb03051c084",
"snapshot": false,
"version": "7.17.0"
},
"event": {
"action": "firewall-rule",
"agent_id_status": "verified",
"category": "network",
"created": "2022-05-20T05:49:52.624Z",
"created": "2022-05-20T07:26:27.445Z",
"dataset": "gcp.firewall",
"id": "1f21ciqfpfssuo",
"ingested": "2022-05-20T05:49:53Z",
"ingested": "2022-05-20T07:26:28Z",
"kind": "event",
"type": "connection"
},
Expand Down
10 changes: 5 additions & 5 deletions packages/gcp/data_stream/vpcflow/sample_event.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
{
"@timestamp": "2019-06-14T03:50:10.845Z",
"agent": {
"ephemeral_id": "f2d2334d-ddaa-4dea-9022-710c606f382e",
"ephemeral_id": "10bb82a5-c0e6-4aed-8589-003f734a7183",
"hostname": "docker-fleet-agent",
"id": "881eed9e-f1d0-4e58-9efd-819f27ff81e7",
"id": "df142714-8028-4ef0-a80c-4eb03051c084",
"name": "docker-fleet-agent",
"type": "filebeat",
"version": "7.17.0"
Expand Down Expand Up @@ -43,18 +43,18 @@
"version": "8.2.0"
},
"elastic_agent": {
"id": "881eed9e-f1d0-4e58-9efd-819f27ff81e7",
"id": "df142714-8028-4ef0-a80c-4eb03051c084",
"snapshot": false,
"version": "7.17.0"
},
"event": {
"agent_id_status": "verified",
"category": "network",
"created": "2022-05-20T05:50:37.694Z",
"created": "2022-05-20T07:27:09.739Z",
"dataset": "gcp.vpcflow",
"end": "2019-06-14T03:49:51.821308944Z",
"id": "ut8lbrffooxyp",
"ingested": "2022-05-20T05:50:38Z",
"ingested": "2022-05-20T07:27:10Z",
"kind": "event",
"start": "2019-06-14T03:40:08.469099728Z",
"type": "connection"
Expand Down
Loading