Skip to content

[gcp]: Preserve gcp field data in flattened fields #3390

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
r00tu53r merged 12 commits into from
May 20, 2022
Merged

[gcp]: Preserve gcp field data in flattened fields #3390

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
9a7b097
Update fields
r00tu53r May 17, 2022
b5b7e2f
remove extraneous spaces
r00tu53r May 18, 2022
609b8d4
* Pipeline refactor and ECS updates
r00tu53r May 19, 2022
e64753f
add .labels as flattened field
r00tu53r May 19, 2022
26dbe3f
add severity to log.level
r00tu53r May 19, 2022
707f579
fix pipeline and system tests
r00tu53r May 19, 2022
3af2462
update readme
r00tu53r May 20, 2022
dbdb69e
Update version, changelog; client.user.id fix.
r00tu53r May 20, 2022
b169079
re-run system tests; regen readme
r00tu53r May 20, 2022
eea507d
fix principal_email
r00tu53r May 20, 2022
64ae9af
Add description to labels field
r00tu53r May 20, 2022
f3a3f27
fix bbuild error
r00tu53r May 20, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
re-run system tests; regen readme
  • Loading branch information
@r00tu53r
r00tu53r committed May 20, 2022
commit b169079b22bf515d36ba0a08303fd4a82957cc4f
33 changes: 23 additions & 10 deletions packages/gcp/data_stream/audit/sample_event.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"@timestamp": "2019-12-19T00:44:25.051Z",
"agent": {
"ephemeral_id": "88282319-d625-409f-a07a-53875ef781fa",
"ephemeral_id": "c80ea3eb-6a00-4668-ba41-ea9ebe4bc255",
"hostname": "docker-fleet-agent",
"id": "881eed9e-f1d0-4e58-9efd-819f27ff81e7",
"name": "docker-fleet-agent",
Expand Down Expand Up @@ -39,18 +39,23 @@
"network",
"configuration"
],
"created": "2022-05-20T05:06:26.185Z",
"created": "2022-05-20T05:48:23.008Z",
"dataset": "gcp.audit",
"id": "yonau3dc2zi",
"ingested": "2022-05-20T05:06:27Z",
"id": "yonau2dg2zi",
"ingested": "2022-05-20T05:48:26Z",
"kind": "event",
"outcome": "failure",
"provider": "data_access"
"outcome": "success",
"provider": "data_access",
"type": [
"access",
"allowed"
]
},
"gcp": {
"audit": {
"authorization_info": [
{
"granted": true,
"permission": "compute.instances.list",
"resource_attributes": {
"name": "projects/elastic-beats",
Expand All @@ -73,11 +78,19 @@
]
},
"resource_name": "projects/elastic-beats/global/instances",
"service_name": "compute.googleapis.com",
"status": {
"code": 7,
"message": "PERMISSION_DENIED"
"response": {
"@type": "core.k8s.io/v1.Status",
"apiVersion": "v1",
"details": {
"group": "batch",
"kind": "jobs",
"name": "gsuite-exporter-1589294700",
"uid": "2beff34a-945f-11ea-bacf-42010a80007f"
},
"kind": "Status",
"status": "Success"
},
"service_name": "compute.googleapis.com",
"type": "type.googleapis.com/google.cloud.audit.AuditLog"
}
},
Expand Down
6 changes: 3 additions & 3 deletions packages/gcp/data_stream/dns/sample_event.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"@timestamp": "2022-01-23T09:16:05.341Z",
"agent": {
"ephemeral_id": "0a8788f9-39fb-4af0-bdb3-8ecfc2ee6454",
"ephemeral_id": "21a4eccb-1fd9-4d8b-beef-3c78c32f03f6",
"hostname": "docker-fleet-agent",
"id": "881eed9e-f1d0-4e58-9efd-819f27ff81e7",
"name": "docker-fleet-agent",
Expand Down Expand Up @@ -56,10 +56,10 @@
},
"event": {
"agent_id_status": "verified",
"created": "2022-05-20T05:07:10.864Z",
"created": "2022-05-20T05:49:10.978Z",
"dataset": "gcp.dns",
"id": "vwroyze8pg7y",
"ingested": "2022-05-20T05:07:11Z",
"ingested": "2022-05-20T05:49:11Z",
"kind": "event",
"outcome": "success"
},
Expand Down
6 changes: 3 additions & 3 deletions packages/gcp/data_stream/firewall/sample_event.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"@timestamp": "2019-10-30T13:52:42.191Z",
"agent": {
"ephemeral_id": "101e0b46-fd1a-4fa3-a18c-05f024eb84b6",
"ephemeral_id": "142bc200-4e4b-4ea0-af2a-acad9400e523",
"hostname": "docker-fleet-agent",
"id": "881eed9e-f1d0-4e58-9efd-819f27ff81e7",
"name": "docker-fleet-agent",
Expand Down Expand Up @@ -39,10 +39,10 @@
"action": "firewall-rule",
"agent_id_status": "verified",
"category": "network",
"created": "2022-05-20T05:07:49.277Z",
"created": "2022-05-20T05:49:52.624Z",
"dataset": "gcp.firewall",
"id": "1f21ciqfpfssuo",
"ingested": "2022-05-20T05:07:50Z",
"ingested": "2022-05-20T05:49:53Z",
"kind": "event",
"type": "connection"
},
Expand Down
91 changes: 55 additions & 36 deletions packages/gcp/data_stream/vpcflow/sample_event.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,26 +1,43 @@
{
"@timestamp": "2019-06-14T03:50:10.845Z",
"agent": {
"ephemeral_id": "1696b0fc-5ccf-41e8-adc7-0eb7a8526939",
"ephemeral_id": "f2d2334d-ddaa-4dea-9022-710c606f382e",
"hostname": "docker-fleet-agent",
"id": "881eed9e-f1d0-4e58-9efd-819f27ff81e7",
"name": "docker-fleet-agent",
"type": "filebeat",
"version": "7.17.0"
},
"cloud": {
"provider": "gcp"
"availability_zone": "us-east1-b",
"project": {
"id": "my-sample-project"
},
"provider": "gcp",
"region": "us-east1"
},
"data_stream": {
"dataset": "gcp.vpcflow",
"namespace": "ep",
"type": "logs"
},
"destination": {
"address": "10.87.40.76",
"domain": "kibana",
"ip": "10.87.40.76",
"port": 5601
"address": "67.43.156.14",
"as": {
"number": 35908
},
"domain": "elasticsearch",
"geo": {
"continent_name": "Asia",
"country_iso_code": "BT",
"country_name": "Bhutan",
"location": {
"lat": 27.5,
"lon": 90.5
}
},
"ip": "67.43.156.14",
"port": 9200
},
"ecs": {
"version": "8.2.0"
Expand All @@ -33,13 +50,13 @@
"event": {
"agent_id_status": "verified",
"category": "network",
"created": "2022-05-20T05:08:34.070Z",
"created": "2022-05-20T05:50:37.694Z",
"dataset": "gcp.vpcflow",
"end": "2019-06-14T03:48:39.076420731Z",
"id": "ut8lbrffooxz7",
"ingested": "2022-05-20T05:08:35Z",
"end": "2019-06-14T03:49:51.821308944Z",
"id": "ut8lbrffooxyp",
"ingested": "2022-05-20T05:50:38Z",
"kind": "event",
"start": "2019-06-14T03:48:38.961050187Z",
"start": "2019-06-14T03:40:08.469099728Z",
"type": "connection"
},
"gcp": {
Expand All @@ -55,10 +72,22 @@
"vpc_name": "default"
}
},
"source": {
"instance": {
"project_id": "my-sample-project",
"region": "us-east1",
"zone": "us-east1-b"
},
"vpc": {
"project_id": "my-sample-project",
"subnetwork_name": "default",
"vpc_name": "default"
}
},
"vpcflow": {
"reporter": "DEST",
"reporter": "SRC",
"rtt": {
"ms": 36
"ms": 3
}
}
},
Expand All @@ -69,38 +98,28 @@
"logger": "projects/my-sample-project/logs/compute.googleapis.com%2Fvpc_flows"
},
"network": {
"bytes": 1467,
"community_id": "1:qoQEykwJ/Fqctc/3YyFJSUPTETc=",
"direction": "inbound",
"bytes": 15169,
"community_id": "1:NAY9D1IuyJAG+Hm34t3LIlP6/4c=",
"direction": "internal",
"iana_number": "6",
"packets": 7,
"name": "default",
"packets": 92,
"transport": "tcp",
"type": "ipv4"
},
"related": {
"ip": [
"67.43.156.13",
"10.87.40.76"
"10.87.40.76",
"67.43.156.14"
]
},
"source": {
"address": "67.43.156.13",
"as": {
"number": 35908
},
"bytes": 1467,
"geo": {
"continent_name": "Asia",
"country_iso_code": "BT",
"country_name": "Bhutan",
"location": {
"lat": 27.5,
"lon": 90.5
}
},
"ip": "67.43.156.13",
"packets": 7,
"port": 34836
"address": "10.87.40.76",
"bytes": 15169,
"domain": "kibana",
"ip": "10.87.40.76",
"packets": 92,
"port": 33880
},
"tags": [
"forwarded",
Expand Down
Loading