Skip to content

Commit 6f4ee21

Browse files
mrodmmrodm
authored
[backport-elastic_agent-2.5] Sync CI configuration with main branch (#21189)
Keep CI pipelines, GitHub Actions, and tooling consistent between active backport branches and `main`, so that automated backport workflows and tooling improvements land on all supported branches.
1 parent 50c274f commit 6f4ee21

195 files changed

Lines changed: 16171 additions & 3386 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.buildkite/hooks/pre-command‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,6 @@ export GO_VERSION
2525
REPO_NAME=$(repo_name "${BUILDKITE_REPO}")
2626
export REPO_NAME
2727

28-
export TMP_FOLDER_TEMPLATE_BASE="tmp.${REPO_NAME}"
29-
export TMP_FOLDER_TEMPLATE="${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX"
30-
3128
REPO_BUILD_TAG="${REPO_NAME}/$(buildkite_pr_branch_build_id)"
3229
export REPO_BUILD_TAG
3330

@@ -113,11 +110,6 @@ if [[ "${BUILDKITE_PIPELINE_SLUG}" =~ ^(integrations|integrations-test-stack)$ ]
113110
BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field buildkite_token "${BUILDKITE_API_TOKEN_PATH}")
114111
export BUILDKITE_API_TOKEN
115112
fi
116-
117-
if [[ "${BUILDKITE_STEP_KEY}" =~ ^test-integrations- ]]; then
118-
BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field buildkite_token "${BUILDKITE_API_TOKEN_PATH}")
119-
export BUILDKITE_API_TOKEN
120-
fi
121113
fi
122114

123115
if [[ "${BUILDKITE_PIPELINE_SLUG}" == "integrations-serverless" ]]; then

‎.buildkite/hooks/pre-exit‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,5 @@ fi
5050

5151
echo "--- Cleaning up"
5252
unset_secrets
53-
cleanup
5453

5554
exit "${exit_code}"

‎.buildkite/pipeline.backport-dispatch.yml‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
2+
3+
env:
4+
SETUP_GVM_VERSION: "v0.6.0"
5+
LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
6+
YQ_VERSION: 'v4.35.2'
7+
GH_CLI_VERSION: "2.29.0"
8+
NOTIFY_TO: "ecosystem-team@elastic.co"
9+
10+
steps:
11+
- label: ":ballot_box_with_check: Backports inventory validation"
12+
key: "check-backports-inventory"
13+
command: ".buildkite/scripts/check_backports_inventory.sh"
14+
agents:
15+
image: "${LINUX_AGENT_IMAGE}"
16+
if_changed:
17+
- ".backports.yml"
18+
- ".buildkite/scripts/run_dev_scripts_tests.sh"
19+
- "dev/scripts/**.sh"
20+
- "cmd/backport/**"
21+
if: |
22+
build.env('BUILDKITE_PULL_REQUEST') == "false" && build.branch == "main"
23+
24+
- label: ":git: Create backport branches for new entries"
25+
key: "trigger-backport-create"
26+
command: ".buildkite/scripts/trigger_backport.sh"
27+
agents:
28+
image: "${LINUX_AGENT_IMAGE}"
29+
plugins:
30+
- elastic/vault-github-token#v0.1.0:
31+
depends_on:
32+
- step: "check-backports-inventory"
33+
allow_failure: false
34+
if_changed:
35+
- ".backports.yml"
36+
if: |
37+
build.env('BUILDKITE_PULL_REQUEST') == "false" &&
38+
build.branch == "main" &&
39+
build.env('BUILDKITE_PIPELINE_SLUG') == "integrations-backport-dispatch"
40+
41+
notify:
42+
- email: "$NOTIFY_TO"
43+
if: "build.state == 'failed' && build.env('BUILDKITE_PULL_REQUEST') == 'false'"

‎.buildkite/pipeline.backport.yml‎

Lines changed: 50 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,21 +3,33 @@
33
name: "integrations-backport"
44

55
env:
6+
SETUP_GVM_VERSION: "v0.6.0"
67
YQ_VERSION: 'v4.35.2'
8+
GH_CLI_VERSION: "2.29.0"
79
# Agent images used in pipeline steps
810
LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
11+
NOTIFY_TO: "ecosystem-team@elastic.co"
912

1013
steps:
1114

1215
- label: "Check that it runs from UI"
1316
key: "check-ui"
1417
command:
15-
- "buildkite-agent annotate \"The $BUILDKITE_PIPELINE_SLUG is used only for running from UI!\" --style 'warning'"
18+
- "buildkite-agent annotate \"The $${BUILDKITE_PIPELINE_SLUG} pipeline can only be triggered from the UI by members of the 'ecosystem' team, or via a trigger step from the 'integrations' or 'integrations-backport-dispatch' pipeline. For the recommended workflow see the [documentation](https://www.elastic.co/guide/en/integrations-developer/current/developer-workflow-support-old-package.html).\" --style 'warning'"
1619
- "exit 1"
17-
if: "build.source != 'ui'"
20+
if: |
21+
!(
22+
(build.source == 'ui' && build.creator.teams includes "ecosystem") ||
23+
(build.source == 'trigger_job' && build.env('BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG') == 'integrations') ||
24+
(build.source == 'trigger_job' && build.env('BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG') == 'integrations-backport-dispatch')
25+
)
26+
27+
# Ensure that the check-ui step runs before any other step
28+
- wait: ~
1829

1930
- input: "Input values for the variables"
2031
key: "input-variables"
32+
if: "build.source != 'trigger_job'"
2133
fields:
2234
- select: "DRY_RUN"
2335
key: "DRY_RUN"
@@ -35,10 +47,6 @@ steps:
3547
key: "PACKAGE_NAME"
3648
required: true
3749
default: ""
38-
- text: "Enter name of the folder for the package (in most cases coincides with PACKAGE_NAME)"
39-
key: "PACKAGE_FOLDER_NAME"
40-
required: true
41-
default: ""
4250
- text: "Enter package version (examples: 1.5.7, 1.0.0-beta1)"
4351
key: "PACKAGE_VERSION"
4452
required: true
@@ -50,7 +58,15 @@ steps:
5058
value: "true"
5159
- label: "False"
5260
value: "false"
53-
default: "false"
61+
default: "true"
62+
- text: "Backport branch name (optional, defaults to backport-<package>-<major>.<minor>)"
63+
key: "BACKPORT_BRANCH_NAME"
64+
required: false
65+
default: ""
66+
- text: "PR number to notify on completion (optional)"
67+
key: "PR_NUMBER"
68+
required: false
69+
default: ""
5470

5571
- label: "Creating the backport branch"
5672
key: "create-backport-branch"
@@ -65,3 +81,30 @@ steps:
6581
depends_on:
6682
- step: "input-variables"
6783
allow_failure: false
84+
85+
- label: ":github: Notify PR"
86+
key: "notify-pr"
87+
command: |
88+
if ! outcome=$$(buildkite-agent step get "outcome" --step "create-backport-branch"); then
89+
echo "Failed to retrieve step outcome, skipping PR notification"
90+
exit 0
91+
fi
92+
if [[ "$${outcome}" == "passed" ]]; then
93+
NOTIFY_STATUS=success .buildkite/scripts/notify_backport_pr.sh
94+
else
95+
NOTIFY_STATUS=failure .buildkite/scripts/notify_backport_pr.sh
96+
fi
97+
agents:
98+
image: "${LINUX_AGENT_IMAGE}"
99+
plugins:
100+
- elastic/vault-github-token#v0.1.0:
101+
depends_on:
102+
- step: "create-backport-branch"
103+
allow_failure: true
104+
105+
notify:
106+
# Only notify for automated backport creation failures (triggered by integrations-backport-dispatch).
107+
# Dry-run failures from PR builds (triggered by integrations) are already surfaced as a failing
108+
# CI status on the PR itself and do not warrant a team email.
109+
- email: "$NOTIFY_TO"
110+
if: "build.state == 'failed' && build.source == 'trigger_job' && build.env('BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG') == 'integrations-backport-dispatch'"

‎.buildkite/pipeline.publish.yml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
22

33
env:
4-
SETUP_GVM_VERSION: "v0.5.2"
4+
SETUP_GVM_VERSION: "v0.6.0"
55
LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
66
DOCKER_COMPOSE_VERSION: "v2.24.1"
77
DOCKER_VERSION: "false"
@@ -38,6 +38,9 @@ steps:
3838
ARTIFACTS_FOLDER: "artifacts-to-sign"
3939
# by default it will publish packages
4040
DRY_RUN: "${DRY_RUN:-false}"
41+
# Comma-separated list of package zip filenames to skip (e.g. "foo-1.0.0.zip,bar-2.0.0.zip")
42+
# kubeletstats_input_otel-0.1.0 was deleted but it is kept in the repository for now, in case it is needed to release a new version if needed
43+
SKIP_PACKAGES: "kubeletstats_input_otel-0.1.0.zip"
4144
depends_on:
4245
- step: "check"
4346
allow_failure: false

‎.buildkite/pipeline.schedule-daily.yml‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
name: integrations-schedule-daily
33

44
env:
5-
SETUP_GVM_VERSION: "v0.5.2"
5+
SETUP_GVM_VERSION: "v0.6.0"
66
LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
77

88
# The pipeline is triggered by the scheduler every day
@@ -34,8 +34,9 @@ steps:
3434
env:
3535
SERVERLESS: "false"
3636
FORCE_CHECK_ALL: "true"
37-
STACK_VERSION: 8.19.4-SNAPSHOT
37+
STACK_VERSION: 8.19.22-SNAPSHOT
3838
PUBLISH_COVERAGE_REPORTS: "true"
39+
ELASTIC_PACKAGE_KIBANA_SKIP_UPLOAD_PACKAGE_VALIDATION: "true"
3940
depends_on:
4041
- step: "check"
4142
allow_failure: false
@@ -48,9 +49,10 @@ steps:
4849
env:
4950
SERVERLESS: "false"
5051
FORCE_CHECK_ALL: "true"
51-
STACK_VERSION: 8.19.4-SNAPSHOT
52+
STACK_VERSION: 8.19.22-SNAPSHOT
5253
STACK_LOGSDB_ENABLED: "true"
5354
PUBLISH_COVERAGE_REPORTS: "false"
55+
ELASTIC_PACKAGE_KIBANA_SKIP_UPLOAD_PACKAGE_VALIDATION: "true"
5456
depends_on:
5557
- step: "check"
5658
allow_failure: false
@@ -86,13 +88,13 @@ steps:
8688
if: |
8789
build.env('TEST_PACKAGES_BASIC_SUBSCRIPTION') == "true"
8890
89-
- label: "Check integrations local stacks - Stack Version v9.2"
91+
- label: "Check integrations local stacks - Stack Version v9.6"
9092
trigger: "integrations"
9193
build:
9294
env:
9395
SERVERLESS: "false"
9496
FORCE_CHECK_ALL: "true"
95-
STACK_VERSION: 9.2.0-SNAPSHOT
97+
STACK_VERSION: 9.6.0-SNAPSHOT
9698
PUBLISH_COVERAGE_REPORTS: "false"
9799
depends_on:
98100
- step: "check"

‎.buildkite/pipeline.schedule-weekly.yml‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
name: integrations-schedule-weekly
33

44
env:
5-
SETUP_GVM_VERSION: "v0.5.2"
5+
SETUP_GVM_VERSION: "v0.6.0"
66
LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
77

88
# The pipeline is triggered by the scheduler every week
@@ -21,20 +21,21 @@ steps:
2121
env:
2222
SERVERLESS: "false"
2323
FORCE_CHECK_ALL: "true"
24-
STACK_VERSION: 8.19.4-SNAPSHOT
24+
STACK_VERSION: 8.19.22-SNAPSHOT
2525
PUBLISH_COVERAGE_REPORTS: "false"
2626
ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI: "true"
27+
ELASTIC_PACKAGE_KIBANA_SKIP_UPLOAD_PACKAGE_VALIDATION: "true"
2728
depends_on:
2829
- step: "check"
2930
allow_failure: false
3031

31-
- label: "Check integrations local stacks and non-wolfi images for Elastic Agent - Stack Version v9.2"
32+
- label: "Check integrations local stacks and non-wolfi images for Elastic Agent - Stack Version v9.6"
3233
trigger: "integrations"
3334
build:
3435
env:
3536
SERVERLESS: "false"
3637
FORCE_CHECK_ALL: "true"
37-
STACK_VERSION: 9.2.0-SNAPSHOT
38+
STACK_VERSION: 9.6.0-SNAPSHOT
3839
PUBLISH_COVERAGE_REPORTS: "false"
3940
ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI: "true"
4041
depends_on:

‎.buildkite/pipeline.serverless.yml‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
11
# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
22

33
env:
4-
SETUP_GVM_VERSION: "v0.5.2"
4+
SETUP_GVM_VERSION: "v0.6.0"
55
LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
66
DOCKER_COMPOSE_VERSION: "v2.24.1"
77
DOCKER_VERSION: "false" # not required to set since system tests are not running yet
88
KIND_VERSION: 'v0.27.0'
99
K8S_VERSION: 'v1.33.0'
1010
YQ_VERSION: 'v4.35.2'
11-
IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204"
11+
IMAGE_UBUNTU_X86_64: "family/platform-obs-integrations-ubuntu-2204"
1212
GH_CLI_VERSION: "2.29.0"
1313
# This pipeline is intended to test packages with Elastic Serverless
1414
SERVERLESS: true
@@ -30,6 +30,8 @@ env:
3030
ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI: "${ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI:-false}"
3131
# Disable checking for newer versions
3232
ELASTIC_PACKAGE_CHECK_UPDATE_DISABLED: "true"
33+
# Required to install packages via API even if they are already present in EPR (only available in QA)
34+
ELASTIC_PACKAGE_SERVERLESS_KIBANA_SKIP_UPLOAD_PACKAGE_VALIDATION: "true"
3335

3436
steps:
3537
- input: "Input values for the variables"
@@ -71,14 +73,13 @@ steps:
7173
# See https://github.com/elastic/oblt-infra/blob/main/conf/resources/repos/integrations/01-aws-buildkite-oidc.tf
7274
# This plugin creates the environment variables required by the service deployer (AWS_SECRET_ACCESS_KEY and AWS_SECRET_KEY_ID)
7375
- elastic/oblt-aws-auth#v0.1.0:
74-
duration: 10800 # seconds
76+
duration: 21600 # seconds
7577
# See https://github.com/elastic/oblt-infra/blob/main/conf/resources/repos/integrations/01-gcp-buildkite-oidc.tf
7678
# This plugin authenticates to CI Google Cloud using the OIDC token.
7779
- elastic/oblt-google-auth#v1.3.0:
78-
lifetime: 10800 # seconds
80+
lifetime: 21600 # seconds
7981
project-id: "elastic-observability-ci"
8082
project-number: "911195782929"
81-
lifetime: 10800 # seconds
8283
- avaly/gcp-secret-manager#v1.2.0:
8384
env:
8485
EC_API_KEY: elastic-cloud-observability-team-qa-api-key
@@ -109,6 +110,8 @@ steps:
109110
- label: ":github: Report failed tests"
110111
key: report-failed-tests
111112
command: ".buildkite/scripts/report_issues.sh"
113+
env:
114+
CI_MAX_TESTS_REPORTED: 30
112115
agents:
113116
image: "${LINUX_AGENT_IMAGE}"
114117
cpu: "8"

0 commit comments

Comments
 (0)