Skip to content

Commit f1f2943

Browse files
openshift-merge-bot[bot]openshift-merge-bot[bot]
authored
Merge pull request #46 from shiftstack/fix/openstack-test-ote-batch-trust-resolver
openstack_test: trust OTE resolver exits; isolate serial coda from LB
2 parents a52cf8b + 9ba59bf commit f1f2943

6 files changed

Lines changed: 269 additions & 70 deletions

File tree

‎collection/stages/roles/lb_tests/tasks/main.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,10 @@
9999
openstack_tests_allowlist_file: "{{ lb_ovn_allowlist_resolved }}"
100100
openstack_tests_blocklist_file: "{{ lb_ingress_blocklist_resolved }}"
101101
openstack_test_ote_run_serial: true
102+
# LB allowlists can include egressIP (a coda matcher) without the Machine
103+
# leak cluster; serial-coda uniqueness then fails. Coda belongs on the
104+
# main openstack_test stage only.
105+
openstack_test_machines_leak_cluster_serial: false
102106
openstack_testsuite_name: openstack_tests_lb_ovn
103107
openstack_reset_result_dir: no # As we want to keep the logs generated in the previous step
104108

‎collection/stages/roles/lb_tests/tasks/run_amphora_openstack_test.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,5 +8,7 @@
88
openstack_tests_allowlist_file: "{{ lb_amphora_openstack_test_allowlist }}"
99
openstack_tests_blocklist_file: "{{ lb_amphora_openstack_test_blocklist }}"
1010
openstack_test_ote_run_serial: true
11+
# Same as OVN LB: do not apply Machine serial coda on LB allowlists.
12+
openstack_test_machines_leak_cluster_serial: false
1113
openstack_testsuite_name: "{{ lb_amphora_openstack_testsuite_name }}"
1214
openstack_reset_result_dir: "{{ lb_amphora_openstack_reset_result_dir | default(false) }}"

‎collection/stages/roles/openstack_test/defaults/main.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,11 @@ openstack_test_ote_run_serial: false
2525
# the matchers are absent (e.g. lb_tests allowlists). Requires OTE
2626
# filtered run path.
2727
openstack_test_machines_leak_cluster_serial: true
28+
# Dedicated Zuul/artifact folder for the serial coda (same pattern as
29+
# lb_ovn / lb_amphora_*): own log, junit, list, and test_results copy.
30+
# Main openstack-test-results stays batch-only after the split.
31+
openstack_test_serial_coda_results_dir: "{{ artifacts_dir }}/openstack_test_serial_coda"
32+
openstack_test_serial_coda_testsuite_name: openstack_tests_serial_coda
2833
# Ordered substrings matched against list_of_tests_to_run.txt (one phrase
2934
# each; written to a matchers file so shell never splits them). For the
3035
# MachineSet replica matcher, ControlPlane variants are skipped.

‎collection/stages/roles/openstack_test/tasks/main.yml‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,15 @@
3838
- name: Include Openstack-Test tasks
3939
ansible.builtin.include_tasks: run_openstack_test.yml
4040

41+
always:
42+
# Always publish batch results when junit exists, even if run_openstack_test
43+
# (or serial coda) failed after the suite wrote artifacts.
44+
- name: Stat main openstack-test junit before post
45+
ansible.builtin.stat:
46+
path: "{{ openstack_test_results_dir }}/junit_e2e_openstack.xml"
47+
register: openstack_test_main_junit_stat
48+
failed_when: false
49+
4150
- name: Post openshift-test
4251
ansible.builtin.include_role:
4352
name: tools_openshift_tests
@@ -47,8 +56,8 @@
4756
key_for_filtering_results: "openstack"
4857
test_name: "{{ openstack_test_name }}"
4958
results_dir: "{{ openstack_test_results_dir }}"
59+
when: openstack_test_main_junit_stat.stat.exists | default(false)
5060

51-
always:
5261
- name: Restore hypervisor's cifmw-dnsmasq service
5362
ansible.builtin.import_role:
5463
name: tools_cifmw_dnsmasq

‎collection/stages/roles/openstack_test/tasks/run_machines_leak_cluster_serial.yml‎

Lines changed: 98 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,9 @@
22
# Serial coda: serially run the ordered coda list after the main batch suite
33
# (Machine leak cluster, egressIP / prometheus, then topology AZ check
44
# before enable_topology=false mutator).
5-
# Expects openstack_test_leak_cluster_path to already list the ordered tests.
5+
# Artifacts go to openstack_test_serial_coda/ (separate from batch
6+
# openstack-test-results/), same layout as lb_ovn / lb_amphora_*.
7+
# Expects openstack_test_leak_cluster_path and coda log/junit paths set.
68
- name: Stat machines leak-cluster serial list
79
ansible.builtin.stat:
810
path: "{{ openstack_test_leak_cluster_path }}"
@@ -26,18 +28,18 @@
2628
(not (openstack_test_leak_cluster_stat.stat.exists | default(false)))
2729
or ((openstack_test_leak_cluster_count.stdout | default('0') | trim | int) == 0)
2830
29-
- name: Run machines leak-cluster tests serially and merge into suite log/junit
31+
- name: Run machines leak-cluster tests serially into dedicated results dir
3032
when:
3133
- openstack_test_leak_cluster_stat.stat.exists | default(false)
3234
- (openstack_test_leak_cluster_count.stdout | default('0') | trim | int) > 0
3335
block:
3436
- name: Execute machines leak-cluster run-test loop
3537
ansible.builtin.shell: |
3638
set -o pipefail
37-
suite_exit=0
3839
py_exit=0
3940
resolve="{{ openstack_test_ote_resolve_script }}"
40-
leak_log="{{ openstack_test_results_dir }}/machines_leak_cluster.log"
41+
leak_log="{{ openstack_test_serial_coda_log_path }}"
42+
leak_junit="{{ openstack_test_serial_coda_junit_path }}"
4143
leak_list="{{ openstack_test_leak_cluster_path }}"
4244
# Pre-count from the list file (do not increment in-loop; avoids
4345
# false UNSTABLE when resolvable!=expected after a successful coda).
@@ -46,37 +48,21 @@
4648
while IFS= read -r tname || [ -n "${tname:-}" ]; do
4749
[ -z "${tname// }" ] && continue
4850
tmp=$(mktemp)
49-
rc=0
51+
# OTE often exits non-zero on pass/skip; ignore per-test rc — final
52+
# resolver counts are the source of truth (same as LB serial / batch).
5053
{{ openstack_test_executable }} run-test --output=json -c 1 "$tname" \
51-
> "$tmp" 2>&1 || rc=$?
54+
> "$tmp" 2>&1 || true
5255
cat "$tmp" >> "$leak_log"
53-
cat "$tmp" >> {{ openstack_test_log_path }}
54-
if [ "$rc" -ne 0 ]; then
55-
failed=$(python3 "$resolve" count "$tmp" failed || echo 1)
56-
skipped=$(python3 "$resolve" count "$tmp" skipped || echo 0)
57-
passed=$(python3 "$resolve" count "$tmp" passed || echo 0)
58-
unknown=$(python3 "$resolve" count "$tmp" unknown || echo 0)
59-
if [ "$failed" -gt 0 ] || [ "$unknown" -gt 0 ]; then
60-
suite_exit=1
61-
elif [ $((failed + skipped + passed + unknown)) -eq 0 ]; then
62-
if grep -q 'FAIL!' "$tmp" 2>/dev/null; then
63-
suite_exit=1
64-
elif ! grep -qE 'SUCCESS!|SKIP' "$tmp" 2>/dev/null; then
65-
suite_exit=1
66-
fi
67-
fi
68-
fi
6956
rm -f "$tmp"
7057
done < "$leak_list"
71-
python3 "$resolve" junit \
72-
"{{ openstack_test_log_path }}" "{{ openstack_test_junit_path }}" || py_exit=$?
58+
python3 "$resolve" junit "$leak_log" "$leak_junit" || py_exit=$?
7359
failed_count=$(python3 "$resolve" count "$leak_log" failed || echo 0)
7460
skipped_count=$(python3 "$resolve" count "$leak_log" skipped || echo 0)
7561
passed_count=$(python3 "$resolve" count "$leak_log" passed || echo 0)
7662
unknown_count=$(python3 "$resolve" count "$leak_log" unknown || echo 0)
7763
resolvable=$(( ${passed_count:-0} + ${skipped_count:-0} + ${failed_count:-0} ))
7864
echo "machines_leak_cluster counts: passed=${passed_count} failed=${failed_count} skipped=${skipped_count} unknown=${unknown_count} expected=${expected} resolvable=${resolvable}"
79-
if [ "$suite_exit" -ne 0 ] || [ "$py_exit" -ne 0 ] \
65+
if [ "$py_exit" -ne 0 ] \
8066
|| [ "${failed_count:-0}" -gt 0 ] || [ "${unknown_count:-0}" -gt 0 ] \
8167
|| [ "$resolvable" -ne "$expected" ]; then
8268
exit 1
@@ -91,6 +77,61 @@
9177
register: openstack_test_leak_cluster_run
9278

9379
rescue:
80+
- name: Resolve machines leak-cluster serial failure counts
81+
ansible.builtin.command:
82+
argv:
83+
- python3
84+
- "{{ openstack_test_ote_resolve_script }}"
85+
- count
86+
- "{{ openstack_test_serial_coda_log_path }}"
87+
- "{{ item }}"
88+
loop:
89+
- failed
90+
- passed
91+
- skipped
92+
- unknown
93+
register: openstack_test_leak_cluster_rescue_counts
94+
changed_when: false
95+
failed_when: false
96+
97+
- name: Normalize machines leak-cluster rescue counts
98+
ansible.builtin.set_fact:
99+
openstack_test_leak_failed: >-
100+
{{ (openstack_test_leak_cluster_rescue_counts.results
101+
| selectattr('item', 'equalto', 'failed')
102+
| map(attribute='stdout') | first | default('0')) | trim }}
103+
openstack_test_leak_passed: >-
104+
{{ (openstack_test_leak_cluster_rescue_counts.results
105+
| selectattr('item', 'equalto', 'passed')
106+
| map(attribute='stdout') | first | default('0')) | trim }}
107+
openstack_test_leak_skipped: >-
108+
{{ (openstack_test_leak_cluster_rescue_counts.results
109+
| selectattr('item', 'equalto', 'skipped')
110+
| map(attribute='stdout') | first | default('0')) | trim }}
111+
openstack_test_leak_unknown: >-
112+
{{ (openstack_test_leak_cluster_rescue_counts.results
113+
| selectattr('item', 'equalto', 'unknown')
114+
| map(attribute='stdout') | first | default('0')) | trim }}
115+
openstack_test_leak_expected: >-
116+
{{ (openstack_test_leak_cluster_count.stdout | default('0')) | trim }}
117+
118+
- name: Compute machines leak-cluster resolvable count
119+
ansible.builtin.set_fact:
120+
openstack_test_leak_resolvable: >-
121+
{{ (
122+
(openstack_test_leak_passed | int)
123+
+ (openstack_test_leak_skipped | int)
124+
+ (openstack_test_leak_failed | int)
125+
) }}
126+
127+
- name: Print machines leak-cluster rescue counts
128+
ansible.builtin.debug:
129+
msg: >-
130+
machines_leak_cluster rescue counts: passed={{ openstack_test_leak_passed }},
131+
failed={{ openstack_test_leak_failed }}, skipped={{ openstack_test_leak_skipped }},
132+
unknown={{ openstack_test_leak_unknown }}, expected={{ openstack_test_leak_expected }},
133+
resolvable={{ openstack_test_leak_resolvable }}
134+
94135
- name: Mark openstack-test UNSTABLE after machines leak-cluster serial failure
95136
ansible.builtin.include_role:
96137
name: tools_stage_results
@@ -99,3 +140,35 @@
99140
unstable_msg: >-
100141
openstack_test serial coda failed (Machine leak cluster, egressIP /
101142
prometheus resize, and/or topology enable_topology pair).
143+
when: >
144+
((openstack_test_leak_failed | default('0')) | int > 0)
145+
or ((openstack_test_leak_unknown | default('0')) | int > 0)
146+
or (
147+
(openstack_test_leak_resolvable | default('0')) | int
148+
!= (openstack_test_leak_expected | default('0')) | int
149+
)
150+
151+
always:
152+
# Publish coda results like lb_ovn when junit exists. No ignore_errors:
153+
# publish failure must fail the stage so results are not silently missing.
154+
- name: Stat serial-coda junit before post
155+
ansible.builtin.stat:
156+
path: "{{ openstack_test_serial_coda_junit_path }}"
157+
register: openstack_test_serial_coda_junit_stat
158+
failed_when: false
159+
when:
160+
- openstack_test_serial_coda_junit_path is defined
161+
- openstack_test_leak_cluster_stat.stat.exists | default(false)
162+
- (openstack_test_leak_cluster_count.stdout | default('0') | trim | int) > 0
163+
164+
- name: Post serial-coda openshift-test results
165+
ansible.builtin.include_role:
166+
name: tools_openshift_tests
167+
tasks_from: post_openshift_tests.yml
168+
vars:
169+
testsuite_name: "{{ openstack_test_serial_coda_testsuite_name }}"
170+
key_for_filtering_results: "openstack"
171+
test_name: "{{ openstack_test_name }}"
172+
results_dir: "{{ openstack_test_serial_coda_results_dir }}"
173+
when:
174+
- openstack_test_serial_coda_junit_stat.stat.exists | default(false)

0 commit comments

Comments
 (0)