|
| 1 | +#!/usr/bin/env bash |
| 2 | +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. |
| 3 | +# SPDX-License-Identifier: Apache-2.0 |
| 4 | + |
| 5 | +set -euo pipefail |
| 6 | +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" |
| 7 | +test_dir="$(mktemp -d)" |
| 8 | +trap 'rm -rf "$test_dir"' EXIT |
| 9 | +mkdir -p "$test_dir/bin" |
| 10 | +export TEST_FIXTURE="$script_dir/fixtures/nix-resumed-416.txt" |
| 11 | +# Fixture excerpt: NVIDIA/OpenShell job 110463784656, October 1, 2026. |
| 12 | + |
| 13 | +cat > "$test_dir/bin/nix" <<'SH' |
| 14 | +#!/usr/bin/env bash |
| 15 | +set -euo pipefail |
| 16 | +[ "$*" = '--log-format raw develop -c true' ] || exit 99 |
| 17 | +count=0 |
| 18 | +[ ! -f "$TEST_CASE/count" ] || count="$(cat "$TEST_CASE/count")" |
| 19 | +count=$((count + 1)) |
| 20 | +echo "$count" > "$TEST_CASE/count" |
| 21 | +case "$TEST_SCENARIO" in |
| 22 | + success) echo 'shell ready'; exit 0 ;; |
| 23 | + warnings) sed -n '1p' "$TEST_FIXTURE" >&2; exit 0 ;; |
| 24 | + recover) [ "$count" -lt 2 ] || exit 0 ;; |
| 25 | + recover-third) [ "$count" -lt 3 ] || exit 0 ;; |
| 26 | + changed-failure) |
| 27 | + if [ "$count" -gt 1 ]; then |
| 28 | + echo "error: builder for '/nix/store/example.drv' failed with exit code 7" >&2 |
| 29 | + exit 7 |
| 30 | + fi |
| 31 | + ;; |
| 32 | + standalone-416) sed -n '2p' "$TEST_FIXTURE" >&2; exit 23 ;; |
| 33 | + other-resume) sed '1s/0gb22ka/1gb22ka/' "$TEST_FIXTURE" >&2; exit 23 ;; |
| 34 | + dependency-only) sed -n '3,7p' "$TEST_FIXTURE" >&2; exit 23 ;; |
| 35 | + other-host) sed 's/cache.nixos.org/example.invalid/g' "$TEST_FIXTURE" >&2; exit 23 ;; |
| 36 | + terminated) cat "$TEST_FIXTURE" >&2; exit 143 ;; |
| 37 | +esac |
| 38 | +cat "$TEST_FIXTURE" >&2 |
| 39 | +case "$TEST_SCENARIO" in |
| 40 | + builder) echo "error: builder for '/nix/store/example.drv' failed with exit code 1" >&2 ;; |
| 41 | + hash) echo 'error: hash mismatch in fixed-output derivation' >&2 ;; |
| 42 | + disk) echo 'error: No space left on device' >&2 ;; |
| 43 | + unknown) echo 'error: unexplained fatal error' >&2 ;; |
| 44 | + auth) echo "error: unable to download 'https://cache.nixos.org/test': HTTP error 401" >&2 ;; |
| 45 | + missing) echo "error: unable to download 'https://cache.nixos.org/test': HTTP error 404" >&2 ;; |
| 46 | +esac |
| 47 | +exit 23 |
| 48 | +SH |
| 49 | +cat > "$test_dir/bin/sleep" <<'SH' |
| 50 | +#!/usr/bin/env bash |
| 51 | +echo "$1" >> "$TEST_CASE/delays" |
| 52 | +if [ "$TEST_SCENARIO" = cancel-backoff ]; then |
| 53 | + kill -TERM "$PPID" |
| 54 | +fi |
| 55 | +SH |
| 56 | +chmod +x "$test_dir/bin/nix" "$test_dir/bin/sleep" |
| 57 | +export PATH="$test_dir/bin:$PATH" |
| 58 | + |
| 59 | +fail() { echo "FAIL: $*" >&2; exit 1; } |
| 60 | +check_case() { |
| 61 | + export TEST_SCENARIO="$1" TEST_CASE="$test_dir/$1" |
| 62 | + export GITHUB_OUTPUT="$TEST_CASE/outputs" |
| 63 | + mkdir -p "$TEST_CASE" |
| 64 | + local status=0 |
| 65 | + bash "$script_dir/realize-nix-dev-shell.sh" "$TEST_CASE/logs" > "$TEST_CASE/console" 2>&1 || status=$? |
| 66 | + [ "$status" -eq "$2" ] || fail "$1: expected exit $2, got $status" |
| 67 | + [ "$(cat "$TEST_CASE/count")" -eq "$3" ] || fail "$1: incorrect invocation count" |
| 68 | + local attempt |
| 69 | + for ((attempt=1; attempt<=$3; attempt++)); do |
| 70 | + [ -f "$TEST_CASE/logs/attempt-$attempt.log" ] || fail "$1: missing attempt $attempt log" |
| 71 | + done |
| 72 | + [ "$(wc -l < "$TEST_CASE/logs/results.tsv" | tr -d ' ')" -eq "$3" ] || fail "$1: missing result statuses" |
| 73 | + if [ "$3" -gt 1 ]; then |
| 74 | + grep -q 'retried=true' "$GITHUB_OUTPUT" || fail "$1: missing retry marker" |
| 75 | + grep -q 'HTTP error 416' "$TEST_CASE/logs/attempt-1.log" || fail "$1: original failure lost" |
| 76 | + local delay lower=10 |
| 77 | + while read -r delay; do |
| 78 | + [ "$delay" -ge "$lower" ] && [ "$delay" -lt "$((lower + 5))" ] || fail "$1: unbounded delay" |
| 79 | + lower=30 |
| 80 | + done < "$TEST_CASE/delays" |
| 81 | + [ "$(wc -l < "$TEST_CASE/delays" | tr -d ' ')" -eq "$(( $3 - 1 ))" ] || fail "$1: incorrect sleep count" |
| 82 | + else |
| 83 | + [ ! -e "$TEST_CASE/delays" ] || fail "$1: unexpected delay" |
| 84 | + [ ! -e "$GITHUB_OUTPUT" ] || fail "$1: unexpected retry marker" |
| 85 | + fi |
| 86 | + if [ "$2" -eq 0 ] && [ "$3" -gt 1 ]; then |
| 87 | + grep -q 'recovered after' "$TEST_CASE/console" || fail "$1: recovery not reported" |
| 88 | + fi |
| 89 | + echo "PASS: $1" |
| 90 | +} |
| 91 | + |
| 92 | +check_case success 0 1 |
| 93 | +check_case warnings 0 1 |
| 94 | +check_case recover 0 2 |
| 95 | +check_case recover-third 0 3 |
| 96 | +check_case exhaust 23 3 |
| 97 | +check_case changed-failure 7 2 |
| 98 | +for scenario in standalone-416 other-resume dependency-only other-host builder hash disk unknown auth missing; do |
| 99 | + check_case "$scenario" 23 1 |
| 100 | +done |
| 101 | +check_case terminated 143 1 |
| 102 | + |
| 103 | +export TEST_SCENARIO=cancel-backoff TEST_CASE="$test_dir/cancel-backoff" |
| 104 | +export GITHUB_OUTPUT="$TEST_CASE/outputs" |
| 105 | +mkdir -p "$TEST_CASE" |
| 106 | +status=0 |
| 107 | +bash "$script_dir/realize-nix-dev-shell.sh" "$TEST_CASE/logs" > "$TEST_CASE/console" 2>&1 || status=$? |
| 108 | +[ "$status" -eq 143 ] || fail "cancellation during backoff hidden" |
| 109 | +[ "$(cat "$TEST_CASE/count")" -eq 1 ] || fail "cancellation during backoff retried" |
| 110 | +echo 'PASS: cancellation during backoff' |
| 111 | + |
| 112 | +# Pipeline status must report a failed log sink even when Nix succeeds. |
| 113 | +cat > "$test_dir/bin/tee" <<'SH' |
| 114 | +#!/usr/bin/env bash |
| 115 | +cat >/dev/null |
| 116 | +exit 9 |
| 117 | +SH |
| 118 | +chmod +x "$test_dir/bin/tee" |
| 119 | +export TEST_SCENARIO=success TEST_CASE="$test_dir/log-sink-failure" |
| 120 | +mkdir -p "$TEST_CASE" |
| 121 | +status=0 |
| 122 | +bash "$script_dir/realize-nix-dev-shell.sh" "$TEST_CASE/logs" > "$TEST_CASE/console" 2>&1 || status=$? |
| 123 | +[ "$status" -eq 9 ] || fail "log sink error hidden" |
| 124 | +[ "$(cat "$TEST_CASE/count")" -eq 1 ] || fail "log sink error retried" |
| 125 | +echo 'PASS: log sink failure' |
| 126 | +echo 'All Nix shell recovery tests passed.' |
0 commit comments