Skip to content

Commit 6534d1c

Browse files
reactive-firewallreactive-firewall
committed
[DEMO] Protoytpe from GHI RustPython#5974 with rpau from PR RustPython#6089
* cc: @youknowone @ShaharNaveh * demo of how to automate RustPython Auto-Patching and upload required testing patches (first attempt, YOLO) * uses ShaharNaveh:auto-updater for rpau (see PR RustPython#6089) (Auto Patching and config) * uses automation idea from GHI RustPython#5974 (reporting and automation)
1 parent d36c8ca commit 6534d1c

2 files changed

Lines changed: 349 additions & 4 deletions

File tree

‎.github/actions/CI-6089-Test-RustPython-rapu/action.yaml‎

Lines changed: 342 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,342 @@
1+
---
2+
name: 'RustPython Auto Patching'
3+
description: 'R.ust P.ython A.utomated U.pdater Integration'
4+
author: 'Mr. Walls & RustPython Team'
5+
branding:
6+
icon: 'check-circle'
7+
color: 'red'
8+
inputs:
9+
override-working-dir:
10+
description: |
11+
Path to integrated RustPython clone to smoke test and auto-patch. Default is 'rustpython'
12+
required: true
13+
default: ${{ github.server_url == 'https://github.com' && 'rustpython' || '' }}
14+
override-rustpython-path:
15+
description: |
16+
override value for path to the Python Lib. The default is to use the value of the environment
17+
variable 'RUSTPYTHONPATH'. Most users will find the default 'Lib' sufficient.
18+
required: true
19+
override-rpau-workers:
20+
description: |
21+
override value for number of workers for rpau to use. Must be an integer ≥1.
22+
required: true
23+
default: '1'
24+
override-rpau-base-url:
25+
description: |
26+
override value for where to base auto-patches on. Default is
27+
'https://raw.githubusercontent.com/python/cpython/refs/heads'
28+
required: true
29+
default: 'https://raw.githubusercontent.com/python/cpython/refs/heads'
30+
test-files:
31+
description: |
32+
List of paths to CPython Test files from source to destination. Default is 'Lib/test/*.py'
33+
required: true
34+
default: 'Lib/test/*.py'
35+
python-version:
36+
description: |
37+
The Cpython version (e.g., any valid release or tag, 3.11, 3.12, 3.13) to override. Note
38+
Python 3.12+ is required for R.P.A.U. function.
39+
The default is to use the value of the environment variable 'PYTHON_VERSION'.
40+
default: '3.13'
41+
required: true
42+
max-test-time:
43+
description: |
44+
The max time in seconds per test module file run before aborting a test attempt. The default
45+
is deliberately short at a value of 30 seconds to keep total run-time down.
46+
default: '30'
47+
required: true
48+
49+
# TODO: add verification steps
50+
51+
runs:
52+
using: composite
53+
steps:
54+
- id: output_python
55+
env:
56+
PYTHON_VERSION_INPUT: ${{ inputs.python-version }}
57+
OVERRIDE_RUSTPYTHONPATH_INPUT: ${{ inputs.override-rustpython-path }}
58+
name: "Detect Python"
59+
if: ${{ !cancelled() && inputs.test-files != '' }}
60+
shell: bash
61+
run: |
62+
printf "%s\n" "::group::detect-python-env"
63+
if [[ -n $PYTHON_VERSION_INPUT ]]; then
64+
printf "python-version=%s\n" "${PYTHON_VERSION_INPUT}" >> "$GITHUB_OUTPUT"
65+
PYTHON_VERSION=${PYTHON_VERSION_INPUT}
66+
else
67+
printf "python-version=%s\n" "${PYTHON_VERSION}" >> "$GITHUB_OUTPUT"
68+
fi
69+
if [[ -n $OVERRIDE_RUSTPYTHONPATH_INPUT ]]; then
70+
printf "override-rustpython-path=%s\n" "${OVERRIDE_RUSTPYTHONPATH_INPUT}" >> "$GITHUB_OUTPUT"
71+
OVERRIDE_RUSTPYTHONPATH=${RUSTPYTHONPATH}
72+
else
73+
printf "override-rustpython-path=%s\n" "${RUSTPYTHONPATH:-Lib}" >> "$GITHUB_OUTPUT"
74+
OVERRIDE_RUSTPYTHONPATH="${RUSTPYTHONPATH:-Lib}"
75+
fi
76+
printf "%s\n" "PYTHON_VERSION=${PYTHON_VERSION}" >> "$GITHUB_ENV"
77+
printf "%s\n" "OVERRIDE_RUSTPYTHONPATH=${OVERRIDE_RUSTPYTHONPATH}" >> "$GITHUB_ENV"
78+
printf "Targeting Cpython %s on %s.\n" '${PYTHON_VERSION}' '${{ runner.os }}' ;
79+
printf "%s\n" "::endgroup::"
80+
- name: "Check Cargo Setup"
81+
id: output_cargo_args
82+
shell: bash
83+
run: |
84+
if [[ -n $CARGO_ARGS ]]; then
85+
if [[ "$RUNNER_DEBUG" == "true" ]] || [[ "$ACTIONS_STEP_DEBUG" == "true" ]]; then printf "::debug::CARGO_ARGS already set to '%s'\n" "${CARGO_ARGS}" ; fi ;
86+
# e.g., CARGO_ARGS=${CARGO_ARGS}
87+
else
88+
CARGO_ARGS="--release" ;
89+
if [[ "$RUNNER_DEBUG" == "true" ]] || [[ "$ACTIONS_STEP_DEBUG" == "true" ]]; then printf "::debug::CARGO_ARGS initialized to '%s'\n" "${CARGO_ARGS}" ; fi ;
90+
fi ;
91+
printf "%s\n" "CARGO_ARGS=${CARGO_ARGS}" >> "$GITHUB_ENV" ;
92+
- name: "Prepare Artifact Name"
93+
id: output_artifact_name
94+
if: ${{ !cancelled() }}
95+
shell: bash
96+
run: |
97+
printf "%s\n" "TEST_STEP_SUMMARY=CPython-Summary-Artifact-${{ runner.os }}-${PYTHON_VERSION}.md" >> "$GITHUB_ENV"
98+
- id: store_old_path
99+
if: ${{ !cancelled() }}
100+
shell: bash
101+
run: |
102+
cd ${PWD:-.} ;
103+
export OLD_PWD=$(pwd) ; # only local use for bootstrap
104+
printf "initial-path=%s\n" "${OLD_PWD}" >> "$GITHUB_OUTPUT"
105+
- name: "Try Smoke Testing"
106+
id: smoke_test
107+
shell: bash
108+
if: ${{ !cancelled() && inputs.test-files != '' }}
109+
env:
110+
INPUT_FILES: ${{ inputs.test-files }}
111+
WORKER_COUNT: ${{ inputs.workers }}
112+
BASE_REF_URL: ${{ inputs.override-rpau-base-url }}
113+
OS: ${{ runner.os }}
114+
CONTEXT_PHRASE: 'for Cpython ${{ steps.output_python.outputs.python-version }} on ${{ runner.os }}'
115+
SUBSHELL_TIMEOUT: ${{ inputs.max-test-time }}
116+
run: |
117+
# TODO: clean this up
118+
# Custom timeout function (GH-5974 - because ulimit is restricted and windows can't ulimit at all)
119+
run_with_timeout() {
120+
local timeout=$1
121+
shift
122+
"$@" &
123+
local pid=$!
124+
( sleep "$timeout" && kill -HUP "$pid" 2>/dev/null ) & disown
125+
# Send HUP signal after timeout
126+
wait "$pid"
127+
local status=$?
128+
if [ $status -eq 0 ]; then
129+
printf "::debug::%s\n" "Command completed successfully."
130+
true ; # force success result
131+
elif [ $status -eq 143 ]; then
132+
printf "::warning title='Timeout'::%s\n" "The command \`$@\` ${CONTEXT_PHRASE} was terminated due to timeout."
133+
false ;
134+
else
135+
printf "%s\n" "The command failed with status $status ${CONTEXT_PHRASE}."
136+
false ;
137+
fi ;
138+
}
139+
140+
export -f run_with_timeout ;
141+
# Usage
142+
# run_with_timeout 360 your_command_here
143+
cd ${{ inputs.override-working-dir }} || exit 13 ;
144+
mkdir -p Lib-rpau/test || exit 15 ;
145+
printf "%s\n\n" "# CPython ${PYTHON_VERSION} Results" > "${TEST_STEP_SUMMARY}" ;
146+
for reference_file in ${INPUT_FILES}; do
147+
if [[ ( -f "${reference_file}" ) ]] ; then
148+
# See https://devguide.python.org/testing/run-write-tests
149+
# Heuristic: "if some module does not have unittest.main(), then most likely it does not support direct invocation."
150+
if grep -qF "unittest.main()" "${reference_file}" 2>/dev/null ; then
151+
printf "Now Testing '%s'\n" "${reference_file}"
152+
# vars for subshell but not for workflow
153+
export REF_FILE_NAME=$(basename "${reference_file}") ;
154+
printf "::group::%s\n" "${REF_FILE_NAME}" ;
155+
# TODO: test with cpython first for baseline
156+
# Execute the testing command in a subshell
157+
time (
158+
export RUSTPYTHONPATH=${OVERRIDE_RUSTPYTHONPATH:-Lib} ;
159+
run_with_timeout ${SUBSHELL_TIMEOUT} cargo run $CARGO_ARGS -- ${RUSTPYTHONPATH:-Lib}/test/"${REF_FILE_NAME}" || RAW_COPY_OUTCOME='failing'
160+
if [[ ( -n ${RAW_COPY_OUTCOME} ) ]] ; then
161+
printf "::warning file='%s',title='test-warning':: Could not copy file %s unmodified, and pass tests %s.\n" "${reference_file}" "${reference_file}" "${CONTEXT_PHRASE}" ;
162+
printf "::debug:: Will now attempt Rust Python Auto Patch with file %s, and pass re-tests %s.\n" "${reference_file}" "${CONTEXT_PHRASE}" ;
163+
(
164+
‎ run_with_timeout ${SUBSHELL_TIMEOUT} cargo run $CARGO_ARGS -- -m tools/rpau/src/rpau --workers ${WORKER_COUNT} --default-version "${PYTHON_VERSION}" --base-upstream-url "${BASE_REF_URL}" --include ${RUSTPYTHONPATH:-Lib}/test/"${REF_FILE_NAME}" --output-dir "Lib-rpau/test" || PATCH_COPY_OUTCOME='unpatchable' ;
165+
) ;
166+
if [[ ( -n ${PATCH_COPY_OUTCOME} ) ]] ; then
167+
printf "::error file='%s',title='testing-failure':: Could not copy and auto-fix tests %s.\n" "${reference_file}" "${CONTEXT_PHRASE}" >&2 ;
168+
printf "::debug:: Will now attempt Auto Suppress Patch with file %s, and pass re-tests %s.\n" "${reference_file}" "${CONTEXT_PHRASE}" ;
169+
(
170+
run_with_timeout ${SUBSHELL_TIMEOUT} cargo run $CARGO_ARGS -- ./scripts/fix_test.py --path ${RUSTPYTHONPATH:-Lib}/test/"${REF_FILE_NAME}" || FIX_COPY_OUTCOME='unfixed' ;
171+
) ;
172+
if [[ ( -n ${FIX_COPY_OUTCOME} ) ]] ; then
173+
printf "::error file='%s',title='testing-failure':: Could not copy and auto-fix tests %s.\n" "${reference_file}" "${CONTEXT_PHRASE}" >&2 ;
174+
# reset broken integration to last rustpython copy
175+
git restore --ignore-unmerged --worktree --staged "${reference_file}" || : ;
176+
git checkout -f --ignore-unmerged -- "${reference_file}" || : ;
177+
# TODO: validate and conditionally set
178+
FIX_COPY_OUTCOME="reverted"
179+
else
180+
PATCH_COPY_OUTCOME="unpatchable"
181+
FIX_COPY_OUTCOME="fixed"
182+
RAW_COPY_OUTCOME="incompatible"
183+
fi ;
184+
else
185+
PATCH_COPY_OUTCOME="patched"
186+
FIX_COPY_OUTCOME="skipped"
187+
RAW_COPY_OUTCOME="incompatible"
188+
fi ;
189+
else
190+
FIX_COPY_OUTCOME="skipped"
191+
PATCH_COPY_OUTCOME="skipped"
192+
RAW_COPY_OUTCOME="compatible"
193+
fi ;
194+
if [ -n ${RAW_COPY_OUTCOME} ]; then
195+
if [[ "${RAW_COPY_OUTCOME}" == "compatible" ]] ; then
196+
printf "%s\n" ":ballot_box_with_check: Directly copying the test file \`${reference_file}\` is ${RAW_COPY_OUTCOME}" >> "${TEST_STEP_SUMMARY}" ;
197+
else
198+
printf "%s\n" ":black_square_button: Directly copying the test file \`${reference_file}\` is ${RAW_COPY_OUTCOME}" >> "${TEST_STEP_SUMMARY}" ;
199+
fi ;
200+
if [ -n ${PATCH_COPY_OUTCOME} ]; then
201+
if [[ "${PATCH_COPY_OUTCOME}" == "patched" ]] ; then
202+
printf "%s\n\n" " :ballot_box_with_check: Copying and RustPython Auto-Patching the test file \`${reference_file}\` was successful" >> "${TEST_STEP_SUMMARY}" ;
203+
else
204+
printf "%s\n\n" " :grey_exclamation: Copying and RustPython Auto-Patching the test file \`${reference_file}\` was ${FIX_COPY_OUTCOME}" >> "${TEST_STEP_SUMMARY}" ;
205+
fi ;
206+
fi ; # end rpau
207+
if [ -n ${FIX_COPY_OUTCOME} ]; then
208+
if [[ "${FIX_COPY_OUTCOME}" == "fixed" ]] ; then
209+
printf "%s\n\n" " :ballot_box_with_check: Copying and Auto-fixing the test file \`${reference_file}\` was successful" >> "${TEST_STEP_SUMMARY}" ;
210+
else
211+
printf "%s\n\n" " :grey_exclamation: Copying and Auto-fixing the test file \`${reference_file}\` was ${FIX_COPY_OUTCOME}" >> "${TEST_STEP_SUMMARY}" ;
212+
fi ; # end auto-fix
213+
else
214+
printf "\n" >> "${TEST_STEP_SUMMARY}" ; # extra space
215+
fi ;
216+
fi ; # end copy
217+
printf "\n---\n%s Outcome:\n\tDirectly:%s\n\tAuto-Patch:%s\n\tAuto-Fix:%s\n\n" "${REF_FILE_NAME}" "${RAW_COPY_OUTCOME}" "${PACTH_COPY_OUTCOME}" "${FIX_COPY_OUTCOME}" ;
218+
printf "FIX_COPY_%s_OUTCOME=%s\n" "${REF_FILE_NAME}" "${FIX_COPY_OUTCOME}" >> "$GITHUB_ENV" ;
219+
printf "RAW_COPY_%s_OUTCOME=%s\n" "${REF_FILE_NAME}" "${RAW_COPY_OUTCOME}" >> "$GITHUB_ENV" ;
220+
printf "PATCH_COPY_%s_OUTCOME=%s\n" "${REF_FILE_NAME}" "${PATCH_COPY_OUTCOME}" >> "$GITHUB_ENV" ;
221+
# should dump a diff or something here
222+
printf "\n\n" ;
223+
unset RUSTPYTHONPATH 2>/dev/null || : ;
224+
) ;
225+
# cleanup temp env
226+
unset FIX_COPY_OUTCOME 2>/dev/null || : ;
227+
unset PATCH_COPY_OUTCOME 2>/dev/null || : ;
228+
unset RAW_COPY_OUTCOME 2>/dev/null || : ;
229+
unset REF_FILE_NAME 2>/dev/null || : ;
230+
printf "\n::endgroup::\n" ;
231+
wait ;
232+
else
233+
# TODO: else can not be run directly and needs to be invoked with -m unittest -v test.
234+
# TODO: cleanup this regular expression for edge-cases
235+
if grep -qE "^[^cC]*([cC]lass)\s*(.+)(Test)" "${reference_file}" 2>/dev/null ; then
236+
printf "Now Testing test-cases in '%s'\n" "${reference_file}"
237+
# vars for subshell but not for workflow
238+
export REF_TEST_NAME=$(basename -s ".py" "${reference_file}") ;
239+
printf "%s\n" "Selected testcase test.${REF_TEST_NAME}"
240+
printf "::group::%s\n" "${REF_FILE_NAME}" ;
241+
# TODO: test with cpython first for baseline
242+
# TODO: add to list of files that need additional prep due to hanging or unexpected failures that need to be removed
243+
# Execute the testing command in a subshell
244+
time (
245+
export RUSTPYTHONPATH=${OVERRIDE_RUSTPYTHONPATH:-Lib} ;
246+
run_with_timeout ${SUBSHELL_TIMEOUT} cargo run $CARGO_ARGS -- -m unittest -v test.${REF_TEST_NAME} || RAW_COPY_OUTCOME='failing'
247+
if [[ ( -n ${RAW_COPY_OUTCOME} ) ]] ; then
248+
printf "::warning file='%s',title='test-warning':: Could not copy file %s unmodified, and pass tests %s.\n" "${reference_file}" "${reference_file}" "${CONTEXT_PHRASE}" ;
249+
printf "::debug:: Will now attempt Rust Python Auto Patch with file %s, and pass re-tests %s.\n" "${reference_file}" "${CONTEXT_PHRASE}" ;
250+
(
251+
‎ run_with_timeout ${SUBSHELL_TIMEOUT} cargo run $CARGO_ARGS -- -m tools/rpau/src/rpau --workers ${WORKER_COUNT} --default-version "${PYTHON_VERSION}" --base-upstream-url "${BASE_REF_URL}" --include ${RUSTPYTHONPATH:-Lib}/test/"${REF_FILE_NAME}".py --output-dir "Lib-rpau/test" || PATCH_COPY_OUTCOME='unpatchable' ;
252+
) ;
253+
if [[ ( -n ${PATCH_COPY_OUTCOME} ) ]] ; then
254+
(
255+
run_with_timeout ${SUBSHELL_TIMEOUT} cargo run $CARGO_ARGS -- ./scripts/fix_test.py --path ${RUSTPYTHONPATH:-Lib}/test/"${REF_FILE_NAME}".py || FIX_COPY_OUTCOME='unfixed' ;
256+
) ;
257+
if [[ ( -n ${FIX_COPY_OUTCOME} ) ]] ; then
258+
printf "::error file='%s',title='testing-failure':: Could not copy and auto-fix tests for file.\n" "${reference_file}" >&2 ;
259+
# reset broken integration to last rustpython copy
260+
git restore --ignore-unmerged --worktree --staged "${reference_file}" || : ;
261+
git checkout -f --ignore-unmerged -- "${reference_file}" || : ;
262+
# TODO: validate and conditionally set
263+
FIX_COPY_OUTCOME="reverted"
264+
else
265+
PATCH_COPY_OUTCOME="unpatchable"
266+
FIX_COPY_OUTCOME="fixed"
267+
RAW_COPY_OUTCOME="incompatible"
268+
fi ;
269+
else
270+
PATCH_COPY_OUTCOME="patched"
271+
FIX_COPY_OUTCOME="skipped"
272+
RAW_COPY_OUTCOME="incompatible"
273+
fi ;
274+
else
275+
FIX_COPY_OUTCOME="skipped"
276+
PATCH_COPY_OUTCOME="skipped"
277+
RAW_COPY_OUTCOME="compatible"
278+
fi ;
279+
if [ -n ${RAW_COPY_OUTCOME} ]; then
280+
if [[ "${RAW_COPY_OUTCOME}" == "compatible" ]] ; then
281+
printf "%s\n" ":ballot_box_with_check: Directly copying the test file \`${reference_file}\` is ${RAW_COPY_OUTCOME}" >> "${TEST_STEP_SUMMARY}" ;
282+
else
283+
printf "%s\n" ":black_square_button: Directly copying the test file \`${reference_file}\` is ${RAW_COPY_OUTCOME}" >> "${TEST_STEP_SUMMARY}" ;
284+
fi ;
285+
if [ -n ${PATCH_COPY_OUTCOME} ]; then
286+
if [[ "${PATCH_COPY_OUTCOME}" == "patched" ]] ; then
287+
printf "%s\n\n" " :ballot_box_with_check: Copying and RustPython Auto-Patching the test file \`${reference_file}\` was successful" >> "${TEST_STEP_SUMMARY}" ;
288+
else
289+
printf "%s\n\n" " :grey_exclamation: Copying and RustPython Auto-Patching the test file \`${reference_file}\` was ${FIX_COPY_OUTCOME}" >> "${TEST_STEP_SUMMARY}" ;
290+
fi ;
291+
fi ; # end rpau
292+
if [ -n ${FIX_COPY_OUTCOME} ]; then
293+
if [[ "${FIX_COPY_OUTCOME}" == "fixed" ]] ; then
294+
printf "%s\n\n" " :ballot_box_with_check: Copying and Auto-fixing the test file \`${reference_file}\` was successful" >> "${TEST_STEP_SUMMARY}" ;
295+
else
296+
printf "%s\n\n" " :grey_exclamation: Copying and Auto-fixing the test file \`${reference_file}\` was ${FIX_COPY_OUTCOME}" >> "${TEST_STEP_SUMMARY}" ;
297+
fi ; # end auto-fix
298+
else
299+
printf "\n" >> "${TEST_STEP_SUMMARY}" ; # extra space
300+
fi ;
301+
fi ; # end copy
302+
printf "\n---\n%s Outcome:\n\tDirectly:%s\n\tAuto-Patch:%s\n\tAuto-Fix:%s\n\n" "${REF_FILE_NAME}" "${RAW_COPY_OUTCOME}" "${PACTH_COPY_OUTCOME}" "${FIX_COPY_OUTCOME}" ;
303+
printf "FIX_COPY_%s_OUTCOME=%s\n" "${REF_FILE_NAME}" "${FIX_COPY_OUTCOME}" >> "$GITHUB_ENV" ;
304+
printf "RAW_COPY_%s_OUTCOME=%s\n" "${REF_FILE_NAME}" "${RAW_COPY_OUTCOME}" >> "$GITHUB_ENV" ;
305+
printf "PATCH_COPY_%s_OUTCOME=%s\n" "${REF_FILE_NAME}" "${PATCH_COPY_OUTCOME}" >> "$GITHUB_ENV" ;
306+
# should dump a diff or something here
307+
printf "\n\n" ;
308+
unset RUSTPYTHONPATH 2>/dev/null || : ;
309+
) ;
310+
# cleanup temp env
311+
unset FIX_COPY_OUTCOME 2>/dev/null || : ;
312+
unset PATCH_COPY_OUTCOME 2>/dev/null || : ;
313+
unset RAW_COPY_OUTCOME 2>/dev/null || : ;
314+
unset REF_FILE_NAME 2>/dev/null || : ;
315+
printf "\n::endgroup::\n" ;
316+
wait ;
317+
else
318+
printf "\nNow Skipping '%s'\n\n" "${reference_file}" ;
319+
printf "%s\n" ":grey_exclamation: Directly copying the filepath \`${reference_file}\` was inconclusive (_testing and validation skipped_)." >> "${TEST_STEP_SUMMARY}" ;
320+
printf "\n" >> "${TEST_STEP_SUMMARY}" ; # extra space
321+
fi ;
322+
fi ; # TODO: else can not be run directly and needs to be invoked with -m unittest -v test.
323+
fi ;
324+
done
325+
cat <"${TEST_STEP_SUMMARY}" >> "$GITHUB_STEP_SUMMARY" ;
326+
cd ${{ steps.store_old_path.outputs.initial-path }} || exit 15 ;
327+
- id: upload-rpau
328+
name: "Upload Lib-rpau"
329+
if: ${{ !cancelled() }}
330+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
331+
with:
332+
path: ${{ steps.output_python.override-rustpython-path }}/Lib-rpau
333+
name: Lib-rpau-${{ github.sha }}-${{ runner.os }}-${{ steps.output_python.outputs.python-version }}
334+
if-no-files-found: ignore
335+
compression-level: 6
336+
overwrite: true
337+
- name: Post-Clean
338+
id: post-bootstrap
339+
run: |
340+
exit 0 ; # don't break CI on regression
341+
if: ${{ always() }}
342+
shell: bash

0 commit comments

Comments
 (0)