Skip to content

Commit 11de79d

Browse files
vstinnervstinner
committed
pythongh-109276: libregrtest calls random.seed() before each test
libregrtest now calls random.seed() before running each test file when -r/--randomize command line option is used. Moreover, it's also called in worker processes. It should help to make tests more deterministic. Previously, it was only called once in the main process before running all test files and it was not called in worker processes. * Convert some f-strings to regular strings in test_regrtest when f-string is not needed. * Remove unused all_methods variables from test_regrtest.
1 parent baa6dc8 commit 11de79d

5 files changed

Lines changed: 92 additions & 33 deletions

File tree

‎Lib/test/libregrtest/main.py‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -112,8 +112,11 @@ def __init__(self, ns: Namespace):
112112
self.junit_filename: StrPath | None = ns.xmlpath
113113
self.memory_limit: str | None = ns.memlimit
114114
self.gc_threshold: int | None = ns.threshold
115-
self.use_resources: list[str] = ns.use_resources
116-
self.python_cmd: list[str] | None = ns.python
115+
self.use_resources: tuple[str] = tuple(ns.use_resources)
116+
if ns.python:
117+
self.python_cmd: tuple[str] = tuple(ns.python)
118+
else:
119+
self.python_cmd = None
117120
self.coverage: bool = ns.trace
118121
self.coverage_dir: StrPath | None = ns.coverdir
119122
self.tmp_dir: StrPath | None = ns.tempdir
@@ -377,8 +380,11 @@ def create_run_tests(self, tests: TestTuple):
377380
return RunTests(
378381
tests,
379382
fail_fast=self.fail_fast,
383+
fail_env_changed=self.fail_env_changed,
380384
match_tests=self.match_tests,
381385
ignore_tests=self.ignore_tests,
386+
match_tests_dict=None,
387+
rerun=None,
382388
forever=self.forever,
383389
pgo=self.pgo,
384390
pgo_extended=self.pgo_extended,
@@ -393,6 +399,8 @@ def create_run_tests(self, tests: TestTuple):
393399
gc_threshold=self.gc_threshold,
394400
use_resources=self.use_resources,
395401
python_cmd=self.python_cmd,
402+
randomize=self.randomize,
403+
random_seed=self.random_seed,
396404
)
397405

398406
def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int:

‎Lib/test/libregrtest/runtests.py‎

Lines changed: 22 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -16,26 +16,28 @@ class HuntRefleak:
1616
@dataclasses.dataclass(slots=True, frozen=True)
1717
class RunTests:
1818
tests: TestTuple
19-
fail_fast: bool = False
20-
fail_env_changed: bool = False
21-
match_tests: FilterTuple | None = None
22-
ignore_tests: FilterTuple | None = None
23-
match_tests_dict: FilterDict | None = None
24-
rerun: bool = False
25-
forever: bool = False
26-
pgo: bool = False
27-
pgo_extended: bool = False
28-
output_on_failure: bool = False
29-
timeout: float | None = None
30-
verbose: int = 0
31-
quiet: bool = False
32-
hunt_refleak: HuntRefleak | None = None
33-
test_dir: StrPath | None = None
34-
use_junit: bool = False
35-
memory_limit: str | None = None
36-
gc_threshold: int | None = None
37-
use_resources: list[str] = dataclasses.field(default_factory=list)
38-
python_cmd: list[str] | None = None
19+
fail_fast: bool
20+
fail_env_changed: bool
21+
match_tests: FilterTuple | None
22+
ignore_tests: FilterTuple | None
23+
match_tests_dict: FilterDict | None
24+
rerun: bool
25+
forever: bool
26+
pgo: bool
27+
pgo_extended: bool
28+
output_on_failure: bool
29+
timeout: float | None
30+
verbose: int
31+
quiet: bool
32+
hunt_refleak: HuntRefleak | None
33+
test_dir: StrPath | None
34+
use_junit: bool
35+
memory_limit: str | None
36+
gc_threshold: int | None
37+
use_resources: tuple[str]
38+
python_cmd: tuple[str] | None
39+
randomize: bool
40+
random_seed: int | None
3941

4042
def copy(self, **override):
4143
state = dataclasses.asdict(self)

‎Lib/test/libregrtest/setup.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import faulthandler
22
import os
3+
import random
34
import signal
45
import sys
56
import unittest
@@ -127,3 +128,6 @@ def setup_tests(runtests: RunTests):
127128

128129
if runtests.gc_threshold is not None:
129130
gc.set_threshold(runtests.gc_threshold)
131+
132+
if runtests.randomize:
133+
random.seed(runtests.random_seed)

‎Lib/test/test_regrtest.py‎

Lines changed: 50 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import locale
1212
import os.path
1313
import platform
14+
import random
1415
import re
1516
import subprocess
1617
import sys
@@ -504,7 +505,7 @@ def list_regex(line_format, tests):
504505
if rerun is not None:
505506
regex = list_regex('%s re-run test%s', [rerun.name])
506507
self.check_line(output, regex)
507-
regex = LOG_PREFIX + fr"Re-running 1 failed tests in verbose mode"
508+
regex = LOG_PREFIX + r"Re-running 1 failed tests in verbose mode"
508509
self.check_line(output, regex)
509510
regex = fr"Re-running {rerun.name} in verbose mode"
510511
if rerun.match:
@@ -1018,13 +1019,13 @@ def test_run(self):
10181019
stats=TestStats(4, 1),
10191020
forever=True)
10201021

1021-
def check_leak(self, code, what, *, multiprocessing=False):
1022+
def check_leak(self, code, what, *, run_workers=False):
10221023
test = self.create_test('huntrleaks', code=code)
10231024

10241025
filename = 'reflog.txt'
10251026
self.addCleanup(os_helper.unlink, filename)
10261027
cmd = ['--huntrleaks', '6:3:']
1027-
if multiprocessing:
1028+
if run_workers:
10281029
cmd.append('-j1')
10291030
cmd.append(test)
10301031
output = self.run_tests(*cmd,
@@ -1043,7 +1044,7 @@ def check_leak(self, code, what, *, multiprocessing=False):
10431044
self.assertIn(line2, reflog)
10441045

10451046
@unittest.skipUnless(support.Py_DEBUG, 'need a debug build')
1046-
def check_huntrleaks(self, *, multiprocessing: bool):
1047+
def check_huntrleaks(self, *, run_workers: bool):
10471048
# test --huntrleaks
10481049
code = textwrap.dedent("""
10491050
import unittest
@@ -1054,13 +1055,13 @@ class RefLeakTest(unittest.TestCase):
10541055
def test_leak(self):
10551056
GLOBAL_LIST.append(object())
10561057
""")
1057-
self.check_leak(code, 'references', multiprocessing=multiprocessing)
1058+
self.check_leak(code, 'references', run_workers=run_workers)
10581059

10591060
def test_huntrleaks(self):
1060-
self.check_huntrleaks(multiprocessing=False)
1061+
self.check_huntrleaks(run_workers=False)
10611062

10621063
def test_huntrleaks_mp(self):
1063-
self.check_huntrleaks(multiprocessing=True)
1064+
self.check_huntrleaks(run_workers=True)
10641065

10651066
@unittest.skipUnless(support.Py_DEBUG, 'need a debug build')
10661067
def test_huntrleaks_fd_leak(self):
@@ -1138,8 +1139,6 @@ def test_method3(self):
11381139
def test_method4(self):
11391140
pass
11401141
""")
1141-
all_methods = ['test_method1', 'test_method2',
1142-
'test_method3', 'test_method4']
11431142
testname = self.create_test(code=code)
11441143

11451144
# only run a subset
@@ -1761,7 +1760,7 @@ def test_mp_decode_error(self):
17611760
if encoding is None:
17621761
encoding = sys.__stdout__.encoding
17631762
if encoding is None:
1764-
self.skipTest(f"cannot get regrtest worker encoding")
1763+
self.skipTest("cannot get regrtest worker encoding")
17651764

17661765
nonascii = b"byte:\xa0\xa9\xff\n"
17671766
try:
@@ -1788,7 +1787,7 @@ def test_mp_decode_error(self):
17881787
stats=0)
17891788

17901789
def test_doctest(self):
1791-
code = textwrap.dedent(fr'''
1790+
code = textwrap.dedent(r'''
17921791
import doctest
17931792
import sys
17941793
from test import support
@@ -1826,6 +1825,46 @@ def load_tests(loader, tests, pattern):
18261825
randomize=True,
18271826
stats=TestStats(1, 1, 0))
18281827

1828+
def _check_random_seed(self, run_workers: bool):
1829+
# gh-109276: When -r/--randomize is used, random.seed() is called
1830+
# with the same random seed before running each test file.
1831+
code = textwrap.dedent(r'''
1832+
import random
1833+
import unittest
1834+
1835+
class RandomSeedTest(unittest.TestCase):
1836+
def test_randint(self):
1837+
numbers = [random.randint(0, 1000) for _ in range(10)]
1838+
print(f"Random numbers: {numbers}")
1839+
''')
1840+
tests = [self.create_test(name=f'test_random{i}', code=code)
1841+
for i in range(1, 3+1)]
1842+
1843+
random_seed = 856_656_202
1844+
cmd = ["--randomize", f"--randseed={random_seed}"]
1845+
if run_workers:
1846+
# run as many worker processes than the number of tests
1847+
cmd.append(f'-j{len(tests)}')
1848+
cmd.extend(tests)
1849+
output = self.run_tests(*cmd)
1850+
1851+
random.seed(random_seed)
1852+
# make the assumption that nothing consume entry between libregrest
1853+
# setup_tests() calls random.seed() and RandomSeedTest calling
1854+
# random.randint().
1855+
numbers = [random.randint(0, 1000) for _ in range(10)]
1856+
expected = f"Random numbers: {numbers}"
1857+
1858+
regex = r'^Random numbers: .*$'
1859+
matches = re.findall(regex, output, flags=re.MULTILINE)
1860+
self.assertEqual(matches, [expected] * len(tests))
1861+
1862+
def test_random_seed(self):
1863+
self._check_random_seed(run_workers=False)
1864+
1865+
def test_random_seed_workers(self):
1866+
self._check_random_seed(run_workers=True)
1867+
18291868

18301869
class TestUtils(unittest.TestCase):
18311870
def test_format_duration(self):

‎Misc/NEWS.d/next/Tests/2023-09-11-19-11-57.gh-issue-109276.qxI4OG.rst‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
libregrtest now calls :func:`random.seed()` before running each test file
2+
when ``-r/--randomize`` command line option is used. Moreover, it's also
3+
called in worker processes. It should help to make tests more
4+
deterministic. Previously, it was only called once in the main process before
5+
running all test files and it was not called in worker processes. Patch by
6+
Victor Stinner.

0 commit comments

Comments
 (0)