gh-108828: Support selecting tests by labels

serhiy-storchaka · Sep 2, 2023 · Sep 3, 2023 · Sep 3, 2023 · Sep 3, 2023 · Oct 29, 2023
commit 6ea2f47b11eb6688ac27d909401ba80d26d6eb17
diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py
@@ -172,6 +172,8 @@ def __init__(self, **kwargs) -> None:
         self.failfast = False
         self.match_tests = None
         self.ignore_tests = None
+        self.accept_labels = None
+        self.ignore_labels = None
         self.pgo = False
         self.pgo_extended = False
         self.worker_json = None
@@ -259,6 +261,12 @@ def _create_parser():
     group.add_argument('-i', '--ignore', metavar='PAT',
                        dest='ignore_tests', action='append',
                        help='ignore test cases and methods with glob pattern PAT')
+    group.add_argument('--label', metavar='NAME',
+                       dest='accept_labels', action='append',
+                       help='match test cases and methods with label NAME')
+    group.add_argument('--no-label', metavar='NAME',
+                       dest='ignore_labels', action='append',
+                       help='ignore test cases and methods with label NAME')
     group.add_argument('--matchfile', metavar='FILENAME',
                        dest='match_filename',
                        help='similar to --match but get patterns from a '

diff --git a/Lib/test/libregrtest/findtests.py b/Lib/test/libregrtest/findtests.py
@@ -77,9 +77,12 @@ def _list_cases(suite):
 def list_cases(tests: TestTuple, *,
                match_tests: FilterTuple | None = None,
                ignore_tests: FilterTuple | None = None,
+               accept_labels: tuple[str, ...] | None = None,
+               ignore_labels: tuple[str, ...] | None = None,
                test_dir: StrPath | None = None):
     support.verbose = False
     support.set_match_tests(match_tests, ignore_tests)
+    support.set_match_tests2(accept_labels, ignore_labels)
 
     skipped = []
     for test_name in tests:

diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py
@@ -78,6 +78,14 @@ def __init__(self, ns: Namespace):
             self.ignore_tests: FilterTuple = tuple(ns.ignore_tests)
         else:
             self.ignore_tests = None
+        if ns.accept_labels:
+            self.accept_labels: tuple[str, ...] = tuple(ns.accept_labels)
+        else:
+            self.accept_labels = None
+        if ns.ignore_labels:
+            self.ignore_labels: tuple[str, ...] = tuple(ns.ignore_labels)
+        else:
+            self.ignore_labels = None
         self.exclude: bool = ns.exclude
         self.fromfile: StrPath | None = ns.fromfile
         self.starting_test: TestName | None = ns.start
@@ -484,6 +492,8 @@ def main(self, tests: TestList | None = None):
             list_cases(selected,
                        match_tests=self.match_tests,
                        ignore_tests=self.ignore_tests,
+                       accept_labels=self.accept_labels,
+                       ignore_labels=self.ignore_labels,
                        test_dir=self.test_dir)
         else:
             exitcode = self.run_tests(selected, tests)

diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py
@@ -20,6 +20,8 @@ class RunTests:
     fail_env_changed: bool = False
     match_tests: FilterTuple | None = None
     ignore_tests: FilterTuple | None = None
+    accept_labels: tuple[str, ...] | None = None
+    ignore_labels: tuple[str, ...] | None = None
     match_tests_dict: FilterDict | None = None
     rerun: bool = False
     forever: bool = False

diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py
@@ -95,6 +95,7 @@ def setup_tests(runtests: RunTests):
     support.PGO_EXTENDED = runtests.pgo_extended
 
     support.set_match_tests(runtests.match_tests, runtests.ignore_tests)
+    support.set_match_tests2(runtests.accept_labels, runtests.ignore_labels)
 
     if runtests.use_junit:
         support.junit_xml_list = []

diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
@@ -472,28 +472,28 @@ def requires_zlib(reason='requires zlib'):
         import zlib
     except ImportError:
         zlib = None
-    return unittest.skipUnless(zlib, reason)
+    return skipUnless(zlib, reason, label='requires_zlib')
 
 def requires_gzip(reason='requires gzip'):
     try:
         import gzip
     except ImportError:
         gzip = None
-    return unittest.skipUnless(gzip, reason)
+    return skipUnless(gzip, reason, label='requires_gzip')
 
 def requires_bz2(reason='requires bz2'):
     try:
         import bz2
     except ImportError:
         bz2 = None
-    return unittest.skipUnless(bz2, reason)
+    return skipUnless(bz2, reason, label='requires_bz2')
 
 def requires_lzma(reason='requires lzma'):
     try:
         import lzma
     except ImportError:
         lzma = None
-    return unittest.skipUnless(lzma, reason)
+    return skipUnless(lzma, reason, label='requires_lzma')
 
 def has_no_debug_ranges():
     try:
@@ -504,7 +504,7 @@ def has_no_debug_ranges():
     return not bool(config['code_debug_ranges'])
 
 def requires_debug_ranges(reason='requires co_positions / debug_ranges'):
-    return unittest.skipIf(has_no_debug_ranges(), reason)
+    return skipIf(has_no_debug_ranges(), reason, label='requires_debug_ranges')
 
 # Is not actually used in tests, but is kept for compatibility.
 is_jython = sys.platform.startswith('java')
@@ -524,13 +524,13 @@ def requires_debug_ranges(reason='requires co_positions / debug_ranges'):
 has_fork_support = hasattr(os, "fork") and not is_emscripten and not is_wasi
 
 def requires_fork():
-    return unittest.skipUnless(has_fork_support, "requires working os.fork()")
+    return skipUnless(has_fork_support, "requires working os.fork()", label='requires_fork')
 
 has_subprocess_support = not is_emscripten and not is_wasi
 
 def requires_subprocess():
     """Used for subprocess, os.spawn calls, fd inheritance"""
-    return unittest.skipUnless(has_subprocess_support, "requires subprocess support")
+    return skipUnless(has_subprocess_support, "requires subprocess support", label='requires_subprocess')
 
 # Emscripten's socket emulation and WASI sockets have limitations.
 has_socket_support = not is_emscripten and not is_wasi
@@ -545,7 +545,7 @@ def requires_working_socket(*, module=False):
         if not has_socket_support:
             raise unittest.SkipTest(msg)
     else:
-        return unittest.skipUnless(has_socket_support, msg)
+        return skipUnless(has_socket_support, msg, label='requires_socket')
 
 # Does strftime() support glibc extension like '%4Y'?
 has_strftime_extensions = False
@@ -950,6 +950,8 @@ def bigmemtest(size, memuse, dry_run=True):
     test doesn't support dummy runs when -M is not specified.
     """
     def decorator(f):
+        @mark('bigmemtest')
+        @functools.wraps(f)
         def wrapper(self):
             size = wrapper.size
             memuse = wrapper.memuse
@@ -986,6 +988,8 @@ def wrapper(self):
 
 def bigaddrspacetest(f):
     """Decorator for tests that fill the address space."""
+    @mark('bigaddrspacetest')
+    @functools.wraps(f)
     def wrapper(self):
         if max_memuse < MAX_Py_ssize_t:
             if MAX_Py_ssize_t >= 2**63 - 1 and max_memuse >= 2**31:
@@ -1002,16 +1006,31 @@ def wrapper(self):
 #=======================================================================
 # unittest integration.
 
-def _id(obj):
-    return obj
+def mark(label):
+    def decorator(test):
+        setattr(test, label, True)
+        return test
+    return decorator
+
+def combine(*decorators):
+    def decorator(test):
+        for deco in reversed(decorators):
+            test = deco(test)
+        return test
+    return decorator
+
+def skipUnless(condition, reason, *, label):
+    return combine(unittest.skipUnless(condition, reason), mark(label))
+
+def skipIf(condition, reason, *, label):
+    return combine(unittest.skipIf(condition, reason), mark(label))
 
 def requires_resource(resource):
     if resource == 'gui' and not _is_gui_available():
-        return unittest.skip(_is_gui_available.reason)
-    if is_resource_enabled(resource):
-        return _id
-    else:
-        return unittest.skip("resource {0!r} is not enabled".format(resource))
+        return skipUnless(False, _is_gui_available.reason, label='requires_gui')
+    return skipUnless(is_resource_enabled(resource),
+                      f"resource {resource!r} is not enabled",
+                      label='requires_' + resource)
 
 def cpython_only(test):
     """
@@ -1020,8 +1039,16 @@ def cpython_only(test):
     return impl_detail(cpython=True)(test)
 
 def impl_detail(msg=None, **guards):
+    guards, _ = _parse_guards(guards)
+    decorators = []
+    for name in reversed(guards):
+        if guards[name]:
+            label = f'impl_detail_{name}'
+        else:
+            label = f'impl_detail_no_{name}'
+        decorators.append(mark(label))
     if check_impl_detail(**guards):
-        return _id
+        return combine(*decorators)
     if msg is None:
         guardnames, default = _parse_guards(guards)
         if default:
@@ -1030,7 +1057,7 @@ def impl_detail(msg=None, **guards):
             msg = "implementation detail specific to {0}"
         guardnames = sorted(guardnames.keys())
         msg = msg.format(' or '.join(guardnames))
-    return unittest.skip(msg)
+    return combine(unittest.skip(msg), *decorators)
 
 def _parse_guards(guards):
     # Returns a tuple ({platform_name: run_me}, default_value)
@@ -1157,17 +1184,16 @@ def _run_suite(suite):
 
 # By default, don't filter tests
 _match_test_func = None
+_match_test_func2 = None
 
 _accept_test_patterns = None
 _ignore_test_patterns = None
 
 
 def match_test(test):
     # Function used by support.run_unittest() and regrtest --list-cases
-    if _match_test_func is None:
-        return True
-    else:
-        return _match_test_func(test.id())
+    return ((_match_test_func is None or _match_test_func(test.id())) and
+            (_match_test_func2 is None or _match_test_func2(test)))
 
 
 def _is_full_match_test(pattern):
@@ -1212,6 +1238,45 @@ def match_function(test_id):
         _match_test_func = match_function
 
 
+def _check_obj_labels(obj, labels):
+    for label in labels:
+        if hasattr(obj, label):
+            return True
+    return False
+
+def _check_test_labels(test, labels):
+    if _check_obj_labels(test, labels):
+        return True
+    testMethod = getattr(test, test._testMethodName)
+    while testMethod is not None:
+        if _check_obj_labels(testMethod, labels):
+            return True
+        testMethod = getattr(testMethod, '__wrapped__', None)
+    return False
+
+def set_match_tests2(accept_labels=None, ignore_labels=None):
+    global _match_test_func2
+
+    if accept_labels is None:
+        accept_labels = ()
+    if ignore_labels is None:
+        ignore_labels = ()
+    # Create a copy since label lists can be mutable and so modified later
+    accept_labels = tuple(accept_labels)
+    ignore_labels = tuple(ignore_labels)
+
+    def match_function(test):
+        accept = True
+        ignore = False
+        if accept_labels:
+            accept = _check_test_labels(test, accept_labels)
+        if ignore_labels:
+            ignore = _check_test_labels(test, ignore_labels)
+        return accept and not ignore
+
+    _match_test_func2 = match_function
+
+
 def _compile_match_function(patterns):
     if not patterns:
         func = None