Significantly optimize; more misc cleanups

JelleZijlstra · Dec 23, 2022 · Dec 18, 2022 · Dec 18, 2022 · Dec 18, 2022 · Dec 18, 2022
commit e9486014c645ff782e6cd12eaf6797f360d621ce
diff --git a/tests/regr_test.py b/tests/regr_test.py
@@ -16,6 +16,7 @@
 
 from utils import (
     PackageInfo,
+    VenvInfo,
     colored,
     get_all_testcase_directories,
     get_mypy_req,
@@ -28,6 +29,10 @@
 
 ReturnCode: TypeAlias = int
 
+TEST_CASES = "test_cases"
+VENV_DIR = ".venv"
+TYPESHED = "typeshed"
+
 SUPPORTED_PLATFORMS = ["linux", "darwin", "win32"]
 SUPPORTED_VERSIONS = ["3.11", "3.10", "3.9", "3.8", "3.7"]
 
@@ -36,7 +41,7 @@ def package_with_test_cases(package_name: str) -> PackageInfo:
     """Helper function for argument-parsing"""
 
     if package_name == "stdlib":
-        return PackageInfo("stdlib", Path("test_cases"))
+        return PackageInfo("stdlib", Path(TEST_CASES))
     test_case_dir = testcase_dir_from_package_name(package_name)
     if test_case_dir.is_dir():
         if not os.listdir(test_case_dir):
@@ -87,90 +92,102 @@ def package_with_test_cases(package_name: str) -> PackageInfo:
 )
 
 
-def run_testcases(
-    package: PackageInfo, flags: list[str], tmpdir_path: Path, python_minor_version: int
-) -> tuple[Path, subprocess.CompletedProcess[str]]:
-    python_exe = sys.executable
-    new_test_case_dir = tmpdir_path / "test_cases"
+def setup_testcase_dir(package: PackageInfo, tempdir: Path, new_test_case_dir: Path) -> None:
+    # --warn-unused-ignores doesn't work for files inside typeshed.
+    # SO, to work around this, we copy the test_cases directory into a TemporaryDirectory,
+    # and run the test cases inside of that.
     shutil.copytree(package.test_case_directory, new_test_case_dir)
+    if package.is_stdlib:
+        return
+
+    # HACK: we want to run these test cases in an isolated environment --
+    # we want mypy to see all stub packages listed in the "requires" field of METADATA.toml
+    # (and all stub packages required by those stub packages, etc. etc.),
+    # but none of the other stubs in typeshed.
+    #
+    # The best way of doing that without stopping --warn-unused-ignore from working
+    # seems to be to create a "new typeshed" directory in a tempdir
+    # that has only the required stubs copied over.
+    new_typeshed = tempdir / TYPESHED
+    new_typeshed.mkdir()
+    shutil.copytree(Path("stdlib"), new_typeshed / "stdlib")
+    requirements = get_recursive_requirements(package.name)
+    # mypy refuses to consider a directory a "valid typeshed directory"
+    # unless there's a stubs/mypy-extensions path inside it,
+    # so add that to the list of stubs to copy over to the new directory
+    for requirement in {package.name, *requirements.typeshed_pkgs, "mypy-extensions"}:
+        shutil.copytree(Path("stubs", requirement), new_typeshed / "stubs" / requirement)
+
+    if requirements.external_pkgs:
+        pip_exe = make_venv(tempdir / VENV_DIR).pip_exe
+        pip_command = [pip_exe, "install", get_mypy_req(), *requirements.external_pkgs]
+        try:
+            subprocess.run(pip_command, check=True, capture_output=True, text=True)
+        except subprocess.CalledProcessError as e:
+            print(e.stderr)
+            raise
+
+
+def run_testcases(package: PackageInfo, version: str, platform: str, *, tempdir: Path) -> subprocess.CompletedProcess[str]:
     env_vars = dict(os.environ)
+    new_test_case_dir = tempdir / TEST_CASES
+    testcasedir_already_setup = new_test_case_dir.exists() and new_test_case_dir.is_dir()
+
+    if not testcasedir_already_setup:
+        setup_testcase_dir(package, tempdir=tempdir, new_test_case_dir=new_test_case_dir)
+
+    # "--enable-error-code ignore-without-code" is purposefully ommited. See https://github.com/python/typeshed/pull/8083
+    flags = [
+        "--python-version",
+        version,
+        "--show-traceback",
+        "--show-error-codes",
+        "--no-error-summary",
+        "--platform",
+        platform,
+        "--strict",
+        "--pretty",
+        "--no-incremental",
+    ]
+
     if package.is_stdlib:
-        flags.extend(["--no-site-packages", "--custom-typeshed-dir", str(Path(__file__).parent.parent)])
+        python_exe = sys.executable
+        custom_typeshed = Path(__file__).parent.parent
+        flags.append("--no-site-packages")
     else:
-        # HACK: we want to run these test cases in an isolated environment --
-        # we want mypy to see all stub packages listed in the "requires" field of METADATA.toml
-        # (and all stub packages required by those stub packages, etc. etc.),
-        # but none of the other stubs in typeshed.
-        #
-        # The best way of doing that without stopping --warn-unused-ignore from working
-        # seems to be to create a "new typeshed" directory in a tempdir
-        # that has only the required stubs copied over.
-        new_typeshed = tmpdir_path / "typeshed"
-        new_typeshed.mkdir()
-        shutil.copytree(Path("stdlib"), new_typeshed / "stdlib")
-        requirements = get_recursive_requirements(package.name)
-        # mypy refuses to consider a directory a "valid typeshed directory"
-        # unless there's a stubs/mypy-extensions path inside it,
-        # so add that to the list of stubs to copy over to the new directory
-        for requirement in {package.name, *requirements.typeshed_pkgs, "mypy-extensions"}:
-            shutil.copytree(Path("stubs", requirement), new_typeshed / "stubs" / requirement)
-
-        if requirements.external_pkgs:
-            pip_exe, python_exe = make_venv(tmpdir_path / ".venv")
-            pip_command = [pip_exe, "install", get_mypy_req(), *requirements.external_pkgs]
-            try:
-                subprocess.run(pip_command, check=True, capture_output=True, text=True)
-            except subprocess.CalledProcessError as e:
-                print(e.stderr)
-                raise
+        custom_typeshed = tempdir / TYPESHED
+        env_vars["MYPYPATH"] = os.pathsep.join(map(str, custom_typeshed.glob("stubs/*")))
+        has_non_types_dependencies = (tempdir / VENV_DIR).exists()
+        if has_non_types_dependencies:
+            python_exe = VenvInfo.of_existing_venv(tempdir / VENV_DIR).python_exe
         else:
+            python_exe = sys.executable
             flags.append("--no-site-packages")
 
-        env_vars["MYPYPATH"] = os.pathsep.join(map(str, new_typeshed.glob("stubs/*")))
-        flags.extend(["--custom-typeshed-dir", str(new_typeshed)])
+    flags.extend(["--custom-typeshed-dir", str(custom_typeshed)])
 
     # If the test-case filename ends with -py39,
     # only run the test if --python-version was set to 3.9 or higher (for example)
     for path in new_test_case_dir.rglob("*.py"):
         if match := re.fullmatch(r".*-py3(\d{1,2})", path.stem):
             minor_version_required = int(match[1])
             assert f"3.{minor_version_required}" in SUPPORTED_VERSIONS
-            if minor_version_required <= python_minor_version:
-                flags.append(str(path))
-        else:
-            flags.append(str(path))
+            python_minor_version = int(version.split(".")[1])
+            if minor_version_required > python_minor_version:
+                continue
+        flags.append(str(path))
 
     mypy_command = [python_exe, "-m", "mypy"] + flags
-    result = subprocess.run(mypy_command, capture_output=True, text=True, env=env_vars)
-    return new_test_case_dir, result
+    return subprocess.run(mypy_command, capture_output=True, text=True, env=env_vars)
 
 
-def test_testcase_directory(package: PackageInfo, version: str, platform: str, quiet: bool) -> ReturnCode:
+def test_testcase_directory(package: PackageInfo, version: str, platform: str, *, quiet: bool, tempdir: Path) -> ReturnCode:
     msg = f"Running mypy --platform {platform} --python-version {version} on the "
     msg += "standard library test cases..." if package.is_stdlib else f"test cases for {package.name!r}..."
     if not quiet:
         print(msg, end=" ", flush=True)
 
-    # "--enable-error-code ignore-without-code" is purposefully ommited. See https://github.com/python/typeshed/pull/8083
-    flags = [
-        "--python-version",
-        version,
-        "--show-traceback",
-        "--show-error-codes",
-        "--no-error-summary",
-        "--platform",
-        platform,
-        "--strict",
-        "--pretty",
-    ]
-
-    # --warn-unused-ignores doesn't work for files inside typeshed.
-    # SO, to work around this, we copy the test_cases directory into a TemporaryDirectory,
-    # and run the test cases inside of that.
-    with tempfile.TemporaryDirectory() as td:
-        new_test_case_dir, result = run_testcases(
-            package=package, flags=flags, tmpdir_path=Path(td), python_minor_version=int(version.split(".")[1])
-        )
+    result = run_testcases(package=package, version=version, platform=platform, tempdir=tempdir)
 
     if result.returncode:
         if quiet:
@@ -179,7 +196,7 @@ def test_testcase_directory(package: PackageInfo, version: str, platform: str, q
             # If there are errors, the output is inscrutable if this isn't printed.
             print(msg, end=" ")
         print_error("failure\n")
-        replacements = (str(new_test_case_dir), str(package.test_case_directory))
+        replacements = (str(tempdir / TEST_CASES), str(package.test_case_directory))
         if result.stderr:
             print_error(result.stderr, fix_path=replacements)
         if result.stdout:
@@ -204,8 +221,12 @@ def main() -> ReturnCode:
         versions_to_test = args.versions_to_test or [f"3.{sys.version_info[1]}"]
 
     code = 0
-    for platform, version, directory in product(platforms_to_test, versions_to_test, testcase_directories):
-        code = max(code, test_testcase_directory(directory, version, platform, args.quiet))
+    for testcase_dir in testcase_directories:
+        with tempfile.TemporaryDirectory() as td:
+            tempdir = Path(td)
+            for platform, version in product(platforms_to_test, versions_to_test):
+                this_code = test_testcase_directory(testcase_dir, version, platform, quiet=args.quiet, tempdir=tempdir)
+                code = max(code, this_code)
     if code:
         print_error("\nTest completed with errors")
     else:

diff --git a/tests/utils.py b/tests/utils.py
@@ -127,6 +127,17 @@ class VenvInfo(NamedTuple):
     pip_exe: Annotated[str, "A path to the venv's pip executable"]
     python_exe: Annotated[str, "A path to the venv's python executable"]
 
+    @staticmethod
+    def of_existing_venv(venv_dir: Path) -> VenvInfo:
+        if sys.platform == "win32":
+            pip = venv_dir / "Scripts" / "pip.exe"
+            python = venv_dir / "Scripts" / "python.exe"
+        else:
+            pip = venv_dir / "bin" / "pip"
+            python = venv_dir / "bin" / "python"
+
+        return VenvInfo(str(pip), str(python))
+
 
 def make_venv(venv_dir: Path) -> VenvInfo:
     try:
@@ -139,14 +150,7 @@ def make_venv(venv_dir: Path) -> VenvInfo:
             )
         raise
 
-    if sys.platform == "win32":
-        pip = venv_dir / "Scripts" / "pip.exe"
-        python = venv_dir / "Scripts" / "python.exe"
-    else:
-        pip = venv_dir / "bin" / "pip"
-        python = venv_dir / "bin" / "python"
-
-    return VenvInfo(str(pip), str(python))
+    return VenvInfo.of_existing_venv(venv_dir)
 
 
 @cache