Skip to content

Commit c614451

Browse files
JukkaLJukkaL
authored
[mypyc] Import setuptools lazily in mypyc.build on Python 3.12+ (#22084)
Importing mypyc.build used to always import setuptools, distutils and mypyc.build_setup, but mypyc_build (type checking and C generation) needs none of them. They are only needed for building extensions, e.g. in mypycify. Importing them is slow, which adds significant fixed overhead to tools that call mypyc_build directly for many small builds, such as Bazel integrations. Move these imports into a helper function, called by the functions that need them. On Python versions before 3.12, distutils is still in the stdlib and setuptools must be imported before it to override it, so keep the eager imports there to preserve backward compatibility. In an interpreted microbenchmark this saved ~34 ms of a ~190 ms run (17%).
1 parent 48755e0 commit c614451

3 files changed

Lines changed: 74 additions & 18 deletions

File tree

‎mypyc/build.py‎

Lines changed: 42 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,13 @@
1313
1414
See the mypycify docs for additional arguments.
1515
16-
mypycify can integrate with either distutils or setuptools, but needs
17-
to know at import-time whether it is using distutils or setuputils. We
18-
hackily decide based on whether setuptools has been imported already.
16+
mypycify requires setuptools.
17+
18+
setuptools and distutils are only imported when they are needed to build
19+
extensions (such as in mypycify), since importing them is slow and
20+
generating C via mypyc_build doesn't need them. On Python versions before
21+
3.12, they are imported when this module is imported, since setuptools must
22+
be imported before distutils to override it.
1923
"""
2024

2125
from __future__ import annotations
@@ -28,7 +32,6 @@
2832
from collections.abc import Iterable
2933
from typing import TYPE_CHECKING, Any, NamedTuple, NoReturn, cast
3034

31-
import mypyc.build_setup # noqa: F401
3235
from mypy.build import BuildSource
3336
from mypy.errors import CompileError
3437
from mypy.fscache import FileSystemCache
@@ -135,12 +138,6 @@ class ModDesc(NamedTuple):
135138
),
136139
]
137140

138-
try:
139-
# Import setuptools so that it monkey-patch overrides distutils
140-
import setuptools
141-
except ImportError:
142-
pass
143-
144141
if TYPE_CHECKING:
145142
if sys.version_info >= (3, 12):
146143
from setuptools import Extension
@@ -152,14 +149,39 @@ class ModDesc(NamedTuple):
152149

153150
Extension: TypeAlias = _setuptools_Extension | _distutils_Extension
154151

155-
if sys.version_info >= (3, 12):
156-
# From setuptools' monkeypatch
157-
from distutils import ccompiler, sysconfig # type: ignore[import-not-found]
158-
else:
159-
from distutils import ccompiler, sysconfig
152+
153+
def import_distutils() -> tuple[Any, Any]:
154+
"""Import and return the distutils ccompiler and sysconfig modules.
155+
156+
This also imports setuptools (if available) first, so that it overrides distutils,
157+
and patches the compiler to support per-file flags. These imports are slow, and
158+
only needed when building C extensions, so this is done lazily (C generation in
159+
mypyc_build doesn't need it).
160+
"""
161+
try:
162+
# Import setuptools so that it monkey-patch overrides distutils
163+
import setuptools # noqa: F401
164+
except ImportError:
165+
pass
166+
167+
import mypyc.build_setup # noqa: F401
168+
169+
if sys.version_info >= (3, 12):
170+
# From setuptools' monkeypatch
171+
from distutils import ccompiler, sysconfig # type: ignore[import-not-found]
172+
else:
173+
from distutils import ccompiler, sysconfig
174+
return ccompiler, sysconfig
175+
176+
177+
if sys.version_info < (3, 12):
178+
# Distutils is still in the stdlib, and setuptools must be imported before it to
179+
# override it, so preserve the old behavior of importing these eagerly.
180+
import_distutils()
160181

161182

162183
def get_extension() -> type[Extension]:
184+
import_distutils()
163185
# We can work with either setuptools or distutils, and pick setuptools
164186
# if it has been imported.
165187
use_setuptools = "setuptools" in sys.modules
@@ -172,6 +194,8 @@ def get_extension() -> type[Extension]:
172194
else:
173195
if not use_setuptools:
174196
sys.exit("error: setuptools not installed")
197+
import setuptools
198+
175199
extension_class = setuptools.Extension
176200

177201
return extension_class
@@ -182,6 +206,7 @@ def setup_mypycify_vars() -> None:
182206
# There has to be a better approach to this.
183207

184208
# The vars can contain ints but we only work with str ones
209+
_, sysconfig = import_distutils()
185210
vars = cast(dict[str, str], sysconfig.get_config_vars())
186211
if sys.platform == "darwin":
187212
# Disable building 32-bit binaries, since we generate too much code
@@ -808,6 +833,7 @@ def get_cflags(
808833
List of compiler flags.
809834
"""
810835
if compiler_type is None:
836+
ccompiler, sysconfig = import_distutils()
811837
compiler: Any = ccompiler.new_compiler()
812838
sysconfig.customize_compiler(compiler)
813839
compiler_type = compiler.compiler_type
@@ -955,6 +981,7 @@ def mypycify(
955981
also needed if using experimental librt features). These
956982
have no backward compatibility guarantees!
957983
"""
984+
ccompiler, sysconfig = import_distutils()
958985

959986
# Skip redundant inplace .so copies on every build_ext invocation.
960987
_patch_setuptools_copy_extensions_to_source()

‎mypyc/test/librt_cache.py‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,15 +29,14 @@
2929

3030
import filelock
3131

32-
from mypyc.build import LIBRT_MODULES, get_cflags, include_dir
32+
from mypyc.build import LIBRT_MODULES, get_cflags, import_distutils, include_dir
3333
from mypyc.common import RUNTIME_C_FILES
3434
from mypyc.test.config import PREFIX
3535

3636

3737
def _librt_build_hash(experimental: bool, opt_level: str) -> str:
3838
"""Compute hash for librt build, including sources and build environment."""
39-
# Import lazily to ensure mypyc.build has ensured that distutils is correctly set up
40-
from distutils import ccompiler
39+
ccompiler, _ = import_distutils()
4140

4241
h = hashlib.sha256()
4342
# Include experimental flag

‎mypyc/test/test_misc.py‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
from __future__ import annotations
22

33
import os
4+
import subprocess
5+
import sys
46
import tempfile
7+
import textwrap
58
import unittest
69

710
from mypy.options import Options
@@ -10,6 +13,7 @@
1013
from mypyc.ir.pprint import format_blocks, generate_names_for_ir
1114
from mypyc.irbuild.ll_builder import LowLevelIRBuilder
1215
from mypyc.options import CompilerOptions
16+
from mypyc.test.config import PREFIX
1317

1418

1519
class TestMisc(unittest.TestCase):
@@ -163,3 +167,29 @@ def test_resolve_search_order_matches_preprocessor(self) -> None:
163167
assert resolve_cfile_deps(
164168
cfile_dir=includer, direct_includes=[(True, "shared.h")], target_dir=target
165169
) == {global_h}
170+
171+
172+
class TestLazySetuptoolsImport(unittest.TestCase):
173+
@unittest.skipIf(sys.version_info < (3, 12), "setuptools is imported eagerly before 3.12")
174+
def test_c_generation_does_not_import_setuptools(self) -> None:
175+
# Importing setuptools is slow, and generating C via mypyc_build doesn't need it.
176+
script = textwrap.dedent("""
177+
import os
178+
import sys
179+
from mypyc.build import mypyc_build
180+
from mypyc.options import CompilerOptions
181+
182+
mypyc_build(["a.py"], CompilerOptions(target_dir="build"), only_compile_paths=["a.py"])
183+
assert any(f.endswith(".c") for f in os.listdir("build")), "no C generated"
184+
loaded = {"setuptools", "distutils", "mypyc.build_setup"} & set(sys.modules)
185+
assert not loaded, f"unexpectedly imported: {sorted(loaded)}"
186+
""")
187+
with tempfile.TemporaryDirectory() as tmpdir:
188+
with open(os.path.join(tmpdir, "a.py"), "w") as f:
189+
f.write("def f(x: int) -> int:\n return x + 1\n")
190+
env = os.environ.copy()
191+
env["PYTHONPATH"] = PREFIX + os.pathsep + env.get("PYTHONPATH", "")
192+
result = subprocess.run(
193+
[sys.executable, "-c", script], cwd=tmpdir, env=env, capture_output=True, text=True
194+
)
195+
assert result.returncode == 0, result.stdout + result.stderr

0 commit comments

Comments
 (0)