Regen frozen.c and Makefile.pre.in.

ericsnowcurrently · Aug 30, 2021 · Jun 10, 2021 · Aug 10, 2021 · Jun 8, 2021 · Aug 10, 2021
commit 1cae77177cb8887cfd301cdaee3d4a98fae8dad5
@@ -986,8 +986,14 @@ regen-opcode-targets:
 Python/ceval.o: $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/ceval_gil.h \
 		$(srcdir)/Python/condvar.h
 
-Python/frozen.o: $(srcdir)/Python/importlib.h $(srcdir)/Python/importlib_external.h \
-		$(srcdir)/Python/importlib_zipimport.h $(srcdir)/Python/frozen_hello.h
+# FROZEN_FILES is auto-generated by Tools/scripts/freeze_modules.py.
+FROZEN_FILES = \
+		$(srcdir)/Python/importlib.h \
+		$(srcdir)/Python/importlib_external.h \
+		$(srcdir)/Python/importlib_zipimport.h \
+		$(srcdir)/Python/frozen_hello.h
+
+Python/frozen.o: $(FROZEN_FILES)
 
 # Generate DTrace probe macros, then rename them (PYTHON_ -> PyDTrace_) to
 # follow our naming conventions. dtrace(1) uses the output filename to generate

@@ -1,10 +1,13 @@
+/* Auto-generated by Tools/scripts/freeze_modules.py */";
 
 /* Frozen modules initializer */
 
 #include "Python.h"
+/* importlib */
 #include "importlib.h"
 #include "importlib_external.h"
 #include "importlib_zipimport.h"
+/* stdlib */
 
 /* In order to test the support for frozen modules, by default we
    define a single frozen module, __hello__.  Loading it will print
@@ -15,22 +18,23 @@
  */
 #include "frozen_hello.h"
 
-#define SIZE (int)sizeof(_Py_M__hello)
-
 static const struct _frozen _PyImport_FrozenModules[] = {
     /* importlib */
     {"_frozen_importlib", _Py_M__importlib__bootstrap,
         (int)sizeof(_Py_M__importlib__bootstrap)},
     {"_frozen_importlib_external", _Py_M__importlib__bootstrap_external,
         (int)sizeof(_Py_M__importlib__bootstrap_external)},
-    {"zipimport", _Py_M__zipimport,
-        (int)sizeof(_Py_M__zipimport)},
+    {"zipimport", _Py_M__zipimport, (int)sizeof(_Py_M__zipimport)},
+
+    /* stdlib */
+    /* without site (python -S) */
+    /* with site */
 
     /* Test module */
-    {"__hello__", _Py_M__hello, SIZE},
+    {"__hello__", _Py_M__hello, (int)sizeof(_Py_M__hello)},
     /* Test package (negative size indicates package-ness) */
-    {"__phello__", _Py_M__hello, -SIZE},
-    {"__phello__.spam", _Py_M__hello, SIZE},
+    {"__phello__", _Py_M__hello, -(int)sizeof(_Py_M__hello)},
+    {"__phello__.spam", _Py_M__hello, (int)sizeof(_Py_M__hello)},
 
     {0, 0, 0} /* sentinel */
 };

@@ -2,16 +2,21 @@
 import os.path
 import subprocess
 import sys
+import textwrap
 
 
 SCRIPTS_DIR = os.path.abspath(os.path.dirname(__file__))
 TOOLS_DIR = os.path.dirname(SCRIPTS_DIR)
 ROOT_DIR = os.path.dirname(TOOLS_DIR)
+
 STDLIB_DIR = os.path.join(ROOT_DIR, 'Lib')
 MODULES_DIR = os.path.join(ROOT_DIR, 'Python')
 #MODULES_DIR = os.path.join(ROOT_DIR, 'Modules')
 TOOL = os.path.join(ROOT_DIR, 'Programs', '_freeze_module')
 
+FROZEN_FILE = os.path.join(ROOT_DIR, 'Python', 'frozen.c')
+MAKEFILE = os.path.join(ROOT_DIR, 'Makefile.pre.in')
+
 # These are modules that get frozen.
 FROZEN = {
     # frozen_id: (pyfile, frozenfile)
@@ -42,6 +47,32 @@
         'hello',
     ],
 }
+# These are the modules defined in frozen.c, in order.
+MODULES = [
+    # frozen_id
+    # (module, frozen_id)
+    # (<package>, frozen_id)
+
+    # importlib
+    '# [importlib]',
+    ('_frozen_importlib', 'importlib._bootstrap'),
+    ('_frozen_importlib_external', 'importlib._bootstrap_external'),
+    'zipimport',
+
+    # stdlib
+    '# [stdlib]',
+    '# without site (python -S)',
+    # ...
+    '# with site',
+    # ...
+
+    # test
+    '# [Test module]',
+    ('__hello__', 'hello'),
+    '# Test package (negative size indicates package-ness)',
+    ('<__phello__>', 'hello'),
+    ('__phello__.spam', 'hello'),
+]
 
 
 def expand_frozen(destdir=MODULES_DIR):
@@ -99,7 +130,44 @@ def expand_frozen(destdir=MODULES_DIR):
             else:
                 group.append(frozenid)
 
-    return groups, frozen
+    # Finally, expand MODULES.
+    rows = []
+    for row in MODULES:
+        if isinstance(row, str) and not row.startswith('#'):
+            if row in packages:
+                for modname in packages[row]:
+                    rows.append((modname, modname))
+                continue
+            row = (row, row)
+        rows.append(row)
+    headers = []
+    specs = []
+    for row in rows:
+        if isinstance(row, str):
+            assert row.startswith('# ')
+            comment = row[2:]
+            if comment.startswith('['):
+                line = f'/* {comment[1:-1]} */'
+                headers.append(line)
+                specs.extend(['', line])
+            else:
+                specs.append(f'/* {comment} */')
+            continue
+        modname, frozenid = row
+        _, frozenfile, ispkg = frozen[frozenid]
+
+        assert frozenfile.startswith(destdir), (frozenfile, destdir)
+        header = os.path.relpath(frozenfile, destdir)
+        if header not in headers:
+            headers.append(header)
+
+        if modname.startswith('<'):
+            modname = modname[1:-1]
+            ispkg = True
+        spec = (modname, frozenid, ispkg)
+        specs.append(spec)
+
+    return groups, frozen, headers, specs
 
 
 #def freeze_stdlib_module(modname, destdir=MODULES_DIR):
@@ -122,6 +190,123 @@ def expand_frozen(destdir=MODULES_DIR):
 #        _freeze_module(modname, pyfile, frozenfile)
 
 
+def regen_frozen(headers, specs):
+    headerlines = []
+    for row in headers:
+        assert isinstance(row, str), row
+        if row == '/* Test module */':
+            comment = textwrap.dedent('''
+                /* In order to test the support for frozen modules, by default we
+                   define a single frozen module, __hello__.  Loading it will print
+                   some famous words... */
+
+                /* Run "make regen-frozen" to regen the file below (e.g. after a bytecode
+                 * format change).  The include file defines _Py_M__hello as an array of bytes.
+                 */
+                 '''.rstrip())
+            headerlines.extend(comment.splitlines())
+            continue
+        elif not row.startswith('/*'):
+            assert row.endswith('.h')
+            assert os.path.basename(row) == row, row
+            row = f'#include "{row}"'
+        headerlines.append(row)
+
+    speclines = []
+    indent = '    '
+    for spec in specs:
+        if isinstance(spec, str):
+            speclines.append(spec)
+            continue
+        modname, frozenid, ispkg = spec
+        # This matches what we do in Programs/_freeze_module.c:
+        symbol = '_Py_M__' + frozenid.replace('.', '_')
+        pkg = '-' if ispkg else ''
+        line = '{"%s", %s, %s(int)sizeof(%s)},' % (modname, symbol, pkg, symbol)
+        if len(line) < 80:
+            speclines.append(line)
+        else:
+            line1, _, line2 = line.rpartition(' ')
+            speclines.append(line1)
+            speclines.append(indent + line2)
+    if not speclines[0]:
+        del speclines[0]
+    for i, line in enumerate(speclines):
+        if line:
+            speclines[i] = indent + line
+
+    text = textwrap.dedent('''\
+        /* Auto-generated by Tools/scripts/freeze_modules.py */";
+
+        /* Frozen modules initializer */
+
+        #include "Python.h"
+        %s
+
+        static const struct _frozen _PyImport_FrozenModules[] = {
+        %s
+
+            {0, 0, 0} /* sentinel */
+        };
+
+        /* Embedding apps may change this pointer to point to their favorite
+           collection of frozen modules: */
+
+        const struct _frozen *PyImport_FrozenModules = _PyImport_FrozenModules;
+        '''
+    ) % (
+        os.linesep.join(headerlines),
+        os.linesep.join(speclines),
+    )
+    with open(FROZEN_FILE, 'w') as outfile:
+        outfile.write(text)
+
+
+def regen_makefile(headers, destdir=MODULES_DIR):
+    reldir = os.path.relpath(destdir, ROOT_DIR)
+    assert destdir.endswith(reldir), destdir
+    frozenfiles = []
+    for row in headers:
+        if not row.startswith('/*'):
+            header = row
+            assert header.endswith('.h'), header
+            assert os.path.basename(header) == header, header
+            relfile = f'{reldir}/{header}'.replace('\\', '/')
+            frozenfiles.append(f'$(srcdir)/{relfile}')
+
+    tmpfile = MAKEFILE + '.tmp'
+    try:
+        with open(tmpfile, 'w') as outfile:
+            with open(MAKEFILE) as infile:
+                lines = iter(infile)
+                # Get to $FROZEN_FILES.
+                for line in lines:
+                    outfile.write(line)
+                    if line.rstrip() == 'FROZEN_FILES = \\':
+                        break
+                else:
+                    raise Exception("did you delete a line you shouldn't have?")
+                # Pop off the existing values.
+                for line in lines:
+                    if line.rstrip() == 'Python/frozen.o: $(FROZEN_FILES)':
+                        break
+                else:
+                    raise Exception("did you delete a line you shouldn't have?")
+                # Add the regen'ed values.
+                for header in frozenfiles[:-1]:
+                    outfile.write(f'\t\t{header} \\{os.linesep}')
+                outfile.write(f'\t\t{frozenfiles[-1]}{os.linesep}')
+                # Keep the rest of the file.
+                outfile.write(os.linesep)
+                outfile.write('Python/frozen.o: $(FROZEN_FILES)' + os.linesep)
+                for line in lines:
+                    outfile.write(line)
+        os.rename(tmpfile, MAKEFILE)
+    finally:
+        if os.path.exists(tmpfile):
+            os.remove(tmpfile)
+
+
 def _freeze_module(frozenid, pyfile, frozenfile):
     tmpfile = frozenfile + '.new'
 
@@ -180,7 +365,7 @@ def parse_args(argv=sys.argv[1:], prog=sys.argv[0]):
 
 
 def main(kinds=None):
-    groups, frozen = expand_frozen(MODULES_DIR)
+    groups, frozen, headers, specs = expand_frozen(MODULES_DIR)
 
     # First, freeze the modules.
     # (We use a consistent order:)
@@ -193,6 +378,10 @@ def main(kinds=None):
                 pyfile, frozenfile, _ = frozen[frozenid]
                 _freeze_module(frozenid, pyfile, frozenfile)
 
+    # Then regen frozen.c and Makefile.pre.in.
+    regen_frozen(headers, specs)
+    regen_makefile(headers, MODULES_DIR)
+
 
 if __name__ == '__main__':
     kwargs = parse_args()