From df1901ed254edf2f4c7c1164d00d9a9120f95960 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 22 Nov 2021 12:28:56 -0800 Subject: [PATCH 01/11] bpo-45873: Get rid of bootstrap_python Instead we use $(PYTHON_FOR_REGEN) .../deepfreeze.py with the frozen .h file as input, as we did for Windows in bpo-45850. We also get rid of the code that generates the .h files when make regen-frozen is run (i.e., .../make_frozen.py). --- Makefile.pre.in | 207 ++++++++++---------------------- Tools/scripts/freeze_modules.py | 65 +--------- 2 files changed, 68 insertions(+), 204 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index d52f73620795c1..2e3aab762a376c 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -275,7 +275,6 @@ LIBOBJS= @LIBOBJS@ PYTHON= python$(EXE) BUILDPYTHON= python$(BUILDEXE) -BOOTSTRAP= _bootstrap_python PYTHON_FOR_REGEN?=@PYTHON_FOR_REGEN@ UPDATE_FILE=$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/update_file.py @@ -710,9 +709,9 @@ platform: $(BUILDPYTHON) pybuilddir.txt # problems by creating a dummy pybuilddir.txt just to allow interpreter # initialization to succeed. It will be overwritten by generate-posix-vars # or removed in case of failure. -pybuilddir.txt: $(BOOTSTRAP) +pybuilddir.txt: $(BUILDPYTHON) @echo "none" > ./pybuilddir.txt - ./$(BOOTSTRAP) -S -m sysconfig --generate-posix-vars ;\ + $(RUNSHARED) $(PYTHON_FOR_BUILD) -S -m sysconfig --generate-posix-vars ;\ if test $$? -ne 0 ; then \ echo "generate-posix-vars failed" ; \ rm -f ./pybuilddir.txt ; \ @@ -934,155 +933,75 @@ regen-test-frozenmain: $(BUILDPYTHON) Programs/_testembed: Programs/_testembed.o $(LIBRARY_DEPS) $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/_testembed.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) -############################################################################ -# "Bootstrap Python" used to run deepfreeze.py - -BOOTSTRAP_HEADERS = \ - Python/frozen_modules/importlib._bootstrap.h \ - Python/frozen_modules/importlib._bootstrap_external.h \ - Python/frozen_modules/zipimport.h - -Python/bootstrap_frozen.o: Python/bootstrap_frozen.c Include/cpython/import.h $(BOOTSTRAP_HEADERS) - -$(BOOTSTRAP): $(LIBRARY_OBJS_OMIT_FROZEN) \ - Python/bootstrap_frozen.o Programs/python.o - $(LINKCC) $(PY_CORE_LDFLAGS) -o $@ $(LIBRARY_OBJS_OMIT_FROZEN) \ - Python/bootstrap_frozen.o \ - Programs/python.o \ - $(LIBS) $(MODLIBS) $(SYSLIBS) - ############################################################################ # Deepfreeze targets .PHONY: regen-deepfreeze regen-deepfreeze: $(DEEPFREEZE_OBJS) -DEEPFREEZE_DEPS = \ - $(BOOTSTRAP) \ - pybuilddir.txt \ - $(srcdir)/Tools/scripts/deepfreeze.py +DEEPFREEZE_DEPS=$(srcdir)/Tools/scripts/deepfreeze.py # BEGIN: deepfreeze modules -Python/deepfreeze/importlib._bootstrap.c: $(srcdir)/Lib/importlib/_bootstrap.py $(DEEPFREEZE_DEPS) - @echo "Deepfreezing Python/deepfreeze/importlib._bootstrap.c from Lib/importlib/_bootstrap.py" - @./$(BOOTSTRAP) \ - $(srcdir)/Tools/scripts/deepfreeze.py \ - $(srcdir)/Lib/importlib/_bootstrap.py -m importlib._bootstrap -o Python/deepfreeze/importlib._bootstrap.c - -Python/deepfreeze/importlib._bootstrap_external.c: $(srcdir)/Lib/importlib/_bootstrap_external.py $(DEEPFREEZE_DEPS) - @echo "Deepfreezing Python/deepfreeze/importlib._bootstrap_external.c from Lib/importlib/_bootstrap_external.py" - @./$(BOOTSTRAP) \ - $(srcdir)/Tools/scripts/deepfreeze.py \ - $(srcdir)/Lib/importlib/_bootstrap_external.py -m importlib._bootstrap_external -o Python/deepfreeze/importlib._bootstrap_external.c - -Python/deepfreeze/zipimport.c: $(srcdir)/Lib/zipimport.py $(DEEPFREEZE_DEPS) - @echo "Deepfreezing Python/deepfreeze/zipimport.c from Lib/zipimport.py" - @./$(BOOTSTRAP) \ - $(srcdir)/Tools/scripts/deepfreeze.py \ - $(srcdir)/Lib/zipimport.py -m zipimport -o Python/deepfreeze/zipimport.c - -Python/deepfreeze/abc.c: $(srcdir)/Lib/abc.py $(DEEPFREEZE_DEPS) - @echo "Deepfreezing Python/deepfreeze/abc.c from Lib/abc.py" - @./$(BOOTSTRAP) \ - $(srcdir)/Tools/scripts/deepfreeze.py \ - $(srcdir)/Lib/abc.py -m abc -o Python/deepfreeze/abc.c - -Python/deepfreeze/codecs.c: $(srcdir)/Lib/codecs.py $(DEEPFREEZE_DEPS) - @echo "Deepfreezing Python/deepfreeze/codecs.c from Lib/codecs.py" - @./$(BOOTSTRAP) \ - $(srcdir)/Tools/scripts/deepfreeze.py \ - $(srcdir)/Lib/codecs.py -m codecs -o Python/deepfreeze/codecs.c - -Python/deepfreeze/io.c: $(srcdir)/Lib/io.py $(DEEPFREEZE_DEPS) - @echo "Deepfreezing Python/deepfreeze/io.c from Lib/io.py" - @./$(BOOTSTRAP) \ - $(srcdir)/Tools/scripts/deepfreeze.py \ - $(srcdir)/Lib/io.py -m io -o Python/deepfreeze/io.c - -Python/deepfreeze/_collections_abc.c: $(srcdir)/Lib/_collections_abc.py $(DEEPFREEZE_DEPS) - @echo "Deepfreezing Python/deepfreeze/_collections_abc.c from Lib/_collections_abc.py" - @./$(BOOTSTRAP) \ - $(srcdir)/Tools/scripts/deepfreeze.py \ - $(srcdir)/Lib/_collections_abc.py -m _collections_abc -o Python/deepfreeze/_collections_abc.c - -Python/deepfreeze/_sitebuiltins.c: $(srcdir)/Lib/_sitebuiltins.py $(DEEPFREEZE_DEPS) - @echo "Deepfreezing Python/deepfreeze/_sitebuiltins.c from Lib/_sitebuiltins.py" - @./$(BOOTSTRAP) \ - $(srcdir)/Tools/scripts/deepfreeze.py \ - $(srcdir)/Lib/_sitebuiltins.py -m _sitebuiltins -o Python/deepfreeze/_sitebuiltins.c - -Python/deepfreeze/genericpath.c: $(srcdir)/Lib/genericpath.py $(DEEPFREEZE_DEPS) - @echo "Deepfreezing Python/deepfreeze/genericpath.c from Lib/genericpath.py" - @./$(BOOTSTRAP) \ - $(srcdir)/Tools/scripts/deepfreeze.py \ - $(srcdir)/Lib/genericpath.py -m genericpath -o Python/deepfreeze/genericpath.c - -Python/deepfreeze/ntpath.c: $(srcdir)/Lib/ntpath.py $(DEEPFREEZE_DEPS) - @echo "Deepfreezing Python/deepfreeze/ntpath.c from Lib/ntpath.py" - @./$(BOOTSTRAP) \ - $(srcdir)/Tools/scripts/deepfreeze.py \ - $(srcdir)/Lib/ntpath.py -m ntpath -o Python/deepfreeze/ntpath.c - -Python/deepfreeze/posixpath.c: $(srcdir)/Lib/posixpath.py $(DEEPFREEZE_DEPS) - @echo "Deepfreezing Python/deepfreeze/posixpath.c from Lib/posixpath.py" - @./$(BOOTSTRAP) \ - $(srcdir)/Tools/scripts/deepfreeze.py \ - $(srcdir)/Lib/posixpath.py -m posixpath -o Python/deepfreeze/posixpath.c - -Python/deepfreeze/os.c: $(srcdir)/Lib/os.py $(DEEPFREEZE_DEPS) - @echo "Deepfreezing Python/deepfreeze/os.c from Lib/os.py" - @./$(BOOTSTRAP) \ - $(srcdir)/Tools/scripts/deepfreeze.py \ - $(srcdir)/Lib/os.py -m os -o Python/deepfreeze/os.c - -Python/deepfreeze/site.c: $(srcdir)/Lib/site.py $(DEEPFREEZE_DEPS) - @echo "Deepfreezing Python/deepfreeze/site.c from Lib/site.py" - @./$(BOOTSTRAP) \ - $(srcdir)/Tools/scripts/deepfreeze.py \ - $(srcdir)/Lib/site.py -m site -o Python/deepfreeze/site.c - -Python/deepfreeze/stat.c: $(srcdir)/Lib/stat.py $(DEEPFREEZE_DEPS) - @echo "Deepfreezing Python/deepfreeze/stat.c from Lib/stat.py" - @./$(BOOTSTRAP) \ - $(srcdir)/Tools/scripts/deepfreeze.py \ - $(srcdir)/Lib/stat.py -m stat -o Python/deepfreeze/stat.c - -Python/deepfreeze/__hello__.c: $(srcdir)/Lib/__hello__.py $(DEEPFREEZE_DEPS) - @echo "Deepfreezing Python/deepfreeze/__hello__.c from Lib/__hello__.py" - @./$(BOOTSTRAP) \ - $(srcdir)/Tools/scripts/deepfreeze.py \ - $(srcdir)/Lib/__hello__.py -m __hello__ -o Python/deepfreeze/__hello__.c - -Python/deepfreeze/__phello__.c: $(srcdir)/Lib/__phello__/__init__.py $(DEEPFREEZE_DEPS) - @echo "Deepfreezing Python/deepfreeze/__phello__.c from Lib/__phello__/__init__.py" - @./$(BOOTSTRAP) \ - $(srcdir)/Tools/scripts/deepfreeze.py \ - $(srcdir)/Lib/__phello__/__init__.py -m __phello__ -o Python/deepfreeze/__phello__.c - -Python/deepfreeze/__phello__.ham.c: $(srcdir)/Lib/__phello__/ham/__init__.py $(DEEPFREEZE_DEPS) - @echo "Deepfreezing Python/deepfreeze/__phello__.ham.c from Lib/__phello__/ham/__init__.py" - @./$(BOOTSTRAP) \ - $(srcdir)/Tools/scripts/deepfreeze.py \ - $(srcdir)/Lib/__phello__/ham/__init__.py -m __phello__.ham -o Python/deepfreeze/__phello__.ham.c - -Python/deepfreeze/__phello__.ham.eggs.c: $(srcdir)/Lib/__phello__/ham/eggs.py $(DEEPFREEZE_DEPS) - @echo "Deepfreezing Python/deepfreeze/__phello__.ham.eggs.c from Lib/__phello__/ham/eggs.py" - @./$(BOOTSTRAP) \ - $(srcdir)/Tools/scripts/deepfreeze.py \ - $(srcdir)/Lib/__phello__/ham/eggs.py -m __phello__.ham.eggs -o Python/deepfreeze/__phello__.ham.eggs.c - -Python/deepfreeze/__phello__.spam.c: $(srcdir)/Lib/__phello__/spam.py $(DEEPFREEZE_DEPS) - @echo "Deepfreezing Python/deepfreeze/__phello__.spam.c from Lib/__phello__/spam.py" - @./$(BOOTSTRAP) \ - $(srcdir)/Tools/scripts/deepfreeze.py \ - $(srcdir)/Lib/__phello__/spam.py -m __phello__.spam -o Python/deepfreeze/__phello__.spam.c - -Python/deepfreeze/frozen_only.c: $(srcdir)/Tools/freeze/flag.py $(DEEPFREEZE_DEPS) - @echo "Deepfreezing Python/deepfreeze/frozen_only.c from Tools/freeze/flag.py" - @./$(BOOTSTRAP) \ - $(srcdir)/Tools/scripts/deepfreeze.py \ - $(srcdir)/Tools/freeze/flag.py -m frozen_only -o Python/deepfreeze/frozen_only.c +Python/deepfreeze/importlib._bootstrap.c: $(srcdir)/Python/frozen_modules/importlib._bootstrap.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/importlib._bootstrap.h -m importlib._bootstrap -o Python/deepfreeze/importlib._bootstrap.c + +Python/deepfreeze/importlib._bootstrap_external.c: $(srcdir)/Python/frozen_modules/importlib._bootstrap_external.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/importlib._bootstrap_external.h -m importlib._bootstrap_external -o Python/deepfreeze/importlib._bootstrap_external.c + +Python/deepfreeze/zipimport.c: $(srcdir)/Python/frozen_modules/zipimport.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/zipimport.h -m zipimport -o Python/deepfreeze/zipimport.c + +Python/deepfreeze/abc.c: $(srcdir)/Python/frozen_modules/abc.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/abc.h -m abc -o Python/deepfreeze/abc.c + +Python/deepfreeze/codecs.c: $(srcdir)/Python/frozen_modules/codecs.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/codecs.h -m codecs -o Python/deepfreeze/codecs.c + +Python/deepfreeze/io.c: $(srcdir)/Python/frozen_modules/io.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/io.h -m io -o Python/deepfreeze/io.c + +Python/deepfreeze/_collections_abc.c: $(srcdir)/Python/frozen_modules/_collections_abc.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/_collections_abc.h -m _collections_abc -o Python/deepfreeze/_collections_abc.c + +Python/deepfreeze/_sitebuiltins.c: $(srcdir)/Python/frozen_modules/_sitebuiltins.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/_sitebuiltins.h -m _sitebuiltins -o Python/deepfreeze/_sitebuiltins.c + +Python/deepfreeze/genericpath.c: $(srcdir)/Python/frozen_modules/genericpath.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/genericpath.h -m genericpath -o Python/deepfreeze/genericpath.c + +Python/deepfreeze/ntpath.c: $(srcdir)/Python/frozen_modules/ntpath.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/ntpath.h -m ntpath -o Python/deepfreeze/ntpath.c + +Python/deepfreeze/posixpath.c: $(srcdir)/Python/frozen_modules/posixpath.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/posixpath.h -m posixpath -o Python/deepfreeze/posixpath.c + +Python/deepfreeze/os.c: $(srcdir)/Python/frozen_modules/os.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/os.h -m os -o Python/deepfreeze/os.c + +Python/deepfreeze/site.c: $(srcdir)/Python/frozen_modules/site.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/site.h -m site -o Python/deepfreeze/site.c + +Python/deepfreeze/stat.c: $(srcdir)/Python/frozen_modules/stat.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/stat.h -m stat -o Python/deepfreeze/stat.c + +Python/deepfreeze/__hello__.c: $(srcdir)/Python/frozen_modules/__hello__.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/__hello__.h -m __hello__ -o Python/deepfreeze/__hello__.c + +Python/deepfreeze/__phello__.c: $(srcdir)/Python/frozen_modules/__phello__.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/__phello__.h -m __phello__ -o Python/deepfreeze/__phello__.c + +Python/deepfreeze/__phello__.ham.c: $(srcdir)/Python/frozen_modules/__phello__.ham.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/__phello__.ham.h -m __phello__.ham -o Python/deepfreeze/__phello__.ham.c + +Python/deepfreeze/__phello__.ham.eggs.c: $(srcdir)/Python/frozen_modules/__phello__.ham.eggs.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/__phello__.ham.eggs.h -m __phello__.ham.eggs -o Python/deepfreeze/__phello__.ham.eggs.c + +Python/deepfreeze/__phello__.spam.c: $(srcdir)/Python/frozen_modules/__phello__.spam.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/__phello__.spam.h -m __phello__.spam -o Python/deepfreeze/__phello__.spam.c + +Python/deepfreeze/frozen_only.c: $(srcdir)/Python/frozen_modules/frozen_only.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/frozen_only.h -m frozen_only -o Python/deepfreeze/frozen_only.c # END: deepfreeze modules @@ -2367,7 +2286,7 @@ clean-retain-profile: pycremoval find build -name '*.py[co]' -exec rm -f {} ';' || true -rm -f pybuilddir.txt -rm -f Lib/lib2to3/*Grammar*.pickle - -rm -f Programs/_testembed Programs/_freeze_module $(BOOTSTRAP) + -rm -f Programs/_testembed Programs/_freeze_module -rm -f Python/deepfreeze/*.[co] -find build -type f -a ! -name '*.gc??' -exec rm -f {} ';' -rm -f Include/pydtrace_probes.h diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index 61ccae61e4f965..59bc50af1d6753 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -25,30 +25,6 @@ # need to be updated. MODULES_DIR = os.path.join(ROOT_DIR, 'Python', 'frozen_modules') -if sys.platform != "win32": - TOOL = os.path.join(ROOT_DIR, 'Programs', '_freeze_module') - if not os.path.isfile(TOOL): - # When building out of the source tree, get the tool from directory - # of the Python executable - TOOL = os.path.dirname(sys.executable) - TOOL = os.path.join(TOOL, 'Programs', '_freeze_module') - TOOL = os.path.abspath(TOOL) - if not os.path.isfile(TOOL): - sys.exit("ERROR: missing _freeze_module") -else: - def find_tool(): - archs = ['amd64', 'win32'] - if platform.machine() == "ARM64": - archs.append('arm64') - for arch in archs: - for exe in ['_freeze_module.exe', '_freeze_module_d.exe']: - tool = os.path.join(ROOT_DIR, 'PCbuild', arch, exe) - if os.path.isfile(tool): - return tool - sys.exit("ERROR: missing _freeze_module.exe; you need to run PCbuild/build.bat") - TOOL = find_tool() - del find_tool - MANIFEST = os.path.join(MODULES_DIR, 'MANIFEST') FROZEN_FILE = os.path.join(ROOT_DIR, 'Python', 'frozen.c') MAKEFILE = os.path.join(ROOT_DIR, 'Makefile.pre.in') @@ -648,11 +624,11 @@ def regen_makefile(modules): deepfreezefiles.append(f"\t\t{ofile} \\") # Also add a deepfreeze rule. - deepfreezerules.append(f'{cfile}: $(srcdir)/{_pyfile} $(DEEPFREEZE_DEPS)') - deepfreezerules.append(f'\t@echo "Deepfreezing {cfile} from {_pyfile}"') - deepfreezerules.append(f"\t@./$(BOOTSTRAP) \\") - deepfreezerules.append(f"\t\t$(srcdir)/Tools/scripts/deepfreeze.py \\") - deepfreezerules.append(f"\t\t$(srcdir)/{_pyfile} -m {src.frozenid} -o {cfile}") + deepfreezerules.append(f'{cfile}: $(srcdir)/{header} $(DEEPFREEZE_DEPS)') + deepfreezerules.append( + f"\t$(PYTHON_FOR_REGEN) " + f"$(srcdir)/Tools/scripts/deepfreeze.py " + f"$(srcdir)/{header} -m {src.frozenid} -o {cfile}") deepfreezerules.append('') for src in _iter_sources(modules): @@ -774,32 +750,6 @@ def regen_pcbuild(modules): outfile.writelines(lines) -####################################### -# freezing modules - -def freeze_module(modname, pyfile=None, destdir=MODULES_DIR): - """Generate the frozen module .h file for the given module.""" - tmpsuffix = f'.{int(time.time())}' - for modname, pyfile, ispkg in resolve_modules(modname, pyfile): - frozenfile = resolve_frozen_file(modname, destdir) - _freeze_module(modname, pyfile, frozenfile, tmpsuffix) - - -def _freeze_module(frozenid, pyfile, frozenfile, tmpsuffix): - tmpfile = f'{frozenfile}.{int(time.time())}' - - argv = [TOOL, frozenid, pyfile, tmpfile] - print('#', ' '.join(os.path.relpath(a) for a in argv), flush=True) - try: - subprocess.run(argv, check=True) - except (FileNotFoundError, subprocess.CalledProcessError): - if not os.path.exists(TOOL): - sys.exit(f'ERROR: missing {TOOL}; you need to run "make regen-frozen"') - raise # re-raise - - update_file_with_tmpfile(frozenfile, tmpfile, create=True) - - ####################################### # the script @@ -811,11 +761,6 @@ def main(): regen_makefile(modules) regen_pcbuild(modules) - # Freeze the target modules. - tmpsuffix = f'.{int(time.time())}' - for src in _iter_sources(modules): - _freeze_module(src.frozenid, src.pyfile, src.frozenfile, tmpsuffix) - # Regen files dependent of frozen file details. regen_frozen(modules) regen_manifest(modules) From 4463a96da33e165b9f64afcaca6d04da5c8b5c40 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 22 Nov 2021 13:11:12 -0800 Subject: [PATCH 02/11] Get rid of the MANIFEST file --- Tools/scripts/freeze_modules.py | 43 --------------------------------- 1 file changed, 43 deletions(-) diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index 59bc50af1d6753..6614a2518f4f08 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -25,7 +25,6 @@ # need to be updated. MODULES_DIR = os.path.join(ROOT_DIR, 'Python', 'frozen_modules') -MANIFEST = os.path.join(MODULES_DIR, 'MANIFEST') FROZEN_FILE = os.path.join(ROOT_DIR, 'Python', 'frozen.c') MAKEFILE = os.path.join(ROOT_DIR, 'Makefile.pre.in') PCBUILD_PROJECT = os.path.join(ROOT_DIR, 'PCbuild', '_freeze_module.vcxproj') @@ -456,45 +455,6 @@ def replace_block(lines, start_marker, end_marker, replacements, file): return lines[:start_pos + 1] + replacements + lines[end_pos:] -def regen_manifest(modules): - header = 'module ispkg source frozen checksum'.split() - widths = [5] * len(header) - rows = [] - for mod in modules: - info = mod.summarize() - row = [] - for i, col in enumerate(header): - value = info[col] - if col == 'checksum': - value = value[:12] - elif col == 'ispkg': - value = 'YES' if value else 'no' - widths[i] = max(widths[i], len(value)) - row.append(value or '-') - rows.append(row) - - modlines = [ - '# The list of frozen modules with key information.', - '# Note that the "check_generated_files" CI job will identify', - '# when source files were changed but regen-frozen wasn\'t run.', - '# This file is auto-generated by Tools/scripts/freeze_modules.py.', - ' '.join(c.center(w) for c, w in zip(header, widths)).rstrip(), - ' '.join('-' * w for w in widths), - ] - for row in rows: - for i, w in enumerate(widths): - if header[i] == 'ispkg': - row[i] = row[i].center(w) - else: - row[i] = row[i].ljust(w) - modlines.append(' '.join(row).rstrip()) - - print(f'# Updating {os.path.relpath(MANIFEST)}') - with open(MANIFEST, 'w', encoding="utf-8") as outfile: - lines = (l + '\n' for l in modlines) - outfile.writelines(lines) - - def regen_frozen(modules): headerlines = [] parentdir = os.path.dirname(FROZEN_FILE) @@ -760,10 +720,7 @@ def main(): # Regen build-related files. regen_makefile(modules) regen_pcbuild(modules) - - # Regen files dependent of frozen file details. regen_frozen(modules) - regen_manifest(modules) if __name__ == '__main__': From b7469b8e01bb81780b5231b356d667da201026b3 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 22 Nov 2021 20:18:49 -0800 Subject: [PATCH 03/11] Make deepfreeze.py palatable to Python 3.7 and up --- Tools/scripts/deepfreeze.py | 87 ++++++------- Tools/scripts/umarshal.py | 237 ++++++++++++++++++------------------ 2 files changed, 164 insertions(+), 160 deletions(-) diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index b6d52b7454965f..ed791fd5b7d6dd 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -7,7 +7,7 @@ import re import time import types -import typing +from typing import Dict, FrozenSet, Tuple, TextIO import umarshal @@ -42,13 +42,14 @@ def get_localsplus(code: types.CodeType): def get_localsplus_counts(code: types.CodeType, - names: tuple[str, ...], - kinds: bytes) -> tuple[int, int, int, int]: + names: Tuple[str, ...], + kinds: bytes) -> Tuple[int, int, int, int]: nlocals = 0 nplaincellvars = 0 ncellvars = 0 nfreevars = 0 - for name, kind in zip(names, kinds, strict=True): + assert len(names) == len(kinds) + for name, kind in zip(names, kinds): if kind & CO_FAST_LOCAL: nlocals += 1 if kind & CO_FAST_CELL: @@ -71,7 +72,7 @@ def get_localsplus_counts(code: types.CodeType, PyUnicode_4BYTE_KIND = 4 -def analyze_character_width(s: str) -> tuple[int, bool]: +def analyze_character_width(s: str) -> Tuple[int, bool]: maxchar = ' ' for c in s: maxchar = max(maxchar, c) @@ -86,12 +87,17 @@ def analyze_character_width(s: str) -> tuple[int, bool]: return kind, ascii +def removesuffix(base, suffix): + if base.endswith(suffix): + return base[:len(base) - len(suffix)] + return base + class Printer: - def __init__(self, file: typing.TextIO): + def __init__(self, file: TextIO): self.level = 0 self.file = file - self.cache: dict[tuple[type, object], str] = {} + self.cache: Dict[Tuple[type, object], str] = {} self.hits, self.misses = 0, 0 self.patchups: list[str] = [] self.write('#include "Python.h"') @@ -231,7 +237,7 @@ def generate_code(self, name: str, code: types.CodeType) -> str: # otherwise MSVC doesn't like it. self.write(f".co_consts = {co_consts},") self.write(f".co_names = {co_names},") - self.write(f".co_firstinstr = (_Py_CODEUNIT *) {co_code.removesuffix('.ob_base.ob_base')}.ob_sval,") + self.write(f".co_firstinstr = (_Py_CODEUNIT *) {removesuffix(co_code, '.ob_base.ob_base')}.ob_sval,") self.write(f".co_exceptiontable = {co_exceptiontable},") self.field(code, "co_flags") self.write(".co_warmup = QUICKENING_INITIAL_WARMUP_VALUE,") @@ -259,7 +265,7 @@ def generate_code(self, name: str, code: types.CodeType) -> str: self.write(f".co_freevars = {co_freevars},") return f"& {name}.ob_base" - def generate_tuple(self, name: str, t: tuple[object, ...]) -> str: + def generate_tuple(self, name: str, t: Tuple[object, ...]) -> str: items = [self.generate(f"{name}_{i}", it) for i, it in enumerate(t)] self.write("static") with self.indent(): @@ -323,7 +329,7 @@ def generate_complex(self, name: str, z: complex) -> str: self.write(f".cval = {{ {z.real}, {z.imag} }},") return f"&{name}.ob_base" - def generate_frozenset(self, name: str, fs: frozenset[object]) -> str: + def generate_frozenset(self, name: str, fs: FrozenSet[object]) -> str: ret = self.generate_tuple(name, tuple(sorted(fs))) self.write("// TODO: The above tuple should be a frozenset") return ret @@ -336,34 +342,33 @@ def generate(self, name: str, obj: object) -> str: # print(f"Cache hit {key!r:.40}: {self.cache[key]!r:.40}") return self.cache[key] self.misses += 1 - match obj: - case types.CodeType() | umarshal.Code() as code: - val = self.generate_code(name, code) - case tuple(t): - val = self.generate_tuple(name, t) - case str(s): - val = self.generate_unicode(name, s) - case bytes(b): - val = self.generate_bytes(name, b) - case True: - return "Py_True" - case False: - return "Py_False" - case int(i): - val = self.generate_int(name, i) - case float(x): - val = self.generate_float(name, x) - case complex() as z: - val = self.generate_complex(name, z) - case frozenset(fs): - val = self.generate_frozenset(name, fs) - case builtins.Ellipsis: - return "Py_Ellipsis" - case None: - return "Py_None" - case _: - raise TypeError( - f"Cannot generate code for {type(obj).__name__} object") + if isinstance(obj, types.CodeType) or isinstance(obj, umarshal.Code): + val = self.generate_code(name, obj) + elif isinstance(obj, tuple): + val = self.generate_tuple(name, obj) + elif isinstance(obj, str): + val = self.generate_unicode(name, obj) + elif isinstance(obj, bytes): + val = self.generate_bytes(name, obj) + elif obj is True: + return "Py_True" + elif obj is False: + return "Py_False" + elif isinstance(obj, int): + val = self.generate_int(name, obj) + elif isinstance(obj, float): + val = self.generate_float(name, obj) + elif isinstance(obj, complex): + val = self.generate_complex(name, obj) + elif isinstance(obj, frozenset): + val = self.generate_frozenset(name, obj) + elif obj is builtins.Ellipsis: + return "Py_Ellipsis" + elif obj is None: + return "Py_None" + else: + raise TypeError( + f"Cannot generate code for {type(obj).__name__} object") # print(f"Cache store {key!r:.40}: {val!r:.40}") self.cache[key] = val return val @@ -393,12 +398,12 @@ def decode_frozen_data(source: str) -> types.CodeType: del lines[0] while lines and re.match(FROZEN_DATA_LINE, lines[-1]) is None: del lines[-1] - values: tuple[int, ...] = ast.literal_eval("".join(lines)) + values: Tuple[int, ...] = ast.literal_eval("".join(lines).strip()) data = bytes(values) return umarshal.loads(data) -def generate(source: str, filename: str, modname: str, file: typing.TextIO) -> None: +def generate(source: str, filename: str, modname: str, file: TextIO) -> None: if is_frozen_header(source): code = decode_frozen_data(source) else: @@ -439,7 +444,7 @@ def main() -> None: verbose = args.verbose with open(args.file, encoding="utf-8") as f: source = f.read() - modname = args.module or os.path.basename(args.file).removesuffix(".py") + modname = args.module or removesuffix(os.path.basename(args.file), ".py") output = args.output or modname + ".c" with open(output, "w", encoding="utf-8") as file: with report_time("generate"): diff --git a/Tools/scripts/umarshal.py b/Tools/scripts/umarshal.py index e0d18c89e75319..2eaaa7ce2d95bc 100644 --- a/Tools/scripts/umarshal.py +++ b/Tools/scripts/umarshal.py @@ -2,7 +2,7 @@ import ast -from typing import Any +from typing import Any, Tuple class Type: @@ -55,10 +55,10 @@ def __init__(self, **kwds: Any): def __repr__(self) -> str: return f"Code(**{self.__dict__})" - co_localsplusnames: tuple[str] - co_localspluskinds: tuple[int] + co_localsplusnames: Tuple[str] + co_localspluskinds: Tuple[int] - def get_localsplus_names(self, select_kind: int) -> tuple[str, ...]: + def get_localsplus_names(self, select_kind: int) -> Tuple[str, ...]: varnames: list[str] = [] for name, kind in zip(self.co_localsplusnames, self.co_localspluskinds): @@ -67,15 +67,15 @@ def get_localsplus_names(self, select_kind: int) -> tuple[str, ...]: return tuple(varnames) @property - def co_varnames(self) -> tuple[str, ...]: + def co_varnames(self) -> Tuple[str, ...]: return self.get_localsplus_names(CO_FAST_LOCAL) @property - def co_cellvars(self) -> tuple[str, ...]: + def co_cellvars(self) -> Tuple[str, ...]: return self.get_localsplus_names(CO_FAST_CELL) @property - def co_freevars(self) -> tuple[str, ...]: + def co_freevars(self) -> Tuple[str, ...]: return self.get_localsplus_names(CO_FAST_FREE) @property @@ -190,118 +190,117 @@ def R_REF(obj: Any) -> Any: obj = self.r_ref(obj, flag) return obj - match type: - case Type.NULL: - return NULL - case Type.NONE: - return None - case Type.ELLIPSIS: - return Ellipsis - case Type.FALSE: - return False - case Type.TRUE: - return True - case Type.INT: - return R_REF(self.r_long()) - case Type.INT64: - return R_REF(self.r_long64()) - case Type.LONG: - return R_REF(self.r_PyLong()) - case Type.FLOAT: - return R_REF(self.r_float_str()) - case Type.BINARY_FLOAT: - return R_REF(self.r_float_bin()) - case Type.COMPLEX: - return R_REF(complex(self.r_float_str(), - self.r_float_str())) - case Type.BINARY_COMPLEX: - return R_REF(complex(self.r_float_bin(), - self.r_float_bin())) - case Type.STRING: - n = self.r_long() - return R_REF(self.r_string(n)) - case Type.ASCII_INTERNED | Type.ASCII: - n = self.r_long() - return R_REF(self.r_string(n).decode("ascii")) - case Type.SHORT_ASCII_INTERNED | Type.SHORT_ASCII: - n = self.r_byte() - return R_REF(self.r_string(n).decode("ascii")) - case Type.INTERNED | Type.UNICODE: - n = self.r_long() - return R_REF(self.r_string(n).decode("utf8", "surrogatepass")) - case Type.SMALL_TUPLE: - n = self.r_byte() - idx = self.r_ref_reserve(flag) - retval: Any = tuple(self.r_object() for _ in range(n)) - self.r_ref_insert(retval, idx, flag) - return retval - case Type.TUPLE: - n = self.r_long() - idx = self.r_ref_reserve(flag) - retval = tuple(self.r_object() for _ in range(n)) - self.r_ref_insert(retval, idx, flag) - return retval - case Type.LIST: - n = self.r_long() - retval = R_REF([]) - for _ in range(n): - retval.append(self.r_object()) - return retval - case Type.DICT: - retval = R_REF({}) - while True: - key = self.r_object() - if key == NULL: - break - val = self.r_object() - retval[key] = val - return retval - case Type.SET: - n = self.r_long() - retval = R_REF(set()) - for _ in range(n): - v = self.r_object() - retval.add(v) - return retval - case Type.FROZENSET: - n = self.r_long() - s: set[Any] = set() - idx = self.r_ref_reserve(flag) - for _ in range(n): - v = self.r_object() - s.add(v) - retval = frozenset(s) - self.r_ref_insert(retval, idx, flag) - return retval - case Type.CODE: - retval = R_REF(Code()) - retval.co_argcount = self.r_long() - retval.co_posonlyargcount = self.r_long() - retval.co_kwonlyargcount = self.r_long() - retval.co_stacksize = self.r_long() - retval.co_flags = self.r_long() - retval.co_code = self.r_object() - retval.co_consts = self.r_object() - retval.co_names = self.r_object() - retval.co_localsplusnames = self.r_object() - retval.co_localspluskinds = self.r_object() - retval.co_filename = self.r_object() - retval.co_name = self.r_object() - retval.co_qualname = self.r_object() - retval.co_firstlineno = self.r_long() - retval.co_linetable = self.r_object() - retval.co_endlinetable = self.r_object() - retval.co_columntable = self.r_object() - retval.co_exceptiontable = self.r_object() - return retval - case Type.REF: - n = self.r_long() - retval = self.refs[n] - assert retval is not None - return retval - case _: - breakpoint() - raise AssertionError(f"Unknown type {type} {chr(type)!r}") + if type == Type.NULL: + return NULL + elif type == Type.NONE: + return None + elif type == Type.ELLIPSIS: + return Ellipsis + elif type == Type.FALSE: + return False + elif type == Type.TRUE: + return True + elif type == Type.INT: + return R_REF(self.r_long()) + elif type == Type.INT64: + return R_REF(self.r_long64()) + elif type == Type.LONG: + return R_REF(self.r_PyLong()) + elif type == Type.FLOAT: + return R_REF(self.r_float_str()) + elif type == Type.BINARY_FLOAT: + return R_REF(self.r_float_bin()) + elif type == Type.COMPLEX: + return R_REF(complex(self.r_float_str(), + self.r_float_str())) + elif type == Type.BINARY_COMPLEX: + return R_REF(complex(self.r_float_bin(), + self.r_float_bin())) + elif type == Type.STRING: + n = self.r_long() + return R_REF(self.r_string(n)) + elif type == Type.ASCII_INTERNED or type == Type.ASCII: + n = self.r_long() + return R_REF(self.r_string(n).decode("ascii")) + elif type == Type.SHORT_ASCII_INTERNED or type == Type.SHORT_ASCII: + n = self.r_byte() + return R_REF(self.r_string(n).decode("ascii")) + elif type == Type.INTERNED or type == Type.UNICODE: + n = self.r_long() + return R_REF(self.r_string(n).decode("utf8", "surrogatepass")) + elif type == Type.SMALL_TUPLE: + n = self.r_byte() + idx = self.r_ref_reserve(flag) + retval: Any = tuple(self.r_object() for _ in range(n)) + self.r_ref_insert(retval, idx, flag) + return retval + elif type == Type.TUPLE: + n = self.r_long() + idx = self.r_ref_reserve(flag) + retval = tuple(self.r_object() for _ in range(n)) + self.r_ref_insert(retval, idx, flag) + return retval + elif type == Type.LIST: + n = self.r_long() + retval = R_REF([]) + for _ in range(n): + retval.append(self.r_object()) + return retval + elif type == Type.DICT: + retval = R_REF({}) + while True: + key = self.r_object() + if key == NULL: + break + val = self.r_object() + retval[key] = val + return retval + elif type == Type.SET: + n = self.r_long() + retval = R_REF(set()) + for _ in range(n): + v = self.r_object() + retval.add(v) + return retval + elif type == Type.FROZENSET: + n = self.r_long() + s: set[Any] = set() + idx = self.r_ref_reserve(flag) + for _ in range(n): + v = self.r_object() + s.add(v) + retval = frozenset(s) + self.r_ref_insert(retval, idx, flag) + return retval + elif type == Type.CODE: + retval = R_REF(Code()) + retval.co_argcount = self.r_long() + retval.co_posonlyargcount = self.r_long() + retval.co_kwonlyargcount = self.r_long() + retval.co_stacksize = self.r_long() + retval.co_flags = self.r_long() + retval.co_code = self.r_object() + retval.co_consts = self.r_object() + retval.co_names = self.r_object() + retval.co_localsplusnames = self.r_object() + retval.co_localspluskinds = self.r_object() + retval.co_filename = self.r_object() + retval.co_name = self.r_object() + retval.co_qualname = self.r_object() + retval.co_firstlineno = self.r_long() + retval.co_linetable = self.r_object() + retval.co_endlinetable = self.r_object() + retval.co_columntable = self.r_object() + retval.co_exceptiontable = self.r_object() + return retval + elif type == Type.REF: + n = self.r_long() + retval = self.refs[n] + assert retval is not None + return retval + else: + breakpoint() + raise AssertionError(f"Unknown type {type} {chr(type)!r}") def loads(data: bytes) -> Any: From 0821457c0e3803191803aa1024ee8edf857b86a7 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 22 Nov 2021 20:22:12 -0800 Subject: [PATCH 04/11] Restore Python 3.8 and 3.9 as Windows host Python again --- PCbuild/find_python.bat | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/PCbuild/find_python.bat b/PCbuild/find_python.bat index a9f14c5277ffa0..a9bbe5a6279a61 100644 --- a/PCbuild/find_python.bat +++ b/PCbuild/find_python.bat @@ -31,13 +31,13 @@ @if "%_Py_EXTERNALS_DIR%"=="" (set _Py_EXTERNALS_DIR=%~dp0\..\externals) @rem If we have Python in externals, use that one -@if exist "%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" ("%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" -Ec "import sys; assert sys.version_info[:2] >= (3, 10)" >nul 2>nul) && (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") && (set _Py_Python_Source=found in externals directory) && goto :found || rmdir /Q /S "%_Py_EXTERNALS_DIR%\pythonx86" +@if exist "%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" ("%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" -Ec "import sys; assert sys.version_info[:2] >= (3, 8)" >nul 2>nul) && (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") && (set _Py_Python_Source=found in externals directory) && goto :found || rmdir /Q /S "%_Py_EXTERNALS_DIR%\pythonx86" @rem If HOST_PYTHON is recent enough, use that -@if NOT "%HOST_PYTHON%"=="" @%HOST_PYTHON% -Ec "import sys; assert sys.version_info[:2] >= (3, 10)" >nul 2>nul && (set PYTHON="%HOST_PYTHON%") && (set _Py_Python_Source=found as HOST_PYTHON) && goto :found +@if NOT "%HOST_PYTHON%"=="" @%HOST_PYTHON% -Ec "import sys; assert sys.version_info[:2] >= (3, 8)" >nul 2>nul && (set PYTHON="%HOST_PYTHON%") && (set _Py_Python_Source=found as HOST_PYTHON) && goto :found @rem If py.exe finds a recent enough version, use that one -@for %%p in (3.10) do @py -%%p -EV >nul 2>&1 && (set PYTHON=py -%%p) && (set _Py_Python_Source=found %%p with py.exe) && goto :found +@for %%p in (3.10 3.9 3.8) do @py -%%p -EV >nul 2>&1 && (set PYTHON=py -%%p) && (set _Py_Python_Source=found %%p with py.exe) && goto :found @if NOT exist "%_Py_EXTERNALS_DIR%" mkdir "%_Py_EXTERNALS_DIR%" @set _Py_NUGET=%NUGET% From 0e6dcf2ebd904cdcebc6797cc7d786d473c376db Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Tue, 23 Nov 2021 04:28:42 +0000 Subject: [PATCH 05/11] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Misc/NEWS.d/next/Build/2021-11-23-04-28-40.bpo-45873.9dldZ4.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Build/2021-11-23-04-28-40.bpo-45873.9dldZ4.rst diff --git a/Misc/NEWS.d/next/Build/2021-11-23-04-28-40.bpo-45873.9dldZ4.rst b/Misc/NEWS.d/next/Build/2021-11-23-04-28-40.bpo-45873.9dldZ4.rst new file mode 100644 index 00000000000000..901a3d6650b61e --- /dev/null +++ b/Misc/NEWS.d/next/Build/2021-11-23-04-28-40.bpo-45873.9dldZ4.rst @@ -0,0 +1 @@ +Get rid of the ``_bootstrap_python`` build step. The deepfreeze.py script is now run using ``$(PYTHON_FOR_REGEN)`` which can be Python 3.7 or newer (on Windows, 3.8 or newer). \ No newline at end of file From f92358084c7165389e404f47281631ce465573e6 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 22 Nov 2021 12:28:56 -0800 Subject: [PATCH 06/11] bpo-45873: Get rid of bootstrap_python Instead we use $(PYTHON_FOR_REGEN) .../deepfreeze.py with the frozen .h file as input, as we did for Windows in bpo-45850. We also get rid of the code that generates the .h files when make regen-frozen is run (i.e., .../make_frozen.py). --- Makefile.pre.in | 207 ++++++++++---------------------- Tools/scripts/freeze_modules.py | 65 +--------- 2 files changed, 68 insertions(+), 204 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index d52f73620795c1..2e3aab762a376c 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -275,7 +275,6 @@ LIBOBJS= @LIBOBJS@ PYTHON= python$(EXE) BUILDPYTHON= python$(BUILDEXE) -BOOTSTRAP= _bootstrap_python PYTHON_FOR_REGEN?=@PYTHON_FOR_REGEN@ UPDATE_FILE=$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/update_file.py @@ -710,9 +709,9 @@ platform: $(BUILDPYTHON) pybuilddir.txt # problems by creating a dummy pybuilddir.txt just to allow interpreter # initialization to succeed. It will be overwritten by generate-posix-vars # or removed in case of failure. -pybuilddir.txt: $(BOOTSTRAP) +pybuilddir.txt: $(BUILDPYTHON) @echo "none" > ./pybuilddir.txt - ./$(BOOTSTRAP) -S -m sysconfig --generate-posix-vars ;\ + $(RUNSHARED) $(PYTHON_FOR_BUILD) -S -m sysconfig --generate-posix-vars ;\ if test $$? -ne 0 ; then \ echo "generate-posix-vars failed" ; \ rm -f ./pybuilddir.txt ; \ @@ -934,155 +933,75 @@ regen-test-frozenmain: $(BUILDPYTHON) Programs/_testembed: Programs/_testembed.o $(LIBRARY_DEPS) $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/_testembed.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) -############################################################################ -# "Bootstrap Python" used to run deepfreeze.py - -BOOTSTRAP_HEADERS = \ - Python/frozen_modules/importlib._bootstrap.h \ - Python/frozen_modules/importlib._bootstrap_external.h \ - Python/frozen_modules/zipimport.h - -Python/bootstrap_frozen.o: Python/bootstrap_frozen.c Include/cpython/import.h $(BOOTSTRAP_HEADERS) - -$(BOOTSTRAP): $(LIBRARY_OBJS_OMIT_FROZEN) \ - Python/bootstrap_frozen.o Programs/python.o - $(LINKCC) $(PY_CORE_LDFLAGS) -o $@ $(LIBRARY_OBJS_OMIT_FROZEN) \ - Python/bootstrap_frozen.o \ - Programs/python.o \ - $(LIBS) $(MODLIBS) $(SYSLIBS) - ############################################################################ # Deepfreeze targets .PHONY: regen-deepfreeze regen-deepfreeze: $(DEEPFREEZE_OBJS) -DEEPFREEZE_DEPS = \ - $(BOOTSTRAP) \ - pybuilddir.txt \ - $(srcdir)/Tools/scripts/deepfreeze.py +DEEPFREEZE_DEPS=$(srcdir)/Tools/scripts/deepfreeze.py # BEGIN: deepfreeze modules -Python/deepfreeze/importlib._bootstrap.c: $(srcdir)/Lib/importlib/_bootstrap.py $(DEEPFREEZE_DEPS) - @echo "Deepfreezing Python/deepfreeze/importlib._bootstrap.c from Lib/importlib/_bootstrap.py" - @./$(BOOTSTRAP) \ - $(srcdir)/Tools/scripts/deepfreeze.py \ - $(srcdir)/Lib/importlib/_bootstrap.py -m importlib._bootstrap -o Python/deepfreeze/importlib._bootstrap.c - -Python/deepfreeze/importlib._bootstrap_external.c: $(srcdir)/Lib/importlib/_bootstrap_external.py $(DEEPFREEZE_DEPS) - @echo "Deepfreezing Python/deepfreeze/importlib._bootstrap_external.c from Lib/importlib/_bootstrap_external.py" - @./$(BOOTSTRAP) \ - $(srcdir)/Tools/scripts/deepfreeze.py \ - $(srcdir)/Lib/importlib/_bootstrap_external.py -m importlib._bootstrap_external -o Python/deepfreeze/importlib._bootstrap_external.c - -Python/deepfreeze/zipimport.c: $(srcdir)/Lib/zipimport.py $(DEEPFREEZE_DEPS) - @echo "Deepfreezing Python/deepfreeze/zipimport.c from Lib/zipimport.py" - @./$(BOOTSTRAP) \ - $(srcdir)/Tools/scripts/deepfreeze.py \ - $(srcdir)/Lib/zipimport.py -m zipimport -o Python/deepfreeze/zipimport.c - -Python/deepfreeze/abc.c: $(srcdir)/Lib/abc.py $(DEEPFREEZE_DEPS) - @echo "Deepfreezing Python/deepfreeze/abc.c from Lib/abc.py" - @./$(BOOTSTRAP) \ - $(srcdir)/Tools/scripts/deepfreeze.py \ - $(srcdir)/Lib/abc.py -m abc -o Python/deepfreeze/abc.c - -Python/deepfreeze/codecs.c: $(srcdir)/Lib/codecs.py $(DEEPFREEZE_DEPS) - @echo "Deepfreezing Python/deepfreeze/codecs.c from Lib/codecs.py" - @./$(BOOTSTRAP) \ - $(srcdir)/Tools/scripts/deepfreeze.py \ - $(srcdir)/Lib/codecs.py -m codecs -o Python/deepfreeze/codecs.c - -Python/deepfreeze/io.c: $(srcdir)/Lib/io.py $(DEEPFREEZE_DEPS) - @echo "Deepfreezing Python/deepfreeze/io.c from Lib/io.py" - @./$(BOOTSTRAP) \ - $(srcdir)/Tools/scripts/deepfreeze.py \ - $(srcdir)/Lib/io.py -m io -o Python/deepfreeze/io.c - -Python/deepfreeze/_collections_abc.c: $(srcdir)/Lib/_collections_abc.py $(DEEPFREEZE_DEPS) - @echo "Deepfreezing Python/deepfreeze/_collections_abc.c from Lib/_collections_abc.py" - @./$(BOOTSTRAP) \ - $(srcdir)/Tools/scripts/deepfreeze.py \ - $(srcdir)/Lib/_collections_abc.py -m _collections_abc -o Python/deepfreeze/_collections_abc.c - -Python/deepfreeze/_sitebuiltins.c: $(srcdir)/Lib/_sitebuiltins.py $(DEEPFREEZE_DEPS) - @echo "Deepfreezing Python/deepfreeze/_sitebuiltins.c from Lib/_sitebuiltins.py" - @./$(BOOTSTRAP) \ - $(srcdir)/Tools/scripts/deepfreeze.py \ - $(srcdir)/Lib/_sitebuiltins.py -m _sitebuiltins -o Python/deepfreeze/_sitebuiltins.c - -Python/deepfreeze/genericpath.c: $(srcdir)/Lib/genericpath.py $(DEEPFREEZE_DEPS) - @echo "Deepfreezing Python/deepfreeze/genericpath.c from Lib/genericpath.py" - @./$(BOOTSTRAP) \ - $(srcdir)/Tools/scripts/deepfreeze.py \ - $(srcdir)/Lib/genericpath.py -m genericpath -o Python/deepfreeze/genericpath.c - -Python/deepfreeze/ntpath.c: $(srcdir)/Lib/ntpath.py $(DEEPFREEZE_DEPS) - @echo "Deepfreezing Python/deepfreeze/ntpath.c from Lib/ntpath.py" - @./$(BOOTSTRAP) \ - $(srcdir)/Tools/scripts/deepfreeze.py \ - $(srcdir)/Lib/ntpath.py -m ntpath -o Python/deepfreeze/ntpath.c - -Python/deepfreeze/posixpath.c: $(srcdir)/Lib/posixpath.py $(DEEPFREEZE_DEPS) - @echo "Deepfreezing Python/deepfreeze/posixpath.c from Lib/posixpath.py" - @./$(BOOTSTRAP) \ - $(srcdir)/Tools/scripts/deepfreeze.py \ - $(srcdir)/Lib/posixpath.py -m posixpath -o Python/deepfreeze/posixpath.c - -Python/deepfreeze/os.c: $(srcdir)/Lib/os.py $(DEEPFREEZE_DEPS) - @echo "Deepfreezing Python/deepfreeze/os.c from Lib/os.py" - @./$(BOOTSTRAP) \ - $(srcdir)/Tools/scripts/deepfreeze.py \ - $(srcdir)/Lib/os.py -m os -o Python/deepfreeze/os.c - -Python/deepfreeze/site.c: $(srcdir)/Lib/site.py $(DEEPFREEZE_DEPS) - @echo "Deepfreezing Python/deepfreeze/site.c from Lib/site.py" - @./$(BOOTSTRAP) \ - $(srcdir)/Tools/scripts/deepfreeze.py \ - $(srcdir)/Lib/site.py -m site -o Python/deepfreeze/site.c - -Python/deepfreeze/stat.c: $(srcdir)/Lib/stat.py $(DEEPFREEZE_DEPS) - @echo "Deepfreezing Python/deepfreeze/stat.c from Lib/stat.py" - @./$(BOOTSTRAP) \ - $(srcdir)/Tools/scripts/deepfreeze.py \ - $(srcdir)/Lib/stat.py -m stat -o Python/deepfreeze/stat.c - -Python/deepfreeze/__hello__.c: $(srcdir)/Lib/__hello__.py $(DEEPFREEZE_DEPS) - @echo "Deepfreezing Python/deepfreeze/__hello__.c from Lib/__hello__.py" - @./$(BOOTSTRAP) \ - $(srcdir)/Tools/scripts/deepfreeze.py \ - $(srcdir)/Lib/__hello__.py -m __hello__ -o Python/deepfreeze/__hello__.c - -Python/deepfreeze/__phello__.c: $(srcdir)/Lib/__phello__/__init__.py $(DEEPFREEZE_DEPS) - @echo "Deepfreezing Python/deepfreeze/__phello__.c from Lib/__phello__/__init__.py" - @./$(BOOTSTRAP) \ - $(srcdir)/Tools/scripts/deepfreeze.py \ - $(srcdir)/Lib/__phello__/__init__.py -m __phello__ -o Python/deepfreeze/__phello__.c - -Python/deepfreeze/__phello__.ham.c: $(srcdir)/Lib/__phello__/ham/__init__.py $(DEEPFREEZE_DEPS) - @echo "Deepfreezing Python/deepfreeze/__phello__.ham.c from Lib/__phello__/ham/__init__.py" - @./$(BOOTSTRAP) \ - $(srcdir)/Tools/scripts/deepfreeze.py \ - $(srcdir)/Lib/__phello__/ham/__init__.py -m __phello__.ham -o Python/deepfreeze/__phello__.ham.c - -Python/deepfreeze/__phello__.ham.eggs.c: $(srcdir)/Lib/__phello__/ham/eggs.py $(DEEPFREEZE_DEPS) - @echo "Deepfreezing Python/deepfreeze/__phello__.ham.eggs.c from Lib/__phello__/ham/eggs.py" - @./$(BOOTSTRAP) \ - $(srcdir)/Tools/scripts/deepfreeze.py \ - $(srcdir)/Lib/__phello__/ham/eggs.py -m __phello__.ham.eggs -o Python/deepfreeze/__phello__.ham.eggs.c - -Python/deepfreeze/__phello__.spam.c: $(srcdir)/Lib/__phello__/spam.py $(DEEPFREEZE_DEPS) - @echo "Deepfreezing Python/deepfreeze/__phello__.spam.c from Lib/__phello__/spam.py" - @./$(BOOTSTRAP) \ - $(srcdir)/Tools/scripts/deepfreeze.py \ - $(srcdir)/Lib/__phello__/spam.py -m __phello__.spam -o Python/deepfreeze/__phello__.spam.c - -Python/deepfreeze/frozen_only.c: $(srcdir)/Tools/freeze/flag.py $(DEEPFREEZE_DEPS) - @echo "Deepfreezing Python/deepfreeze/frozen_only.c from Tools/freeze/flag.py" - @./$(BOOTSTRAP) \ - $(srcdir)/Tools/scripts/deepfreeze.py \ - $(srcdir)/Tools/freeze/flag.py -m frozen_only -o Python/deepfreeze/frozen_only.c +Python/deepfreeze/importlib._bootstrap.c: $(srcdir)/Python/frozen_modules/importlib._bootstrap.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/importlib._bootstrap.h -m importlib._bootstrap -o Python/deepfreeze/importlib._bootstrap.c + +Python/deepfreeze/importlib._bootstrap_external.c: $(srcdir)/Python/frozen_modules/importlib._bootstrap_external.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/importlib._bootstrap_external.h -m importlib._bootstrap_external -o Python/deepfreeze/importlib._bootstrap_external.c + +Python/deepfreeze/zipimport.c: $(srcdir)/Python/frozen_modules/zipimport.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/zipimport.h -m zipimport -o Python/deepfreeze/zipimport.c + +Python/deepfreeze/abc.c: $(srcdir)/Python/frozen_modules/abc.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/abc.h -m abc -o Python/deepfreeze/abc.c + +Python/deepfreeze/codecs.c: $(srcdir)/Python/frozen_modules/codecs.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/codecs.h -m codecs -o Python/deepfreeze/codecs.c + +Python/deepfreeze/io.c: $(srcdir)/Python/frozen_modules/io.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/io.h -m io -o Python/deepfreeze/io.c + +Python/deepfreeze/_collections_abc.c: $(srcdir)/Python/frozen_modules/_collections_abc.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/_collections_abc.h -m _collections_abc -o Python/deepfreeze/_collections_abc.c + +Python/deepfreeze/_sitebuiltins.c: $(srcdir)/Python/frozen_modules/_sitebuiltins.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/_sitebuiltins.h -m _sitebuiltins -o Python/deepfreeze/_sitebuiltins.c + +Python/deepfreeze/genericpath.c: $(srcdir)/Python/frozen_modules/genericpath.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/genericpath.h -m genericpath -o Python/deepfreeze/genericpath.c + +Python/deepfreeze/ntpath.c: $(srcdir)/Python/frozen_modules/ntpath.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/ntpath.h -m ntpath -o Python/deepfreeze/ntpath.c + +Python/deepfreeze/posixpath.c: $(srcdir)/Python/frozen_modules/posixpath.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/posixpath.h -m posixpath -o Python/deepfreeze/posixpath.c + +Python/deepfreeze/os.c: $(srcdir)/Python/frozen_modules/os.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/os.h -m os -o Python/deepfreeze/os.c + +Python/deepfreeze/site.c: $(srcdir)/Python/frozen_modules/site.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/site.h -m site -o Python/deepfreeze/site.c + +Python/deepfreeze/stat.c: $(srcdir)/Python/frozen_modules/stat.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/stat.h -m stat -o Python/deepfreeze/stat.c + +Python/deepfreeze/__hello__.c: $(srcdir)/Python/frozen_modules/__hello__.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/__hello__.h -m __hello__ -o Python/deepfreeze/__hello__.c + +Python/deepfreeze/__phello__.c: $(srcdir)/Python/frozen_modules/__phello__.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/__phello__.h -m __phello__ -o Python/deepfreeze/__phello__.c + +Python/deepfreeze/__phello__.ham.c: $(srcdir)/Python/frozen_modules/__phello__.ham.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/__phello__.ham.h -m __phello__.ham -o Python/deepfreeze/__phello__.ham.c + +Python/deepfreeze/__phello__.ham.eggs.c: $(srcdir)/Python/frozen_modules/__phello__.ham.eggs.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/__phello__.ham.eggs.h -m __phello__.ham.eggs -o Python/deepfreeze/__phello__.ham.eggs.c + +Python/deepfreeze/__phello__.spam.c: $(srcdir)/Python/frozen_modules/__phello__.spam.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/__phello__.spam.h -m __phello__.spam -o Python/deepfreeze/__phello__.spam.c + +Python/deepfreeze/frozen_only.c: $(srcdir)/Python/frozen_modules/frozen_only.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/frozen_only.h -m frozen_only -o Python/deepfreeze/frozen_only.c # END: deepfreeze modules @@ -2367,7 +2286,7 @@ clean-retain-profile: pycremoval find build -name '*.py[co]' -exec rm -f {} ';' || true -rm -f pybuilddir.txt -rm -f Lib/lib2to3/*Grammar*.pickle - -rm -f Programs/_testembed Programs/_freeze_module $(BOOTSTRAP) + -rm -f Programs/_testembed Programs/_freeze_module -rm -f Python/deepfreeze/*.[co] -find build -type f -a ! -name '*.gc??' -exec rm -f {} ';' -rm -f Include/pydtrace_probes.h diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index 61ccae61e4f965..59bc50af1d6753 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -25,30 +25,6 @@ # need to be updated. MODULES_DIR = os.path.join(ROOT_DIR, 'Python', 'frozen_modules') -if sys.platform != "win32": - TOOL = os.path.join(ROOT_DIR, 'Programs', '_freeze_module') - if not os.path.isfile(TOOL): - # When building out of the source tree, get the tool from directory - # of the Python executable - TOOL = os.path.dirname(sys.executable) - TOOL = os.path.join(TOOL, 'Programs', '_freeze_module') - TOOL = os.path.abspath(TOOL) - if not os.path.isfile(TOOL): - sys.exit("ERROR: missing _freeze_module") -else: - def find_tool(): - archs = ['amd64', 'win32'] - if platform.machine() == "ARM64": - archs.append('arm64') - for arch in archs: - for exe in ['_freeze_module.exe', '_freeze_module_d.exe']: - tool = os.path.join(ROOT_DIR, 'PCbuild', arch, exe) - if os.path.isfile(tool): - return tool - sys.exit("ERROR: missing _freeze_module.exe; you need to run PCbuild/build.bat") - TOOL = find_tool() - del find_tool - MANIFEST = os.path.join(MODULES_DIR, 'MANIFEST') FROZEN_FILE = os.path.join(ROOT_DIR, 'Python', 'frozen.c') MAKEFILE = os.path.join(ROOT_DIR, 'Makefile.pre.in') @@ -648,11 +624,11 @@ def regen_makefile(modules): deepfreezefiles.append(f"\t\t{ofile} \\") # Also add a deepfreeze rule. - deepfreezerules.append(f'{cfile}: $(srcdir)/{_pyfile} $(DEEPFREEZE_DEPS)') - deepfreezerules.append(f'\t@echo "Deepfreezing {cfile} from {_pyfile}"') - deepfreezerules.append(f"\t@./$(BOOTSTRAP) \\") - deepfreezerules.append(f"\t\t$(srcdir)/Tools/scripts/deepfreeze.py \\") - deepfreezerules.append(f"\t\t$(srcdir)/{_pyfile} -m {src.frozenid} -o {cfile}") + deepfreezerules.append(f'{cfile}: $(srcdir)/{header} $(DEEPFREEZE_DEPS)') + deepfreezerules.append( + f"\t$(PYTHON_FOR_REGEN) " + f"$(srcdir)/Tools/scripts/deepfreeze.py " + f"$(srcdir)/{header} -m {src.frozenid} -o {cfile}") deepfreezerules.append('') for src in _iter_sources(modules): @@ -774,32 +750,6 @@ def regen_pcbuild(modules): outfile.writelines(lines) -####################################### -# freezing modules - -def freeze_module(modname, pyfile=None, destdir=MODULES_DIR): - """Generate the frozen module .h file for the given module.""" - tmpsuffix = f'.{int(time.time())}' - for modname, pyfile, ispkg in resolve_modules(modname, pyfile): - frozenfile = resolve_frozen_file(modname, destdir) - _freeze_module(modname, pyfile, frozenfile, tmpsuffix) - - -def _freeze_module(frozenid, pyfile, frozenfile, tmpsuffix): - tmpfile = f'{frozenfile}.{int(time.time())}' - - argv = [TOOL, frozenid, pyfile, tmpfile] - print('#', ' '.join(os.path.relpath(a) for a in argv), flush=True) - try: - subprocess.run(argv, check=True) - except (FileNotFoundError, subprocess.CalledProcessError): - if not os.path.exists(TOOL): - sys.exit(f'ERROR: missing {TOOL}; you need to run "make regen-frozen"') - raise # re-raise - - update_file_with_tmpfile(frozenfile, tmpfile, create=True) - - ####################################### # the script @@ -811,11 +761,6 @@ def main(): regen_makefile(modules) regen_pcbuild(modules) - # Freeze the target modules. - tmpsuffix = f'.{int(time.time())}' - for src in _iter_sources(modules): - _freeze_module(src.frozenid, src.pyfile, src.frozenfile, tmpsuffix) - # Regen files dependent of frozen file details. regen_frozen(modules) regen_manifest(modules) From 1912356bbcc5ba3bdacfaab54329b1de293942b9 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 22 Nov 2021 13:11:12 -0800 Subject: [PATCH 07/11] Get rid of the MANIFEST file --- Tools/scripts/freeze_modules.py | 43 --------------------------------- 1 file changed, 43 deletions(-) diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index 59bc50af1d6753..6614a2518f4f08 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -25,7 +25,6 @@ # need to be updated. MODULES_DIR = os.path.join(ROOT_DIR, 'Python', 'frozen_modules') -MANIFEST = os.path.join(MODULES_DIR, 'MANIFEST') FROZEN_FILE = os.path.join(ROOT_DIR, 'Python', 'frozen.c') MAKEFILE = os.path.join(ROOT_DIR, 'Makefile.pre.in') PCBUILD_PROJECT = os.path.join(ROOT_DIR, 'PCbuild', '_freeze_module.vcxproj') @@ -456,45 +455,6 @@ def replace_block(lines, start_marker, end_marker, replacements, file): return lines[:start_pos + 1] + replacements + lines[end_pos:] -def regen_manifest(modules): - header = 'module ispkg source frozen checksum'.split() - widths = [5] * len(header) - rows = [] - for mod in modules: - info = mod.summarize() - row = [] - for i, col in enumerate(header): - value = info[col] - if col == 'checksum': - value = value[:12] - elif col == 'ispkg': - value = 'YES' if value else 'no' - widths[i] = max(widths[i], len(value)) - row.append(value or '-') - rows.append(row) - - modlines = [ - '# The list of frozen modules with key information.', - '# Note that the "check_generated_files" CI job will identify', - '# when source files were changed but regen-frozen wasn\'t run.', - '# This file is auto-generated by Tools/scripts/freeze_modules.py.', - ' '.join(c.center(w) for c, w in zip(header, widths)).rstrip(), - ' '.join('-' * w for w in widths), - ] - for row in rows: - for i, w in enumerate(widths): - if header[i] == 'ispkg': - row[i] = row[i].center(w) - else: - row[i] = row[i].ljust(w) - modlines.append(' '.join(row).rstrip()) - - print(f'# Updating {os.path.relpath(MANIFEST)}') - with open(MANIFEST, 'w', encoding="utf-8") as outfile: - lines = (l + '\n' for l in modlines) - outfile.writelines(lines) - - def regen_frozen(modules): headerlines = [] parentdir = os.path.dirname(FROZEN_FILE) @@ -760,10 +720,7 @@ def main(): # Regen build-related files. regen_makefile(modules) regen_pcbuild(modules) - - # Regen files dependent of frozen file details. regen_frozen(modules) - regen_manifest(modules) if __name__ == '__main__': From 2b1f844eeef6b27354d0038a23c55fc7ef031260 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 22 Nov 2021 20:18:49 -0800 Subject: [PATCH 08/11] Make deepfreeze.py palatable to Python 3.7 and up --- Tools/scripts/deepfreeze.py | 87 ++++++------- Tools/scripts/umarshal.py | 237 ++++++++++++++++++------------------ 2 files changed, 164 insertions(+), 160 deletions(-) diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index b6d52b7454965f..ed791fd5b7d6dd 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -7,7 +7,7 @@ import re import time import types -import typing +from typing import Dict, FrozenSet, Tuple, TextIO import umarshal @@ -42,13 +42,14 @@ def get_localsplus(code: types.CodeType): def get_localsplus_counts(code: types.CodeType, - names: tuple[str, ...], - kinds: bytes) -> tuple[int, int, int, int]: + names: Tuple[str, ...], + kinds: bytes) -> Tuple[int, int, int, int]: nlocals = 0 nplaincellvars = 0 ncellvars = 0 nfreevars = 0 - for name, kind in zip(names, kinds, strict=True): + assert len(names) == len(kinds) + for name, kind in zip(names, kinds): if kind & CO_FAST_LOCAL: nlocals += 1 if kind & CO_FAST_CELL: @@ -71,7 +72,7 @@ def get_localsplus_counts(code: types.CodeType, PyUnicode_4BYTE_KIND = 4 -def analyze_character_width(s: str) -> tuple[int, bool]: +def analyze_character_width(s: str) -> Tuple[int, bool]: maxchar = ' ' for c in s: maxchar = max(maxchar, c) @@ -86,12 +87,17 @@ def analyze_character_width(s: str) -> tuple[int, bool]: return kind, ascii +def removesuffix(base, suffix): + if base.endswith(suffix): + return base[:len(base) - len(suffix)] + return base + class Printer: - def __init__(self, file: typing.TextIO): + def __init__(self, file: TextIO): self.level = 0 self.file = file - self.cache: dict[tuple[type, object], str] = {} + self.cache: Dict[Tuple[type, object], str] = {} self.hits, self.misses = 0, 0 self.patchups: list[str] = [] self.write('#include "Python.h"') @@ -231,7 +237,7 @@ def generate_code(self, name: str, code: types.CodeType) -> str: # otherwise MSVC doesn't like it. self.write(f".co_consts = {co_consts},") self.write(f".co_names = {co_names},") - self.write(f".co_firstinstr = (_Py_CODEUNIT *) {co_code.removesuffix('.ob_base.ob_base')}.ob_sval,") + self.write(f".co_firstinstr = (_Py_CODEUNIT *) {removesuffix(co_code, '.ob_base.ob_base')}.ob_sval,") self.write(f".co_exceptiontable = {co_exceptiontable},") self.field(code, "co_flags") self.write(".co_warmup = QUICKENING_INITIAL_WARMUP_VALUE,") @@ -259,7 +265,7 @@ def generate_code(self, name: str, code: types.CodeType) -> str: self.write(f".co_freevars = {co_freevars},") return f"& {name}.ob_base" - def generate_tuple(self, name: str, t: tuple[object, ...]) -> str: + def generate_tuple(self, name: str, t: Tuple[object, ...]) -> str: items = [self.generate(f"{name}_{i}", it) for i, it in enumerate(t)] self.write("static") with self.indent(): @@ -323,7 +329,7 @@ def generate_complex(self, name: str, z: complex) -> str: self.write(f".cval = {{ {z.real}, {z.imag} }},") return f"&{name}.ob_base" - def generate_frozenset(self, name: str, fs: frozenset[object]) -> str: + def generate_frozenset(self, name: str, fs: FrozenSet[object]) -> str: ret = self.generate_tuple(name, tuple(sorted(fs))) self.write("// TODO: The above tuple should be a frozenset") return ret @@ -336,34 +342,33 @@ def generate(self, name: str, obj: object) -> str: # print(f"Cache hit {key!r:.40}: {self.cache[key]!r:.40}") return self.cache[key] self.misses += 1 - match obj: - case types.CodeType() | umarshal.Code() as code: - val = self.generate_code(name, code) - case tuple(t): - val = self.generate_tuple(name, t) - case str(s): - val = self.generate_unicode(name, s) - case bytes(b): - val = self.generate_bytes(name, b) - case True: - return "Py_True" - case False: - return "Py_False" - case int(i): - val = self.generate_int(name, i) - case float(x): - val = self.generate_float(name, x) - case complex() as z: - val = self.generate_complex(name, z) - case frozenset(fs): - val = self.generate_frozenset(name, fs) - case builtins.Ellipsis: - return "Py_Ellipsis" - case None: - return "Py_None" - case _: - raise TypeError( - f"Cannot generate code for {type(obj).__name__} object") + if isinstance(obj, types.CodeType) or isinstance(obj, umarshal.Code): + val = self.generate_code(name, obj) + elif isinstance(obj, tuple): + val = self.generate_tuple(name, obj) + elif isinstance(obj, str): + val = self.generate_unicode(name, obj) + elif isinstance(obj, bytes): + val = self.generate_bytes(name, obj) + elif obj is True: + return "Py_True" + elif obj is False: + return "Py_False" + elif isinstance(obj, int): + val = self.generate_int(name, obj) + elif isinstance(obj, float): + val = self.generate_float(name, obj) + elif isinstance(obj, complex): + val = self.generate_complex(name, obj) + elif isinstance(obj, frozenset): + val = self.generate_frozenset(name, obj) + elif obj is builtins.Ellipsis: + return "Py_Ellipsis" + elif obj is None: + return "Py_None" + else: + raise TypeError( + f"Cannot generate code for {type(obj).__name__} object") # print(f"Cache store {key!r:.40}: {val!r:.40}") self.cache[key] = val return val @@ -393,12 +398,12 @@ def decode_frozen_data(source: str) -> types.CodeType: del lines[0] while lines and re.match(FROZEN_DATA_LINE, lines[-1]) is None: del lines[-1] - values: tuple[int, ...] = ast.literal_eval("".join(lines)) + values: Tuple[int, ...] = ast.literal_eval("".join(lines).strip()) data = bytes(values) return umarshal.loads(data) -def generate(source: str, filename: str, modname: str, file: typing.TextIO) -> None: +def generate(source: str, filename: str, modname: str, file: TextIO) -> None: if is_frozen_header(source): code = decode_frozen_data(source) else: @@ -439,7 +444,7 @@ def main() -> None: verbose = args.verbose with open(args.file, encoding="utf-8") as f: source = f.read() - modname = args.module or os.path.basename(args.file).removesuffix(".py") + modname = args.module or removesuffix(os.path.basename(args.file), ".py") output = args.output or modname + ".c" with open(output, "w", encoding="utf-8") as file: with report_time("generate"): diff --git a/Tools/scripts/umarshal.py b/Tools/scripts/umarshal.py index e0d18c89e75319..2eaaa7ce2d95bc 100644 --- a/Tools/scripts/umarshal.py +++ b/Tools/scripts/umarshal.py @@ -2,7 +2,7 @@ import ast -from typing import Any +from typing import Any, Tuple class Type: @@ -55,10 +55,10 @@ def __init__(self, **kwds: Any): def __repr__(self) -> str: return f"Code(**{self.__dict__})" - co_localsplusnames: tuple[str] - co_localspluskinds: tuple[int] + co_localsplusnames: Tuple[str] + co_localspluskinds: Tuple[int] - def get_localsplus_names(self, select_kind: int) -> tuple[str, ...]: + def get_localsplus_names(self, select_kind: int) -> Tuple[str, ...]: varnames: list[str] = [] for name, kind in zip(self.co_localsplusnames, self.co_localspluskinds): @@ -67,15 +67,15 @@ def get_localsplus_names(self, select_kind: int) -> tuple[str, ...]: return tuple(varnames) @property - def co_varnames(self) -> tuple[str, ...]: + def co_varnames(self) -> Tuple[str, ...]: return self.get_localsplus_names(CO_FAST_LOCAL) @property - def co_cellvars(self) -> tuple[str, ...]: + def co_cellvars(self) -> Tuple[str, ...]: return self.get_localsplus_names(CO_FAST_CELL) @property - def co_freevars(self) -> tuple[str, ...]: + def co_freevars(self) -> Tuple[str, ...]: return self.get_localsplus_names(CO_FAST_FREE) @property @@ -190,118 +190,117 @@ def R_REF(obj: Any) -> Any: obj = self.r_ref(obj, flag) return obj - match type: - case Type.NULL: - return NULL - case Type.NONE: - return None - case Type.ELLIPSIS: - return Ellipsis - case Type.FALSE: - return False - case Type.TRUE: - return True - case Type.INT: - return R_REF(self.r_long()) - case Type.INT64: - return R_REF(self.r_long64()) - case Type.LONG: - return R_REF(self.r_PyLong()) - case Type.FLOAT: - return R_REF(self.r_float_str()) - case Type.BINARY_FLOAT: - return R_REF(self.r_float_bin()) - case Type.COMPLEX: - return R_REF(complex(self.r_float_str(), - self.r_float_str())) - case Type.BINARY_COMPLEX: - return R_REF(complex(self.r_float_bin(), - self.r_float_bin())) - case Type.STRING: - n = self.r_long() - return R_REF(self.r_string(n)) - case Type.ASCII_INTERNED | Type.ASCII: - n = self.r_long() - return R_REF(self.r_string(n).decode("ascii")) - case Type.SHORT_ASCII_INTERNED | Type.SHORT_ASCII: - n = self.r_byte() - return R_REF(self.r_string(n).decode("ascii")) - case Type.INTERNED | Type.UNICODE: - n = self.r_long() - return R_REF(self.r_string(n).decode("utf8", "surrogatepass")) - case Type.SMALL_TUPLE: - n = self.r_byte() - idx = self.r_ref_reserve(flag) - retval: Any = tuple(self.r_object() for _ in range(n)) - self.r_ref_insert(retval, idx, flag) - return retval - case Type.TUPLE: - n = self.r_long() - idx = self.r_ref_reserve(flag) - retval = tuple(self.r_object() for _ in range(n)) - self.r_ref_insert(retval, idx, flag) - return retval - case Type.LIST: - n = self.r_long() - retval = R_REF([]) - for _ in range(n): - retval.append(self.r_object()) - return retval - case Type.DICT: - retval = R_REF({}) - while True: - key = self.r_object() - if key == NULL: - break - val = self.r_object() - retval[key] = val - return retval - case Type.SET: - n = self.r_long() - retval = R_REF(set()) - for _ in range(n): - v = self.r_object() - retval.add(v) - return retval - case Type.FROZENSET: - n = self.r_long() - s: set[Any] = set() - idx = self.r_ref_reserve(flag) - for _ in range(n): - v = self.r_object() - s.add(v) - retval = frozenset(s) - self.r_ref_insert(retval, idx, flag) - return retval - case Type.CODE: - retval = R_REF(Code()) - retval.co_argcount = self.r_long() - retval.co_posonlyargcount = self.r_long() - retval.co_kwonlyargcount = self.r_long() - retval.co_stacksize = self.r_long() - retval.co_flags = self.r_long() - retval.co_code = self.r_object() - retval.co_consts = self.r_object() - retval.co_names = self.r_object() - retval.co_localsplusnames = self.r_object() - retval.co_localspluskinds = self.r_object() - retval.co_filename = self.r_object() - retval.co_name = self.r_object() - retval.co_qualname = self.r_object() - retval.co_firstlineno = self.r_long() - retval.co_linetable = self.r_object() - retval.co_endlinetable = self.r_object() - retval.co_columntable = self.r_object() - retval.co_exceptiontable = self.r_object() - return retval - case Type.REF: - n = self.r_long() - retval = self.refs[n] - assert retval is not None - return retval - case _: - breakpoint() - raise AssertionError(f"Unknown type {type} {chr(type)!r}") + if type == Type.NULL: + return NULL + elif type == Type.NONE: + return None + elif type == Type.ELLIPSIS: + return Ellipsis + elif type == Type.FALSE: + return False + elif type == Type.TRUE: + return True + elif type == Type.INT: + return R_REF(self.r_long()) + elif type == Type.INT64: + return R_REF(self.r_long64()) + elif type == Type.LONG: + return R_REF(self.r_PyLong()) + elif type == Type.FLOAT: + return R_REF(self.r_float_str()) + elif type == Type.BINARY_FLOAT: + return R_REF(self.r_float_bin()) + elif type == Type.COMPLEX: + return R_REF(complex(self.r_float_str(), + self.r_float_str())) + elif type == Type.BINARY_COMPLEX: + return R_REF(complex(self.r_float_bin(), + self.r_float_bin())) + elif type == Type.STRING: + n = self.r_long() + return R_REF(self.r_string(n)) + elif type == Type.ASCII_INTERNED or type == Type.ASCII: + n = self.r_long() + return R_REF(self.r_string(n).decode("ascii")) + elif type == Type.SHORT_ASCII_INTERNED or type == Type.SHORT_ASCII: + n = self.r_byte() + return R_REF(self.r_string(n).decode("ascii")) + elif type == Type.INTERNED or type == Type.UNICODE: + n = self.r_long() + return R_REF(self.r_string(n).decode("utf8", "surrogatepass")) + elif type == Type.SMALL_TUPLE: + n = self.r_byte() + idx = self.r_ref_reserve(flag) + retval: Any = tuple(self.r_object() for _ in range(n)) + self.r_ref_insert(retval, idx, flag) + return retval + elif type == Type.TUPLE: + n = self.r_long() + idx = self.r_ref_reserve(flag) + retval = tuple(self.r_object() for _ in range(n)) + self.r_ref_insert(retval, idx, flag) + return retval + elif type == Type.LIST: + n = self.r_long() + retval = R_REF([]) + for _ in range(n): + retval.append(self.r_object()) + return retval + elif type == Type.DICT: + retval = R_REF({}) + while True: + key = self.r_object() + if key == NULL: + break + val = self.r_object() + retval[key] = val + return retval + elif type == Type.SET: + n = self.r_long() + retval = R_REF(set()) + for _ in range(n): + v = self.r_object() + retval.add(v) + return retval + elif type == Type.FROZENSET: + n = self.r_long() + s: set[Any] = set() + idx = self.r_ref_reserve(flag) + for _ in range(n): + v = self.r_object() + s.add(v) + retval = frozenset(s) + self.r_ref_insert(retval, idx, flag) + return retval + elif type == Type.CODE: + retval = R_REF(Code()) + retval.co_argcount = self.r_long() + retval.co_posonlyargcount = self.r_long() + retval.co_kwonlyargcount = self.r_long() + retval.co_stacksize = self.r_long() + retval.co_flags = self.r_long() + retval.co_code = self.r_object() + retval.co_consts = self.r_object() + retval.co_names = self.r_object() + retval.co_localsplusnames = self.r_object() + retval.co_localspluskinds = self.r_object() + retval.co_filename = self.r_object() + retval.co_name = self.r_object() + retval.co_qualname = self.r_object() + retval.co_firstlineno = self.r_long() + retval.co_linetable = self.r_object() + retval.co_endlinetable = self.r_object() + retval.co_columntable = self.r_object() + retval.co_exceptiontable = self.r_object() + return retval + elif type == Type.REF: + n = self.r_long() + retval = self.refs[n] + assert retval is not None + return retval + else: + breakpoint() + raise AssertionError(f"Unknown type {type} {chr(type)!r}") def loads(data: bytes) -> Any: From eb1b3caf6cac3600c3f82439129d940d23badc99 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 22 Nov 2021 20:22:12 -0800 Subject: [PATCH 09/11] Restore Python 3.8 and 3.9 as Windows host Python again --- PCbuild/find_python.bat | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/PCbuild/find_python.bat b/PCbuild/find_python.bat index a9f14c5277ffa0..a9bbe5a6279a61 100644 --- a/PCbuild/find_python.bat +++ b/PCbuild/find_python.bat @@ -31,13 +31,13 @@ @if "%_Py_EXTERNALS_DIR%"=="" (set _Py_EXTERNALS_DIR=%~dp0\..\externals) @rem If we have Python in externals, use that one -@if exist "%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" ("%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" -Ec "import sys; assert sys.version_info[:2] >= (3, 10)" >nul 2>nul) && (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") && (set _Py_Python_Source=found in externals directory) && goto :found || rmdir /Q /S "%_Py_EXTERNALS_DIR%\pythonx86" +@if exist "%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" ("%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" -Ec "import sys; assert sys.version_info[:2] >= (3, 8)" >nul 2>nul) && (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") && (set _Py_Python_Source=found in externals directory) && goto :found || rmdir /Q /S "%_Py_EXTERNALS_DIR%\pythonx86" @rem If HOST_PYTHON is recent enough, use that -@if NOT "%HOST_PYTHON%"=="" @%HOST_PYTHON% -Ec "import sys; assert sys.version_info[:2] >= (3, 10)" >nul 2>nul && (set PYTHON="%HOST_PYTHON%") && (set _Py_Python_Source=found as HOST_PYTHON) && goto :found +@if NOT "%HOST_PYTHON%"=="" @%HOST_PYTHON% -Ec "import sys; assert sys.version_info[:2] >= (3, 8)" >nul 2>nul && (set PYTHON="%HOST_PYTHON%") && (set _Py_Python_Source=found as HOST_PYTHON) && goto :found @rem If py.exe finds a recent enough version, use that one -@for %%p in (3.10) do @py -%%p -EV >nul 2>&1 && (set PYTHON=py -%%p) && (set _Py_Python_Source=found %%p with py.exe) && goto :found +@for %%p in (3.10 3.9 3.8) do @py -%%p -EV >nul 2>&1 && (set PYTHON=py -%%p) && (set _Py_Python_Source=found %%p with py.exe) && goto :found @if NOT exist "%_Py_EXTERNALS_DIR%" mkdir "%_Py_EXTERNALS_DIR%" @set _Py_NUGET=%NUGET% From 34a623ad3e8a330820dc66cc87b1ce0b1fd429f7 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 23 Nov 2021 08:18:44 -0800 Subject: [PATCH 10/11] Annotate removesuffix() Co-authored-by: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> --- Tools/scripts/deepfreeze.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index ed791fd5b7d6dd..b840c4b51d73a3 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -87,7 +87,7 @@ def analyze_character_width(s: str) -> Tuple[int, bool]: return kind, ascii -def removesuffix(base, suffix): +def removesuffix(base: str, suffix: str) -> str: if base.endswith(suffix): return base[:len(base) - len(suffix)] return base From 5197758c65aef88e7b119c5b6f1b6f49b07d8be8 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 23 Nov 2021 08:21:14 -0800 Subject: [PATCH 11/11] Make out-of-tree builds pass --- Makefile.pre.in | 120 ++++++++++++++++---------------- Tools/scripts/freeze_modules.py | 6 +- configure | 1 + configure.ac | 1 + 4 files changed, 65 insertions(+), 63 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 2e3aab762a376c..1e7c7a51faf498 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -943,65 +943,65 @@ DEEPFREEZE_DEPS=$(srcdir)/Tools/scripts/deepfreeze.py # BEGIN: deepfreeze modules -Python/deepfreeze/importlib._bootstrap.c: $(srcdir)/Python/frozen_modules/importlib._bootstrap.h $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/importlib._bootstrap.h -m importlib._bootstrap -o Python/deepfreeze/importlib._bootstrap.c +Python/deepfreeze/importlib._bootstrap.c: Python/frozen_modules/importlib._bootstrap.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/importlib._bootstrap.h -m importlib._bootstrap -o Python/deepfreeze/importlib._bootstrap.c -Python/deepfreeze/importlib._bootstrap_external.c: $(srcdir)/Python/frozen_modules/importlib._bootstrap_external.h $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/importlib._bootstrap_external.h -m importlib._bootstrap_external -o Python/deepfreeze/importlib._bootstrap_external.c +Python/deepfreeze/importlib._bootstrap_external.c: Python/frozen_modules/importlib._bootstrap_external.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/importlib._bootstrap_external.h -m importlib._bootstrap_external -o Python/deepfreeze/importlib._bootstrap_external.c -Python/deepfreeze/zipimport.c: $(srcdir)/Python/frozen_modules/zipimport.h $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/zipimport.h -m zipimport -o Python/deepfreeze/zipimport.c +Python/deepfreeze/zipimport.c: Python/frozen_modules/zipimport.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/zipimport.h -m zipimport -o Python/deepfreeze/zipimport.c -Python/deepfreeze/abc.c: $(srcdir)/Python/frozen_modules/abc.h $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/abc.h -m abc -o Python/deepfreeze/abc.c +Python/deepfreeze/abc.c: Python/frozen_modules/abc.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/abc.h -m abc -o Python/deepfreeze/abc.c -Python/deepfreeze/codecs.c: $(srcdir)/Python/frozen_modules/codecs.h $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/codecs.h -m codecs -o Python/deepfreeze/codecs.c +Python/deepfreeze/codecs.c: Python/frozen_modules/codecs.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/codecs.h -m codecs -o Python/deepfreeze/codecs.c -Python/deepfreeze/io.c: $(srcdir)/Python/frozen_modules/io.h $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/io.h -m io -o Python/deepfreeze/io.c +Python/deepfreeze/io.c: Python/frozen_modules/io.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/io.h -m io -o Python/deepfreeze/io.c -Python/deepfreeze/_collections_abc.c: $(srcdir)/Python/frozen_modules/_collections_abc.h $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/_collections_abc.h -m _collections_abc -o Python/deepfreeze/_collections_abc.c +Python/deepfreeze/_collections_abc.c: Python/frozen_modules/_collections_abc.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/_collections_abc.h -m _collections_abc -o Python/deepfreeze/_collections_abc.c -Python/deepfreeze/_sitebuiltins.c: $(srcdir)/Python/frozen_modules/_sitebuiltins.h $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/_sitebuiltins.h -m _sitebuiltins -o Python/deepfreeze/_sitebuiltins.c +Python/deepfreeze/_sitebuiltins.c: Python/frozen_modules/_sitebuiltins.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/_sitebuiltins.h -m _sitebuiltins -o Python/deepfreeze/_sitebuiltins.c -Python/deepfreeze/genericpath.c: $(srcdir)/Python/frozen_modules/genericpath.h $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/genericpath.h -m genericpath -o Python/deepfreeze/genericpath.c +Python/deepfreeze/genericpath.c: Python/frozen_modules/genericpath.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/genericpath.h -m genericpath -o Python/deepfreeze/genericpath.c -Python/deepfreeze/ntpath.c: $(srcdir)/Python/frozen_modules/ntpath.h $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/ntpath.h -m ntpath -o Python/deepfreeze/ntpath.c +Python/deepfreeze/ntpath.c: Python/frozen_modules/ntpath.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/ntpath.h -m ntpath -o Python/deepfreeze/ntpath.c -Python/deepfreeze/posixpath.c: $(srcdir)/Python/frozen_modules/posixpath.h $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/posixpath.h -m posixpath -o Python/deepfreeze/posixpath.c +Python/deepfreeze/posixpath.c: Python/frozen_modules/posixpath.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/posixpath.h -m posixpath -o Python/deepfreeze/posixpath.c -Python/deepfreeze/os.c: $(srcdir)/Python/frozen_modules/os.h $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/os.h -m os -o Python/deepfreeze/os.c +Python/deepfreeze/os.c: Python/frozen_modules/os.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/os.h -m os -o Python/deepfreeze/os.c -Python/deepfreeze/site.c: $(srcdir)/Python/frozen_modules/site.h $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/site.h -m site -o Python/deepfreeze/site.c +Python/deepfreeze/site.c: Python/frozen_modules/site.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/site.h -m site -o Python/deepfreeze/site.c -Python/deepfreeze/stat.c: $(srcdir)/Python/frozen_modules/stat.h $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/stat.h -m stat -o Python/deepfreeze/stat.c +Python/deepfreeze/stat.c: Python/frozen_modules/stat.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/stat.h -m stat -o Python/deepfreeze/stat.c -Python/deepfreeze/__hello__.c: $(srcdir)/Python/frozen_modules/__hello__.h $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/__hello__.h -m __hello__ -o Python/deepfreeze/__hello__.c +Python/deepfreeze/__hello__.c: Python/frozen_modules/__hello__.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__hello__.h -m __hello__ -o Python/deepfreeze/__hello__.c -Python/deepfreeze/__phello__.c: $(srcdir)/Python/frozen_modules/__phello__.h $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/__phello__.h -m __phello__ -o Python/deepfreeze/__phello__.c +Python/deepfreeze/__phello__.c: Python/frozen_modules/__phello__.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__phello__.h -m __phello__ -o Python/deepfreeze/__phello__.c -Python/deepfreeze/__phello__.ham.c: $(srcdir)/Python/frozen_modules/__phello__.ham.h $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/__phello__.ham.h -m __phello__.ham -o Python/deepfreeze/__phello__.ham.c +Python/deepfreeze/__phello__.ham.c: Python/frozen_modules/__phello__.ham.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__phello__.ham.h -m __phello__.ham -o Python/deepfreeze/__phello__.ham.c -Python/deepfreeze/__phello__.ham.eggs.c: $(srcdir)/Python/frozen_modules/__phello__.ham.eggs.h $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/__phello__.ham.eggs.h -m __phello__.ham.eggs -o Python/deepfreeze/__phello__.ham.eggs.c +Python/deepfreeze/__phello__.ham.eggs.c: Python/frozen_modules/__phello__.ham.eggs.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__phello__.ham.eggs.h -m __phello__.ham.eggs -o Python/deepfreeze/__phello__.ham.eggs.c -Python/deepfreeze/__phello__.spam.c: $(srcdir)/Python/frozen_modules/__phello__.spam.h $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/__phello__.spam.h -m __phello__.spam -o Python/deepfreeze/__phello__.spam.c +Python/deepfreeze/__phello__.spam.c: Python/frozen_modules/__phello__.spam.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__phello__.spam.h -m __phello__.spam -o Python/deepfreeze/__phello__.spam.c -Python/deepfreeze/frozen_only.c: $(srcdir)/Python/frozen_modules/frozen_only.h $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py $(srcdir)/Python/frozen_modules/frozen_only.h -m frozen_only -o Python/deepfreeze/frozen_only.c +Python/deepfreeze/frozen_only.c: Python/frozen_modules/frozen_only.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/frozen_only.h -m frozen_only -o Python/deepfreeze/frozen_only.c # END: deepfreeze modules @@ -1062,64 +1062,64 @@ Programs/_freeze_module: Programs/_freeze_module.o $(LIBRARY_OBJS_OMIT_FROZEN) # BEGIN: freezing modules Python/frozen_modules/importlib._bootstrap.h: Programs/_freeze_module Lib/importlib/_bootstrap.py - Programs/_freeze_module importlib._bootstrap $(srcdir)/Lib/importlib/_bootstrap.py $(srcdir)/Python/frozen_modules/importlib._bootstrap.h + Programs/_freeze_module importlib._bootstrap $(srcdir)/Lib/importlib/_bootstrap.py Python/frozen_modules/importlib._bootstrap.h Python/frozen_modules/importlib._bootstrap_external.h: Programs/_freeze_module Lib/importlib/_bootstrap_external.py - Programs/_freeze_module importlib._bootstrap_external $(srcdir)/Lib/importlib/_bootstrap_external.py $(srcdir)/Python/frozen_modules/importlib._bootstrap_external.h + Programs/_freeze_module importlib._bootstrap_external $(srcdir)/Lib/importlib/_bootstrap_external.py Python/frozen_modules/importlib._bootstrap_external.h Python/frozen_modules/zipimport.h: Programs/_freeze_module Lib/zipimport.py - Programs/_freeze_module zipimport $(srcdir)/Lib/zipimport.py $(srcdir)/Python/frozen_modules/zipimport.h + Programs/_freeze_module zipimport $(srcdir)/Lib/zipimport.py Python/frozen_modules/zipimport.h Python/frozen_modules/abc.h: Programs/_freeze_module Lib/abc.py - Programs/_freeze_module abc $(srcdir)/Lib/abc.py $(srcdir)/Python/frozen_modules/abc.h + Programs/_freeze_module abc $(srcdir)/Lib/abc.py Python/frozen_modules/abc.h Python/frozen_modules/codecs.h: Programs/_freeze_module Lib/codecs.py - Programs/_freeze_module codecs $(srcdir)/Lib/codecs.py $(srcdir)/Python/frozen_modules/codecs.h + Programs/_freeze_module codecs $(srcdir)/Lib/codecs.py Python/frozen_modules/codecs.h Python/frozen_modules/io.h: Programs/_freeze_module Lib/io.py - Programs/_freeze_module io $(srcdir)/Lib/io.py $(srcdir)/Python/frozen_modules/io.h + Programs/_freeze_module io $(srcdir)/Lib/io.py Python/frozen_modules/io.h Python/frozen_modules/_collections_abc.h: Programs/_freeze_module Lib/_collections_abc.py - Programs/_freeze_module _collections_abc $(srcdir)/Lib/_collections_abc.py $(srcdir)/Python/frozen_modules/_collections_abc.h + Programs/_freeze_module _collections_abc $(srcdir)/Lib/_collections_abc.py Python/frozen_modules/_collections_abc.h Python/frozen_modules/_sitebuiltins.h: Programs/_freeze_module Lib/_sitebuiltins.py - Programs/_freeze_module _sitebuiltins $(srcdir)/Lib/_sitebuiltins.py $(srcdir)/Python/frozen_modules/_sitebuiltins.h + Programs/_freeze_module _sitebuiltins $(srcdir)/Lib/_sitebuiltins.py Python/frozen_modules/_sitebuiltins.h Python/frozen_modules/genericpath.h: Programs/_freeze_module Lib/genericpath.py - Programs/_freeze_module genericpath $(srcdir)/Lib/genericpath.py $(srcdir)/Python/frozen_modules/genericpath.h + Programs/_freeze_module genericpath $(srcdir)/Lib/genericpath.py Python/frozen_modules/genericpath.h Python/frozen_modules/ntpath.h: Programs/_freeze_module Lib/ntpath.py - Programs/_freeze_module ntpath $(srcdir)/Lib/ntpath.py $(srcdir)/Python/frozen_modules/ntpath.h + Programs/_freeze_module ntpath $(srcdir)/Lib/ntpath.py Python/frozen_modules/ntpath.h Python/frozen_modules/posixpath.h: Programs/_freeze_module Lib/posixpath.py - Programs/_freeze_module posixpath $(srcdir)/Lib/posixpath.py $(srcdir)/Python/frozen_modules/posixpath.h + Programs/_freeze_module posixpath $(srcdir)/Lib/posixpath.py Python/frozen_modules/posixpath.h Python/frozen_modules/os.h: Programs/_freeze_module Lib/os.py - Programs/_freeze_module os $(srcdir)/Lib/os.py $(srcdir)/Python/frozen_modules/os.h + Programs/_freeze_module os $(srcdir)/Lib/os.py Python/frozen_modules/os.h Python/frozen_modules/site.h: Programs/_freeze_module Lib/site.py - Programs/_freeze_module site $(srcdir)/Lib/site.py $(srcdir)/Python/frozen_modules/site.h + Programs/_freeze_module site $(srcdir)/Lib/site.py Python/frozen_modules/site.h Python/frozen_modules/stat.h: Programs/_freeze_module Lib/stat.py - Programs/_freeze_module stat $(srcdir)/Lib/stat.py $(srcdir)/Python/frozen_modules/stat.h + Programs/_freeze_module stat $(srcdir)/Lib/stat.py Python/frozen_modules/stat.h Python/frozen_modules/__hello__.h: Programs/_freeze_module Lib/__hello__.py - Programs/_freeze_module __hello__ $(srcdir)/Lib/__hello__.py $(srcdir)/Python/frozen_modules/__hello__.h + Programs/_freeze_module __hello__ $(srcdir)/Lib/__hello__.py Python/frozen_modules/__hello__.h Python/frozen_modules/__phello__.h: Programs/_freeze_module Lib/__phello__/__init__.py - Programs/_freeze_module __phello__ $(srcdir)/Lib/__phello__/__init__.py $(srcdir)/Python/frozen_modules/__phello__.h + Programs/_freeze_module __phello__ $(srcdir)/Lib/__phello__/__init__.py Python/frozen_modules/__phello__.h Python/frozen_modules/__phello__.ham.h: Programs/_freeze_module Lib/__phello__/ham/__init__.py - Programs/_freeze_module __phello__.ham $(srcdir)/Lib/__phello__/ham/__init__.py $(srcdir)/Python/frozen_modules/__phello__.ham.h + Programs/_freeze_module __phello__.ham $(srcdir)/Lib/__phello__/ham/__init__.py Python/frozen_modules/__phello__.ham.h Python/frozen_modules/__phello__.ham.eggs.h: Programs/_freeze_module Lib/__phello__/ham/eggs.py - Programs/_freeze_module __phello__.ham.eggs $(srcdir)/Lib/__phello__/ham/eggs.py $(srcdir)/Python/frozen_modules/__phello__.ham.eggs.h + Programs/_freeze_module __phello__.ham.eggs $(srcdir)/Lib/__phello__/ham/eggs.py Python/frozen_modules/__phello__.ham.eggs.h Python/frozen_modules/__phello__.spam.h: Programs/_freeze_module Lib/__phello__/spam.py - Programs/_freeze_module __phello__.spam $(srcdir)/Lib/__phello__/spam.py $(srcdir)/Python/frozen_modules/__phello__.spam.h + Programs/_freeze_module __phello__.spam $(srcdir)/Lib/__phello__/spam.py Python/frozen_modules/__phello__.spam.h Python/frozen_modules/frozen_only.h: Programs/_freeze_module Tools/freeze/flag.py - Programs/_freeze_module frozen_only $(srcdir)/Tools/freeze/flag.py $(srcdir)/Python/frozen_modules/frozen_only.h + Programs/_freeze_module frozen_only $(srcdir)/Tools/freeze/flag.py Python/frozen_modules/frozen_only.h # END: freezing modules diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index 6614a2518f4f08..4281b67e27cb3e 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -584,11 +584,11 @@ def regen_makefile(modules): deepfreezefiles.append(f"\t\t{ofile} \\") # Also add a deepfreeze rule. - deepfreezerules.append(f'{cfile}: $(srcdir)/{header} $(DEEPFREEZE_DEPS)') + deepfreezerules.append(f'{cfile}: {header} $(DEEPFREEZE_DEPS)') deepfreezerules.append( f"\t$(PYTHON_FOR_REGEN) " f"$(srcdir)/Tools/scripts/deepfreeze.py " - f"$(srcdir)/{header} -m {src.frozenid} -o {cfile}") + f"{header} -m {src.frozenid} -o {cfile}") deepfreezerules.append('') for src in _iter_sources(modules): @@ -599,7 +599,7 @@ def regen_makefile(modules): pyfiles.append(f'\t\t{pyfile} \\') freeze = (f'Programs/_freeze_module {src.frozenid} ' - f'$(srcdir)/{pyfile} $(srcdir)/{header}') + f'$(srcdir)/{pyfile} {header}') rules.extend([ f'{header}: Programs/_freeze_module {pyfile}', f'\t{freeze}', diff --git a/configure b/configure index dafcce61d132df..b99ee20f640377 100755 --- a/configure +++ b/configure @@ -19439,6 +19439,7 @@ SRCDIRS="\ Parser \ Programs \ Python \ + Python/frozen_modules \ Python/deepfreeze" { $as_echo "$as_me:${as_lineno-$LINENO}: checking for build directories" >&5 $as_echo_n "checking for build directories... " >&6; } diff --git a/configure.ac b/configure.ac index ec7d1a48a0beeb..fde91b9f44ae03 100644 --- a/configure.ac +++ b/configure.ac @@ -5613,6 +5613,7 @@ SRCDIRS="\ Parser \ Programs \ Python \ + Python/frozen_modules \ Python/deepfreeze" AC_MSG_CHECKING(for build directories) for dir in $SRCDIRS; do pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy