From 85dd6ec9a2460283942281ce931d0dfcd662da5b Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Wed, 1 Dec 2021 17:27:33 +0100 Subject: [PATCH 1/3] bpo-45949: Pure Python freeze module for cross builds Use `_bootstrap_python` interpreter and pure Python implementation of `freeze_module` to generate frozen byte code files. Only importlib bootstrap files are generated with `Programs/_freeze_module`. This simplifies cross building, as the build system no longer needs a `_freeze_module` binary. A standard Python installation with same version is sufficient. Signed-off-by: Christian Heimes --- Doc/using/configure.rst | 14 +- Makefile.pre.in | 133 +++++++++++------- .../2021-12-02-23-21-18.bpo-45949.OTSo9X.rst | 3 + Tools/scripts/deepfreeze.py | 5 +- Tools/scripts/freeze_module.py | 69 +++++++++ Tools/scripts/freeze_modules.py | 19 ++- configure | 45 ++---- configure.ac | 32 +---- 8 files changed, 189 insertions(+), 131 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2021-12-02-23-21-18.bpo-45949.OTSo9X.rst create mode 100644 Tools/scripts/freeze_module.py diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 7802a7c810b29e..b2a2936db738bc 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -518,9 +518,8 @@ Cross Compiling Options Cross compiling, also known as cross building, can be used to build Python for another CPU architecture or platform. Cross compiling requires a Python -interpreter and the :program:`_freeze_module` binary from another build. The -version of the build Python and :program:`_freeze_module` command must be -the same as the cross compiled host Python. +interpreter for the build platform. The version of the build Python must match +the version of the cross compiled host Python. .. cmdoption:: --build=BUILD @@ -530,13 +529,7 @@ the same as the cross compiled host Python. cross-compile to build programs to run on HOST (target platform) -.. cmdoption:: --with-freeze-module=Programs/_freeze_module - - path to ``_freeze_module`` binary for cross compiling. - - .. versionadded:: 3.11 - -.. cmdoption:: --with-build-python=python3.xx +.. cmdoption:: --with-build-python=path/to/python path to build ``python`` binary for cross compiling @@ -559,7 +552,6 @@ Cross compiling example:: CONFIG_SITE=config.site-aarch64 ../configure \ --build=x86_64-pc-linux-gnu \ --host=aarch64-unknown-linux-gnu \ - --with-freeze-module=../x86_64/Programs/_freeze_module \ --with-build-python=../x86_64/python diff --git a/Makefile.pre.in b/Makefile.pre.in index a182786e5b5873..db11892b0f3f80 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -286,11 +286,14 @@ PYTHON_FOR_BUILD=@PYTHON_FOR_BUILD@ # Standard builds use _bootstrap_python for freezing, cross compiling # uses build Python, which must have the same version and bytecode, PYTHON_FOR_FREEZE?=@PYTHON_FOR_FREEZE@ +FREEZE_MODULE_C=Programs/_freeze_module +FREEZE_MODULE_PY=$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/freeze_module.py +# bootstrap freezing uses FREEZE_MODULE_C for normal builds and _PY for cross builds +FREEZE_MODULE_BOOTSTRAP=@FREEZE_MODULE_BOOTSTRAP@ + _PYTHON_HOST_PLATFORM=@_PYTHON_HOST_PLATFORM@ BUILD_GNU_TYPE= @build@ HOST_GNU_TYPE= @host@ -# Allow developers to override freeze_module command for cross building (bpo-45886) -FREEZE_MODULE?=@FREEZE_MODULE@ # Tcl and Tk config info from --with-tcltk-includes and -libs options TCLTK_INCLUDES= @TCLTK_INCLUDES@ @@ -1044,6 +1047,32 @@ Python/deepfreeze/frozen_only.c: Python/frozen_modules/frozen_only.h $(DEEPFREEZ ############################################################################ # frozen modules (including importlib) +# +# Freezing is a multi step process. It works differently for standard builds +# and cross builds. Standard builds use Programs/_freeze_module and +# _bootstrap_python for freezing and deepfreezing, so users can build Python +# without an existing Python installation. Cross builds cannot execute +# compiled binaries and therefore rely on an external build Python +# interpreter. The build interpreter must have same version and same bytecode +# as the host (target) binary. +# +# Standard build process: +# 1) compile minimal core objects for Py_Compile*() and PyMarshal_Write*(). +# 2) build Programs/_freeze_module binary. +# 3) create frozen module headers for importlib and getpath. +# 4) build _bootstrap_python binary. +# 5) create remaining frozen module headers with _bootstrap_python and +# Python freeze_module.py (so we test this code path). +# 6) deepfreeze modules with _bootstrap_python +# +# Cross compile proces: +# 1) create all frozen module headers with external build Python and +# freeze_module.py script. +# 2) deepfreeze modules with external build Python. +# +# Cross compiling also creates Programs/_freeze_module and _bootstrap_python +# artifacts, but does not use or ship them. +# # FROZEN_FILES_* are auto-generated by Tools/scripts/freeze_modules.py. FROZEN_FILES_IN = \ @@ -1104,83 +1133,83 @@ Modules/getpath_noop.o: $(srcdir)/Modules/getpath_noop.c Makefile Programs/_freeze_module: Programs/_freeze_module.o Modules/getpath_noop.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LINKCC) $(PY_CORE_LDFLAGS) -o $@ Programs/_freeze_module.o Modules/getpath_noop.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LIBS) $(MODLIBS) $(SYSLIBS) +# We manually freeze getpath.py rather than through freeze_modules +Python/frozen_modules/getpath.h: $(FREEZE_MODULE_C) Modules/getpath.py + $(FREEZE_MODULE_BOOTSTRAP) getpath $(srcdir)/Modules/getpath.py Python/frozen_modules/getpath.h + # BEGIN: freezing modules -Python/frozen_modules/importlib._bootstrap.h: $(FREEZE_MODULE) Lib/importlib/_bootstrap.py - $(FREEZE_MODULE) importlib._bootstrap $(srcdir)/Lib/importlib/_bootstrap.py Python/frozen_modules/importlib._bootstrap.h +Python/frozen_modules/importlib._bootstrap.h: Lib/importlib/_bootstrap.py $(FREEZE_MODULE_C) $(srcdir)/Tools/scripts/freeze_module.py + $(FREEZE_MODULE_BOOTSTRAP) importlib._bootstrap $(srcdir)/Lib/importlib/_bootstrap.py Python/frozen_modules/importlib._bootstrap.h -Python/frozen_modules/importlib._bootstrap_external.h: $(FREEZE_MODULE) Lib/importlib/_bootstrap_external.py - $(FREEZE_MODULE) importlib._bootstrap_external $(srcdir)/Lib/importlib/_bootstrap_external.py Python/frozen_modules/importlib._bootstrap_external.h +Python/frozen_modules/importlib._bootstrap_external.h: Lib/importlib/_bootstrap_external.py $(FREEZE_MODULE_C) $(srcdir)/Tools/scripts/freeze_module.py + $(FREEZE_MODULE_BOOTSTRAP) importlib._bootstrap_external $(srcdir)/Lib/importlib/_bootstrap_external.py Python/frozen_modules/importlib._bootstrap_external.h -Python/frozen_modules/zipimport.h: $(FREEZE_MODULE) Lib/zipimport.py - $(FREEZE_MODULE) zipimport $(srcdir)/Lib/zipimport.py Python/frozen_modules/zipimport.h +Python/frozen_modules/zipimport.h: Lib/zipimport.py $(FREEZE_MODULE_C) $(srcdir)/Tools/scripts/freeze_module.py + $(FREEZE_MODULE_BOOTSTRAP) zipimport $(srcdir)/Lib/zipimport.py Python/frozen_modules/zipimport.h -Python/frozen_modules/abc.h: $(FREEZE_MODULE) Lib/abc.py - $(FREEZE_MODULE) abc $(srcdir)/Lib/abc.py Python/frozen_modules/abc.h +Python/frozen_modules/abc.h: Lib/abc.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py + $(FREEZE_MODULE_PY) abc $(srcdir)/Lib/abc.py Python/frozen_modules/abc.h -Python/frozen_modules/codecs.h: $(FREEZE_MODULE) Lib/codecs.py - $(FREEZE_MODULE) codecs $(srcdir)/Lib/codecs.py Python/frozen_modules/codecs.h +Python/frozen_modules/codecs.h: Lib/codecs.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py + $(FREEZE_MODULE_PY) codecs $(srcdir)/Lib/codecs.py Python/frozen_modules/codecs.h -Python/frozen_modules/io.h: $(FREEZE_MODULE) Lib/io.py - $(FREEZE_MODULE) io $(srcdir)/Lib/io.py Python/frozen_modules/io.h +Python/frozen_modules/io.h: Lib/io.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py + $(FREEZE_MODULE_PY) io $(srcdir)/Lib/io.py Python/frozen_modules/io.h -Python/frozen_modules/_collections_abc.h: $(FREEZE_MODULE) Lib/_collections_abc.py - $(FREEZE_MODULE) _collections_abc $(srcdir)/Lib/_collections_abc.py Python/frozen_modules/_collections_abc.h +Python/frozen_modules/_collections_abc.h: Lib/_collections_abc.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py + $(FREEZE_MODULE_PY) _collections_abc $(srcdir)/Lib/_collections_abc.py Python/frozen_modules/_collections_abc.h -Python/frozen_modules/_sitebuiltins.h: $(FREEZE_MODULE) Lib/_sitebuiltins.py - $(FREEZE_MODULE) _sitebuiltins $(srcdir)/Lib/_sitebuiltins.py Python/frozen_modules/_sitebuiltins.h +Python/frozen_modules/_sitebuiltins.h: Lib/_sitebuiltins.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py + $(FREEZE_MODULE_PY) _sitebuiltins $(srcdir)/Lib/_sitebuiltins.py Python/frozen_modules/_sitebuiltins.h -Python/frozen_modules/genericpath.h: $(FREEZE_MODULE) Lib/genericpath.py - $(FREEZE_MODULE) genericpath $(srcdir)/Lib/genericpath.py Python/frozen_modules/genericpath.h +Python/frozen_modules/genericpath.h: Lib/genericpath.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py + $(FREEZE_MODULE_PY) genericpath $(srcdir)/Lib/genericpath.py Python/frozen_modules/genericpath.h -Python/frozen_modules/ntpath.h: $(FREEZE_MODULE) Lib/ntpath.py - $(FREEZE_MODULE) ntpath $(srcdir)/Lib/ntpath.py Python/frozen_modules/ntpath.h +Python/frozen_modules/ntpath.h: Lib/ntpath.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py + $(FREEZE_MODULE_PY) ntpath $(srcdir)/Lib/ntpath.py Python/frozen_modules/ntpath.h -Python/frozen_modules/posixpath.h: $(FREEZE_MODULE) Lib/posixpath.py - $(FREEZE_MODULE) posixpath $(srcdir)/Lib/posixpath.py Python/frozen_modules/posixpath.h +Python/frozen_modules/posixpath.h: Lib/posixpath.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py + $(FREEZE_MODULE_PY) posixpath $(srcdir)/Lib/posixpath.py Python/frozen_modules/posixpath.h -Python/frozen_modules/os.h: $(FREEZE_MODULE) Lib/os.py - $(FREEZE_MODULE) os $(srcdir)/Lib/os.py Python/frozen_modules/os.h +Python/frozen_modules/os.h: Lib/os.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py + $(FREEZE_MODULE_PY) os $(srcdir)/Lib/os.py Python/frozen_modules/os.h -Python/frozen_modules/site.h: $(FREEZE_MODULE) Lib/site.py - $(FREEZE_MODULE) site $(srcdir)/Lib/site.py Python/frozen_modules/site.h +Python/frozen_modules/site.h: Lib/site.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py + $(FREEZE_MODULE_PY) site $(srcdir)/Lib/site.py Python/frozen_modules/site.h -Python/frozen_modules/stat.h: $(FREEZE_MODULE) Lib/stat.py - $(FREEZE_MODULE) stat $(srcdir)/Lib/stat.py Python/frozen_modules/stat.h +Python/frozen_modules/stat.h: Lib/stat.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py + $(FREEZE_MODULE_PY) stat $(srcdir)/Lib/stat.py Python/frozen_modules/stat.h -Python/frozen_modules/importlib.util.h: $(FREEZE_MODULE) Lib/importlib/util.py - $(FREEZE_MODULE) importlib.util $(srcdir)/Lib/importlib/util.py Python/frozen_modules/importlib.util.h +Python/frozen_modules/importlib.util.h: Lib/importlib/util.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py + $(FREEZE_MODULE_PY) importlib.util $(srcdir)/Lib/importlib/util.py Python/frozen_modules/importlib.util.h -Python/frozen_modules/importlib.machinery.h: $(FREEZE_MODULE) Lib/importlib/machinery.py - $(FREEZE_MODULE) importlib.machinery $(srcdir)/Lib/importlib/machinery.py Python/frozen_modules/importlib.machinery.h +Python/frozen_modules/importlib.machinery.h: Lib/importlib/machinery.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py + $(FREEZE_MODULE_PY) importlib.machinery $(srcdir)/Lib/importlib/machinery.py Python/frozen_modules/importlib.machinery.h -Python/frozen_modules/runpy.h: $(FREEZE_MODULE) Lib/runpy.py - $(FREEZE_MODULE) runpy $(srcdir)/Lib/runpy.py Python/frozen_modules/runpy.h +Python/frozen_modules/runpy.h: Lib/runpy.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py + $(FREEZE_MODULE_PY) runpy $(srcdir)/Lib/runpy.py Python/frozen_modules/runpy.h -Python/frozen_modules/__hello__.h: $(FREEZE_MODULE) Lib/__hello__.py - $(FREEZE_MODULE) __hello__ $(srcdir)/Lib/__hello__.py Python/frozen_modules/__hello__.h +Python/frozen_modules/__hello__.h: Lib/__hello__.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py + $(FREEZE_MODULE_PY) __hello__ $(srcdir)/Lib/__hello__.py Python/frozen_modules/__hello__.h -Python/frozen_modules/__phello__.h: $(FREEZE_MODULE) Lib/__phello__/__init__.py - $(FREEZE_MODULE) __phello__ $(srcdir)/Lib/__phello__/__init__.py Python/frozen_modules/__phello__.h +Python/frozen_modules/__phello__.h: Lib/__phello__/__init__.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py + $(FREEZE_MODULE_PY) __phello__ $(srcdir)/Lib/__phello__/__init__.py Python/frozen_modules/__phello__.h -Python/frozen_modules/__phello__.ham.h: $(FREEZE_MODULE) Lib/__phello__/ham/__init__.py - $(FREEZE_MODULE) __phello__.ham $(srcdir)/Lib/__phello__/ham/__init__.py Python/frozen_modules/__phello__.ham.h +Python/frozen_modules/__phello__.ham.h: Lib/__phello__/ham/__init__.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py + $(FREEZE_MODULE_PY) __phello__.ham $(srcdir)/Lib/__phello__/ham/__init__.py Python/frozen_modules/__phello__.ham.h -Python/frozen_modules/__phello__.ham.eggs.h: $(FREEZE_MODULE) Lib/__phello__/ham/eggs.py - $(FREEZE_MODULE) __phello__.ham.eggs $(srcdir)/Lib/__phello__/ham/eggs.py Python/frozen_modules/__phello__.ham.eggs.h +Python/frozen_modules/__phello__.ham.eggs.h: Lib/__phello__/ham/eggs.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py + $(FREEZE_MODULE_PY) __phello__.ham.eggs $(srcdir)/Lib/__phello__/ham/eggs.py Python/frozen_modules/__phello__.ham.eggs.h -Python/frozen_modules/__phello__.spam.h: $(FREEZE_MODULE) Lib/__phello__/spam.py - $(FREEZE_MODULE) __phello__.spam $(srcdir)/Lib/__phello__/spam.py Python/frozen_modules/__phello__.spam.h +Python/frozen_modules/__phello__.spam.h: Lib/__phello__/spam.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py + $(FREEZE_MODULE_PY) __phello__.spam $(srcdir)/Lib/__phello__/spam.py Python/frozen_modules/__phello__.spam.h -Python/frozen_modules/frozen_only.h: $(FREEZE_MODULE) Tools/freeze/flag.py - $(FREEZE_MODULE) frozen_only $(srcdir)/Tools/freeze/flag.py Python/frozen_modules/frozen_only.h +Python/frozen_modules/frozen_only.h: Tools/freeze/flag.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py + $(FREEZE_MODULE_PY) frozen_only $(srcdir)/Tools/freeze/flag.py Python/frozen_modules/frozen_only.h # END: freezing modules -# We manually freeze getpath.py rather than through freeze_modules -Python/frozen_modules/getpath.h: $(FREEZE_MODULE) Modules/getpath.py - $(FREEZE_MODULE) getpath $(srcdir)/Modules/getpath.py Python/frozen_modules/getpath.h - Tools/scripts/freeze_modules.py: $(FREEZE_MODULE) .PHONY: regen-frozen diff --git a/Misc/NEWS.d/next/Build/2021-12-02-23-21-18.bpo-45949.OTSo9X.rst b/Misc/NEWS.d/next/Build/2021-12-02-23-21-18.bpo-45949.OTSo9X.rst new file mode 100644 index 00000000000000..c746d71b1dcae9 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2021-12-02-23-21-18.bpo-45949.OTSo9X.rst @@ -0,0 +1,3 @@ +Use pure Python ``freeze_module`` for all but importlib bootstrap files. +``--with-freeze-module`` :program:`configure` option is no longer needed for +cross builds. diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index 30ca7bb0b6090e..7cd7eb674d4511 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -393,13 +393,14 @@ def generate(self, name: str, obj: object) -> str: } """ -FROZEN_COMMENT = "/* Auto-generated by Programs/_freeze_module.c */" +FROZEN_COMMENT_C = "/* Auto-generated by Programs/_freeze_module.c */" +FROZEN_COMMENT_PY = "/* Auto-generated by Tools/scripts/freeze_module.py */" FROZEN_DATA_LINE = r"\s*(\d+,\s*)+\s*" def is_frozen_header(source: str) -> bool: - return source.startswith(FROZEN_COMMENT) + return source.startswith((FROZEN_COMMENT_C, FROZEN_COMMENT_PY)) def decode_frozen_data(source: str) -> types.CodeType: diff --git a/Tools/scripts/freeze_module.py b/Tools/scripts/freeze_module.py new file mode 100644 index 00000000000000..6b5ad35708f788 --- /dev/null +++ b/Tools/scripts/freeze_module.py @@ -0,0 +1,69 @@ +"""Python implementation of Programs/_freeze_module.c + +The pure Python implementation uses same functions and arguments as the C +implementation. + +The generated byte code is slightly different because +compile() sets the PyCF_SOURCE_IS_UTF8 flag and objects have a +reference count > 1. Marshal adds the `FLAG_REF` flag and creates a +reference `hashtable`. +""" + +import marshal +import sys + +header = "/* Auto-generated by Tools/scripts/freeze_module.py */" + + +def read_text(inpath: str) -> bytes: + with open(inpath, "rb") as f: + return f.read() + + +def compile_and_marshal(name: str, text: bytes) -> bytes: + filename = f"" + # exec == Py_file_input + code = compile(text, filename, "exec", optimize=0, dont_inherit=True) + # defaults to Py_MARSHAL_VERSION) + return marshal.dumps(code) + + +def get_varname(name: str, prefix: str) -> str: + return f"{prefix}{name.replace('.', '_')}" + + +def write_code(outfile, marshalled: bytes, varname: str) -> None: + data_size = len(marshalled) + + outfile.write(f"const unsigned char {varname}[] = {{\n") + + for n in range(0, data_size, 16): + outfile.write(" ") + outfile.write(",".join(str(i) for i in marshalled[n : n + 16])) + outfile.write(",\n") + outfile.write("};\n") + + +def write_frozen(outpath: str, inpath: str, name: str, marshalled: bytes) -> None: + with open(outpath, "w") as outfile: + outfile.write(header) + outfile.write("\n") + arrayname = get_varname(name, "_Py_M__") + write_code(outfile, marshalled, arrayname) + + +def main(): + if len(sys.argv) != 4: + sys.exit("need to specify the name, input and output paths\n") + + name = sys.argv[1] + inpath = sys.argv[2] + outpath = sys.argv[3] + + text = read_text(inpath) + marshalled = compile_and_marshal(name, text) + write_frozen(outpath, inpath, name, marshalled) + + +if __name__ == "__main__": + main() diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index 58d13985608359..05d59cc6bcd452 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -264,6 +264,10 @@ def ispkg(self): else: return os.path.basename(self.pyfile) == '__init__.py' + @property + def isbootstrap(self): + return self.id in BOOTSTRAP + def resolve_frozen_file(frozenid, destdir): """Return the filename corresponding to the given frozen ID. @@ -476,7 +480,7 @@ def regen_frozen(modules): indent = ' ' lastsection = None for mod in modules: - if mod.frozenid in BOOTSTRAP: + if mod.isbootstrap: lines = bootstraplines elif mod.section == TESTS_SECTION: lines = testlines @@ -585,10 +589,17 @@ def regen_makefile(modules): pyfile = relpath_for_posix_display(src.pyfile, ROOT_DIR) pyfiles.append(f'\t\t{pyfile} \\') - freeze = (f'$(FREEZE_MODULE) {src.frozenid} ' - f'$(srcdir)/{pyfile} {frozen_header}') + if src.isbootstrap: + freezecmd = '$(FREEZE_MODULE_BOOTSTRAP)' + freezedep = '$(FREEZE_MODULE_C)' + else: + freezecmd = '$(FREEZE_MODULE_PY)' + freezedep = '_bootstrap_python' + + freeze = (f'{freezecmd} {src.frozenid} ' + f'$(srcdir)/{pyfile} {frozen_header}') rules.extend([ - f'{frozen_header}: $(FREEZE_MODULE) {pyfile}', + f'{frozen_header}: {pyfile} {freezedep} $(srcdir)/Tools/scripts/freeze_module.py', f'\t{freeze}', '', ]) diff --git a/configure b/configure index 3c2a9cf0a249dd..e3ad4a8875c5dd 100755 --- a/configure +++ b/configure @@ -933,9 +933,9 @@ CONFIG_ARGS SOVERSION VERSION PYTHON_FOR_REGEN +FREEZE_MODULE_BOOTSTRAP PYTHON_FOR_FREEZE PYTHON_FOR_BUILD -FREEZE_MODULE host_os host_vendor host_cpu @@ -991,7 +991,6 @@ SHELL' ac_subst_files='' ac_user_opts=' enable_option_checking -with_freeze_module with_build_python with_pkg_config enable_universalsdk @@ -1726,8 +1725,6 @@ Optional Features: Optional Packages: --with-PACKAGE[=ARG] use PACKAGE [ARG=yes] --without-PACKAGE do not use PACKAGE (same as --with-PACKAGE=no) - --with-freeze-module=Programs/_freeze_module - path to _freeze_module binary for cross compiling --with-build-python=python3.11 path to build python binary for cross compiling (default: _bootstrap_python or python3.11) @@ -3196,38 +3193,6 @@ fi rm -f pybuilddir.txt -# Check whether --with-freeze-module was given. -if test "${with_freeze_module+set}" = set; then : - withval=$with_freeze_module; - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-freeze-module" >&5 -$as_echo_n "checking for --with-freeze-module... " >&6; } - if test "x$cross_compiling" = xno; then : - as_fn_error $? "--with-freeze-module only applies to cross compiling" "$LINENO" 5 -fi - if test "$with_freeze_module" = yes -o "$with_freeze_module" = no; then - as_fn_error $? "invalid --with-freeze-module option: expected path, not \"$with_freeze_module\"" "$LINENO" 5 - fi - if ! $(command -v "$with_freeze_module" >/dev/null 2>&1); then - as_fn_error $? "invalid or missing freeze module binary \"$with_freeze_module\"" "$LINENO" 5 - fi - FREEZE_MODULE="$with_freeze_module" - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $FREEZE_MODULE" >&5 -$as_echo "$FREEZE_MODULE" >&6; } - -else - - if test "x$cross_compiling" = xyes; then : - as_fn_error $? "Cross compiling requires --with-freeze-module" "$LINENO" 5 - -fi - FREEZE_MODULE=Programs/_freeze_module - - -fi - - - - # Check whether --with-build-python was given. if test "${with_build_python+set}" = set; then : withval=$with_build_python; @@ -3274,6 +3239,14 @@ $as_echo_n "checking for Python interpreter freezing... " >&6; } $as_echo "$PYTHON_FOR_FREEZE" >&6; } +if test "x$cross_compiling" = xyes; then : + FREEZE_MODULE_BOOTSTRAP='$(FREEZE_MODULE_PY)' +else + FREEZE_MODULE_BOOTSTRAP='$(FREEZE_MODULE_C)' + +fi + + for ac_prog in python$PACKAGE_VERSION python3.10 python3.9 python3.8 python3.7 python3.6 python3 python do # Extract the first word of "$ac_prog", so it can be a program name with args. diff --git a/configure.ac b/configure.ac index fa5e63cf5c3f98..f5e6ea46b455ee 100644 --- a/configure.ac +++ b/configure.ac @@ -100,32 +100,6 @@ AS_VAR_IF([cross_compiling], [maybe], # pybuilddir.txt will be created by --generate-posix-vars in the Makefile rm -f pybuilddir.txt -dnl cross-compiling needs a freeze_module binary for build platform -AC_ARG_WITH( - [freeze-module], - [AS_HELP_STRING([--with-freeze-module=Programs/_freeze_module], - [path to _freeze_module binary for cross compiling])], - [ - AC_MSG_CHECKING([for --with-freeze-module]) - AS_VAR_IF([cross_compiling], [no], AC_MSG_ERROR([--with-freeze-module only applies to cross compiling])) - if test "$with_freeze_module" = yes -o "$with_freeze_module" = no; then - AC_MSG_ERROR([invalid --with-freeze-module option: expected path, not "$with_freeze_module"]) - fi - if ! $(command -v "$with_freeze_module" >/dev/null 2>&1); then - AC_MSG_ERROR([invalid or missing freeze module binary "$with_freeze_module"]) - fi - FREEZE_MODULE="$with_freeze_module" - AC_MSG_RESULT([$FREEZE_MODULE]) - ], [ - AS_VAR_IF([cross_compiling], [yes], - [AC_MSG_ERROR([Cross compiling requires --with-freeze-module])] - ) - FREEZE_MODULE=Programs/_freeze_module - ] -) -AC_SUBST([FREEZE_MODULE]) - -dnl build-python is used for cross compiling and macOS framework builds. AC_ARG_WITH( [build-python], [AS_HELP_STRING([--with-build-python=python]PYTHON_VERSION, @@ -162,6 +136,12 @@ AC_MSG_CHECKING([for Python interpreter freezing]) AC_MSG_RESULT([$PYTHON_FOR_FREEZE]) AC_SUBST([PYTHON_FOR_FREEZE]) +AS_VAR_IF([cross_compiling], [yes], + [FREEZE_MODULE_BOOTSTRAP='$(FREEZE_MODULE_PY)'], + [FREEZE_MODULE_BOOTSTRAP='$(FREEZE_MODULE_C)'] +) +AC_SUBST([FREEZE_MODULE_BOOTSTRAP]) + AC_CHECK_PROGS([PYTHON_FOR_REGEN], [python$PACKAGE_VERSION python3.10 python3.9 python3.8 python3.7 python3.6 python3 python], [python3]) From df657a9b980cce1c4d0deeb8b87be96d657d30c1 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Thu, 9 Dec 2021 15:25:35 +0100 Subject: [PATCH 2/3] Move script next to _freeze_module.c --- Makefile.pre.in | 59 ++++++++++--------- .../_freeze_module.py | 2 +- Tools/scripts/deepfreeze.py | 2 +- Tools/scripts/freeze_modules.py | 4 +- 4 files changed, 34 insertions(+), 33 deletions(-) rename Tools/scripts/freeze_module.py => Programs/_freeze_module.py (96%) diff --git a/Makefile.pre.in b/Makefile.pre.in index db11892b0f3f80..4eff3ab415f193 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -287,7 +287,7 @@ PYTHON_FOR_BUILD=@PYTHON_FOR_BUILD@ # uses build Python, which must have the same version and bytecode, PYTHON_FOR_FREEZE?=@PYTHON_FOR_FREEZE@ FREEZE_MODULE_C=Programs/_freeze_module -FREEZE_MODULE_PY=$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/freeze_module.py +FREEZE_MODULE_PY=$(PYTHON_FOR_FREEZE) $(srcdir)/Programs/_freeze_module.py # bootstrap freezing uses FREEZE_MODULE_C for normal builds and _PY for cross builds FREEZE_MODULE_BOOTSTRAP=@FREEZE_MODULE_BOOTSTRAP@ @@ -1061,13 +1061,14 @@ Python/deepfreeze/frozen_only.c: Python/frozen_modules/frozen_only.h $(DEEPFREEZ # 2) build Programs/_freeze_module binary. # 3) create frozen module headers for importlib and getpath. # 4) build _bootstrap_python binary. -# 5) create remaining frozen module headers with _bootstrap_python and -# Python freeze_module.py (so we test this code path). +# 5) create remaining frozen module headers with +# ``./_bootstrap_python Programs/_freeze_module.py``. The pure Python +# script is used to test the cross compile code path. # 6) deepfreeze modules with _bootstrap_python # # Cross compile proces: # 1) create all frozen module headers with external build Python and -# freeze_module.py script. +# Programs/_freeze_module.py script. # 2) deepfreeze modules with external build Python. # # Cross compiling also creates Programs/_freeze_module and _bootstrap_python @@ -1130,82 +1131,82 @@ Programs/_freeze_module.o: Programs/_freeze_module.c Makefile Modules/getpath_noop.o: $(srcdir)/Modules/getpath_noop.c Makefile -Programs/_freeze_module: Programs/_freeze_module.o Modules/getpath_noop.o $(LIBRARY_OBJS_OMIT_FROZEN) +$(FREEZE_MODULE_C): Programs/_freeze_module.o Modules/getpath_noop.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LINKCC) $(PY_CORE_LDFLAGS) -o $@ Programs/_freeze_module.o Modules/getpath_noop.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LIBS) $(MODLIBS) $(SYSLIBS) # We manually freeze getpath.py rather than through freeze_modules -Python/frozen_modules/getpath.h: $(FREEZE_MODULE_C) Modules/getpath.py +Python/frozen_modules/getpath.h: Modules/getpath.py $(FREEZE_MODULE_C) $(FREEZE_MODULE_BOOTSTRAP) getpath $(srcdir)/Modules/getpath.py Python/frozen_modules/getpath.h # BEGIN: freezing modules -Python/frozen_modules/importlib._bootstrap.h: Lib/importlib/_bootstrap.py $(FREEZE_MODULE_C) $(srcdir)/Tools/scripts/freeze_module.py +Python/frozen_modules/importlib._bootstrap.h: Lib/importlib/_bootstrap.py $(FREEZE_MODULE_C) $(FREEZE_MODULE_BOOTSTRAP) importlib._bootstrap $(srcdir)/Lib/importlib/_bootstrap.py Python/frozen_modules/importlib._bootstrap.h -Python/frozen_modules/importlib._bootstrap_external.h: Lib/importlib/_bootstrap_external.py $(FREEZE_MODULE_C) $(srcdir)/Tools/scripts/freeze_module.py +Python/frozen_modules/importlib._bootstrap_external.h: Lib/importlib/_bootstrap_external.py $(FREEZE_MODULE_C) $(FREEZE_MODULE_BOOTSTRAP) importlib._bootstrap_external $(srcdir)/Lib/importlib/_bootstrap_external.py Python/frozen_modules/importlib._bootstrap_external.h -Python/frozen_modules/zipimport.h: Lib/zipimport.py $(FREEZE_MODULE_C) $(srcdir)/Tools/scripts/freeze_module.py +Python/frozen_modules/zipimport.h: Lib/zipimport.py $(FREEZE_MODULE_C) $(FREEZE_MODULE_BOOTSTRAP) zipimport $(srcdir)/Lib/zipimport.py Python/frozen_modules/zipimport.h -Python/frozen_modules/abc.h: Lib/abc.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py +Python/frozen_modules/abc.h: Lib/abc.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py $(FREEZE_MODULE_PY) abc $(srcdir)/Lib/abc.py Python/frozen_modules/abc.h -Python/frozen_modules/codecs.h: Lib/codecs.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py +Python/frozen_modules/codecs.h: Lib/codecs.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py $(FREEZE_MODULE_PY) codecs $(srcdir)/Lib/codecs.py Python/frozen_modules/codecs.h -Python/frozen_modules/io.h: Lib/io.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py +Python/frozen_modules/io.h: Lib/io.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py $(FREEZE_MODULE_PY) io $(srcdir)/Lib/io.py Python/frozen_modules/io.h -Python/frozen_modules/_collections_abc.h: Lib/_collections_abc.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py +Python/frozen_modules/_collections_abc.h: Lib/_collections_abc.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py $(FREEZE_MODULE_PY) _collections_abc $(srcdir)/Lib/_collections_abc.py Python/frozen_modules/_collections_abc.h -Python/frozen_modules/_sitebuiltins.h: Lib/_sitebuiltins.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py +Python/frozen_modules/_sitebuiltins.h: Lib/_sitebuiltins.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py $(FREEZE_MODULE_PY) _sitebuiltins $(srcdir)/Lib/_sitebuiltins.py Python/frozen_modules/_sitebuiltins.h -Python/frozen_modules/genericpath.h: Lib/genericpath.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py +Python/frozen_modules/genericpath.h: Lib/genericpath.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py $(FREEZE_MODULE_PY) genericpath $(srcdir)/Lib/genericpath.py Python/frozen_modules/genericpath.h -Python/frozen_modules/ntpath.h: Lib/ntpath.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py +Python/frozen_modules/ntpath.h: Lib/ntpath.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py $(FREEZE_MODULE_PY) ntpath $(srcdir)/Lib/ntpath.py Python/frozen_modules/ntpath.h -Python/frozen_modules/posixpath.h: Lib/posixpath.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py +Python/frozen_modules/posixpath.h: Lib/posixpath.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py $(FREEZE_MODULE_PY) posixpath $(srcdir)/Lib/posixpath.py Python/frozen_modules/posixpath.h -Python/frozen_modules/os.h: Lib/os.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py +Python/frozen_modules/os.h: Lib/os.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py $(FREEZE_MODULE_PY) os $(srcdir)/Lib/os.py Python/frozen_modules/os.h -Python/frozen_modules/site.h: Lib/site.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py +Python/frozen_modules/site.h: Lib/site.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py $(FREEZE_MODULE_PY) site $(srcdir)/Lib/site.py Python/frozen_modules/site.h -Python/frozen_modules/stat.h: Lib/stat.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py +Python/frozen_modules/stat.h: Lib/stat.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py $(FREEZE_MODULE_PY) stat $(srcdir)/Lib/stat.py Python/frozen_modules/stat.h -Python/frozen_modules/importlib.util.h: Lib/importlib/util.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py +Python/frozen_modules/importlib.util.h: Lib/importlib/util.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py $(FREEZE_MODULE_PY) importlib.util $(srcdir)/Lib/importlib/util.py Python/frozen_modules/importlib.util.h -Python/frozen_modules/importlib.machinery.h: Lib/importlib/machinery.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py +Python/frozen_modules/importlib.machinery.h: Lib/importlib/machinery.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py $(FREEZE_MODULE_PY) importlib.machinery $(srcdir)/Lib/importlib/machinery.py Python/frozen_modules/importlib.machinery.h -Python/frozen_modules/runpy.h: Lib/runpy.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py +Python/frozen_modules/runpy.h: Lib/runpy.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py $(FREEZE_MODULE_PY) runpy $(srcdir)/Lib/runpy.py Python/frozen_modules/runpy.h -Python/frozen_modules/__hello__.h: Lib/__hello__.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py +Python/frozen_modules/__hello__.h: Lib/__hello__.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py $(FREEZE_MODULE_PY) __hello__ $(srcdir)/Lib/__hello__.py Python/frozen_modules/__hello__.h -Python/frozen_modules/__phello__.h: Lib/__phello__/__init__.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py +Python/frozen_modules/__phello__.h: Lib/__phello__/__init__.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py $(FREEZE_MODULE_PY) __phello__ $(srcdir)/Lib/__phello__/__init__.py Python/frozen_modules/__phello__.h -Python/frozen_modules/__phello__.ham.h: Lib/__phello__/ham/__init__.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py +Python/frozen_modules/__phello__.ham.h: Lib/__phello__/ham/__init__.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py $(FREEZE_MODULE_PY) __phello__.ham $(srcdir)/Lib/__phello__/ham/__init__.py Python/frozen_modules/__phello__.ham.h -Python/frozen_modules/__phello__.ham.eggs.h: Lib/__phello__/ham/eggs.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py +Python/frozen_modules/__phello__.ham.eggs.h: Lib/__phello__/ham/eggs.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py $(FREEZE_MODULE_PY) __phello__.ham.eggs $(srcdir)/Lib/__phello__/ham/eggs.py Python/frozen_modules/__phello__.ham.eggs.h -Python/frozen_modules/__phello__.spam.h: Lib/__phello__/spam.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py +Python/frozen_modules/__phello__.spam.h: Lib/__phello__/spam.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py $(FREEZE_MODULE_PY) __phello__.spam $(srcdir)/Lib/__phello__/spam.py Python/frozen_modules/__phello__.spam.h -Python/frozen_modules/frozen_only.h: Tools/freeze/flag.py _bootstrap_python $(srcdir)/Tools/scripts/freeze_module.py +Python/frozen_modules/frozen_only.h: Tools/freeze/flag.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py $(FREEZE_MODULE_PY) frozen_only $(srcdir)/Tools/freeze/flag.py Python/frozen_modules/frozen_only.h # END: freezing modules diff --git a/Tools/scripts/freeze_module.py b/Programs/_freeze_module.py similarity index 96% rename from Tools/scripts/freeze_module.py rename to Programs/_freeze_module.py index 6b5ad35708f788..cc97e4195bfa2e 100644 --- a/Tools/scripts/freeze_module.py +++ b/Programs/_freeze_module.py @@ -12,7 +12,7 @@ import marshal import sys -header = "/* Auto-generated by Tools/scripts/freeze_module.py */" +header = "/* Auto-generated by Programs/_freeze_module.py */" def read_text(inpath: str) -> bytes: diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index 7cd7eb674d4511..002d680e10c2f6 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -394,7 +394,7 @@ def generate(self, name: str, obj: object) -> str: """ FROZEN_COMMENT_C = "/* Auto-generated by Programs/_freeze_module.c */" -FROZEN_COMMENT_PY = "/* Auto-generated by Tools/scripts/freeze_module.py */" +FROZEN_COMMENT_PY = "/* Auto-generated by Programs/_freeze_module.py */" FROZEN_DATA_LINE = r"\s*(\d+,\s*)+\s*" diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index 05d59cc6bcd452..f68a0932142bd9 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -594,12 +594,12 @@ def regen_makefile(modules): freezedep = '$(FREEZE_MODULE_C)' else: freezecmd = '$(FREEZE_MODULE_PY)' - freezedep = '_bootstrap_python' + freezedep = '_bootstrap_python $(srcdir)/Programs/_freeze_module.py' freeze = (f'{freezecmd} {src.frozenid} ' f'$(srcdir)/{pyfile} {frozen_header}') rules.extend([ - f'{frozen_header}: {pyfile} {freezedep} $(srcdir)/Tools/scripts/freeze_module.py', + f'{frozen_header}: {pyfile} {freezedep}', f'\t{freeze}', '', ]) From 0d328a9377a7eba677e5ae38f812bd51ad18a835 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Sat, 11 Dec 2021 15:14:17 +0100 Subject: [PATCH 3/3] Address review comments - Fix typo - Remove confusing comment - Improve dependency handling for freezing --- Makefile.pre.in | 111 ++++++++++++++++---------------- Programs/_freeze_module.c | 2 + Programs/_freeze_module.py | 1 - Tools/scripts/freeze_modules.py | 6 +- configure | 20 +++++- configure.ac | 19 +++++- 6 files changed, 95 insertions(+), 64 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 4eff3ab415f193..fb114979a5dff9 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -283,13 +283,15 @@ BUILDPYTHON= python$(BUILDEXE) PYTHON_FOR_REGEN?=@PYTHON_FOR_REGEN@ UPDATE_FILE=$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/update_file.py PYTHON_FOR_BUILD=@PYTHON_FOR_BUILD@ -# Standard builds use _bootstrap_python for freezing, cross compiling -# uses build Python, which must have the same version and bytecode, -PYTHON_FOR_FREEZE?=@PYTHON_FOR_FREEZE@ -FREEZE_MODULE_C=Programs/_freeze_module -FREEZE_MODULE_PY=$(PYTHON_FOR_FREEZE) $(srcdir)/Programs/_freeze_module.py -# bootstrap freezing uses FREEZE_MODULE_C for normal builds and _PY for cross builds + +# Normal builds use Programs/_freeze_module.c for bootstrapping and +# ./_bootstrap_python Programs/_freeze_module.py for remaining modules +# Cross builds use an external "build Python" for all modules. +PYTHON_FOR_FREEZE=@PYTHON_FOR_FREEZE@ FREEZE_MODULE_BOOTSTRAP=@FREEZE_MODULE_BOOTSTRAP@ +FREEZE_MODULE_BOOTSTRAP_DEPS=@FREEZE_MODULE_BOOTSTRAP_DEPS@ +FREEZE_MODULE=@FREEZE_MODULE@ +FREEZE_MODULE_DEPS=@FREEZE_MODULE_DEPS@ _PYTHON_HOST_PLATFORM=@_PYTHON_HOST_PLATFORM@ BUILD_GNU_TYPE= @build@ @@ -970,7 +972,7 @@ _bootstrap_python: $(LIBRARY_OBJS_OMIT_FROZEN) Programs/_bootstrap_python.o Modu .PHONY: regen-deepfreeze regen-deepfreeze: $(DEEPFREEZE_OBJS) -DEEPFREEZE_DEPS=$(srcdir)/Tools/scripts/deepfreeze.py _bootstrap_python +DEEPFREEZE_DEPS=$(srcdir)/Tools/scripts/deepfreeze.py $(FREEZE_MODULE_DEPS) # BEGIN: deepfreeze modules @@ -1066,14 +1068,11 @@ Python/deepfreeze/frozen_only.c: Python/frozen_modules/frozen_only.h $(DEEPFREEZ # script is used to test the cross compile code path. # 6) deepfreeze modules with _bootstrap_python # -# Cross compile proces: +# Cross compile process: # 1) create all frozen module headers with external build Python and # Programs/_freeze_module.py script. # 2) deepfreeze modules with external build Python. # -# Cross compiling also creates Programs/_freeze_module and _bootstrap_python -# artifacts, but does not use or ship them. -# # FROZEN_FILES_* are auto-generated by Tools/scripts/freeze_modules.py. FROZEN_FILES_IN = \ @@ -1131,83 +1130,83 @@ Programs/_freeze_module.o: Programs/_freeze_module.c Makefile Modules/getpath_noop.o: $(srcdir)/Modules/getpath_noop.c Makefile -$(FREEZE_MODULE_C): Programs/_freeze_module.o Modules/getpath_noop.o $(LIBRARY_OBJS_OMIT_FROZEN) +Programs/_freeze_module: Programs/_freeze_module.o Modules/getpath_noop.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LINKCC) $(PY_CORE_LDFLAGS) -o $@ Programs/_freeze_module.o Modules/getpath_noop.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LIBS) $(MODLIBS) $(SYSLIBS) # We manually freeze getpath.py rather than through freeze_modules -Python/frozen_modules/getpath.h: Modules/getpath.py $(FREEZE_MODULE_C) +Python/frozen_modules/getpath.h: Modules/getpath.py $(FREEZE_MODULE_BOOTSTRAP_DEPS) $(FREEZE_MODULE_BOOTSTRAP) getpath $(srcdir)/Modules/getpath.py Python/frozen_modules/getpath.h # BEGIN: freezing modules -Python/frozen_modules/importlib._bootstrap.h: Lib/importlib/_bootstrap.py $(FREEZE_MODULE_C) +Python/frozen_modules/importlib._bootstrap.h: Lib/importlib/_bootstrap.py $(FREEZE_MODULE_BOOTSTRAP_DEPS) $(FREEZE_MODULE_BOOTSTRAP) importlib._bootstrap $(srcdir)/Lib/importlib/_bootstrap.py Python/frozen_modules/importlib._bootstrap.h -Python/frozen_modules/importlib._bootstrap_external.h: Lib/importlib/_bootstrap_external.py $(FREEZE_MODULE_C) +Python/frozen_modules/importlib._bootstrap_external.h: Lib/importlib/_bootstrap_external.py $(FREEZE_MODULE_BOOTSTRAP_DEPS) $(FREEZE_MODULE_BOOTSTRAP) importlib._bootstrap_external $(srcdir)/Lib/importlib/_bootstrap_external.py Python/frozen_modules/importlib._bootstrap_external.h -Python/frozen_modules/zipimport.h: Lib/zipimport.py $(FREEZE_MODULE_C) +Python/frozen_modules/zipimport.h: Lib/zipimport.py $(FREEZE_MODULE_BOOTSTRAP_DEPS) $(FREEZE_MODULE_BOOTSTRAP) zipimport $(srcdir)/Lib/zipimport.py Python/frozen_modules/zipimport.h -Python/frozen_modules/abc.h: Lib/abc.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py - $(FREEZE_MODULE_PY) abc $(srcdir)/Lib/abc.py Python/frozen_modules/abc.h +Python/frozen_modules/abc.h: Lib/abc.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) abc $(srcdir)/Lib/abc.py Python/frozen_modules/abc.h -Python/frozen_modules/codecs.h: Lib/codecs.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py - $(FREEZE_MODULE_PY) codecs $(srcdir)/Lib/codecs.py Python/frozen_modules/codecs.h +Python/frozen_modules/codecs.h: Lib/codecs.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) codecs $(srcdir)/Lib/codecs.py Python/frozen_modules/codecs.h -Python/frozen_modules/io.h: Lib/io.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py - $(FREEZE_MODULE_PY) io $(srcdir)/Lib/io.py Python/frozen_modules/io.h +Python/frozen_modules/io.h: Lib/io.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) io $(srcdir)/Lib/io.py Python/frozen_modules/io.h -Python/frozen_modules/_collections_abc.h: Lib/_collections_abc.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py - $(FREEZE_MODULE_PY) _collections_abc $(srcdir)/Lib/_collections_abc.py Python/frozen_modules/_collections_abc.h +Python/frozen_modules/_collections_abc.h: Lib/_collections_abc.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) _collections_abc $(srcdir)/Lib/_collections_abc.py Python/frozen_modules/_collections_abc.h -Python/frozen_modules/_sitebuiltins.h: Lib/_sitebuiltins.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py - $(FREEZE_MODULE_PY) _sitebuiltins $(srcdir)/Lib/_sitebuiltins.py Python/frozen_modules/_sitebuiltins.h +Python/frozen_modules/_sitebuiltins.h: Lib/_sitebuiltins.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) _sitebuiltins $(srcdir)/Lib/_sitebuiltins.py Python/frozen_modules/_sitebuiltins.h -Python/frozen_modules/genericpath.h: Lib/genericpath.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py - $(FREEZE_MODULE_PY) genericpath $(srcdir)/Lib/genericpath.py Python/frozen_modules/genericpath.h +Python/frozen_modules/genericpath.h: Lib/genericpath.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) genericpath $(srcdir)/Lib/genericpath.py Python/frozen_modules/genericpath.h -Python/frozen_modules/ntpath.h: Lib/ntpath.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py - $(FREEZE_MODULE_PY) ntpath $(srcdir)/Lib/ntpath.py Python/frozen_modules/ntpath.h +Python/frozen_modules/ntpath.h: Lib/ntpath.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) ntpath $(srcdir)/Lib/ntpath.py Python/frozen_modules/ntpath.h -Python/frozen_modules/posixpath.h: Lib/posixpath.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py - $(FREEZE_MODULE_PY) posixpath $(srcdir)/Lib/posixpath.py Python/frozen_modules/posixpath.h +Python/frozen_modules/posixpath.h: Lib/posixpath.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) posixpath $(srcdir)/Lib/posixpath.py Python/frozen_modules/posixpath.h -Python/frozen_modules/os.h: Lib/os.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py - $(FREEZE_MODULE_PY) os $(srcdir)/Lib/os.py Python/frozen_modules/os.h +Python/frozen_modules/os.h: Lib/os.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) os $(srcdir)/Lib/os.py Python/frozen_modules/os.h -Python/frozen_modules/site.h: Lib/site.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py - $(FREEZE_MODULE_PY) site $(srcdir)/Lib/site.py Python/frozen_modules/site.h +Python/frozen_modules/site.h: Lib/site.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) site $(srcdir)/Lib/site.py Python/frozen_modules/site.h -Python/frozen_modules/stat.h: Lib/stat.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py - $(FREEZE_MODULE_PY) stat $(srcdir)/Lib/stat.py Python/frozen_modules/stat.h +Python/frozen_modules/stat.h: Lib/stat.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) stat $(srcdir)/Lib/stat.py Python/frozen_modules/stat.h -Python/frozen_modules/importlib.util.h: Lib/importlib/util.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py - $(FREEZE_MODULE_PY) importlib.util $(srcdir)/Lib/importlib/util.py Python/frozen_modules/importlib.util.h +Python/frozen_modules/importlib.util.h: Lib/importlib/util.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) importlib.util $(srcdir)/Lib/importlib/util.py Python/frozen_modules/importlib.util.h -Python/frozen_modules/importlib.machinery.h: Lib/importlib/machinery.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py - $(FREEZE_MODULE_PY) importlib.machinery $(srcdir)/Lib/importlib/machinery.py Python/frozen_modules/importlib.machinery.h +Python/frozen_modules/importlib.machinery.h: Lib/importlib/machinery.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) importlib.machinery $(srcdir)/Lib/importlib/machinery.py Python/frozen_modules/importlib.machinery.h -Python/frozen_modules/runpy.h: Lib/runpy.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py - $(FREEZE_MODULE_PY) runpy $(srcdir)/Lib/runpy.py Python/frozen_modules/runpy.h +Python/frozen_modules/runpy.h: Lib/runpy.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) runpy $(srcdir)/Lib/runpy.py Python/frozen_modules/runpy.h -Python/frozen_modules/__hello__.h: Lib/__hello__.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py - $(FREEZE_MODULE_PY) __hello__ $(srcdir)/Lib/__hello__.py Python/frozen_modules/__hello__.h +Python/frozen_modules/__hello__.h: Lib/__hello__.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) __hello__ $(srcdir)/Lib/__hello__.py Python/frozen_modules/__hello__.h -Python/frozen_modules/__phello__.h: Lib/__phello__/__init__.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py - $(FREEZE_MODULE_PY) __phello__ $(srcdir)/Lib/__phello__/__init__.py Python/frozen_modules/__phello__.h +Python/frozen_modules/__phello__.h: Lib/__phello__/__init__.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) __phello__ $(srcdir)/Lib/__phello__/__init__.py Python/frozen_modules/__phello__.h -Python/frozen_modules/__phello__.ham.h: Lib/__phello__/ham/__init__.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py - $(FREEZE_MODULE_PY) __phello__.ham $(srcdir)/Lib/__phello__/ham/__init__.py Python/frozen_modules/__phello__.ham.h +Python/frozen_modules/__phello__.ham.h: Lib/__phello__/ham/__init__.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) __phello__.ham $(srcdir)/Lib/__phello__/ham/__init__.py Python/frozen_modules/__phello__.ham.h -Python/frozen_modules/__phello__.ham.eggs.h: Lib/__phello__/ham/eggs.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py - $(FREEZE_MODULE_PY) __phello__.ham.eggs $(srcdir)/Lib/__phello__/ham/eggs.py Python/frozen_modules/__phello__.ham.eggs.h +Python/frozen_modules/__phello__.ham.eggs.h: Lib/__phello__/ham/eggs.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) __phello__.ham.eggs $(srcdir)/Lib/__phello__/ham/eggs.py Python/frozen_modules/__phello__.ham.eggs.h -Python/frozen_modules/__phello__.spam.h: Lib/__phello__/spam.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py - $(FREEZE_MODULE_PY) __phello__.spam $(srcdir)/Lib/__phello__/spam.py Python/frozen_modules/__phello__.spam.h +Python/frozen_modules/__phello__.spam.h: Lib/__phello__/spam.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) __phello__.spam $(srcdir)/Lib/__phello__/spam.py Python/frozen_modules/__phello__.spam.h -Python/frozen_modules/frozen_only.h: Tools/freeze/flag.py _bootstrap_python $(srcdir)/Programs/_freeze_module.py - $(FREEZE_MODULE_PY) frozen_only $(srcdir)/Tools/freeze/flag.py Python/frozen_modules/frozen_only.h +Python/frozen_modules/frozen_only.h: Tools/freeze/flag.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) frozen_only $(srcdir)/Tools/freeze/flag.py Python/frozen_modules/frozen_only.h # END: freezing modules diff --git a/Programs/_freeze_module.c b/Programs/_freeze_module.c index e3f6c11c8b0731..d50787666f81cc 100644 --- a/Programs/_freeze_module.c +++ b/Programs/_freeze_module.c @@ -5,6 +5,8 @@ This is used directly by Tools/scripts/freeze_modules.py, and indirectly by "make regen-frozen". See Python/frozen.c for more info. + + Keep this file in sync with Programs/_freeze_module.py. */ #include diff --git a/Programs/_freeze_module.py b/Programs/_freeze_module.py index cc97e4195bfa2e..ba638eef6c4cd6 100644 --- a/Programs/_freeze_module.py +++ b/Programs/_freeze_module.py @@ -24,7 +24,6 @@ def compile_and_marshal(name: str, text: bytes) -> bytes: filename = f"" # exec == Py_file_input code = compile(text, filename, "exec", optimize=0, dont_inherit=True) - # defaults to Py_MARSHAL_VERSION) return marshal.dumps(code) diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index f68a0932142bd9..cbe8bf1ce60cdc 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -591,10 +591,10 @@ def regen_makefile(modules): if src.isbootstrap: freezecmd = '$(FREEZE_MODULE_BOOTSTRAP)' - freezedep = '$(FREEZE_MODULE_C)' + freezedep = '$(FREEZE_MODULE_BOOTSTRAP_DEPS)' else: - freezecmd = '$(FREEZE_MODULE_PY)' - freezedep = '_bootstrap_python $(srcdir)/Programs/_freeze_module.py' + freezecmd = '$(FREEZE_MODULE)' + freezedep = '$(FREEZE_MODULE_DEPS)' freeze = (f'{freezecmd} {src.frozenid} ' f'$(srcdir)/{pyfile} {frozen_header}') diff --git a/configure b/configure index e3ad4a8875c5dd..583e7d1fe3b32c 100755 --- a/configure +++ b/configure @@ -933,6 +933,9 @@ CONFIG_ARGS SOVERSION VERSION PYTHON_FOR_REGEN +FREEZE_MODULE_DEPS +FREEZE_MODULE +FREEZE_MODULE_BOOTSTRAP_DEPS FREEZE_MODULE_BOOTSTRAP PYTHON_FOR_FREEZE PYTHON_FOR_BUILD @@ -3240,13 +3243,26 @@ $as_echo "$PYTHON_FOR_FREEZE" >&6; } if test "x$cross_compiling" = xyes; then : - FREEZE_MODULE_BOOTSTRAP='$(FREEZE_MODULE_PY)' + + FREEZE_MODULE_BOOTSTRAP='$(PYTHON_FOR_FREEZE) $(srcdir)/Programs/_freeze_module.py' + FREEZE_MODULE_BOOTSTRAP_DEPS='$(srcdir)/Programs/_freeze_module.py' + FREEZE_MODULE='$(FREEZE_MODULE_BOOTSTRAP)' + FREEZE_MODULE_DEPS='$(FREEZE_MODULE_BOOTSTRAP_DEPS)' + else - FREEZE_MODULE_BOOTSTRAP='$(FREEZE_MODULE_C)' + + FREEZE_MODULE_BOOTSTRAP='./Programs/_freeze_module' + FREEZE_MODULE_BOOTSTRAP_DEPS="Programs/_freeze_module" + FREEZE_MODULE='$(PYTHON_FOR_FREEZE) $(srcdir)/Programs/_freeze_module.py' + FREEZE_MODULE_DEPS="_bootstrap_python \$(srcdir)/Programs/_freeze_module.py" + fi + + + for ac_prog in python$PACKAGE_VERSION python3.10 python3.9 python3.8 python3.7 python3.6 python3 python do # Extract the first word of "$ac_prog", so it can be a program name with args. diff --git a/configure.ac b/configure.ac index f5e6ea46b455ee..5256a61289e223 100644 --- a/configure.ac +++ b/configure.ac @@ -137,10 +137,25 @@ AC_MSG_RESULT([$PYTHON_FOR_FREEZE]) AC_SUBST([PYTHON_FOR_FREEZE]) AS_VAR_IF([cross_compiling], [yes], - [FREEZE_MODULE_BOOTSTRAP='$(FREEZE_MODULE_PY)'], - [FREEZE_MODULE_BOOTSTRAP='$(FREEZE_MODULE_C)'] + [ + dnl external build Python, freezing depends on Programs/_freeze_module.py + FREEZE_MODULE_BOOTSTRAP='$(PYTHON_FOR_FREEZE) $(srcdir)/Programs/_freeze_module.py' + FREEZE_MODULE_BOOTSTRAP_DEPS='$(srcdir)/Programs/_freeze_module.py' + FREEZE_MODULE='$(FREEZE_MODULE_BOOTSTRAP)' + FREEZE_MODULE_DEPS='$(FREEZE_MODULE_BOOTSTRAP_DEPS)' + ], + [ + dnl internal build tools also depend on Programs/_freeze_module and _bootstrap_python. + FREEZE_MODULE_BOOTSTRAP='./Programs/_freeze_module' + FREEZE_MODULE_BOOTSTRAP_DEPS="Programs/_freeze_module" + FREEZE_MODULE='$(PYTHON_FOR_FREEZE) $(srcdir)/Programs/_freeze_module.py' + FREEZE_MODULE_DEPS="_bootstrap_python \$(srcdir)/Programs/_freeze_module.py" + ] ) AC_SUBST([FREEZE_MODULE_BOOTSTRAP]) +AC_SUBST([FREEZE_MODULE_BOOTSTRAP_DEPS]) +AC_SUBST([FREEZE_MODULE]) +AC_SUBST([FREEZE_MODULE_DEPS]) AC_CHECK_PROGS([PYTHON_FOR_REGEN], [python$PACKAGE_VERSION python3.10 python3.9 python3.8 python3.7 python3.6 python3 python], pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy