Skip to content

Commit 84ca123

Browse files
tirantiran
authored
bpo-45950: Introduce Bootstrap Python again (#29859)
The build system now uses a :program:`_bootstrap_python` interpreter for freezing and deepfreezing again. To speed up build process the build tools :program:`_bootstrap_python` and :program:`_freeze_module` are no longer build with LTO. Cross building depends on a build Python interpreter, which must have same version and bytecode as target host Python.
1 parent ccb73a0 commit 84ca123

File tree

9 files changed

+185
-71
lines changed

9 files changed

+185
-71
lines changed

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ Lib/distutils/command/*.pdb
6060
Lib/lib2to3/*.pickle
6161
Lib/test/data/*
6262
!Lib/test/data/README
63+
/_bootstrap_python
6364
/Makefile
6465
/Makefile.pre
6566
Mac/Makefile

‎Makefile.pre.in

Lines changed: 42 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,9 @@ CONFIGURE_CFLAGS_NODIST=@CFLAGS_NODIST@
9090
# Use it when a linker flag should _not_ be part of the distutils LDFLAGS
9191
# once Python is installed (bpo-35257)
9292
CONFIGURE_LDFLAGS_NODIST=@LDFLAGS_NODIST@
93+
# LDFLAGS_NOLTO is an extra flag to disable lto. It is used to speed up building
94+
# of _bootstrap_python and _freeze_module tools, which don't need LTO.
95+
CONFIGURE_LDFLAGS_NOLTO=@LDFLAGS_NOLTO@
9396
CONFIGURE_CPPFLAGS= @CPPFLAGS@
9497
CONFIGURE_LDFLAGS= @LDFLAGS@
9598
# Avoid assigning CFLAGS, LDFLAGS, etc. so users can use them on the
@@ -103,6 +106,7 @@ PY_CFLAGS_NODIST=$(CONFIGURE_CFLAGS_NODIST) $(CFLAGS_NODIST) -I$(srcdir)/Include
103106
PY_CPPFLAGS= $(BASECPPFLAGS) -I. -I$(srcdir)/Include $(CONFIGURE_CPPFLAGS) $(CPPFLAGS)
104107
PY_LDFLAGS= $(CONFIGURE_LDFLAGS) $(LDFLAGS)
105108
PY_LDFLAGS_NODIST=$(CONFIGURE_LDFLAGS_NODIST) $(LDFLAGS_NODIST)
109+
PY_LDFLAGS_NOLTO=$(PY_LDFLAGS) $(CONFIGURE_LDFLAGS_NOLTO) $(LDFLAGS_NODIST)
106110
NO_AS_NEEDED= @NO_AS_NEEDED@
107111
CCSHARED= @CCSHARED@
108112
# LINKFORSHARED are the flags passed to the $(CC) command that links
@@ -279,6 +283,9 @@ BUILDPYTHON= python$(BUILDEXE)
279283
PYTHON_FOR_REGEN?=@PYTHON_FOR_REGEN@
280284
UPDATE_FILE=$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/update_file.py
281285
PYTHON_FOR_BUILD=@PYTHON_FOR_BUILD@
286+
# Standard builds use _bootstrap_python for freezing, cross compiling
287+
# uses build Python, which must have the same version and bytecode,
288+
PYTHON_FOR_FREEZE?=@PYTHON_FOR_FREEZE@
282289
_PYTHON_HOST_PLATFORM=@_PYTHON_HOST_PLATFORM@
283290
BUILD_GNU_TYPE= @build@
284291
HOST_GNU_TYPE= @host@
@@ -938,75 +945,88 @@ regen-test-frozenmain: $(BUILDPYTHON)
938945
Programs/_testembed: Programs/_testembed.o $(LIBRARY_DEPS)
939946
$(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/_testembed.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS)
940947

948+
############################################################################
949+
# "Bootstrap Python" used to run deepfreeze.py
950+
951+
BOOTSTRAP_HEADERS = \
952+
Python/frozen_modules/importlib._bootstrap.h \
953+
Python/frozen_modules/importlib._bootstrap_external.h
954+
955+
Programs/_bootstrap_python.o: Programs/_bootstrap_python.c $(BOOTSTRAP_HEADERS) $(PYTHON_HEADERS)
956+
957+
_bootstrap_python: $(LIBRARY_OBJS_OMIT_FROZEN) Programs/_bootstrap_python.o Modules/getpath.o Modules/Setup.local
958+
$(LINKCC) $(PY_LDFLAGS_NOLTO) -o $@ $(LIBRARY_OBJS_OMIT_FROZEN) \
959+
Programs/_bootstrap_python.o Modules/getpath.o $(LIBS) $(MODLIBS) $(SYSLIBS)
960+
941961
############################################################################
942962
# Deepfreeze targets
943963

944964
.PHONY: regen-deepfreeze
945965
regen-deepfreeze: $(DEEPFREEZE_OBJS)
946966

947-
DEEPFREEZE_DEPS=$(srcdir)/Tools/scripts/deepfreeze.py
967+
DEEPFREEZE_DEPS=$(srcdir)/Tools/scripts/deepfreeze.py _bootstrap_python
948968

949969
# BEGIN: deepfreeze modules
950970

951971
Python/deepfreeze/importlib._bootstrap.c: Python/frozen_modules/importlib._bootstrap.h $(DEEPFREEZE_DEPS)
952-
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/importlib._bootstrap.h -m importlib._bootstrap -o Python/deepfreeze/importlib._bootstrap.c
972+
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/importlib._bootstrap.h -m importlib._bootstrap -o Python/deepfreeze/importlib._bootstrap.c
953973

954974
Python/deepfreeze/importlib._bootstrap_external.c: Python/frozen_modules/importlib._bootstrap_external.h $(DEEPFREEZE_DEPS)
955-
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/importlib._bootstrap_external.h -m importlib._bootstrap_external -o Python/deepfreeze/importlib._bootstrap_external.c
975+
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/importlib._bootstrap_external.h -m importlib._bootstrap_external -o Python/deepfreeze/importlib._bootstrap_external.c
956976

957977
Python/deepfreeze/zipimport.c: Python/frozen_modules/zipimport.h $(DEEPFREEZE_DEPS)
958-
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/zipimport.h -m zipimport -o Python/deepfreeze/zipimport.c
978+
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/zipimport.h -m zipimport -o Python/deepfreeze/zipimport.c
959979

960980
Python/deepfreeze/abc.c: Python/frozen_modules/abc.h $(DEEPFREEZE_DEPS)
961-
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/abc.h -m abc -o Python/deepfreeze/abc.c
981+
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/abc.h -m abc -o Python/deepfreeze/abc.c
962982

963983
Python/deepfreeze/codecs.c: Python/frozen_modules/codecs.h $(DEEPFREEZE_DEPS)
964-
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/codecs.h -m codecs -o Python/deepfreeze/codecs.c
984+
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/codecs.h -m codecs -o Python/deepfreeze/codecs.c
965985

966986
Python/deepfreeze/io.c: Python/frozen_modules/io.h $(DEEPFREEZE_DEPS)
967-
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/io.h -m io -o Python/deepfreeze/io.c
987+
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/io.h -m io -o Python/deepfreeze/io.c
968988

969989
Python/deepfreeze/_collections_abc.c: Python/frozen_modules/_collections_abc.h $(DEEPFREEZE_DEPS)
970-
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/_collections_abc.h -m _collections_abc -o Python/deepfreeze/_collections_abc.c
990+
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/_collections_abc.h -m _collections_abc -o Python/deepfreeze/_collections_abc.c
971991

972992
Python/deepfreeze/_sitebuiltins.c: Python/frozen_modules/_sitebuiltins.h $(DEEPFREEZE_DEPS)
973-
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/_sitebuiltins.h -m _sitebuiltins -o Python/deepfreeze/_sitebuiltins.c
993+
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/_sitebuiltins.h -m _sitebuiltins -o Python/deepfreeze/_sitebuiltins.c
974994

975995
Python/deepfreeze/genericpath.c: Python/frozen_modules/genericpath.h $(DEEPFREEZE_DEPS)
976-
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/genericpath.h -m genericpath -o Python/deepfreeze/genericpath.c
996+
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/genericpath.h -m genericpath -o Python/deepfreeze/genericpath.c
977997

978998
Python/deepfreeze/ntpath.c: Python/frozen_modules/ntpath.h $(DEEPFREEZE_DEPS)
979-
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/ntpath.h -m ntpath -o Python/deepfreeze/ntpath.c
999+
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/ntpath.h -m ntpath -o Python/deepfreeze/ntpath.c
9801000

9811001
Python/deepfreeze/posixpath.c: Python/frozen_modules/posixpath.h $(DEEPFREEZE_DEPS)
982-
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/posixpath.h -m posixpath -o Python/deepfreeze/posixpath.c
1002+
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/posixpath.h -m posixpath -o Python/deepfreeze/posixpath.c
9831003

9841004
Python/deepfreeze/os.c: Python/frozen_modules/os.h $(DEEPFREEZE_DEPS)
985-
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/os.h -m os -o Python/deepfreeze/os.c
1005+
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/os.h -m os -o Python/deepfreeze/os.c
9861006

9871007
Python/deepfreeze/site.c: Python/frozen_modules/site.h $(DEEPFREEZE_DEPS)
988-
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/site.h -m site -o Python/deepfreeze/site.c
1008+
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/site.h -m site -o Python/deepfreeze/site.c
9891009

9901010
Python/deepfreeze/stat.c: Python/frozen_modules/stat.h $(DEEPFREEZE_DEPS)
991-
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/stat.h -m stat -o Python/deepfreeze/stat.c
1011+
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/stat.h -m stat -o Python/deepfreeze/stat.c
9921012

9931013
Python/deepfreeze/__hello__.c: Python/frozen_modules/__hello__.h $(DEEPFREEZE_DEPS)
994-
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__hello__.h -m __hello__ -o Python/deepfreeze/__hello__.c
1014+
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__hello__.h -m __hello__ -o Python/deepfreeze/__hello__.c
9951015

9961016
Python/deepfreeze/__phello__.c: Python/frozen_modules/__phello__.h $(DEEPFREEZE_DEPS)
997-
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__phello__.h -m __phello__ -o Python/deepfreeze/__phello__.c
1017+
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__phello__.h -m __phello__ -o Python/deepfreeze/__phello__.c
9981018

9991019
Python/deepfreeze/__phello__.ham.c: Python/frozen_modules/__phello__.ham.h $(DEEPFREEZE_DEPS)
1000-
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__phello__.ham.h -m __phello__.ham -o Python/deepfreeze/__phello__.ham.c
1020+
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__phello__.ham.h -m __phello__.ham -o Python/deepfreeze/__phello__.ham.c
10011021

10021022
Python/deepfreeze/__phello__.ham.eggs.c: Python/frozen_modules/__phello__.ham.eggs.h $(DEEPFREEZE_DEPS)
1003-
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__phello__.ham.eggs.h -m __phello__.ham.eggs -o Python/deepfreeze/__phello__.ham.eggs.c
1023+
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__phello__.ham.eggs.h -m __phello__.ham.eggs -o Python/deepfreeze/__phello__.ham.eggs.c
10041024

10051025
Python/deepfreeze/__phello__.spam.c: Python/frozen_modules/__phello__.spam.h $(DEEPFREEZE_DEPS)
1006-
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__phello__.spam.h -m __phello__.spam -o Python/deepfreeze/__phello__.spam.c
1026+
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__phello__.spam.h -m __phello__.spam -o Python/deepfreeze/__phello__.spam.c
10071027

10081028
Python/deepfreeze/frozen_only.c: Python/frozen_modules/frozen_only.h $(DEEPFREEZE_DEPS)
1009-
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/frozen_only.h -m frozen_only -o Python/deepfreeze/frozen_only.c
1029+
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/frozen_only.h -m frozen_only -o Python/deepfreeze/frozen_only.c
10101030

10111031
# END: deepfreeze modules
10121032

@@ -2295,6 +2315,7 @@ clean-retain-profile: pycremoval
22952315
find build -name '*.py[co]' -exec rm -f {} ';' || true
22962316
-rm -f pybuilddir.txt
22972317
-rm -f Lib/lib2to3/*Grammar*.pickle
2318+
-rm -f _bootstrap_python
22982319
-rm -f Programs/_testembed Programs/_freeze_module
22992320
-rm -f Python/deepfreeze/*.[co]
23002321
-rm -f Python/frozen_modules/*.h

‎Misc/NEWS.d/next/Build/2021-12-01-17-28-39.bpo-45950.eEVLoz.rst

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
The build system now uses a :program:`_bootstrap_python` interpreter for
2+
freezing and deepfreezing again. To speed up build process the build tools
3+
:program:`_bootstrap_python` and :program:`_freeze_module` are no longer
4+
build with LTO.

‎Programs/_bootstrap_python.c

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
2+
/* Frozen modules bootstrap
3+
*
4+
* Limited and restricted Python interpreter to run
5+
* "Tools/scripts/deepfreeze.py" on systems with no or older Python
6+
* interpreter.
7+
*/
8+
9+
#include "Python.h"
10+
#include "pycore_import.h"
11+
12+
/* Includes for frozen modules: */
13+
#include "Python/frozen_modules/importlib._bootstrap.h"
14+
#include "Python/frozen_modules/importlib._bootstrap_external.h"
15+
/* End includes */
16+
17+
/* Note that a negative size indicates a package. */
18+
19+
static const struct _frozen bootstrap_modules[] = {
20+
{"_frozen_importlib", _Py_M__importlib__bootstrap, (int)sizeof(_Py_M__importlib__bootstrap)},
21+
{"_frozen_importlib_external", _Py_M__importlib__bootstrap_external, (int)sizeof(_Py_M__importlib__bootstrap_external)},
22+
{0, 0, 0} /* bootstrap sentinel */
23+
};
24+
static const struct _frozen stdlib_modules[] = {
25+
{0, 0, 0} /* stdlib sentinel */
26+
};
27+
static const struct _frozen test_modules[] = {
28+
{0, 0, 0} /* test sentinel */
29+
};
30+
const struct _frozen *_PyImport_FrozenBootstrap = bootstrap_modules;
31+
const struct _frozen *_PyImport_FrozenStdlib = stdlib_modules;
32+
const struct _frozen *_PyImport_FrozenTest = test_modules;
33+
34+
static const struct _module_alias aliases[] = {
35+
{"_frozen_importlib", "importlib._bootstrap"},
36+
{"_frozen_importlib_external", "importlib._bootstrap_external"},
37+
{0, 0} /* aliases sentinel */
38+
};
39+
const struct _module_alias *_PyImport_FrozenAliases = aliases;
40+
41+
/* Embedding apps may change this pointer to point to their favorite
42+
collection of frozen modules: */
43+
44+
const struct _frozen *PyImport_FrozenModules = NULL;
45+
46+
int
47+
#ifdef MS_WINDOWS
48+
wmain(int argc, wchar_t **argv)
49+
#else
50+
main(int argc, char **argv)
51+
#endif
52+
{
53+
PyStatus status;
54+
55+
PyConfig config;
56+
PyConfig_InitIsolatedConfig(&config);
57+
// don't warn, pybuilddir.txt does not exist yet
58+
config.pathconfig_warnings = 0;
59+
// parse arguments
60+
config.parse_argv = 1;
61+
// add current script dir to sys.path
62+
config.isolated = 0;
63+
64+
#ifdef MS_WINDOWS
65+
status = PyConfig_SetArgv(&config, argc, argv);
66+
#else
67+
status = PyConfig_SetBytesArgv(&config, argc, argv);
68+
#endif
69+
if (PyStatus_Exception(status)) {
70+
goto error;
71+
}
72+
73+
status = PyConfig_Read(&config);
74+
if (config.run_filename == NULL) {
75+
status = PyStatus_Error("Run filename expected");
76+
goto error;
77+
}
78+
79+
#define CLEAR(ATTR) \
80+
do { \
81+
PyMem_RawFree(ATTR); \
82+
ATTR = NULL; \
83+
} while (0)
84+
85+
// isolate from system Python
86+
CLEAR(config.base_prefix);
87+
CLEAR(config.prefix);
88+
CLEAR(config.base_exec_prefix);
89+
CLEAR(config.exec_prefix);
90+
91+
status = Py_InitializeFromConfig(&config);
92+
if (PyStatus_Exception(status)) {
93+
goto error;
94+
}
95+
PyConfig_Clear(&config);
96+
97+
return Py_RunMain();
98+
99+
error:
100+
PyConfig_Clear(&config);
101+
if (PyStatus_IsExit(status)) {
102+
return status.exitcode;
103+
}
104+
Py_ExitStatusException(status);
105+
}

‎Python/bootstrap_frozen.c

Lines changed: 0 additions & 45 deletions
This file was deleted.

‎Tools/scripts/deepfreeze.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,8 @@
1+
"""Deep freeze
2+
3+
The script is executed by _bootstrap_python interpreter. Shared library
4+
extension modules are not available.
5+
"""
16
import argparse
27
import ast
38
import builtins
@@ -8,7 +13,6 @@
813
import sys
914
import time
1015
import types
11-
import unicodedata
1216
from typing import Dict, FrozenSet, Tuple, TextIO
1317

1418
import umarshal

‎Tools/scripts/freeze_modules.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -589,7 +589,7 @@ def regen_makefile(modules):
589589
])
590590
deepfreezerules.append(f'{cfile}: {frozen_header} $(DEEPFREEZE_DEPS)')
591591
deepfreezerules.append(
592-
f"\t$(PYTHON_FOR_REGEN) "
592+
f"\t$(PYTHON_FOR_FREEZE) "
593593
f"$(srcdir)/Tools/scripts/deepfreeze.py "
594594
f"{frozen_header} -m {src.frozenid} -o {cfile}")
595595
deepfreezerules.append('')

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy