From b72a43666c4fdebf4acbaae898ca496730a9cbb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Mon, 6 Jan 2025 18:50:56 +0000 Subject: [PATCH 1/3] GH-127178: improve compatibility in _sysconfig_vars_(...).json MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch improves environment and platform compatibility - Data now matches `sysconfig.get_config_vars` after install - `userbase` now correctly reflects the target platform when cross-compiling - `test_sysconfigdata_json` now takes into account the following situations: - Running with a non-default sys.executable path - Running under virtual environments - Running under relocatable installs To simplify the detection of relocatable installs, which needs to look at `_sysconfigdata_(...)`, this module is now saved in `sys.modules`. As part of this change, the code to import the module from a specific directory was refactored to use `PathFinder`, simplifying the implementation. Signed-off-by: Filipe Laíns = "5": # SunOS 5 == Solaris 2 + osname = "solaris" + release = f"{int(release[0]) - 3}.{release[2:]}" + # We can't use "platform.architecture()[0]" because a + # bootstrap problem. We use a dict to get an error + # if some suspicious happens. + bitness = {2147483647:"32bit", 9223372036854775807:"64bit"} + machine += f".{bitness[sys.maxsize]}" + # fall through to standard osname-release-machine representation + elif osname[:3] == "aix": + from _aix_support import aix_platform + return aix_platform() + elif osname[:6] == "cygwin": + osname = "cygwin" + import re + rel_re = re.compile(r'[\d.]+') + m = rel_re.match(release) + if m: + release = m.group() + elif osname[:6] == "darwin": + if sys.platform == "ios": + release = get_config_vars().get("IPHONEOS_DEPLOYMENT_TARGET", "13.0") + osname = sys.platform + machine = sys.implementation._multiarch + else: + import _osx_support + osname, release, machine = _osx_support.get_platform_osx( + get_config_vars(), + osname, release, machine) + + return f"{osname}-{release}-{machine}" + # NOTE: site.py has copy of this function. # Sync it when modify this function. def _getuserbase(): @@ -116,8 +213,10 @@ def _getuserbase(): if env_base: return env_base - # Emscripten, iOS, tvOS, VxWorks, WASI, and watchOS have no home directories - if sys.platform in {"emscripten", "ios", "tvos", "vxworks", "wasi", "watchos"}: + # Emscripten, iOS, tvOS, VxWorks, WASI, and watchOS have no home directories. + # Use sysconfig.get_platform() to get the correct platform when cross-compiling. + system_name = get_platform().split('-')[0] + if system_name in {"emscripten", "ios", "tvos", "vxworks", "wasi", "watchos"}: return None def joinuser(*args): @@ -335,6 +434,18 @@ def get_makefile_filename(): return os.path.join(get_path('stdlib'), config_dir_name, 'Makefile') +def _import_from_directory(path, name): + if name not in sys.modules: + import importlib.machinery + import importlib.util + + spec = importlib.machinery.PathFinder.find_spec(name, [path]) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + sys.modules[name] = module + return sys.modules[name] + + def _get_sysconfigdata_name(): multiarch = getattr(sys.implementation, '_multiarch', '') return os.environ.get( @@ -342,27 +453,34 @@ def _get_sysconfigdata_name(): f'_sysconfigdata_{sys.abiflags}_{sys.platform}_{multiarch}', ) -def _init_posix(vars): - """Initialize the module as appropriate for POSIX systems.""" - # _sysconfigdata is generated at build time, see _generate_posix_vars() + +def _get_sysconfigdata(): + import importlib + name = _get_sysconfigdata_name() + path = os.environ.get('_PYTHON_SYSCONFIGDATA_PATH') + module = _import_from_directory(path, name) if path else importlib.import_module(name) - # For cross builds, the path to the target's sysconfigdata must be specified - # so it can be imported. It cannot be in PYTHONPATH, as foreign modules in - # sys.path can cause crashes when loaded by the host interpreter. - # Rely on truthiness as a valueless env variable is still an empty string. - # See OS X note in _generate_posix_vars re _sysconfigdata. - if (path := os.environ.get('_PYTHON_SYSCONFIGDATA_PATH')): - from importlib.machinery import FileFinder, SourceFileLoader, SOURCE_SUFFIXES - from importlib.util import module_from_spec - spec = FileFinder(path, (SourceFileLoader, SOURCE_SUFFIXES)).find_spec(name) - _temp = module_from_spec(spec) - spec.loader.exec_module(_temp) - else: - _temp = __import__(name, globals(), locals(), ['build_time_vars'], 0) - build_time_vars = _temp.build_time_vars + return module.build_time_vars + + +def _installation_is_relocated(): + """Is the Python installation running from a different prefix than what was targetted when building?""" + if os.name != 'posix': + raise NotImplementedError('sysconfig._installation_is_relocated() is currently only supported on POSIX') + + data = _get_sysconfigdata() + return ( + data['prefix'] != getattr(sys, 'base_prefix', '') + or data['exec_prefix'] != getattr(sys, 'base_exec_prefix', '') + ) + + +def _init_posix(vars): + """Initialize the module as appropriate for POSIX systems.""" # GH-126920: Make sure we don't overwrite any of the keys already set - vars.update(build_time_vars | vars) + vars.update(_get_sysconfigdata() | vars) + def _init_non_posix(vars): """Initialize the module as appropriate for NT""" @@ -601,104 +719,6 @@ def get_config_var(name): return get_config_vars().get(name) -def get_platform(): - """Return a string that identifies the current platform. - - This is used mainly to distinguish platform-specific build directories and - platform-specific built distributions. Typically includes the OS name and - version and the architecture (as supplied by 'os.uname()'), although the - exact information included depends on the OS; on Linux, the kernel version - isn't particularly important. - - Examples of returned values: - linux-i586 - linux-alpha (?) - solaris-2.6-sun4u - - Windows will return one of: - win-amd64 (64bit Windows on AMD64 (aka x86_64, Intel64, EM64T, etc) - win32 (all others - specifically, sys.platform is returned) - - For other non-POSIX platforms, currently just returns 'sys.platform'. - - """ - if os.name == 'nt': - if 'amd64' in sys.version.lower(): - return 'win-amd64' - if '(arm)' in sys.version.lower(): - return 'win-arm32' - if '(arm64)' in sys.version.lower(): - return 'win-arm64' - return sys.platform - - if os.name != "posix" or not hasattr(os, 'uname'): - # XXX what about the architecture? NT is Intel or Alpha - return sys.platform - - # Set for cross builds explicitly - if "_PYTHON_HOST_PLATFORM" in os.environ: - return os.environ["_PYTHON_HOST_PLATFORM"] - - # Try to distinguish various flavours of Unix - osname, host, release, version, machine = os.uname() - - # Convert the OS name to lowercase, remove '/' characters, and translate - # spaces (for "Power Macintosh") - osname = osname.lower().replace('/', '') - machine = machine.replace(' ', '_') - machine = machine.replace('/', '-') - - if osname[:5] == "linux": - if sys.platform == "android": - osname = "android" - release = get_config_var("ANDROID_API_LEVEL") - - # Wheel tags use the ABI names from Android's own tools. - machine = { - "x86_64": "x86_64", - "i686": "x86", - "aarch64": "arm64_v8a", - "armv7l": "armeabi_v7a", - }[machine] - else: - # At least on Linux/Intel, 'machine' is the processor -- - # i386, etc. - # XXX what about Alpha, SPARC, etc? - return f"{osname}-{machine}" - elif osname[:5] == "sunos": - if release[0] >= "5": # SunOS 5 == Solaris 2 - osname = "solaris" - release = f"{int(release[0]) - 3}.{release[2:]}" - # We can't use "platform.architecture()[0]" because a - # bootstrap problem. We use a dict to get an error - # if some suspicious happens. - bitness = {2147483647:"32bit", 9223372036854775807:"64bit"} - machine += f".{bitness[sys.maxsize]}" - # fall through to standard osname-release-machine representation - elif osname[:3] == "aix": - from _aix_support import aix_platform - return aix_platform() - elif osname[:6] == "cygwin": - osname = "cygwin" - import re - rel_re = re.compile(r'[\d.]+') - m = rel_re.match(release) - if m: - release = m.group() - elif osname[:6] == "darwin": - if sys.platform == "ios": - release = get_config_vars().get("IPHONEOS_DEPLOYMENT_TARGET", "13.0") - osname = sys.platform - machine = sys.implementation._multiarch - else: - import _osx_support - osname, release, machine = _osx_support.get_platform_osx( - get_config_vars(), - osname, release, machine) - - return f"{osname}-{release}-{machine}" - - def get_python_version(): return _PY_VERSION_SHORT diff --git a/Lib/sysconfig/__main__.py b/Lib/sysconfig/__main__.py index 10728c709e1811..bc2197cfe79402 100644 --- a/Lib/sysconfig/__main__.py +++ b/Lib/sysconfig/__main__.py @@ -232,10 +232,14 @@ def _generate_posix_vars(): print(f'Written {destfile}') + install_vars = get_config_vars() + # Fix config vars to match the values after install (of the default environment) + install_vars['projectbase'] = install_vars['BINDIR'] + install_vars['srcdir'] = install_vars['LIBPL'] # Write a JSON file with the output of sysconfig.get_config_vars jsonfile = os.path.join(pybuilddir, _get_json_data_name()) with open(jsonfile, 'w') as f: - json.dump(get_config_vars(), f, indent=2) + json.dump(install_vars, f, indent=2) print(f'Written {jsonfile}') diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index ce504dc21af85f..ef9f8c0cd801b3 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -650,8 +650,18 @@ def test_sysconfigdata_json(self): system_config_vars = get_config_vars() - # Ignore keys in the check - for key in ('projectbase', 'srcdir'): + ignore_keys = set() + # Keys dependent on the executable location + if os.path.dirname(sys.executable) != system_config_vars['BINDIR']: + ignore_keys |= {'projectbase'} + # Keys dependent on the environment (different inside virtual environments) + if sys.prefix != sys.base_prefix: + ignore_keys |= {'prefix', 'exec_prefix', 'base', 'platbase'} + # Keys dependent on Python being run from the prefix targetted when building (different on relocatable installs) + if sysconfig._installation_is_relocated(): + ignore_keys |= {'prefix', 'exec_prefix', 'base', 'platbase', 'installed_base', 'installed_platbase'} + + for key in ignore_keys: json_config_vars.pop(key) system_config_vars.pop(key) From 396ac89f8bd3dfae69a8f60813cee72ddd66487f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Mon, 6 Jan 2025 19:53:04 +0000 Subject: [PATCH 2/3] Avoid get_platform() in _getuserbase() because of macOS get_config_vars() dependency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns = "5": # SunOS 5 == Solaris 2 - osname = "solaris" - release = f"{int(release[0]) - 3}.{release[2:]}" - # We can't use "platform.architecture()[0]" because a - # bootstrap problem. We use a dict to get an error - # if some suspicious happens. - bitness = {2147483647:"32bit", 9223372036854775807:"64bit"} - machine += f".{bitness[sys.maxsize]}" - # fall through to standard osname-release-machine representation - elif osname[:3] == "aix": - from _aix_support import aix_platform - return aix_platform() - elif osname[:6] == "cygwin": - osname = "cygwin" - import re - rel_re = re.compile(r'[\d.]+') - m = rel_re.match(release) - if m: - release = m.group() - elif osname[:6] == "darwin": - if sys.platform == "ios": - release = get_config_vars().get("IPHONEOS_DEPLOYMENT_TARGET", "13.0") - osname = sys.platform - machine = sys.implementation._multiarch - else: - import _osx_support - osname, release, machine = _osx_support.get_platform_osx( - get_config_vars(), - osname, release, machine) - - return f"{osname}-{release}-{machine}" - # NOTE: site.py has copy of this function. # Sync it when modify this function. def _getuserbase(): @@ -214,8 +117,8 @@ def _getuserbase(): return env_base # Emscripten, iOS, tvOS, VxWorks, WASI, and watchOS have no home directories. - # Use sysconfig.get_platform() to get the correct platform when cross-compiling. - system_name = get_platform().split('-')[0] + # Use _PYTHON_HOST_PLATFORM to get the correct platform when cross-compiling. + system_name = os.environ.get('_PYTHON_HOST_PLATFORM', sys.platform).split('-')[0] if system_name in {"emscripten", "ios", "tvos", "vxworks", "wasi", "watchos"}: return None @@ -719,6 +622,104 @@ def get_config_var(name): return get_config_vars().get(name) +def get_platform(): + """Return a string that identifies the current platform. + + This is used mainly to distinguish platform-specific build directories and + platform-specific built distributions. Typically includes the OS name and + version and the architecture (as supplied by 'os.uname()'), although the + exact information included depends on the OS; on Linux, the kernel version + isn't particularly important. + + Examples of returned values: + linux-i586 + linux-alpha (?) + solaris-2.6-sun4u + + Windows will return one of: + win-amd64 (64bit Windows on AMD64 (aka x86_64, Intel64, EM64T, etc) + win32 (all others - specifically, sys.platform is returned) + + For other non-POSIX platforms, currently just returns 'sys.platform'. + + """ + if os.name == 'nt': + if 'amd64' in sys.version.lower(): + return 'win-amd64' + if '(arm)' in sys.version.lower(): + return 'win-arm32' + if '(arm64)' in sys.version.lower(): + return 'win-arm64' + return sys.platform + + if os.name != "posix" or not hasattr(os, 'uname'): + # XXX what about the architecture? NT is Intel or Alpha + return sys.platform + + # Set for cross builds explicitly + if "_PYTHON_HOST_PLATFORM" in os.environ: + return os.environ["_PYTHON_HOST_PLATFORM"] + + # Try to distinguish various flavours of Unix + osname, host, release, version, machine = os.uname() + + # Convert the OS name to lowercase, remove '/' characters, and translate + # spaces (for "Power Macintosh") + osname = osname.lower().replace('/', '') + machine = machine.replace(' ', '_') + machine = machine.replace('/', '-') + + if osname[:5] == "linux": + if sys.platform == "android": + osname = "android" + release = get_config_var("ANDROID_API_LEVEL") + + # Wheel tags use the ABI names from Android's own tools. + machine = { + "x86_64": "x86_64", + "i686": "x86", + "aarch64": "arm64_v8a", + "armv7l": "armeabi_v7a", + }[machine] + else: + # At least on Linux/Intel, 'machine' is the processor -- + # i386, etc. + # XXX what about Alpha, SPARC, etc? + return f"{osname}-{machine}" + elif osname[:5] == "sunos": + if release[0] >= "5": # SunOS 5 == Solaris 2 + osname = "solaris" + release = f"{int(release[0]) - 3}.{release[2:]}" + # We can't use "platform.architecture()[0]" because a + # bootstrap problem. We use a dict to get an error + # if some suspicious happens. + bitness = {2147483647:"32bit", 9223372036854775807:"64bit"} + machine += f".{bitness[sys.maxsize]}" + # fall through to standard osname-release-machine representation + elif osname[:3] == "aix": + from _aix_support import aix_platform + return aix_platform() + elif osname[:6] == "cygwin": + osname = "cygwin" + import re + rel_re = re.compile(r'[\d.]+') + m = rel_re.match(release) + if m: + release = m.group() + elif osname[:6] == "darwin": + if sys.platform == "ios": + release = get_config_vars().get("IPHONEOS_DEPLOYMENT_TARGET", "13.0") + osname = sys.platform + machine = sys.implementation._multiarch + else: + import _osx_support + osname, release, machine = _osx_support.get_platform_osx( + get_config_vars(), + osname, release, machine) + + return f"{osname}-{release}-{machine}" + + def get_python_version(): return _PY_VERSION_SHORT From 2c26b59af569457c30a74dc37db6f4314be0c217 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Wed, 29 Jan 2025 22:16:44 +0000 Subject: [PATCH 3/3] Skip srcdir in builddir MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Lib/test/test_sysconfig.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index ff1a80d75aed82..3950d8888d4c8c 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -651,6 +651,9 @@ def test_sysconfigdata_json(self): system_config_vars = get_config_vars() ignore_keys = set() + # Keys dependent on Python being run outside the build directrory + if sysconfig.is_python_build(): + ignore_keys |= {'srcdir'} # Keys dependent on the executable location if os.path.dirname(sys.executable) != system_config_vars['BINDIR']: ignore_keys |= {'projectbase'} pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy