From 919efccc188c0175056053164a70cff9d6fd3ef6 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Mon, 20 Jul 2020 13:42:17 +0200 Subject: [PATCH 01/11] Add support for ARM64 to the pythonw tool --- Mac/Tools/pythonw.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Mac/Tools/pythonw.c b/Mac/Tools/pythonw.c index c8bd3ba8d68c15..8d65da79517420 100644 --- a/Mac/Tools/pythonw.c +++ b/Mac/Tools/pythonw.c @@ -95,9 +95,6 @@ setup_spawnattr(posix_spawnattr_t* spawnattr) size_t count; cpu_type_t cpu_types[1]; short flags = 0; -#ifdef __LP64__ - int ch; -#endif if ((errno = posix_spawnattr_init(spawnattr)) != 0) { err(2, "posix_spawnattr_int"); @@ -119,10 +116,16 @@ setup_spawnattr(posix_spawnattr_t* spawnattr) #elif defined(__ppc__) cpu_types[0] = CPU_TYPE_POWERPC; + #elif defined(__i386__) cpu_types[0] = CPU_TYPE_X86; + +#elif defined(__arm64__) + cpu_types[0] = CPU_TYPE_ARM64; + #else # error "Unknown CPU" + #endif if (posix_spawnattr_setbinpref_np(spawnattr, count, From 69c39f36f6efe9b13b2dc06892133ac31e6ca7bc Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Mon, 20 Jul 2020 13:42:43 +0200 Subject: [PATCH 02/11] Add support for "universal2" as a fat binary target on macOS --- Lib/_osx_support.py | 2 ++ configure | 20 ++++++++++++++------ configure.ac | 18 +++++++++++++----- 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/Lib/_osx_support.py b/Lib/_osx_support.py index e9efce7d7ed5bd..8ed1eeac8525da 100644 --- a/Lib/_osx_support.py +++ b/Lib/_osx_support.py @@ -481,6 +481,8 @@ def get_platform_osx(_config_vars, osname, release, machine): if len(archs) == 1: machine = archs[0] + elif archs == ('arm64', 'x86_64'): + machine = 'universal2' elif archs == ('i386', 'ppc'): machine = 'fat' elif archs == ('i386', 'x86_64'): diff --git a/configure b/configure index 5024860ca4395a..f811d2c23e4a7a 100755 --- a/configure +++ b/configure @@ -1509,8 +1509,8 @@ Optional Packages: specify the kind of universal binary that should be created. this option is only valid when --enable-universalsdk is set; options are: - ("32-bit", "64-bit", "3-way", "intel", "intel-32", - "intel-64", or "all") see Mac/README.rst + ("universal2", "32-bit", "64-bit", "3-way", "intel", + "intel-32", "intel-64", or "all") see Mac/README.rst --with-framework-name=FRAMEWORK specify the name for the python framework on macOS only valid when --enable-framework is set. see @@ -6945,7 +6945,7 @@ fi -# The -arch flags for universal builds on OSX +# The -arch flags for universal builds on macOS UNIVERSAL_ARCH_FLAGS= @@ -7472,6 +7472,11 @@ $as_echo "$CC" >&6; } LIPO_32BIT_FLAGS="-extract ppc7400 -extract i386" ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc" ;; + universal2) + UNIVERSAL_ARCH_FLAGS="-arch arm64 -arch x86_64" + LIPO_32BIT_FLAGS="" + ARCH_RUN_32BIT="true" + ;; intel) UNIVERSAL_ARCH_FLAGS="-arch i386 -arch x86_64" LIPO_32BIT_FLAGS="-extract i386" @@ -7493,7 +7498,7 @@ $as_echo "$CC" >&6; } ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc" ;; *) - as_fn_error $? "proper usage is --with-universal-arch=32-bit|64-bit|all|intel|3-way" "$LINENO" 5 + as_fn_error $? "proper usage is --with-universal-arch=universal2|32-bit|64-bit|all|intel|3-way" "$LINENO" 5 ;; esac @@ -9322,7 +9327,7 @@ fi MACOSX_DEFAULT_ARCH="ppc" ;; *) - as_fn_error $? "Unexpected output of 'arch' on OSX" "$LINENO" 5 + as_fn_error $? "Unexpected output of 'arch' on macOS" "$LINENO" 5 ;; esac else @@ -9332,9 +9337,12 @@ fi ;; ppc) MACOSX_DEFAULT_ARCH="ppc64" + ;; + arm64) + MACOSX_DEFAULT_ARCH="arm64" ;; *) - as_fn_error $? "Unexpected output of 'arch' on OSX" "$LINENO" 5 + as_fn_error $? "Unexpected output of 'arch' on macOS" "$LINENO" 5 ;; esac diff --git a/configure.ac b/configure.ac index 5a3e340aa3e72b..d2044d0520e679 100644 --- a/configure.ac +++ b/configure.ac @@ -218,7 +218,7 @@ AC_ARG_WITH(universal-archs, AS_HELP_STRING([--with-universal-archs=ARCH], [specify the kind of universal binary that should be created. this option is only valid when --enable-universalsdk is set; options are: - ("32-bit", "64-bit", "3-way", "intel", "intel-32", "intel-64", or "all") + ("universal2", "32-bit", "64-bit", "3-way", "intel", "intel-32", "intel-64", or "all") see Mac/README.rst]), [ UNIVERSAL_ARCHS="$withval" @@ -1578,7 +1578,7 @@ AC_SUBST(BASECFLAGS) AC_SUBST(CFLAGS_NODIST) AC_SUBST(LDFLAGS_NODIST) -# The -arch flags for universal builds on OSX +# The -arch flags for universal builds on macOS UNIVERSAL_ARCH_FLAGS= AC_SUBST(UNIVERSAL_ARCH_FLAGS) @@ -1879,6 +1879,11 @@ yes) LIPO_32BIT_FLAGS="-extract ppc7400 -extract i386" ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc" ;; + universal2) + UNIVERSAL_ARCH_FLAGS="-arch arm64 -arch x86_64" + LIPO_32BIT_FLAGS="" + ARCH_RUN_32BIT="true" + ;; intel) UNIVERSAL_ARCH_FLAGS="-arch i386 -arch x86_64" LIPO_32BIT_FLAGS="-extract i386" @@ -1900,7 +1905,7 @@ yes) ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc" ;; *) - AC_MSG_ERROR([proper usage is --with-universal-arch=32-bit|64-bit|all|intel|3-way]) + AC_MSG_ERROR([proper usage is --with-universal-arch=universal2|32-bit|64-bit|all|intel|3-way]) ;; esac @@ -2467,7 +2472,7 @@ case $ac_sys_system/$ac_sys_release in MACOSX_DEFAULT_ARCH="ppc" ;; *) - AC_MSG_ERROR([Unexpected output of 'arch' on OSX]) + AC_MSG_ERROR([Unexpected output of 'arch' on macOS]) ;; esac else @@ -2477,9 +2482,12 @@ case $ac_sys_system/$ac_sys_release in ;; ppc) MACOSX_DEFAULT_ARCH="ppc64" + ;; + arm64) + MACOSX_DEFAULT_ARCH="arm64" ;; *) - AC_MSG_ERROR([Unexpected output of 'arch' on OSX]) + AC_MSG_ERROR([Unexpected output of 'arch' on macOS]) ;; esac From 3940c867b6394d8950bfff725133e8a8241081b0 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Mon, 20 Jul 2020 15:47:44 +0200 Subject: [PATCH 03/11] Tweak ctypes.macholib.dyld to work with the shared library cache. --- Lib/ctypes/macholib/dyld.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/Lib/ctypes/macholib/dyld.py b/Lib/ctypes/macholib/dyld.py index 9d86b058765a3e..9ab9e79ab816c4 100644 --- a/Lib/ctypes/macholib/dyld.py +++ b/Lib/ctypes/macholib/dyld.py @@ -122,7 +122,19 @@ def dyld_find(name, executable_path=None, env=None): dyld_executable_path_search(name, executable_path), dyld_default_search(name, env), ), env): - if os.path.isfile(path): + + # on macOS 11 system libraries are in a shared library + # cache, not in the regular place in the filesystem. + # There still are symlinks (libz.dylib -> libz.1.dylib), + # but the target of the symlink no longer is there. + # + # There shouldn't be 3th-party libraries in these + # system locations + if path.startswith("/System/") and os.path.islink(path): + return path + elif path.startswith("/usr/lib/") and os.path.islink(path): + return path + elif os.path.isfile(path): return path raise ValueError("dylib %s could not be found" % (name,)) From deda5f0c830517969c619821f5d932495d7d27ef Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Mon, 20 Jul 2020 16:11:40 +0200 Subject: [PATCH 04/11] Silence compile time warning We should consider just dropping support for macOS 10.4 here to simplify the code. --- Mac/Tools/pythonw.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Mac/Tools/pythonw.c b/Mac/Tools/pythonw.c index 8d65da79517420..78813e818e7dac 100644 --- a/Mac/Tools/pythonw.c +++ b/Mac/Tools/pythonw.c @@ -223,7 +223,8 @@ main(int argc, char **argv) { /* We're weak-linking to posix-spawnv to ensure that * an executable build on 10.5 can work on 10.4. */ - if (posix_spawn != NULL) { + + if (&posix_spawn != NULL) { posix_spawnattr_t spawnattr = NULL; setup_spawnattr(&spawnattr); From ea3c200890e1dc606745f6a5308725018595d001 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Mon, 20 Jul 2020 16:12:37 +0200 Subject: [PATCH 05/11] macOS/arm64 support, based on GH-21249 This is support for ctypes on macOS/arm64 based on PR 21249 by Lawrence D'Anna (Apple). Changes: - changed __builtin_available tests from 11.0 to 10.15 - added test to setup.py for ffi_closure_alloc and use that in malloc_closure.c - Minor change in the code path for ffi_prep_closure_var (coding style change) --- Lib/test/test_bytes.py | 1 + Lib/test/test_unicode.py | 2 + Modules/_ctypes/callbacks.c | 39 +++++++++--- Modules/_ctypes/callproc.c | 72 ++++++++++++++++++---- Modules/_ctypes/ctypes.h | 8 +++ Modules/_ctypes/malloc_closure.c | 15 ++++- setup.py | 102 ++++++++++++++++--------------- 7 files changed, 168 insertions(+), 71 deletions(-) diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 61b4b9162ccc54..e26ea4704bf5c7 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1035,6 +1035,7 @@ def test_from_format(self): c_char_p) PyBytes_FromFormat = pythonapi.PyBytes_FromFormat + PyBytes_FromFormat.argtypes = (c_char_p,) PyBytes_FromFormat.restype = py_object # basic tests diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index d485bc7ede2b92..4cf60591d2dd4d 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -2513,11 +2513,13 @@ class CAPITest(unittest.TestCase): def test_from_format(self): import_helper.import_module('ctypes') from ctypes import ( + c_char_p, pythonapi, py_object, sizeof, c_int, c_long, c_longlong, c_ssize_t, c_uint, c_ulong, c_ulonglong, c_size_t, c_void_p) name = "PyUnicode_FromFormat" _PyUnicode_FromFormat = getattr(pythonapi, name) + _PyUnicode_FromFormat.argtypes = (c_char_p,) _PyUnicode_FromFormat.restype = py_object def PyUnicode_FromFormat(format, *args): diff --git a/Modules/_ctypes/callbacks.c b/Modules/_ctypes/callbacks.c index 2abfa67cdc06bf..77d4855277ecec 100644 --- a/Modules/_ctypes/callbacks.c +++ b/Modules/_ctypes/callbacks.c @@ -1,6 +1,8 @@ #include "Python.h" #include "frameobject.h" +#include + #include #ifdef MS_WIN32 #include @@ -18,7 +20,7 @@ CThunkObject_dealloc(PyObject *myself) Py_XDECREF(self->callable); Py_XDECREF(self->restype); if (self->pcl_write) - ffi_closure_free(self->pcl_write); + Py_ffi_closure_free(self->pcl_write); PyObject_GC_Del(self); } @@ -361,8 +363,7 @@ CThunkObject *_ctypes_alloc_callback(PyObject *callable, assert(CThunk_CheckExact((PyObject *)p)); - p->pcl_write = ffi_closure_alloc(sizeof(ffi_closure), - &p->pcl_exec); + p->pcl_write = Py_ffi_closure_alloc(sizeof(ffi_closure), &p->pcl_exec); if (p->pcl_write == NULL) { PyErr_NoMemory(); goto error; @@ -408,13 +409,35 @@ CThunkObject *_ctypes_alloc_callback(PyObject *callable, "ffi_prep_cif failed with %d", result); goto error; } -#if defined(X86_DARWIN) || defined(POWERPC_DARWIN) - result = ffi_prep_closure(p->pcl_write, &p->cif, closure_fcn, p); +#if HAVE_FFI_PREP_CLOSURE_LOC +# if USING_APPLE_OS_LIBFFI +# define HAVE_FFI_PREP_CLOSURE_LOC_RUNTIME __builtin_available(macos 10.15, ios 13, watchos 6, tvos 13, *) +# else +# define HAVE_FFI_PREP_CLOSURE_LOC_RUNTIME true +# endif + if (HAVE_FFI_PREP_CLOSURE_LOC_RUNTIME) { + result = ffi_prep_closure_loc(p->pcl_write, &p->cif, closure_fcn, + p, + p->pcl_exec); + } else +#endif + { +#if USING_APPLE_OS_LIBFFI && defined(__arm64__) + PyErr_Format(PyExc_NotImplementedError, "ffi_prep_closure_loc() is missing"); + goto error; #else - result = ffi_prep_closure_loc(p->pcl_write, &p->cif, closure_fcn, - p, - p->pcl_exec); +#ifdef MACOSX + #pragma clang diagnostic push + #pragma clang diagnostic ignored "-Wdeprecated-declarations" #endif + result = ffi_prep_closure(p->pcl_write, &p->cif, closure_fcn, p); + +#ifdef MACOSX + #pragma clang diagnostic pop +#endif + +#endif + } if (result != FFI_OK) { PyErr_Format(PyExc_RuntimeError, "ffi_prep_closure failed with %d", result); diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 261ae5ceb9e48a..ac407c93c240e4 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -57,6 +57,8 @@ #include "Python.h" #include "structmember.h" // PyMemberDef +#include + #ifdef MS_WIN32 #include #include @@ -812,7 +814,8 @@ static int _call_function_pointer(int flags, ffi_type **atypes, ffi_type *restype, void *resmem, - int argcount) + int argcount, + int argtypecount) { PyThreadState *_save = NULL; /* For Py_BLOCK_THREADS and Py_UNBLOCK_THREADS */ PyObject *error_object = NULL; @@ -835,14 +838,60 @@ static int _call_function_pointer(int flags, if ((flags & FUNCFLAG_CDECL) == 0) cc = FFI_STDCALL; #endif - if (FFI_OK != ffi_prep_cif(&cif, - cc, - argcount, - restype, - atypes)) { - PyErr_SetString(PyExc_RuntimeError, - "ffi_prep_cif failed"); - return -1; + +# if USING_APPLE_OS_LIBFFI +# define HAVE_FFI_PREP_CIF_VAR_RUNTIME __builtin_available(macos 10.15, ios 13, watchos 6, tvos 13, *) +# elif HAVE_FFI_PREP_CIF_VAR +# define HAVE_FFI_PREP_CIF_VAR_RUNTIME true +# else +# define HAVE_FFI_PREP_CIF_VAR_RUNTIME false +# endif + + /* Even on Apple-arm64 the calling convention for variadic functions conincides + * with the standard calling convention in the case that the function called + * only with its fixed arguments. Thus, we do not need a special flag to be + * set on variadic functions. We treat a function as variadic if it is called + * with a nonzero number of variadic arguments */ + bool is_variadic = (argtypecount != 0 && argcount > argtypecount); + (void) is_variadic; + +#if defined(__APPLE__) && defined(__arm64__) + if (is_variadic) { + if (HAVE_FFI_PREP_CIF_VAR_RUNTIME) { + } else { + PyErr_SetString(PyExc_NotImplementedError, "ffi_prep_cif_var() is missing"); + return -1; + } + } +#endif + +#if HAVE_FFI_PREP_CIF_VAR + if (is_variadic) { + if (HAVE_FFI_PREP_CIF_VAR_RUNTIME) { + if (FFI_OK != ffi_prep_cif_var(&cif, + cc, + argtypecount, + argcount, + restype, + atypes)) { + PyErr_SetString(PyExc_RuntimeError, + "ffi_prep_cif_var failed"); + return -1; + } + } + } else +#endif + + { + if (FFI_OK != ffi_prep_cif(&cif, + cc, + argcount, + restype, + atypes)) { + PyErr_SetString(PyExc_RuntimeError, + "ffi_prep_cif failed"); + return -1; + } } if (flags & (FUNCFLAG_USE_ERRNO | FUNCFLAG_USE_LASTERROR)) { @@ -1212,9 +1261,8 @@ PyObject *_ctypes_callproc(PPROC pProc, if (-1 == _call_function_pointer(flags, pProc, avalues, atypes, rtype, resbuf, - Py_SAFE_DOWNCAST(argcount, - Py_ssize_t, - int))) + Py_SAFE_DOWNCAST(argcount, Py_ssize_t, int), + Py_SAFE_DOWNCAST(argtype_count, Py_ssize_t, int))) goto cleanup; #ifdef WORDS_BIGENDIAN diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 1effccf9cc5ff9..3f20031d671a8a 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -366,6 +366,14 @@ PyObject *_ctypes_get_errobj(int **pspace); extern PyObject *ComError; #endif +#if USING_MALLOC_CLOSURE_DOT_C +void Py_ffi_closure_free(void *p); +void *Py_ffi_closure_alloc(size_t size, void** codeloc); +#else +#define Py_ffi_closure_free ffi_closure_free +#define Py_ffi_closure_alloc ffi_closure_alloc +#endif + /* Local Variables: compile-command: "python setup.py -q build install --home ~" diff --git a/Modules/_ctypes/malloc_closure.c b/Modules/_ctypes/malloc_closure.c index f9cdb336958c6f..4f220e42ff3fcc 100644 --- a/Modules/_ctypes/malloc_closure.c +++ b/Modules/_ctypes/malloc_closure.c @@ -89,16 +89,27 @@ static void more_core(void) /******************************************************************/ /* put the item back into the free list */ -void ffi_closure_free(void *p) +void Py_ffi_closure_free(void *p) { +#if USING_APPLE_OS_LIBFFI && HAVE_FFI_CLOSURE_ALLOC + if (__builtin_available(macos 10.15, ios 13, watchos 6, tvos 13, *)) { + ffi_closure_free(p); + return; + } +#endif ITEM *item = (ITEM *)p; item->next = free_list; free_list = item; } /* return one item from the free list, allocating more if needed */ -void *ffi_closure_alloc(size_t ignored, void** codeloc) +void *Py_ffi_closure_alloc(size_t size, void** codeloc) { +#if USING_APPLE_OS_LIBFFI && HAVE_FFI_CLOSURE_ALLOC + if (__builtin_available(macos 10.15, ios 13, watchos 6, tvos 13, *)) { + return ffi_closure_alloc(size, codeloc); + } +#endif ITEM *item; if (!free_list) more_core(); diff --git a/setup.py b/setup.py index 21a5a58981fc15..65d479fcb184c6 100644 --- a/setup.py +++ b/setup.py @@ -229,6 +229,13 @@ def macosx_sdk_specified(): macosx_sdk_root() return MACOS_SDK_SPECIFIED +def is_macosx_at_least(vers): + if MACOS: + dep_target = sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET') + if dep_target: + return tuple(map(int, dep_target.split('.'))) >= vers + return False + def is_macosx_sdk_path(path): """ @@ -239,6 +246,13 @@ def is_macosx_sdk_path(path): or path.startswith('/Library/') ) +def grep_headers_for(function, headers): + for header in headers: + with open(header, 'r') as f: + if function in f.read(): + return True + return False + def find_file(filename, std_dirs, paths): """Searches for the directory where a given file is located, and returns a possibly-empty list of additional directories, or None @@ -2100,43 +2114,18 @@ def detect_tkinter(self): library_dirs=added_lib_dirs)) return True - def configure_ctypes_darwin(self, ext): - # Darwin (OS X) uses preconfigured files, in - # the Modules/_ctypes/libffi_osx directory. - ffi_srcdir = os.path.abspath(os.path.join(self.srcdir, 'Modules', - '_ctypes', 'libffi_osx')) - sources = [os.path.join(ffi_srcdir, p) - for p in ['ffi.c', - 'x86/darwin64.S', - 'x86/x86-darwin.S', - 'x86/x86-ffi_darwin.c', - 'x86/x86-ffi64.c', - 'powerpc/ppc-darwin.S', - 'powerpc/ppc-darwin_closure.S', - 'powerpc/ppc-ffi_darwin.c', - 'powerpc/ppc64-darwin_closure.S', - ]] - - # Add .S (preprocessed assembly) to C compiler source extensions. - self.compiler.src_extensions.append('.S') - - include_dirs = [os.path.join(ffi_srcdir, 'include'), - os.path.join(ffi_srcdir, 'powerpc')] - ext.include_dirs.extend(include_dirs) - ext.sources.extend(sources) - return True - def configure_ctypes(self, ext): - if not self.use_system_libffi: - if MACOS: - return self.configure_ctypes_darwin(ext) - print('INFO: Could not locate ffi libs and/or headers') - return False return True def detect_ctypes(self): # Thomas Heller's _ctypes module - self.use_system_libffi = False + + if (not sysconfig.get_config_var("LIBFFI_INCLUDEDIR") and MACOS and + (is_macosx_at_least((10,15)) or '-arch arm64' in sysconfig.get_config_var("CFLAGS"))): + self.use_system_libffi = True + else: + self.use_system_libffi = '--with-system-ffi' in sysconfig.get_config_var("CONFIG_ARGS") + include_dirs = [] extra_compile_args = ['-DPy_BUILD_CORE_MODULE'] extra_link_args = [] @@ -2149,11 +2138,10 @@ def detect_ctypes(self): if MACOS: sources.append('_ctypes/malloc_closure.c') - sources.append('_ctypes/darwin/dlfcn_simple.c') + extra_compile_args.append('-DUSING_MALLOC_CLOSURE_DOT_C=1') + #sources.append('_ctypes/darwin/dlfcn_simple.c') extra_compile_args.append('-DMACOSX') include_dirs.append('_ctypes/darwin') - # XXX Is this still needed? - # extra_link_args.extend(['-read_only_relocs', 'warning']) elif HOST_PLATFORM == 'sunos5': # XXX This shouldn't be necessary; it appears that some @@ -2183,31 +2171,47 @@ def detect_ctypes(self): sources=['_ctypes/_ctypes_test.c'], libraries=['m'])) + ffi_inc = sysconfig.get_config_var("LIBFFI_INCLUDEDIR") + ffi_lib = None + ffi_inc_dirs = self.inc_dirs.copy() if MACOS: - if '--with-system-ffi' not in sysconfig.get_config_var("CONFIG_ARGS"): - return - # OS X 10.5 comes with libffi.dylib; the include files are - # in /usr/include/ffi - ffi_inc_dirs.append('/usr/include/ffi') - - ffi_inc = [sysconfig.get_config_var("LIBFFI_INCLUDEDIR")] - if not ffi_inc or ffi_inc[0] == '': - ffi_inc = find_file('ffi.h', [], ffi_inc_dirs) - if ffi_inc is not None: - ffi_h = ffi_inc[0] + '/ffi.h' + # XXX: The define should only be added when actually using the system + # version (and not a locally compiled one) + ext.extra_compile_args.append("-DUSING_APPLE_OS_LIBFFI=1") + ffi_in_sdk = os.path.join(macosx_sdk_root(), "usr/include/ffi") + if os.path.exists(ffi_in_sdk): + ffi_inc = ffi_in_sdk + ffi_lib = 'ffi' + else: + # OS X 10.5 comes with libffi.dylib; the include files are + # in /usr/include/ffi + ffi_inc_dirs.append('/usr/include/ffi') + + if not ffi_inc: + found = find_file('ffi.h', [], ffi_inc_dirs) + if found: + ffi_inc = found[0] + if ffi_inc: + ffi_h = ffi_inc + '/ffi.h' if not os.path.exists(ffi_h): ffi_inc = None print('Header file {} does not exist'.format(ffi_h)) - ffi_lib = None - if ffi_inc is not None: + if ffi_lib is None and ffi_inc: for lib_name in ('ffi', 'ffi_pic'): if (self.compiler.find_library_file(self.lib_dirs, lib_name)): ffi_lib = lib_name break if ffi_inc and ffi_lib: - ext.include_dirs.extend(ffi_inc) + ffi_headers = glob(os.path.join(ffi_inc, '*.h')) + if grep_headers_for('ffi_prep_cif_var', ffi_headers): + ext.extra_compile_args.append("-DHAVE_FFI_PREP_CIF_VAR=1") + if grep_headers_for('ffi_prep_closure_loc', ffi_headers): + ext.extra_compile_args.append("-DHAVE_FFI_PREP_CLOSURE_LOC=1") + if grep_headers_for('ffi_closure_alloc', ffi_headers): + ext.extra_compile_args.append("-DHAVE_FFI_CLOSURE_ALLOC=1") + ext.include_dirs.append(ffi_inc) ext.libraries.append(ffi_lib) self.use_system_libffi = True From 552bca86a2e7049d0483b779a7776245608d172c Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Mon, 20 Jul 2020 16:23:50 +0200 Subject: [PATCH 06/11] Fix test for macOS before 10.10 --- Lib/distutils/tests/test_build_ext.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/distutils/tests/test_build_ext.py b/Lib/distutils/tests/test_build_ext.py index f9e0d766d870e0..6bb009a86f41eb 100644 --- a/Lib/distutils/tests/test_build_ext.py +++ b/Lib/distutils/tests/test_build_ext.py @@ -493,7 +493,7 @@ def _try_compile_deployment_target(self, operator, target): # format the target value as defined in the Apple # Availability Macros. We can't use the macro names since # at least one value we test with will not exist yet. - if target[1] < 10: + if target[:2] < (10, 10): # for 10.1 through 10.9.x -> "10n0" target = '%02d%01d0' % target else: From e0c23a1d14446079e5cbee0b1d1624f242eebbc5 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Mon, 20 Jul 2020 16:25:35 +0200 Subject: [PATCH 07/11] ARM64 is a valid platform for macOS --- Lib/test/test_platform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_platform.py b/Lib/test/test_platform.py index 5ad306e0ed5795..45ca08819fb36c 100644 --- a/Lib/test/test_platform.py +++ b/Lib/test/test_platform.py @@ -246,7 +246,7 @@ def test_mac_ver(self): self.assertEqual(res[1], ('', '', '')) if sys.byteorder == 'little': - self.assertIn(res[2], ('i386', 'x86_64')) + self.assertIn(res[2], ('i386', 'x86_64', 'arm64')) else: self.assertEqual(res[2], 'PowerPC') From e637a77f3017d3149b90c25932805112b52d43c4 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Mon, 20 Jul 2020 17:21:37 +0200 Subject: [PATCH 08/11] Unconditionally use uint32_t here. The preprocessor guard in the old version doesn't work, and isn't really needed (10.4 only supported 32-bit code where unsigned long is the same as uint32_t). --- Modules/getpath.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Modules/getpath.c b/Modules/getpath.c index f7a6dd40443054..44453f29df703a 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -923,11 +923,7 @@ static PyStatus calculate_program_macos(wchar_t **abs_path_p) { char execpath[MAXPATHLEN + 1]; -#if MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_4 uint32_t nsexeclength = Py_ARRAY_LENGTH(execpath) - 1; -#else - unsigned long nsexeclength = Py_ARRAY_LENGTH(execpath) - 1; -#endif /* On Mac OS X, if a script uses an interpreter of the form "#!/opt/python2.3/bin/python", the kernel only passes "python" From ba2f5a324fba1f16c0c4f4e98fd906e5a470bb69 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Sun, 26 Jul 2020 14:52:51 +0200 Subject: [PATCH 09/11] Update build-installer.py for universal2 builds This also adds an option to stop building after compiling the 3th-party dependencies, as well as a script for archiving those dependencies. This makes it easier to work on the build. There are three changes to build-installer.py related to universal2 support: 1. Add 'universal2' information to build-installer.py; 2. Building OpenSSL for arm64 requires a patch at this time; 3. For some reason I had to patch the Tcl build to avoid a build error. --- Mac/BuildScript/archive-deps.py | 39 ++++++++++++++++++++ Mac/BuildScript/build-installer.py | 47 +++++++++++++++++++++++-- Mac/BuildScript/openssl-mac-arm64.patch | 41 +++++++++++++++++++++ 3 files changed, 124 insertions(+), 3 deletions(-) create mode 100755 Mac/BuildScript/archive-deps.py create mode 100644 Mac/BuildScript/openssl-mac-arm64.patch diff --git a/Mac/BuildScript/archive-deps.py b/Mac/BuildScript/archive-deps.py new file mode 100755 index 00000000000000..06d73a0f07617c --- /dev/null +++ b/Mac/BuildScript/archive-deps.py @@ -0,0 +1,39 @@ +#!/usr/bin/python + +import shutil +import getopt +import os +import sys + +WORKDIR = "/tmp/_py" + +def main(): + workdir = WORKDIR + + try: + options, args = getopt.getopt(sys.argv[1:], "?hbo", + [ "build-dir=", "output=", "help" ]) + except getopt.GetoptError: + print(sys.exc_info()[1]) + sys.exit(1) + + if args: + print("Additional arguments") + sys.exit(1) + + for k, v in options: + if k in ("-h", "-?", "--help"): + print(USAGE) + sys.exit(0) + + elif k in ("b", "--build-dir"): + workdir=v + + else: + raise SystemExit("Unknown option") + + root = os.path.join(workdir, "libraries", "Library", "Frameworks") + shutil.make_archive("third-party", "zip", root_dir=root) + +if __name__ == "__main__": + main() diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index a58b922ce30b83..56fc20390c3f97 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -116,7 +116,8 @@ def getFullVersion(): DEPSRC = os.path.join(WORKDIR, 'third-party') DEPSRC = os.path.expanduser('~/Universal/other-sources') -universal_opts_map = { '32-bit': ('i386', 'ppc',), +universal_opts_map = { 'universal2': ('arm64', 'x86_64'), + '32-bit': ('i386', 'ppc',), '64-bit': ('x86_64', 'ppc64',), 'intel': ('i386', 'x86_64'), 'intel-32': ('i386',), @@ -124,6 +125,7 @@ def getFullVersion(): '3-way': ('ppc', 'i386', 'x86_64'), 'all': ('i386', 'ppc', 'x86_64', 'ppc64',) } default_target_map = { + 'universal2': '10.9', '64-bit': '10.5', '3-way': '10.5', 'intel': '10.5', @@ -148,6 +150,10 @@ def getFullVersion(): # $MACOSX_DEPLOYMENT_TARGET -> minimum OS X level DEPTARGET = '10.5' +# If true only builds the 3th-party dependencies +# in $WORKDIR +DEPS_ONLY=False + def getDeptargetTuple(): return tuple([int(n) for n in DEPTARGET.split('.')[0:2]]) @@ -190,6 +196,27 @@ def getTargetCompilers(): def internalTk(): return getDeptargetTuple() >= (10, 6) + +def tweak_tcl_build(basedir, archList): + with open("Makefile", "r") as fp: + contents = fp.readlines() + + # For reasons I don't understand the tcl configure script + # decides that some stdlib symbols aren't present, before + # deciding that strtod is broken. + new_contents = [] + for line in contents: + if line.startswith("COMPAT_OBJS"): + # note: the space before strtod.o is intentional, + # the detection of a broken strtod results in + # "fixstrod.o" on this line. + for nm in ("strstr.o", "strtoul.o", " strtod.o"): + line = line.replace(nm, "") + new_contents.append(line) + + with open("Makefile", "w") as fp: + fp.writelines(new_contents) + # List of names of third party software built with this installer. # The names will be inserted into the rtf version of the License. THIRD_PARTY_LIBS = [] @@ -215,6 +242,9 @@ def library_recipes(): buildrecipe=build_universal_openssl, configure=None, install=None, + patches=[ + "openssl-mac-arm64.patch", + ], ), ]) @@ -231,6 +261,7 @@ def library_recipes(): '--libdir=/Library/Frameworks/Python.framework/Versions/%s/lib'%(getVersion(),), ], useLDFlags=False, + buildrecipe=tweak_tcl_build, install='make TCL_LIBRARY=%(TCL_LIBRARY)s && make install TCL_LIBRARY=%(TCL_LIBRARY)s DESTDIR=%(DESTDIR)s'%{ "DESTDIR": shellQuote(os.path.join(WORKDIR, 'libraries')), "TCL_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tcl8.6'%(getVersion())), @@ -596,7 +627,7 @@ def checkEnvironment(): ev, os.environ[ev])) del os.environ[ev] - base_path = '/bin:/sbin:/usr/bin:/usr/sbin' + base_path = '/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin' if 'SDK_TOOLS_BIN' in os.environ: base_path = os.environ['SDK_TOOLS_BIN'] + ':' + base_path # Xcode 2.5 on OS X 10.4 does not include SetFile in its usr/bin; @@ -618,6 +649,7 @@ def parseOptions(args=None): global UNIVERSALOPTS, UNIVERSALARCHS, ARCHLIST, CC, CXX global FW_VERSION_PREFIX global FW_SSL_DIRECTORY + global DEPS_ONLY if args is None: args = sys.argv[1:] @@ -625,7 +657,7 @@ def parseOptions(args=None): try: options, args = getopt.getopt(args, '?hb', [ 'build-dir=', 'third-party=', 'sdk-path=' , 'src-dir=', - 'dep-target=', 'universal-archs=', 'help' ]) + 'dep-target=', 'universal-archs=', 'deps-only', 'help' ]) except getopt.GetoptError: print(sys.exc_info()[1]) sys.exit(1) @@ -646,6 +678,9 @@ def parseOptions(args=None): elif k in ('--third-party',): DEPSRC=v + elif k in ('--deps-only',): + DEPS_ONLY=True + elif k in ('--sdk-path',): print(" WARNING: --sdk-path is no longer supported") @@ -691,6 +726,8 @@ def parseOptions(args=None): print(" -- Building a Python %s framework at patch level %s" % (getVersion(), getFullVersion())) print("") + if DEPS_ONLY: + print("Stopping after building third-party libraries") def extractArchive(builddir, archiveName): """ @@ -801,6 +838,7 @@ def build_openssl_arch(archbase, arch): arch_opts = { "i386": ["darwin-i386-cc"], "x86_64": ["darwin64-x86_64-cc", "enable-ec_nistp_64_gcc_128"], + "arm64": ["darwin64-arm64-cc" ], "ppc": ["darwin-ppc-cc"], "ppc64": ["darwin64-ppc-cc"], } @@ -1656,6 +1694,9 @@ def main(): # Then build third-party libraries such as sleepycat DB4. buildLibraries() + if DEPS_ONLY: + sys.exit(1) + # Now build python itself buildPython() diff --git a/Mac/BuildScript/openssl-mac-arm64.patch b/Mac/BuildScript/openssl-mac-arm64.patch new file mode 100644 index 00000000000000..11267fb118744a --- /dev/null +++ b/Mac/BuildScript/openssl-mac-arm64.patch @@ -0,0 +1,41 @@ +diff -ur openssl-1.1.1g-orig/Configurations/10-main.conf openssl-1.1.1g/Configurations/10-main.conf +--- openssl-1.1.1g-orig/Configurations/10-main.conf 2020-04-21 14:22:39.000000000 +0200 ++++ openssl-1.1.1g/Configurations/10-main.conf 2020-07-26 12:21:32.000000000 +0200 +@@ -1557,6 +1557,14 @@ + bn_ops => "SIXTY_FOUR_BIT_LONG", + perlasm_scheme => "macosx", + }, ++ "darwin64-arm64-cc" => { ++ inherit_from => [ "darwin-common", asm("aarch64_asm") ], ++ CFLAGS => add("-Wall"), ++ cflags => add("-arch arm64"), ++ lib_cppflags => add("-DL_ENDIAN"), ++ bn_ops => "SIXTY_FOUR_BIT_LONG", ++ perlasm_scheme => "ios64", ++ }, + + ##### GNU Hurd + "hurd-x86" => { +diff -ur openssl-1.1.1g-orig/config openssl-1.1.1g/config +--- openssl-1.1.1g-orig/config 2020-04-21 14:22:39.000000000 +0200 ++++ openssl-1.1.1g/config 2020-07-26 12:21:59.000000000 +0200 +@@ -255,6 +255,9 @@ + ;; + x86_64) + echo "x86_64-apple-darwin${VERSION}" ++ ;; ++ arm64) ++ echo "arm64-apple-darwin${VERSION}" + ;; + *) + echo "i686-apple-darwin${VERSION}" +@@ -497,6 +500,9 @@ + else + OUT="darwin64-x86_64-cc" + fi ;; ++ x86_64-apple-darwin*) ++ OUT="darwin64-arm64-cc" ++ ;; + armv6+7-*-iphoneos) + __CNF_CFLAGS="$__CNF_CFLAGS -arch armv6 -arch armv7" + __CNF_CXXFLAGS="$__CNF_CXXFLAGS -arch armv6 -arch armv7" From 8e3b4548f3843518996a9b754f213740479f290d Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Sun, 26 Jul 2020 16:06:43 +0200 Subject: [PATCH 10/11] Merge ctypes changes from Apple (PR 21241) Needed because my previous workaround doesn't work anymore. This uses a private API, that should be made public later... --- Lib/ctypes/macholib/dyld.py | 24 ++++++++-------- Lib/ctypes/test/test_macholib.py | 15 ++++++---- Modules/_ctypes/callproc.c | 48 ++++++++++++++++++++++++++++++++ setup.py | 1 + 4 files changed, 70 insertions(+), 18 deletions(-) diff --git a/Lib/ctypes/macholib/dyld.py b/Lib/ctypes/macholib/dyld.py index 9ab9e79ab816c4..22825d0d15b290 100644 --- a/Lib/ctypes/macholib/dyld.py +++ b/Lib/ctypes/macholib/dyld.py @@ -6,6 +6,11 @@ from ctypes.macholib.framework import framework_info from ctypes.macholib.dylib import dylib_info from itertools import * +try: + from _ctypes import _dyld_shared_cache_contains_path +except ImportError: + def _dyld_shared_cache_contains_path(*args): + raise NotImplementedError __all__ = [ 'dyld_find', 'framework_find', @@ -123,19 +128,14 @@ def dyld_find(name, executable_path=None, env=None): dyld_default_search(name, env), ), env): - # on macOS 11 system libraries are in a shared library - # cache, not in the regular place in the filesystem. - # There still are symlinks (libz.dylib -> libz.1.dylib), - # but the target of the symlink no longer is there. - # - # There shouldn't be 3th-party libraries in these - # system locations - if path.startswith("/System/") and os.path.islink(path): - return path - elif path.startswith("/usr/lib/") and os.path.islink(path): - return path - elif os.path.isfile(path): + if os.path.isfile(path): return path + try: + if _dyld_shared_cache_contains_path(path): + return path + except NotImplementedError: + pass + raise ValueError("dylib %s could not be found" % (name,)) def framework_find(fn, executable_path=None, env=None): diff --git a/Lib/ctypes/test/test_macholib.py b/Lib/ctypes/test/test_macholib.py index 6b3526951acfab..a1bac26a7df058 100644 --- a/Lib/ctypes/test/test_macholib.py +++ b/Lib/ctypes/test/test_macholib.py @@ -45,19 +45,22 @@ def find_lib(name): class MachOTest(unittest.TestCase): @unittest.skipUnless(sys.platform == "darwin", 'OSX-specific test') def test_find(self): - - self.assertEqual(find_lib('pthread'), - '/usr/lib/libSystem.B.dylib') + # On Mac OS 11, system dylibs are only present in the shared cache, + # so symlinks like libpthread.dylib -> libSystem.B.dylib will not + # be resolved by dyld_find + self.assertIn(find_lib('pthread'), + ('/usr/lib/libSystem.B.dylib', '/usr/lib/libpthread.dylib')) result = find_lib('z') # Issue #21093: dyld default search path includes $HOME/lib and # /usr/local/lib before /usr/lib, which caused test failures if # a local copy of libz exists in one of them. Now ignore the head # of the path. - self.assertRegex(result, r".*/lib/libz\..*.*\.dylib") + self.assertRegex(result, r".*/lib/libz.*\.dylib") - self.assertEqual(find_lib('IOKit'), - '/System/Library/Frameworks/IOKit.framework/Versions/A/IOKit') + self.assertIn(find_lib('IOKit'), + ('/System/Library/Frameworks/IOKit.framework/Versions/A/IOKit', + '/System/Library/Frameworks/IOKit.framework/IOKit')) if __name__ == "__main__": unittest.main() diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index ac407c93c240e4..824b00af0e60c2 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -66,6 +66,18 @@ #include "ctypes_dlfcn.h" #endif +#ifdef __APPLE__ +/* + * The API to query if a shared library is in the shared cache is + * private for now, this should change in beta 4. + * + * TODO: + * - Switch to that API + * - Add feature macro and runtime guards (as with the ffi.*loc API's) + */ +extern bool _dyld_shared_cache_contains_path(const char* path) __attribute__((weak_import)); +#endif + #ifdef MS_WIN32 #include #endif @@ -1446,6 +1458,37 @@ copy_com_pointer(PyObject *self, PyObject *args) } #else +#ifdef __APPLE__ + static PyObject *py_dyld_shared_cache_contains_path(PyObject *self, PyObject *args) + { + PyObject *name, *name2; + char *name_str; + + if (_dyld_shared_cache_contains_path == NULL) { + PyErr_SetString(PyExc_NotImplementedError, "_dyld_shared_cache_contains_path symbol is missing"); + return NULL; + } + + if (!PyArg_ParseTuple(args, "O", &name)) + return NULL; + + if (name == Py_None) + Py_RETURN_FALSE; + + if (PyUnicode_FSConverter(name, &name2) == 0) + return NULL; + if (PyBytes_Check(name2)) + name_str = PyBytes_AS_STRING(name2); + else + name_str = PyByteArray_AS_STRING(name2); + + if(_dyld_shared_cache_contains_path(name_str)) + Py_RETURN_TRUE; + else + Py_RETURN_FALSE; + } + #endif + static PyObject *py_dl_open(PyObject *self, PyObject *args) { PyObject *name, *name2; @@ -1935,6 +1978,8 @@ buffer_info(PyObject *self, PyObject *arg) return Py_BuildValue("siN", dict->format, dict->ndim, shape); } + + PyMethodDef _ctypes_module_methods[] = { {"get_errno", get_errno, METH_NOARGS}, {"set_errno", set_errno, METH_VARARGS}, @@ -1956,6 +2001,9 @@ PyMethodDef _ctypes_module_methods[] = { "dlopen(name, flag={RTLD_GLOBAL|RTLD_LOCAL}) open a shared library"}, {"dlclose", py_dl_close, METH_VARARGS, "dlclose a library"}, {"dlsym", py_dl_sym, METH_VARARGS, "find symbol in shared library"}, +#endif +#ifdef __APPLE__ + {"_dyld_shared_cache_contains_path", py_dyld_shared_cache_contains_path, METH_VARARGS, "check if path is in the shared cache"}, #endif {"alignment", align_func, METH_O, alignment_doc}, {"sizeof", sizeof_func, METH_O, sizeof_doc}, diff --git a/setup.py b/setup.py index 65d479fcb184c6..3bbf17a1c2283e 100644 --- a/setup.py +++ b/setup.py @@ -2211,6 +2211,7 @@ def detect_ctypes(self): ext.extra_compile_args.append("-DHAVE_FFI_PREP_CLOSURE_LOC=1") if grep_headers_for('ffi_closure_alloc', ffi_headers): ext.extra_compile_args.append("-DHAVE_FFI_CLOSURE_ALLOC=1") + ext.include_dirs.append(ffi_inc) ext.libraries.append(ffi_lib) self.use_system_libffi = True From 87c942becc00ac0c4bcb06f049e69aa2df8622e0 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Sun, 26 Jul 2020 16:36:23 +0200 Subject: [PATCH 11/11] The archive-deps.py script isn't really useful --- Mac/BuildScript/archive-deps.py | 39 --------------------------------- 1 file changed, 39 deletions(-) delete mode 100755 Mac/BuildScript/archive-deps.py diff --git a/Mac/BuildScript/archive-deps.py b/Mac/BuildScript/archive-deps.py deleted file mode 100755 index 06d73a0f07617c..00000000000000 --- a/Mac/BuildScript/archive-deps.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/python - -import shutil -import getopt -import os -import sys - -WORKDIR = "/tmp/_py" - -def main(): - workdir = WORKDIR - - try: - options, args = getopt.getopt(sys.argv[1:], "?hbo", - [ "build-dir=", "output=", "help" ]) - except getopt.GetoptError: - print(sys.exc_info()[1]) - sys.exit(1) - - if args: - print("Additional arguments") - sys.exit(1) - - for k, v in options: - if k in ("-h", "-?", "--help"): - print(USAGE) - sys.exit(0) - - elif k in ("b", "--build-dir"): - workdir=v - - else: - raise SystemExit("Unknown option") - - root = os.path.join(workdir, "libraries", "Library", "Frameworks") - shutil.make_archive("third-party", "zip", root_dir=root) - -if __name__ == "__main__": - main() pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy