From f3a362dab8d975652b33eeeded96ac13005d6c67 Mon Sep 17 00:00:00 2001 From: Alex Turner Date: Thu, 9 May 2024 07:22:29 -0700 Subject: [PATCH 01/13] gh-117657: initialize_new_array --- Include/internal/pycore_qsbr.h | 2 +- Python/pystate.c | 2 +- Python/qsbr.c | 11 ++++++----- Tools/tsan/suppressions_free_threading.txt | 1 - 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Include/internal/pycore_qsbr.h b/Include/internal/pycore_qsbr.h index c3680a205542f7..20e643e172b38d 100644 --- a/Include/internal/pycore_qsbr.h +++ b/Include/internal/pycore_qsbr.h @@ -140,7 +140,7 @@ _Py_qsbr_register(struct _PyThreadStateImpl *tstate, // Disassociates a PyThreadState from the QSBR state and frees the QSBR state. extern void -_Py_qsbr_unregister(struct _PyThreadStateImpl *tstate); +_Py_qsbr_unregister(PyThreadState *tstate); extern void _Py_qsbr_fini(PyInterpreterState *interp); diff --git a/Python/pystate.c b/Python/pystate.c index b1e085bb806915..0a226268c558c7 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1785,7 +1785,7 @@ tstate_delete_common(PyThreadState *tstate) HEAD_UNLOCK(runtime); #ifdef Py_GIL_DISABLED - _Py_qsbr_unregister((_PyThreadStateImpl *)tstate); + _Py_qsbr_unregister(tstate); #endif // XXX Unbind in PyThreadState_Clear(), or earlier diff --git a/Python/qsbr.c b/Python/qsbr.c index d7ac8f479cda1b..1e02ff9c2e45f0 100644 --- a/Python/qsbr.c +++ b/Python/qsbr.c @@ -231,20 +231,21 @@ _Py_qsbr_register(_PyThreadStateImpl *tstate, PyInterpreterState *interp, } void -_Py_qsbr_unregister(_PyThreadStateImpl *tstate) +_Py_qsbr_unregister(PyThreadState *tstate) { - struct _qsbr_shared *shared = tstate->qsbr->shared; + struct _qsbr_shared *shared = &tstate->interp->qsbr; + struct _PyThreadStateImpl *tstate_imp = (_PyThreadStateImpl*) tstate; PyMutex_Lock(&shared->mutex); // NOTE: we must load (or reload) the thread state's qbsr inside the mutex // because the array may have been resized (changing tstate->qsbr) while // we waited to acquire the mutex. - struct _qsbr_thread_state *qsbr = tstate->qsbr; + struct _qsbr_thread_state *qsbr = tstate_imp->qsbr; assert(qsbr->seq == 0 && "thread state must be detached"); - assert(qsbr->allocated && qsbr->tstate == (PyThreadState *)tstate); + assert(qsbr->allocated && qsbr->tstate == tstate); - tstate->qsbr = NULL; + tstate_imp->qsbr = NULL; qsbr->tstate = NULL; qsbr->allocated = false; qsbr->freelist_next = shared->freelist; diff --git a/Tools/tsan/suppressions_free_threading.txt b/Tools/tsan/suppressions_free_threading.txt index a669bc4d1d5c30..88fe7183e59051 100644 --- a/Tools/tsan/suppressions_free_threading.txt +++ b/Tools/tsan/suppressions_free_threading.txt @@ -24,7 +24,6 @@ race:_PyParkingLot_Park race:_PyType_HasFeature race:assign_version_tag race:gc_restore_tid -race:initialize_new_array race:insertdict race:lookup_tp_dict race:mi_heap_visit_pages From ec921f7276a74509784a09f27f0c99bcb0214ebc Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Thu, 9 May 2024 18:20:46 +0300 Subject: [PATCH 02/13] gh-118817: Fix `asyncio REPL` on Windows (#118819) --- Lib/asyncio/__main__.py | 7 ++++--- Lib/test/test_repl.py | 5 ++++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/Lib/asyncio/__main__.py b/Lib/asyncio/__main__.py index cbc1d7c93ef76f..9041b8b8316c1e 100644 --- a/Lib/asyncio/__main__.py +++ b/Lib/asyncio/__main__.py @@ -108,7 +108,7 @@ def run(self): try: import readline # NoQA except ImportError: - pass + readline = None interactive_hook = getattr(sys, "__interactivehook__", None) @@ -122,8 +122,9 @@ def run(self): except: pass else: - completer = rlcompleter.Completer(console.locals) - readline.set_completer(completer.complete) + if readline is not None: + completer = rlcompleter.Completer(console.locals) + readline.set_completer(completer.complete) repl_thread = REPLThread() repl_thread.daemon = True diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py index 457279a4db687d..340178366fc13a 100644 --- a/Lib/test/test_repl.py +++ b/Lib/test/test_repl.py @@ -7,7 +7,7 @@ from textwrap import dedent from test import support from test.support import cpython_only, has_subprocess_support, SuppressCrashReport -from test.support.script_helper import kill_python +from test.support.script_helper import kill_python, assert_python_ok from test.support.import_helper import import_module @@ -195,6 +195,9 @@ def bar(x): expected = "(30, None, [\'def foo(x):\\n\', \' return x + 1\\n\', \'\\n\'], \'\')" self.assertIn(expected, output, expected) + def test_asyncio_repl_is_ok(self): + assert_python_ok("-m", "asyncio") + class TestInteractiveModeSyntaxErrors(unittest.TestCase): From 154a3f575d0c2cdcc2acf055257b1b0d32a37037 Mon Sep 17 00:00:00 2001 From: mpage Date: Thu, 9 May 2024 09:05:52 -0700 Subject: [PATCH 03/13] gh-117657: Acquire a critical section around `SemLock.__{enter,exit}__` (#118812) These methods are purely wrappers around `Semlock.{acquire,release}`, which expect a critical section to be held. --- Modules/_multiprocessing/clinic/semaphore.c.h | 12 ++++++++++-- Modules/_multiprocessing/semaphore.c | 6 ++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/Modules/_multiprocessing/clinic/semaphore.c.h b/Modules/_multiprocessing/clinic/semaphore.c.h index 64e666b5af6f5b..512e5a016192fb 100644 --- a/Modules/_multiprocessing/clinic/semaphore.c.h +++ b/Modules/_multiprocessing/clinic/semaphore.c.h @@ -473,7 +473,13 @@ _multiprocessing_SemLock___enter___impl(SemLockObject *self); static PyObject * _multiprocessing_SemLock___enter__(SemLockObject *self, PyObject *Py_UNUSED(ignored)) { - return _multiprocessing_SemLock___enter___impl(self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _multiprocessing_SemLock___enter___impl(self); + Py_END_CRITICAL_SECTION(); + + return return_value; } #endif /* defined(HAVE_MP_SEMAPHORE) */ @@ -518,7 +524,9 @@ _multiprocessing_SemLock___exit__(SemLockObject *self, PyObject *const *args, Py } exc_tb = args[2]; skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _multiprocessing_SemLock___exit___impl(self, exc_type, exc_value, exc_tb); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -565,4 +573,4 @@ _multiprocessing_SemLock___exit__(SemLockObject *self, PyObject *const *args, Py #ifndef _MULTIPROCESSING_SEMLOCK___EXIT___METHODDEF #define _MULTIPROCESSING_SEMLOCK___EXIT___METHODDEF #endif /* !defined(_MULTIPROCESSING_SEMLOCK___EXIT___METHODDEF) */ -/*[clinic end generated code: output=713b597256233716 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=dea36482d23a355f input=a9049054013a1b77]*/ diff --git a/Modules/_multiprocessing/semaphore.c b/Modules/_multiprocessing/semaphore.c index 5bb055f501e35b..4de4ee6c78fbd1 100644 --- a/Modules/_multiprocessing/semaphore.c +++ b/Modules/_multiprocessing/semaphore.c @@ -682,6 +682,7 @@ _multiprocessing_SemLock__after_fork_impl(SemLockObject *self) } /*[clinic input] +@critical_section _multiprocessing.SemLock.__enter__ Enter the semaphore/lock. @@ -689,12 +690,13 @@ Enter the semaphore/lock. static PyObject * _multiprocessing_SemLock___enter___impl(SemLockObject *self) -/*[clinic end generated code: output=beeb2f07c858511f input=c5e27d594284690b]*/ +/*[clinic end generated code: output=beeb2f07c858511f input=d35c9860992ee790]*/ { return _multiprocessing_SemLock_acquire_impl(self, 1, Py_None); } /*[clinic input] +@critical_section _multiprocessing.SemLock.__exit__ exc_type: object = None @@ -709,7 +711,7 @@ static PyObject * _multiprocessing_SemLock___exit___impl(SemLockObject *self, PyObject *exc_type, PyObject *exc_value, PyObject *exc_tb) -/*[clinic end generated code: output=3b37c1a9f8b91a03 input=7d644b64a89903f8]*/ +/*[clinic end generated code: output=3b37c1a9f8b91a03 input=1610c8cc3e0e337e]*/ { return _multiprocessing_SemLock_release_impl(self); } From a6bc4e3f97bfd416250a8064ebf85415c6e362f7 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Thu, 9 May 2024 17:43:21 +0100 Subject: [PATCH 04/13] gh-118773: Use language-invariant SDDL string instead of aliases for ACLs. (GH-118800) --- ...-05-08-21-59-38.gh-issue-118773.7dFRJY.rst | 2 + Modules/posixmodule.c | 176 +++--------------- 2 files changed, 24 insertions(+), 154 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2024-05-08-21-59-38.gh-issue-118773.7dFRJY.rst diff --git a/Misc/NEWS.d/next/Security/2024-05-08-21-59-38.gh-issue-118773.7dFRJY.rst b/Misc/NEWS.d/next/Security/2024-05-08-21-59-38.gh-issue-118773.7dFRJY.rst new file mode 100644 index 00000000000000..bfec178f6318a7 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-05-08-21-59-38.gh-issue-118773.7dFRJY.rst @@ -0,0 +1,2 @@ +Fixes creation of ACLs in :func:`os.mkdir` on Windows to work correctly on +non-English machines. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 9f4be98b35186e..5fe6036b3817e4 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5541,146 +5541,6 @@ os__path_normpath_impl(PyObject *module, PyObject *path) return result; } -#ifdef MS_WINDOWS - -/* We centralise SECURITY_ATTRIBUTE initialization based around -templates that will probably mostly match common POSIX mode settings. -The _Py_SECURITY_ATTRIBUTE_DATA structure contains temporary data, as -a constructed SECURITY_ATTRIBUTE structure typically refers to memory -that has to be alive while it's being used. - -Typical use will look like: - SECURITY_ATTRIBUTES *pSecAttr = NULL; - struct _Py_SECURITY_ATTRIBUTE_DATA secAttrData; - int error, error2; - - Py_BEGIN_ALLOW_THREADS - switch (mode) { - case 0x1C0: // 0o700 - error = initializeMkdir700SecurityAttributes(&pSecAttr, &secAttrData); - break; - ... - default: - error = initializeDefaultSecurityAttributes(&pSecAttr, &secAttrData); - break; - } - - if (!error) { - // do operation, passing pSecAttr - } - - // Unconditionally clear secAttrData. - error2 = clearSecurityAttributes(&pSecAttr, &secAttrData); - if (!error) { - error = error2; - } - Py_END_ALLOW_THREADS - - if (error) { - PyErr_SetFromWindowsErr(error); - return NULL; - } -*/ - -struct _Py_SECURITY_ATTRIBUTE_DATA { - SECURITY_ATTRIBUTES securityAttributes; - PACL acl; - SECURITY_DESCRIPTOR sd; - EXPLICIT_ACCESS_W ea[4]; - char sid[64]; -}; - -static int -initializeDefaultSecurityAttributes( - PSECURITY_ATTRIBUTES *securityAttributes, - struct _Py_SECURITY_ATTRIBUTE_DATA *data -) { - assert(securityAttributes); - assert(data); - *securityAttributes = NULL; - memset(data, 0, sizeof(*data)); - return 0; -} - -static int -initializeMkdir700SecurityAttributes( - PSECURITY_ATTRIBUTES *securityAttributes, - struct _Py_SECURITY_ATTRIBUTE_DATA *data -) { - assert(securityAttributes); - assert(data); - *securityAttributes = NULL; - memset(data, 0, sizeof(*data)); - - if (!InitializeSecurityDescriptor(&data->sd, SECURITY_DESCRIPTOR_REVISION) - || !SetSecurityDescriptorGroup(&data->sd, NULL, TRUE)) { - return GetLastError(); - } - - int use_alias = 0; - DWORD cbSid = sizeof(data->sid); - if (!CreateWellKnownSid(WinCreatorOwnerRightsSid, NULL, (PSID)data->sid, &cbSid)) { - use_alias = 1; - } - - data->securityAttributes.nLength = sizeof(SECURITY_ATTRIBUTES); - data->ea[0].grfAccessPermissions = GENERIC_ALL; - data->ea[0].grfAccessMode = SET_ACCESS; - data->ea[0].grfInheritance = SUB_CONTAINERS_AND_OBJECTS_INHERIT; - if (use_alias) { - data->ea[0].Trustee.TrusteeForm = TRUSTEE_IS_NAME; - data->ea[0].Trustee.TrusteeType = TRUSTEE_IS_ALIAS; - data->ea[0].Trustee.ptstrName = L"CURRENT_USER"; - } else { - data->ea[0].Trustee.TrusteeForm = TRUSTEE_IS_SID; - data->ea[0].Trustee.TrusteeType = TRUSTEE_IS_WELL_KNOWN_GROUP; - data->ea[0].Trustee.ptstrName = (LPWCH)(SID*)data->sid; - } - - data->ea[1].grfAccessPermissions = GENERIC_ALL; - data->ea[1].grfAccessMode = SET_ACCESS; - data->ea[1].grfInheritance = SUB_CONTAINERS_AND_OBJECTS_INHERIT; - data->ea[1].Trustee.TrusteeForm = TRUSTEE_IS_NAME; - data->ea[1].Trustee.TrusteeType = TRUSTEE_IS_ALIAS; - data->ea[1].Trustee.ptstrName = L"SYSTEM"; - - data->ea[2].grfAccessPermissions = GENERIC_ALL; - data->ea[2].grfAccessMode = SET_ACCESS; - data->ea[2].grfInheritance = SUB_CONTAINERS_AND_OBJECTS_INHERIT; - data->ea[2].Trustee.TrusteeForm = TRUSTEE_IS_NAME; - data->ea[2].Trustee.TrusteeType = TRUSTEE_IS_ALIAS; - data->ea[2].Trustee.ptstrName = L"ADMINISTRATORS"; - - int r = SetEntriesInAclW(3, data->ea, NULL, &data->acl); - if (r) { - return r; - } - if (!SetSecurityDescriptorDacl(&data->sd, TRUE, data->acl, FALSE)) { - return GetLastError(); - } - data->securityAttributes.lpSecurityDescriptor = &data->sd; - *securityAttributes = &data->securityAttributes; - return 0; -} - -static int -clearSecurityAttributes( - PSECURITY_ATTRIBUTES *securityAttributes, - struct _Py_SECURITY_ATTRIBUTE_DATA *data -) { - assert(securityAttributes); - assert(data); - *securityAttributes = NULL; - if (data->acl) { - if (LocalFree((void *)data->acl)) { - return GetLastError(); - } - } - return 0; -} - -#endif - /*[clinic input] os.mkdir @@ -5713,8 +5573,8 @@ os_mkdir_impl(PyObject *module, path_t *path, int mode, int dir_fd) #ifdef MS_WINDOWS int error = 0; int pathError = 0; + SECURITY_ATTRIBUTES secAttr = { sizeof(secAttr) }; SECURITY_ATTRIBUTES *pSecAttr = NULL; - struct _Py_SECURITY_ATTRIBUTE_DATA secAttrData; #endif #ifdef HAVE_MKDIRAT int mkdirat_unavailable = 0; @@ -5727,26 +5587,34 @@ os_mkdir_impl(PyObject *module, path_t *path, int mode, int dir_fd) #ifdef MS_WINDOWS Py_BEGIN_ALLOW_THREADS - switch (mode) { - case 0x1C0: // 0o700 - error = initializeMkdir700SecurityAttributes(&pSecAttr, &secAttrData); - break; - default: - error = initializeDefaultSecurityAttributes(&pSecAttr, &secAttrData); - break; + if (mode == 0700 /* 0o700 */) { + ULONG sdSize; + pSecAttr = &secAttr; + // Set a discreationary ACL (D) that is protected (P) and includes + // inheritable (OICI) entries that allow (A) full control (FA) to + // SYSTEM (SY), Administrators (BA), and the owner (OW). + if (!ConvertStringSecurityDescriptorToSecurityDescriptorW( + L"D:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FA;;;OW)", + SDDL_REVISION_1, + &secAttr.lpSecurityDescriptor, + &sdSize + )) { + error = GetLastError(); + } } if (!error) { result = CreateDirectoryW(path->wide, pSecAttr); - error = clearSecurityAttributes(&pSecAttr, &secAttrData); - } else { - // Ignore error from "clear" - we have a more interesting one already - clearSecurityAttributes(&pSecAttr, &secAttrData); + if (secAttr.lpSecurityDescriptor && + // uncommonly, LocalFree returns non-zero on error, but still uses + // GetLastError() to see what the error code is + LocalFree(secAttr.lpSecurityDescriptor)) { + error = GetLastError(); + } } Py_END_ALLOW_THREADS if (error) { - PyErr_SetFromWindowsErr(error); - return NULL; + return PyErr_SetFromWindowsErr(error); } if (!result) { return path_error(path); From 63af4b52bcd0b62236632b5720e32b249f97acad Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Thu, 9 May 2024 20:15:14 +0300 Subject: [PATCH 05/13] gh-118849: Fix "code will never be executed" warning in `dictobject.c` (#118850) --- Objects/dictobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index b0fce09d7940e0..985a326a176c94 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -5396,6 +5396,7 @@ static int dictiter_iternext_threadsafe(PyDictObject *d, PyObject *self, PyObject **out_key, PyObject **out_value) { + int res; dictiterobject *di = (dictiterobject *)self; Py_ssize_t i; PyDictKeysObject *k; @@ -5491,7 +5492,6 @@ dictiter_iternext_threadsafe(PyDictObject *d, PyObject *self, Py_DECREF(d); return -1; - int res; try_locked: Py_BEGIN_CRITICAL_SECTION(d); res = dictiter_iternextitem_lock_held(d, self, out_key, out_value); From fa80517cbd7e82f701cd3b33f39aa134464b7ebd Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Thu, 9 May 2024 14:52:27 -0400 Subject: [PATCH 06/13] gh-118561: Fix crash involving list.extend in free-threaded build (#118723) The `list_preallocate_exact` function did not zero initialize array contents. In the free-threaded build, this could expose uninitialized memory to concurrent readers between the call to `list_preallocate_exact` and the filling of the array contents with items. --- .../2024-05-07-16-57-56.gh-issue-118561.wNMKVd.rst | 2 ++ Objects/listobject.c | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-07-16-57-56.gh-issue-118561.wNMKVd.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-07-16-57-56.gh-issue-118561.wNMKVd.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-07-16-57-56.gh-issue-118561.wNMKVd.rst new file mode 100644 index 00000000000000..9eaf0abb8a6128 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-07-16-57-56.gh-issue-118561.wNMKVd.rst @@ -0,0 +1,2 @@ +Fix race condition in free-threaded build where :meth:`list.extend` could expose +uninitialied memory to concurrent readers. diff --git a/Objects/listobject.c b/Objects/listobject.c index 3c4e2d2e6ed7de..7070165014f137 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -192,6 +192,7 @@ list_preallocate_exact(PyListObject *self, Py_ssize_t size) return -1; } items = array->ob_item; + memset(items, 0, size * sizeof(PyObject *)); #else items = PyMem_New(PyObject*, size); if (items == NULL) { @@ -199,7 +200,7 @@ list_preallocate_exact(PyListObject *self, Py_ssize_t size) return -1; } #endif - self->ob_item = items; + FT_ATOMIC_STORE_PTR_RELEASE(self->ob_item, items); self->allocated = size; return 0; } From e0bb384da1eff6868e638f00a5fa2ff22563c055 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Thu, 9 May 2024 16:03:45 -0400 Subject: [PATCH 07/13] gh-116984: Make mimalloc header includes relative to the current file (#118808) Some embedders and extensions include parts of the internal API. The pycore_mimalloc.h file is transitively include by a number of other internal headers. This avoids include errors for code that was already including those headers. --- Include/internal/mimalloc/mimalloc/internal.h | 4 ++-- Include/internal/mimalloc/mimalloc/types.h | 2 +- Include/internal/pycore_mimalloc.h | 6 +++--- .../C API/2024-05-08-23-14-06.gh-issue-116984.5sgcDo.rst | 3 +++ 4 files changed, 9 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2024-05-08-23-14-06.gh-issue-116984.5sgcDo.rst diff --git a/Include/internal/mimalloc/mimalloc/internal.h b/Include/internal/mimalloc/mimalloc/internal.h index 8af841cfdffc01..94f88fb603af25 100644 --- a/Include/internal/mimalloc/mimalloc/internal.h +++ b/Include/internal/mimalloc/mimalloc/internal.h @@ -14,8 +14,8 @@ terms of the MIT license. A copy of the license can be found in the file // functions and macros. // -------------------------------------------------------------------------- -#include "mimalloc/types.h" -#include "mimalloc/track.h" +#include "types.h" +#include "track.h" #if (MI_DEBUG>0) #define mi_trace_message(...) _mi_trace_message(__VA_ARGS__) diff --git a/Include/internal/mimalloc/mimalloc/types.h b/Include/internal/mimalloc/mimalloc/types.h index 17e440848ecae5..354839ba955b36 100644 --- a/Include/internal/mimalloc/mimalloc/types.h +++ b/Include/internal/mimalloc/mimalloc/types.h @@ -21,7 +21,7 @@ terms of the MIT license. A copy of the license can be found in the file #include // ptrdiff_t #include // uintptr_t, uint16_t, etc -#include "mimalloc/atomic.h" // _Atomic +#include "atomic.h" // _Atomic #ifdef _MSC_VER #pragma warning(disable:4214) // bitfield is not int diff --git a/Include/internal/pycore_mimalloc.h b/Include/internal/pycore_mimalloc.h index 10d451398f1410..100f78d53021ee 100644 --- a/Include/internal/pycore_mimalloc.h +++ b/Include/internal/pycore_mimalloc.h @@ -36,9 +36,9 @@ typedef enum { # define MI_TSAN 1 #endif -#include "mimalloc.h" -#include "mimalloc/types.h" -#include "mimalloc/internal.h" +#include "mimalloc/mimalloc.h" +#include "mimalloc/mimalloc/types.h" +#include "mimalloc/mimalloc/internal.h" #endif #ifdef Py_GIL_DISABLED diff --git a/Misc/NEWS.d/next/C API/2024-05-08-23-14-06.gh-issue-116984.5sgcDo.rst b/Misc/NEWS.d/next/C API/2024-05-08-23-14-06.gh-issue-116984.5sgcDo.rst new file mode 100644 index 00000000000000..561417b80d444d --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-05-08-23-14-06.gh-issue-116984.5sgcDo.rst @@ -0,0 +1,3 @@ +Make mimalloc includes relative to the current file to avoid embedders or +extensions needing to include ``Internal/mimalloc`` if they are already +including internal CPython headers. From 888d3d79c9339ae93a14d804d64316ed37723f00 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Thu, 9 May 2024 16:06:20 -0400 Subject: [PATCH 08/13] gh-118846: Fix PGO tests in free-threaded build (#118862) Avoid immortalizing objects in tests that verify garbage collection of classes or modules. This fixes test_ordered_dict and test_struct. --- Lib/test/test_ordered_dict.py | 3 ++- Lib/test/test_struct.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_ordered_dict.py b/Lib/test/test_ordered_dict.py index 4571b23dfe7c1a..06a0e81227188c 100644 --- a/Lib/test/test_ordered_dict.py +++ b/Lib/test/test_ordered_dict.py @@ -10,7 +10,7 @@ import weakref from collections.abc import MutableMapping from test import mapping_tests, support -from test.support import import_helper +from test.support import import_helper, suppress_immortalization py_coll = import_helper.import_fresh_module('collections', @@ -667,6 +667,7 @@ def test_dict_update(self): dict.update(od, [('spam', 1)]) self.assertNotIn('NULL', repr(od)) + @suppress_immortalization() def test_reference_loop(self): # Issue 25935 OrderedDict = self.OrderedDict diff --git a/Lib/test/test_struct.py b/Lib/test/test_struct.py index 15f6ee06ffe19b..5508cc3eec85c8 100644 --- a/Lib/test/test_struct.py +++ b/Lib/test/test_struct.py @@ -9,7 +9,7 @@ import weakref from test import support -from test.support import import_helper +from test.support import import_helper, suppress_immortalization from test.support.script_helper import assert_python_ok ISBIGENDIAN = sys.byteorder == "big" @@ -674,6 +674,7 @@ def __del__(self): self.assertIn(b"Exception ignored in:", stderr) self.assertIn(b"C.__del__", stderr) + @suppress_immortalization() def test__struct_reference_cycle_cleaned_up(self): # Regression test for python/cpython#94207. From 64a7f498a8bf6106efb42a25287ff8981f456285 Mon Sep 17 00:00:00 2001 From: Brett Simmers Date: Thu, 9 May 2024 14:02:39 -0700 Subject: [PATCH 09/13] gh-117657: Replace TSAN suppresions with more specific rules (#118722) Using `race:` filters out warnings if the function appears anywhere in the stack trace. This can hide a lot of unrelated warnings, especially for a function like `_PyEval_EvalFrameDefault`, which is somewhere on the stack more often than not. Change all free-threaded suppressions to `race_top:`, which only matches the top frame, and add any new suppressions this exposes. --- Tools/tsan/suppressions_free_threading.txt | 104 ++++++++++++++++----- 1 file changed, 81 insertions(+), 23 deletions(-) diff --git a/Tools/tsan/suppressions_free_threading.txt b/Tools/tsan/suppressions_free_threading.txt index 88fe7183e59051..7f91a9113c4e1a 100644 --- a/Tools/tsan/suppressions_free_threading.txt +++ b/Tools/tsan/suppressions_free_threading.txt @@ -11,30 +11,88 @@ race:set_allocator_unlocked ## Free-threaded suppressions -race:_add_to_weak_set -race:_in_weak_set -race:_mi_heap_delayed_free_partial -race:_PyEval_EvalFrameDefault -race:_PyImport_AcquireLock -race:_PyImport_ReleaseLock -race:_PyInterpreterState_SetNotRunningMain -race:_PyInterpreterState_IsRunningMain -# https://gist.github.com/mpage/0a24eb2dd458441ededb498e9b0e5de8 + +# These entries are for warnings that trigger in a library function, as called +# by a CPython function. + +# https://gist.github.com/swtaarrs/9d41251e603fa6dedd604191a6da820d +race:park_detached_threads +# https://gist.github.com/swtaarrs/8e0e365e1d9cecece3269a2fb2f2b8b8 +race:sock_recv_impl +# https://gist.github.com/swtaarrs/08dfe7883b4c975c31ecb39388987a67 +race:free_threadstate +# https://gist.github.com/swtaarrs/cd6aec2006e0c1b561b68d65e9f1a872 race:_PyParkingLot_Park -race:_PyType_HasFeature -race:assign_version_tag -race:gc_restore_tid -race:insertdict -race:lookup_tp_dict -race:mi_heap_visit_pages -race:PyMember_GetOne -race:PyMember_SetOne -race:new_reference -race:set_contains_key -race:set_inheritable -race:start_the_world -race:tstate_set_detached -race:unicode_hash + + +# These warnings trigger directly in a CPython function. + +race_top:_add_to_weak_set +race_top:_in_weak_set +race_top:_mi_heap_delayed_free_partial +race_top:_PyEval_EvalFrameDefault +race_top:_PyImport_AcquireLock +race_top:_PyImport_ReleaseLock +race_top:_PyInterpreterState_SetNotRunningMain +race_top:_PyInterpreterState_IsRunningMain +# https://gist.github.com/mpage/0a24eb2dd458441ededb498e9b0e5de8 +race_top:_PyParkingLot_Park +race_top:_PyType_HasFeature +race_top:assign_version_tag +race_top:gc_restore_tid +race_top:initialize_new_array +race_top:insertdict +race_top:lookup_tp_dict +race_top:mi_heap_visit_pages +race_top:PyMember_GetOne +race_top:PyMember_SetOne +race_top:new_reference +race_top:set_contains_key +race_top:set_inheritable +race_top:start_the_world +race_top:tstate_set_detached +race_top:unicode_hash +race_top:Py_SET_TYPE +race_top:_PyDict_CheckConsistency +race_top:_PyImport_AcquireLock +race_top:_Py_dict_lookup_threadsafe +race_top:_imp_release_lock +race_top:_multiprocessing_SemLock_acquire_impl +race_top:builtin_compile_impl +race_top:count_next +race_top:dictiter_new +race_top:dictresize +race_top:insert_to_emptydict +race_top:insertdict +race_top:list_get_item_ref +race_top:make_pending_calls +race_top:set_add_entry +race_top:should_intern_string +race_top:worklist_pop +race_top:_PyEval_IsGILEnabled +race_top:llist_insert_tail +race_top:_Py_slot_tp_getattr_hook +race_top:add_threadstate +race_top:dump_traceback +race_top:fatal_error +race_top:mi_page_decode_padding +race_top:_multiprocessing_SemLock_release_impl +race_top:_PyFrame_GetCode +race_top:_PyFrame_Initialize +race_top:PyInterpreterState_ThreadHead +race_top:_PyObject_TryGetInstanceAttribute +race_top:_Py_qsbr_unregister +race_top:_Py_qsbr_poll +race_top:PyThreadState_Next +race_top:Py_TYPE +race_top:PyUnstable_InterpreterFrame_GetLine +race_top:sock_close +race_top:tstate_delete_common +race_top:tstate_is_freed +race_top:type_modified_unlocked +race_top:update_refs +race_top:write_thread_id +race_top:PyThreadState_Clear # https://gist.github.com/mpage/6962e8870606cfc960e159b407a0cb40 thread:pthread_create From bb8677dd7f019e4fe4b7d951ad866a582911650e Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 9 May 2024 15:30:14 -0700 Subject: [PATCH 10/13] gh-118851: Default ctx arguments to AST constructors to Load() (#118854) Co-authored-by: Alex Waygood --- Doc/library/ast.rst | 6 +++--- Doc/whatsnew/3.13.rst | 8 +++++--- Lib/test/test_ast.py | 17 +++++++++++++++++ ...24-05-09-08-46-12.gh-issue-118851.aPAoJw.rst | 2 ++ Parser/asdl_c.py | 7 +++++++ Python/Python-ast.c | 7 +++++++ 6 files changed, 41 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-09-08-46-12.gh-issue-118851.aPAoJw.rst diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 24c56f17ebb002..d4ccf282a5d00a 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -120,7 +120,8 @@ Node classes If a field that is optional in the grammar is omitted from the constructor, it defaults to ``None``. If a list field is omitted, it defaults to the empty - list. If any other field is omitted, a :exc:`DeprecationWarning` is raised + list. If a field of type :class:`!ast.expr_context` is omitted, it defaults to + :class:`Load() `. If any other field is omitted, a :exc:`DeprecationWarning` is raised and the AST node will not have this field. In Python 3.15, this condition will raise an error. @@ -596,8 +597,7 @@ Expressions * ``keywords`` holds a list of :class:`.keyword` objects representing arguments passed by keyword. - When creating a ``Call`` node, ``args`` and ``keywords`` are required, but - they can be empty lists. + The ``args`` and ``keywords`` arguments are optional and default to empty lists. .. doctest:: diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 44555718184e19..9dab458b210093 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -521,8 +521,10 @@ ast If an optional field on an AST node is not included as an argument when constructing an instance, the field will now be set to ``None``. Similarly, - if a list field is omitted, that field will now be set to an empty list. - (Previously, in both cases, the attribute would be missing on the newly + if a list field is omitted, that field will now be set to an empty list, + and if a :class:`!ast.expr_context` field is omitted, it defaults to + :class:`Load() `. + (Previously, in all cases, the attribute would be missing on the newly constructed AST node instance.) If other arguments are omitted, a :exc:`DeprecationWarning` is emitted. @@ -534,7 +536,7 @@ ast unless the class opts in to the new behavior by setting the attribute :attr:`ast.AST._field_types`. - (Contributed by Jelle Zijlstra in :gh:`105858` and :gh:`117486`.) + (Contributed by Jelle Zijlstra in :gh:`105858`, :gh:`117486`, and :gh:`118851`.) * :func:`ast.parse` now accepts an optional argument *optimize* which is passed on to the :func:`compile` built-in. This makes it diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index f6e22d44406d9e..5422c861ffb5c0 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -3036,6 +3036,23 @@ def test_FunctionDef(self): self.assertEqual(node.name, 'foo') self.assertEqual(node.decorator_list, []) + def test_expr_context(self): + name = ast.Name("x") + self.assertEqual(name.id, "x") + self.assertIsInstance(name.ctx, ast.Load) + + name2 = ast.Name("x", ast.Store()) + self.assertEqual(name2.id, "x") + self.assertIsInstance(name2.ctx, ast.Store) + + name3 = ast.Name("x", ctx=ast.Del()) + self.assertEqual(name3.id, "x") + self.assertIsInstance(name3.ctx, ast.Del) + + with self.assertWarnsRegex(DeprecationWarning, + r"Name\.__init__ missing 1 required positional argument: 'id'"): + name3 = ast.Name() + def test_custom_subclass_with_no_fields(self): class NoInit(ast.AST): pass diff --git a/Misc/NEWS.d/next/Library/2024-05-09-08-46-12.gh-issue-118851.aPAoJw.rst b/Misc/NEWS.d/next/Library/2024-05-09-08-46-12.gh-issue-118851.aPAoJw.rst new file mode 100644 index 00000000000000..d036d0cda617ef --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-09-08-46-12.gh-issue-118851.aPAoJw.rst @@ -0,0 +1,2 @@ +``ctx`` arguments to the constructors of :mod:`ast` node classes now default +to :class:`ast.Load() `. Patch by Jelle Zijlstra. diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 11d59faeb0d42c..9961d23629abc5 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -1022,6 +1022,13 @@ def visitModule(self, mod): goto set_remaining_cleanup; } } + else if (type == state->expr_context_type) { + // special case for expr_context: default to Load() + res = PyObject_SetAttr(self, name, state->Load_singleton); + if (res < 0) { + goto set_remaining_cleanup; + } + } else { // simple field (e.g., identifier) if (PyErr_WarnFormat( diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 4956d04f719de9..7aa1c5119d8f28 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -5221,6 +5221,13 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) goto set_remaining_cleanup; } } + else if (type == state->expr_context_type) { + // special case for expr_context: default to Load() + res = PyObject_SetAttr(self, name, state->Load_singleton); + if (res < 0) { + goto set_remaining_cleanup; + } + } else { // simple field (e.g., identifier) if (PyErr_WarnFormat( From 9c4ae0270d4183215383fd116fa055ba683daaf9 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Thu, 9 May 2024 18:33:53 -0400 Subject: [PATCH 11/13] Revert "gh-115432: Add critical section variant that handles a NULL object (#115433)" (#118861) This reverts commit ad4f909e0e7890e027c4ae7fea74586667242ad3. The API ended up not being used. --- Include/internal/pycore_critical_section.h | 29 ------------------- .../test_critical_sections.c | 9 ------ 2 files changed, 38 deletions(-) diff --git a/Include/internal/pycore_critical_section.h b/Include/internal/pycore_critical_section.h index 23b85c2f9e9bb2..573d09a09683ef 100644 --- a/Include/internal/pycore_critical_section.h +++ b/Include/internal/pycore_critical_section.h @@ -99,15 +99,6 @@ extern "C" { _PyCriticalSection_End(&_cs); \ } -# define Py_XBEGIN_CRITICAL_SECTION(op) \ - { \ - _PyCriticalSection _cs_opt = {0}; \ - _PyCriticalSection_XBegin(&_cs_opt, _PyObject_CAST(op)) - -# define Py_XEND_CRITICAL_SECTION() \ - _PyCriticalSection_XEnd(&_cs_opt); \ - } - # define Py_BEGIN_CRITICAL_SECTION2(a, b) \ { \ _PyCriticalSection2 _cs2; \ @@ -144,8 +135,6 @@ extern "C" { # define Py_BEGIN_CRITICAL_SECTION_MUT(mut) # define Py_BEGIN_CRITICAL_SECTION(op) # define Py_END_CRITICAL_SECTION() -# define Py_XBEGIN_CRITICAL_SECTION(op) -# define Py_XEND_CRITICAL_SECTION() # define Py_BEGIN_CRITICAL_SECTION2(a, b) # define Py_END_CRITICAL_SECTION2() # define _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(mutex) @@ -202,16 +191,6 @@ _PyCriticalSection_Begin(_PyCriticalSection *c, PyMutex *m) } } -static inline void -_PyCriticalSection_XBegin(_PyCriticalSection *c, PyObject *op) -{ -#ifdef Py_GIL_DISABLED - if (op != NULL) { - _PyCriticalSection_Begin(c, &_PyObject_CAST(op)->ob_mutex); - } -#endif -} - // Removes the top-most critical section from the thread's stack of critical // sections. If the new top-most critical section is inactive, then it is // resumed. @@ -234,14 +213,6 @@ _PyCriticalSection_End(_PyCriticalSection *c) _PyCriticalSection_Pop(c); } -static inline void -_PyCriticalSection_XEnd(_PyCriticalSection *c) -{ - if (c->mutex) { - _PyCriticalSection_End(c); - } -} - static inline void _PyCriticalSection2_Begin(_PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2) { diff --git a/Modules/_testinternalcapi/test_critical_sections.c b/Modules/_testinternalcapi/test_critical_sections.c index cdf8a70fc79ff3..1c0e049efafcb7 100644 --- a/Modules/_testinternalcapi/test_critical_sections.c +++ b/Modules/_testinternalcapi/test_critical_sections.c @@ -49,15 +49,6 @@ test_critical_sections(PyObject *self, PyObject *Py_UNUSED(args)) Py_END_CRITICAL_SECTION2(); assert_nogil(!PyMutex_IsLocked(&d2->ob_mutex)); - // Optional variant behaves the same if the object is non-NULL - Py_XBEGIN_CRITICAL_SECTION(d1); - assert_nogil(PyMutex_IsLocked(&d1->ob_mutex)); - Py_XEND_CRITICAL_SECTION(); - - // No-op - Py_XBEGIN_CRITICAL_SECTION(NULL); - Py_XEND_CRITICAL_SECTION(); - Py_DECREF(d2); Py_DECREF(d1); Py_RETURN_NONE; From 6502eb30b9f75af189f6ed6f3d246f0955fe5695 Mon Sep 17 00:00:00 2001 From: Xie Yanbo Date: Fri, 10 May 2024 16:11:50 +0800 Subject: [PATCH 12/13] Docs: fix typos in documentation (GH-118815) --- Doc/library/importlib.rst | 4 ++-- Doc/library/logging.rst | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index b58ef359378e4f..2ec15dd171c18a 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -1252,7 +1252,7 @@ find and load modules. be only a single binary per framework, and there can be no executable binary material outside the Frameworks folder. - To accomodate this requirement, when running on iOS, extension module + To accommodate this requirement, when running on iOS, extension module binaries are *not* packaged as ``.so`` files on ``sys.path``, but as individual standalone frameworks. To discover those frameworks, this loader is be registered against the ``.fwork`` file extension, with a ``.fwork`` @@ -1279,7 +1279,7 @@ find and load modules. When a module is loaded with this loader, the ``__file__`` for the module will report as the location of the ``.fwork`` file. This allows code to use - the ``__file__`` of a module as an anchor for file system traveral. + the ``__file__`` of a module as an anchor for file system traversal. However, the spec origin will reference the location of the *actual* binary in the ``.framework`` folder. diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index fb6ca38ba72aba..564b34bcf1bb37 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -1204,7 +1204,7 @@ functions. most programs will want to carefully and explicitly control the logging configuration, and should therefore prefer creating a module-level logger and calling :meth:`Logger.debug` (or other level-specific methods) on it, as - described at the beginnning of this documentation. + described at the beginning of this documentation. .. function:: info(msg, *args, **kwargs) From 0f901c9bae98e725c13b5ae3d16c4f5296d8a631 Mon Sep 17 00:00:00 2001 From: Alex Turner Date: Fri, 10 May 2024 02:22:30 -0700 Subject: [PATCH 13/13] gh-117657: initialize_new_array remove suppresion --- Tools/tsan/suppressions_free_threading.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/Tools/tsan/suppressions_free_threading.txt b/Tools/tsan/suppressions_free_threading.txt index 7f91a9113c4e1a..387338f5d5fb0f 100644 --- a/Tools/tsan/suppressions_free_threading.txt +++ b/Tools/tsan/suppressions_free_threading.txt @@ -40,7 +40,6 @@ race_top:_PyParkingLot_Park race_top:_PyType_HasFeature race_top:assign_version_tag race_top:gc_restore_tid -race_top:initialize_new_array race_top:insertdict race_top:lookup_tp_dict race_top:mi_heap_visit_pages pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy