merge with upstream

python · colesbury · Apr 29, 2024 · Apr 26, 2024 · Apr 26, 2024 · Apr 26, 2024
commit d83833c8d36d9326dc857e4bbf9285e8592cc567
diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml
@@ -74,7 +74,7 @@ jobs:
     - name: 'Set up Python'
       uses: actions/setup-python@v5
       with:
-        python-version: '3.11'  # known to work with Sphinx 4.2
+        python-version: '3.12'  # known to work with Sphinx 6.2.1
         cache: 'pip'
         cache-dependency-path: 'Doc/requirements-oldest-sphinx.txt'
     - name: 'Install build dependencies'

diff --git a/Doc/conf.py b/Doc/conf.py
@@ -298,8 +298,8 @@
     'languages': ['ja', 'fr', 'zh_TW', 'zh_CN'], 'builders': ['man', 'text'],
 }
 
-# Avoid a warning with Sphinx >= 2.0
-master_doc = 'contents'
+# Avoid a warning with Sphinx >= 4.0
+root_doc = 'contents'
 
 # Allow translation of index directives
 gettext_additional_targets = [

diff --git a/Doc/requirements-oldest-sphinx.txt b/Doc/requirements-oldest-sphinx.txt
@@ -7,29 +7,29 @@ blurb
 python-docs-theme>=2022.1
 
 # Generated from:
-#     pip install "Sphinx~=4.2.0"
+#     pip install "Sphinx~=6.2.1"
 #     pip freeze
 #
-# Sphinx 4.2 comes from ``needs_sphinx = '4.2'`` in ``Doc/conf.py``.
+# Sphinx 6.2.1 comes from ``needs_sphinx = '6.2.1'`` in ``Doc/conf.py``.
 
-alabaster==0.7.13
-Babel==2.13.0
-certifi==2023.7.22
-charset-normalizer==3.3.0
-docutils==0.17.1
-idna==3.4
+alabaster==0.7.16
+Babel==2.14.0
+certifi==2024.2.2
+charset-normalizer==3.3.2
+docutils==0.19
+idna==3.7
 imagesize==1.4.1
-Jinja2==3.1.2
-MarkupSafe==2.1.3
-packaging==23.2
-Pygments==2.16.1
+Jinja2==3.1.3
+MarkupSafe==2.1.5
+packaging==24.0
+Pygments==2.17.2
 requests==2.31.0
 snowballstemmer==2.2.0
-Sphinx==4.2.0
-sphinxcontrib-applehelp==1.0.4
-sphinxcontrib-devhelp==1.0.2
-sphinxcontrib-htmlhelp==2.0.1
+Sphinx==6.2.1
+sphinxcontrib-applehelp==1.0.8
+sphinxcontrib-devhelp==1.0.6
+sphinxcontrib-htmlhelp==2.0.5
 sphinxcontrib-jsmath==1.0.1
-sphinxcontrib-qthelp==1.0.3
-sphinxcontrib-serializinghtml==1.1.5
-urllib3==2.0.7
+sphinxcontrib-qthelp==1.0.7
+sphinxcontrib-serializinghtml==1.1.10
+urllib3==2.2.1
diff --git a/Doc/tools/extensions/c_annotations.py b/Doc/tools/extensions/c_annotations.py
@@ -19,7 +19,6 @@
 """
 
 from os import path
-import docutils
 from docutils import nodes
 from docutils.parsers.rst import directives
 from docutils.parsers.rst import Directive
@@ -40,16 +39,6 @@
 }
 
 
-# Monkeypatch nodes.Node.findall for forwards compatibility
-# This patch can be dropped when the minimum Sphinx version is 4.4.0
-# or the minimum Docutils version is 0.18.1.
-if docutils.__version_info__ < (0, 18, 1):
-    def findall(self, *args, **kwargs):
-        return iter(self.traverse(*args, **kwargs))
-
-    nodes.Node.findall = findall
-
-
 class RCEntry:
     def __init__(self, name):
         self.name = name

diff --git a/Doc/tools/extensions/peg_highlight.py b/Doc/tools/extensions/peg_highlight.py
@@ -16,6 +16,7 @@ class PEGLexer(RegexLexer):
         - Rule types
         - Rule options
         - Rules named `invalid_*` or `incorrect_*`
+        - Rules with `RAISE_SYNTAX_ERROR`
     """
 
     name = "PEG"
@@ -59,6 +60,7 @@ class PEGLexer(RegexLexer):
             (r"^(\s+\|\s+.*invalid_\w+.*\n)", bygroups(None)),
             (r"^(\s+\|\s+.*incorrect_\w+.*\n)", bygroups(None)),
             (r"^(#.*invalid syntax.*(?:.|\n)*)", bygroups(None),),
+            (r"^(\s+\|\s+.*\{[^}]*RAISE_SYNTAX_ERROR[^}]*\})\n", bygroups(None)),
         ],
         "root": [
             include("invalids"),

diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py
@@ -27,13 +27,7 @@
 from sphinx.util import logging
 from sphinx.util.docutils import SphinxDirective
 from sphinx.writers.text import TextWriter, TextTranslator
-
-try:
-    # Sphinx 6+
-    from sphinx.util.display import status_iterator
-except ImportError:
-    # Deprecated in Sphinx 6.1, will be removed in Sphinx 8
-    from sphinx.util import status_iterator
+from sphinx.util.display import status_iterator
 
 
 ISSUE_URI = 'https://bugs.python.org/issue?@action=redirect&bpo=%s'

diff --git a/Include/cpython/dictobject.h b/Include/cpython/dictobject.h
@@ -56,7 +56,11 @@ static inline Py_ssize_t PyDict_GET_SIZE(PyObject *op) {
     PyDictObject *mp;
     assert(PyDict_Check(op));
     mp = _Py_CAST(PyDictObject*, op);
+#ifdef Py_GIL_DISABLED
+    return _Py_atomic_load_ssize_relaxed(&mp->ma_used);
+#else
     return mp->ma_used;
+#endif
 }
 #define PyDict_GET_SIZE(op) PyDict_GET_SIZE(_PyObject_CAST(op))
 

diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h
@@ -48,8 +48,12 @@ extern void _PyEval_SignalReceived(void);
 #define _Py_PENDING_MAINTHREADONLY 1
 #define _Py_PENDING_RAWFREE 2
 
+typedef int _Py_add_pending_call_result;
+#define _Py_ADD_PENDING_SUCCESS 0
+#define _Py_ADD_PENDING_FULL -1
+
 // Export for '_testinternalcapi' shared extension
-PyAPI_FUNC(int) _PyEval_AddPendingCall(
+PyAPI_FUNC(_Py_add_pending_call_result) _PyEval_AddPendingCall(
     PyInterpreterState *interp,
     _Py_pending_call_func func,
     void *arg,

diff --git a/Include/internal/pycore_ceval_state.h b/Include/internal/pycore_ceval_state.h
@@ -14,28 +14,56 @@ extern "C" {
 
 typedef int (*_Py_pending_call_func)(void *);
 
+struct _pending_call {
+    _Py_pending_call_func func;
+    void *arg;
+    int flags;
+};
+
+#define PENDINGCALLSARRAYSIZE 32
+
+#define MAXPENDINGCALLS PENDINGCALLSARRAYSIZE
+/* For interpreter-level pending calls, we want to avoid spending too
+   much time on pending calls in any one thread, so we apply a limit. */
+#if MAXPENDINGCALLS > 100
+#  define MAXPENDINGCALLSLOOP 100
+#else
+#  define MAXPENDINGCALLSLOOP MAXPENDINGCALLS
+#endif
+
+#define MAXPENDINGCALLS_MAIN PENDINGCALLSARRAYSIZE
+/* For the main thread, we want to make sure all pending calls are
+   run at once, for the sake of prompt signal handling.  This is
+   unlikely to cause any problems since there should be very few
+   pending calls for the main thread. */
+#define MAXPENDINGCALLSLOOP_MAIN 0
+
 struct _pending_calls {
     int busy;
     PyMutex mutex;
     /* Request for running pending calls. */
-    int32_t calls_to_do;
-#define NPENDINGCALLS 32
-    struct _pending_call {
-        _Py_pending_call_func func;
-        void *arg;
-        int flags;
-    } calls[NPENDINGCALLS];
+    int32_t npending;
+    /* The maximum allowed number of pending calls.
+       If the queue fills up to this point then _PyEval_AddPendingCall()
+       will return _Py_ADD_PENDING_FULL. */
+    int32_t max;
+    /* We don't want a flood of pending calls to interrupt any one thread
+       for too long, so we keep a limit on the number handled per pass.
+       A value of 0 means there is no limit (other than the maximum
+       size of the list of pending calls). */
+    int32_t maxloop;
+    struct _pending_call calls[PENDINGCALLSARRAYSIZE];
     int first;
-    int last;
+    int next;
 };
 
+
 typedef enum {
     PERF_STATUS_FAILED = -1,  // Perf trampoline is in an invalid state
     PERF_STATUS_NO_INIT = 0,  // Perf trampoline is not initialized
     PERF_STATUS_OK = 1,       // Perf trampoline is ready to be executed
 } perf_status_t;
 
-
 #ifdef PY_HAVE_PERF_TRAMPOLINE
 struct code_arena_st;
 
@@ -48,6 +76,7 @@ struct trampoline_api_st {
 };
 #endif
 
+
 struct _ceval_runtime_state {
     struct {
 #ifdef PY_HAVE_PERF_TRAMPOLINE
@@ -62,10 +91,15 @@ struct _ceval_runtime_state {
 #endif
     } perf;
     /* Pending calls to be made only on the main thread. */
+    // The signal machinery falls back on this
+    // so it must be especially stable and efficient.
+    // For example, we use a preallocated array
+    // for the list of pending calls.
     struct _pending_calls pending_mainthread;
     PyMutex sys_trace_profile_mutex;
 };
 
+
 #ifdef PY_HAVE_PERF_TRAMPOLINE
 # define _PyEval_RUNTIME_PERF_INIT \
     { \

diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h
@@ -688,7 +688,7 @@ static inline PyDictObject *
 _PyObject_GetManagedDict(PyObject *obj)
 {
     PyManagedDictPointer *dorv = _PyObject_ManagedDictPointer(obj);
-    return (PyDictObject *)FT_ATOMIC_LOAD_PTR_RELAXED(dorv->dict);
+    return (PyDictObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(dorv->dict);
 }
 
 static inline PyDictValues *

diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h
@@ -114,6 +114,10 @@ extern PyTypeObject _PyExc_MemoryError;
         .autoTSSkey = Py_tss_NEEDS_INIT, \
         .parser = _parser_runtime_state_INIT, \
         .ceval = { \
+            .pending_mainthread = { \
+                .max = MAXPENDINGCALLS_MAIN, \
+                .maxloop = MAXPENDINGCALLSLOOP_MAIN, \
+            }, \
             .perf = _PyEval_RUNTIME_PERF_INIT, \
         }, \
         .gilstate = { \
@@ -166,6 +170,10 @@ extern PyTypeObject _PyExc_MemoryError;
         .imports = IMPORTS_INIT, \
         .ceval = { \
             .recursion_limit = Py_DEFAULT_RECURSION_LIMIT, \
+            .pending = { \
+                .max = MAXPENDINGCALLS, \
+                .maxloop = MAXPENDINGCALLSLOOP, \
+            }, \
         }, \
         .gc = { \
             .enabled = 1, \

diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h
@@ -164,6 +164,7 @@ extern PyObject * _PyType_GetBases(PyTypeObject *type);
 extern PyObject * _PyType_GetMRO(PyTypeObject *type);
 extern PyObject* _PyType_GetSubclasses(PyTypeObject *);
 extern int _PyType_HasSubclasses(PyTypeObject *);
+PyAPI_FUNC(PyObject *) _PyType_GetModuleByDef2(PyTypeObject *, PyTypeObject *, PyModuleDef *);
 
 // PyType_Ready() must be called if _PyType_IsReady() is false.
 // See also the Py_TPFLAGS_READY flag.

diff --git a/Include/object.h b/Include/object.h
@@ -349,7 +349,8 @@ static inline Py_ssize_t Py_SIZE(PyObject *ob) {
 static inline Py_ALWAYS_INLINE int _Py_IsImmortal(PyObject *op)
 {
 #if defined(Py_GIL_DISABLED)
-    return (op->ob_ref_local == _Py_IMMORTAL_REFCNT_LOCAL);
+    return (_Py_atomic_load_uint32_relaxed(&op->ob_ref_local) ==
+            _Py_IMMORTAL_REFCNT_LOCAL);
 #elif SIZEOF_VOID_P > 4
     return (_Py_CAST(PY_INT32_T, op->ob_refcnt) < 0);
 #else

diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py
@@ -2142,6 +2142,9 @@ def is_loopback(self):
             RFC 2373 2.5.3.
 
         """
+        ipv4_mapped = self.ipv4_mapped
+        if ipv4_mapped is not None:
+            return ipv4_mapped.is_loopback
         return self._ip == 1
 
     @property
@@ -2258,7 +2261,7 @@ def is_unspecified(self):
 
     @property
     def is_loopback(self):
-        return self._ip == 1 and self.network.is_loopback
+        return super().is_loopback and self.network.is_loopback
 
 
 class IPv6Network(_BaseV6, _BaseNetwork):

diff --git a/Lib/test/test_capi/test_mem.py b/Lib/test/test_capi/test_mem.py
@@ -153,7 +153,7 @@ class C(): pass
 
 
 # free-threading requires mimalloc (not malloc)
-@support.requires_gil_enabled
+@support.requires_gil_enabled()
 class PyMemMallocDebugTests(PyMemDebugTests):
     PYTHONMALLOC = 'malloc_debug'