diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index e716d7bb0f2a5c..c5b80e010bab46 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -375,10 +375,18 @@ zlib Optimizations ============= -module_name ------------ +abc +--- + +* Reduce memory usage of :func:`issubclass` checks for classes inheriting abstract classes. + + :class:`abc.ABCMeta` hook ``__subclasscheck__`` now includes + a guard which is triggered then the hook is called from a parent class + (``issubclass(cls, RootClass)`` -> ``issubclass(cls, NestedClass)`` -> ...). + This guard prevents adding ``cls`` to ``NestedClass`` positive and negative caches, + preventing memory bloat in some cases (thousands of classes inherited from ABC). -* TODO + (Contributed by Maxim Martynov in :gh:`92810`.) diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index 3e41e2fd1569ca..09540b613d2dbf 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -45,6 +45,8 @@ extern "C" { _Py_atomic_load_uint16_relaxed(&value) #define FT_ATOMIC_LOAD_UINT32_RELAXED(value) \ _Py_atomic_load_uint32_relaxed(&value) +#define FT_ATOMIC_LOAD_UINT64_RELAXED(value) \ + _Py_atomic_load_uint64_relaxed(&value) #define FT_ATOMIC_LOAD_ULONG_RELAXED(value) \ _Py_atomic_load_ulong_relaxed(&value) #define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) \ @@ -61,6 +63,8 @@ extern "C" { _Py_atomic_store_uint16_relaxed(&value, new_value) #define FT_ATOMIC_STORE_UINT32_RELAXED(value, new_value) \ _Py_atomic_store_uint32_relaxed(&value, new_value) +#define FT_ATOMIC_STORE_UINT64_RELAXED(value, new_value) \ + _Py_atomic_store_uint64_relaxed(&value, new_value) #define FT_ATOMIC_STORE_CHAR_RELAXED(value, new_value) \ _Py_atomic_store_char_relaxed(&value, new_value) #define FT_ATOMIC_LOAD_CHAR_RELAXED(value) \ @@ -111,6 +115,8 @@ extern "C" { _Py_atomic_load_ullong_relaxed(&value) #define FT_ATOMIC_ADD_SSIZE(value, new_value) \ (void)_Py_atomic_add_ssize(&value, new_value) +#define FT_ATOMIC_ADD_UINT64(value, new_value) \ + (void)_Py_atomic_add_uint64(&value, new_value) #else #define FT_ATOMIC_LOAD_PTR(value) value @@ -126,6 +132,7 @@ extern "C" { #define FT_ATOMIC_LOAD_UINT8_RELAXED(value) value #define FT_ATOMIC_LOAD_UINT16_RELAXED(value) value #define FT_ATOMIC_LOAD_UINT32_RELAXED(value) value +#define FT_ATOMIC_LOAD_UINT64_RELAXED(value) value #define FT_ATOMIC_LOAD_ULONG_RELAXED(value) value #define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_PTR_RELEASE(value, new_value) value = new_value @@ -134,6 +141,7 @@ extern "C" { #define FT_ATOMIC_STORE_UINT8_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINT16_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINT32_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_STORE_UINT64_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_LOAD_CHAR_RELAXED(value) value #define FT_ATOMIC_STORE_CHAR_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_LOAD_UCHAR_RELAXED(value) value @@ -159,6 +167,7 @@ extern "C" { #define FT_ATOMIC_LOAD_ULLONG_RELAXED(value) value #define FT_ATOMIC_STORE_ULLONG_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_ADD_SSIZE(value, new_value) (void)(value += new_value) +#define FT_ATOMIC_ADD_UINT64(value, new_value) (void)(value += new_value) #endif diff --git a/Lib/_py_abc.py b/Lib/_py_abc.py index c870ae9048b4f1..eeb18f8a60c618 100644 --- a/Lib/_py_abc.py +++ b/Lib/_py_abc.py @@ -49,6 +49,7 @@ def __new__(mcls, name, bases, namespace, /, **kwargs): cls._abc_cache = WeakSet() cls._abc_negative_cache = WeakSet() cls._abc_negative_cache_version = ABCMeta._abc_invalidation_counter + cls._abc_issubclasscheck_recursive = False return cls def register(cls, subclass): @@ -139,9 +140,24 @@ def __subclasscheck__(cls, subclass): return True # Check if it's a subclass of a subclass (recursive) for scls in cls.__subclasses__(): - if issubclass(subclass, scls): - cls._abc_cache.add(subclass) + # If inside recursive issubclass check, avoid adding classes + # to any cache because this may drastically increase memory usage. + # Unfortunately, issubclass/__subclasscheck__ don't accept third + # argument with context, so using global context within ABCMeta. + # This is done only on first method call, next will use cache. + scls_is_abc = hasattr(scls, "_abc_issubclasscheck_recursive") + if scls_is_abc: + scls._abc_issubclasscheck_recursive = True + try: + result = issubclass(subclass, scls) + finally: + if scls_is_abc: + scls._abc_issubclasscheck_recursive = False + if result: + if not cls._abc_issubclasscheck_recursive: + cls._abc_cache.add(subclass) return True # No dice; update negative cache - cls._abc_negative_cache.add(subclass) + if not cls._abc_issubclasscheck_recursive: + cls._abc_negative_cache.add(subclass) return False diff --git a/Lib/test/test_abc.py b/Lib/test/test_abc.py index 80ee9e0ba56e75..3cf7c7529e51ba 100644 --- a/Lib/test/test_abc.py +++ b/Lib/test/test_abc.py @@ -70,6 +70,25 @@ def foo(): return 4 class TestABC(unittest.TestCase): + def check_isinstance(self, obj, target_class): + self.assertIsInstance(obj, target_class) + self.assertIsInstance(obj, (target_class,)) + self.assertIsInstance(obj, target_class | target_class) + + def check_not_isinstance(self, obj, target_class): + self.assertNotIsInstance(obj, target_class) + self.assertNotIsInstance(obj, (target_class,)) + self.assertNotIsInstance(obj, target_class | target_class) + + def check_issubclass(self, klass, target_class): + self.assertIsSubclass(klass, target_class) + self.assertIsSubclass(klass, (target_class,)) + self.assertIsSubclass(klass, target_class | target_class) + + def check_not_issubclass(self, klass, target_class): + self.assertNotIsSubclass(klass, target_class) + self.assertNotIsSubclass(klass, (target_class,)) + self.assertNotIsSubclass(klass, target_class | target_class) def test_ABC_helper(self): # create an ABC using the helper class and perform basic checks @@ -270,29 +289,75 @@ def x(self): class C(metaclass=meta): pass + def test_isinstance_direct_inheritance(self): + class A(metaclass=abc_ABCMeta): + pass + class B(A): + pass + class C(A): + pass + + a = A() + b = B() + c = C() + # trigger caching + for _ in range(2): + self.check_isinstance(a, A) + self.check_not_isinstance(a, B) + self.check_not_isinstance(a, C) + + self.check_isinstance(b, B) + self.check_isinstance(b, A) + self.check_not_isinstance(b, C) + + self.check_isinstance(c, C) + self.check_isinstance(c, A) + self.check_not_isinstance(c, B) + + self.check_issubclass(B, A) + self.check_issubclass(C, A) + self.check_not_issubclass(B, C) + self.check_not_issubclass(C, B) + self.check_not_issubclass(A, B) + self.check_not_issubclass(A, C) + def test_registration_basics(self): class A(metaclass=abc_ABCMeta): pass class B(object): pass + + a = A() b = B() - self.assertNotIsSubclass(B, A) - self.assertNotIsSubclass(B, (A,)) - self.assertNotIsInstance(b, A) - self.assertNotIsInstance(b, (A,)) + # trigger caching + for _ in range(2): + self.check_not_issubclass(B, A) + self.check_not_isinstance(b, A) + + self.check_not_issubclass(A, B) + self.check_not_isinstance(a, B) + B1 = A.register(B) - self.assertIsSubclass(B, A) - self.assertIsSubclass(B, (A,)) - self.assertIsInstance(b, A) - self.assertIsInstance(b, (A,)) - self.assertIs(B1, B) + # trigger caching + for _ in range(2): + self.check_issubclass(B, A) + self.check_isinstance(b, A) + self.assertIs(B1, B) + + self.check_not_issubclass(A, B) + self.check_not_isinstance(a, B) + class C(B): pass + c = C() - self.assertIsSubclass(C, A) - self.assertIsSubclass(C, (A,)) - self.assertIsInstance(c, A) - self.assertIsInstance(c, (A,)) + # trigger caching + for _ in range(2): + self.check_issubclass(C, A) + self.check_isinstance(c, A) + + self.check_not_issubclass(A, C) + self.check_not_isinstance(a, C) def test_register_as_class_deco(self): class A(metaclass=abc_ABCMeta): @@ -377,39 +442,73 @@ class A(metaclass=abc_ABCMeta): pass self.assertIsSubclass(A, A) self.assertIsSubclass(A, (A,)) + class B(metaclass=abc_ABCMeta): pass self.assertNotIsSubclass(A, B) self.assertNotIsSubclass(A, (B,)) self.assertNotIsSubclass(B, A) self.assertNotIsSubclass(B, (A,)) + class C(metaclass=abc_ABCMeta): pass A.register(B) class B1(B): pass - self.assertIsSubclass(B1, A) - self.assertIsSubclass(B1, (A,)) + # trigger caching + for _ in range(2): + self.assertIsSubclass(B1, A) + self.assertIsSubclass(B1, (A,)) + class C1(C): pass B1.register(C1) - self.assertNotIsSubclass(C, B) - self.assertNotIsSubclass(C, (B,)) - self.assertNotIsSubclass(C, B1) - self.assertNotIsSubclass(C, (B1,)) - self.assertIsSubclass(C1, A) - self.assertIsSubclass(C1, (A,)) - self.assertIsSubclass(C1, B) - self.assertIsSubclass(C1, (B,)) - self.assertIsSubclass(C1, B1) - self.assertIsSubclass(C1, (B1,)) + # trigger caching + for _ in range(2): + self.assertNotIsSubclass(C, B) + self.assertNotIsSubclass(C, (B,)) + self.assertNotIsSubclass(C, B1) + self.assertNotIsSubclass(C, (B1,)) + self.assertIsSubclass(C1, A) + self.assertIsSubclass(C1, (A,)) + self.assertIsSubclass(C1, B) + self.assertIsSubclass(C1, (B,)) + self.assertIsSubclass(C1, B1) + self.assertIsSubclass(C1, (B1,)) + C1.register(int) class MyInt(int): pass - self.assertIsSubclass(MyInt, A) - self.assertIsSubclass(MyInt, (A,)) - self.assertIsInstance(42, A) - self.assertIsInstance(42, (A,)) + # trigger caching + for _ in range(2): + self.assertIsSubclass(MyInt, A) + self.assertIsSubclass(MyInt, (A,)) + self.assertIsInstance(42, A) + self.assertIsInstance(42, (A,)) + + def test_custom_subclasses(self): + class A: pass + class B: pass + + class Parent1(metaclass=abc_ABCMeta): + @classmethod + def __subclasses__(cls): + return [A] + + class Parent2(metaclass=abc_ABCMeta): + __subclasses__ = lambda: [A] + + # trigger caching + for _ in range(2): + self.check_isinstance(A(), Parent1) + self.check_issubclass(A, Parent1) + self.check_not_isinstance(B(), Parent1) + self.check_not_issubclass(B, Parent1) + + self.check_isinstance(A(), Parent2) + self.check_issubclass(A, Parent2) + self.check_not_isinstance(B(), Parent2) + self.check_not_issubclass(B, Parent2) def test_issubclass_bad_arguments(self): class A(metaclass=abc_ABCMeta): @@ -460,8 +559,32 @@ class S(metaclass=abc_ABCMeta): with self.assertRaisesRegex(CustomError, exc_msg): issubclass(int, S) - def test_subclasshook(self): + def test_issubclass_bad_class(self): class A(metaclass=abc.ABCMeta): + pass + + A._abc_impl = 1 + error_msg = "_abc_impl is set to a wrong type" + with self.assertRaisesRegex(TypeError, error_msg): + issubclass(A, A) + + class B(metaclass=_py_abc.ABCMeta): + pass + + B._abc_cache = 1 + error_msg = "argument of type 'int' is not a container or iterable" + with self.assertRaisesRegex(TypeError, error_msg): + issubclass(B, B) + + class C(metaclass=_py_abc.ABCMeta): + pass + + C._abc_negative_cache = 1 + with self.assertRaisesRegex(TypeError, error_msg): + issubclass(C, C) + + def test_subclasshook(self): + class A(metaclass=abc_ABCMeta): @classmethod def __subclasshook__(cls, C): if cls is A: @@ -478,6 +601,26 @@ class C: self.assertNotIsSubclass(C, A) self.assertNotIsSubclass(C, (A,)) + def test_subclasshook_exception(self): + # Check that issubclass() propagates exceptions raised by + # __subclasshook__. + class CustomError(Exception): ... + exc_msg = "exception from __subclasshook__" + class A(metaclass=abc_ABCMeta): + @classmethod + def __subclasshook__(cls, C): + raise CustomError(exc_msg) + with self.assertRaisesRegex(CustomError, exc_msg): + issubclass(A, A) + class B(A): + pass + with self.assertRaisesRegex(CustomError, exc_msg): + issubclass(B, A) + class C: + pass + with self.assertRaisesRegex(CustomError, exc_msg): + issubclass(C, A) + def test_all_new_methods_are_called(self): class A(metaclass=abc_ABCMeta): pass @@ -522,7 +665,6 @@ def foo(self): self.assertEqual(A.__abstractmethods__, set()) A() - def test_update_new_abstractmethods(self): class A(metaclass=abc_ABCMeta): @abc.abstractmethod diff --git a/Lib/test/test_isinstance.py b/Lib/test/test_isinstance.py index f440fc28ee7b7d..1786f052f37923 100644 --- a/Lib/test/test_isinstance.py +++ b/Lib/test/test_isinstance.py @@ -353,6 +353,28 @@ class B: with support.infinite_recursion(25): self.assertRaises(RecursionError, issubclass, X(), int) + def test_custom_subclasses_are_ignored(self): + class A: pass + class B: pass + + class Parent1: + @classmethod + def __subclasses__(cls): + return [A, B] + + class Parent2: + __subclasses__ = lambda: [A, B] + + self.assertNotIsInstance(A(), Parent1) + self.assertNotIsInstance(B(), Parent1) + self.assertNotIsSubclass(A, Parent1) + self.assertNotIsSubclass(B, Parent1) + + self.assertNotIsInstance(A(), Parent2) + self.assertNotIsInstance(B(), Parent2) + self.assertNotIsSubclass(A, Parent2) + self.assertNotIsSubclass(B, Parent2) + def blowstack(fxn, arg, compare_to): # Make sure that calling isinstance with a deeply nested tuple for its diff --git a/Misc/NEWS.d/next/Library/2025-06-13-17-22-34.gh-issue-92810.Tb6x6C.rst b/Misc/NEWS.d/next/Library/2025-06-13-17-22-34.gh-issue-92810.Tb6x6C.rst new file mode 100644 index 00000000000000..13cd06eb821bf4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-06-13-17-22-34.gh-issue-92810.Tb6x6C.rst @@ -0,0 +1,2 @@ +Reduce memory usage by :meth:`~type.__subclasscheck__` +for :class:`abc.ABCMeta` and large class trees. diff --git a/Modules/_abc.c b/Modules/_abc.c index d6a953b336025d..73013ba065c198 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -35,21 +35,13 @@ get_abc_state(PyObject *module) static inline uint64_t get_invalidation_counter(_abcmodule_state *state) { -#ifdef Py_GIL_DISABLED - return _Py_atomic_load_uint64(&state->abc_invalidation_counter); -#else - return state->abc_invalidation_counter; -#endif + return FT_ATOMIC_LOAD_UINT64_RELAXED(state->abc_invalidation_counter); } static inline void increment_invalidation_counter(_abcmodule_state *state) { -#ifdef Py_GIL_DISABLED - _Py_atomic_add_uint64(&state->abc_invalidation_counter, 1); -#else - state->abc_invalidation_counter++; -#endif + FT_ATOMIC_ADD_UINT64(state->abc_invalidation_counter, 1); } /* This object stores internal state for ABCs. @@ -65,6 +57,7 @@ typedef struct { PyObject *_abc_cache; PyObject *_abc_negative_cache; uint64_t _abc_negative_cache_version; + uint8_t _abc_issubclasscheck_recursive; } _abc_data; #define _abc_data_CAST(op) ((_abc_data *)(op)) @@ -72,21 +65,31 @@ typedef struct { static inline uint64_t get_cache_version(_abc_data *impl) { -#ifdef Py_GIL_DISABLED - return _Py_atomic_load_uint64(&impl->_abc_negative_cache_version); -#else - return impl->_abc_negative_cache_version; -#endif + return FT_ATOMIC_LOAD_UINT64_RELAXED(impl->_abc_negative_cache_version); } static inline void set_cache_version(_abc_data *impl, uint64_t version) { -#ifdef Py_GIL_DISABLED - _Py_atomic_store_uint64(&impl->_abc_negative_cache_version, version); -#else - impl->_abc_negative_cache_version = version; -#endif + FT_ATOMIC_STORE_UINT64_RELAXED(impl->_abc_negative_cache_version, version); +} + +static inline uint8_t +is_issubclasscheck_recursive(_abc_data *impl) +{ + return FT_ATOMIC_LOAD_UINT8_RELAXED(impl->_abc_issubclasscheck_recursive); +} + +static inline void +set_issubclasscheck_recursive(_abc_data *impl) +{ + FT_ATOMIC_STORE_UINT8_RELAXED(impl->_abc_issubclasscheck_recursive, 1); +} + +static inline void +unset_issubclasscheck_recursive(_abc_data *impl) +{ + FT_ATOMIC_STORE_UINT8_RELAXED(impl->_abc_issubclasscheck_recursive, 0); } static int @@ -139,6 +142,7 @@ abc_data_new(PyTypeObject *type, PyObject *args, PyObject *kwds) self->_abc_cache = NULL; self->_abc_negative_cache = NULL; self->_abc_negative_cache_version = get_invalidation_counter(state); + self->_abc_issubclasscheck_recursive = 0; return (PyObject *) self; } @@ -177,6 +181,30 @@ _get_impl(PyObject *module, PyObject *self) return (_abc_data *)impl; } +/* If class is inherited from ABC, set data to point to internal ABC state of class, and return 1. + If object is not inherited from ABC, return 0. + If error is encountered, return -1. + */ +static int +_get_optional_impl(_abcmodule_state *state, PyObject *self, _abc_data **data) +{ + assert(data != NULL); + PyObject *impl = NULL; + int res = PyObject_GetOptionalAttr(self, &_Py_ID(_abc_impl), &impl); + if (res <= 0) { + *data = NULL; + return res; + } + if (!Py_IS_TYPE(impl, state->_abc_data_type)) { + PyErr_SetString(PyExc_TypeError, "_abc_impl is set to a wrong type"); + Py_DECREF(impl); + *data = NULL; + return -1; + } + *data = (_abc_data *)impl; + return 1; +} + static int _in_weak_set(_abc_data *impl, PyObject **pset, PyObject *obj) { @@ -347,11 +375,12 @@ _abc__get_dump(PyObject *module, PyObject *self) } PyObject *res; Py_BEGIN_CRITICAL_SECTION(impl); - res = Py_BuildValue("NNNK", + res = Py_BuildValue("NNNKK", PySet_New(impl->_abc_registry), PySet_New(impl->_abc_cache), PySet_New(impl->_abc_negative_cache), - get_cache_version(impl)); + get_cache_version(impl), + is_issubclasscheck_recursive(impl)); Py_END_CRITICAL_SECTION(); Py_DECREF(impl); return res; @@ -814,23 +843,52 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, if (scls == NULL) { goto end; } + + _abc_data *scls_impl; + int scls_is_abc = _get_optional_impl(state, scls, &scls_impl); + if (scls_is_abc < 0) { + Py_DECREF(scls); + goto end; + } + if (scls_is_abc > 0) { + /* + If inside recursive issubclass check, avoid adding classes + to any cache because this may drastically increase memory usage. + Unfortunately, issubclass/__subclasscheck__ don't accept third + argument with context, so using global context within ABCMeta. + This is done only on first method call, next will use cache. + */ + set_issubclasscheck_recursive(scls_impl); + } + int r = PyObject_IsSubclass(subclass, scls); Py_DECREF(scls); + + if (scls_is_abc > 0) { + // reset recursion guard even if exception was raised in __subclasscheck__ + unset_issubclasscheck_recursive(scls_impl); + } + Py_XDECREF(scls_impl); + + if (r < 0) { + goto end; + } if (r > 0) { - if (_add_to_weak_set(impl, &impl->_abc_cache, subclass) < 0) { - goto end; + if (!is_issubclasscheck_recursive(impl)) { + if (_add_to_weak_set(impl, &impl->_abc_cache, subclass) < 0) { + goto end; + } } result = Py_True; goto end; } - if (r < 0) { - goto end; - } } /* No dice; update negative cache. */ - if (_add_to_weak_set(impl, &impl->_abc_negative_cache, subclass) < 0) { - goto end; + if (!is_issubclasscheck_recursive(impl)) { + if (_add_to_weak_set(impl, &impl->_abc_negative_cache, subclass) < 0) { + goto end; + } } result = Py_False; pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy