From 289f94be5f10f9e57c6e241fa9340566a15d0c6c Mon Sep 17 00:00:00 2001 From: Anson Mansfield Date: Fri, 19 Jul 2024 13:01:09 -0400 Subject: [PATCH 1/3] tests/basics/class_setname: Test for PEP487 __set_name__. Including the stochastic tests needed to guarantee sensitivity to the potential iterate-while-modifying hazard a naive implementation might have. Signed-off-by: Anson Mansfield --- tests/basics/class_descriptor.py | 20 ++- tests/basics/class_setname_hazard.py | 179 ++++++++++++++++++++++ tests/basics/class_setname_hazard_rand.py | 130 ++++++++++++++++ 3 files changed, 322 insertions(+), 7 deletions(-) create mode 100644 tests/basics/class_setname_hazard.py create mode 100644 tests/basics/class_setname_hazard_rand.py diff --git a/tests/basics/class_descriptor.py b/tests/basics/class_descriptor.py index 83d31674301d5..feaed2fbb2a43 100644 --- a/tests/basics/class_descriptor.py +++ b/tests/basics/class_descriptor.py @@ -1,22 +1,28 @@ class Descriptor: def __get__(self, obj, cls): - print('get') + print("get") print(type(obj) is Main) print(cls is Main) - return 'result' + return "result" def __set__(self, obj, val): - print('set') + print("set") print(type(obj) is Main) print(val) def __delete__(self, obj): - print('delete') + print("delete") print(type(obj) is Main) + def __set_name__(self, owner, name): + print("set_name", name) + print(owner.__name__ == "Main") + + class Main: Forward = Descriptor() + m = Main() try: m.__class__ @@ -26,15 +32,15 @@ class Main: raise SystemExit r = m.Forward -if 'Descriptor' in repr(r.__class__): +if "Descriptor" in repr(r.__class__): # Target doesn't support descriptors. - print('SKIP') + print("SKIP") raise SystemExit # Test assignment and deletion. print(r) -m.Forward = 'a' +m.Forward = "a" del m.Forward # Test that lookup of descriptors like __get__ are not passed into __getattr__. diff --git a/tests/basics/class_setname_hazard.py b/tests/basics/class_setname_hazard.py new file mode 100644 index 0000000000000..dad4089a66610 --- /dev/null +++ b/tests/basics/class_setname_hazard.py @@ -0,0 +1,179 @@ +def skip_if_no_descriptors(): + class Descriptor: + def __get__(self, obj, cls): + return + + class TestClass: + Forward = Descriptor() + + a = TestClass() + try: + a.__class__ + except AttributeError: + # Target doesn't support __class__. + print("SKIP") + raise SystemExit + + b = a.Forward + if "Descriptor" in repr(b.__class__): + # Target doesn't support descriptors. + print("SKIP") + raise SystemExit + + +skip_if_no_descriptors() + + +# Test basic hazard-free mutations of the enclosing class. + + +class GetSibling: + def __set_name__(self, owner, name): + print(getattr(owner, name + "_sib")) + + +class GetSiblingTest: + desc = GetSibling() + desc_sib = 111 + + +t110 = GetSiblingTest() + + +class SetSibling: + def __set_name__(self, owner, name): + setattr(owner, name + "_sib", 121) + + +class SetSiblingTest: + desc = SetSibling() + + +t120 = SetSiblingTest() + +print(t120.desc_sib) + + +class DelSibling: + def __set_name__(self, owner, name): + delattr(owner, name + "_sib") + + +class DelSiblingTest: + desc = DelSibling() + desc_sib = 131 + + +t130 = DelSiblingTest() + +try: + print(t130.desc_sib) +except AttributeError: + print("AttributeError") + + +class GetSelf: + x = 211 + + def __set_name__(self, owner, name): + print(getattr(owner, name).x) + + +class GetSelfTest: + desc = GetSelf() + + +t210 = GetSelfTest() + + +class SetSelf: + def __set_name__(self, owner, name): + setattr(owner, name, 221) + + +class SetSelfTest: + desc = SetSelf() + + +t220 = SetSelfTest() + +print(t220.desc) + + +class DelSelf: + def __set_name__(self, owner, name): + delattr(owner, name) + + +class DelSelfTest: + desc = DelSelf() + + +t230 = DelSelfTest() + +try: + print(t230.desc) +except AttributeError: + print("AttributeError") + + +# Test exception behavior + + +class Raise: + def __set_name__(self, owner, name): + raise Exception() + + +try: + + class RaiseTest: + desc = Raise() +except Exception as e: + print("Exception") + + +# Test simple hazards: whether other class attributes still get __set_name__ even if removed before being run + + +class SetSpecific: + def __init__(self, sib_name, sib_replace): + self.sib_name = sib_name + self.sib_replace = sib_replace + + def __set_name__(self, owner, name): + setattr(owner, self.sib_name, self.sib_replace) + + +class SetReplaceTest: + a = SetSpecific("b", 312) # one of these is changed first + b = SetSpecific("a", 311) + + +t310 = SetReplaceTest() +print(t310.a) +print(t310.b) + + +class DelSpecific: + def __init__(self, sib_name): + self.sib_name = sib_name + + def __set_name__(self, owner, name): + delattr(owner, self.sib_name) + + +class DelReplaceTest: + a = DelSpecific("b") # one of these is removed first + b = DelSpecific("a") + + +t320 = DelReplaceTest() +try: + print(t320.a) +except AttributeError: + print("AttributeError") +try: + print(t320.b) +except AttributeError: + print("AttributeError") diff --git a/tests/basics/class_setname_hazard_rand.py b/tests/basics/class_setname_hazard_rand.py new file mode 100644 index 0000000000000..10270285dc5a4 --- /dev/null +++ b/tests/basics/class_setname_hazard_rand.py @@ -0,0 +1,130 @@ +# Test to make sure there's no sequence hazard even when a __set_name__ implementation +# mutates and reorders the class namespace. +# VERY hard bug to prove out except via a stochastic test. + + +def skip_if_no_descriptors(): + class Descriptor: + def __get__(self, obj, cls): + return + + class TestClass: + Forward = Descriptor() + + a = TestClass() + try: + a.__class__ + except AttributeError: + # Target doesn't support __class__. + print("SKIP") + raise SystemExit + + b = a.Forward + if "Descriptor" in repr(b.__class__): + # Target doesn't support descriptors. + print("SKIP") + raise SystemExit + + +def skip_if_no_libs(): + try: + import random + except ImportError: + # Target doesn't have a random library + print("SKIP") + raise SystemExit + + try: + random.choice + except AttributeError: + # Target doesn't have an ACTUAL random library + print("SKIP") + raise SystemExit + + try: + import re + except ImportError: + # Target doesn't have a regex library + print("SKIP") + raise SystemExit + + +skip_if_no_descriptors() +skip_if_no_libs() + + +import random +import re + +letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + +junk_re = re.compile(r"[A-Z][A-Z][A-Z][A-Z][A-Z]") + + +def junk_fill(obj, n=10): # Add randomly-generated attributes to an object. + for i in range(n): + name = "".join(random.choice(letters) for j in range(5)) + setattr(obj, name, object()) + + +def junk_clear(obj): # Remove attributes added by junk_fill. + to_del = [name for name in dir(obj) if junk_re.match(name)] + for name in to_del: + delattr(obj, name) + + +def junk_sequencer(): + global runs + try: + while True: + owner, name = yield + runs[name] = runs.get(name, 0) + 1 + junk_fill(owner) + finally: + junk_clear(owner) + + +class JunkMaker: + def __set_name__(self, owner, name): + global seq + seq.send((owner, name)) + + +runs = {} +seq = junk_sequencer() +next(seq) + + +class Main: + a = JunkMaker() + b = JunkMaker() + c = JunkMaker() + d = JunkMaker() + e = JunkMaker() + f = JunkMaker() + g = JunkMaker() + h = JunkMaker() + i = JunkMaker() + j = JunkMaker() + k = JunkMaker() + l = JunkMaker() + m = JunkMaker() + n = JunkMaker() + o = JunkMaker() + p = JunkMaker() + q = JunkMaker() + r = JunkMaker() + s = JunkMaker() + t = JunkMaker() + u = JunkMaker() + v = JunkMaker() + w = JunkMaker() + x = JunkMaker() + y = JunkMaker() + z = JunkMaker() + + +seq.close() + +for k in letters.lower(): + print(k, runs.get(k, 0)) From f6acdfad8704e8c0423c7423758755cc643dfd79 Mon Sep 17 00:00:00 2001 From: Anson Mansfield Date: Fri, 19 Jul 2024 18:23:45 -0400 Subject: [PATCH 2/3] py/objtype: Add support for PEP487 __set_name__. This PR adds support for the `__set_name__` data model method specified by PEP487 - Simpler customisation of class creation. This includes support for methods that mutate the owner class, and avoids the naive modify-while-iterating hazard possible in a naive implementation like micropython/micropython#15503. Note that based on the benchmarks in micropython/micropython#16825, this is also as fast or faster than the naive implementation, thanks to clever data layout in `setname_list_t`, and the way this allows the capture step to run during an existing loop through the class dict. Other rejected approaches for dealing with the hazard include: - python/cpython#72983 During the implementation of this feature for MicroPython, it was discovered that some versions of CPython also have this naive hazard. CPython resolved this bug in BPO-28797 and now makes a complete flat copy of the class's dict to iterate. This design decision doesn't make much sense for a microcontroller though, even if it's perfectly reasonable in the desktop world where memcpy might actually be cheaper than a hard-to-branch-predict conditional; and it's also motivated in their case by error-tracing considerations. - micropython/micropython#16816 This is an equivalent implementation to CPython's approach that places this copy directly on the stack; however it is both slower and has larger code size than the approach taken here. - micropython/micropython#15503 The simplest implementation is to just not worry about it and let the user face the consequences if they mutate the owner class. That's not a very friendly behavior, though, and it's not actually much more performant than this implementation on either time or code size. - micropython/micropython#17693 Another alternative is to do the same as #15503 but leverage MicroPython's existing `is_fixed` field in its dict type to convert attempted mutations of the owner dict into `AttributeError`s. This is safer than just leaving the open hazard, but there's still important use-cases for owner-mutating descriptors, and the performance ain is small enough that it isn't worth missing support for those cases. - combined micropython/micropython#17693 with this Another version of this feature used a new feature define, `MICROPY_PY_METACLASSES_LITE`, to control whether this algorithm or the naive version is used. This was rejected in favor of simplicity, based on the very limited performance margin the naive version has (which in some cases even goes _against_ it). Signed-off-by: Anson Mansfield --- py/objtype.c | 90 ++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 77 insertions(+), 13 deletions(-) diff --git a/py/objtype.c b/py/objtype.c index b9af1008995ef..7455384541a49 100644 --- a/py/objtype.c +++ b/py/objtype.c @@ -661,8 +661,8 @@ static void mp_obj_instance_load_attr(mp_obj_t self_in, qstr attr, mp_obj_t *des // try __getattr__ if (attr != MP_QSTR___getattr__) { #if MICROPY_PY_DESCRIPTORS - // With descriptors enabled, don't delegate lookups of __get__/__set__/__delete__. - if (attr == MP_QSTR___get__ || attr == MP_QSTR___set__ || attr == MP_QSTR___delete__) { + // With descriptors enabled, don't delegate lookups of __get__/__set__/__delete__/__set_name__. + if (attr == MP_QSTR___get__ || attr == MP_QSTR___set__ || attr == MP_QSTR___delete__ || attr == MP_QSTR___set_name__) { return; } #endif @@ -960,7 +960,7 @@ static bool check_for_special_accessors(mp_obj_t key, mp_obj_t value) { #endif #if MICROPY_PY_DESCRIPTORS static const uint8_t to_check[] = { - MP_QSTR___get__, MP_QSTR___set__, MP_QSTR___delete__, + MP_QSTR___get__, MP_QSTR___set__, MP_QSTR___delete__, // not needed for MP_QSTR___set_name__ tho }; for (size_t i = 0; i < MP_ARRAY_SIZE(to_check); ++i) { mp_obj_t dest_temp[2]; @@ -974,6 +974,48 @@ static bool check_for_special_accessors(mp_obj_t key, mp_obj_t value) { } #endif +#if MICROPY_PY_DESCRIPTORS +// Shared data layout for the __set_name__ call and a linked list of calls to be made. +typedef union _setname_list_t setname_list_t; +union _setname_list_t { + mp_obj_t call[4]; + struct { + mp_obj_t _meth; + mp_obj_t _self; + setname_list_t *next; // can use the "owner" argument position temporarily for the linked list + mp_obj_t _name; + }; +}; + +// Append any `__set_name__` method on `value` to the setname list, with its per-attr args +static setname_list_t *setname_maybe_bind_append(setname_list_t *tail, mp_obj_t name, mp_obj_t value) { + // make certain our type-punning is safe: + MP_STATIC_ASSERT_NONCONSTEXPR(offsetof(setname_list_t, next) == offsetof(setname_list_t, call[2])); + + // tail is a blank list entry + mp_load_method_maybe(value, MP_QSTR___set_name__, tail->call); + if (tail->call[1] != MP_OBJ_NULL) { + // Each time a __set_name__ is found, leave it in-place in the former tail and allocate a new tail + tail->next = m_new_obj(setname_list_t); + tail->next->next = NULL; + tail->call[3] = name; + return tail->next; + } else { + return tail; + } +} + +// Execute the captured `__set_name__` calls, destroying the setname list in the process. +static inline void setname_consume_call_all(setname_list_t *head, mp_obj_t owner) { + setname_list_t *next; + while ((next = head->next) != NULL) { + head->call[2] = owner; + mp_call_method_n_kw(2, 0, head->call); + head = next; + } +} +#endif + static void type_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kind_t kind) { (void)kind; mp_obj_type_t *self = MP_OBJ_TO_PTR(self_in); @@ -1210,20 +1252,38 @@ mp_obj_t mp_obj_new_type(qstr name, mp_obj_t bases_tuple, mp_obj_t locals_dict) } } + #if MICROPY_PY_DESCRIPTORS + // To avoid any dynamic allocations when no __set_name__ exists, + // the head of this list is kept on the stack (marked blank with `next = NULL`). + setname_list_t setname_list = { .next = NULL }; + setname_list_t *setname_tail = &setname_list; + #endif + #if ENABLE_SPECIAL_ACCESSORS - // Check if the class has any special accessor methods - if (!(o->flags & MP_TYPE_FLAG_HAS_SPECIAL_ACCESSORS)) { - for (size_t i = 0; i < locals_ptr->map.alloc; i++) { - if (mp_map_slot_is_filled(&locals_ptr->map, i)) { - const mp_map_elem_t *elem = &locals_ptr->map.table[i]; - if (check_for_special_accessors(elem->key, elem->value)) { - o->flags |= MP_TYPE_FLAG_HAS_SPECIAL_ACCESSORS; - break; - } + // Check if the class has any special accessor methods, + // and accumulate bound __set_name__ methods that need to be called + for (size_t i = 0; i < locals_ptr->map.alloc; i++) { + #if !MICROPY_PY_DESCRIPTORS + // __set_name__ needs to scan the entire locals map, can't early-terminate + if (o->flags & MP_TYPE_FLAG_HAS_SPECIAL_ACCESSORS) { + break; + } + #endif + + if (mp_map_slot_is_filled(&locals_ptr->map, i)) { + const mp_map_elem_t *elem = &locals_ptr->map.table[i]; + + if (!(o->flags & MP_TYPE_FLAG_HAS_SPECIAL_ACCESSORS) // elidable when the early-termination check is enabled + && check_for_special_accessors(elem->key, elem->value)) { + o->flags |= MP_TYPE_FLAG_HAS_SPECIAL_ACCESSORS; } + + #if MICROPY_PY_DESCRIPTORS + setname_tail = setname_maybe_bind_append(setname_tail, elem->key, elem->value); + #endif } } - #endif + #endif // ENABLE_SPECIAL_ACCESSORS const mp_obj_type_t *native_base; size_t num_native_bases = instance_count_native_bases(o, &native_base); @@ -1241,6 +1301,10 @@ mp_obj_t mp_obj_new_type(qstr name, mp_obj_t bases_tuple, mp_obj_t locals_dict) } } + #if MICROPY_PY_DESCRIPTORS + setname_consume_call_all(&setname_list, MP_OBJ_FROM_PTR(o)); + #endif + return MP_OBJ_FROM_PTR(o); } From 391636986587c48e4dfd21809ec523e5a75d4ffe Mon Sep 17 00:00:00 2001 From: Anson Mansfield Date: Mon, 21 Jul 2025 08:33:17 -0400 Subject: [PATCH 3/3] docs: Document PEP487 __set_name__ implementation. Signed-off-by: Anson Mansfield --- docs/differences/python_36.rst | 7 ++++++- py/mpconfig.h | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/differences/python_36.rst b/docs/differences/python_36.rst index 3315b0594dafc..7d294fbf6f136 100644 --- a/docs/differences/python_36.rst +++ b/docs/differences/python_36.rst @@ -25,7 +25,8 @@ Python 3.6 beta 1 was released on 12 Sep 2016, and a summary of the new features +--------------------------------------------------------+--------------------------------------------------+-----------------+ | `PEP 468 `_ | Preserving the order of *kwargs* in a function | | +--------------------------------------------------------+--------------------------------------------------+-----------------+ - | `PEP 487 `_ | Simpler customization of class creation | | + | `PEP 487 `_ | Simpler customization of class creation | Partial | + | | | [#setname]_ | +--------------------------------------------------------+--------------------------------------------------+-----------------+ | `PEP 520 `_ | Preserving Class Attribute Definition Order | | +--------------------------------------------------------+--------------------------------------------------+-----------------+ @@ -198,3 +199,7 @@ Changes to built-in modules: +--------------------------------------------------------------------------------------------------------------+----------------+ | The *compress()* and *decompress()* functions now accept keyword arguments | | +--------------------------------------------------------------------------------------------------------------+----------------+ + +.. rubric:: Notes + +.. [#setname] Only :func:`__set_name__` is implemented. diff --git a/py/mpconfig.h b/py/mpconfig.h index a1025fe5e1b18..787477358a06d 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -1124,7 +1124,7 @@ typedef time_t mp_timestamp_t; #define MICROPY_PY_FUNCTION_ATTRS_CODE (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_FULL_FEATURES) #endif -// Whether to support the descriptors __get__, __set__, __delete__ +// Whether to support the descriptors __get__, __set__, __delete__, __set_name__ // This costs some code size and makes load/store/delete of instance // attributes slower for the classes that use this feature #ifndef MICROPY_PY_DESCRIPTORS pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy