From 04472325877f4f561fd8018713692d8ad8b4ad00 Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Mon, 3 Oct 2022 15:47:16 -0700 Subject: [PATCH 1/6] gh-91052: Add C API for watching dictionaries --- Doc/c-api/dict.rst | 41 ++++ Include/cpython/dictobject.h | 22 ++ Include/internal/pycore_dict.h | 27 ++- Include/internal/pycore_interp.h | 2 + ...2-10-03-16-12-39.gh-issue-91052.MsYL9d.rst | 1 + Modules/_testcapimodule.c | 201 ++++++++++++++++++ Objects/dictobject.c | 100 +++++++-- Python/ceval.c | 5 +- 8 files changed, 382 insertions(+), 17 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-10-03-16-12-39.gh-issue-91052.MsYL9d.rst diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index d257c9b5f763d1..70b8dae2e5a3e2 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -238,3 +238,44 @@ Dictionary Objects for key, value in seq2: if override or key not in a: a[key] = value + +.. c:function:: int PyDict_AddWatcher(PyDict_WatchCallback callback) + + Register *callback* as a dictionary watcher. Return a non-negative integer + id which must be passed to future calls to :c:func:`PyDict_Watch`. In case + of error (e.g. no more watcher IDs available), return ``-1`` and set an + exception. + +.. c:function:: int PyDict_Watch(int watcher_id, PyObject *dict) + + Mark dictionary *dict* as watched. The callback granted *watcher_id* by + :c:func:`PyDict_AddWatcher` will be called when *dict* is modified or + deallocated. + +.. c:type:: PyDict_WatchEvent + + Enumeration of possible dictionary watcher events: ``PyDict_EVENT_ADDED``, + ``PyDict_EVENT_MODIFIED``, ``PyDict_EVENT_DELETED``, ``PyDict_EVENT_CLONED``, + ``PyDict_EVENT_CLEARED``, or ``PyDict_EVENT_DEALLOCED``. + +.. c:type:: void (*PyDict_WatchCallback)(PyDict_WatchEvent event, PyObject *dict, PyObject *key, PyObject *new_value) + + Type of a dict watcher callback function. + + If *event* is ``PyDict_EVENT_CLEARED`` or ``PyDict_EVENT_DEALLOCED``, both + *key* and *new_value* will be ``NULL``. If *event* is + ``PyDict_EVENT_ADDED`` or ``PyDict_EVENT_MODIFIED``, *new_value* will be the + new value for *key*. If *event* is ``PyDict_EVENT_DELETED``, *key* is being + deleted from the dictionary and *new_value* will be ``NULL``. + + ``PyDict_EVENT_CLONED`` occurs when *dict* was previously empty and another + dict is merged into it. To maintain efficiency of this operation, per-key + ``PyDict_EVENT_ADDED`` events are not issued in this case; instead a + single ``PyDict_EVENT_CLONED`` is issued, and *key* will be the source + dictionary. + + The callback may inspect but should not modify *dict*; doing so could have + unpredictable effects, including infinite recursion. + + Callbacks occur before the notified modification to *dict* takes place, so + the prior state of *dict* can be inspected. diff --git a/Include/cpython/dictobject.h b/Include/cpython/dictobject.h index 565ad791a6cb28..ea9d37f0837a6e 100644 --- a/Include/cpython/dictobject.h +++ b/Include/cpython/dictobject.h @@ -83,3 +83,25 @@ typedef struct { PyAPI_FUNC(PyObject *) _PyDictView_New(PyObject *, PyTypeObject *); PyAPI_FUNC(PyObject *) _PyDictView_Intersect(PyObject* self, PyObject *other); + +/* Dictionary watchers */ + +typedef enum { + PyDict_EVENT_ADDED, + PyDict_EVENT_MODIFIED, + PyDict_EVENT_DELETED, + PyDict_EVENT_CLONED, + PyDict_EVENT_CLEARED, + PyDict_EVENT_DEALLOCED, +} PyDict_WatchEvent; + +// Callback to be invoked when a watched dict is cleared, dealloced, or modified. +// In clear/dealloc case, key and new_value will be NULL. Otherwise, new_value will be the +// new value for key, NULL if key is being deleted. +typedef void(*PyDict_WatchCallback)(PyDict_WatchEvent event, PyObject* dict, PyObject* key, PyObject* new_value); + +// Register a dict-watcher callback +PyAPI_FUNC(int) PyDict_AddWatcher(PyDict_WatchCallback callback); + +// Mark given dictionary as "watched" (callback will be called if it is modified) +PyAPI_FUNC(int) PyDict_Watch(int watcher_id, PyObject* dict); diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 464092996cae00..fda97dc0faeec3 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -154,7 +154,32 @@ struct _dictvalues { extern uint64_t _pydict_global_version; -#define DICT_NEXT_VERSION() (++_pydict_global_version) +#define DICT_MAX_WATCHERS 8 +#define DICT_VERSION_MASK 255 +#define DICT_VERSION_INCREMENT 256 + +#define DICT_NEXT_VERSION() (_pydict_global_version += DICT_VERSION_INCREMENT) + +void +_PyDict_SendEvent(int watcher_bits, + PyDict_WatchEvent event, + PyDictObject *mp, + PyObject *key, + PyObject *value); + +static inline uint64_t +_PyDict_NotifyEvent(PyDict_WatchEvent event, + PyDictObject *mp, + PyObject *key, + PyObject *value) +{ + int watcher_bits = mp->ma_version_tag & DICT_VERSION_MASK; + if (watcher_bits) { + _PyDict_SendEvent(watcher_bits, event, mp, key, value); + return DICT_NEXT_VERSION() | watcher_bits; + } + return DICT_NEXT_VERSION(); +} extern PyObject *_PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values); extern PyObject *_PyDict_FromItems( diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index b21708a388b339..d327ead8473378 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -144,6 +144,8 @@ struct _is { // Initialized to _PyEval_EvalFrameDefault(). _PyFrameEvalFunction eval_frame; + void *dict_watchers[8]; + Py_ssize_t co_extra_user_count; freefunc co_extra_freefuncs[MAX_CO_EXTRA_USERS]; diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-03-16-12-39.gh-issue-91052.MsYL9d.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-03-16-12-39.gh-issue-91052.MsYL9d.rst new file mode 100644 index 00000000000000..c7db4da494fe68 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-10-03-16-12-39.gh-issue-91052.MsYL9d.rst @@ -0,0 +1 @@ +Add API for subscribing to modification events on selected dictionaries. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 3d6535f50be957..633c47a50ff269 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -5169,6 +5169,206 @@ test_tstate_capi(PyObject *self, PyObject *Py_UNUSED(args)) } +// Test dict watching +static PyObject *g_dict_watch_events; + +static void +dict_watch_callback(PyDict_WatchEvent event, + PyObject *dict, + PyObject *key, + PyObject *new_value) +{ + PyObject *msg; + switch(event) { + case PyDict_EVENT_CLEARED: + msg = PyUnicode_FromString("clear"); + break; + case PyDict_EVENT_DEALLOCED: + msg = PyUnicode_FromString("dealloc"); + break; + case PyDict_EVENT_CLONED: + msg = PyUnicode_FromString("clone"); + break; + case PyDict_EVENT_ADDED: + msg = PyUnicode_FromFormat("new:%S:%S", key, new_value); + break; + case PyDict_EVENT_MODIFIED: + msg = PyUnicode_FromFormat("mod:%S:%S", key, new_value); + break; + case PyDict_EVENT_DELETED: + msg = PyUnicode_FromFormat("del:%S", key); + break; + default: + msg = PyUnicode_FromString("unknown"); + } + assert(PyList_Check(g_dict_watch_events)); + PyList_Append(g_dict_watch_events, msg); +} + +static int +dict_watch_assert(Py_ssize_t expected_num_events, + const char *expected_last_msg) +{ + char buf[512]; + Py_ssize_t actual_num_events = PyList_Size(g_dict_watch_events); + if (expected_num_events != actual_num_events) { + snprintf(buf, + 512, + "got %d dict watch events, expected %d", + (int)actual_num_events, + (int)expected_num_events); + raiseTestError("test_watch_dict", (const char *)&buf); + return -1; + } + PyObject *last_msg = PyList_GetItem(g_dict_watch_events, + PyList_Size(g_dict_watch_events)-1); + if (PyUnicode_CompareWithASCIIString(last_msg, expected_last_msg)) { + snprintf(buf, + 512, + "last event is '%s', expected '%s'", + PyUnicode_AsUTF8(last_msg), + expected_last_msg); + raiseTestError("test_watch_dict", (const char *)&buf); + return -1; + } + return 0; +} + +static int +try_watch(int watcher_id, PyObject *obj) { + if (PyDict_Watch(watcher_id, obj)) { + raiseTestError("test_watch_dict", "PyDict_Watch() failed on dict"); + return -1; + } + return 0; +} + +static int +dict_watch_assert_error(int watcher_id, PyObject *obj, const char *fail_msg) +{ + if (!PyDict_Watch(watcher_id, obj)) { + raiseTestError("test_watch_dict", fail_msg); + return -1; + } else if (!PyErr_Occurred()) { + raiseTestError("test_watch_dict", "PyDict_Watch() returned error code without exception set"); + return -1; + } else { + PyErr_Clear(); + } + return 0; +} + +static PyObject * +test_watch_dict(PyObject *self, PyObject *Py_UNUSED(args)) +{ + PyObject *watched = PyDict_New(); + PyObject *unwatched = PyDict_New(); + PyObject *one = PyLong_FromLong(1); + PyObject *two = PyLong_FromLong(2); + PyObject *key1 = PyUnicode_FromString("key1"); + PyObject *key2 = PyUnicode_FromString("key2"); + + g_dict_watch_events = PyList_New(0); + + int wid = PyDict_AddWatcher(dict_watch_callback); + if (try_watch(wid, watched)) { + return NULL; + } + + PyDict_SetItem(unwatched, key1, two); + PyDict_Merge(watched, unwatched, 1); + + if (dict_watch_assert(1, "clone")) { + return NULL; + } + + PyDict_SetItem(watched, key1, one); + PyDict_SetItem(unwatched, key1, one); + + if (dict_watch_assert(2, "mod:key1:1")) { + return NULL; + } + + PyDict_SetItemString(watched, "key1", two); + PyDict_SetItemString(unwatched, "key1", two); + + if (dict_watch_assert(3, "mod:key1:2")) { + return NULL; + } + + PyDict_SetItem(watched, key2, one); + PyDict_SetItem(unwatched, key2, one); + + if (dict_watch_assert(4, "new:key2:1")) { + return NULL; + } + + _PyDict_Pop(watched, key2, Py_None); + _PyDict_Pop(unwatched, key2, Py_None); + + if (dict_watch_assert(5, "del:key2")) { + return NULL; + } + + PyDict_DelItemString(watched, "key1"); + PyDict_DelItemString(unwatched, "key1"); + + if (dict_watch_assert(6, "del:key1")) { + return NULL; + } + + PyDict_SetDefault(watched, key1, one); + PyDict_SetDefault(unwatched, key1, one); + + if (dict_watch_assert(7, "new:key1:1")) { + return NULL; + } + + PyDict_Clear(watched); + PyDict_Clear(unwatched); + + if (dict_watch_assert(8, "clear")) { + return NULL; + } + + PyObject *copy = PyDict_Copy(watched); + // copied dict is not watched, so this does not add an event + Py_CLEAR(copy); + + Py_CLEAR(watched); + + if (dict_watch_assert(9, "dealloc")) { + return NULL; + } + + // it is an error to try to watch a non-dict + if (dict_watch_assert_error(wid, one, "PyDict_Watch() succeeded on non-dict")) { + return NULL; + } + + // It is an error to pass an out-of-range watcher ID + if (dict_watch_assert_error(-1, unwatched, "PyDict_Watch() succeeded on negative watcher ID")) { + return NULL; + } + if (dict_watch_assert_error(8, unwatched, "PyDict_Watch() succeeded on too-large watcher ID")) { + return NULL; + } + + // It is an error to pass a never-registered watcher ID + if (dict_watch_assert_error(7, unwatched, "PyDict_Watch() succeeded on unused watcher ID")) { + return NULL; + } + + Py_CLEAR(unwatched); + Py_CLEAR(g_dict_watch_events); + Py_DECREF(one); + Py_DECREF(two); + Py_DECREF(key1); + Py_DECREF(key2); + Py_RETURN_NONE; +} + + // Test PyFloat_Pack2(), PyFloat_Pack4() and PyFloat_Pack8() static PyObject * test_float_pack(PyObject *self, PyObject *args) @@ -5762,6 +5962,7 @@ static PyMethodDef TestMethods[] = { {"settrace_to_record", settrace_to_record, METH_O, NULL}, {"test_macros", test_macros, METH_NOARGS, NULL}, {"clear_managed_dict", clear_managed_dict, METH_O, NULL}, + {"test_watch_dict", test_watch_dict, METH_NOARGS, NULL}, {NULL, NULL} /* sentinel */ }; diff --git a/Objects/dictobject.c b/Objects/dictobject.c index fecdfa86370193..8f25416475e3e1 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -1240,6 +1240,7 @@ insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) MAINTAIN_TRACKING(mp, key, value); if (ix == DKIX_EMPTY) { + uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_ADDED, mp, key, value); /* Insert into new slot. */ mp->ma_keys->dk_version = 0; assert(old_value == NULL); @@ -1274,7 +1275,7 @@ insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) ep->me_value = value; } mp->ma_used++; - mp->ma_version_tag = DICT_NEXT_VERSION(); + mp->ma_version_tag = new_version; mp->ma_keys->dk_usable--; mp->ma_keys->dk_nentries++; assert(mp->ma_keys->dk_usable >= 0); @@ -1283,6 +1284,7 @@ insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) } if (old_value != value) { + uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, mp, key, value); if (_PyDict_HasSplitTable(mp)) { mp->ma_values->values[ix] = value; if (old_value == NULL) { @@ -1299,7 +1301,7 @@ insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) DK_ENTRIES(mp->ma_keys)[ix].me_value = value; } } - mp->ma_version_tag = DICT_NEXT_VERSION(); + mp->ma_version_tag = new_version; } Py_XDECREF(old_value); /* which **CAN** re-enter (see issue #22653) */ ASSERT_CONSISTENT(mp); @@ -1320,6 +1322,8 @@ insert_to_emptydict(PyDictObject *mp, PyObject *key, Py_hash_t hash, { assert(mp->ma_keys == Py_EMPTY_KEYS); + uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_ADDED, mp, key, value); + int unicode = PyUnicode_CheckExact(key); PyDictKeysObject *newkeys = new_keys_object(PyDict_LOG_MINSIZE, unicode); if (newkeys == NULL) { @@ -1347,7 +1351,7 @@ insert_to_emptydict(PyDictObject *mp, PyObject *key, Py_hash_t hash, ep->me_value = value; } mp->ma_used++; - mp->ma_version_tag = DICT_NEXT_VERSION(); + mp->ma_version_tag = new_version; mp->ma_keys->dk_usable--; mp->ma_keys->dk_nentries++; return 0; @@ -1910,7 +1914,7 @@ delete_index_from_values(PyDictValues *values, Py_ssize_t ix) static int delitem_common(PyDictObject *mp, Py_hash_t hash, Py_ssize_t ix, - PyObject *old_value) + PyObject *old_value, uint64_t new_version) { PyObject *old_key; @@ -1918,7 +1922,7 @@ delitem_common(PyDictObject *mp, Py_hash_t hash, Py_ssize_t ix, assert(hashpos >= 0); mp->ma_used--; - mp->ma_version_tag = DICT_NEXT_VERSION(); + mp->ma_version_tag = new_version; if (mp->ma_values) { assert(old_value == mp->ma_values->values[ix]); mp->ma_values->values[ix] = NULL; @@ -1987,7 +1991,8 @@ _PyDict_DelItem_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) return -1; } - return delitem_common(mp, hash, ix, old_value); + uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_DELETED, mp, key, NULL); + return delitem_common(mp, hash, ix, old_value, new_version); } /* This function promises that the predicate -> deletion sequence is atomic @@ -2028,10 +2033,12 @@ _PyDict_DelItemIf(PyObject *op, PyObject *key, hashpos = lookdict_index(mp->ma_keys, hash, ix); assert(hashpos >= 0); - if (res > 0) - return delitem_common(mp, hashpos, ix, old_value); - else + if (res > 0) { + uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_DELETED, mp, key, NULL); + return delitem_common(mp, hashpos, ix, old_value, new_version); + } else { return 0; + } } @@ -2052,11 +2059,12 @@ PyDict_Clear(PyObject *op) return; } /* Empty the dict... */ + uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_CLEARED, mp, NULL, NULL); dictkeys_incref(Py_EMPTY_KEYS); mp->ma_keys = Py_EMPTY_KEYS; mp->ma_values = NULL; mp->ma_used = 0; - mp->ma_version_tag = DICT_NEXT_VERSION(); + mp->ma_version_tag = new_version; /* ...then clear the keys and values */ if (oldvalues != NULL) { n = oldkeys->dk_nentries; @@ -2196,7 +2204,8 @@ _PyDict_Pop_KnownHash(PyObject *dict, PyObject *key, Py_hash_t hash, PyObject *d } assert(old_value != NULL); Py_INCREF(old_value); - delitem_common(mp, hash, ix, old_value); + uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_DELETED, mp, key, NULL); + delitem_common(mp, hash, ix, old_value, new_version); ASSERT_CONSISTENT(mp); return old_value; @@ -2321,6 +2330,7 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) static void dict_dealloc(PyDictObject *mp) { + _PyDict_NotifyEvent(PyDict_EVENT_DEALLOCED, mp, NULL, NULL); PyDictValues *values = mp->ma_values; PyDictKeysObject *keys = mp->ma_keys; Py_ssize_t i, n; @@ -2809,6 +2819,7 @@ dict_merge(PyObject *a, PyObject *b, int override) other->ma_used == okeys->dk_nentries && (DK_LOG_SIZE(okeys) == PyDict_LOG_MINSIZE || USABLE_FRACTION(DK_SIZE(okeys)/2) < other->ma_used)) { + uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_CLONED, mp, b, NULL); PyDictKeysObject *keys = clone_combined_dict_keys(other); if (keys == NULL) { return -1; @@ -2822,7 +2833,7 @@ dict_merge(PyObject *a, PyObject *b, int override) } mp->ma_used = other->ma_used; - mp->ma_version_tag = DICT_NEXT_VERSION(); + mp->ma_version_tag = new_version; ASSERT_CONSISTENT(mp); if (_PyObject_GC_IS_TRACKED(other) && !_PyObject_GC_IS_TRACKED(mp)) { @@ -3294,6 +3305,7 @@ PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *defaultobj) return NULL; if (ix == DKIX_EMPTY) { + uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_ADDED, mp, key, defaultobj); mp->ma_keys->dk_version = 0; value = defaultobj; if (mp->ma_keys->dk_usable <= 0) { @@ -3328,12 +3340,13 @@ PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *defaultobj) Py_INCREF(value); MAINTAIN_TRACKING(mp, key, value); mp->ma_used++; - mp->ma_version_tag = DICT_NEXT_VERSION(); + mp->ma_version_tag = new_version; mp->ma_keys->dk_usable--; mp->ma_keys->dk_nentries++; assert(mp->ma_keys->dk_usable >= 0); } else if (value == NULL) { + uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_ADDED, mp, key, defaultobj); value = defaultobj; assert(_PyDict_HasSplitTable(mp)); assert(mp->ma_values->values[ix] == NULL); @@ -3342,7 +3355,7 @@ PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *defaultobj) mp->ma_values->values[ix] = value; _PyDictValues_AddToInsertionOrder(mp->ma_values, ix); mp->ma_used++; - mp->ma_version_tag = DICT_NEXT_VERSION(); + mp->ma_version_tag = new_version; } ASSERT_CONSISTENT(mp); @@ -3415,6 +3428,7 @@ dict_popitem_impl(PyDictObject *self) { Py_ssize_t i, j; PyObject *res; + uint64_t new_version; /* Allocate the result tuple before checking the size. Believe it * or not, this allocation could trigger a garbage collection which @@ -3454,6 +3468,7 @@ dict_popitem_impl(PyDictObject *self) assert(i >= 0); key = ep0[i].me_key; + new_version = _PyDict_NotifyEvent(PyDict_EVENT_DELETED, self, key, NULL); hash = unicode_get_hash(key); value = ep0[i].me_value; ep0[i].me_key = NULL; @@ -3468,6 +3483,7 @@ dict_popitem_impl(PyDictObject *self) assert(i >= 0); key = ep0[i].me_key; + new_version = _PyDict_NotifyEvent(PyDict_EVENT_DELETED, self, key, NULL); hash = ep0[i].me_hash; value = ep0[i].me_value; ep0[i].me_key = NULL; @@ -3485,7 +3501,7 @@ dict_popitem_impl(PyDictObject *self) /* We can't dk_usable++ since there is DKIX_DUMMY in indices */ self->ma_keys->dk_nentries = i; self->ma_used--; - self->ma_version_tag = DICT_NEXT_VERSION(); + self->ma_version_tag = new_version; ASSERT_CONSISTENT(self); return res; } @@ -5703,3 +5719,57 @@ uint32_t _PyDictKeys_GetVersionForCurrentState(PyDictKeysObject *dictkeys) dictkeys->dk_version = v; return v; } + +int +PyDict_Watch(int watcher_id, PyObject* dict) +{ + if (!PyDict_Check(dict)) { + PyErr_SetString(PyExc_ValueError, "Cannot watch non-dictionary"); + return -1; + } + if (watcher_id < 0 || watcher_id >= DICT_MAX_WATCHERS) { + PyErr_Format(PyExc_ValueError, "Invalid dict watcher ID %d", watcher_id); + return -1; + } + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (!interp->dict_watchers[watcher_id]) { + PyErr_Format(PyExc_ValueError, "No dict watcher set for ID %d", watcher_id); + return -1; + } + ((PyDictObject*)dict)->ma_version_tag |= (1 << watcher_id); + return 0; +} + +int +PyDict_AddWatcher(PyDict_WatchCallback callback) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + + for (int i = 0; i < DICT_MAX_WATCHERS; i++) { + if (!interp->dict_watchers[i]) { + interp->dict_watchers[i] = (void*)callback; + return i; + } + } + + PyErr_SetString(PyExc_RuntimeError, "no more dict watcher IDs available"); + return -1; +} + +void +_PyDict_SendEvent(int watcher_bits, + PyDict_WatchEvent event, + PyDictObject *mp, + PyObject *key, + PyObject *value) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + for (int i = 0; i < DICT_MAX_WATCHERS; i++) { + if (watcher_bits & (1 << i)) { + PyDict_WatchCallback cb = (PyDict_WatchCallback)interp->dict_watchers[i]; + if (cb) { + cb(event, (PyObject*)mp, key, value); + } + } + } +} diff --git a/Python/ceval.c b/Python/ceval.c index 945377044170ac..7659a28fee57e3 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -3221,6 +3221,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int uint16_t hint = cache->index; DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR); PyObject *value, *old_value; + uint64_t new_version; if (DK_IS_UNICODE(dict->ma_keys)) { PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; DEOPT_IF(ep->me_key != name, STORE_ATTR); @@ -3228,6 +3229,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int DEOPT_IF(old_value == NULL, STORE_ATTR); STACK_SHRINK(1); value = POP(); + new_version = _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, dict, name, value); ep->me_value = value; } else { @@ -3237,6 +3239,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int DEOPT_IF(old_value == NULL, STORE_ATTR); STACK_SHRINK(1); value = POP(); + new_version = _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, dict, name, value); ep->me_value = value; } Py_DECREF(old_value); @@ -3246,7 +3249,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int _PyObject_GC_TRACK(dict); } /* PEP 509 */ - dict->ma_version_tag = DICT_NEXT_VERSION(); + dict->ma_version_tag = new_version; Py_DECREF(owner); JUMPBY(INLINE_CACHE_ENTRIES_STORE_ATTR); DISPATCH(); From 1c7f814623f8a6023b74767bf8146c839c16cc1d Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Tue, 4 Oct 2022 09:51:14 -0700 Subject: [PATCH 2/6] Add second-watcher tests, fix MSVC warning --- Modules/_testcapimodule.c | 24 ++++++++++++++++++++++-- Objects/dictobject.c | 2 +- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 633c47a50ff269..10c133c602d60b 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -5205,6 +5205,16 @@ dict_watch_callback(PyDict_WatchEvent event, PyList_Append(g_dict_watch_events, msg); } +static void +dict_watch_callback_2(PyDict_WatchEvent event, + PyObject *dict, + PyObject *key, + PyObject *new_value) +{ + PyObject *msg = PyUnicode_FromString("second"); + PyList_Append(g_dict_watch_events, msg); +} + static int dict_watch_assert(Py_ssize_t expected_num_events, const char *expected_last_msg) @@ -5324,20 +5334,30 @@ test_watch_dict(PyObject *self, PyObject *Py_UNUSED(args)) return NULL; } + int wid2 = PyDict_AddWatcher(dict_watch_callback_2); + if (try_watch(wid2, unwatched)) { + return NULL; + } + PyDict_Clear(watched); - PyDict_Clear(unwatched); if (dict_watch_assert(8, "clear")) { return NULL; } + PyDict_Clear(unwatched); + + if (dict_watch_assert(9, "second")) { + return NULL; + } + PyObject *copy = PyDict_Copy(watched); // copied dict is not watched, so this does not add an event Py_CLEAR(copy); Py_CLEAR(watched); - if (dict_watch_assert(9, "dealloc")) { + if (dict_watch_assert(10, "dealloc")) { return NULL; } diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 8f25416475e3e1..35637d34f0fd06 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -5736,7 +5736,7 @@ PyDict_Watch(int watcher_id, PyObject* dict) PyErr_Format(PyExc_ValueError, "No dict watcher set for ID %d", watcher_id); return -1; } - ((PyDictObject*)dict)->ma_version_tag |= (1 << watcher_id); + ((PyDictObject*)dict)->ma_version_tag |= (1LL << watcher_id); return 0; } From 6e5cee2a9ef5c6785498dc3dc3f6885d3fd8731f Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Tue, 4 Oct 2022 10:12:26 -0700 Subject: [PATCH 3/6] Over-engineer the watcher bits check --- Objects/dictobject.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 35637d34f0fd06..395b8a77c5fafd 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -5765,11 +5765,12 @@ _PyDict_SendEvent(int watcher_bits, { PyInterpreterState *interp = _PyInterpreterState_GET(); for (int i = 0; i < DICT_MAX_WATCHERS; i++) { - if (watcher_bits & (1 << i)) { + if (watcher_bits & 1) { PyDict_WatchCallback cb = (PyDict_WatchCallback)interp->dict_watchers[i]; if (cb) { cb(event, (PyObject*)mp, key, value); } } + watcher_bits >>= 1; } } From 7dcf9b04a7389e5f3eb557c596cdb56d24ad4017 Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Thu, 6 Oct 2022 10:34:59 -0700 Subject: [PATCH 4/6] Address review comments --- Doc/c-api/dict.rst | 12 +- Include/cpython/dictobject.h | 5 +- Include/internal/pycore_dict.h | 4 +- Include/internal/pycore_interp.h | 2 +- Lib/test/test_capi.py | 132 ++++++++++++++++++ Modules/_testcapimodule.c | 223 ++++++++++--------------------- Objects/dictobject.c | 26 +++- 7 files changed, 243 insertions(+), 161 deletions(-) diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index 0fedf2c7d9030c..97f83d522af2d4 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -246,6 +246,12 @@ Dictionary Objects of error (e.g. no more watcher IDs available), return ``-1`` and set an exception. +.. c:function:: int PyDict_ClearWatcher(int watcher_id) + + Clear watcher identified by *watcher_id* previously returned from + :c:func:`PyDict_AddWatcher`. Return ``0`` on success, ``-1`` on error (e.g. + if the given *watcher_id* was never registered.) + .. c:function:: int PyDict_Watch(int watcher_id, PyObject *dict) Mark dictionary *dict* as watched. The callback granted *watcher_id* by @@ -258,7 +264,7 @@ Dictionary Objects ``PyDict_EVENT_MODIFIED``, ``PyDict_EVENT_DELETED``, ``PyDict_EVENT_CLONED``, ``PyDict_EVENT_CLEARED``, or ``PyDict_EVENT_DEALLOCED``. -.. c:type:: void (*PyDict_WatchCallback)(PyDict_WatchEvent event, PyObject *dict, PyObject *key, PyObject *new_value) +.. c:type:: int (*PyDict_WatchCallback)(PyDict_WatchEvent event, PyObject *dict, PyObject *key, PyObject *new_value) Type of a dict watcher callback function. @@ -279,3 +285,7 @@ Dictionary Objects Callbacks occur before the notified modification to *dict* takes place, so the prior state of *dict* can be inspected. + + If an error occurs in the callback, it may return ``-1`` with an exception + set; this exception will be printed as an unraisable exception using + :c:func:`PyErr_WriteUnraisable`. On success it should return ``0``. diff --git a/Include/cpython/dictobject.h b/Include/cpython/dictobject.h index ea9d37f0837a6e..05c751ff4b22c3 100644 --- a/Include/cpython/dictobject.h +++ b/Include/cpython/dictobject.h @@ -98,10 +98,11 @@ typedef enum { // Callback to be invoked when a watched dict is cleared, dealloced, or modified. // In clear/dealloc case, key and new_value will be NULL. Otherwise, new_value will be the // new value for key, NULL if key is being deleted. -typedef void(*PyDict_WatchCallback)(PyDict_WatchEvent event, PyObject* dict, PyObject* key, PyObject* new_value); +typedef int(*PyDict_WatchCallback)(PyDict_WatchEvent event, PyObject* dict, PyObject* key, PyObject* new_value); -// Register a dict-watcher callback +// Register/unregister a dict-watcher callback PyAPI_FUNC(int) PyDict_AddWatcher(PyDict_WatchCallback callback); +PyAPI_FUNC(int) PyDict_ClearWatcher(int watcher_id); // Mark given dictionary as "watched" (callback will be called if it is modified) PyAPI_FUNC(int) PyDict_Watch(int watcher_id, PyObject* dict); diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index fda97dc0faeec3..ae4094a095d879 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -155,8 +155,8 @@ struct _dictvalues { extern uint64_t _pydict_global_version; #define DICT_MAX_WATCHERS 8 -#define DICT_VERSION_MASK 255 -#define DICT_VERSION_INCREMENT 256 +#define DICT_VERSION_INCREMENT (1 << DICT_MAX_WATCHERS) +#define DICT_VERSION_MASK (DICT_VERSION_INCREMENT - 1) #define DICT_NEXT_VERSION() (_pydict_global_version += DICT_VERSION_INCREMENT) diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index d327ead8473378..8cecd5ab3e541e 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -144,7 +144,7 @@ struct _is { // Initialized to _PyEval_EvalFrameDefault(). _PyFrameEvalFunction eval_frame; - void *dict_watchers[8]; + PyDict_WatchCallback dict_watchers[DICT_MAX_WATCHERS]; Py_ssize_t co_extra_user_count; freefunc co_extra_freefuncs[MAX_CO_EXTRA_USERS]; diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 2c6fe34d3b788c..cb90d55941cae7 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -2,6 +2,7 @@ # these are all functions _testcapi exports whose name begins with 'test_'. from collections import OrderedDict +from contextlib import contextmanager import _thread import importlib.machinery import importlib.util @@ -1393,5 +1394,136 @@ def func2(x=None): self.do_test(func2) +class TestDictWatchers(unittest.TestCase): + # types of watchers testcapimodule can add: + EVENTS = 0 # appends dict events as strings to global event list + ERROR = 1 # unconditionally sets and signals a RuntimeException + SECOND = 2 # always appends "second" to global event list + + def add_watcher(self, kind=EVENTS): + return _testcapi.add_dict_watcher(kind) + + def clear_watcher(self, watcher_id): + _testcapi.clear_dict_watcher(watcher_id) + + @contextmanager + def watcher(self, kind=EVENTS): + wid = self.add_watcher(kind) + try: + yield wid + finally: + self.clear_watcher(wid) + + def assert_events(self, expected): + actual = _testcapi.get_dict_watcher_events() + self.assertEqual(actual, expected) + + def watch(self, wid, d): + _testcapi.watch_dict(wid, d) + + def test_set_new_item(self): + d = {} + with self.watcher() as wid: + self.watch(wid, d) + d["foo"] = "bar" + self.assert_events(["new:foo:bar"]) + + def test_set_existing_item(self): + d = {"foo": "bar"} + with self.watcher() as wid: + self.watch(wid, d) + d["foo"] = "baz" + self.assert_events(["mod:foo:baz"]) + + def test_clone(self): + d = {} + d2 = {"foo": "bar"} + with self.watcher() as wid: + self.watch(wid, d) + d.update(d2) + self.assert_events(["clone"]) + + def test_no_event_if_not_watched(self): + d = {} + with self.watcher() as wid: + d["foo"] = "bar" + self.assert_events([]) + + def test_del(self): + d = {"foo": "bar"} + with self.watcher() as wid: + self.watch(wid, d) + del d["foo"] + self.assert_events(["del:foo"]) + + def test_pop(self): + d = {"foo": "bar"} + with self.watcher() as wid: + self.watch(wid, d) + d.pop("foo") + self.assert_events(["del:foo"]) + + def test_clear(self): + d = {"foo": "bar"} + with self.watcher() as wid: + self.watch(wid, d) + d.clear() + self.assert_events(["clear"]) + + def test_dealloc(self): + d = {"foo": "bar"} + with self.watcher() as wid: + self.watch(wid, d) + del d + self.assert_events(["dealloc"]) + + def test_error(self): + d = {} + unraisables = [] + def unraisable_hook(unraisable): + unraisables.append(unraisable) + with self.watcher(kind=self.ERROR) as wid: + self.watch(wid, d) + orig_unraisable_hook = sys.unraisablehook + sys.unraisablehook = unraisable_hook + try: + d["foo"] = "bar" + finally: + sys.unraisablehook = orig_unraisable_hook + self.assert_events([]) + self.assertEqual(len(unraisables), 1) + unraisable = unraisables[0] + self.assertIs(unraisable.object, d) + self.assertEqual(str(unraisable.exc_value), "boom!") + + def test_two_watchers(self): + d1 = {} + d2 = {} + with self.watcher() as wid1: + with self.watcher(kind=self.SECOND) as wid2: + self.watch(wid1, d1) + self.watch(wid2, d2) + d1["foo"] = "bar" + d2["hmm"] = "baz" + self.assert_events(["new:foo:bar", "second"]) + + def test_watch_non_dict(self): + with self.watcher() as wid: + with self.assertRaisesRegex(ValueError, r"Cannot watch non-dictionary"): + self.watch(wid, 1) + + def test_watch_out_of_range_watcher_id(self): + d = {} + with self.assertRaisesRegex(ValueError, r"Invalid dict watcher ID -1"): + self.watch(-1, d) + with self.assertRaisesRegex(ValueError, r"Invalid dict watcher ID 8"): + self.watch(8, d) # DICT_MAX_WATCHERS = 8 + + def test_unassigned_watcher_id(self): + d = {} + with self.assertRaisesRegex(ValueError, r"No dict watcher set for ID 1"): + self.watch(1, d) + + if __name__ == "__main__": unittest.main() diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 10c133c602d60b..624aaed4ba918a 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -5171,8 +5171,9 @@ test_tstate_capi(PyObject *self, PyObject *Py_UNUSED(args)) // Test dict watching static PyObject *g_dict_watch_events; +static int g_dict_watchers_installed; -static void +static int dict_watch_callback(PyDict_WatchEvent event, PyObject *dict, PyObject *key, @@ -5201,191 +5202,106 @@ dict_watch_callback(PyDict_WatchEvent event, default: msg = PyUnicode_FromString("unknown"); } - assert(PyList_Check(g_dict_watch_events)); - PyList_Append(g_dict_watch_events, msg); -} - -static void -dict_watch_callback_2(PyDict_WatchEvent event, - PyObject *dict, - PyObject *key, - PyObject *new_value) -{ - PyObject *msg = PyUnicode_FromString("second"); - PyList_Append(g_dict_watch_events, msg); -} - -static int -dict_watch_assert(Py_ssize_t expected_num_events, - const char *expected_last_msg) -{ - char buf[512]; - Py_ssize_t actual_num_events = PyList_Size(g_dict_watch_events); - if (expected_num_events != actual_num_events) { - snprintf(buf, - 512, - "got %d dict watch events, expected %d", - (int)actual_num_events, - (int)expected_num_events); - raiseTestError("test_watch_dict", (const char *)&buf); + if (!msg) { return -1; } - PyObject *last_msg = PyList_GetItem(g_dict_watch_events, - PyList_Size(g_dict_watch_events)-1); - if (PyUnicode_CompareWithASCIIString(last_msg, expected_last_msg)) { - snprintf(buf, - 512, - "last event is '%s', expected '%s'", - PyUnicode_AsUTF8(last_msg), - expected_last_msg); - raiseTestError("test_watch_dict", (const char *)&buf); + assert(PyList_Check(g_dict_watch_events)); + if (PyList_Append(g_dict_watch_events, msg) < 0) { + Py_DECREF(msg); return -1; } return 0; } static int -try_watch(int watcher_id, PyObject *obj) { - if (PyDict_Watch(watcher_id, obj)) { - raiseTestError("test_watch_dict", "PyDict_Watch() failed on dict"); +dict_watch_callback_second(PyDict_WatchEvent event, + PyObject *dict, + PyObject *key, + PyObject *new_value) +{ + PyObject *msg = PyUnicode_FromString("second"); + if (!msg) { + return -1; + } + if (PyList_Append(g_dict_watch_events, msg) < 0) { return -1; } return 0; } static int -dict_watch_assert_error(int watcher_id, PyObject *obj, const char *fail_msg) +dict_watch_callback_error(PyDict_WatchEvent event, + PyObject *dict, + PyObject *key, + PyObject *new_value) { - if (!PyDict_Watch(watcher_id, obj)) { - raiseTestError("test_watch_dict", fail_msg); - return -1; - } else if (!PyErr_Occurred()) { - raiseTestError("test_watch_dict", "PyDict_Watch() returned error code without exception set"); - return -1; - } else { - PyErr_Clear(); - } - return 0; + PyErr_SetString(PyExc_RuntimeError, "boom!"); + return -1; } static PyObject * -test_watch_dict(PyObject *self, PyObject *Py_UNUSED(args)) +add_dict_watcher(PyObject *self, PyObject *kind) { - PyObject *watched = PyDict_New(); - PyObject *unwatched = PyDict_New(); - PyObject *one = PyLong_FromLong(1); - PyObject *two = PyLong_FromLong(2); - PyObject *key1 = PyUnicode_FromString("key1"); - PyObject *key2 = PyUnicode_FromString("key2"); - - g_dict_watch_events = PyList_New(0); - - int wid = PyDict_AddWatcher(dict_watch_callback); - if (try_watch(wid, watched)) { - return NULL; - } - - PyDict_SetItem(unwatched, key1, two); - PyDict_Merge(watched, unwatched, 1); - - if (dict_watch_assert(1, "clone")) { - return NULL; - } - - PyDict_SetItem(watched, key1, one); - PyDict_SetItem(unwatched, key1, one); - - if (dict_watch_assert(2, "mod:key1:1")) { - return NULL; - } - - PyDict_SetItemString(watched, "key1", two); - PyDict_SetItemString(unwatched, "key1", two); - - if (dict_watch_assert(3, "mod:key1:2")) { - return NULL; - } - - PyDict_SetItem(watched, key2, one); - PyDict_SetItem(unwatched, key2, one); - - if (dict_watch_assert(4, "new:key2:1")) { - return NULL; - } - - _PyDict_Pop(watched, key2, Py_None); - _PyDict_Pop(unwatched, key2, Py_None); - - if (dict_watch_assert(5, "del:key2")) { - return NULL; - } - - PyDict_DelItemString(watched, "key1"); - PyDict_DelItemString(unwatched, "key1"); - - if (dict_watch_assert(6, "del:key1")) { - return NULL; - } - - PyDict_SetDefault(watched, key1, one); - PyDict_SetDefault(unwatched, key1, one); - - if (dict_watch_assert(7, "new:key1:1")) { - return NULL; - } - - int wid2 = PyDict_AddWatcher(dict_watch_callback_2); - if (try_watch(wid2, unwatched)) { - return NULL; + int watcher_id; + assert(PyLong_Check(kind)); + long kind_l = PyLong_AsLong(kind); + if (kind_l == 2) { + watcher_id = PyDict_AddWatcher(dict_watch_callback_second); + } else if (kind_l == 1) { + watcher_id = PyDict_AddWatcher(dict_watch_callback_error); + } else { + watcher_id = PyDict_AddWatcher(dict_watch_callback); } - - PyDict_Clear(watched); - - if (dict_watch_assert(8, "clear")) { + if (watcher_id < 0) { return NULL; } - - PyDict_Clear(unwatched); - - if (dict_watch_assert(9, "second")) { - return NULL; + if (!g_dict_watchers_installed) { + assert(!g_dict_watch_events); + if (!(g_dict_watch_events = PyList_New(0))) { + return NULL; + } } + g_dict_watchers_installed++; + return PyLong_FromLong(watcher_id); +} - PyObject *copy = PyDict_Copy(watched); - // copied dict is not watched, so this does not add an event - Py_CLEAR(copy); - - Py_CLEAR(watched); - - if (dict_watch_assert(10, "dealloc")) { +static PyObject * +clear_dict_watcher(PyObject *self, PyObject *watcher_id) +{ + if (PyDict_ClearWatcher(PyLong_AsLong(watcher_id))) { return NULL; } - - // it is an error to try to watch a non-dict - if (dict_watch_assert_error(wid, one, "PyDict_Watch() succeeded on non-dict")) { - return NULL; + g_dict_watchers_installed--; + if (!g_dict_watchers_installed) { + assert(g_dict_watch_events); + Py_CLEAR(g_dict_watch_events); } + Py_RETURN_NONE; +} - // It is an error to pass an out-of-range watcher ID - if (dict_watch_assert_error(-1, unwatched, "PyDict_Watch() succeeded on negative watcher ID")) { +static PyObject * +watch_dict(PyObject *self, PyObject *args) +{ + PyObject *dict; + int watcher_id; + if (!PyArg_ParseTuple(args, "iO", &watcher_id, &dict)) { return NULL; } - if (dict_watch_assert_error(8, unwatched, "PyDict_Watch() succeeded on too-large watcher ID")) { + if (PyDict_Watch(watcher_id, dict)) { return NULL; } + Py_RETURN_NONE; +} - // It is an error to pass a never-registered watcher ID - if (dict_watch_assert_error(7, unwatched, "PyDict_Watch() succeeded on unused watcher ID")) { +static PyObject * +get_dict_watcher_events(PyObject *self, PyObject *Py_UNUSED(args)) +{ + if (!g_dict_watch_events) { + PyErr_SetString(PyExc_RuntimeError, "no watchers active"); return NULL; } - - Py_CLEAR(unwatched); - Py_CLEAR(g_dict_watch_events); - Py_DECREF(one); - Py_DECREF(two); - Py_DECREF(key1); - Py_DECREF(key2); - Py_RETURN_NONE; + Py_INCREF(g_dict_watch_events); + return g_dict_watch_events; } @@ -5982,7 +5898,10 @@ static PyMethodDef TestMethods[] = { {"settrace_to_record", settrace_to_record, METH_O, NULL}, {"test_macros", test_macros, METH_NOARGS, NULL}, {"clear_managed_dict", clear_managed_dict, METH_O, NULL}, - {"test_watch_dict", test_watch_dict, METH_NOARGS, NULL}, + {"add_dict_watcher", add_dict_watcher, METH_O, NULL}, + {"clear_dict_watcher", clear_dict_watcher, METH_O, NULL}, + {"watch_dict", watch_dict, METH_VARARGS, NULL}, + {"get_dict_watcher_events", get_dict_watcher_events, METH_NOARGS, NULL}, {NULL, NULL} /* sentinel */ }; diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 395b8a77c5fafd..f85385017b199d 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -5747,7 +5747,7 @@ PyDict_AddWatcher(PyDict_WatchCallback callback) for (int i = 0; i < DICT_MAX_WATCHERS; i++) { if (!interp->dict_watchers[i]) { - interp->dict_watchers[i] = (void*)callback; + interp->dict_watchers[i] = callback; return i; } } @@ -5756,6 +5756,22 @@ PyDict_AddWatcher(PyDict_WatchCallback callback) return -1; } +int +PyDict_ClearWatcher(int watcher_id) +{ + if (watcher_id < 0 || watcher_id >= DICT_MAX_WATCHERS) { + PyErr_Format(PyExc_ValueError, "Invalid dict watcher ID %d", watcher_id); + return -1; + } + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (!interp->dict_watchers[watcher_id]) { + PyErr_Format(PyExc_ValueError, "No dict watcher set for ID %d", watcher_id); + return -1; + } + interp->dict_watchers[watcher_id] = NULL; + return 0; +} + void _PyDict_SendEvent(int watcher_bits, PyDict_WatchEvent event, @@ -5766,9 +5782,13 @@ _PyDict_SendEvent(int watcher_bits, PyInterpreterState *interp = _PyInterpreterState_GET(); for (int i = 0; i < DICT_MAX_WATCHERS; i++) { if (watcher_bits & 1) { - PyDict_WatchCallback cb = (PyDict_WatchCallback)interp->dict_watchers[i]; + PyDict_WatchCallback cb = interp->dict_watchers[i]; if (cb) { - cb(event, (PyObject*)mp, key, value); + if (cb(event, (PyObject*)mp, key, value) < 0) { + // some dict modification paths (e.g. PyDict_Clear) can't raise, so we + // can't propagate exceptions from dict watchers. + PyErr_WriteUnraisable((PyObject *)mp); + } } } watcher_bits >>= 1; From fb1961799b8f388d4bc8bbc21a947a7f91121d7a Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Thu, 6 Oct 2022 10:49:00 -0700 Subject: [PATCH 5/6] Minor cleanups --- Objects/dictobject.c | 10 ++++------ Python/pystate.c | 4 ++++ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index f85385017b199d..510b24ac980cd0 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -5783,12 +5783,10 @@ _PyDict_SendEvent(int watcher_bits, for (int i = 0; i < DICT_MAX_WATCHERS; i++) { if (watcher_bits & 1) { PyDict_WatchCallback cb = interp->dict_watchers[i]; - if (cb) { - if (cb(event, (PyObject*)mp, key, value) < 0) { - // some dict modification paths (e.g. PyDict_Clear) can't raise, so we - // can't propagate exceptions from dict watchers. - PyErr_WriteUnraisable((PyObject *)mp); - } + if (cb && (cb(event, (PyObject*)mp, key, value) < 0)) { + // some dict modification paths (e.g. PyDict_Clear) can't raise, so we + // can't propagate exceptions from dict watchers. + PyErr_WriteUnraisable((PyObject *)mp); } } watcher_bits >>= 1; diff --git a/Python/pystate.c b/Python/pystate.c index 50ae0ce682170b..c74868ddfa20f3 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -451,6 +451,10 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) Py_CLEAR(interp->sysdict); Py_CLEAR(interp->builtins); + for (int i=0; i < DICT_MAX_WATCHERS; i++) { + interp->dict_watchers[i] = NULL; + } + // XXX Once we have one allocator per interpreter (i.e. // per-interpreter GC) we must ensure that all of the interpreter's // objects have been cleaned up at the point. From cc1d0b7711dc3692c6287a097a99d2b3418eda7e Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Thu, 6 Oct 2022 16:07:09 -0700 Subject: [PATCH 6/6] Address further review comments --- Doc/c-api/dict.rst | 20 ++++++++++---------- Include/cpython/dictobject.h | 2 +- Modules/_testcapimodule.c | 2 +- Objects/dictobject.c | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index 97f83d522af2d4..7bebea0c97de5a 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -262,17 +262,17 @@ Dictionary Objects Enumeration of possible dictionary watcher events: ``PyDict_EVENT_ADDED``, ``PyDict_EVENT_MODIFIED``, ``PyDict_EVENT_DELETED``, ``PyDict_EVENT_CLONED``, - ``PyDict_EVENT_CLEARED``, or ``PyDict_EVENT_DEALLOCED``. + ``PyDict_EVENT_CLEARED``, or ``PyDict_EVENT_DEALLOCATED``. .. c:type:: int (*PyDict_WatchCallback)(PyDict_WatchEvent event, PyObject *dict, PyObject *key, PyObject *new_value) Type of a dict watcher callback function. - If *event* is ``PyDict_EVENT_CLEARED`` or ``PyDict_EVENT_DEALLOCED``, both - *key* and *new_value* will be ``NULL``. If *event* is - ``PyDict_EVENT_ADDED`` or ``PyDict_EVENT_MODIFIED``, *new_value* will be the - new value for *key*. If *event* is ``PyDict_EVENT_DELETED``, *key* is being - deleted from the dictionary and *new_value* will be ``NULL``. + If *event* is ``PyDict_EVENT_CLEARED`` or ``PyDict_EVENT_DEALLOCATED``, both + *key* and *new_value* will be ``NULL``. If *event* is ``PyDict_EVENT_ADDED`` + or ``PyDict_EVENT_MODIFIED``, *new_value* will be the new value for *key*. + If *event* is ``PyDict_EVENT_DELETED``, *key* is being deleted from the + dictionary and *new_value* will be ``NULL``. ``PyDict_EVENT_CLONED`` occurs when *dict* was previously empty and another dict is merged into it. To maintain efficiency of this operation, per-key @@ -280,12 +280,12 @@ Dictionary Objects single ``PyDict_EVENT_CLONED`` is issued, and *key* will be the source dictionary. - The callback may inspect but should not modify *dict*; doing so could have + The callback may inspect but must not modify *dict*; doing so could have unpredictable effects, including infinite recursion. Callbacks occur before the notified modification to *dict* takes place, so the prior state of *dict* can be inspected. - If an error occurs in the callback, it may return ``-1`` with an exception - set; this exception will be printed as an unraisable exception using - :c:func:`PyErr_WriteUnraisable`. On success it should return ``0``. + If the callback returns with an exception set, it must return ``-1``; this + exception will be printed as an unraisable exception using + :c:func:`PyErr_WriteUnraisable`. Otherwise it should return ``0``. diff --git a/Include/cpython/dictobject.h b/Include/cpython/dictobject.h index 05c751ff4b22c3..f8a74a597b0ea2 100644 --- a/Include/cpython/dictobject.h +++ b/Include/cpython/dictobject.h @@ -92,7 +92,7 @@ typedef enum { PyDict_EVENT_DELETED, PyDict_EVENT_CLONED, PyDict_EVENT_CLEARED, - PyDict_EVENT_DEALLOCED, + PyDict_EVENT_DEALLOCATED, } PyDict_WatchEvent; // Callback to be invoked when a watched dict is cleared, dealloced, or modified. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 624aaed4ba918a..c57dba4a5bf39e 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -5184,7 +5184,7 @@ dict_watch_callback(PyDict_WatchEvent event, case PyDict_EVENT_CLEARED: msg = PyUnicode_FromString("clear"); break; - case PyDict_EVENT_DEALLOCED: + case PyDict_EVENT_DEALLOCATED: msg = PyUnicode_FromString("dealloc"); break; case PyDict_EVENT_CLONED: diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 510b24ac980cd0..6542b1803ffa2e 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -2330,7 +2330,7 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) static void dict_dealloc(PyDictObject *mp) { - _PyDict_NotifyEvent(PyDict_EVENT_DEALLOCED, mp, NULL, NULL); + _PyDict_NotifyEvent(PyDict_EVENT_DEALLOCATED, mp, NULL, NULL); PyDictValues *values = mp->ma_values; PyDictKeysObject *keys = mp->ma_keys; Py_ssize_t i, n; pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy