From efd5f33336df065d594f6a33f8aea10717867bc5 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 29 Feb 2024 18:12:04 -0700 Subject: [PATCH 01/15] Make Interpreter instances pickleable. --- Lib/test/support/interpreters/__init__.py | 8 ++++++++ Lib/test/test_interpreters/test_api.py | 7 +++++++ 2 files changed, 15 insertions(+) diff --git a/Lib/test/support/interpreters/__init__.py b/Lib/test/support/interpreters/__init__.py index d02ffbae1113c0..d8e6654fc96efd 100644 --- a/Lib/test/support/interpreters/__init__.py +++ b/Lib/test/support/interpreters/__init__.py @@ -129,6 +129,14 @@ def __hash__(self): def __del__(self): self._decref() + # for pickling: + def __getnewargs__(self): + return (self._id,) + + # for pickling: + def __getstate__(self): + return None + def _decref(self): if not self._ownsref: return diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index 363143fa810f35..3cde9bd0014d9a 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -1,4 +1,5 @@ import os +import pickle import threading from textwrap import dedent import unittest @@ -261,6 +262,12 @@ def test_equality(self): self.assertEqual(interp1, interp1) self.assertNotEqual(interp1, interp2) + def test_pickle(self): + interp = interpreters.create() + data = pickle.dumps(interp) + unpickled = pickle.loads(data) + self.assertEqual(unpickled, interp) + class TestInterpreterIsRunning(TestBase): From acd472a0b6c1be443062a94e7048f4c5177fbd05 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 29 Feb 2024 18:11:47 -0700 Subject: [PATCH 02/15] Make interpreters.Queue instances pickleable. --- Lib/test/support/interpreters/queues.py | 8 ++++++++ Lib/test/test_interpreters/test_queues.py | 7 +++++++ 2 files changed, 15 insertions(+) diff --git a/Lib/test/support/interpreters/queues.py b/Lib/test/support/interpreters/queues.py index f9978f0bec5a62..4933f2d8fbf2c5 100644 --- a/Lib/test/support/interpreters/queues.py +++ b/Lib/test/support/interpreters/queues.py @@ -93,6 +93,14 @@ def __repr__(self): def __hash__(self): return hash(self._id) + # for pickling: + def __getnewargs__(self): + return (self._id,) + + # for pickling: + def __getstate__(self): + return None + @property def id(self): return self._id diff --git a/Lib/test/test_interpreters/test_queues.py b/Lib/test/test_interpreters/test_queues.py index 0a1fdb41f73166..d0a2d823c88138 100644 --- a/Lib/test/test_interpreters/test_queues.py +++ b/Lib/test/test_interpreters/test_queues.py @@ -1,4 +1,5 @@ import importlib +import pickle import threading from textwrap import dedent import unittest @@ -127,6 +128,12 @@ def test_equality(self): self.assertEqual(queue1, queue1) self.assertNotEqual(queue1, queue2) + def test_pickle(self): + queue = queues.create() + data = pickle.dumps(queue) + unpickled = pickle.loads(data) + self.assertEqual(unpickled, queue) + class TestQueueOps(TestBase): From 98a03d2a9756872182a71bf94f24cea7266dd92a Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 4 Mar 2024 13:55:15 -0700 Subject: [PATCH 03/15] Make interpreters.*Channel instances pickleable. --- Lib/test/support/interpreters/channels.py | 12 +++++++++++- Lib/test/test_interpreters/test_channels.py | 13 +++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/Lib/test/support/interpreters/channels.py b/Lib/test/support/interpreters/channels.py index 75a5a60f54f926..f7f523b1fc5a77 100644 --- a/Lib/test/support/interpreters/channels.py +++ b/Lib/test/support/interpreters/channels.py @@ -38,7 +38,8 @@ class _ChannelEnd: _end = None - def __init__(self, cid): + def __new__(cls, cid): + self = super().__new__(cls) if self._end == 'send': cid = _channels._channel_id(cid, send=True, force=True) elif self._end == 'recv': @@ -46,6 +47,7 @@ def __init__(self, cid): else: raise NotImplementedError(self._end) self._id = cid + return self def __repr__(self): return f'{type(self).__name__}(id={int(self._id)})' @@ -61,6 +63,14 @@ def __eq__(self, other): return NotImplemented return other._id == self._id + # for pickling: + def __getnewargs__(self): + return (int(self._id),) + + # for pickling: + def __getstate__(self): + return None + @property def id(self): return self._id diff --git a/Lib/test/test_interpreters/test_channels.py b/Lib/test/test_interpreters/test_channels.py index 57204e2776468d..7e0b82884c33d3 100644 --- a/Lib/test/test_interpreters/test_channels.py +++ b/Lib/test/test_interpreters/test_channels.py @@ -1,4 +1,5 @@ import importlib +import pickle import threading from textwrap import dedent import unittest @@ -100,6 +101,12 @@ def test_equality(self): self.assertEqual(ch1, ch1) self.assertNotEqual(ch1, ch2) + def test_pickle(self): + ch, _ = channels.create() + data = pickle.dumps(ch) + unpickled = pickle.loads(data) + self.assertEqual(unpickled, ch) + class TestSendChannelAttrs(TestBase): @@ -125,6 +132,12 @@ def test_equality(self): self.assertEqual(ch1, ch1) self.assertNotEqual(ch1, ch2) + def test_pickle(self): + _, ch = channels.create() + data = pickle.dumps(ch) + unpickled = pickle.loads(data) + self.assertEqual(unpickled, ch) + class TestSendRecv(TestBase): From 46c6fb58fa0ca6bd2e0205ea45b41a04d7218832 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 1 Mar 2024 13:10:22 -0700 Subject: [PATCH 04/15] Raise QueueNotFoundError from _interpqueues.release(). --- Modules/_xxinterpqueuesmodule.c | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/Modules/_xxinterpqueuesmodule.c b/Modules/_xxinterpqueuesmodule.c index 1b76b6963ae0f1..f95ca7710a8e24 100644 --- a/Modules/_xxinterpqueuesmodule.c +++ b/Modules/_xxinterpqueuesmodule.c @@ -824,17 +824,17 @@ _queues_incref(_queues *queues, int64_t qid) static void _queue_free(_queue *); -static void +static int _queues_decref(_queues *queues, int64_t qid) { + int res = -1; PyThread_acquire_lock(queues->mutex, WAIT_LOCK); _queueref *prev = NULL; _queueref *ref = _queuerefs_find(queues->head, qid, &prev); if (ref == NULL) { assert(!PyErr_Occurred()); - // Already destroyed. - // XXX Warn? + res = ERR_QUEUE_NOT_FOUND; goto finally; } assert(ref->refcount > 0); @@ -852,8 +852,10 @@ _queues_decref(_queues *queues, int64_t qid) return; } + res = 0 finally: PyThread_release_lock(queues->mutex); + return res; } struct queue_id_and_fmt { @@ -1152,7 +1154,14 @@ _queueid_xid_free(void *data) int64_t qid = ((struct _queueid_xid *)data)->qid; PyMem_RawFree(data); _queues *queues = _get_global_queues(); - _queues_decref(queues, qid); + int res = _queues_decref(queues, qid); + if (res == ERR_QUEUE_NOT_FOUND) { + // Already destroyed. + // XXX Warn? + } + else { + assert(res == 0); + } } static PyObject * @@ -1491,7 +1500,10 @@ queuesmod_release(PyObject *self, PyObject *args, PyObject *kwds) // XXX Check module state if bound already. // XXX Update module state. - _queues_decref(&_globals.queues, qid); + int err = _queues_decref(&_globals.queues, qid); + if (handle_queue_error(err, self, qid)) { + return NULL; + } Py_RETURN_NONE; } From 7df489d29eca8b1470104974b3b88b5b328c78f1 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 1 Mar 2024 14:11:22 -0700 Subject: [PATCH 05/15] Combine the two QueueEmpty (and QueueFull) exception types. --- Lib/test/support/interpreters/queues.py | 21 ++-- Modules/_xxinterpqueuesmodule.c | 126 ++++++++++++++++++------ 2 files changed, 105 insertions(+), 42 deletions(-) diff --git a/Lib/test/support/interpreters/queues.py b/Lib/test/support/interpreters/queues.py index 4933f2d8fbf2c5..5c5bcce8d2cf07 100644 --- a/Lib/test/support/interpreters/queues.py +++ b/Lib/test/support/interpreters/queues.py @@ -18,14 +18,14 @@ ] -class QueueEmpty(_queues.QueueEmpty, queue.Empty): +class QueueEmpty(QueueError, queue.Empty): """Raised from get_nowait() when the queue is empty. It is also raised from get() if it times out. """ -class QueueFull(_queues.QueueFull, queue.Full): +class QueueFull(QueueError, queue.Full): """Raised from put_nowait() when the queue is full. It is also raised from put() if it times out. @@ -167,9 +167,8 @@ def put(self, obj, timeout=None, *, while True: try: _queues.put(self._id, obj, fmt) - except _queues.QueueFull as exc: + except QueueFull as exc: if timeout is not None and time.time() >= end: - exc.__class__ = QueueFull raise # re-raise time.sleep(_delay) else: @@ -182,11 +181,7 @@ def put_nowait(self, obj, *, syncobj=None): fmt = _SHARED_ONLY if syncobj else _PICKLED if fmt is _PICKLED: obj = pickle.dumps(obj) - try: - _queues.put(self._id, obj, fmt) - except _queues.QueueFull as exc: - exc.__class__ = QueueFull - raise # re-raise + _queues.put(self._id, obj, fmt) def get(self, timeout=None, *, _delay=10 / 1000, # 10 milliseconds @@ -203,9 +198,8 @@ def get(self, timeout=None, *, while True: try: obj, fmt = _queues.get(self._id) - except _queues.QueueEmpty as exc: + except QueueEmpty as exc: if timeout is not None and time.time() >= end: - exc.__class__ = QueueEmpty raise # re-raise time.sleep(_delay) else: @@ -224,8 +218,7 @@ def get_nowait(self): """ try: obj, fmt = _queues.get(self._id) - except _queues.QueueEmpty as exc: - exc.__class__ = QueueEmpty + except QueueEmpty as exc: raise # re-raise if fmt == _PICKLED: obj = pickle.loads(obj) @@ -234,4 +227,4 @@ def get_nowait(self): return obj -_queues._register_queue_type(Queue) +_queues._register_heap_types(Queue, QueueEmpty, QueueFull) diff --git a/Modules/_xxinterpqueuesmodule.c b/Modules/_xxinterpqueuesmodule.c index f95ca7710a8e24..8c848a28e23e12 100644 --- a/Modules/_xxinterpqueuesmodule.c +++ b/Modules/_xxinterpqueuesmodule.c @@ -128,6 +128,22 @@ idarg_int64_converter(PyObject *arg, void *ptr) } +static int +ensure_highlevel_module_loaded(void) +{ + PyObject *highlevel = PyImport_ImportModule("interpreters.queues"); + if (highlevel == NULL) { + PyErr_Clear(); + highlevel = PyImport_ImportModule("test.support.interpreters.queues"); + if (highlevel == NULL) { + return -1; + } + } + Py_DECREF(highlevel); + return 0; +} + + /* module state *************************************************************/ typedef struct { @@ -196,6 +212,8 @@ clear_module_state(module_state *state) #define ERR_QUEUE_EMPTY (-21) #define ERR_QUEUE_FULL (-22) +static int ensure_external_exc_types(module_state *); + static int resolve_module_errcode(module_state *state, int errcode, int64_t qid, PyObject **p_exctype, PyObject **p_msgobj) @@ -212,10 +230,16 @@ resolve_module_errcode(module_state *state, int errcode, int64_t qid, msg = PyUnicode_FromFormat("queue %" PRId64 " not found", qid); break; case ERR_QUEUE_EMPTY: + if (ensure_external_exc_types(state) < 0) { + return -1; + } exctype = state->QueueEmpty; msg = PyUnicode_FromFormat("queue %" PRId64 " is empty", qid); break; case ERR_QUEUE_FULL: + if (ensure_external_exc_types(state) < 0) { + return -1; + } exctype = state->QueueFull; msg = PyUnicode_FromFormat("queue %" PRId64 " is full", qid); break; @@ -267,20 +291,59 @@ add_QueueError(PyObject *mod) #define PREFIX "test.support.interpreters." #define ADD_EXCTYPE(NAME, BASE, DOC) \ + assert(state->NAME == NULL); \ if (add_exctype(mod, &state->NAME, PREFIX #NAME, DOC, BASE) < 0) { \ return -1; \ } ADD_EXCTYPE(QueueError, PyExc_RuntimeError, "Indicates that a queue-related error happened.") ADD_EXCTYPE(QueueNotFoundError, state->QueueError, NULL) - ADD_EXCTYPE(QueueEmpty, state->QueueError, NULL) - ADD_EXCTYPE(QueueFull, state->QueueError, NULL) + // QueueEmpty and QueueFull are set by set_external_exc_types(). + state->QueueEmpty = NULL; + state->QueueFull = NULL; #undef ADD_EXCTYPE #undef PREFIX return 0; } +static int +set_external_exc_types(module_state *state, + PyObject *emptyerror, PyObject *fullerror) +{ + if (state->QueueEmpty != NULL) { + assert(state->QueueFull != NULL); + Py_CLEAR(state->QueueEmpty); + Py_CLEAR(state->QueueFull); + } + else { + assert(state->QueueFull == NULL); + } + assert(PyObject_IsSubclass(emptyerror, state->QueueError)); + assert(PyObject_IsSubclass(fullerror, state->QueueError)); + state->QueueEmpty = Py_NewRef(emptyerror); + state->QueueFull = Py_NewRef(fullerror); + return 0; +} + +static int +ensure_external_exc_types(module_state *state) +{ + if (state->QueueEmpty != NULL) { + assert(state->QueueFull != NULL); + return 0; + } + assert(state->QueueFull == NULL); + + // Force the module to be loaded, to register the type. + if (ensure_highlevel_module_loaded() < 0) { + return -1; + } + assert(state->QueueEmpty != NULL); + assert(state->QueueFull != NULL); + return 0; +} + static int handle_queue_error(int err, PyObject *mod, int64_t qid) { @@ -849,10 +912,10 @@ _queues_decref(_queues *queues, int64_t qid) _queue_kill_and_wait(queue); _queue_free(queue); - return; + return 0; } - res = 0 + res = 0; finally: PyThread_release_lock(queues->mutex); return res; @@ -1079,10 +1142,8 @@ static int _queueobj_shared(PyThreadState *, PyObject *, _PyCrossInterpreterData *); static int -set_external_queue_type(PyObject *module, PyTypeObject *queue_type) +set_external_queue_type(module_state *state, PyTypeObject *queue_type) { - module_state *state = get_module_state(module); - // Clear the old value if the .py module was reloaded. if (state->queue_type != NULL) { (void)_PyCrossInterpreterData_UnregisterClass( @@ -1107,15 +1168,9 @@ get_external_queue_type(PyObject *module) PyTypeObject *cls = state->queue_type; if (cls == NULL) { // Force the module to be loaded, to register the type. - PyObject *highlevel = PyImport_ImportModule("interpreters.queue"); - if (highlevel == NULL) { - PyErr_Clear(); - highlevel = PyImport_ImportModule("test.support.interpreters.queue"); - if (highlevel == NULL) { - return NULL; - } + if (ensure_highlevel_module_loaded() < 0) { + return NULL; } - Py_DECREF(highlevel); cls = state->queue_type; assert(cls != NULL); } @@ -1407,6 +1462,7 @@ queuesmod_put(PyObject *self, PyObject *args, PyObject *kwds) /* Queue up the object. */ int err = queue_put(&_globals.queues, qid, obj, fmt); + // This is the only place that raises QueueFull. if (handle_queue_error(err, self, qid)) { return NULL; } @@ -1434,11 +1490,8 @@ queuesmod_get(PyObject *self, PyObject *args, PyObject *kwds) PyObject *obj = NULL; int fmt = 0; int err = queue_get(&_globals.queues, qid, &obj, &fmt); - if (err == ERR_QUEUE_EMPTY && dflt != NULL) { - assert(obj == NULL); - obj = Py_NewRef(dflt); - } - else if (handle_queue_error(err, self, qid)) { + // This is the only place that raises QueueEmpty. + if (handle_queue_error(err, self, qid)) { return NULL; } @@ -1621,22 +1674,39 @@ PyDoc_STRVAR(queuesmod_get_count_doc, Return the number of items in the queue."); static PyObject * -queuesmod__register_queue_type(PyObject *self, PyObject *args, PyObject *kwds) +queuesmod__register_heap_types(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"queuetype", NULL}; + static char *kwlist[] = {"queuetype", "emptyerror", "fullerror", NULL}; PyObject *queuetype; + PyObject *emptyerror; + PyObject *fullerror; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O:_register_queue_type", kwlist, - &queuetype)) { + "OOO:_register_heap_types", kwlist, + &queuetype, &emptyerror, &fullerror)) { return NULL; } if (!PyType_Check(queuetype)) { - PyErr_SetString(PyExc_TypeError, "expected a type for 'queuetype'"); + PyErr_SetString(PyExc_TypeError, + "expected a type for 'queuetype'"); + return NULL; + } + if (!PyExceptionClass_Check(emptyerror)) { + PyErr_SetString(PyExc_TypeError, + "expected an exception type for 'emptyerror'"); + return NULL; + } + if (!PyExceptionClass_Check(fullerror)) { + PyErr_SetString(PyExc_TypeError, + "expected an exception type for 'fullerror'"); return NULL; } - PyTypeObject *cls_queue = (PyTypeObject *)queuetype; - if (set_external_queue_type(self, cls_queue) < 0) { + module_state *state = get_module_state(self); + + if (set_external_queue_type(state, (PyTypeObject *)queuetype) < 0) { + return NULL; + } + if (set_external_exc_types(state, emptyerror, fullerror) < 0) { return NULL; } @@ -1666,7 +1736,7 @@ static PyMethodDef module_functions[] = { METH_VARARGS | METH_KEYWORDS, queuesmod_is_full_doc}, {"get_count", _PyCFunction_CAST(queuesmod_get_count), METH_VARARGS | METH_KEYWORDS, queuesmod_get_count_doc}, - {"_register_queue_type", _PyCFunction_CAST(queuesmod__register_queue_type), + {"_register_heap_types", _PyCFunction_CAST(queuesmod__register_heap_types), METH_VARARGS | METH_KEYWORDS, NULL}, {NULL, NULL} /* sentinel */ From dddd40bcf20dd83c90c16b4aaea27b7d679040bd Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 1 Mar 2024 16:26:59 -0700 Subject: [PATCH 06/15] Fix docstrings in the low-level module. --- Modules/_xxinterpqueuesmodule.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Modules/_xxinterpqueuesmodule.c b/Modules/_xxinterpqueuesmodule.c index 8c848a28e23e12..bfa535c6b08945 100644 --- a/Modules/_xxinterpqueuesmodule.c +++ b/Modules/_xxinterpqueuesmodule.c @@ -1383,7 +1383,7 @@ queuesmod_create(PyObject *self, PyObject *args, PyObject *kwds) } PyDoc_STRVAR(queuesmod_create_doc, -"create() -> qid\n\ +"create(maxsize, fmt) -> qid\n\ \n\ Create a new cross-interpreter queue and return its unique generated ID.\n\ It is a new reference as though bind() had been called on the queue."); @@ -1443,9 +1443,10 @@ queuesmod_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) } PyDoc_STRVAR(queuesmod_list_all_doc, -"list_all() -> [qid]\n\ +"list_all() -> [(qid, fmt)]\n\ \n\ -Return the list of IDs for all queues."); +Return the list of IDs for all queues.\n\ +Each corresponding default format is also included."); static PyObject * queuesmod_put(PyObject *self, PyObject *args, PyObject *kwds) @@ -1471,7 +1472,7 @@ queuesmod_put(PyObject *self, PyObject *args, PyObject *kwds) } PyDoc_STRVAR(queuesmod_put_doc, -"put(qid, obj, sharedonly=False)\n\ +"put(qid, obj, fmt)\n\ \n\ Add the object's data to the queue."); @@ -1501,9 +1502,10 @@ queuesmod_get(PyObject *self, PyObject *args, PyObject *kwds) } PyDoc_STRVAR(queuesmod_get_doc, -"get(qid, [default]) -> obj\n\ +"get(qid, [default]) -> (obj, fmt)\n\ \n\ Return a new object from the data at the front of the queue.\n\ +The object's format is also returned.\n\ \n\ If there is nothing to receive then raise QueueEmpty, unless\n\ a default value is provided. In that case return it."); From 512bed1231da6c7815975fb034a89ac808d6b187 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 1 Mar 2024 16:27:44 -0700 Subject: [PATCH 07/15] get_default_fmt -> get_queue_defaults --- Lib/test/support/interpreters/queues.py | 2 +- Modules/_xxinterpqueuesmodule.c | 30 ++++++++++++++++--------- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/Lib/test/support/interpreters/queues.py b/Lib/test/support/interpreters/queues.py index 5c5bcce8d2cf07..5849a1cc15e447 100644 --- a/Lib/test/support/interpreters/queues.py +++ b/Lib/test/support/interpreters/queues.py @@ -66,7 +66,7 @@ def __new__(cls, id, /, *, _fmt=None): else: raise TypeError(f'id must be an int, got {id!r}') if _fmt is None: - _fmt = _queues.get_default_fmt(id) + _fmt, = _queues.get_queue_defaults(id) try: self = _known_queues[id] except KeyError: diff --git a/Modules/_xxinterpqueuesmodule.c b/Modules/_xxinterpqueuesmodule.c index bfa535c6b08945..3ea92009dbffdf 100644 --- a/Modules/_xxinterpqueuesmodule.c +++ b/Modules/_xxinterpqueuesmodule.c @@ -1595,12 +1595,12 @@ PyDoc_STRVAR(queuesmod_get_maxsize_doc, Return the maximum number of items in the queue."); static PyObject * -queuesmod_get_default_fmt(PyObject *self, PyObject *args, PyObject *kwds) +queuesmod_get_queue_defaults(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"qid", NULL}; qidarg_converter_data qidarg; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O&:get_default_fmt", kwlist, + "O&:get_queue_defaults", kwlist, qidarg_converter, &qidarg)) { return NULL; } @@ -1613,13 +1613,21 @@ queuesmod_get_default_fmt(PyObject *self, PyObject *args, PyObject *kwds) } int fmt = queue->fmt; _queue_unmark_waiter(queue, _globals.queues.mutex); - return PyLong_FromLong(fmt); + + PyObject *fmt_obj = PyLong_FromLong(fmt); + if (fmt_obj == NULL) { + return NULL; + } + // For now queues only have one default. + PyObject *res = PyTuple_Pack(1, fmt_obj); + Py_DECREF(fmt_obj); + return res; } -PyDoc_STRVAR(queuesmod_get_default_fmt_doc, -"get_default_fmt(qid)\n\ +PyDoc_STRVAR(queuesmod_get_queue_defaults_doc, +"get_queue_defaults(qid)\n\ \n\ -Return the default format to use for the queue."); +Return the queue's default values, set when it was created."); static PyObject * queuesmod_is_full(PyObject *self, PyObject *args, PyObject *kwds) @@ -1722,18 +1730,18 @@ static PyMethodDef module_functions[] = { METH_VARARGS | METH_KEYWORDS, queuesmod_destroy_doc}, {"list_all", queuesmod_list_all, METH_NOARGS, queuesmod_list_all_doc}, - {"put", _PyCFunction_CAST(queuesmod_put), + {"put", _PyCFunction_CAST(queuesmod_put), METH_VARARGS | METH_KEYWORDS, queuesmod_put_doc}, - {"get", _PyCFunction_CAST(queuesmod_get), + {"get", _PyCFunction_CAST(queuesmod_get), METH_VARARGS | METH_KEYWORDS, queuesmod_get_doc}, - {"bind", _PyCFunction_CAST(queuesmod_bind), + {"bind", _PyCFunction_CAST(queuesmod_bind), METH_VARARGS | METH_KEYWORDS, queuesmod_bind_doc}, {"release", _PyCFunction_CAST(queuesmod_release), METH_VARARGS | METH_KEYWORDS, queuesmod_release_doc}, {"get_maxsize", _PyCFunction_CAST(queuesmod_get_maxsize), METH_VARARGS | METH_KEYWORDS, queuesmod_get_maxsize_doc}, - {"get_default_fmt", _PyCFunction_CAST(queuesmod_get_default_fmt), - METH_VARARGS | METH_KEYWORDS, queuesmod_get_default_fmt_doc}, + {"get_queue_defaults", _PyCFunction_CAST(queuesmod_get_queue_defaults), + METH_VARARGS | METH_KEYWORDS, queuesmod_get_queue_defaults_doc}, {"is_full", _PyCFunction_CAST(queuesmod_is_full), METH_VARARGS | METH_KEYWORDS, queuesmod_is_full_doc}, {"get_count", _PyCFunction_CAST(queuesmod_get_count), From 55e0e925c264ea23a7778456c004d75f0a8b744b Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 1 Mar 2024 16:32:58 -0700 Subject: [PATCH 08/15] Drop the default arg from _interpqueues.get(). --- Modules/_xxinterpqueuesmodule.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/Modules/_xxinterpqueuesmodule.c b/Modules/_xxinterpqueuesmodule.c index 3ea92009dbffdf..f137359b3a36f2 100644 --- a/Modules/_xxinterpqueuesmodule.c +++ b/Modules/_xxinterpqueuesmodule.c @@ -1479,11 +1479,10 @@ Add the object's data to the queue."); static PyObject * queuesmod_get(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"qid", "default", NULL}; + static char *kwlist[] = {"qid", NULL}; qidarg_converter_data qidarg; - PyObject *dflt = NULL; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&|O:get", kwlist, - qidarg_converter, &qidarg, &dflt)) { + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&:get", kwlist, + qidarg_converter, &qidarg)) { return NULL; } int64_t qid = qidarg.id; @@ -1502,13 +1501,12 @@ queuesmod_get(PyObject *self, PyObject *args, PyObject *kwds) } PyDoc_STRVAR(queuesmod_get_doc, -"get(qid, [default]) -> (obj, fmt)\n\ +"get(qid) -> (obj, fmt)\n\ \n\ Return a new object from the data at the front of the queue.\n\ The object's format is also returned.\n\ \n\ -If there is nothing to receive then raise QueueEmpty, unless\n\ -a default value is provided. In that case return it."); +If there is nothing to receive then raise QueueEmpty."); static PyObject * queuesmod_bind(PyObject *self, PyObject *args, PyObject *kwds) From b15e706773d00ecb90533c2023ed3ac89242f4c1 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 1 Mar 2024 18:20:55 -0700 Subject: [PATCH 09/15] Add low-level tests. --- Lib/test/test_interpreters/test_queues.py | 65 ++++++++++++++++++++++- Modules/_xxinterpqueuesmodule.c | 5 +- 2 files changed, 67 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_interpreters/test_queues.py b/Lib/test/test_interpreters/test_queues.py index d0a2d823c88138..01f153ef85c27c 100644 --- a/Lib/test/test_interpreters/test_queues.py +++ b/Lib/test/test_interpreters/test_queues.py @@ -10,10 +10,14 @@ _queues = import_helper.import_module('_xxinterpqueues') from test.support import interpreters from test.support.interpreters import queues -from .utils import _run_output, TestBase +from .utils import _run_output, TestBase as _TestBase -class TestBase(TestBase): +def get_num_queues(): + return len(_queues.list_all()) + + +class TestBase(_TestBase): def tearDown(self): for qid in _queues.list_all(): try: @@ -35,6 +39,63 @@ def test_highlevel_reloaded(self): # See gh-115490 (https://github.com/python/cpython/issues/115490). importlib.reload(queues) + def test_create_destroy(self): + qid = _queues.create(2, 0) + _queues.destroy(qid) + self.assertEqual(get_num_queues(), 0) + with self.assertRaises(queues.QueueNotFoundError): + _queues.get(qid) + with self.assertRaises(queues.QueueNotFoundError): + _queues.destroy(qid) + + def test_not_destroyed(self): + stdout, stderr = self.assert_python_failure( + '-c', + dedent(f""" + import {_queues.__name__} as _queues + _queues.create(2, 0) + """), + ) + # It should have aborted due to an assert. + self.assertEqual(stdout, '') + self.assertNotEqual(stderr, '') + + def test_bind_release(self): + with self.subTest('typical'): + qid = _queues.create(2, 0) + _queues.bind(qid) + _queues.release(qid) + self.assertEqual(get_num_queues(), 0) + + with self.subTest('bind too much'): + qid = _queues.create(2, 0) + _queues.bind(qid) + _queues.bind(qid) + _queues.release(qid) + _queues.destroy(qid) + self.assertEqual(get_num_queues(), 0) + + with self.subTest('nested'): + qid = _queues.create(2, 0) + _queues.bind(qid) + _queues.bind(qid) + _queues.release(qid) + _queues.release(qid) + self.assertEqual(get_num_queues(), 0) + + with self.subTest('release without binding'): + stdout, stderr = self.assert_python_failure( + '-c', + dedent(f""" + import {_queues.__name__} as _queues + _queues.create(2, 0) + _queues.release(qid) + """), + ) + # It should have aborted due to an assert. + self.assertEqual(stdout, '') + self.assertNotEqual(stderr, '') + class QueueTests(TestBase): diff --git a/Modules/_xxinterpqueuesmodule.c b/Modules/_xxinterpqueuesmodule.c index f137359b3a36f2..b41951a459e1cb 100644 --- a/Modules/_xxinterpqueuesmodule.c +++ b/Modules/_xxinterpqueuesmodule.c @@ -1386,7 +1386,10 @@ PyDoc_STRVAR(queuesmod_create_doc, "create(maxsize, fmt) -> qid\n\ \n\ Create a new cross-interpreter queue and return its unique generated ID.\n\ -It is a new reference as though bind() had been called on the queue."); +It is a new reference as though bind() had been called on the queue.\n\ +\n\ +The caller is responsible for calling destroy() for the new queue\n\ +before the runtime is finalized."); static PyObject * queuesmod_destroy(PyObject *self, PyObject *args, PyObject *kwds) From 38fe300665bf05d69309ffc1297e884b2a6d72d3 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 4 Mar 2024 12:21:40 -0700 Subject: [PATCH 10/15] _PyCrossInterpreterData_UnregisterClass -> clear_xid_class --- Modules/_interpreters_common.h | 8 ++++++++ Modules/_xxinterpchannelsmodule.c | 14 ++++++++------ Modules/_xxinterpqueuesmodule.c | 7 ++++--- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/Modules/_interpreters_common.h b/Modules/_interpreters_common.h index 5661a26d8790d1..07120f6ccc7207 100644 --- a/Modules/_interpreters_common.h +++ b/Modules/_interpreters_common.h @@ -11,3 +11,11 @@ ensure_xid_class(PyTypeObject *cls, crossinterpdatafunc getdata) //assert(cls->tp_flags & Py_TPFLAGS_HEAPTYPE); return _PyCrossInterpreterData_RegisterClass(cls, getdata); } + +#ifdef REGISTERS_HEAP_TYPES +static int +clear_xid_class(PyTypeObject *cls) +{ + return _PyCrossInterpreterData_UnregisterClass(cls); +} +#endif diff --git a/Modules/_xxinterpchannelsmodule.c b/Modules/_xxinterpchannelsmodule.c index 0ad184a78e8c1a..28ec00a159d6cd 100644 --- a/Modules/_xxinterpchannelsmodule.c +++ b/Modules/_xxinterpchannelsmodule.c @@ -17,7 +17,9 @@ #include // sched_yield() #endif +#define REGISTERS_HEAP_TYPES #include "_interpreters_common.h" +#undef REGISTERS_HEAP_TYPES /* @@ -281,17 +283,17 @@ clear_xid_types(module_state *state) { /* external types */ if (state->send_channel_type != NULL) { - (void)_PyCrossInterpreterData_UnregisterClass(state->send_channel_type); + (void)clear_xid_class(state->send_channel_type); Py_CLEAR(state->send_channel_type); } if (state->recv_channel_type != NULL) { - (void)_PyCrossInterpreterData_UnregisterClass(state->recv_channel_type); + (void)clear_xid_class(state->recv_channel_type); Py_CLEAR(state->recv_channel_type); } /* heap types */ if (state->ChannelIDType != NULL) { - (void)_PyCrossInterpreterData_UnregisterClass(state->ChannelIDType); + (void)clear_xid_class(state->ChannelIDType); Py_CLEAR(state->ChannelIDType); } } @@ -2677,11 +2679,11 @@ set_channelend_types(PyObject *mod, PyTypeObject *send, PyTypeObject *recv) // Clear the old values if the .py module was reloaded. if (state->send_channel_type != NULL) { - (void)_PyCrossInterpreterData_UnregisterClass(state->send_channel_type); + (void)clear_xid_class(state->send_channel_type); Py_CLEAR(state->send_channel_type); } if (state->recv_channel_type != NULL) { - (void)_PyCrossInterpreterData_UnregisterClass(state->recv_channel_type); + (void)clear_xid_class(state->recv_channel_type); Py_CLEAR(state->recv_channel_type); } @@ -2694,7 +2696,7 @@ set_channelend_types(PyObject *mod, PyTypeObject *send, PyTypeObject *recv) return -1; } if (ensure_xid_class(recv, _channelend_shared) < 0) { - (void)_PyCrossInterpreterData_UnregisterClass(state->send_channel_type); + (void)clear_xid_class(state->send_channel_type); Py_CLEAR(state->send_channel_type); Py_CLEAR(state->recv_channel_type); return -1; diff --git a/Modules/_xxinterpqueuesmodule.c b/Modules/_xxinterpqueuesmodule.c index b41951a459e1cb..2b5516d47cb99e 100644 --- a/Modules/_xxinterpqueuesmodule.c +++ b/Modules/_xxinterpqueuesmodule.c @@ -8,7 +8,9 @@ #include "Python.h" #include "pycore_crossinterp.h" // struct _xid +#define REGISTERS_HEAP_TYPES #include "_interpreters_common.h" +#undef REGISTERS_HEAP_TYPES #define MODULE_NAME _xxinterpqueues @@ -186,7 +188,7 @@ clear_module_state(module_state *state) { /* external types */ if (state->queue_type != NULL) { - (void)_PyCrossInterpreterData_UnregisterClass(state->queue_type); + (void)clear_xid_class(state->queue_type); } Py_CLEAR(state->queue_type); @@ -1146,8 +1148,7 @@ set_external_queue_type(module_state *state, PyTypeObject *queue_type) { // Clear the old value if the .py module was reloaded. if (state->queue_type != NULL) { - (void)_PyCrossInterpreterData_UnregisterClass( - state->queue_type); + (void)clear_xid_class(state->queue_type); Py_CLEAR(state->queue_type); } From 9496113df439c77692ac3d31a37afa7ae592ef5e Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 4 Mar 2024 15:37:23 -0700 Subject: [PATCH 11/15] Destroy any remaining queues. --- Lib/test/test_interpreters/test_queues.py | 11 +++--- Modules/_xxinterpqueuesmodule.c | 42 ++++++++++++++++++++--- 2 files changed, 45 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_interpreters/test_queues.py b/Lib/test/test_interpreters/test_queues.py index 01f153ef85c27c..b2654be28fc6ec 100644 --- a/Lib/test/test_interpreters/test_queues.py +++ b/Lib/test/test_interpreters/test_queues.py @@ -5,7 +5,7 @@ import unittest import time -from test.support import import_helper +from test.support import import_helper, Py_DEBUG # Raise SkipTest if subinterpreters not supported. _queues = import_helper.import_module('_xxinterpqueues') from test.support import interpreters @@ -49,16 +49,19 @@ def test_create_destroy(self): _queues.destroy(qid) def test_not_destroyed(self): - stdout, stderr = self.assert_python_failure( + # It should have cleaned up any remaining queues. + stdout, stderr = self.assert_python_ok( '-c', dedent(f""" import {_queues.__name__} as _queues _queues.create(2, 0) """), ) - # It should have aborted due to an assert. self.assertEqual(stdout, '') - self.assertNotEqual(stderr, '') + if Py_DEBUG: + self.assertNotEqual(stderr, '') + else: + self.assertEqual(stderr, '') def test_bind_release(self): with self.subTest('typical'): diff --git a/Modules/_xxinterpqueuesmodule.c b/Modules/_xxinterpqueuesmodule.c index 2b5516d47cb99e..3600b20c0861a0 100644 --- a/Modules/_xxinterpqueuesmodule.c +++ b/Modules/_xxinterpqueuesmodule.c @@ -458,6 +458,7 @@ _queueitem_popped(_queueitem *item, /* the queue */ + typedef struct _queue { Py_ssize_t num_waiters; // protected by global lock PyThread_type_lock mutex; @@ -500,6 +501,8 @@ _queue_clear(_queue *queue) *queue = (_queue){0}; } +static void _queue_free(_queue *); + static void _queue_kill_and_wait(_queue *queue) { @@ -732,6 +735,32 @@ _queuerefs_find(_queueref *first, int64_t qid, _queueref **pprev) return ref; } +static void +_queuerefs_clear(_queueref *head) +{ + _queueref *next = head; + while (next != NULL) { + _queueref *ref = next; + next = ref->next; + int64_t qid = ref->qid; + +#ifdef Py_DEBUG + fprintf(stderr, "queue %ld still exists\n", qid); +#endif + _queue *queue = ref->queue; + GLOBAL_FREE(ref); + + _queue_kill_and_wait(queue); +#ifdef Py_DEBUG + if (queue->items.count > 0) { + fprintf(stderr, "queue %ld still holds %ld items\n", + qid, queue->items.count); + } +#endif + _queue_free(queue); + } +} + /* a collection of queues ***************************************************/ @@ -754,8 +783,15 @@ _queues_init(_queues *queues, PyThread_type_lock mutex) static void _queues_fini(_queues *queues) { - assert(queues->count == 0); - assert(queues->head == NULL); + if (queues->count > 0) { + PyThread_acquire_lock(queues->mutex, WAIT_LOCK); + assert((queues->count == 0) != (queues->head != NULL)); + _queueref *head = queues->head; + queues->head = NULL; + queues->count = 0; + PyThread_release_lock(queues->mutex); + _queuerefs_clear(head); + } if (queues->mutex != NULL) { PyThread_free_lock(queues->mutex); queues->mutex = NULL; @@ -887,8 +923,6 @@ _queues_incref(_queues *queues, int64_t qid) return res; } -static void _queue_free(_queue *); - static int _queues_decref(_queues *queues, int64_t qid) { From 5645bcff9b444d38dc06d42b0672f966b3337414 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 4 Mar 2024 15:46:49 -0700 Subject: [PATCH 12/15] Do not crash for an unbalanced queue release. --- Lib/test/test_interpreters/test_queues.py | 15 ++++----------- Modules/_xxinterpqueuesmodule.c | 9 +++++++++ 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/Lib/test/test_interpreters/test_queues.py b/Lib/test/test_interpreters/test_queues.py index b2654be28fc6ec..b27cb75188e620 100644 --- a/Lib/test/test_interpreters/test_queues.py +++ b/Lib/test/test_interpreters/test_queues.py @@ -87,17 +87,10 @@ def test_bind_release(self): self.assertEqual(get_num_queues(), 0) with self.subTest('release without binding'): - stdout, stderr = self.assert_python_failure( - '-c', - dedent(f""" - import {_queues.__name__} as _queues - _queues.create(2, 0) - _queues.release(qid) - """), - ) - # It should have aborted due to an assert. - self.assertEqual(stdout, '') - self.assertNotEqual(stderr, '') + qid = _queues.create(2, 0) + self.addCleanup(lambda: _queues.destroy(qid)) + with self.assertRaises(queues.QueueError): + _queues.release(qid) class QueueTests(TestBase): diff --git a/Modules/_xxinterpqueuesmodule.c b/Modules/_xxinterpqueuesmodule.c index 3600b20c0861a0..d6daa075ac59a0 100644 --- a/Modules/_xxinterpqueuesmodule.c +++ b/Modules/_xxinterpqueuesmodule.c @@ -213,6 +213,7 @@ clear_module_state(module_state *state) // single-queue errors #define ERR_QUEUE_EMPTY (-21) #define ERR_QUEUE_FULL (-22) +#define ERR_QUEUE_NEVER_BOUND (-23) static int ensure_external_exc_types(module_state *); @@ -245,6 +246,10 @@ resolve_module_errcode(module_state *state, int errcode, int64_t qid, exctype = state->QueueFull; msg = PyUnicode_FromFormat("queue %" PRId64 " is full", qid); break; + case ERR_QUEUE_NEVER_BOUND: + exctype = state->QueueError; + msg = PyUnicode_FromFormat("queue %" PRId64 " never bound", qid); + break; default: PyErr_Format(PyExc_ValueError, "unsupported error code %d", errcode); @@ -936,6 +941,10 @@ _queues_decref(_queues *queues, int64_t qid) res = ERR_QUEUE_NOT_FOUND; goto finally; } + if (ref->refcount == 0) { + res = ERR_QUEUE_NEVER_BOUND; + goto finally; + } assert(ref->refcount > 0); ref->refcount -= 1; From c43da0cb8dc1c53a64c74a2acbcd1454fc82423e Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 4 Mar 2024 17:15:36 -0700 Subject: [PATCH 13/15] Fix a compiler warning. --- Modules/_xxinterpqueuesmodule.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Modules/_xxinterpqueuesmodule.c b/Modules/_xxinterpqueuesmodule.c index d6daa075ac59a0..f449339acbfb1d 100644 --- a/Modules/_xxinterpqueuesmodule.c +++ b/Modules/_xxinterpqueuesmodule.c @@ -747,10 +747,9 @@ _queuerefs_clear(_queueref *head) while (next != NULL) { _queueref *ref = next; next = ref->next; - int64_t qid = ref->qid; #ifdef Py_DEBUG - fprintf(stderr, "queue %ld still exists\n", qid); + fprintf(stderr, "queue %ld still exists\n", ref->qid); #endif _queue *queue = ref->queue; GLOBAL_FREE(ref); From 50a84f4f44ac1b79c231c88d0707f85a48944ea3 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 4 Mar 2024 17:15:53 -0700 Subject: [PATCH 14/15] Fix TestBase.tearDown(). --- Lib/test/test_interpreters/test_queues.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/test/test_interpreters/test_queues.py b/Lib/test/test_interpreters/test_queues.py index b27cb75188e620..d16d294b82d044 100644 --- a/Lib/test/test_interpreters/test_queues.py +++ b/Lib/test/test_interpreters/test_queues.py @@ -19,7 +19,7 @@ def get_num_queues(): class TestBase(_TestBase): def tearDown(self): - for qid in _queues.list_all(): + for qid, _ in _queues.list_all(): try: _queues.destroy(qid) except Exception: @@ -88,7 +88,6 @@ def test_bind_release(self): with self.subTest('release without binding'): qid = _queues.create(2, 0) - self.addCleanup(lambda: _queues.destroy(qid)) with self.assertRaises(queues.QueueError): _queues.release(qid) From 3bd92b71c02750ed167b11179bb976f9e0fceb3b Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 4 Mar 2024 17:28:02 -0700 Subject: [PATCH 15/15] Fix a compiler warning. --- Modules/_xxinterpqueuesmodule.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Modules/_xxinterpqueuesmodule.c b/Modules/_xxinterpqueuesmodule.c index f449339acbfb1d..cb8b9e4a661d5a 100644 --- a/Modules/_xxinterpqueuesmodule.c +++ b/Modules/_xxinterpqueuesmodule.c @@ -749,7 +749,8 @@ _queuerefs_clear(_queueref *head) next = ref->next; #ifdef Py_DEBUG - fprintf(stderr, "queue %ld still exists\n", ref->qid); + int64_t qid = ref->qid; + fprintf(stderr, "queue %ld still exists\n", qid); #endif _queue *queue = ref->queue; GLOBAL_FREE(ref); pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy