From 223d650804eae20c9a3969b24a67983c8963839d Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Sat, 23 Nov 2024 21:04:14 +0100 Subject: [PATCH 01/17] Make methodcalled thread-safe --- Modules/_operator.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/Modules/_operator.c b/Modules/_operator.c index 6c1945174ab7cd..98f9664b1e48ac 100644 --- a/Modules/_operator.c +++ b/Modules/_operator.c @@ -1648,7 +1648,9 @@ static PyObject * methodcaller_vectorcall( methodcallerobject *mc, PyObject *const *args, size_t nargsf, PyObject* kwnames) { - if (!_PyArg_CheckPositional("methodcaller", PyVectorcall_NARGS(nargsf), 1, 1) + Py_ssize_t number_of_arguments = PyVectorcall_NARGS(nargsf); + + if (!_PyArg_CheckPositional("methodcaller", number_of_arguments, 1, 1) || !_PyArg_NoKwnames("methodcaller", kwnames)) { return NULL; } @@ -1659,11 +1661,16 @@ methodcaller_vectorcall( } assert(mc->vectorcall_args != 0); - mc->vectorcall_args[0] = args[0]; + number_of_arguments++; + PyObject **tmp_args = (PyObject **) PyMem_Malloc(number_of_arguments * sizeof(PyObject *)); + memcpy(tmp_args, mc->vectorcall_args, sizeof(PyObject *) * number_of_arguments ); + tmp_args[0] = args[0]; return PyObject_VectorcallMethod( - mc->name, mc->vectorcall_args, + mc->name, tmp_args, (PyTuple_GET_SIZE(mc->xargs)) | PY_VECTORCALL_ARGUMENTS_OFFSET, mc->vectorcall_kwnames); + + PyMem_Free(tmp_args); } #endif From b6d454a730b12bb55cfd562dedff4ca06069cba0 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Sat, 23 Nov 2024 21:17:59 +0100 Subject: [PATCH 02/17] check result of PyMem_Malloc --- Modules/_operator.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Modules/_operator.c b/Modules/_operator.c index 98f9664b1e48ac..8b227d39dd79bf 100644 --- a/Modules/_operator.c +++ b/Modules/_operator.c @@ -1661,9 +1661,13 @@ methodcaller_vectorcall( } assert(mc->vectorcall_args != 0); - number_of_arguments++; - PyObject **tmp_args = (PyObject **) PyMem_Malloc(number_of_arguments * sizeof(PyObject *)); - memcpy(tmp_args, mc->vectorcall_args, sizeof(PyObject *) * number_of_arguments ); + size_t buffer_size = sizeof(PyObject *) * (number_of_arguments + 1); + PyObject **tmp_args = (PyObject **) PyMem_Malloc buffer_size); + if (tmp_args == NULL) { + PyErr_NoMemory(); + return -1; + } + memcpy(tmp_args, mc->vectorcall_args, buffer_size); tmp_args[0] = args[0]; return PyObject_VectorcallMethod( mc->name, tmp_args, From 4ce123381731cd7178fdd8eabd3e11440a24423e Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Sat, 23 Nov 2024 21:23:54 +0100 Subject: [PATCH 03/17] enable ft --- Modules/_operator.c | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/Modules/_operator.c b/Modules/_operator.c index 8b227d39dd79bf..9c1ef897ba0536 100644 --- a/Modules/_operator.c +++ b/Modules/_operator.c @@ -1602,7 +1602,7 @@ typedef struct { vectorcallfunc vectorcall; } methodcallerobject; -#ifndef Py_GIL_DISABLED + static int _methodcaller_initialize_vectorcall(methodcallerobject* mc) { PyObject* args = mc->xargs; @@ -1662,10 +1662,10 @@ methodcaller_vectorcall( assert(mc->vectorcall_args != 0); size_t buffer_size = sizeof(PyObject *) * (number_of_arguments + 1); - PyObject **tmp_args = (PyObject **) PyMem_Malloc buffer_size); + PyObject **tmp_args = (PyObject **) PyMem_Malloc(buffer_size); if (tmp_args == NULL) { PyErr_NoMemory(); - return -1; + return NULL; } memcpy(tmp_args, mc->vectorcall_args, buffer_size); tmp_args[0] = args[0]; @@ -1676,7 +1676,6 @@ methodcaller_vectorcall( PyMem_Free(tmp_args); } -#endif /* AC 3.5: variable number of arguments, not currently support by AC */ @@ -1715,15 +1714,7 @@ methodcaller_new(PyTypeObject *type, PyObject *args, PyObject *kwds) mc->kwds = Py_XNewRef(kwds); mc->vectorcall_args = 0; - -#ifdef Py_GIL_DISABLED - // gh-127065: The current implementation of methodcaller_vectorcall - // is not thread-safe because it modifies the `vectorcall_args` array, - // which is shared across calls. - mc->vectorcall = NULL; -#else mc->vectorcall = (vectorcallfunc)methodcaller_vectorcall; -#endif PyObject_GC_Track(mc); return (PyObject *)mc; From cf6b79bccecdf6c9bedbf1aa066b790f45ec95e9 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Mon, 25 Nov 2024 11:35:15 +0100 Subject: [PATCH 04/17] fix memory error --- Modules/_operator.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Modules/_operator.c b/Modules/_operator.c index 9c1ef897ba0536..0e71dac7fa178b 100644 --- a/Modules/_operator.c +++ b/Modules/_operator.c @@ -1648,9 +1648,7 @@ static PyObject * methodcaller_vectorcall( methodcallerobject *mc, PyObject *const *args, size_t nargsf, PyObject* kwnames) { - Py_ssize_t number_of_arguments = PyVectorcall_NARGS(nargsf); - - if (!_PyArg_CheckPositional("methodcaller", number_of_arguments, 1, 1) + if (!_PyArg_CheckPositional("methodcaller", PyVectorcall_NARGS(nargsf), 1, 1) || !_PyArg_NoKwnames("methodcaller", kwnames)) { return NULL; } @@ -1661,7 +1659,12 @@ methodcaller_vectorcall( } assert(mc->vectorcall_args != 0); + + Py_ssize_t number_of_arguments = PyTuple_GET_SIZE(mc->xargs) + + (mc->vectorcall_kwnames? PyTuple_GET_SIZE(mc->vectorcall_kwnames):0) + 1; size_t buffer_size = sizeof(PyObject *) * (number_of_arguments + 1); + + // for number_of_arguments == 1 we could optimize by setting tmp_args equal to &args PyObject **tmp_args = (PyObject **) PyMem_Malloc(buffer_size); if (tmp_args == NULL) { PyErr_NoMemory(); From 2476ce4a0a2fc32f8b9f96f0d0f18fc106cdf0f9 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Thu, 28 Nov 2024 22:38:05 +0100 Subject: [PATCH 05/17] wip --- Modules/_operator.c | 147 ++++++++++++++++++++++---------------------- 1 file changed, 75 insertions(+), 72 deletions(-) diff --git a/Modules/_operator.c b/Modules/_operator.c index 0e71dac7fa178b..830e4dd0d14eda 100644 --- a/Modules/_operator.c +++ b/Modules/_operator.c @@ -1595,7 +1595,7 @@ static PyType_Spec attrgetter_type_spec = { typedef struct { PyObject_HEAD PyObject *name; - PyObject *xargs; // reference to arguments passed in constructor + PyObject *args; PyObject *kwds; PyObject **vectorcall_args; /* Borrowed references */ PyObject *vectorcall_kwnames; @@ -1603,13 +1603,43 @@ typedef struct { } methodcallerobject; +#define _METHODCALLER_MAX_ARGS 8 + +static PyObject * +methodcaller_vectorcall( + methodcallerobject *mc, PyObject *const *args, size_t nargsf, PyObject* kwnames) +{ + printf("methodcaller_vectorcall\n"); + if (!_PyArg_CheckPositional("methodcaller", PyVectorcall_NARGS(nargsf), 1, 1) + || !_PyArg_NoKwnames("methodcaller", kwnames)) { + return NULL; + } + assert(mc->vectorcall_args != NULL); + + Py_ssize_t number_of_arguments = 1 + PyTuple_GET_SIZE(mc->args) + + (mc->vectorcall_kwnames? PyTuple_GET_SIZE(mc->vectorcall_kwnames):0) + 1; + + PyObject *tmp_args[_METHODCALLER_MAX_ARGS]; + tmp_args[0] = args[0]; + if (number_of_arguments) { + assert(1 + number_of_arguments <= _METHODCALLER_MAX_ARGS); + memcpy(tmp_args + 1, mc->vectorcall_args, sizeof(PyObject *) * number_of_arguments); + } + PyObject *result = PyObject_VectorcallMethod( + mc->name, tmp_args, + (1 + PyTuple_GET_SIZE(mc->args)) | PY_VECTORCALL_ARGUMENTS_OFFSET, + mc->vectorcall_kwnames); + + PyMem_Free(tmp_args); + return result; +} + static int _methodcaller_initialize_vectorcall(methodcallerobject* mc) { - PyObject* args = mc->xargs; + PyObject* args = mc->args; PyObject* kwds = mc->kwds; - Py_ssize_t nargs = PyTuple_GET_SIZE(args); - assert(nargs > 0); + Py_ssize_t nargs = 1 + PyTuple_GET_SIZE(args); mc->vectorcall_args = PyMem_Calloc( nargs + (kwds ? PyDict_Size(kwds) : 0), sizeof(PyObject*)); @@ -1619,8 +1649,8 @@ static int _methodcaller_initialize_vectorcall(methodcallerobject* mc) } /* The first item of vectorcall_args will be filled with obj later */ if (nargs > 1) { - memcpy(mc->vectorcall_args, PySequence_Fast_ITEMS(args), - nargs * sizeof(PyObject*)); + memcpy(mc->vectorcall_args + 1, PySequence_Fast_ITEMS(args), + (nargs - 1) * sizeof(PyObject*)); } if (kwds) { const Py_ssize_t nkwds = PyDict_Size(kwds); @@ -1640,47 +1670,11 @@ static int _methodcaller_initialize_vectorcall(methodcallerobject* mc) else { mc->vectorcall_kwnames = NULL; } - return 1; -} + //mc->vectorcall = (vectorcallfunc)methodcaller_vectorcall; - -static PyObject * -methodcaller_vectorcall( - methodcallerobject *mc, PyObject *const *args, size_t nargsf, PyObject* kwnames) -{ - if (!_PyArg_CheckPositional("methodcaller", PyVectorcall_NARGS(nargsf), 1, 1) - || !_PyArg_NoKwnames("methodcaller", kwnames)) { - return NULL; - } - if (mc->vectorcall_args == NULL) { - if (_methodcaller_initialize_vectorcall(mc) < 0) { - return NULL; - } - } - - assert(mc->vectorcall_args != 0); - - Py_ssize_t number_of_arguments = PyTuple_GET_SIZE(mc->xargs) + - (mc->vectorcall_kwnames? PyTuple_GET_SIZE(mc->vectorcall_kwnames):0) + 1; - size_t buffer_size = sizeof(PyObject *) * (number_of_arguments + 1); - - // for number_of_arguments == 1 we could optimize by setting tmp_args equal to &args - PyObject **tmp_args = (PyObject **) PyMem_Malloc(buffer_size); - if (tmp_args == NULL) { - PyErr_NoMemory(); - return NULL; - } - memcpy(tmp_args, mc->vectorcall_args, buffer_size); - tmp_args[0] = args[0]; - return PyObject_VectorcallMethod( - mc->name, tmp_args, - (PyTuple_GET_SIZE(mc->xargs)) | PY_VECTORCALL_ARGUMENTS_OFFSET, - mc->vectorcall_kwnames); - - PyMem_Free(tmp_args); + return 1; } - /* AC 3.5: variable number of arguments, not currently support by AC */ static PyObject * methodcaller_new(PyTypeObject *type, PyObject *args, PyObject *kwds) @@ -1713,11 +1707,27 @@ methodcaller_new(PyTypeObject *type, PyObject *args, PyObject *kwds) _PyUnicode_InternMortal(interp, &name); mc->name = name; - mc->xargs = Py_XNewRef(args); // allows us to use borrowed references + mc->args = PyTuple_GetSlice(args, 1, PyTuple_GET_SIZE(args)); + if (mc->args == NULL) { + Py_DECREF(mc); + return NULL; + } mc->kwds = Py_XNewRef(kwds); - mc->vectorcall_args = 0; + mc->vectorcall_args = NULL; + + Py_ssize_t vectorcall_size = PyTuple_GET_SIZE(args) + + (kwds ? PyDict_Size(kwds) : 0); + printf("methodcaller_new %d %d\n", PyTuple_GET_SIZE(args), vectorcall_size); + if (vectorcall_size < (_METHODCALLER_MAX_ARGS-10)) { + printf("hih\n"); + if (_methodcaller_initialize_vectorcall(mc) < 0) { + Py_XDECREF(mc->args); + Py_XDECREF(mc->kwds); + return NULL; + } + } - mc->vectorcall = (vectorcallfunc)methodcaller_vectorcall; + //mc->vectorcall = (vectorcallfunc)methodcaller_vectorcall; PyObject_GC_Track(mc); return (PyObject *)mc; @@ -1726,12 +1736,13 @@ methodcaller_new(PyTypeObject *type, PyObject *args, PyObject *kwds) static void methodcaller_clear(methodcallerobject *mc) { + printf("methodcaller_clear\n"); + Py_CLEAR(mc->name); - Py_CLEAR(mc->xargs); + Py_CLEAR(mc->args); Py_CLEAR(mc->kwds); if (mc->vectorcall_args != NULL) { PyMem_Free(mc->vectorcall_args); - mc->vectorcall_args = 0; Py_CLEAR(mc->vectorcall_kwnames); } } @@ -1739,6 +1750,8 @@ methodcaller_clear(methodcallerobject *mc) static void methodcaller_dealloc(methodcallerobject *mc) { + printf("methodcaller_dealloc\n"); + PyTypeObject *tp = Py_TYPE(mc); PyObject_GC_UnTrack(mc); methodcaller_clear(mc); @@ -1749,8 +1762,9 @@ methodcaller_dealloc(methodcallerobject *mc) static int methodcaller_traverse(methodcallerobject *mc, visitproc visit, void *arg) { + printf("methodcaller_traverse\n"); Py_VISIT(mc->name); - Py_VISIT(mc->xargs); + Py_VISIT(mc->args); Py_VISIT(mc->kwds); Py_VISIT(Py_TYPE(mc)); return 0; @@ -1759,6 +1773,8 @@ methodcaller_traverse(methodcallerobject *mc, visitproc visit, void *arg) static PyObject * methodcaller_call(methodcallerobject *mc, PyObject *args, PyObject *kw) { + printf("methodcaller_call\n"); + return Py_None; PyObject *method, *obj, *result; if (!_PyArg_NoKeywords("methodcaller", kw)) @@ -1770,15 +1786,7 @@ methodcaller_call(methodcallerobject *mc, PyObject *args, PyObject *kw) if (method == NULL) return NULL; - - PyObject *cargs = PyTuple_GetSlice(mc->xargs, 1, PyTuple_GET_SIZE(mc->xargs)); - if (cargs == NULL) { - Py_DECREF(method); - return NULL; - } - - result = PyObject_Call(method, cargs, mc->kwds); - Py_DECREF(cargs); + result = PyObject_Call(method, mc->args, mc->kwds); Py_DECREF(method); return result; } @@ -1796,7 +1804,7 @@ methodcaller_repr(methodcallerobject *mc) } numkwdargs = mc->kwds != NULL ? PyDict_GET_SIZE(mc->kwds) : 0; - numposargs = PyTuple_GET_SIZE(mc->xargs) - 1; + numposargs = PyTuple_GET_SIZE(mc->args); numtotalargs = numposargs + numkwdargs; if (numtotalargs == 0) { @@ -1812,7 +1820,7 @@ methodcaller_repr(methodcallerobject *mc) } for (i = 0; i < numposargs; ++i) { - PyObject *onerepr = PyObject_Repr(PyTuple_GET_ITEM(mc->xargs, i+1)); + PyObject *onerepr = PyObject_Repr(PyTuple_GET_ITEM(mc->args, i)); if (onerepr == NULL) goto done; PyTuple_SET_ITEM(argreprs, i, onerepr); @@ -1864,14 +1872,14 @@ methodcaller_reduce(methodcallerobject *mc, PyObject *Py_UNUSED(ignored)) { if (!mc->kwds || PyDict_GET_SIZE(mc->kwds) == 0) { Py_ssize_t i; - Py_ssize_t newarg_size = PyTuple_GET_SIZE(mc->xargs); - PyObject *newargs = PyTuple_New(newarg_size); + Py_ssize_t callargcount = PyTuple_GET_SIZE(mc->args); + PyObject *newargs = PyTuple_New(1 + callargcount); if (newargs == NULL) return NULL; PyTuple_SET_ITEM(newargs, 0, Py_NewRef(mc->name)); - for (i = 1; i < newarg_size; ++i) { - PyObject *arg = PyTuple_GET_ITEM(mc->xargs, i); - PyTuple_SET_ITEM(newargs, i, Py_NewRef(arg)); + for (i = 0; i < callargcount; ++i) { + PyObject *arg = PyTuple_GET_ITEM(mc->args, i); + PyTuple_SET_ITEM(newargs, i + 1, Py_NewRef(arg)); } return Py_BuildValue("ON", Py_TYPE(mc), newargs); } @@ -1889,12 +1897,7 @@ methodcaller_reduce(methodcallerobject *mc, PyObject *Py_UNUSED(ignored)) constructor = PyObject_VectorcallDict(partial, newargs, 2, mc->kwds); Py_DECREF(partial); - PyObject *args = PyTuple_GetSlice(mc->xargs, 1, PyTuple_GET_SIZE(mc->xargs)); - if (!args) { - Py_DECREF(constructor); - return NULL; - } - return Py_BuildValue("NO", constructor, args); + return Py_BuildValue("NO", constructor, mc->args); } } From 5ecf87681f19194a96a0e2c97251087fe95e8e86 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Thu, 28 Nov 2024 23:43:33 +0100 Subject: [PATCH 06/17] add tests --- Lib/test/test_operator.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/Lib/test/test_operator.py b/Lib/test/test_operator.py index 812d46482e238a..82578a0ef1e6f2 100644 --- a/Lib/test/test_operator.py +++ b/Lib/test/test_operator.py @@ -482,6 +482,8 @@ def bar(self, f=42): return f def baz(*args, **kwds): return kwds['name'], kwds['self'] + def return_arguments(self, *args, **kwds): + return args, kwds a = A() f = operator.methodcaller('foo') self.assertRaises(IndexError, f, a) @@ -498,6 +500,17 @@ def baz(*args, **kwds): f = operator.methodcaller('baz', name='spam', self='eggs') self.assertEqual(f(a), ('spam', 'eggs')) + many_positional_arguments = tuple(range(10)) + many_kw_arguments = dict(zip('abcdefghij', range(10))) + f = operator.methodcaller('return_arguments', *many_positional_arguments) + self.assertEqual(f(a), (many_positional_arguments, {})) + + f = operator.methodcaller('return_arguments', **many_kw_arguments) + self.assertEqual(f(a), ((), many_kw_arguments)) + + f = operator.methodcaller('return_arguments', *many_positional_arguments, **many_kw_arguments) + self.assertEqual(f(a), (many_positional_arguments, many_kw_arguments)) + def test_inplace(self): operator = self.module class C(object): From 709010d3d46174d2970a5596b74cf7667d22e2e8 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Thu, 28 Nov 2024 23:44:21 +0100 Subject: [PATCH 07/17] wip --- Modules/_operator.c | 37 +++++++++++++++++-------------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/Modules/_operator.c b/Modules/_operator.c index 830e4dd0d14eda..d4785fa4ae5db0 100644 --- a/Modules/_operator.c +++ b/Modules/_operator.c @@ -1609,18 +1609,21 @@ static PyObject * methodcaller_vectorcall( methodcallerobject *mc, PyObject *const *args, size_t nargsf, PyObject* kwnames) { - printf("methodcaller_vectorcall\n"); + //printf("methodcaller_vectorcall\n"); if (!_PyArg_CheckPositional("methodcaller", PyVectorcall_NARGS(nargsf), 1, 1) || !_PyArg_NoKwnames("methodcaller", kwnames)) { return NULL; } assert(mc->vectorcall_args != NULL); - Py_ssize_t number_of_arguments = 1 + PyTuple_GET_SIZE(mc->args) + - (mc->vectorcall_kwnames? PyTuple_GET_SIZE(mc->vectorcall_kwnames):0) + 1; + Py_ssize_t number_of_arguments = PyTuple_GET_SIZE(mc->args) + + (mc->vectorcall_kwnames? PyTuple_GET_SIZE(mc->vectorcall_kwnames):0); + + //printf("methodcaller_vectorcall: number_of_arguments %ld\n", number_of_arguments); PyObject *tmp_args[_METHODCALLER_MAX_ARGS]; tmp_args[0] = args[0]; + tmp_args[1] = Py_None; if (number_of_arguments) { assert(1 + number_of_arguments <= _METHODCALLER_MAX_ARGS); memcpy(tmp_args + 1, mc->vectorcall_args, sizeof(PyObject *) * number_of_arguments); @@ -1630,7 +1633,7 @@ methodcaller_vectorcall( (1 + PyTuple_GET_SIZE(mc->args)) | PY_VECTORCALL_ARGUMENTS_OFFSET, mc->vectorcall_kwnames); - PyMem_Free(tmp_args); + //printf("methodcaller_vectorcall done\n"); return result; } @@ -1639,7 +1642,9 @@ static int _methodcaller_initialize_vectorcall(methodcallerobject* mc) PyObject* args = mc->args; PyObject* kwds = mc->kwds; - Py_ssize_t nargs = 1 + PyTuple_GET_SIZE(args); + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + // printf("_methodcaller_initialize_vectorcall: nargs %ld, vectorcall_args size \n", nargs, nargs + (kwds ? PyDict_Size(kwds) : 0)); + mc->vectorcall_args = PyMem_Calloc( nargs + (kwds ? PyDict_Size(kwds) : 0), sizeof(PyObject*)); @@ -1648,9 +1653,9 @@ static int _methodcaller_initialize_vectorcall(methodcallerobject* mc) return -1; } /* The first item of vectorcall_args will be filled with obj later */ - if (nargs > 1) { - memcpy(mc->vectorcall_args + 1, PySequence_Fast_ITEMS(args), - (nargs - 1) * sizeof(PyObject*)); + if (nargs > 0) { + memcpy(mc->vectorcall_args, PySequence_Fast_ITEMS(args), + nargs * sizeof(PyObject*)); } if (kwds) { const Py_ssize_t nkwds = PyDict_Size(kwds); @@ -1670,7 +1675,7 @@ static int _methodcaller_initialize_vectorcall(methodcallerobject* mc) else { mc->vectorcall_kwnames = NULL; } - //mc->vectorcall = (vectorcallfunc)methodcaller_vectorcall; + mc->vectorcall = (vectorcallfunc)methodcaller_vectorcall; return 1; } @@ -1707,19 +1712,19 @@ methodcaller_new(PyTypeObject *type, PyObject *args, PyObject *kwds) _PyUnicode_InternMortal(interp, &name); mc->name = name; + //printf("methodcaller_new: %ld \n", PyTuple_GET_SIZE(args)); mc->args = PyTuple_GetSlice(args, 1, PyTuple_GET_SIZE(args)); if (mc->args == NULL) { Py_DECREF(mc); return NULL; } mc->kwds = Py_XNewRef(kwds); + mc->vectorcall = NULL; mc->vectorcall_args = NULL; Py_ssize_t vectorcall_size = PyTuple_GET_SIZE(args) + (kwds ? PyDict_Size(kwds) : 0); - printf("methodcaller_new %d %d\n", PyTuple_GET_SIZE(args), vectorcall_size); - if (vectorcall_size < (_METHODCALLER_MAX_ARGS-10)) { - printf("hih\n"); + if (vectorcall_size < (_METHODCALLER_MAX_ARGS)) { if (_methodcaller_initialize_vectorcall(mc) < 0) { Py_XDECREF(mc->args); Py_XDECREF(mc->kwds); @@ -1727,8 +1732,6 @@ methodcaller_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } } - //mc->vectorcall = (vectorcallfunc)methodcaller_vectorcall; - PyObject_GC_Track(mc); return (PyObject *)mc; } @@ -1736,8 +1739,6 @@ methodcaller_new(PyTypeObject *type, PyObject *args, PyObject *kwds) static void methodcaller_clear(methodcallerobject *mc) { - printf("methodcaller_clear\n"); - Py_CLEAR(mc->name); Py_CLEAR(mc->args); Py_CLEAR(mc->kwds); @@ -1750,8 +1751,6 @@ methodcaller_clear(methodcallerobject *mc) static void methodcaller_dealloc(methodcallerobject *mc) { - printf("methodcaller_dealloc\n"); - PyTypeObject *tp = Py_TYPE(mc); PyObject_GC_UnTrack(mc); methodcaller_clear(mc); @@ -1762,7 +1761,6 @@ methodcaller_dealloc(methodcallerobject *mc) static int methodcaller_traverse(methodcallerobject *mc, visitproc visit, void *arg) { - printf("methodcaller_traverse\n"); Py_VISIT(mc->name); Py_VISIT(mc->args); Py_VISIT(mc->kwds); @@ -1774,7 +1772,6 @@ static PyObject * methodcaller_call(methodcallerobject *mc, PyObject *args, PyObject *kw) { printf("methodcaller_call\n"); - return Py_None; PyObject *method, *obj, *result; if (!_PyArg_NoKeywords("methodcaller", kw)) From 6bd2c2e70fbca04ff4da856532e24a5ca973a6bb Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Thu, 28 Nov 2024 23:49:12 +0100 Subject: [PATCH 08/17] wip --- Modules/_operator.c | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/Modules/_operator.c b/Modules/_operator.c index d4785fa4ae5db0..fc710861ff8312 100644 --- a/Modules/_operator.c +++ b/Modules/_operator.c @@ -1609,7 +1609,6 @@ static PyObject * methodcaller_vectorcall( methodcallerobject *mc, PyObject *const *args, size_t nargsf, PyObject* kwnames) { - //printf("methodcaller_vectorcall\n"); if (!_PyArg_CheckPositional("methodcaller", PyVectorcall_NARGS(nargsf), 1, 1) || !_PyArg_NoKwnames("methodcaller", kwnames)) { return NULL; @@ -1619,21 +1618,16 @@ methodcaller_vectorcall( Py_ssize_t number_of_arguments = PyTuple_GET_SIZE(mc->args) + (mc->vectorcall_kwnames? PyTuple_GET_SIZE(mc->vectorcall_kwnames):0); - //printf("methodcaller_vectorcall: number_of_arguments %ld\n", number_of_arguments); - PyObject *tmp_args[_METHODCALLER_MAX_ARGS]; tmp_args[0] = args[0]; - tmp_args[1] = Py_None; - if (number_of_arguments) { - assert(1 + number_of_arguments <= _METHODCALLER_MAX_ARGS); - memcpy(tmp_args + 1, mc->vectorcall_args, sizeof(PyObject *) * number_of_arguments); - } + assert(1 + number_of_arguments <= _METHODCALLER_MAX_ARGS); + memcpy(tmp_args + 1, mc->vectorcall_args, sizeof(PyObject *) * number_of_arguments); + PyObject *result = PyObject_VectorcallMethod( mc->name, tmp_args, (1 + PyTuple_GET_SIZE(mc->args)) | PY_VECTORCALL_ARGUMENTS_OFFSET, mc->vectorcall_kwnames); - //printf("methodcaller_vectorcall done\n"); return result; } @@ -1643,8 +1637,6 @@ static int _methodcaller_initialize_vectorcall(methodcallerobject* mc) PyObject* kwds = mc->kwds; Py_ssize_t nargs = PyTuple_GET_SIZE(args); - // printf("_methodcaller_initialize_vectorcall: nargs %ld, vectorcall_args size \n", nargs, nargs + (kwds ? PyDict_Size(kwds) : 0)); - mc->vectorcall_args = PyMem_Calloc( nargs + (kwds ? PyDict_Size(kwds) : 0), sizeof(PyObject*)); @@ -1712,7 +1704,6 @@ methodcaller_new(PyTypeObject *type, PyObject *args, PyObject *kwds) _PyUnicode_InternMortal(interp, &name); mc->name = name; - //printf("methodcaller_new: %ld \n", PyTuple_GET_SIZE(args)); mc->args = PyTuple_GetSlice(args, 1, PyTuple_GET_SIZE(args)); if (mc->args == NULL) { Py_DECREF(mc); From c9e3898e1b7e233f2955a9119c3d068429ab6422 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Thu, 28 Nov 2024 23:51:27 +0100 Subject: [PATCH 09/17] wip --- Modules/_operator.c | 1 - 1 file changed, 1 deletion(-) diff --git a/Modules/_operator.c b/Modules/_operator.c index fc710861ff8312..f3bdff3b45aae6 100644 --- a/Modules/_operator.c +++ b/Modules/_operator.c @@ -1762,7 +1762,6 @@ methodcaller_traverse(methodcallerobject *mc, visitproc visit, void *arg) static PyObject * methodcaller_call(methodcallerobject *mc, PyObject *args, PyObject *kw) { - printf("methodcaller_call\n"); PyObject *method, *obj, *result; if (!_PyArg_NoKeywords("methodcaller", kw)) From 8ef7a04d7f94d2a0cd63f3ae9176ca53c7a2f65b Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Fri, 29 Nov 2024 13:42:32 +0100 Subject: [PATCH 10/17] fix memory error --- Modules/_operator.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Modules/_operator.c b/Modules/_operator.c index f3bdff3b45aae6..0d5d0a99892100 100644 --- a/Modules/_operator.c +++ b/Modules/_operator.c @@ -1649,9 +1649,8 @@ static int _methodcaller_initialize_vectorcall(methodcallerobject* mc) memcpy(mc->vectorcall_args, PySequence_Fast_ITEMS(args), nargs * sizeof(PyObject*)); } - if (kwds) { + if (kwds && PyDict_Size(kwds)) { const Py_ssize_t nkwds = PyDict_Size(kwds); - mc->vectorcall_kwnames = PyTuple_New(nkwds); if (!mc->vectorcall_kwnames) { return -1; @@ -1712,6 +1711,7 @@ methodcaller_new(PyTypeObject *type, PyObject *args, PyObject *kwds) mc->kwds = Py_XNewRef(kwds); mc->vectorcall = NULL; mc->vectorcall_args = NULL; + mc->vectorcall_kwnames = NULL; Py_ssize_t vectorcall_size = PyTuple_GET_SIZE(args) + (kwds ? PyDict_Size(kwds) : 0); @@ -1735,6 +1735,7 @@ methodcaller_clear(methodcallerobject *mc) Py_CLEAR(mc->kwds); if (mc->vectorcall_args != NULL) { PyMem_Free(mc->vectorcall_args); + mc->vectorcall_args = NULL; Py_CLEAR(mc->vectorcall_kwnames); } } From b4f30d39728614d29215ffe3d5bbd4dae5e25a15 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Sun, 1 Dec 2024 23:26:51 +0100 Subject: [PATCH 11/17] skip check on zero size for memcpy --- Modules/_operator.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Modules/_operator.c b/Modules/_operator.c index 0d5d0a99892100..387b4696842d75 100644 --- a/Modules/_operator.c +++ b/Modules/_operator.c @@ -1644,11 +1644,8 @@ static int _methodcaller_initialize_vectorcall(methodcallerobject* mc) PyErr_NoMemory(); return -1; } - /* The first item of vectorcall_args will be filled with obj later */ - if (nargs > 0) { - memcpy(mc->vectorcall_args, PySequence_Fast_ITEMS(args), + memcpy(mc->vectorcall_args, PySequence_Fast_ITEMS(args), nargs * sizeof(PyObject*)); - } if (kwds && PyDict_Size(kwds)) { const Py_ssize_t nkwds = PyDict_Size(kwds); mc->vectorcall_kwnames = PyTuple_New(nkwds); From 6d062014c89bb4e4bec94404ea71dc3e976e3e07 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Sun, 1 Dec 2024 22:28:44 +0000 Subject: [PATCH 12/17] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Library/2024-12-01-22-28-41.gh-issue-127065.tFpRer.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2024-12-01-22-28-41.gh-issue-127065.tFpRer.rst diff --git a/Misc/NEWS.d/next/Library/2024-12-01-22-28-41.gh-issue-127065.tFpRer.rst b/Misc/NEWS.d/next/Library/2024-12-01-22-28-41.gh-issue-127065.tFpRer.rst new file mode 100644 index 00000000000000..03d6953b9ddfa1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-01-22-28-41.gh-issue-127065.tFpRer.rst @@ -0,0 +1 @@ +Make :func:`operator.methodcaller` thread-safe and re-entrant safe. From ad66951c9ddc92c34bf46f3c18ab901f59420edf Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Tue, 3 Dec 2024 20:45:57 +0100 Subject: [PATCH 13/17] review comments --- Modules/_operator.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/Modules/_operator.c b/Modules/_operator.c index 387b4696842d75..617773033e9303 100644 --- a/Modules/_operator.c +++ b/Modules/_operator.c @@ -1644,7 +1644,7 @@ static int _methodcaller_initialize_vectorcall(methodcallerobject* mc) PyErr_NoMemory(); return -1; } - memcpy(mc->vectorcall_args, PySequence_Fast_ITEMS(args), + memcpy(mc->vectorcall_args, _PyTuple_ITEMS(args), nargs * sizeof(PyObject*)); if (kwds && PyDict_Size(kwds)) { const Py_ssize_t nkwds = PyDict_Size(kwds); @@ -1694,6 +1694,10 @@ methodcaller_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (mc == NULL) { return NULL; } + mc->vectorcall = NULL; + mc->vectorcall_args = NULL; + mc->vectorcall_kwnames = NULL; + mc->kwds = Py_XNewRef(kwds); Py_INCREF(name); PyInterpreterState *interp = _PyInterpreterState_GET(); @@ -1705,17 +1709,12 @@ methodcaller_new(PyTypeObject *type, PyObject *args, PyObject *kwds) Py_DECREF(mc); return NULL; } - mc->kwds = Py_XNewRef(kwds); - mc->vectorcall = NULL; - mc->vectorcall_args = NULL; - mc->vectorcall_kwnames = NULL; Py_ssize_t vectorcall_size = PyTuple_GET_SIZE(args) + (kwds ? PyDict_Size(kwds) : 0); if (vectorcall_size < (_METHODCALLER_MAX_ARGS)) { if (_methodcaller_initialize_vectorcall(mc) < 0) { - Py_XDECREF(mc->args); - Py_XDECREF(mc->kwds); + Py_DECREF(mc); return NULL; } } From bc3fe2a1688291995978eb71b82f68797461f0fb Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Tue, 3 Dec 2024 20:52:42 +0100 Subject: [PATCH 14/17] review comments --- Modules/_operator.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Modules/_operator.c b/Modules/_operator.c index 617773033e9303..b142b96ab882f7 100644 --- a/Modules/_operator.c +++ b/Modules/_operator.c @@ -1644,8 +1644,10 @@ static int _methodcaller_initialize_vectorcall(methodcallerobject* mc) PyErr_NoMemory(); return -1; } + // the objects in mc->vectorcall_args have references borrowed + // from mc->args and the keys from mc->kwds memcpy(mc->vectorcall_args, _PyTuple_ITEMS(args), - nargs * sizeof(PyObject*)); + nargs * sizeof(PyObject*)); // borrowed references if (kwds && PyDict_Size(kwds)) { const Py_ssize_t nkwds = PyDict_Size(kwds); mc->vectorcall_kwnames = PyTuple_New(nkwds); From e9a1fa696337ffda17072928e5aa4253d1f307b8 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Fri, 6 Dec 2024 21:53:40 +0100 Subject: [PATCH 15/17] Update Modules/_operator.c Co-authored-by: Sam Gross --- Modules/_operator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_operator.c b/Modules/_operator.c index b142b96ab882f7..932daf3282455b 100644 --- a/Modules/_operator.c +++ b/Modules/_operator.c @@ -1616,7 +1616,7 @@ methodcaller_vectorcall( assert(mc->vectorcall_args != NULL); Py_ssize_t number_of_arguments = PyTuple_GET_SIZE(mc->args) + - (mc->vectorcall_kwnames? PyTuple_GET_SIZE(mc->vectorcall_kwnames):0); + (mc->vectorcall_kwnames ? PyTuple_GET_SIZE(mc->vectorcall_kwnames) : 0); PyObject *tmp_args[_METHODCALLER_MAX_ARGS]; tmp_args[0] = args[0]; From 00ab65414a908846fa6cd71f1c95071d26a38f4c Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Fri, 6 Dec 2024 21:54:04 +0100 Subject: [PATCH 16/17] Update Modules/_operator.c Co-authored-by: Sam Gross --- Modules/_operator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_operator.c b/Modules/_operator.c index 932daf3282455b..48cec29a393453 100644 --- a/Modules/_operator.c +++ b/Modules/_operator.c @@ -1647,7 +1647,7 @@ static int _methodcaller_initialize_vectorcall(methodcallerobject* mc) // the objects in mc->vectorcall_args have references borrowed // from mc->args and the keys from mc->kwds memcpy(mc->vectorcall_args, _PyTuple_ITEMS(args), - nargs * sizeof(PyObject*)); // borrowed references + nargs * sizeof(PyObject *)); // borrowed references if (kwds && PyDict_Size(kwds)) { const Py_ssize_t nkwds = PyDict_Size(kwds); mc->vectorcall_kwnames = PyTuple_New(nkwds); From 5a7344b49f6f2613808e721e353372bdf91305db Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Fri, 6 Dec 2024 21:54:19 +0100 Subject: [PATCH 17/17] review comments --- Modules/_operator.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/Modules/_operator.c b/Modules/_operator.c index b142b96ab882f7..4571f94f03b0ca 100644 --- a/Modules/_operator.c +++ b/Modules/_operator.c @@ -1606,8 +1606,8 @@ typedef struct { #define _METHODCALLER_MAX_ARGS 8 static PyObject * -methodcaller_vectorcall( - methodcallerobject *mc, PyObject *const *args, size_t nargsf, PyObject* kwnames) +methodcaller_vectorcall(methodcallerobject *mc, PyObject *const *args, + size_t nargsf, PyObject* kwnames) { if (!_PyArg_CheckPositional("methodcaller", PyVectorcall_NARGS(nargsf), 1, 1) || !_PyArg_NoKwnames("methodcaller", kwnames)) { @@ -1623,15 +1623,13 @@ methodcaller_vectorcall( assert(1 + number_of_arguments <= _METHODCALLER_MAX_ARGS); memcpy(tmp_args + 1, mc->vectorcall_args, sizeof(PyObject *) * number_of_arguments); - PyObject *result = PyObject_VectorcallMethod( - mc->name, tmp_args, + return PyObject_VectorcallMethod(mc->name, tmp_args, (1 + PyTuple_GET_SIZE(mc->args)) | PY_VECTORCALL_ARGUMENTS_OFFSET, mc->vectorcall_kwnames); - - return result; } -static int _methodcaller_initialize_vectorcall(methodcallerobject* mc) +static int +_methodcaller_initialize_vectorcall(methodcallerobject* mc) { PyObject* args = mc->args; PyObject* kwds = mc->kwds; pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy