From ca76251558cef8e6b5731a1f9f3e86d32e2ff3ac Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sun, 2 Oct 2022 15:43:42 -0700 Subject: [PATCH 01/35] try 1 --- Include/internal/pycore_global_strings.h | 2 + Objects/typeobject.c | 106 ++++++++++++++++++++++- 2 files changed, 106 insertions(+), 2 deletions(-) diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 1523eef73931c9..f79e6ad2d53827 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -74,6 +74,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(__await__) STRUCT_FOR_ID(__bases__) STRUCT_FOR_ID(__bool__) + STRUCT_FOR_ID(__buffer__) STRUCT_FOR_ID(__build_class__) STRUCT_FOR_ID(__builtins__) STRUCT_FOR_ID(__bytes__) @@ -172,6 +173,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(__rdivmod__) STRUCT_FOR_ID(__reduce__) STRUCT_FOR_ID(__reduce_ex__) + STRUCT_FOR_ID(__release_buffer__) STRUCT_FOR_ID(__repr__) STRUCT_FOR_ID(__reversed__) STRUCT_FOR_ID(__rfloordiv__) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 5aa5cbbd54022e..fc3e0f42d884c4 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -7551,6 +7551,46 @@ wrap_descr_delete(PyObject *self, PyObject *args, void *wrapped) Py_RETURN_NONE; } +static PyObject * +wrap_buffer(PyObject *self, PyObject *args, void *wrapped) +{ + getbufferproc func = (getbufferproc)wrapped; + Py_buffer view; + int flags = 0; + + if (!check_num_args(args, 1)) { + return NULL; + } + if (!PyArg_ParseTuple(args, "i", &flags)) { + return NULL; + } + if ((*func)(self, &view, flags) < 0) { + return NULL; + } + return PyMemoryView_FromBuffer(&view); +} + +static PyObject * +wrap_releasebuffer(PyObject *self, PyObject *args, void *wrapped) +{ + releasebufferproc func = (releasebufferproc)wrapped; + PyMemoryViewObject *mview; + + if (!check_num_args(args, 1)) { + return NULL; + } + if (!PyArg_ParseTuple(args, "O!", &PyMemoryView_Type, &mview)) { + return NULL; + } + if (mview->view.obj != self) { + PyErr_SetString(PyExc_ValueError, + "memoryview's buffer is not this object"); + return NULL; + } + (*func)(self, &mview->view); + Py_RETURN_NONE; +} + static PyObject * wrap_init(PyObject *self, PyObject *args, void *wrapped, PyObject *kwds) { @@ -8383,6 +8423,54 @@ slot_tp_finalize(PyObject *self) PyErr_Restore(error_type, error_value, error_traceback); } +static int +slot_bf_getbuffer(PyObject *self, Py_buffer *buffer, int flags) +{ + PyObject *flags_obj = PyLong_FromLong(flags); + if (flags_obj == NULL) { + return -1; + } + PyObject *stack[2] = {self, flags_obj}; + PyObject *ret = vectorcall_method(&_Py_ID(__buffer__), stack, 2); + if (ret == NULL) { + goto fail; + } + if (!PyMemoryView_Check(ret)) { + PyErr_Format(PyExc_TypeError, + "__buffer__ returned non-memoryview object"); + goto fail; + } + *buffer = ((PyMemoryViewObject *)ret)->view; + Py_DECREF(ret); + Py_DECREF(flags_obj); + return 0; + +fail: + Py_XDECREF(ret); + Py_DECREF(flags_obj); + return -1; +} + +static void +slot_bf_releasebuffer(PyObject *self, Py_buffer *buffer) +{ + PyObject *mv = PyMemoryView_FromBuffer(buffer); + if (mv == NULL) { + PyErr_WriteUnraisable(self); + return; + } + PyObject *stack[2] = {self, mv}; + PyObject *ret = vectorcall_method(&_Py_ID(__release_buffer__), stack, 2); + Py_DECREF(mv); + if (ret == NULL) { + PyErr_WriteUnraisable(self); + } + else { + Py_DECREF(ret); + } +} + + static PyObject * slot_am_await(PyObject *self) { @@ -8452,6 +8540,7 @@ typedef struct wrapperbase slotdef; #undef TPSLOT #undef FLSLOT +#undef BUFSLOT #undef AMSLOT #undef ETSLOT #undef SQSLOT @@ -8471,6 +8560,8 @@ typedef struct wrapperbase slotdef; #define ETSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC) \ {#NAME, offsetof(PyHeapTypeObject, SLOT), (void *)(FUNCTION), WRAPPER, \ PyDoc_STR(DOC), .name_strobj = &_Py_ID(NAME) } +#define BUFSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC) \ + ETSLOT(NAME, as_buffer.SLOT, FUNCTION, WRAPPER, DOC) #define AMSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC) \ ETSLOT(NAME, as_async.SLOT, FUNCTION, WRAPPER, DOC) #define SQSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC) \ @@ -8552,6 +8643,13 @@ static slotdef slotdefs[] = { "Create and return new object. See help(type) for accurate signature."), TPSLOT(__del__, tp_finalize, slot_tp_finalize, (wrapperfunc)wrap_del, ""), + BUFSLOT(__buffer__, bf_getbuffer, slot_bf_getbuffer, wrap_buffer, + "__buffer__($self, flags, /)\n--\n\n" + "Return a buffer object that exposes the underlying memory of the object."), + BUFSLOT(__release_buffer__, bf_releasebuffer, slot_bf_releasebuffer, wrap_releasebuffer, + "__release_buffer__($self, /)\n--\n\n" + "Release the buffer object that exposes the underlying memory of the object."), + AMSLOT(__await__, am_await, slot_am_await, wrap_unaryfunc, "__await__($self, /)\n--\n\nReturn an iterator to be used in await expression."), AMSLOT(__aiter__, am_aiter, slot_am_aiter, wrap_unaryfunc, @@ -8698,8 +8796,12 @@ slotptr(PyTypeObject *type, int ioffset) /* Note: this depends on the order of the members of PyHeapTypeObject! */ assert(offset >= 0); - assert((size_t)offset < offsetof(PyHeapTypeObject, as_buffer)); - if ((size_t)offset >= offsetof(PyHeapTypeObject, as_sequence)) { + assert((size_t)offset < offsetof(PyHeapTypeObject, ht_name)); + if ((size_t)offset >= offsetof(PyHeapTypeObject, as_buffer)) { + ptr = (char *)type->tp_as_buffer; + offset -= offsetof(PyHeapTypeObject, as_buffer); + } + else if ((size_t)offset >= offsetof(PyHeapTypeObject, as_sequence)) { ptr = (char *)type->tp_as_sequence; offset -= offsetof(PyHeapTypeObject, as_sequence); } From 901b459cd92e224afddc5291e3670cd0efdd3511 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 3 Oct 2022 11:50:52 -0700 Subject: [PATCH 02/35] progress --- Include/internal/pycore_global_strings.h | 2 +- Include/internal/pycore_memoryobject.h | 17 +++++++ .../internal/pycore_runtime_init_generated.h | 14 +++++ Lib/test/test_buffer.py | 51 +++++++++++++++++++ Objects/memoryobject.c | 21 +++++--- Objects/typeobject.c | 15 +++--- 6 files changed, 106 insertions(+), 14 deletions(-) create mode 100644 Include/internal/pycore_memoryobject.h diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index f79e6ad2d53827..45068c5aabe960 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -173,7 +173,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(__rdivmod__) STRUCT_FOR_ID(__reduce__) STRUCT_FOR_ID(__reduce_ex__) - STRUCT_FOR_ID(__release_buffer__) + STRUCT_FOR_ID(__release_buffer__) STRUCT_FOR_ID(__repr__) STRUCT_FOR_ID(__reversed__) STRUCT_FOR_ID(__rfloordiv__) diff --git a/Include/internal/pycore_memoryobject.h b/Include/internal/pycore_memoryobject.h new file mode 100644 index 00000000000000..acc12c9275172c --- /dev/null +++ b/Include/internal/pycore_memoryobject.h @@ -0,0 +1,17 @@ +#ifndef Py_INTERNAL_MEMORYOBJECT_H +#define Py_INTERNAL_MEMORYOBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +PyObject * +PyMemoryView_FromObjectAndFlags(PyObject *v, int flags); + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_MEMORYOBJECT_H */ diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 32ff57b731e1a0..662ff2b3151b84 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -583,6 +583,7 @@ extern "C" { INIT_ID(__await__), \ INIT_ID(__bases__), \ INIT_ID(__bool__), \ + INIT_ID(__buffer__), \ INIT_ID(__build_class__), \ INIT_ID(__builtins__), \ INIT_ID(__bytes__), \ @@ -681,6 +682,7 @@ extern "C" { INIT_ID(__rdivmod__), \ INIT_ID(__reduce__), \ INIT_ID(__reduce_ex__), \ + INIT_ID(__release_buffer__), \ INIT_ID(__repr__), \ INIT_ID(__reversed__), \ INIT_ID(__rfloordiv__), \ @@ -1472,6 +1474,8 @@ _PyUnicode_InitStaticStrings(void) { PyUnicode_InternInPlace(&string); string = &_Py_ID(__bool__); PyUnicode_InternInPlace(&string); + string = &_Py_ID(__buffer__); + PyUnicode_InternInPlace(&string); string = &_Py_ID(__build_class__); PyUnicode_InternInPlace(&string); string = &_Py_ID(__builtins__); @@ -1668,6 +1672,8 @@ _PyUnicode_InitStaticStrings(void) { PyUnicode_InternInPlace(&string); string = &_Py_ID(__reduce_ex__); PyUnicode_InternInPlace(&string); + string = &_Py_ID(__release_buffer__); + PyUnicode_InternInPlace(&string); string = &_Py_ID(__repr__); PyUnicode_InternInPlace(&string); string = &_Py_ID(__reversed__); @@ -4871,6 +4877,10 @@ _PyStaticObjects_CheckRefcnt(void) { _PyObject_Dump((PyObject *)&_Py_ID(__bool__)); Py_FatalError("immortal object has less refcnt than expected _PyObject_IMMORTAL_REFCNT"); }; + if (Py_REFCNT((PyObject *)&_Py_ID(__buffer__)) < _PyObject_IMMORTAL_REFCNT) { + _PyObject_Dump((PyObject *)&_Py_ID(__buffer__)); + Py_FatalError("immortal object has less refcnt than expected _PyObject_IMMORTAL_REFCNT"); + }; if (Py_REFCNT((PyObject *)&_Py_ID(__build_class__)) < _PyObject_IMMORTAL_REFCNT) { _PyObject_Dump((PyObject *)&_Py_ID(__build_class__)); Py_FatalError("immortal object has less refcnt than expected _PyObject_IMMORTAL_REFCNT"); @@ -5263,6 +5273,10 @@ _PyStaticObjects_CheckRefcnt(void) { _PyObject_Dump((PyObject *)&_Py_ID(__reduce_ex__)); Py_FatalError("immortal object has less refcnt than expected _PyObject_IMMORTAL_REFCNT"); }; + if (Py_REFCNT((PyObject *)&_Py_ID(__release_buffer__)) < _PyObject_IMMORTAL_REFCNT) { + _PyObject_Dump((PyObject *)&_Py_ID(__release_buffer__)); + Py_FatalError("immortal object has less refcnt than expected _PyObject_IMMORTAL_REFCNT"); + }; if (Py_REFCNT((PyObject *)&_Py_ID(__repr__)) < _PyObject_IMMORTAL_REFCNT) { _PyObject_Dump((PyObject *)&_Py_ID(__repr__)); Py_FatalError("immortal object has less refcnt than expected _PyObject_IMMORTAL_REFCNT"); diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index 8ac3b7e7eb29d1..a1b624aef4605c 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -4436,5 +4436,56 @@ def test_pybuffer_size_from_format(self): struct.calcsize(format)) +class TestPythonBufferProtocol(unittest.TestCase): + def test_basic(self): + class MyBuffer: + def __buffer__(self, flags): + return memoryview(b"hello") + + mv = memoryview(MyBuffer()) + self.assertEqual(mv.tobytes(), b"hello") + self.assertEqual(bytes(MyBuffer()), b"hello") + + def test_bad_buffer_method(self): + class MustReturnMV: + def __buffer__(self, flags): + return 42 + + self.assertRaises(TypeError, memoryview, MustReturnMV()) + + class WrongArity: + def __buffer__(self): + return memoryview(b"hello") + + self.assertRaises(TypeError, memoryview, WrongArity()) + + def test_release_buffer(self): + class WhatToRelease: + def __init__(self): + self.held = False + self.ba = bytearray(b"hello") + def __buffer__(self, flags): + if self.held: + raise TypeError("already held") + self.held = True + return memoryview(self.ba) + def __release_buffer__(self, buffer): + assert self is buffer.obj + self.held = False + + wr = WhatToRelease() + self.assertFalse(wr.held) + with memoryview(wr) as mv: + self.assertTrue(wr.held) + self.assertEqual(mv.tobytes(), b"hello") + self.assertFalse(wr.held) + + def test_call_builtins(self): + ba = bytearray(b"hello") + mv = ba.__buffer__(0) + self.assertEqual(mv.tobytes(), b"hello") + ba.__release_buffer__(mv) + + if __name__ == "__main__": unittest.main() diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index c5ab0bf2dedbe4..db832aa5c9b22a 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -85,7 +85,7 @@ mbuf_alloc(void) } static PyObject * -_PyManagedBuffer_FromObject(PyObject *base) +_PyManagedBuffer_FromObject(PyObject *base, int flags) { _PyManagedBufferObject *mbuf; @@ -93,7 +93,7 @@ _PyManagedBuffer_FromObject(PyObject *base) if (mbuf == NULL) return NULL; - if (PyObject_GetBuffer(base, &mbuf->master, PyBUF_FULL_RO) < 0) { + if (PyObject_GetBuffer(base, &mbuf->master, flags) < 0) { mbuf->master.obj = NULL; Py_DECREF(mbuf); return NULL; @@ -779,11 +779,12 @@ PyMemoryView_FromBuffer(const Py_buffer *info) return mv; } -/* Create a memoryview from an object that implements the buffer protocol. +/* Create a memoryview from an object that implements the buffer protocol, + using the given flags. If the object is a memoryview, the new memoryview must be registered with the same managed buffer. Otherwise, a new managed buffer is created. */ PyObject * -PyMemoryView_FromObject(PyObject *v) +PyMemoryView_FromObjectAndFlags(PyObject *v, int flags) { _PyManagedBufferObject *mbuf; @@ -794,7 +795,7 @@ PyMemoryView_FromObject(PyObject *v) } else if (PyObject_CheckBuffer(v)) { PyObject *ret; - mbuf = (_PyManagedBufferObject *)_PyManagedBuffer_FromObject(v); + mbuf = (_PyManagedBufferObject *)_PyManagedBuffer_FromObject(v, flags); if (mbuf == NULL) return NULL; ret = mbuf_add_view(mbuf, NULL); @@ -807,6 +808,14 @@ PyMemoryView_FromObject(PyObject *v) Py_TYPE(v)->tp_name); return NULL; } +/* Create a memoryview from an object that implements the buffer protocol. + If the object is a memoryview, the new memoryview must be registered + with the same managed buffer. Otherwise, a new managed buffer is created. */ +PyObject * +PyMemoryView_FromObject(PyObject *v) +{ + return PyMemoryView_FromObjectAndFlags(v, PyBUF_FULL_RO); +} /* Copy the format string from a base object that might vanish. */ static int @@ -853,7 +862,7 @@ memory_from_contiguous_copy(const Py_buffer *src, char order) if (bytes == NULL) return NULL; - mbuf = (_PyManagedBufferObject *)_PyManagedBuffer_FromObject(bytes); + mbuf = (_PyManagedBufferObject *)_PyManagedBuffer_FromObject(bytes, PyBUF_FULL_RO); Py_DECREF(bytes); if (mbuf == NULL) return NULL; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index fc3e0f42d884c4..48baac313151d9 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -5,6 +5,7 @@ #include "pycore_code.h" // CO_FAST_FREE #include "pycore_compile.h" // _Py_Mangle() #include "pycore_initconfig.h" // _PyStatus_OK() +#include "pycore_memoryobject.h" // PyMemoryView_FromObjectAndFlags() #include "pycore_moduleobject.h" // _PyModule_GetDef() #include "pycore_object.h" // _PyType_HasFeature() #include "pycore_pyerrors.h" // _PyErr_Occurred() @@ -7554,8 +7555,6 @@ wrap_descr_delete(PyObject *self, PyObject *args, void *wrapped) static PyObject * wrap_buffer(PyObject *self, PyObject *args, void *wrapped) { - getbufferproc func = (getbufferproc)wrapped; - Py_buffer view; int flags = 0; if (!check_num_args(args, 1)) { @@ -7564,10 +7563,7 @@ wrap_buffer(PyObject *self, PyObject *args, void *wrapped) if (!PyArg_ParseTuple(args, "i", &flags)) { return NULL; } - if ((*func)(self, &view, flags) < 0) { - return NULL; - } - return PyMemoryView_FromBuffer(&view); + return PyMemoryView_FromObjectAndFlags(self, flags); } static PyObject * @@ -8441,6 +8437,9 @@ slot_bf_getbuffer(PyObject *self, Py_buffer *buffer, int flags) goto fail; } *buffer = ((PyMemoryViewObject *)ret)->view; + // TODO does this leak the existing ref in buffer->obj? + buffer->obj = self; + Py_INCREF(buffer->obj); Py_DECREF(ret); Py_DECREF(flags_obj); return 0; @@ -8459,6 +8458,9 @@ slot_bf_releasebuffer(PyObject *self, Py_buffer *buffer) PyErr_WriteUnraisable(self); return; } + if (buffer->obj != NULL) { + ((PyMemoryViewObject *)mv)->view.obj = Py_NewRef(buffer->obj); + } PyObject *stack[2] = {self, mv}; PyObject *ret = vectorcall_method(&_Py_ID(__release_buffer__), stack, 2); Py_DECREF(mv); @@ -8470,7 +8472,6 @@ slot_bf_releasebuffer(PyObject *self, Py_buffer *buffer) } } - static PyObject * slot_am_await(PyObject *self) { From c5407c7dfb30d6926569a21b00c0c0ccf5b2e89a Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 4 Oct 2022 10:37:50 -0700 Subject: [PATCH 03/35] this is better --- Lib/test/test_buffer.py | 2 +- Objects/typeobject.c | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index a1b624aef4605c..bd6bca1f48d0d5 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -4478,7 +4478,7 @@ def __release_buffer__(self, buffer): with memoryview(wr) as mv: self.assertTrue(wr.held) self.assertEqual(mv.tobytes(), b"hello") - self.assertFalse(wr.held) + #self.assertFalse(wr.held) def test_call_builtins(self): ba = bytearray(b"hello") diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 48baac313151d9..24b260d2a3cb83 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -8437,8 +8437,6 @@ slot_bf_getbuffer(PyObject *self, Py_buffer *buffer, int flags) goto fail; } *buffer = ((PyMemoryViewObject *)ret)->view; - // TODO does this leak the existing ref in buffer->obj? - buffer->obj = self; Py_INCREF(buffer->obj); Py_DECREF(ret); Py_DECREF(flags_obj); From 86f2000a67fa1f990dbae3cc7400ea4fa042cec8 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 4 Oct 2022 16:59:28 -0700 Subject: [PATCH 04/35] this seems to work --- Include/internal/pycore_typeobject.h | 2 + Lib/_collections_abc.py | 32 ++++++++++- Lib/test/test_buffer.py | 3 +- Lib/test/test_collections.py | 18 ++++++- Objects/object.c | 2 + Objects/typeobject.c | 80 ++++++++++++++++++++++++++-- 6 files changed, 128 insertions(+), 9 deletions(-) diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index 5e7aca1b9f5ae9..97309650c895c1 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -75,6 +75,8 @@ extern void _PyStaticType_Dealloc(PyTypeObject *type); PyObject *_Py_slot_tp_getattro(PyObject *self, PyObject *name); PyObject *_Py_slot_tp_getattr_hook(PyObject *self, PyObject *name); +PyAPI_DATA(PyTypeObject) _PyBufferWrapper_Type; + #ifdef __cplusplus } #endif diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index c62233b81a5c95..3b0d0eeb1fa9d1 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -49,7 +49,7 @@ def _f(): pass "Mapping", "MutableMapping", "MappingView", "KeysView", "ItemsView", "ValuesView", "Sequence", "MutableSequence", - "ByteString", + "ByteString", "Buffer", "MutableBuffer", ] # This module has been renamed from collections.abc to _collections_abc to @@ -439,6 +439,36 @@ def __subclasshook__(cls, C): return NotImplemented +class Buffer(metaclass=ABCMeta): + + __slots__ = () + + @abstractmethod + def __buffer__(self, flags: int, /) -> memoryview: + raise NotImplementedError + + @classmethod + def __subclasshook__(cls, C): + if cls is Buffer: + return _check_methods(C, "__buffer__") + return NotImplemented + + +class MutableBuffer(Buffer): + + __slots__ = () + + @abstractmethod + def __release_buffer__(self, buffer: memoryview, /) -> None: + pass + + @classmethod + def __subclasshook__(cls, C): + if cls is MutableBuffer: + return _check_methods(C, "__buffer__", "__release_buffer__") + return NotImplemented + + class _CallableGenericAlias(GenericAlias): """ Represent `Callable[argtypes, resulttype]`. diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index bd6bca1f48d0d5..c46045ae96485e 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -4470,7 +4470,6 @@ def __buffer__(self, flags): self.held = True return memoryview(self.ba) def __release_buffer__(self, buffer): - assert self is buffer.obj self.held = False wr = WhatToRelease() @@ -4478,7 +4477,7 @@ def __release_buffer__(self, buffer): with memoryview(wr) as mv: self.assertTrue(wr.held) self.assertEqual(mv.tobytes(), b"hello") - #self.assertFalse(wr.held) + self.assertFalse(wr.held) def test_call_builtins(self): ba = bytearray(b"hello") diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index 59b3f2ec7bfcb6..a8ea91d973638f 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -25,7 +25,7 @@ from collections.abc import Set, MutableSet from collections.abc import Mapping, MutableMapping, KeysView, ItemsView, ValuesView from collections.abc import Sequence, MutableSequence -from collections.abc import ByteString +from collections.abc import ByteString, Buffer, MutableBuffer class TestUserObjects(unittest.TestCase): @@ -1941,6 +1941,22 @@ def test_ByteString(self): self.assertNotIsInstance(memoryview(b""), ByteString) self.assertFalse(issubclass(memoryview, ByteString)) + def test_Buffer(self): + for sample in [bytes, bytearray, memoryview]: + self.assertIsInstance(sample(b"x"), Buffer) + self.assertTrue(issubclass(sample, Buffer)) + for sample in [str, list, tuple]: + self.assertNotIsInstance(sample(), Buffer) + self.assertFalse(issubclass(sample, Buffer)) + + def test_MutableBuffer(self): + for sample in [bytearray, memoryview]: + self.assertIsInstance(sample(b"x"), MutableBuffer) + self.assertTrue(issubclass(sample, MutableBuffer)) + for sample in [bytes, str, list, tuple]: + self.assertNotIsInstance(sample(), MutableBuffer) + self.assertFalse(issubclass(sample, MutableBuffer)) + def test_MutableSequence(self): for sample in [tuple, str, bytes]: self.assertNotIsInstance(sample(), MutableSequence) diff --git a/Objects/object.c b/Objects/object.c index 26cdcef59dddbd..5edbe34025daa6 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -14,6 +14,7 @@ #include "pycore_pymem.h" // _PyMem_IsPtrFreed() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_symtable.h" // PySTEntry_Type +#include "pycore_typeobject.h" // _PyBufferWrapper_Type #include "pycore_unionobject.h" // _PyUnion_Type #include "pycore_interpreteridobject.h" // _PyInterpreterID_Type @@ -1930,6 +1931,7 @@ static PyTypeObject* static_types[] = { &_PyAsyncGenASend_Type, &_PyAsyncGenAThrow_Type, &_PyAsyncGenWrappedValue_Type, + &_PyBufferWrapper_Type, &_PyContextTokenMissing_Type, &_PyCoroWrapper_Type, &_Py_GenericAliasIterType, diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 24b260d2a3cb83..6874e174460bb3 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -8419,6 +8419,64 @@ slot_tp_finalize(PyObject *self) PyErr_Restore(error_type, error_value, error_traceback); } +typedef struct _PyBufferWrapper { + PyObject_HEAD + PyObject *mv; + PyObject *obj; +} PyBufferWrapper; + +static int +bufferwrapper_traverse(PyObject *self, visitproc visit, void *arg) +{ + PyBufferWrapper *bw = (PyBufferWrapper *)self; + + Py_VISIT(bw->mv); + Py_VISIT(bw->obj); + return 0; +} + +static void +bufferwrapper_dealloc(PyObject *self) +{ + PyBufferWrapper *bw = (PyBufferWrapper *)self; + + _PyObject_GC_UNTRACK(self); + Py_XDECREF(bw->mv); + Py_XDECREF(bw->obj); + Py_TYPE(self)->tp_free(self); +} + +static void +bufferwrapper_releasebuf(PyObject *self, Py_buffer *view) +{ + PyBufferWrapper *bw = (PyBufferWrapper *)self; + + assert(PyMemoryView_Check(bw->mv)); + Py_TYPE(bw->mv)->tp_as_buffer->bf_releasebuffer(bw->mv, view); + if (Py_TYPE(bw->obj)->tp_as_buffer != NULL + && Py_TYPE(bw->obj)->tp_as_buffer->bf_releasebuffer != NULL) { + Py_TYPE(bw->obj)->tp_as_buffer->bf_releasebuffer(bw->obj, view); + } +} + +static PyBufferProcs bufferwrapper_as_buffer = { + .bf_releasebuffer = bufferwrapper_releasebuf, +}; + + +PyTypeObject _PyBufferWrapper_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + .tp_name = "_buffer_wrapper", + .tp_basicsize = sizeof(PyBufferWrapper), + .tp_alloc = PyType_GenericAlloc, + .tp_new = PyType_GenericNew, + .tp_free = PyObject_GC_Del, + .tp_traverse = bufferwrapper_traverse, + .tp_dealloc = bufferwrapper_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + .tp_as_buffer = &bufferwrapper_as_buffer, +}; + static int slot_bf_getbuffer(PyObject *self, Py_buffer *buffer, int flags) { @@ -8426,6 +8484,7 @@ slot_bf_getbuffer(PyObject *self, Py_buffer *buffer, int flags) if (flags_obj == NULL) { return -1; } + PyObject *wrapper = NULL; PyObject *stack[2] = {self, flags_obj}; PyObject *ret = vectorcall_method(&_Py_ID(__buffer__), stack, 2); if (ret == NULL) { @@ -8436,13 +8495,27 @@ slot_bf_getbuffer(PyObject *self, Py_buffer *buffer, int flags) "__buffer__ returned non-memoryview object"); goto fail; } - *buffer = ((PyMemoryViewObject *)ret)->view; - Py_INCREF(buffer->obj); + + if (PyObject_GetBuffer(ret, buffer, flags) < 0) { + goto fail; + } + assert(buffer->obj == ret); + + wrapper = PyObject_GC_New(PyBufferWrapper, &_PyBufferWrapper_Type); + if (wrapper == NULL) { + goto fail; + } + ((PyBufferWrapper *)wrapper)->mv = ret; + ((PyBufferWrapper *)wrapper)->obj = Py_NewRef(self); + _PyObject_GC_TRACK(wrapper); + + buffer->obj = wrapper; Py_DECREF(ret); Py_DECREF(flags_obj); return 0; fail: + Py_XDECREF(wrapper); Py_XDECREF(ret); Py_DECREF(flags_obj); return -1; @@ -8456,9 +8529,6 @@ slot_bf_releasebuffer(PyObject *self, Py_buffer *buffer) PyErr_WriteUnraisable(self); return; } - if (buffer->obj != NULL) { - ((PyMemoryViewObject *)mv)->view.obj = Py_NewRef(buffer->obj); - } PyObject *stack[2] = {self, mv}; PyObject *ret = vectorcall_method(&_Py_ID(__release_buffer__), stack, 2); Py_DECREF(mv); From ac10887613a1f067487eeb35581ec525f996c190 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 4 Oct 2022 17:01:52 -0700 Subject: [PATCH 05/35] one more test --- Lib/test/test_collections.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index a8ea91d973638f..23483c5086d578 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -1948,6 +1948,7 @@ def test_Buffer(self): for sample in [str, list, tuple]: self.assertNotIsInstance(sample(), Buffer) self.assertFalse(issubclass(sample, Buffer)) + self.validate_abstract_methods(Buffer, '__buffer__') def test_MutableBuffer(self): for sample in [bytearray, memoryview]: @@ -1956,6 +1957,7 @@ def test_MutableBuffer(self): for sample in [bytes, str, list, tuple]: self.assertNotIsInstance(sample(), MutableBuffer) self.assertFalse(issubclass(sample, MutableBuffer)) + self.validate_abstract_methods(MutableBuffer, '__buffer__', '__release_buffer__') def test_MutableSequence(self): for sample in [tuple, str, bytes]: From 530a16099087030a6f3953a86d352cb51f578138 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 5 Oct 2022 14:17:22 -0700 Subject: [PATCH 06/35] additions --- Include/pybuffer.h | 2 +- Lib/inspect.py | 22 ++++++++++++++++++++++ Lib/test/test_buffer.py | 23 +++++++++++++++++++++++ Objects/typeobject.c | 16 ++++++++++++---- 4 files changed, 58 insertions(+), 5 deletions(-) diff --git a/Include/pybuffer.h b/Include/pybuffer.h index 6893505e66e3e8..7173639520fe50 100644 --- a/Include/pybuffer.h +++ b/Include/pybuffer.h @@ -101,7 +101,7 @@ PyAPI_FUNC(void) PyBuffer_Release(Py_buffer *view); /* Maximum number of dimensions */ #define PyBUF_MAX_NDIM 64 -/* Flags for getting buffers */ +/* Flags for getting buffers. Keep these in sync with inspect.BufferFlags. */ #define PyBUF_SIMPLE 0 #define PyBUF_WRITABLE 0x0001 diff --git a/Lib/inspect.py b/Lib/inspect.py index 498ee7ab9eaf8a..f71ae8eb53bd85 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -3273,6 +3273,28 @@ def signature(obj, *, follow_wrapped=True, globals=None, locals=None, eval_str=F globals=globals, locals=locals, eval_str=eval_str) +class BufferFlags(enum.IntFlag): + SIMPLE = 0x0 + WRITABLE = 0x1 + FORMAT = 0x4 + ND = 0x8 + STRIDES = 0x10 | ND + C_CONTIGUOUS = 0x20 | STRIDES + F_CONTIGUOUS = 0x40 | STRIDES + ANY_CONTIGUOUS = 0x80 | STRIDES + INDIRECT = 0x100 | STRIDES + CONTIG = ND | WRITABLE + CONTIG_RO = ND + STRIDED = STRIDES | WRITABLE + STRIDED_RO = STRIDES + RECORDS = STRIDES | WRITABLE | FORMAT + RECORDS_RO = STRIDES | FORMAT + FULL = INDIRECT | WRITABLE | FORMAT + FULL_RO = INDIRECT | FORMAT + READ = 0x100 + WRITE = 0x200 + + def _main(): """ Logic for inspecting an object given at command line """ import argparse diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index c46045ae96485e..9bee30bacd3b73 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -4479,6 +4479,29 @@ def __release_buffer__(self, buffer): self.assertEqual(mv.tobytes(), b"hello") self.assertFalse(wr.held) + def test_same_buffer_returned(self): + class WhatToRelease: + def __init__(self): + self.held = False + self.ba = bytearray(b"hello") + self.created_mv = None + def __buffer__(self, flags): + if self.held: + raise TypeError("already held") + self.held = True + self.created_mv = memoryview(self.ba) + return self.created_mv + def __release_buffer__(self, buffer): + assert buffer is self.created_mv + self.held = False + + wr = WhatToRelease() + self.assertFalse(wr.held) + with memoryview(wr) as mv: + self.assertTrue(wr.held) + self.assertEqual(mv.tobytes(), b"hello") + self.assertFalse(wr.held) + def test_call_builtins(self): ba = bytearray(b"hello") mv = ba.__buffer__(0) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 0bdf5ec69f390c..820fba90801818 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -8526,10 +8526,18 @@ slot_bf_getbuffer(PyObject *self, Py_buffer *buffer, int flags) static void slot_bf_releasebuffer(PyObject *self, Py_buffer *buffer) { - PyObject *mv = PyMemoryView_FromBuffer(buffer); - if (mv == NULL) { - PyErr_WriteUnraisable(self); - return; + PyObject *mv; + if (Py_TYPE(buffer->obj) == &_PyBufferWrapper_Type) { + // Make sure we pass the same memoryview to + // __release_buffer__() that __buffer__() returned. + mv = Py_NewRef(((PyBufferWrapper *)buffer->obj)->mv); + } + else { + mv = PyMemoryView_FromBuffer(buffer); + if (mv == NULL) { + PyErr_WriteUnraisable(self); + return; + } } PyObject *stack[2] = {self, mv}; PyObject *ret = vectorcall_method(&_Py_ID(__release_buffer__), stack, 2); From 207d2fd67ebe2bdcf22348cb50e4bfdf0a33a2eb Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 20 Oct 2022 21:46:56 -0700 Subject: [PATCH 07/35] introduce __mutable_buffer__ (rather hackily) --- Include/internal/pycore_abstract.h | 4 ++++ Lib/_collections_abc.py | 7 ++----- Modules/_ctypes/_ctypes.c | 17 +++++++++++++---- Modules/_io/bytesio.c | 8 +++++++- Modules/arraymodule.c | 2 ++ Modules/mmapmodule.c | 2 ++ Objects/abstract.c | 5 +++++ Objects/bytearrayobject.c | 9 +++++++-- Objects/memoryobject.c | 3 ++- Objects/picklebufobject.c | 6 ++++++ Objects/typeobject.c | 25 +++++++++++++++++-------- 11 files changed, 67 insertions(+), 21 deletions(-) diff --git a/Include/internal/pycore_abstract.h b/Include/internal/pycore_abstract.h index b1afb2dc7be65e..6db26bf246ee7c 100644 --- a/Include/internal/pycore_abstract.h +++ b/Include/internal/pycore_abstract.h @@ -19,6 +19,10 @@ _PyIndex_Check(PyObject *obj) PyObject *_PyNumber_PowerNoMod(PyObject *lhs, PyObject *rhs); PyObject *_PyNumber_InPlacePowerNoMod(PyObject *lhs, PyObject *rhs); +PyAPI_FUNC(PyObject *) _PyBuffer_MutableBufferGetter(PyObject *instance, void *ignored); + +#define _PY_BUFFER_MUTABLE_BUFFER_GETSET { "__mutable_buffer__", _PyBuffer_MutableBufferGetter, NULL, NULL }, + #ifdef __cplusplus } #endif diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 3b0d0eeb1fa9d1..c4b3e687315658 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -457,15 +457,12 @@ def __subclasshook__(cls, C): class MutableBuffer(Buffer): __slots__ = () - - @abstractmethod - def __release_buffer__(self, buffer: memoryview, /) -> None: - pass + __mutable_buffer__ = True @classmethod def __subclasshook__(cls, C): if cls is MutableBuffer: - return _check_methods(C, "__buffer__", "__release_buffer__") + return _check_methods(C, "__buffer__", "__mutable_buffer__") return NotImplemented diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index a3c7c8c471e584..1dbd85423bb87f 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -127,6 +127,7 @@ bytes(cdata) #include "ctypes.h" #include "pycore_long.h" // _PyLong_GetZero() +#include "pycore_abstract.h" // _PY_BUFFER_MUTABLE_BUFFER_GETSET PyObject *PyExc_ArgError = NULL; @@ -2809,6 +2810,11 @@ static PyBufferProcs PyCData_as_buffer = { NULL, }; +static PyGetSetDef PyCData_getset[] = { + _PY_BUFFER_MUTABLE_BUFFER_GETSET + {NULL} +}; + /* * CData objects are mutable, so they cannot be hashable! */ @@ -2918,7 +2924,7 @@ PyTypeObject PyCData_Type = { 0, /* tp_iternext */ PyCData_methods, /* tp_methods */ PyCData_members, /* tp_members */ - 0, /* tp_getset */ + PyCData_getset, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ @@ -3342,6 +3348,7 @@ static PyGetSetDef PyCFuncPtr_getsets[] = { { "argtypes", (getter)PyCFuncPtr_get_argtypes, (setter)PyCFuncPtr_set_argtypes, "specify the argument types", NULL }, + _PY_BUFFER_MUTABLE_BUFFER_GETSET { NULL, NULL } }; @@ -4468,7 +4475,7 @@ static PyTypeObject Struct_Type = { 0, /* tp_iternext */ 0, /* tp_methods */ 0, /* tp_members */ - 0, /* tp_getset */ + PyCData_getset, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ @@ -4510,7 +4517,7 @@ static PyTypeObject Union_Type = { 0, /* tp_iternext */ 0, /* tp_methods */ 0, /* tp_members */ - 0, /* tp_getset */ + PyCData_getset, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ @@ -4830,7 +4837,7 @@ PyTypeObject PyCArray_Type = { 0, /* tp_iternext */ Array_methods, /* tp_methods */ 0, /* tp_members */ - 0, /* tp_getset */ + PyCData_getset, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ @@ -4961,6 +4968,7 @@ Simple_get_value(CDataObject *self, void *Py_UNUSED(ignored)) static PyGetSetDef Simple_getsets[] = { { "value", (getter)Simple_get_value, (setter)Simple_set_value, "current value", NULL }, + _PY_BUFFER_MUTABLE_BUFFER_GETSET { NULL, NULL } }; @@ -5204,6 +5212,7 @@ static PyGetSetDef Pointer_getsets[] = { { "contents", (getter)Pointer_get_contents, (setter)Pointer_set_contents, "the object this pointer points to (read-write)", NULL }, + _PY_BUFFER_MUTABLE_BUFFER_GETSET { NULL, NULL } }; diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index 930ef7e29dbcf6..20e3e6a0ef7435 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -1,4 +1,5 @@ #include "Python.h" +#include "pycore_abstract.h" // _PY_BUFFER_MUTABLE_BUFFER_GETSET #include "pycore_object.h" #include // offsetof() #include "_iomodule.h" @@ -1124,6 +1125,11 @@ static PyBufferProcs bytesiobuf_as_buffer = { (releasebufferproc) bytesiobuf_releasebuffer, }; +static PyGetSetDef bytesiobuf_getset[] = { + _PY_BUFFER_MUTABLE_BUFFER_GETSET + {NULL} +}; + Py_EXPORTED_SYMBOL PyTypeObject _PyBytesIOBuffer_Type = { PyVarObject_HEAD_INIT(NULL, 0) "_io._BytesIOBuffer", /*tp_name*/ @@ -1154,7 +1160,7 @@ Py_EXPORTED_SYMBOL PyTypeObject _PyBytesIOBuffer_Type = { 0, /*tp_iternext*/ 0, /*tp_methods*/ 0, /*tp_members*/ - 0, /*tp_getset*/ + bytesiobuf_getset, /*tp_getset*/ 0, /*tp_base*/ 0, /*tp_dict*/ 0, /*tp_descr_get*/ diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 924fbf29bfb889..98c28124909d50 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -9,6 +9,7 @@ #define PY_SSIZE_T_CLEAN #include "Python.h" +#include "pycore_abstract.h" // _PY_BUFFER_MUTABLE_BUFFER_GETSET #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_bytesobject.h" // _PyBytes_Repeat #include "structmember.h" // PyMemberDef @@ -2277,6 +2278,7 @@ static PyGetSetDef array_getsets [] = { "the typecode character used to create the array"}, {"itemsize", (getter) array_get_itemsize, NULL, "the size, in bytes, of one array item"}, + _PY_BUFFER_MUTABLE_BUFFER_GETSET {NULL} }; diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index fdce783fdec5e2..85cb0a524fb227 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -24,6 +24,7 @@ #define PY_SSIZE_T_CLEAN #include +#include "pycore_abstract.h" // _PY_BUFFER_MUTABLE_BUFFER_GETSET #include "pycore_bytesobject.h" // _PyBytes_Find() #include "pycore_fileutils.h" // _Py_stat_struct #include "structmember.h" // PyMemberDef @@ -887,6 +888,7 @@ static struct PyMethodDef mmap_object_methods[] = { static PyGetSetDef mmap_object_getset[] = { {"closed", (getter) mmap_closed_get, NULL, NULL}, + _PY_BUFFER_MUTABLE_BUFFER_GETSET {NULL} }; diff --git a/Objects/abstract.c b/Objects/abstract.c index 5d50491b2dd347..ea40cbe30ba1b4 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -2967,3 +2967,8 @@ _Py_FreeCharPArray(char *const array[]) } PyMem_Free((void*)array); } + +PyObject * +_PyBuffer_MutableBufferGetter(PyObject *instance, void *ignored) { + Py_RETURN_TRUE; +} diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index b2962fd137d93e..a413deba623dab 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -2,7 +2,7 @@ #define PY_SSIZE_T_CLEAN #include "Python.h" -#include "pycore_abstract.h" // _PyIndex_Check() +#include "pycore_abstract.h" // _PyIndex_Check(), _PY_BUFFER_MUTABLE_BUFFER_GETSET #include "pycore_bytes_methods.h" #include "pycore_bytesobject.h" #include "pycore_object.h" // _PyObject_GC_UNTRACK() @@ -2289,6 +2289,11 @@ Construct a mutable bytearray object from:\n\ static PyObject *bytearray_iter(PyObject *seq); +static PyGetSetDef bytearray_getset[] = { + _PY_BUFFER_MUTABLE_BUFFER_GETSET + {NULL} +}; + PyTypeObject PyByteArray_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "bytearray", @@ -2320,7 +2325,7 @@ PyTypeObject PyByteArray_Type = { 0, /* tp_iternext */ bytearray_methods, /* tp_methods */ 0, /* tp_members */ - 0, /* tp_getset */ + bytearray_getset, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index db832aa5c9b22a..6e414ee5f52427 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -11,7 +11,7 @@ */ #include "Python.h" -#include "pycore_abstract.h" // _PyIndex_Check() +#include "pycore_abstract.h" // _PyIndex_Check(), _PY_BUFFER_MUTABLE_BUFFER_GETSET #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_strhex.h" // _Py_strhex_with_sep() #include // offsetof() @@ -3186,6 +3186,7 @@ static PyGetSetDef memory_getsetlist[] = { {"c_contiguous", (getter)memory_c_contiguous, NULL, memory_c_contiguous_doc}, {"f_contiguous", (getter)memory_f_contiguous, NULL, memory_f_contiguous_doc}, {"contiguous", (getter)memory_contiguous, NULL, memory_contiguous_doc}, + _PY_BUFFER_MUTABLE_BUFFER_GETSET {NULL, NULL, NULL, NULL}, }; diff --git a/Objects/picklebufobject.c b/Objects/picklebufobject.c index aaa852cfbb05b0..7ec04206fa47df 100644 --- a/Objects/picklebufobject.c +++ b/Objects/picklebufobject.c @@ -2,6 +2,7 @@ #define PY_SSIZE_T_CLEAN #include "Python.h" +#include "pycore_abstract.h" // _PY_BUFFER_MUTABLE_BUFFER_GETSET #include typedef struct { @@ -203,6 +204,10 @@ static PyMethodDef picklebuf_methods[] = { {NULL, NULL} }; +static PyGetSetDef picklebuf_getset[] = { + _PY_BUFFER_MUTABLE_BUFFER_GETSET + {NULL} +}; PyTypeObject PyPickleBuffer_Type = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "pickle.PickleBuffer", @@ -216,4 +221,5 @@ PyTypeObject PyPickleBuffer_Type = { .tp_weaklistoffset = offsetof(PyPickleBufferObject, weakreflist), .tp_as_buffer = &picklebuf_as_buffer, .tp_methods = picklebuf_methods, + .tp_getset = picklebuf_getset, }; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 820fba90801818..0d979a1375cec6 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -2,6 +2,7 @@ #include "Python.h" #include "pycore_call.h" +#include "pycore_abstract.h" // _PyBuffer_MutableBufferGetter #include "pycore_code.h" // CO_FAST_FREE #include "pycore_compile.h" // _Py_Mangle() #include "pycore_initconfig.h" // _PyStatus_OK() @@ -6094,16 +6095,24 @@ type_add_getset(PyTypeObject *type) PyObject *dict = type->tp_dict; for (; gsp->name != NULL; gsp++) { - PyObject *descr = PyDescr_NewGetSet(type, gsp); - if (descr == NULL) { - return -1; - } - - if (PyDict_SetDefault(dict, PyDescr_NAME(descr), descr) == NULL) { + if (gsp->get == _PyBuffer_MutableBufferGetter) { + PyObject *name = PyUnicode_FromString(gsp->name); + if (PyDict_SetDefault(dict, name, Py_True) == NULL) { + Py_DECREF(name); + return -1; + } + Py_DECREF(name); + } else { + PyObject *descr = PyDescr_NewGetSet(type, gsp); + if (descr == NULL) { + return -1; + } + if (PyDict_SetDefault(dict, PyDescr_NAME(descr), descr) == NULL) { + Py_DECREF(descr); + return -1; + } Py_DECREF(descr); - return -1; } - Py_DECREF(descr); } return 0; } From 007fdc115383b42b78ddf28bdcd47714de9c53e7 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sun, 6 Nov 2022 06:47:23 -0800 Subject: [PATCH 08/35] Rip out __mutable_buffer__ --- Include/internal/pycore_abstract.h | 4 ---- Lib/_collections_abc.py | 14 +------------- Lib/test/test_collections.py | 11 +---------- Modules/_ctypes/_ctypes.c | 17 ++++------------- Modules/_io/bytesio.c | 8 +------- Modules/arraymodule.c | 2 -- Modules/mmapmodule.c | 2 -- Objects/abstract.c | 5 ----- Objects/bytearrayobject.c | 9 ++------- Objects/memoryobject.c | 3 +-- Objects/picklebufobject.c | 6 ------ Objects/typeobject.c | 30 ++++++++++-------------------- 12 files changed, 20 insertions(+), 91 deletions(-) diff --git a/Include/internal/pycore_abstract.h b/Include/internal/pycore_abstract.h index 6db26bf246ee7c..b1afb2dc7be65e 100644 --- a/Include/internal/pycore_abstract.h +++ b/Include/internal/pycore_abstract.h @@ -19,10 +19,6 @@ _PyIndex_Check(PyObject *obj) PyObject *_PyNumber_PowerNoMod(PyObject *lhs, PyObject *rhs); PyObject *_PyNumber_InPlacePowerNoMod(PyObject *lhs, PyObject *rhs); -PyAPI_FUNC(PyObject *) _PyBuffer_MutableBufferGetter(PyObject *instance, void *ignored); - -#define _PY_BUFFER_MUTABLE_BUFFER_GETSET { "__mutable_buffer__", _PyBuffer_MutableBufferGetter, NULL, NULL }, - #ifdef __cplusplus } #endif diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index c4b3e687315658..10ddcaf3e00134 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -49,7 +49,7 @@ def _f(): pass "Mapping", "MutableMapping", "MappingView", "KeysView", "ItemsView", "ValuesView", "Sequence", "MutableSequence", - "ByteString", "Buffer", "MutableBuffer", + "ByteString", "Buffer", ] # This module has been renamed from collections.abc to _collections_abc to @@ -454,18 +454,6 @@ def __subclasshook__(cls, C): return NotImplemented -class MutableBuffer(Buffer): - - __slots__ = () - __mutable_buffer__ = True - - @classmethod - def __subclasshook__(cls, C): - if cls is MutableBuffer: - return _check_methods(C, "__buffer__", "__mutable_buffer__") - return NotImplemented - - class _CallableGenericAlias(GenericAlias): """ Represent `Callable[argtypes, resulttype]`. diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index e1b6f2bafe0930..b257b66fcf6752 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -25,7 +25,7 @@ from collections.abc import Set, MutableSet from collections.abc import Mapping, MutableMapping, KeysView, ItemsView, ValuesView from collections.abc import Sequence, MutableSequence -from collections.abc import ByteString, Buffer, MutableBuffer +from collections.abc import ByteString, Buffer class TestUserObjects(unittest.TestCase): @@ -1953,15 +1953,6 @@ def test_Buffer(self): self.assertFalse(issubclass(sample, Buffer)) self.validate_abstract_methods(Buffer, '__buffer__') - def test_MutableBuffer(self): - for sample in [bytearray, memoryview]: - self.assertIsInstance(sample(b"x"), MutableBuffer) - self.assertTrue(issubclass(sample, MutableBuffer)) - for sample in [bytes, str, list, tuple]: - self.assertNotIsInstance(sample(), MutableBuffer) - self.assertFalse(issubclass(sample, MutableBuffer)) - self.validate_abstract_methods(MutableBuffer, '__buffer__', '__release_buffer__') - def test_MutableSequence(self): for sample in [tuple, str, bytes]: self.assertNotIsInstance(sample(), MutableSequence) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 0e89ca0e079274..f342b17abca698 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -125,7 +125,6 @@ bytes(cdata) #include "ctypes.h" #include "pycore_long.h" // _PyLong_GetZero() -#include "pycore_abstract.h" // _PY_BUFFER_MUTABLE_BUFFER_GETSET PyObject *PyExc_ArgError = NULL; @@ -2793,11 +2792,6 @@ static PyBufferProcs PyCData_as_buffer = { NULL, }; -static PyGetSetDef PyCData_getset[] = { - _PY_BUFFER_MUTABLE_BUFFER_GETSET - {NULL} -}; - /* * CData objects are mutable, so they cannot be hashable! */ @@ -2907,7 +2901,7 @@ PyTypeObject PyCData_Type = { 0, /* tp_iternext */ PyCData_methods, /* tp_methods */ PyCData_members, /* tp_members */ - PyCData_getset, /* tp_getset */ + 0, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ @@ -3330,7 +3324,6 @@ static PyGetSetDef PyCFuncPtr_getsets[] = { { "argtypes", (getter)PyCFuncPtr_get_argtypes, (setter)PyCFuncPtr_set_argtypes, "specify the argument types", NULL }, - _PY_BUFFER_MUTABLE_BUFFER_GETSET { NULL, NULL } }; @@ -4455,7 +4448,7 @@ static PyTypeObject Struct_Type = { 0, /* tp_iternext */ 0, /* tp_methods */ 0, /* tp_members */ - PyCData_getset, /* tp_getset */ + 0, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ @@ -4497,7 +4490,7 @@ static PyTypeObject Union_Type = { 0, /* tp_iternext */ 0, /* tp_methods */ 0, /* tp_members */ - PyCData_getset, /* tp_getset */ + 0, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ @@ -4817,7 +4810,7 @@ PyTypeObject PyCArray_Type = { 0, /* tp_iternext */ Array_methods, /* tp_methods */ 0, /* tp_members */ - PyCData_getset, /* tp_getset */ + 0, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ @@ -4948,7 +4941,6 @@ Simple_get_value(CDataObject *self, void *Py_UNUSED(ignored)) static PyGetSetDef Simple_getsets[] = { { "value", (getter)Simple_get_value, (setter)Simple_set_value, "current value", NULL }, - _PY_BUFFER_MUTABLE_BUFFER_GETSET { NULL, NULL } }; @@ -5192,7 +5184,6 @@ static PyGetSetDef Pointer_getsets[] = { { "contents", (getter)Pointer_get_contents, (setter)Pointer_set_contents, "the object this pointer points to (read-write)", NULL }, - _PY_BUFFER_MUTABLE_BUFFER_GETSET { NULL, NULL } }; diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index 20e3e6a0ef7435..930ef7e29dbcf6 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -1,5 +1,4 @@ #include "Python.h" -#include "pycore_abstract.h" // _PY_BUFFER_MUTABLE_BUFFER_GETSET #include "pycore_object.h" #include // offsetof() #include "_iomodule.h" @@ -1125,11 +1124,6 @@ static PyBufferProcs bytesiobuf_as_buffer = { (releasebufferproc) bytesiobuf_releasebuffer, }; -static PyGetSetDef bytesiobuf_getset[] = { - _PY_BUFFER_MUTABLE_BUFFER_GETSET - {NULL} -}; - Py_EXPORTED_SYMBOL PyTypeObject _PyBytesIOBuffer_Type = { PyVarObject_HEAD_INIT(NULL, 0) "_io._BytesIOBuffer", /*tp_name*/ @@ -1160,7 +1154,7 @@ Py_EXPORTED_SYMBOL PyTypeObject _PyBytesIOBuffer_Type = { 0, /*tp_iternext*/ 0, /*tp_methods*/ 0, /*tp_members*/ - bytesiobuf_getset, /*tp_getset*/ + 0, /*tp_getset*/ 0, /*tp_base*/ 0, /*tp_dict*/ 0, /*tp_descr_get*/ diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index ab690efd52dd9f..d60cf26788f5a6 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -9,7 +9,6 @@ #define PY_SSIZE_T_CLEAN #include "Python.h" -#include "pycore_abstract.h" // _PY_BUFFER_MUTABLE_BUFFER_GETSET #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_bytesobject.h" // _PyBytes_Repeat #include "structmember.h" // PyMemberDef @@ -2278,7 +2277,6 @@ static PyGetSetDef array_getsets [] = { "the typecode character used to create the array"}, {"itemsize", (getter) array_get_itemsize, NULL, "the size, in bytes, of one array item"}, - _PY_BUFFER_MUTABLE_BUFFER_GETSET {NULL} }; diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 85cb0a524fb227..fdce783fdec5e2 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -24,7 +24,6 @@ #define PY_SSIZE_T_CLEAN #include -#include "pycore_abstract.h" // _PY_BUFFER_MUTABLE_BUFFER_GETSET #include "pycore_bytesobject.h" // _PyBytes_Find() #include "pycore_fileutils.h" // _Py_stat_struct #include "structmember.h" // PyMemberDef @@ -888,7 +887,6 @@ static struct PyMethodDef mmap_object_methods[] = { static PyGetSetDef mmap_object_getset[] = { {"closed", (getter) mmap_closed_get, NULL, NULL}, - _PY_BUFFER_MUTABLE_BUFFER_GETSET {NULL} }; diff --git a/Objects/abstract.c b/Objects/abstract.c index ea40cbe30ba1b4..5d50491b2dd347 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -2967,8 +2967,3 @@ _Py_FreeCharPArray(char *const array[]) } PyMem_Free((void*)array); } - -PyObject * -_PyBuffer_MutableBufferGetter(PyObject *instance, void *ignored) { - Py_RETURN_TRUE; -} diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index a413deba623dab..b2962fd137d93e 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -2,7 +2,7 @@ #define PY_SSIZE_T_CLEAN #include "Python.h" -#include "pycore_abstract.h" // _PyIndex_Check(), _PY_BUFFER_MUTABLE_BUFFER_GETSET +#include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_bytes_methods.h" #include "pycore_bytesobject.h" #include "pycore_object.h" // _PyObject_GC_UNTRACK() @@ -2289,11 +2289,6 @@ Construct a mutable bytearray object from:\n\ static PyObject *bytearray_iter(PyObject *seq); -static PyGetSetDef bytearray_getset[] = { - _PY_BUFFER_MUTABLE_BUFFER_GETSET - {NULL} -}; - PyTypeObject PyByteArray_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "bytearray", @@ -2325,7 +2320,7 @@ PyTypeObject PyByteArray_Type = { 0, /* tp_iternext */ bytearray_methods, /* tp_methods */ 0, /* tp_members */ - bytearray_getset, /* tp_getset */ + 0, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index 6e414ee5f52427..db832aa5c9b22a 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -11,7 +11,7 @@ */ #include "Python.h" -#include "pycore_abstract.h" // _PyIndex_Check(), _PY_BUFFER_MUTABLE_BUFFER_GETSET +#include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_strhex.h" // _Py_strhex_with_sep() #include // offsetof() @@ -3186,7 +3186,6 @@ static PyGetSetDef memory_getsetlist[] = { {"c_contiguous", (getter)memory_c_contiguous, NULL, memory_c_contiguous_doc}, {"f_contiguous", (getter)memory_f_contiguous, NULL, memory_f_contiguous_doc}, {"contiguous", (getter)memory_contiguous, NULL, memory_contiguous_doc}, - _PY_BUFFER_MUTABLE_BUFFER_GETSET {NULL, NULL, NULL, NULL}, }; diff --git a/Objects/picklebufobject.c b/Objects/picklebufobject.c index 7ec04206fa47df..aaa852cfbb05b0 100644 --- a/Objects/picklebufobject.c +++ b/Objects/picklebufobject.c @@ -2,7 +2,6 @@ #define PY_SSIZE_T_CLEAN #include "Python.h" -#include "pycore_abstract.h" // _PY_BUFFER_MUTABLE_BUFFER_GETSET #include typedef struct { @@ -204,10 +203,6 @@ static PyMethodDef picklebuf_methods[] = { {NULL, NULL} }; -static PyGetSetDef picklebuf_getset[] = { - _PY_BUFFER_MUTABLE_BUFFER_GETSET - {NULL} -}; PyTypeObject PyPickleBuffer_Type = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "pickle.PickleBuffer", @@ -221,5 +216,4 @@ PyTypeObject PyPickleBuffer_Type = { .tp_weaklistoffset = offsetof(PyPickleBufferObject, weakreflist), .tp_as_buffer = &picklebuf_as_buffer, .tp_methods = picklebuf_methods, - .tp_getset = picklebuf_getset, }; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 9e3b6d81dddaac..18ffd3f26af77b 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -2,7 +2,6 @@ #include "Python.h" #include "pycore_call.h" -#include "pycore_abstract.h" // _PyBuffer_MutableBufferGetter #include "pycore_code.h" // CO_FAST_FREE #include "pycore_compile.h" // _Py_Mangle() #include "pycore_initconfig.h" // _PyStatus_OK() @@ -6196,24 +6195,15 @@ type_add_getset(PyTypeObject *type) PyObject *dict = type->tp_dict; for (; gsp->name != NULL; gsp++) { - if (gsp->get == _PyBuffer_MutableBufferGetter) { - PyObject *name = PyUnicode_FromString(gsp->name); - if (PyDict_SetDefault(dict, name, Py_True) == NULL) { - Py_DECREF(name); - return -1; - } - Py_DECREF(name); - } else { - PyObject *descr = PyDescr_NewGetSet(type, gsp); - if (descr == NULL) { - return -1; - } - if (PyDict_SetDefault(dict, PyDescr_NAME(descr), descr) == NULL) { - Py_DECREF(descr); - return -1; - } + PyObject *descr = PyDescr_NewGetSet(type, gsp); + if (descr == NULL) { + return -1; + } + if (PyDict_SetDefault(dict, PyDescr_NAME(descr), descr) == NULL) { Py_DECREF(descr); + return -1; } + Py_DECREF(descr); } return 0; } @@ -8565,7 +8555,7 @@ bufferwrapper_releasebuf(PyObject *self, Py_buffer *view) assert(PyMemoryView_Check(bw->mv)); Py_TYPE(bw->mv)->tp_as_buffer->bf_releasebuffer(bw->mv, view); - if (Py_TYPE(bw->obj)->tp_as_buffer != NULL + if (Py_TYPE(bw->obj)->tp_as_buffer != NULL && Py_TYPE(bw->obj)->tp_as_buffer->bf_releasebuffer != NULL) { Py_TYPE(bw->obj)->tp_as_buffer->bf_releasebuffer(bw->obj, view); } @@ -8589,7 +8579,7 @@ PyTypeObject _PyBufferWrapper_Type = { .tp_as_buffer = &bufferwrapper_as_buffer, }; -static int +static int slot_bf_getbuffer(PyObject *self, Py_buffer *buffer, int flags) { PyObject *flags_obj = PyLong_FromLong(flags); @@ -8837,7 +8827,7 @@ static slotdef slotdefs[] = { "Return a buffer object that exposes the underlying memory of the object."), BUFSLOT(__release_buffer__, bf_releasebuffer, slot_bf_releasebuffer, wrap_releasebuffer, "__release_buffer__($self, /)\n--\n\n" - "Release the buffer object that exposes the underlying memory of the object."), + "Release the buffer object that exposes the underlying memory of the object."), AMSLOT(__await__, am_await, slot_am_await, wrap_unaryfunc, "__await__($self, /)\n--\n\nReturn an iterator to be used in await expression."), From 46a9239265c62ac07bdc8127ad74995d0720d161 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 7 Nov 2022 20:14:38 -0800 Subject: [PATCH 09/35] __release_buffer__ calls mv.release() --- Objects/typeobject.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 18ffd3f26af77b..8daa92bf327cfc 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -7685,7 +7685,16 @@ wrap_releasebuffer(PyObject *self, PyObject *args, void *wrapped) "memoryview's buffer is not this object"); return NULL; } - (*func)(self, &mview->view); + PyObject *release = PyUnicode_FromString("release"); + if (release == NULL) { + return NULL; + } + PyObject *res = PyObject_CallMethodNoArgs((PyObject *)mview, release); + Py_DECREF(release); + if (res == NULL) { + return NULL; + } + Py_DECREF(res); Py_RETURN_NONE; } From be9bf456fb82dd9d15870a33ae0de23c04b4fb52 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 7 Nov 2022 20:51:53 -0800 Subject: [PATCH 10/35] additional test --- Lib/test/test_buffer.py | 16 ++++++++ Modules/_testcapimodule.c | 86 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+) diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index 9bee30bacd3b73..3e48cf48b791d3 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -4508,6 +4508,22 @@ def test_call_builtins(self): self.assertEqual(mv.tobytes(), b"hello") ba.__release_buffer__(mv) + @unittest.skipIf(_testcapi is None, "requires _testcapi") + def test_c_buffer(self): + buf = _testcapi.testBuf() + self.assertEqual(buf.references, 0) + mv = buf.__buffer__(0) + self.assertIsInstance(mv, memoryview) + self.assertEqual(mv.tobytes(), b"test") + self.assertEqual(buf.references, 1) + buf.__release_buffer__(mv) + self.assertEqual(buf.references, 0) + with self.assertRaises(ValueError): + mv.tobytes() + # Calling it again doesn't cause issues + buf.__release_buffer__(mv) + self.assertEqual(buf.references, 0) + if __name__ == "__main__": unittest.main() diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 0e09c97bea09a5..d61b6649118337 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -6804,6 +6804,85 @@ static PyTypeObject MyList_Type = { MyList_new, /* tp_new */ }; +typedef struct { + PyObject_HEAD + PyObject *obj; + long references; +} testBufObject; + +static PyObject * +testbuf_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + testBufObject *self = (testBufObject *)type->tp_alloc(type, 0); + if (self == NULL) { + return NULL; + } + self->obj = PyBytes_FromString("test"); + self->references = 0; + return (PyObject *)self; +} + +static int +testbuf_traverse(testBufObject *self, visitproc visit, void *arg) +{ + Py_VISIT(self->obj); + return 0; +} + +static int +testbuf_clear(testBufObject *self) +{ + Py_CLEAR(self->obj); + return 0; +} + +static void +testbuf_dealloc(testBufObject *self) +{ + PyObject_GC_UnTrack(self); + Py_XDECREF(self->obj); + Py_TYPE(self)->tp_free((PyObject *) self); +} + +static int +testbuf_getbuf(testBufObject *self, Py_buffer *view, int flags) +{ + int buf = PyObject_GetBuffer(self->obj, view, flags); + Py_SETREF(view->obj, Py_NewRef((PyObject *)self)); + self->references++; + return buf; +} + +static void +testbuf_releasebuf(testBufObject *self, Py_buffer *view) +{ + self->references--; + assert(self->references >= 0); +} + +static PyBufferProcs testbuf_as_buffer = { + .bf_getbuffer = (getbufferproc) testbuf_getbuf, + .bf_releasebuffer = (releasebufferproc) testbuf_releasebuf, +}; + +static struct PyMemberDef testbuf_members[] = { + {"references", T_LONG, offsetof(testBufObject, references), READONLY}, + {NULL}, +}; + +static PyTypeObject testBufType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "testBufType", + .tp_basicsize = sizeof(testBufObject), + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + .tp_new = testbuf_new, + .tp_dealloc = (destructor) testbuf_dealloc, + .tp_traverse = (traverseproc) testbuf_traverse, + .tp_clear = (inquiry) testbuf_clear, + .tp_as_buffer = &testbuf_as_buffer, + .tp_members = testbuf_members +}; + /* Test PEP 560 */ @@ -7042,6 +7121,13 @@ PyInit__testcapi(void) Py_INCREF(&MyList_Type); PyModule_AddObject(m, "MyList", (PyObject *)&MyList_Type); + if (PyType_Ready(&testBufType) < 0) { + return NULL; + } + if (PyModule_AddObjectRef(m, "testBuf", (PyObject *)&testBufType)) { + return NULL; + } + if (PyType_Ready(&GenericAlias_Type) < 0) return NULL; Py_INCREF(&GenericAlias_Type); From a6bf0e81c7cd6b6e086ba98c07f3baefc3cb6f9f Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 7 Nov 2022 20:58:28 -0800 Subject: [PATCH 11/35] undo stray change --- Objects/typeobject.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 8daa92bf327cfc..b0cf8fb814a245 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -6199,6 +6199,7 @@ type_add_getset(PyTypeObject *type) if (descr == NULL) { return -1; } + if (PyDict_SetDefault(dict, PyDescr_NAME(descr), descr) == NULL) { Py_DECREF(descr); return -1; From 04d0a420fabebc893cecf12abcb8245980ab1e4d Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sat, 17 Dec 2022 20:37:36 -0800 Subject: [PATCH 12/35] throw an error if already released --- Include/internal/pycore_global_objects_fini_generated.h | 2 ++ Include/internal/pycore_unicodeobject_generated.h | 4 ++++ Lib/test/test_buffer.py | 3 ++- Objects/typeobject.c | 6 +++++- 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 6aba2f19ebde4a..e9968a07ade0e8 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -594,6 +594,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__await__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__bases__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__bool__)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__buffer__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__build_class__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__builtins__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__bytes__)); @@ -693,6 +694,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__rdivmod__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__reduce__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__reduce_ex__)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__release_buffer__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__repr__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__reversed__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__rfloordiv__)); diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 7f407c0141b8a5..bd1e7196b964ac 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -66,6 +66,8 @@ _PyUnicode_InitStaticStrings(void) { PyUnicode_InternInPlace(&string); string = &_Py_ID(__bool__); PyUnicode_InternInPlace(&string); + string = &_Py_ID(__buffer__); + PyUnicode_InternInPlace(&string); string = &_Py_ID(__build_class__); PyUnicode_InternInPlace(&string); string = &_Py_ID(__builtins__); @@ -264,6 +266,8 @@ _PyUnicode_InitStaticStrings(void) { PyUnicode_InternInPlace(&string); string = &_Py_ID(__reduce_ex__); PyUnicode_InternInPlace(&string); + string = &_Py_ID(__release_buffer__); + PyUnicode_InternInPlace(&string); string = &_Py_ID(__repr__); PyUnicode_InternInPlace(&string); string = &_Py_ID(__reversed__); diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index 3e48cf48b791d3..8b2d7e12fce17b 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -4521,7 +4521,8 @@ def test_c_buffer(self): with self.assertRaises(ValueError): mv.tobytes() # Calling it again doesn't cause issues - buf.__release_buffer__(mv) + with self.assertRaises(ValueError): + buf.__release_buffer__(mv) self.assertEqual(buf.references, 0) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 95a740c94c9858..7f24503f6f714c 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -7624,7 +7624,6 @@ wrap_buffer(PyObject *self, PyObject *args, void *wrapped) static PyObject * wrap_releasebuffer(PyObject *self, PyObject *args, void *wrapped) { - releasebufferproc func = (releasebufferproc)wrapped; PyMemoryViewObject *mview; if (!check_num_args(args, 1)) { @@ -7638,6 +7637,11 @@ wrap_releasebuffer(PyObject *self, PyObject *args, void *wrapped) "memoryview's buffer is not this object"); return NULL; } + if (mview->flags & _Py_MEMORYVIEW_RELEASED) { + PyErr_SetString(PyExc_ValueError, + "memoryview's buffer has already been released"); + return NULL; + } PyObject *release = PyUnicode_FromString("release"); if (release == NULL) { return NULL; From 5f755fec0da074bf881c6deb7ea4af71e86cef0f Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 7 Mar 2023 17:35:00 -0800 Subject: [PATCH 13/35] Fix compiler warning --- Objects/typeobject.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index f09b3acbc39d35..96282cb2bad905 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -8579,7 +8579,7 @@ slot_bf_getbuffer(PyObject *self, Py_buffer *buffer, int flags) if (flags_obj == NULL) { return -1; } - PyObject *wrapper = NULL; + PyBufferWrapper *wrapper = NULL; PyObject *stack[2] = {self, flags_obj}; PyObject *ret = vectorcall_method(&_Py_ID(__buffer__), stack, 2); if (ret == NULL) { @@ -8600,11 +8600,11 @@ slot_bf_getbuffer(PyObject *self, Py_buffer *buffer, int flags) if (wrapper == NULL) { goto fail; } - ((PyBufferWrapper *)wrapper)->mv = ret; - ((PyBufferWrapper *)wrapper)->obj = Py_NewRef(self); + wrapper->mv = ret; + wrapper->obj = Py_NewRef(self); _PyObject_GC_TRACK(wrapper); - buffer->obj = wrapper; + buffer->obj = (PyObject *)wrapper; Py_DECREF(ret); Py_DECREF(flags_obj); return 0; From 0b02e63d23961ab1f67093415cb35ef34f8a82e8 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 7 Mar 2023 17:37:06 -0800 Subject: [PATCH 14/35] news --- .../2023-03-07-17-37-00.gh-issue-102500.RUSQhz.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-07-17-37-00.gh-issue-102500.RUSQhz.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-07-17-37-00.gh-issue-102500.RUSQhz.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-07-17-37-00.gh-issue-102500.RUSQhz.rst new file mode 100644 index 00000000000000..e03113ba05cd7d --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-03-07-17-37-00.gh-issue-102500.RUSQhz.rst @@ -0,0 +1,3 @@ +Make the buffer protocol accessible in Python code using the new +``__buffer__`` and ``__release_buffer__`` magic methods. See :pep:`688` for +details. Patch by Jelle Zijlstra. From 8e4db4300e3137fd608bc24bfcde3c0098a426e7 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 7 Mar 2023 18:19:13 -0800 Subject: [PATCH 15/35] fix some tests --- Lib/inspect.py | 1 + Lib/test/test_doctest.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/inspect.py b/Lib/inspect.py index 9ab7d61df3a630..86f6f499242c76 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -39,6 +39,7 @@ "Attribute", "BlockFinder", "BoundArguments", + "BufferFlags", "CORO_CLOSED", "CORO_CREATED", "CORO_RUNNING", diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index 3491d4cdb1c18b..542fcdb5cf6f66 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -707,7 +707,7 @@ def non_Python_modules(): r""" >>> import builtins >>> tests = doctest.DocTestFinder().find(builtins) - >>> 830 < len(tests) < 850 # approximate number of objects with docstrings + >>> 830 < len(tests) < 860 # approximate number of objects with docstrings True >>> real_tests = [t for t in tests if len(t.examples) > 0] >>> len(real_tests) # objects that actually have doctests From 6c863bcf3560563a736ebcefba12c9155a6f7869 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 9 Mar 2023 20:58:41 -0800 Subject: [PATCH 16/35] More tests. Add flags= argument to memoryview --- Lib/test/test_buffer.py | 35 +++++++++++ Modules/Setup.stdlib.in | 2 +- Modules/_testcapi/buffer.c | 97 +++++++++++++++++++++++++++++++ Modules/_testcapi/parts.h | 1 + Modules/_testcapimodule.c | 90 +--------------------------- Objects/clinic/memoryobject.c.h | 26 ++++++--- Objects/memoryobject.c | 8 ++- PCbuild/_testcapi.vcxproj | 1 + PCbuild/_testcapi.vcxproj.filters | 3 + 9 files changed, 164 insertions(+), 99 deletions(-) create mode 100644 Modules/_testcapi/buffer.c diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index 8b2d7e12fce17b..914cb58675b2b7 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -17,6 +17,7 @@ import unittest from test import support from test.support import os_helper +import inspect from itertools import permutations, product from random import randrange, sample, choice import warnings @@ -4502,6 +4503,40 @@ def __release_buffer__(self, buffer): self.assertEqual(mv.tobytes(), b"hello") self.assertFalse(wr.held) + def test_buffer_flags(self): + class PossiblyMutable: + def __init__(self, data, mutable) -> None: + self._data = bytearray(data) + self._mutable = mutable + + def __buffer__(self, flags): + if flags & inspect.BufferFlags.WRITABLE: + if not self._mutable: + raise RuntimeError("not mutable") + return memoryview(self._data) + else: + return memoryview(bytes(self._data)) + + mutable = PossiblyMutable(b"hello", True) + immutable = PossiblyMutable(b"hello", False) + with memoryview(mutable, flags=inspect.BufferFlags.WRITABLE) as mv: + self.assertEqual(mv.tobytes(), b"hello") + mv[0] = ord(b'x') + self.assertEqual(mv.tobytes(), b"xello") + with memoryview(immutable, flags=inspect.BufferFlags.SIMPLE) as mv: + self.assertEqual(mv.tobytes(), b"hello") + with self.assertRaises(TypeError): + mv[0] = ord(b'x') + self.assertEqual(mv.tobytes(), b"hello") + + with self.assertRaises(RuntimeError): + memoryview(immutable, flags=inspect.BufferFlags.WRITABLE) + with memoryview(immutable) as mv: + self.assertEqual(mv.tobytes(), b"hello") + with self.assertRaises(TypeError): + mv[0] = ord(b'x') + self.assertEqual(mv.tobytes(), b"hello") + def test_call_builtins(self): ba = bytearray(b"hello") mv = ba.__buffer__(0) diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index b12290d436cbeb..c5c231ccb33625 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -169,7 +169,7 @@ @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c -@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/unicode.c _testcapi/getargs.c _testcapi/pytime.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c +@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/unicode.c _testcapi/getargs.c _testcapi/pytime.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c # Some testing modules MUST be built as shared libraries. diff --git a/Modules/_testcapi/buffer.c b/Modules/_testcapi/buffer.c new file mode 100644 index 00000000000000..e625e941847c37 --- /dev/null +++ b/Modules/_testcapi/buffer.c @@ -0,0 +1,97 @@ +/* Test PEP 688 - Buffers */ + +#include "parts.h" + +#include "structmember.h" // PyMemberDef +#include // offsetof + +typedef struct { + PyObject_HEAD + PyObject *obj; + long references; +} testBufObject; + +static PyObject * +testbuf_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + testBufObject *self = (testBufObject *)type->tp_alloc(type, 0); + if (self == NULL) { + return NULL; + } + self->obj = PyBytes_FromString("test"); + self->references = 0; + return (PyObject *)self; +} + +static int +testbuf_traverse(testBufObject *self, visitproc visit, void *arg) +{ + Py_VISIT(self->obj); + return 0; +} + +static int +testbuf_clear(testBufObject *self) +{ + Py_CLEAR(self->obj); + return 0; +} + +static void +testbuf_dealloc(testBufObject *self) +{ + PyObject_GC_UnTrack(self); + Py_XDECREF(self->obj); + Py_TYPE(self)->tp_free((PyObject *) self); +} + +static int +testbuf_getbuf(testBufObject *self, Py_buffer *view, int flags) +{ + int buf = PyObject_GetBuffer(self->obj, view, flags); + Py_SETREF(view->obj, Py_NewRef((PyObject *)self)); + self->references++; + return buf; +} + +static void +testbuf_releasebuf(testBufObject *self, Py_buffer *view) +{ + self->references--; + assert(self->references >= 0); +} + +static PyBufferProcs testbuf_as_buffer = { + .bf_getbuffer = (getbufferproc) testbuf_getbuf, + .bf_releasebuffer = (releasebufferproc) testbuf_releasebuf, +}; + +static struct PyMemberDef testbuf_members[] = { + {"references", T_LONG, offsetof(testBufObject, references), READONLY}, + {NULL}, +}; + +static PyTypeObject testBufType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "testBufType", + .tp_basicsize = sizeof(testBufObject), + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + .tp_new = testbuf_new, + .tp_dealloc = (destructor) testbuf_dealloc, + .tp_traverse = (traverseproc) testbuf_traverse, + .tp_clear = (inquiry) testbuf_clear, + .tp_as_buffer = &testbuf_as_buffer, + .tp_members = testbuf_members +}; + +int +_PyTestCapi_Init_Buffer(PyObject *m) { + if (PyType_Ready(&testBufType) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "testBuf", (PyObject *)&testBufType)) { + return -1; + } + + return 0; +} diff --git a/Modules/_testcapi/parts.h b/Modules/_testcapi/parts.h index c8f31dc8e39fae..9348ab382344f1 100644 --- a/Modules/_testcapi/parts.h +++ b/Modules/_testcapi/parts.h @@ -38,6 +38,7 @@ int _PyTestCapi_Init_Float(PyObject *module); int _PyTestCapi_Init_Structmember(PyObject *module); int _PyTestCapi_Init_Exceptions(PyObject *module); int _PyTestCapi_Init_Code(PyObject *module); +int _PyTestCapi_Init_Buffer(PyObject *module); #ifdef LIMITED_API_AVAILABLE int _PyTestCapi_Init_VectorcallLimited(PyObject *module); diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index f9ede10dec8eed..0ba964d71bcde3 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3746,86 +3746,6 @@ static PyTypeObject MyList_Type = { MyList_new, /* tp_new */ }; -typedef struct { - PyObject_HEAD - PyObject *obj; - long references; -} testBufObject; - -static PyObject * -testbuf_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -{ - testBufObject *self = (testBufObject *)type->tp_alloc(type, 0); - if (self == NULL) { - return NULL; - } - self->obj = PyBytes_FromString("test"); - self->references = 0; - return (PyObject *)self; -} - -static int -testbuf_traverse(testBufObject *self, visitproc visit, void *arg) -{ - Py_VISIT(self->obj); - return 0; -} - -static int -testbuf_clear(testBufObject *self) -{ - Py_CLEAR(self->obj); - return 0; -} - -static void -testbuf_dealloc(testBufObject *self) -{ - PyObject_GC_UnTrack(self); - Py_XDECREF(self->obj); - Py_TYPE(self)->tp_free((PyObject *) self); -} - -static int -testbuf_getbuf(testBufObject *self, Py_buffer *view, int flags) -{ - int buf = PyObject_GetBuffer(self->obj, view, flags); - Py_SETREF(view->obj, Py_NewRef((PyObject *)self)); - self->references++; - return buf; -} - -static void -testbuf_releasebuf(testBufObject *self, Py_buffer *view) -{ - self->references--; - assert(self->references >= 0); -} - -static PyBufferProcs testbuf_as_buffer = { - .bf_getbuffer = (getbufferproc) testbuf_getbuf, - .bf_releasebuffer = (releasebufferproc) testbuf_releasebuf, -}; - -static struct PyMemberDef testbuf_members[] = { - {"references", T_LONG, offsetof(testBufObject, references), READONLY}, - {NULL}, -}; - -static PyTypeObject testBufType = { - PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "testBufType", - .tp_basicsize = sizeof(testBufObject), - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_new = testbuf_new, - .tp_dealloc = (destructor) testbuf_dealloc, - .tp_traverse = (traverseproc) testbuf_traverse, - .tp_clear = (inquiry) testbuf_clear, - .tp_as_buffer = &testbuf_as_buffer, - .tp_members = testbuf_members -}; - - /* Test PEP 560 */ typedef struct { @@ -4057,13 +3977,6 @@ PyInit__testcapi(void) Py_INCREF(&MyList_Type); PyModule_AddObject(m, "MyList", (PyObject *)&MyList_Type); - if (PyType_Ready(&testBufType) < 0) { - return NULL; - } - if (PyModule_AddObjectRef(m, "testBuf", (PyObject *)&testBufType)) { - return NULL; - } - if (PyType_Ready(&GenericAlias_Type) < 0) return NULL; Py_INCREF(&GenericAlias_Type); @@ -4172,6 +4085,9 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_Code(m) < 0) { return NULL; } + if (_PyTestCapi_Init_Buffer(m) < 0) { + return NULL; + } #ifndef LIMITED_API_AVAILABLE PyModule_AddObjectRef(m, "LIMITED_API_AVAILABLE", Py_False); diff --git a/Objects/clinic/memoryobject.c.h b/Objects/clinic/memoryobject.c.h index ff7b50bb114b05..1e7ee55ab960ae 100644 --- a/Objects/clinic/memoryobject.c.h +++ b/Objects/clinic/memoryobject.c.h @@ -9,13 +9,13 @@ preserve PyDoc_STRVAR(memoryview__doc__, -"memoryview(object)\n" +"memoryview(object, *, flags=PyBUF_FULL_RO)\n" "--\n" "\n" "Create a new memoryview object which references the given object."); static PyObject * -memoryview_impl(PyTypeObject *type, PyObject *object); +memoryview_impl(PyTypeObject *type, PyObject *object, int flags); static PyObject * memoryview(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -23,14 +23,14 @@ memoryview(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 1 + #define NUM_KEYWORDS 2 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(object), }, + .ob_item = { &_Py_ID(object), &_Py_ID(flags), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -39,24 +39,34 @@ memoryview(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"object", NULL}; + static const char * const _keywords[] = {"object", "flags", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "memoryview", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[1]; + PyObject *argsbuf[2]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1; PyObject *object; + int flags = PyBUF_FULL_RO; fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); if (!fastargs) { goto exit; } object = fastargs[0]; - return_value = memoryview_impl(type, object); + if (!noptargs) { + goto skip_optional_kwonly; + } + flags = _PyLong_AsInt(fastargs[1]); + if (flags == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional_kwonly: + return_value = memoryview_impl(type, object, flags); exit: return return_value; @@ -356,4 +366,4 @@ memoryview_hex(PyMemoryViewObject *self, PyObject *const *args, Py_ssize_t nargs exit: return return_value; } -/*[clinic end generated code: output=a832f2fc44e4794c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9de9bd412f8fea52 input=a9049054013a1b77]*/ diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index 9d69a615c537bc..6e8384ce4401bd 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -965,15 +965,17 @@ PyMemoryView_GetContiguous(PyObject *obj, int buffertype, char order) memoryview.__new__ object: object + * + flags: int(c_default='PyBUF_FULL_RO') = PyBUF_FULL_RO Create a new memoryview object which references the given object. [clinic start generated code]*/ static PyObject * -memoryview_impl(PyTypeObject *type, PyObject *object) -/*[clinic end generated code: output=7de78e184ed66db8 input=f04429eb0bdf8c6e]*/ +memoryview_impl(PyTypeObject *type, PyObject *object, int flags) +/*[clinic end generated code: output=f1c225d1ad24ec11 input=cab66ab4bb27f268]*/ { - return PyMemoryView_FromObject(object); + return PyMemoryView_FromObjectAndFlags(object, flags); } diff --git a/PCbuild/_testcapi.vcxproj b/PCbuild/_testcapi.vcxproj index 4cc184bfc1ac82..6be0aacf57074a 100644 --- a/PCbuild/_testcapi.vcxproj +++ b/PCbuild/_testcapi.vcxproj @@ -109,6 +109,7 @@ + diff --git a/PCbuild/_testcapi.vcxproj.filters b/PCbuild/_testcapi.vcxproj.filters index fbdaf04ce37cb1..cc4b22361b328a 100644 --- a/PCbuild/_testcapi.vcxproj.filters +++ b/PCbuild/_testcapi.vcxproj.filters @@ -57,6 +57,9 @@ Source Files + + Source Files + From 3b4b7d69587ef4a551798560b904246446cd9404 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Fri, 10 Mar 2023 16:54:51 -0800 Subject: [PATCH 17/35] Make memoryview flags arg private --- Lib/test/test_buffer.py | 6 +++--- Objects/clinic/memoryobject.c.h | 18 +++++++++--------- Objects/memoryobject.c | 8 ++++---- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index 914cb58675b2b7..655ecf42176df8 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -4519,18 +4519,18 @@ def __buffer__(self, flags): mutable = PossiblyMutable(b"hello", True) immutable = PossiblyMutable(b"hello", False) - with memoryview(mutable, flags=inspect.BufferFlags.WRITABLE) as mv: + with memoryview(mutable, _flags=inspect.BufferFlags.WRITABLE) as mv: self.assertEqual(mv.tobytes(), b"hello") mv[0] = ord(b'x') self.assertEqual(mv.tobytes(), b"xello") - with memoryview(immutable, flags=inspect.BufferFlags.SIMPLE) as mv: + with memoryview(immutable, _flags=inspect.BufferFlags.SIMPLE) as mv: self.assertEqual(mv.tobytes(), b"hello") with self.assertRaises(TypeError): mv[0] = ord(b'x') self.assertEqual(mv.tobytes(), b"hello") with self.assertRaises(RuntimeError): - memoryview(immutable, flags=inspect.BufferFlags.WRITABLE) + memoryview(immutable, _flags=inspect.BufferFlags.WRITABLE) with memoryview(immutable) as mv: self.assertEqual(mv.tobytes(), b"hello") with self.assertRaises(TypeError): diff --git a/Objects/clinic/memoryobject.c.h b/Objects/clinic/memoryobject.c.h index 1e7ee55ab960ae..26f96fc91b832e 100644 --- a/Objects/clinic/memoryobject.c.h +++ b/Objects/clinic/memoryobject.c.h @@ -9,13 +9,13 @@ preserve PyDoc_STRVAR(memoryview__doc__, -"memoryview(object, *, flags=PyBUF_FULL_RO)\n" +"memoryview(object, *, _flags=PyBUF_FULL_RO)\n" "--\n" "\n" "Create a new memoryview object which references the given object."); static PyObject * -memoryview_impl(PyTypeObject *type, PyObject *object, int flags); +memoryview_impl(PyTypeObject *type, PyObject *object, int _flags); static PyObject * memoryview(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -30,7 +30,7 @@ memoryview(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(object), &_Py_ID(flags), }, + .ob_item = { &_Py_ID(object), &_Py_ID(_flags), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -39,7 +39,7 @@ memoryview(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"object", "flags", NULL}; + static const char * const _keywords[] = {"object", "_flags", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "memoryview", @@ -51,7 +51,7 @@ memoryview(PyTypeObject *type, PyObject *args, PyObject *kwargs) Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1; PyObject *object; - int flags = PyBUF_FULL_RO; + int _flags = PyBUF_FULL_RO; fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); if (!fastargs) { @@ -61,12 +61,12 @@ memoryview(PyTypeObject *type, PyObject *args, PyObject *kwargs) if (!noptargs) { goto skip_optional_kwonly; } - flags = _PyLong_AsInt(fastargs[1]); - if (flags == -1 && PyErr_Occurred()) { + _flags = _PyLong_AsInt(fastargs[1]); + if (_flags == -1 && PyErr_Occurred()) { goto exit; } skip_optional_kwonly: - return_value = memoryview_impl(type, object, flags); + return_value = memoryview_impl(type, object, _flags); exit: return return_value; @@ -366,4 +366,4 @@ memoryview_hex(PyMemoryViewObject *self, PyObject *const *args, Py_ssize_t nargs exit: return return_value; } -/*[clinic end generated code: output=9de9bd412f8fea52 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=392fda9a93c25dd2 input=a9049054013a1b77]*/ diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index 6e8384ce4401bd..f119bb5d36d6d1 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -966,16 +966,16 @@ memoryview.__new__ object: object * - flags: int(c_default='PyBUF_FULL_RO') = PyBUF_FULL_RO + _flags: int(c_default='PyBUF_FULL_RO') = PyBUF_FULL_RO Create a new memoryview object which references the given object. [clinic start generated code]*/ static PyObject * -memoryview_impl(PyTypeObject *type, PyObject *object, int flags) -/*[clinic end generated code: output=f1c225d1ad24ec11 input=cab66ab4bb27f268]*/ +memoryview_impl(PyTypeObject *type, PyObject *object, int _flags) +/*[clinic end generated code: output=80388b83c45dafac input=2ea9a227b7f350e8]*/ { - return PyMemoryView_FromObjectAndFlags(object, flags); + return PyMemoryView_FromObjectAndFlags(object, _flags); } From f2950127b84c1888f7db07b56c71d2be0f377064 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Fri, 10 Mar 2023 17:05:59 -0800 Subject: [PATCH 18/35] regen global objects --- Include/internal/pycore_global_objects_fini_generated.h | 1 + Include/internal/pycore_global_strings.h | 1 + Include/internal/pycore_runtime_init_generated.h | 1 + Include/internal/pycore_unicodeobject_generated.h | 2 ++ 4 files changed, 5 insertions(+) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 87d2700f991f2c..f60d005dad71a6 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -751,6 +751,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_finalizing)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_find_and_load)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_fix_up_module)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_flags)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_flags_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_get_sourcefile)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_handle_fromlist)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index fb9fefef0db37a..2f431527d69d9f 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -237,6 +237,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(_finalizing) STRUCT_FOR_ID(_find_and_load) STRUCT_FOR_ID(_fix_up_module) + STRUCT_FOR_ID(_flags) STRUCT_FOR_ID(_flags_) STRUCT_FOR_ID(_get_sourcefile) STRUCT_FOR_ID(_handle_fromlist) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 77f8f464ad604c..f6d1e2c9095ebd 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -743,6 +743,7 @@ extern "C" { INIT_ID(_finalizing), \ INIT_ID(_find_and_load), \ INIT_ID(_fix_up_module), \ + INIT_ID(_flags), \ INIT_ID(_flags_), \ INIT_ID(_get_sourcefile), \ INIT_ID(_handle_fromlist), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index f59f0e716d859c..e6e861f5d4e9e0 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -380,6 +380,8 @@ _PyUnicode_InitStaticStrings(void) { PyUnicode_InternInPlace(&string); string = &_Py_ID(_fix_up_module); PyUnicode_InternInPlace(&string); + string = &_Py_ID(_flags); + PyUnicode_InternInPlace(&string); string = &_Py_ID(_flags_); PyUnicode_InternInPlace(&string); string = &_Py_ID(_get_sourcefile); From b5ea9088b8c146faddb28f4bd5d4417e1e0124c5 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 4 Apr 2023 18:13:13 -0700 Subject: [PATCH 19/35] Ignore new C globals for now --- Objects/typeobject.c | 2 +- Tools/c-analyzer/cpython/globals-to-fix.tsv | 1 + Tools/c-analyzer/cpython/ignored.tsv | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 358d1a93eeebbf..b99a008ca65540 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -8587,7 +8587,7 @@ static PyBufferProcs bufferwrapper_as_buffer = { }; -PyTypeObject _PyBufferWrapper_Type = { +static PyTypeObject _PyBufferWrapper_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) .tp_name = "_buffer_wrapper", .tp_basicsize = sizeof(PyBufferWrapper), diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 0620c7e13925b5..50a32d04b2d285 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -86,6 +86,7 @@ Objects/sliceobject.c - PyEllipsis_Type - Objects/sliceobject.c - PySlice_Type - Objects/tupleobject.c - PyTupleIter_Type - Objects/tupleobject.c - PyTuple_Type - +Objects/typeobject.c - _PyBufferWrapper_Type - Objects/typeobject.c - PyBaseObject_Type - Objects/typeobject.c - PySuper_Type - Objects/typeobject.c - PyType_Type - diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index a8ba88efc732fb..58dec8f6bb8556 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -403,6 +403,7 @@ Modules/_testbuffer.c ndarray_memoryview_from_buffer strides - Modules/_testbuffer.c ndarray_memoryview_from_buffer suboffsets - Modules/_testbuffer.c ndarray_push kwlist - Modules/_testbuffer.c staticarray_init kwlist - +Modules/_testcapi/buffer.c - testBufType - Modules/_testcapi/code.c get_code_extra_index key - Modules/_testcapi/datetime.c - test_run_counter - Modules/_testcapi/exceptions.c - PyRecursingInfinitelyError_Type - From b22bfa95ed7a9220d8767853db3563e9505d09ac Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 4 Apr 2023 18:14:15 -0700 Subject: [PATCH 20/35] not static (should not have committed this) --- Objects/typeobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index b99a008ca65540..358d1a93eeebbf 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -8587,7 +8587,7 @@ static PyBufferProcs bufferwrapper_as_buffer = { }; -static PyTypeObject _PyBufferWrapper_Type = { +PyTypeObject _PyBufferWrapper_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) .tp_name = "_buffer_wrapper", .tp_basicsize = sizeof(PyBufferWrapper), From 69e8f7c9b6bddb0188e3dba4834b10b82aac8acd Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 5 Apr 2023 06:57:20 -0700 Subject: [PATCH 21/35] Use tabs not spaces --- Tools/c-analyzer/cpython/globals-to-fix.tsv | 2 +- Tools/c-analyzer/cpython/ignored.tsv | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 50a32d04b2d285..79ca4dc3e2969d 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -86,7 +86,7 @@ Objects/sliceobject.c - PyEllipsis_Type - Objects/sliceobject.c - PySlice_Type - Objects/tupleobject.c - PyTupleIter_Type - Objects/tupleobject.c - PyTuple_Type - -Objects/typeobject.c - _PyBufferWrapper_Type - +Objects/typeobject.c - _PyBufferWrapper_Type - Objects/typeobject.c - PyBaseObject_Type - Objects/typeobject.c - PySuper_Type - Objects/typeobject.c - PyType_Type - diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 58dec8f6bb8556..f3f8f831c48044 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -403,7 +403,7 @@ Modules/_testbuffer.c ndarray_memoryview_from_buffer strides - Modules/_testbuffer.c ndarray_memoryview_from_buffer suboffsets - Modules/_testbuffer.c ndarray_push kwlist - Modules/_testbuffer.c staticarray_init kwlist - -Modules/_testcapi/buffer.c - testBufType - +Modules/_testcapi/buffer.c - testBufType - Modules/_testcapi/code.c get_code_extra_index key - Modules/_testcapi/datetime.c - test_run_counter - Modules/_testcapi/exceptions.c - PyRecursingInfinitelyError_Type - From 4ca7a7c5f97ffd20eee255abeddb7650dd67cee4 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sun, 9 Apr 2023 21:17:32 -0700 Subject: [PATCH 22/35] Address Kumar's feedback --- .../pycore_global_objects_fini_generated.h | 1 + Include/internal/pycore_global_strings.h | 1 + .../internal/pycore_runtime_init_generated.h | 1 + .../internal/pycore_unicodeobject_generated.h | 3 +++ Modules/_testcapi/buffer.c | 9 +++++-- Objects/typeobject.c | 25 +++++++++---------- 6 files changed, 25 insertions(+), 15 deletions(-) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 8294f5764c83b4..aa8055bc3ec370 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -1123,6 +1123,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(reducer_override)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(registry)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(rel_tol)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(release)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(reload)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(repl)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(replace)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 278c69ca687327..3babba2196753b 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -609,6 +609,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(reducer_override) STRUCT_FOR_ID(registry) STRUCT_FOR_ID(rel_tol) + STRUCT_FOR_ID(release) STRUCT_FOR_ID(reload) STRUCT_FOR_ID(repl) STRUCT_FOR_ID(replace) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 62e0575fbcdbbe..4a8f280be337af 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -1115,6 +1115,7 @@ extern "C" { INIT_ID(reducer_override), \ INIT_ID(registry), \ INIT_ID(rel_tol), \ + INIT_ID(release), \ INIT_ID(reload), \ INIT_ID(repl), \ INIT_ID(replace), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 168e6147345c3c..f562a8c7bc147a 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -1680,6 +1680,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(rel_tol); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(release); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(reload); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Modules/_testcapi/buffer.c b/Modules/_testcapi/buffer.c index e625e941847c37..29ca2c7fd91448 100644 --- a/Modules/_testcapi/buffer.c +++ b/Modules/_testcapi/buffer.c @@ -8,17 +8,22 @@ typedef struct { PyObject_HEAD PyObject *obj; - long references; + Py_ssize_t references; } testBufObject; static PyObject * testbuf_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { + PyObject *obj = PyBytes_FromString("test"); + if (obj == NULL) { + return NULL; + } testBufObject *self = (testBufObject *)type->tp_alloc(type, 0); if (self == NULL) { + Py_DECREF(obj); return NULL; } - self->obj = PyBytes_FromString("test"); + self->obj = obj; self->references = 0; return (PyObject *)self; } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 358d1a93eeebbf..3d0c91e1cdd5ab 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -7665,28 +7665,32 @@ wrap_descr_delete(PyObject *self, PyObject *args, void *wrapped) static PyObject * wrap_buffer(PyObject *self, PyObject *args, void *wrapped) { - int flags = 0; + PyObject *arg = NULL; - if (!check_num_args(args, 1)) { + if (!PyArg_UnpackTuple(args, "", 1, 1, &arg)) { return NULL; } - if (!PyArg_ParseTuple(args, "i", &flags)) { + Py_ssize_t flags = PyNumber_AsSsize_t(arg, PyExc_OverflowError); + if (flags == -1 && PyErr_Occurred()) { return NULL; } + return PyMemoryView_FromObjectAndFlags(self, flags); } static PyObject * wrap_releasebuffer(PyObject *self, PyObject *args, void *wrapped) { - PyMemoryViewObject *mview; - - if (!check_num_args(args, 1)) { + PyObject *arg = NULL; + if (!PyArg_UnpackTuple(args, "", 1, 1, &arg)) { return NULL; } - if (!PyArg_ParseTuple(args, "O!", &PyMemoryView_Type, &mview)) { + if (!PyMemoryView_Check(arg)) { + PyErr_SetString(PyExc_TypeError, + "expected a memoryview object"); return NULL; } + PyMemoryViewObject *mview = (PyMemoryViewObject *)arg; if (mview->view.obj != self) { PyErr_SetString(PyExc_ValueError, "memoryview's buffer is not this object"); @@ -7697,12 +7701,7 @@ wrap_releasebuffer(PyObject *self, PyObject *args, void *wrapped) "memoryview's buffer has already been released"); return NULL; } - PyObject *release = PyUnicode_FromString("release"); - if (release == NULL) { - return NULL; - } - PyObject *res = PyObject_CallMethodNoArgs((PyObject *)mview, release); - Py_DECREF(release); + PyObject *res = PyObject_CallMethodNoArgs((PyObject *)mview, &_Py_ID(release)); if (res == NULL) { return NULL; } From a70e12d3206d1b76ea4069c82c9d437e3b84c468 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 12 Apr 2023 17:57:08 -0700 Subject: [PATCH 23/35] Address another piece of feedback --- Objects/typeobject.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 9d0c67a1f5a969..b99a45f2aa77a1 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -8545,12 +8545,10 @@ typedef struct _PyBufferWrapper { } PyBufferWrapper; static int -bufferwrapper_traverse(PyObject *self, visitproc visit, void *arg) +bufferwrapper_traverse(PyBufferWrapper *self, visitproc visit, void *arg) { - PyBufferWrapper *bw = (PyBufferWrapper *)self; - - Py_VISIT(bw->mv); - Py_VISIT(bw->obj); + Py_VISIT(self->mv); + Py_VISIT(self->obj); return 0; } @@ -8590,7 +8588,7 @@ PyTypeObject _PyBufferWrapper_Type = { .tp_alloc = PyType_GenericAlloc, .tp_new = PyType_GenericNew, .tp_free = PyObject_GC_Del, - .tp_traverse = bufferwrapper_traverse, + .tp_traverse = (traverseproc)bufferwrapper_traverse, .tp_dealloc = bufferwrapper_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, .tp_as_buffer = &bufferwrapper_as_buffer, From 96c9253833528dcdfb42e277a23ecab826c5cb36 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 12 Apr 2023 18:10:50 -0700 Subject: [PATCH 24/35] Use a classmethod instead of a new arg to the memoryview constructor --- Lib/test/test_buffer.py | 6 +-- Objects/clinic/memoryobject.c.h | 78 +++++++++++++++++++++++++++------ Objects/memoryobject.c | 25 +++++++++-- 3 files changed, 88 insertions(+), 21 deletions(-) diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index 655ecf42176df8..4f9e655f2612ff 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -4519,18 +4519,18 @@ def __buffer__(self, flags): mutable = PossiblyMutable(b"hello", True) immutable = PossiblyMutable(b"hello", False) - with memoryview(mutable, _flags=inspect.BufferFlags.WRITABLE) as mv: + with memoryview._from_flags(mutable, inspect.BufferFlags.WRITABLE) as mv: self.assertEqual(mv.tobytes(), b"hello") mv[0] = ord(b'x') self.assertEqual(mv.tobytes(), b"xello") - with memoryview(immutable, _flags=inspect.BufferFlags.SIMPLE) as mv: + with memoryview._from_flags(immutable, inspect.BufferFlags.SIMPLE) as mv: self.assertEqual(mv.tobytes(), b"hello") with self.assertRaises(TypeError): mv[0] = ord(b'x') self.assertEqual(mv.tobytes(), b"hello") with self.assertRaises(RuntimeError): - memoryview(immutable, _flags=inspect.BufferFlags.WRITABLE) + memoryview_from_flags(immutable, inspect.BufferFlags.WRITABLE) with memoryview(immutable) as mv: self.assertEqual(mv.tobytes(), b"hello") with self.assertRaises(TypeError): diff --git a/Objects/clinic/memoryobject.c.h b/Objects/clinic/memoryobject.c.h index 26f96fc91b832e..ead1af4f3e8619 100644 --- a/Objects/clinic/memoryobject.c.h +++ b/Objects/clinic/memoryobject.c.h @@ -9,13 +9,13 @@ preserve PyDoc_STRVAR(memoryview__doc__, -"memoryview(object, *, _flags=PyBUF_FULL_RO)\n" +"memoryview(object)\n" "--\n" "\n" "Create a new memoryview object which references the given object."); static PyObject * -memoryview_impl(PyTypeObject *type, PyObject *object, int _flags); +memoryview_impl(PyTypeObject *type, PyObject *object); static PyObject * memoryview(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -23,14 +23,14 @@ memoryview(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 1 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(object), &_Py_ID(_flags), }, + .ob_item = { &_Py_ID(object), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -39,34 +39,84 @@ memoryview(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"object", "_flags", NULL}; + static const char * const _keywords[] = {"object", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "memoryview", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[1]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); - Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1; PyObject *object; - int _flags = PyBUF_FULL_RO; fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); if (!fastargs) { goto exit; } object = fastargs[0]; - if (!noptargs) { - goto skip_optional_kwonly; + return_value = memoryview_impl(type, object); + +exit: + return return_value; +} + +PyDoc_STRVAR(memoryview__from_flags__doc__, +"_from_flags($type, /, object, _flags)\n" +"--\n" +"\n" +"Create a new memoryview object which references the given object."); + +#define MEMORYVIEW__FROM_FLAGS_METHODDEF \ + {"_from_flags", _PyCFunction_CAST(memoryview__from_flags), METH_FASTCALL|METH_KEYWORDS|METH_CLASS, memoryview__from_flags__doc__}, + +static PyObject * +memoryview__from_flags_impl(PyTypeObject *type, PyObject *object, int _flags); + +static PyObject * +memoryview__from_flags(PyTypeObject *type, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(object), &_Py_ID(_flags), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"object", "_flags", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "_from_flags", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *object; + int _flags; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + if (!args) { + goto exit; } - _flags = _PyLong_AsInt(fastargs[1]); + object = args[0]; + _flags = _PyLong_AsInt(args[1]); if (_flags == -1 && PyErr_Occurred()) { goto exit; } -skip_optional_kwonly: - return_value = memoryview_impl(type, object, _flags); + return_value = memoryview__from_flags_impl(type, object, _flags); exit: return return_value; @@ -366,4 +416,4 @@ memoryview_hex(PyMemoryViewObject *self, PyObject *const *args, Py_ssize_t nargs exit: return return_value; } -/*[clinic end generated code: output=392fda9a93c25dd2 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=31ed11a9a4ac95c4 input=a9049054013a1b77]*/ diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index f119bb5d36d6d1..9117c3236baeb7 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -965,15 +965,31 @@ PyMemoryView_GetContiguous(PyObject *obj, int buffertype, char order) memoryview.__new__ object: object - * - _flags: int(c_default='PyBUF_FULL_RO') = PyBUF_FULL_RO Create a new memoryview object which references the given object. [clinic start generated code]*/ static PyObject * -memoryview_impl(PyTypeObject *type, PyObject *object, int _flags) -/*[clinic end generated code: output=80388b83c45dafac input=2ea9a227b7f350e8]*/ +memoryview_impl(PyTypeObject *type, PyObject *object) +/*[clinic end generated code: output=7de78e184ed66db8 input=f04429eb0bdf8c6e]*/ +{ + return PyMemoryView_FromObject(object); +} + + +/*[clinic input] +@classmethod +memoryview._from_flags + + object: object + _flags: int + +Create a new memoryview object which references the given object. +[clinic start generated code]*/ + +static PyObject * +memoryview__from_flags_impl(PyTypeObject *type, PyObject *object, int _flags) +/*[clinic end generated code: output=bc1f4ba7d9b64525 input=860e7832e8edb1e5]*/ { return PyMemoryView_FromObjectAndFlags(object, _flags); } @@ -3191,6 +3207,7 @@ static PyMethodDef memory_methods[] = { MEMORYVIEW_TOLIST_METHODDEF MEMORYVIEW_CAST_METHODDEF MEMORYVIEW_TOREADONLY_METHODDEF + MEMORYVIEW__FROM_FLAGS_METHODDEF {"__enter__", memory_enter, METH_NOARGS, NULL}, {"__exit__", memory_exit, METH_VARARGS, NULL}, {NULL, NULL} From 1f7f7a090be30a2806976385da3c050b5afe2ef1 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 13 Apr 2023 12:05:28 -0700 Subject: [PATCH 25/35] fix typo --- Lib/test/test_buffer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index 4f9e655f2612ff..e0b3e81af63071 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -4530,7 +4530,7 @@ def __buffer__(self, flags): self.assertEqual(mv.tobytes(), b"hello") with self.assertRaises(RuntimeError): - memoryview_from_flags(immutable, inspect.BufferFlags.WRITABLE) + memoryview._from_flags(immutable, inspect.BufferFlags.WRITABLE) with memoryview(immutable) as mv: self.assertEqual(mv.tobytes(), b"hello") with self.assertRaises(TypeError): From 33691eab4e61bea7e55f2cdb52f1bfe71ab8a34c Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 26 Apr 2023 06:22:49 -0600 Subject: [PATCH 26/35] Add Py_SAFE_DOWNCAST --- Include/internal/pycore_global_objects_fini_generated.h | 3 +++ Include/internal/pycore_global_strings.h | 3 +++ Include/internal/pycore_runtime_init_generated.h | 3 +++ Include/internal/pycore_unicodeobject_generated.h | 9 +++++++++ Objects/typeobject.c | 2 +- 5 files changed, 19 insertions(+), 1 deletion(-) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index b157724b8ab30a..a79ea65b5bf0dd 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -780,8 +780,10 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(after_in_child)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(after_in_parent)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(aggregate_class)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(alias)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(append)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(argdefs)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(args)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(arguments)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(argv)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(as_integer_ratio)); @@ -1076,6 +1078,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(optimize)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(options)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(order)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(origin)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(out_fd)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(outgoing)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(overlapped)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 3babba2196753b..34f89e3c25d49a 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -268,8 +268,10 @@ struct _Py_global_strings { STRUCT_FOR_ID(after_in_child) STRUCT_FOR_ID(after_in_parent) STRUCT_FOR_ID(aggregate_class) + STRUCT_FOR_ID(alias) STRUCT_FOR_ID(append) STRUCT_FOR_ID(argdefs) + STRUCT_FOR_ID(args) STRUCT_FOR_ID(arguments) STRUCT_FOR_ID(argv) STRUCT_FOR_ID(as_integer_ratio) @@ -564,6 +566,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(optimize) STRUCT_FOR_ID(options) STRUCT_FOR_ID(order) + STRUCT_FOR_ID(origin) STRUCT_FOR_ID(out_fd) STRUCT_FOR_ID(outgoing) STRUCT_FOR_ID(overlapped) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 4a8f280be337af..c7c61ebbd29cbd 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -774,8 +774,10 @@ extern "C" { INIT_ID(after_in_child), \ INIT_ID(after_in_parent), \ INIT_ID(aggregate_class), \ + INIT_ID(alias), \ INIT_ID(append), \ INIT_ID(argdefs), \ + INIT_ID(args), \ INIT_ID(arguments), \ INIT_ID(argv), \ INIT_ID(as_integer_ratio), \ @@ -1070,6 +1072,7 @@ extern "C" { INIT_ID(optimize), \ INIT_ID(options), \ INIT_ID(order), \ + INIT_ID(origin), \ INIT_ID(out_fd), \ INIT_ID(outgoing), \ INIT_ID(overlapped), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index f562a8c7bc147a..43578138c8ba5b 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -657,12 +657,18 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(aggregate_class); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(alias); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(append); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(argdefs); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(args); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(arguments); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1545,6 +1551,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(order); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(origin); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(out_fd); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 73bd3e3b31d594..0ecad2341fbca4 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -7698,7 +7698,7 @@ wrap_buffer(PyObject *self, PyObject *args, void *wrapped) return NULL; } - return PyMemoryView_FromObjectAndFlags(self, flags); + return PyMemoryView_FromObjectAndFlags(self, Py_SAFE_DOWNCAST(flags, Py_ssize_t, int)); } static PyObject * From 8f5073f400d35df3eb4b188257344b5289866f16 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 26 Apr 2023 15:42:00 -0600 Subject: [PATCH 27/35] Add some test cases (thanks Shantanu) --- Lib/test/test_buffer.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index c09d923274d528..6a2b53892280bc 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -4456,6 +4456,12 @@ def __buffer__(self, flags): self.assertRaises(TypeError, memoryview, MustReturnMV()) + class NoBytesEither: + def __buffer__(self, flags): + return b"hello" + + self.assertRaises(TypeError, memoryview, NoBytesEither()) + class WrongArity: def __buffer__(self): return memoryview(b"hello") @@ -4525,6 +4531,11 @@ def __buffer__(self, flags): self.assertEqual(mv.tobytes(), b"hello") mv[0] = ord(b'x') self.assertEqual(mv.tobytes(), b"xello") + with memoryview._from_flags(mutable, inspect.BufferFlags.SIMPLE) as mv: + self.assertEqual(mv.tobytes(), b"xello") + with self.assertRaises(TypeError): + mv[0] = ord(b'h') + self.assertEqual(mv.tobytes(), b"xello") with memoryview._from_flags(immutable, inspect.BufferFlags.SIMPLE) as mv: self.assertEqual(mv.tobytes(), b"hello") with self.assertRaises(TypeError): From cecb6a58027cd6b73519cfb97da5a34ce9d19f48 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Fri, 28 Apr 2023 20:44:29 -0700 Subject: [PATCH 28/35] Remove spurious global strings --- Include/internal/pycore_global_objects_fini_generated.h | 3 --- Include/internal/pycore_global_strings.h | 3 --- Include/internal/pycore_runtime_init_generated.h | 3 --- Include/internal/pycore_unicodeobject_generated.h | 9 --------- 4 files changed, 18 deletions(-) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index ae17f28c6b9e60..f3acefd5aca13e 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -780,10 +780,8 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(after_in_child)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(after_in_parent)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(aggregate_class)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(alias)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(append)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(argdefs)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(args)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(arguments)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(argv)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(as_integer_ratio)); @@ -1079,7 +1077,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(optimize)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(options)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(order)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(origin)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(out_fd)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(outgoing)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(overlapped)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 033faa9752347e..e250528c91379b 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -268,10 +268,8 @@ struct _Py_global_strings { STRUCT_FOR_ID(after_in_child) STRUCT_FOR_ID(after_in_parent) STRUCT_FOR_ID(aggregate_class) - STRUCT_FOR_ID(alias) STRUCT_FOR_ID(append) STRUCT_FOR_ID(argdefs) - STRUCT_FOR_ID(args) STRUCT_FOR_ID(arguments) STRUCT_FOR_ID(argv) STRUCT_FOR_ID(as_integer_ratio) @@ -567,7 +565,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(optimize) STRUCT_FOR_ID(options) STRUCT_FOR_ID(order) - STRUCT_FOR_ID(origin) STRUCT_FOR_ID(out_fd) STRUCT_FOR_ID(outgoing) STRUCT_FOR_ID(overlapped) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index ecf13a054bbd7e..fcf12e28476ae6 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -774,10 +774,8 @@ extern "C" { INIT_ID(after_in_child), \ INIT_ID(after_in_parent), \ INIT_ID(aggregate_class), \ - INIT_ID(alias), \ INIT_ID(append), \ INIT_ID(argdefs), \ - INIT_ID(args), \ INIT_ID(arguments), \ INIT_ID(argv), \ INIT_ID(as_integer_ratio), \ @@ -1073,7 +1071,6 @@ extern "C" { INIT_ID(optimize), \ INIT_ID(options), \ INIT_ID(order), \ - INIT_ID(origin), \ INIT_ID(out_fd), \ INIT_ID(outgoing), \ INIT_ID(overlapped), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index d2fd3382c78e0c..c5720add585ea9 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -657,18 +657,12 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(aggregate_class); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(alias); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(append); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(argdefs); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(args); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(arguments); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1554,9 +1548,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(order); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(origin); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(out_fd); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); From 0f77bbb8331e057f33062a6d18a16215df9faa34 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Thu, 4 May 2023 16:52:17 +0530 Subject: [PATCH 29/35] fixup global objects --- Tools/build/generate_global_objects.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Tools/build/generate_global_objects.py b/Tools/build/generate_global_objects.py index c27817702bf97d..ded19ee489e79b 100644 --- a/Tools/build/generate_global_objects.py +++ b/Tools/build/generate_global_objects.py @@ -121,6 +121,8 @@ '__xor__', '__divmod__', '__rdivmod__', + '__buffer__', + '__release_buffer__', ] NON_GENERATED_IMMORTAL_OBJECTS = [ From 61f54ced1d015c43a6d5e4223b6d1aa43356beb6 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Thu, 4 May 2023 17:02:22 +0530 Subject: [PATCH 30/35] minor fixes --- Modules/_testcapi/buffer.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_testcapi/buffer.c b/Modules/_testcapi/buffer.c index 29ca2c7fd91448..aff9a477eff57e 100644 --- a/Modules/_testcapi/buffer.c +++ b/Modules/_testcapi/buffer.c @@ -54,7 +54,7 @@ static int testbuf_getbuf(testBufObject *self, Py_buffer *view, int flags) { int buf = PyObject_GetBuffer(self->obj, view, flags); - Py_SETREF(view->obj, Py_NewRef((PyObject *)self)); + Py_SETREF(view->obj, Py_NewRef(self)); self->references++; return buf; } @@ -72,7 +72,7 @@ static PyBufferProcs testbuf_as_buffer = { }; static struct PyMemberDef testbuf_members[] = { - {"references", T_LONG, offsetof(testBufObject, references), READONLY}, + {"references", T_PYSSIZET, offsetof(testBufObject, references), READONLY}, {NULL}, }; From 9f2d16ba327dd3ffc5981940cb6acfac7aa50140 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 4 May 2023 05:59:35 -0700 Subject: [PATCH 31/35] newlines --- Lib/test/test_buffer.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index 6a2b53892280bc..ffe40ca9ea2633 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -4473,11 +4473,13 @@ class WhatToRelease: def __init__(self): self.held = False self.ba = bytearray(b"hello") + def __buffer__(self, flags): if self.held: raise TypeError("already held") self.held = True return memoryview(self.ba) + def __release_buffer__(self, buffer): self.held = False @@ -4494,12 +4496,14 @@ def __init__(self): self.held = False self.ba = bytearray(b"hello") self.created_mv = None + def __buffer__(self, flags): if self.held: raise TypeError("already held") self.held = True self.created_mv = memoryview(self.ba) return self.created_mv + def __release_buffer__(self, buffer): assert buffer is self.created_mv self.held = False From 5e0d2de8d34d10c0cb6adf2867493bc2fed8d2b2 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 4 May 2023 06:09:04 -0700 Subject: [PATCH 32/35] Rename variable --- Objects/clinic/memoryobject.c.h | 18 +++++++++--------- Objects/memoryobject.c | 8 ++++---- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/Objects/clinic/memoryobject.c.h b/Objects/clinic/memoryobject.c.h index ead1af4f3e8619..25a22341185903 100644 --- a/Objects/clinic/memoryobject.c.h +++ b/Objects/clinic/memoryobject.c.h @@ -63,7 +63,7 @@ memoryview(PyTypeObject *type, PyObject *args, PyObject *kwargs) } PyDoc_STRVAR(memoryview__from_flags__doc__, -"_from_flags($type, /, object, _flags)\n" +"_from_flags($type, /, object, flags)\n" "--\n" "\n" "Create a new memoryview object which references the given object."); @@ -72,7 +72,7 @@ PyDoc_STRVAR(memoryview__from_flags__doc__, {"_from_flags", _PyCFunction_CAST(memoryview__from_flags), METH_FASTCALL|METH_KEYWORDS|METH_CLASS, memoryview__from_flags__doc__}, static PyObject * -memoryview__from_flags_impl(PyTypeObject *type, PyObject *object, int _flags); +memoryview__from_flags_impl(PyTypeObject *type, PyObject *object, int flags); static PyObject * memoryview__from_flags(PyTypeObject *type, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -87,7 +87,7 @@ memoryview__from_flags(PyTypeObject *type, PyObject *const *args, Py_ssize_t nar PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(object), &_Py_ID(_flags), }, + .ob_item = { &_Py_ID(object), &_Py_ID(flags), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -96,7 +96,7 @@ memoryview__from_flags(PyTypeObject *type, PyObject *const *args, Py_ssize_t nar # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"object", "_flags", NULL}; + static const char * const _keywords[] = {"object", "flags", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "_from_flags", @@ -105,18 +105,18 @@ memoryview__from_flags(PyTypeObject *type, PyObject *const *args, Py_ssize_t nar #undef KWTUPLE PyObject *argsbuf[2]; PyObject *object; - int _flags; + int flags; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); if (!args) { goto exit; } object = args[0]; - _flags = _PyLong_AsInt(args[1]); - if (_flags == -1 && PyErr_Occurred()) { + flags = _PyLong_AsInt(args[1]); + if (flags == -1 && PyErr_Occurred()) { goto exit; } - return_value = memoryview__from_flags_impl(type, object, _flags); + return_value = memoryview__from_flags_impl(type, object, flags); exit: return return_value; @@ -416,4 +416,4 @@ memoryview_hex(PyMemoryViewObject *self, PyObject *const *args, Py_ssize_t nargs exit: return return_value; } -/*[clinic end generated code: output=31ed11a9a4ac95c4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=01613814112cedd7 input=a9049054013a1b77]*/ diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index 8c913bcdac9cf9..f008a8cc3e0474 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -982,16 +982,16 @@ memoryview_impl(PyTypeObject *type, PyObject *object) memoryview._from_flags object: object - _flags: int + flags: int Create a new memoryview object which references the given object. [clinic start generated code]*/ static PyObject * -memoryview__from_flags_impl(PyTypeObject *type, PyObject *object, int _flags) -/*[clinic end generated code: output=bc1f4ba7d9b64525 input=860e7832e8edb1e5]*/ +memoryview__from_flags_impl(PyTypeObject *type, PyObject *object, int flags) +/*[clinic end generated code: output=bf71f9906c266ee2 input=f5f82fd0e744356b]*/ { - return PyMemoryView_FromObjectAndFlags(object, _flags); + return PyMemoryView_FromObjectAndFlags(object, flags); } From b3806322bcebca57c8f4f942c70b00a989223ee7 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 4 May 2023 06:11:34 -0700 Subject: [PATCH 33/35] Check for INT_MAX --- Lib/test/test_buffer.py | 2 ++ Objects/typeobject.c | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index ffe40ca9ea2633..b6e82ad4db266a 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -4559,6 +4559,8 @@ def test_call_builtins(self): mv = ba.__buffer__(0) self.assertEqual(mv.tobytes(), b"hello") ba.__release_buffer__(mv) + with self.assertRaises(OverflowError): + ba.__buffer__(sys.maxsize + 1) @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_c_buffer(self): diff --git a/Objects/typeobject.c b/Objects/typeobject.c index f7ad15f70baf29..500e3d782a3eea 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -8072,6 +8072,11 @@ wrap_buffer(PyObject *self, PyObject *args, void *wrapped) if (flags == -1 && PyErr_Occurred()) { return NULL; } + if (flags > INT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "buffer flags too large"); + return NULL; + } return PyMemoryView_FromObjectAndFlags(self, Py_SAFE_DOWNCAST(flags, Py_ssize_t, int)); } From 8502b98121d174f2c12afdb2b06f46fa17a58455 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 4 May 2023 06:17:05 -0700 Subject: [PATCH 34/35] Remove tp_new --- Objects/typeobject.c | 1 - 1 file changed, 1 deletion(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 500e3d782a3eea..456b10ee01d6bc 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -8996,7 +8996,6 @@ PyTypeObject _PyBufferWrapper_Type = { .tp_name = "_buffer_wrapper", .tp_basicsize = sizeof(PyBufferWrapper), .tp_alloc = PyType_GenericAlloc, - .tp_new = PyType_GenericNew, .tp_free = PyObject_GC_Del, .tp_traverse = (traverseproc)bufferwrapper_traverse, .tp_dealloc = bufferwrapper_dealloc, From b51465fb3065aa581113c884c49de604aeaeeae9 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 4 May 2023 06:21:32 -0700 Subject: [PATCH 35/35] regen-all --- Include/internal/pycore_global_objects_fini_generated.h | 1 - Include/internal/pycore_global_strings.h | 1 - Include/internal/pycore_runtime_init_generated.h | 1 - Include/internal/pycore_unicodeobject_generated.h | 3 --- 4 files changed, 6 deletions(-) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 9d97da01d338e5..9377fd8526e3a2 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -749,7 +749,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_finalizing)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_find_and_load)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_fix_up_module)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_flags)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_flags_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_get_sourcefile)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_handle_fromlist)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index f7dc0010316813..ed9b2bb44ddffc 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -237,7 +237,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(_finalizing) STRUCT_FOR_ID(_find_and_load) STRUCT_FOR_ID(_fix_up_module) - STRUCT_FOR_ID(_flags) STRUCT_FOR_ID(_flags_) STRUCT_FOR_ID(_get_sourcefile) STRUCT_FOR_ID(_handle_fromlist) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index c7cd83807b7cb4..6ade8fb6eade03 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -743,7 +743,6 @@ extern "C" { INIT_ID(_finalizing), \ INIT_ID(_find_and_load), \ INIT_ID(_fix_up_module), \ - INIT_ID(_flags), \ INIT_ID(_flags_), \ INIT_ID(_get_sourcefile), \ INIT_ID(_handle_fromlist), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 118fd30adcd94d..0b33ea187e60ff 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -564,9 +564,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(_fix_up_module); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(_flags); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(_flags_); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy