diff --git a/src/embed_tests/TestPyBuffer.cs b/src/embed_tests/TestPyBuffer.cs index 43ed5ffd4..a1bcc161d 100644 --- a/src/embed_tests/TestPyBuffer.cs +++ b/src/embed_tests/TestPyBuffer.cs @@ -1,4 +1,5 @@ using System; +using System.Runtime.CompilerServices; using System.Text; using NUnit.Framework; using Python.Runtime; @@ -24,24 +25,20 @@ public void Dispose() public void TestBufferWrite() { string bufferTestString = "hello world! !$%&/()=?"; + string bufferTestString2 = "h llo world! !$%&/()=?"; - using (Py.GIL()) + using var _ = Py.GIL(); + + using var pythonArray = ByteArrayFromAsciiString(bufferTestString); + + using (PyBuffer buf = pythonArray.GetBuffer(PyBUF.WRITABLE)) { - using (var scope = Py.CreateScope()) - { - scope.Exec($"arr = bytearray({bufferTestString.Length})"); - PyObject pythonArray = scope.Get("arr"); - byte[] managedArray = new UTF8Encoding().GetBytes(bufferTestString); - - using (PyBuffer buf = pythonArray.GetBuffer()) - { - buf.Write(managedArray, 0, managedArray.Length); - } - - string result = scope.Eval("arr.decode('utf-8')").ToString(); - Assert.IsTrue(result == bufferTestString); - } + byte[] managedArray = { (byte)' ' }; + buf.Write(managedArray, 0, managedArray.Length, 1); } + + string result = pythonArray.InvokeMethod("decode", "utf-8".ToPython()).As(); + Assert.IsTrue(result == bufferTestString2); } [Test] @@ -49,23 +46,19 @@ public void TestBufferRead() { string bufferTestString = "hello world! !$%&/()=?"; - using (Py.GIL()) + using var _ = Py.GIL(); + + using var pythonArray = ByteArrayFromAsciiString(bufferTestString); + byte[] managedArray = new byte[bufferTestString.Length]; + + using (PyBuffer buf = pythonArray.GetBuffer()) { - using (var scope = Py.CreateScope()) - { - scope.Exec($"arr = b'{bufferTestString}'"); - PyObject pythonArray = scope.Get("arr"); - byte[] managedArray = new byte[bufferTestString.Length]; - - using (PyBuffer buf = pythonArray.GetBuffer()) - { - buf.Read(managedArray, 0, managedArray.Length); - } - - string result = new UTF8Encoding().GetString(managedArray); - Assert.IsTrue(result == bufferTestString); - } + managedArray[0] = (byte)' '; + buf.Read(managedArray, 1, managedArray.Length - 1, 1); } + + string result = new UTF8Encoding().GetString(managedArray); + Assert.IsTrue(result == " " + bufferTestString.Substring(1)); } [Test] @@ -77,5 +70,56 @@ public void ArrayHasBuffer() Assert.AreEqual(1, mem[(0, 0).ToPython()].As()); Assert.AreEqual(array[1,0], mem[(1, 0).ToPython()].As()); } + + [Test] + public void RefCount() + { + using var _ = Py.GIL(); + using var arr = ByteArrayFromAsciiString("hello world! !$%&/()=?"); + + Assert.AreEqual(1, arr.Refcount); + + using (PyBuffer buf = arr.GetBuffer()) + { + Assert.AreEqual(2, arr.Refcount); + } + + Assert.AreEqual(1, arr.Refcount); + } + + [Test] + public void Finalization() + { + if (Type.GetType("Mono.Runtime") is not null) + { + Assert.Inconclusive("test unreliable in Mono"); + return; + } + + using var _ = Py.GIL(); + using var arr = ByteArrayFromAsciiString("hello world! !$%&/()=?"); + + Assert.AreEqual(1, arr.Refcount); + + MakeBufAndLeak(arr); + + GC.Collect(); + GC.WaitForPendingFinalizers(); + Finalizer.Instance.Collect(); + + Assert.AreEqual(1, arr.Refcount); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + static void MakeBufAndLeak(PyObject bufProvider) + { + PyBuffer buf = bufProvider.GetBuffer(); + } + + static PyObject ByteArrayFromAsciiString(string str) + { + using var scope = Py.CreateScope(); + return Runtime.Runtime.PyByteArray_FromStringAndSize(str).MoveToPyObject(); + } } } diff --git a/src/runtime/bufferinterface.cs b/src/runtime/bufferinterface.cs index e0b71a925..431d609f4 100644 --- a/src/runtime/bufferinterface.cs +++ b/src/runtime/bufferinterface.cs @@ -8,10 +8,11 @@ namespace Python.Runtime internal struct Py_buffer { public IntPtr buf; public IntPtr obj; /* owned reference */ + /// Buffer size in bytes [MarshalAs(UnmanagedType.SysInt)] - public IntPtr len; + public nint len; [MarshalAs(UnmanagedType.SysInt)] - public IntPtr itemsize; /* This is Py_ssize_t so it can be + public nint itemsize; /* This is Py_ssize_t so it can be pointed to by strides in simple case.*/ [MarshalAs(UnmanagedType.Bool)] public bool _readonly; diff --git a/src/runtime/finalizer.cs b/src/runtime/finalizer.cs index bfb1e228d..be17d62e3 100644 --- a/src/runtime/finalizer.cs +++ b/src/runtime/finalizer.cs @@ -43,6 +43,7 @@ public ErrorArgs(Exception error) private ConcurrentQueue _objQueue = new(); private readonly ConcurrentQueue _derivedQueue = new(); + private readonly ConcurrentQueue _bufferQueue = new(); private int _throttled; #region FINALIZER_CHECK @@ -165,6 +166,19 @@ internal void AddDerivedFinalizedObject(ref IntPtr derived, int run) _derivedQueue.Enqueue(pending); } + internal void AddFinalizedBuffer(ref Py_buffer buffer) + { + if (buffer.obj == IntPtr.Zero) + throw new ArgumentNullException(nameof(buffer)); + + if (!Enable) + return; + + var pending = buffer; + buffer = default; + _bufferQueue.Enqueue(pending); + } + internal static void Initialize() { Instance.started = true; @@ -178,7 +192,7 @@ internal static void Shutdown() internal nint DisposeAll() { - if (_objQueue.IsEmpty && _derivedQueue.IsEmpty) + if (_objQueue.IsEmpty && _derivedQueue.IsEmpty && _bufferQueue.IsEmpty) return 0; nint collected = 0; @@ -242,6 +256,15 @@ internal nint DisposeAll() collected++; } + + while (!_bufferQueue.IsEmpty) + { + if (!_bufferQueue.TryDequeue(out var buffer)) + continue; + + Runtime.PyBuffer_Release(ref buffer); + collected++; + } } finally { diff --git a/src/runtime/pybuffer.cs b/src/runtime/pybuffer.cs index 60aeaf0b9..de0e7122f 100644 --- a/src/runtime/pybuffer.cs +++ b/src/runtime/pybuffer.cs @@ -89,7 +89,9 @@ public static long SizeFromFormat(string format) { if (Runtime.PyVersion < new Version(3,9)) throw new NotSupportedException("SizeFromFormat requires at least Python 3.9"); - return (long)Runtime.PyBuffer_SizeFromFormat(format); + nint result = Runtime.PyBuffer_SizeFromFormat(format); + if (result == -1) throw PythonException.ThrowLastAsClrException(); + return result; } /// @@ -113,7 +115,7 @@ public IntPtr GetPointer(long[] indices) throw new ObjectDisposedException(nameof(PyBuffer)); if (Runtime.PyVersion < new Version(3, 7)) throw new NotSupportedException("GetPointer requires at least Python 3.7"); - return Runtime.PyBuffer_GetPointer(ref _view, indices.Select(x => (IntPtr)x).ToArray()); + return Runtime.PyBuffer_GetPointer(ref _view, indices.Select(x => checked((nint)x)).ToArray()); } /// @@ -126,7 +128,7 @@ public void FromContiguous(IntPtr buf, long len, BufferOrderStyle fort) if (Runtime.PyVersion < new Version(3, 7)) throw new NotSupportedException("FromContiguous requires at least Python 3.7"); - if (Runtime.PyBuffer_FromContiguous(ref _view, buf, (IntPtr)len, OrderStyleToChar(fort, false)) < 0) + if (Runtime.PyBuffer_FromContiguous(ref _view, buf, checked((nint)len), OrderStyleToChar(fort, false)) < 0) throw PythonException.ThrowLastAsClrException(); } @@ -173,44 +175,60 @@ internal void FillInfo(BorrowedReference exporter, IntPtr buf, long len, bool _r /// /// Writes a managed byte array into the buffer of a python object. This can be used to pass data like images from managed to python. /// - public void Write(byte[] buffer, int offset, int count) + public void Write(byte[] buffer, int sourceOffset, int count, nint destinationOffset) { if (disposedValue) throw new ObjectDisposedException(nameof(PyBuffer)); + if (_view.ndim != 1) + throw new NotImplementedException("Multidimensional arrays, scalars and objects without a buffer are not supported."); + if (!this.IsContiguous(BufferOrderStyle.C)) + throw new NotImplementedException("Only continuous buffers are supported"); if (ReadOnly) throw new InvalidOperationException("Buffer is read-only"); - if ((long)_view.len > int.MaxValue) - throw new NotSupportedException("Python buffers bigger than int.MaxValue are not supported."); - if (count > buffer.Length) + if (buffer is null) + throw new ArgumentNullException(nameof(buffer)); + + if (sourceOffset < 0) + throw new IndexOutOfRangeException($"{nameof(sourceOffset)} is negative"); + if (destinationOffset < 0) + throw new IndexOutOfRangeException($"{nameof(destinationOffset)} is negative"); + if (count < 0) + throw new ArgumentOutOfRangeException(nameof(count), count, "Value must be >= 0"); + + if (checked(count + sourceOffset) > buffer.Length) throw new ArgumentOutOfRangeException("count", "Count is bigger than the buffer."); - if (count > (int)_view.len) + if (checked(count + destinationOffset) > _view.len) throw new ArgumentOutOfRangeException("count", "Count is bigger than the python buffer."); - if (_view.ndim != 1) - throw new NotSupportedException("Multidimensional arrays, scalars and objects without a buffer are not supported."); - if (!this.IsContiguous(BufferOrderStyle.C)) - throw new NotImplementedException("Only continuous buffers are supported"); - Marshal.Copy(buffer, offset, _view.buf, count); + Marshal.Copy(buffer, sourceOffset, _view.buf + destinationOffset, count); } /// /// Reads the buffer of a python object into a managed byte array. This can be used to pass data like images from python to managed. /// - public int Read(byte[] buffer, int offset, int count) { + public void Read(byte[] buffer, int destinationOffset, int count, nint sourceOffset) { if (disposedValue) throw new ObjectDisposedException(nameof(PyBuffer)); - if (count > buffer.Length) - throw new ArgumentOutOfRangeException("count", "Count is bigger than the buffer."); if (_view.ndim != 1) - throw new NotSupportedException("Multidimensional arrays, scalars and objects without a buffer are not supported."); - if (_view.len.ToInt64() > int.MaxValue) - throw new NotSupportedException("Python buffers bigger than int.MaxValue are not supported."); + throw new NotImplementedException("Multidimensional arrays, scalars and objects without a buffer are not supported."); if (!this.IsContiguous(BufferOrderStyle.C)) throw new NotImplementedException("Only continuous buffers are supported"); + if (buffer is null) + throw new ArgumentNullException(nameof(buffer)); + + if (sourceOffset < 0) + throw new IndexOutOfRangeException($"{nameof(sourceOffset)} is negative"); + if (destinationOffset < 0) + throw new IndexOutOfRangeException($"{nameof(destinationOffset)} is negative"); + if (count < 0) + throw new ArgumentOutOfRangeException(nameof(count), count, "Value must be >= 0"); + + if (checked(count + destinationOffset) > buffer.Length) + throw new ArgumentOutOfRangeException("count", "Count is bigger than the buffer."); + if (checked(count + sourceOffset) > _view.len) + throw new ArgumentOutOfRangeException("count", "Count is bigger than the python buffer."); - int copylen = count < (int)_view.len ? count : (int)_view.len; - Marshal.Copy(_view.buf, buffer, offset, copylen); - return copylen; + Marshal.Copy(_view.buf + sourceOffset, buffer, destinationOffset, count); } private bool disposedValue = false; // To detect redundant calls @@ -240,11 +258,7 @@ private void Dispose(bool disposing) if (_view.obj != IntPtr.Zero) { - Finalizer.Instance.AddFinalizedObject(ref _view.obj, _exporter.run -#if TRACE_ALLOC - , _exporter.Traceback -#endif - ); + Finalizer.Instance.AddFinalizedBuffer(ref _view); } Dispose(false); diff --git a/src/runtime/pyobject.cs b/src/runtime/pyobject.cs index 709bc11c4..373751cf6 100644 --- a/src/runtime/pyobject.cs +++ b/src/runtime/pyobject.cs @@ -179,6 +179,11 @@ public static PyObject FromManagedObject(object ob) internal bool IsDisposed => rawPtr == IntPtr.Zero; + void CheckDisposed() + { + if (IsDisposed) throw new ObjectDisposedException(nameof(PyObject)); + } + protected virtual void Dispose(bool disposing) { if (IsDisposed) @@ -1114,6 +1119,7 @@ public override int GetHashCode() /// public PyBuffer GetBuffer(PyBUF flags = PyBUF.SIMPLE) { + CheckDisposed(); return new PyBuffer(this, flags); } diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index a2ee45e9c..21d64db4a 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -1109,7 +1109,7 @@ internal static nint PyBuffer_SizeFromFormat(string format) internal static int PyBuffer_IsContiguous(ref Py_buffer view, char order) => Delegates.PyBuffer_IsContiguous(ref view, order); - internal static IntPtr PyBuffer_GetPointer(ref Py_buffer view, IntPtr[] indices) => Delegates.PyBuffer_GetPointer(ref view, indices); + internal static IntPtr PyBuffer_GetPointer(ref Py_buffer view, nint[] indices) => Delegates.PyBuffer_GetPointer(ref view, indices); internal static int PyBuffer_FromContiguous(ref Py_buffer view, IntPtr buf, IntPtr len, char fort) => Delegates.PyBuffer_FromContiguous(ref view, buf, len, fort); @@ -1364,6 +1364,13 @@ internal static NewReference EmptyPyBytes() return Delegates.PyBytes_FromString((IntPtr)bytes); } + internal static NewReference PyByteArray_FromStringAndSize(IntPtr strPtr, nint len) => Delegates.PyByteArray_FromStringAndSize(strPtr, len); + internal static NewReference PyByteArray_FromStringAndSize(string s) + { + using var ptr = new StrPtr(s, Encoding.UTF8); + return PyByteArray_FromStringAndSize(ptr.RawPointer, checked((nint)ptr.ByteCount)); + } + internal static IntPtr PyBytes_AsString(BorrowedReference ob) { Debug.Assert(ob != null); @@ -1992,7 +1999,7 @@ static Delegates() // only in 3.9+ } PyBuffer_IsContiguous = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyBuffer_IsContiguous), GetUnmanagedDll(_PythonDll)); - PyBuffer_GetPointer = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyBuffer_GetPointer), GetUnmanagedDll(_PythonDll)); + PyBuffer_GetPointer = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyBuffer_GetPointer), GetUnmanagedDll(_PythonDll)); PyBuffer_FromContiguous = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyBuffer_FromContiguous), GetUnmanagedDll(_PythonDll)); PyBuffer_ToContiguous = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyBuffer_ToContiguous), GetUnmanagedDll(_PythonDll)); PyBuffer_FillContiguousStrides = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyBuffer_FillContiguousStrides), GetUnmanagedDll(_PythonDll)); @@ -2052,6 +2059,7 @@ static Delegates() PySequence_List = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PySequence_List), GetUnmanagedDll(_PythonDll)); PyBytes_AsString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyBytes_AsString), GetUnmanagedDll(_PythonDll)); PyBytes_FromString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyBytes_FromString), GetUnmanagedDll(_PythonDll)); + PyByteArray_FromStringAndSize = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyByteArray_FromStringAndSize), GetUnmanagedDll(_PythonDll)); PyBytes_Size = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyBytes_Size), GetUnmanagedDll(_PythonDll)); PyUnicode_AsUTF8 = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_AsUTF8), GetUnmanagedDll(_PythonDll)); PyUnicode_DecodeUTF16 = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_DecodeUTF16), GetUnmanagedDll(_PythonDll)); @@ -2264,7 +2272,7 @@ static Delegates() internal static delegate* unmanaged[Cdecl] PyBuffer_Release { get; } internal static delegate* unmanaged[Cdecl] PyBuffer_SizeFromFormat { get; } internal static delegate* unmanaged[Cdecl] PyBuffer_IsContiguous { get; } - internal static delegate* unmanaged[Cdecl] PyBuffer_GetPointer { get; } + internal static delegate* unmanaged[Cdecl] PyBuffer_GetPointer { get; } internal static delegate* unmanaged[Cdecl] PyBuffer_FromContiguous { get; } internal static delegate* unmanaged[Cdecl] PyBuffer_ToContiguous { get; } internal static delegate* unmanaged[Cdecl] PyBuffer_FillContiguousStrides { get; } @@ -2324,6 +2332,7 @@ static Delegates() internal static delegate* unmanaged[Cdecl] PySequence_List { get; } internal static delegate* unmanaged[Cdecl] PyBytes_AsString { get; } internal static delegate* unmanaged[Cdecl] PyBytes_FromString { get; } + internal static delegate* unmanaged[Cdecl] PyByteArray_FromStringAndSize { get; } internal static delegate* unmanaged[Cdecl] PyBytes_Size { get; } internal static delegate* unmanaged[Cdecl] PyUnicode_AsUTF8 { get; } internal static delegate* unmanaged[Cdecl] PyUnicode_DecodeUTF16 { get; } pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy