Pybuffer (pythonnet#1195): add Python buffer api support

koubaa · thesn10 · web-flow · commit 8e029ef646f6 · 2020-08-06T20:33:22.000-07:00
Co-authored-by: SnGmng &lt;38666407+SnGmng@users.noreply.github.com&gt;
diff --git a/AUTHORS.md b/AUTHORS.md
@@ -71,6 +71,7 @@
 -   ([@OneBlue](https://github.com/OneBlue))
 -   ([@rico-chet](https://github.com/rico-chet))
 -   ([@rmadsen-ks](https://github.com/rmadsen-ks))
+-   ([@SnGmng](https://github.com/SnGmng))
 -   ([@stonebig](https://github.com/stonebig))
 -   ([@testrunner123](https://github.com/testrunner123))
 -   ([@DanBarzilian](https://github.com/DanBarzilian))
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -35,6 +35,7 @@ This version improves performance on benchmarks significantly compared to 2.3.
 -   Support for Python 3.8
 -   Codecs as the designated way to handle automatic conversions between
     .NET and Python types
+-   Added Python 3 buffer api support and PyBuffer interface for fast byte and numpy array read/write ([#980][p980])
 
 ### Changed
 
diff --git a/src/embed_tests/Python.EmbeddingTest.csproj b/src/embed_tests/Python.EmbeddingTest.csproj
@@ -97,6 +97,7 @@
     <Compile Include="TestFinalizer.cs" />
     <Compile Include="TestInstanceWrapping.cs" />
     <Compile Include="TestPyAnsiString.cs" />
+	<Compile Include="TestPyBuffer.cs" />
     <Compile Include="TestPyFloat.cs" />
     <Compile Include="TestPyInt.cs" />
     <Compile Include="TestPyList.cs" />
diff --git a/src/embed_tests/TestPyBuffer.cs b/src/embed_tests/TestPyBuffer.cs
@@ -0,0 +1,68 @@
+using System.Text;
+using NUnit.Framework;
+using Python.Runtime;
+
+namespace Python.EmbeddingTest {
+    class TestPyBuffer
+    {
+        [OneTimeSetUp]
+        public void SetUp()
+        {
+            PythonEngine.Initialize();
+        }
+
+        [OneTimeTearDown]
+        public void Dispose()
+        {
+            PythonEngine.Shutdown();
+        }
+
+        [Test]
+        public void TestBufferWrite()
+        {
+            string bufferTestString = "hello world! !$%&/()=?";
+
+            using (Py.GIL())
+            {
+                using (var scope = Py.CreateScope())
+                {
+                    scope.Exec($"arr = bytearray({bufferTestString.Length})");
+                    PyObject pythonArray = scope.Get("arr");
+                    byte[] managedArray = new UTF8Encoding().GetBytes(bufferTestString);
+
+                    using (PyBuffer buf = pythonArray.GetBuffer())
+                    {
+                        buf.Write(managedArray, 0, managedArray.Length);
+                    }
+
+                    string result = scope.Eval("arr.decode('utf-8')").ToString();
+                    Assert.IsTrue(result == bufferTestString);
+                }
+            }
+        }
+
+        [Test]
+        public void TestBufferRead()
+        {
+            string bufferTestString = "hello world! !$%&/()=?";
+
+            using (Py.GIL())
+            {
+                using (var scope = Py.CreateScope())
+                {
+                    scope.Exec($"arr = b'{bufferTestString}'");
+                    PyObject pythonArray = scope.Get("arr");
+                    byte[] managedArray = new byte[bufferTestString.Length];
+
+                    using (PyBuffer buf = pythonArray.GetBuffer())
+                    {
+                        buf.Read(managedArray, 0, managedArray.Length);
+                    }
+
+                    string result = new UTF8Encoding().GetString(managedArray);
+                    Assert.IsTrue(result == bufferTestString);
+                }
+            }
+        }
+    }
+}
diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj
@@ -89,6 +89,7 @@
     <Compile Include="arrayobject.cs" />
     <Compile Include="assemblymanager.cs" />
     <Compile Include="BorrowedReference.cs" />
+    <Compile Include="bufferinterface.cs" />
     <Compile Include="classderived.cs" />
     <Compile Include="classbase.cs" />
     <Compile Include="classmanager.cs" />
@@ -130,6 +131,7 @@
     <Compile Include="overload.cs" />
     <Compile Include="propertyobject.cs" />
     <Compile Include="pyansistring.cs" />
+    <Compile Include="pybuffer.cs" />
     <Compile Include="pydict.cs" />
     <Compile Include="PyExportAttribute.cs" />
     <Compile Include="pyfloat.cs" />
diff --git a/src/runtime/bufferinterface.cs b/src/runtime/bufferinterface.cs
@@ -0,0 +1,106 @@
+using System;
+using System.Runtime.InteropServices;
+
+namespace Python.Runtime
+{
+    /* buffer interface */
+    [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Auto)]
+    internal struct Py_buffer {
+        public IntPtr buf;
+        public IntPtr obj;        /* owned reference */
+        [MarshalAs(UnmanagedType.SysInt)]
+        public IntPtr len;
+        [MarshalAs(UnmanagedType.SysInt)]
+        public IntPtr itemsize;  /* This is Py_ssize_t so it can be
+                             pointed to by strides in simple case.*/
+        [MarshalAs(UnmanagedType.Bool)]
+        public bool _readonly;
+        public int ndim;
+        [MarshalAs(UnmanagedType.LPStr)]
+        public string format;
+        public IntPtr shape;
+        public IntPtr strides;
+        public IntPtr suboffsets;
+        public IntPtr _internal;
+    }
+
+    public enum BufferOrderStyle
+    {
+        C,
+        Fortran,
+        EitherOne,
+    }
+
+    /* Flags for getting buffers */
+    public enum PyBUF
+    {
+        /// <summary>
+        /// Simple buffer without shape strides and suboffsets
+        /// </summary>
+        SIMPLE = 0,
+        /// <summary>
+        /// Controls the <see cref="PyBuffer.ReadOnly"/> field. If set, the exporter MUST provide a writable buffer or else report failure. Otherwise, the exporter MAY provide either a read-only or writable buffer, but the choice MUST be consistent for all consumers.
+        /// </summary>
+        WRITABLE = 0x0001,
+        /// <summary>
+        /// Controls the <see cref="PyBuffer.Format"/> field. If set, this field MUST be filled in correctly. Otherwise, this field MUST be NULL.
+        /// </summary>
+        FORMATS = 0x0004,
+        /// <summary>
+        /// N-Dimensional buffer with shape
+        /// </summary>
+        ND = 0x0008,
+        /// <summary>
+        /// Buffer with strides and shape
+        /// </summary>
+        STRIDES = (0x0010 | ND),
+        /// <summary>
+        /// C-Contigous buffer with strides and shape
+        /// </summary>
+        C_CONTIGUOUS = (0x0020 | STRIDES),
+        /// <summary>
+        /// F-Contigous buffer with strides and shape
+        /// </summary>
+        F_CONTIGUOUS = (0x0040 | STRIDES),
+        /// <summary>
+        /// C or Fortran contigous buffer with strides and shape
+        /// </summary>
+        ANY_CONTIGUOUS = (0x0080 | STRIDES),
+        /// <summary>
+        /// Buffer with suboffsets (if needed)
+        /// </summary>
+        INDIRECT = (0x0100 | STRIDES),
+        /// <summary>
+        /// Writable C-Contigous buffer with shape
+        /// </summary>
+        CONTIG = (ND | WRITABLE),
+        /// <summary>
+        /// Readonly C-Contigous buffer with shape
+        /// </summary>
+        CONTIG_RO = (ND),
+        /// <summary>
+        /// Writable buffer with shape and strides
+        /// </summary>
+        STRIDED = (STRIDES | WRITABLE),
+        /// <summary>
+        /// Readonly buffer with shape and strides
+        /// </summary>
+        STRIDED_RO = (STRIDES),
+        /// <summary>
+        /// Writable buffer with shape, strides and format
+        /// </summary>
+        RECORDS = (STRIDES | WRITABLE | FORMATS),
+        /// <summary>
+        /// Readonly buffer with shape, strides and format
+        /// </summary>
+        RECORDS_RO = (STRIDES | FORMATS),
+        /// <summary>
+        /// Writable indirect buffer with shape, strides, format and suboffsets (if needed)
+        /// </summary>
+        FULL = (INDIRECT | WRITABLE | FORMATS),
+        /// <summary>
+        /// Readonly indirect buffer with shape, strides, format and suboffsets (if needed)
+        /// </summary>
+        FULL_RO = (INDIRECT | FORMATS),
+    }
+}
diff --git a/src/runtime/pybuffer.cs b/src/runtime/pybuffer.cs
diff --git a/src/runtime/pyobject.cs b/src/runtime/pyobject.cs
diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs
