From 21c5dcc950272d84cb739bc6d61ba4d280882180 Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Thu, 30 Jun 2022 15:24:00 -0700 Subject: [PATCH 01/34] implemented dynamic equality and inequality for PyObject instances fixed unhandled Python errors during comparison attempts fixes https://github.com/pythonnet/pythonnet/issues/1848 --- CHANGELOG.md | 2 + src/embed_tests/dynamic.cs | 22 +++++++++ src/runtime/PythonTypes/PyObject.cs | 75 +++++++++++++++++++++++------ src/runtime/Runtime.cs | 25 ---------- 4 files changed, 83 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 52ee08484..9b5dd1816 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,8 @@ details about the cause of the failure able to access members that are part of the implementation class, but not the interface. Use the new `__implementation__` or `__raw_implementation__` properties to if you need to "downcast" to the implementation class. +- BREAKING: `==` and `!=` operators on `PyObject` instances now use Python comparison + (previously was equivalent to `object.ReferenceEquals(,)`) - BREAKING: Parameters marked with `ParameterAttributes.Out` are no longer returned in addition to the regular method return value (unless they are passed with `ref` or `out` keyword). - BREAKING: Drop support for the long-deprecated CLR.* prefix. diff --git a/src/embed_tests/dynamic.cs b/src/embed_tests/dynamic.cs index 0a181231c..6e3bfc4cb 100644 --- a/src/embed_tests/dynamic.cs +++ b/src/embed_tests/dynamic.cs @@ -128,6 +128,28 @@ public void PassPyObjectInNet() Assert.IsTrue(sys.testattr1.Equals(sys.testattr2)); } + // regression test for https://github.com/pythonnet/pythonnet/issues/1848 + [Test] + public void EnumEquality() + { + using var scope = Py.CreateScope(); + scope.Exec(@" +import enum + +class MyEnum(enum.IntEnum): + OK = 1 + ERROR = 2 + +def get_status(): + return MyEnum.OK +" +); + + dynamic MyEnum = scope.Get("MyEnum"); + dynamic status = scope.Get("get_status").Invoke(); + Assert.IsTrue(status == MyEnum.OK); + } + // regression test for https://github.com/pythonnet/pythonnet/issues/1680 [Test] public void ForEach() diff --git a/src/runtime/PythonTypes/PyObject.cs b/src/runtime/PythonTypes/PyObject.cs index cfd3e7158..3d48e22ed 100644 --- a/src/runtime/PythonTypes/PyObject.cs +++ b/src/runtime/PythonTypes/PyObject.cs @@ -1075,12 +1075,9 @@ public virtual bool Equals(PyObject? other) { return true; } - int r = Runtime.PyObject_Compare(this, other); - if (Exceptions.ErrorOccurred()) - { - throw PythonException.ThrowLastAsClrException(); - } - return r == 0; + int result = Runtime.PyObject_RichCompareBool(obj, other.obj, Runtime.Py_EQ); + if (result < 0) throw PythonException.ThrowLastAsClrException(); + return result != 0; } @@ -1304,6 +1301,18 @@ public override bool TryConvert(ConvertBinder binder, out object? result) return false; } + private bool TryCompare(PyObject arg, int op, out object @out) + { + int result = Runtime.PyObject_RichCompareBool(this.obj, arg.obj, op); + @out = result != 0; + if (result < 0) + { + Exceptions.Clear(); + return false; + } + return true; + } + public override bool TryBinaryOperation(BinaryOperationBinder binder, object arg, out object? result) { using var _ = Py.GIL(); @@ -1352,11 +1361,9 @@ public override bool TryBinaryOperation(BinaryOperationBinder binder, object arg res = Runtime.PyNumber_InPlaceXor(this.obj, ((PyObject)arg).obj); break; case ExpressionType.GreaterThan: - result = Runtime.PyObject_Compare(this.obj, ((PyObject)arg).obj) > 0; - return true; + return this.TryCompare((PyObject)arg, Runtime.Py_GT, out result); case ExpressionType.GreaterThanOrEqual: - result = Runtime.PyObject_Compare(this.obj, ((PyObject)arg).obj) >= 0; - return true; + return this.TryCompare((PyObject)arg, Runtime.Py_GE, out result); case ExpressionType.LeftShift: res = Runtime.PyNumber_Lshift(this.obj, ((PyObject)arg).obj); break; @@ -1364,11 +1371,9 @@ public override bool TryBinaryOperation(BinaryOperationBinder binder, object arg res = Runtime.PyNumber_InPlaceLshift(this.obj, ((PyObject)arg).obj); break; case ExpressionType.LessThan: - result = Runtime.PyObject_Compare(this.obj, ((PyObject)arg).obj) < 0; - return true; + return this.TryCompare((PyObject)arg, Runtime.Py_LT, out result); case ExpressionType.LessThanOrEqual: - result = Runtime.PyObject_Compare(this.obj, ((PyObject)arg).obj) <= 0; - return true; + return this.TryCompare((PyObject)arg, Runtime.Py_LE, out result); case ExpressionType.Modulo: res = Runtime.PyNumber_Remainder(this.obj, ((PyObject)arg).obj); break; @@ -1376,8 +1381,9 @@ public override bool TryBinaryOperation(BinaryOperationBinder binder, object arg res = Runtime.PyNumber_InPlaceRemainder(this.obj, ((PyObject)arg).obj); break; case ExpressionType.NotEqual: - result = Runtime.PyObject_Compare(this.obj, ((PyObject)arg).obj) != 0; - return true; + return this.TryCompare((PyObject)arg, Runtime.Py_NE, out result); + case ExpressionType.Equal: + return this.TryCompare((PyObject)arg, Runtime.Py_EQ, out result); case ExpressionType.Or: res = Runtime.PyNumber_Or(this.obj, ((PyObject)arg).obj); break; @@ -1402,6 +1408,40 @@ public override bool TryBinaryOperation(BinaryOperationBinder binder, object arg return true; } + public static bool operator ==(PyObject? a, PyObject? b) + { + if (a is null && b is null) + { + return true; + } + if (a is null || b is null) + { + return false; + } + + using var _ = Py.GIL(); + int result = Runtime.PyObject_RichCompareBool(a.obj, b.obj, Runtime.Py_EQ); + if (result < 0) throw PythonException.ThrowLastAsClrException(); + return result != 0; + } + + public static bool operator !=(PyObject? a, PyObject? b) + { + if (a is null && b is null) + { + return false; + } + if (a is null || b is null) + { + return true; + } + + using var _ = Py.GIL(); + int result = Runtime.PyObject_RichCompareBool(a.obj, b.obj, Runtime.Py_NE); + if (result < 0) throw PythonException.ThrowLastAsClrException(); + return result != 0; + } + // Workaround for https://bugzilla.xamarin.com/show_bug.cgi?id=41509 // See https://github.com/pythonnet/pythonnet/pull/219 internal static object? CheckNone(PyObject pyObj) @@ -1436,14 +1476,17 @@ public override bool TryUnaryOperation(UnaryOperationBinder binder, out object? case ExpressionType.Not: r = Runtime.PyObject_Not(this.obj); result = r == 1; + if (r == -1) Exceptions.Clear(); return r != -1; case ExpressionType.IsFalse: r = Runtime.PyObject_IsTrue(this.obj); result = r == 0; + if (r == -1) Exceptions.Clear(); return r != -1; case ExpressionType.IsTrue: r = Runtime.PyObject_IsTrue(this.obj); result = r == 1; + if (r == -1) Exceptions.Clear(); return r != -1; case ExpressionType.Decrement: case ExpressionType.Increment: diff --git a/src/runtime/Runtime.cs b/src/runtime/Runtime.cs index 6ad1d459f..1eeb96b54 100644 --- a/src/runtime/Runtime.cs +++ b/src/runtime/Runtime.cs @@ -962,31 +962,6 @@ internal static IntPtr PyObject_CallObject(IntPtr pointer, IntPtr args) internal static int PyObject_RichCompareBool(BorrowedReference value1, BorrowedReference value2, int opid) => Delegates.PyObject_RichCompareBool(value1, value2, opid); - internal static int PyObject_Compare(BorrowedReference value1, BorrowedReference value2) - { - int res; - res = PyObject_RichCompareBool(value1, value2, Py_LT); - if (-1 == res) - return -1; - else if (1 == res) - return -1; - - res = PyObject_RichCompareBool(value1, value2, Py_EQ); - if (-1 == res) - return -1; - else if (1 == res) - return 0; - - res = PyObject_RichCompareBool(value1, value2, Py_GT); - if (-1 == res) - return -1; - else if (1 == res) - return 1; - - Exceptions.SetError(Exceptions.SystemError, "Error comparing objects"); - return -1; - } - internal static int PyObject_IsInstance(BorrowedReference ob, BorrowedReference type) => Delegates.PyObject_IsInstance(ob, type); From e241a9d1ccd6b5168aed2457a6c96117d708306f Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Thu, 30 Jun 2022 16:25:15 -0700 Subject: [PATCH 02/34] got rid of a few deprecation warnings that pollute GitHub code review --- src/embed_tests/TestConverter.cs | 9 ++++++--- src/embed_tests/TestDomainReload.cs | 3 +-- src/embed_tests/TestFinalizer.cs | 7 +++++-- src/embed_tests/TestNativeTypeOffset.cs | 3 ++- src/embed_tests/TestPythonException.cs | 2 +- src/runtime/InternString.cs | 4 ++-- src/runtime/PythonTypes/PyObject.cs | 4 +++- src/runtime/Types/ReflectedClrType.cs | 2 +- src/runtime/Util/PythonReferenceComparer.cs | 4 ++-- 9 files changed, 23 insertions(+), 15 deletions(-) diff --git a/src/embed_tests/TestConverter.cs b/src/embed_tests/TestConverter.cs index e586eda1b..0686d528b 100644 --- a/src/embed_tests/TestConverter.cs +++ b/src/embed_tests/TestConverter.cs @@ -148,7 +148,7 @@ public void PyIntImplicit() { var i = new PyInt(1); var ni = (PyObject)i.As(); - Assert.AreEqual(i.rawPtr, ni.rawPtr); + Assert.IsTrue(PythonReferenceComparer.Instance.Equals(i, ni)); } [Test] @@ -178,8 +178,11 @@ public void RawPyObjectProxy() var clrObject = (CLRObject)ManagedType.GetManagedObject(pyObjectProxy); Assert.AreSame(pyObject, clrObject.inst); - var proxiedHandle = pyObjectProxy.GetAttr("Handle").As(); - Assert.AreEqual(pyObject.Handle, proxiedHandle); +#pragma warning disable CS0612 // Type or member is obsolete + const string handlePropertyName = nameof(PyObject.Handle); +#pragma warning restore CS0612 // Type or member is obsolete + var proxiedHandle = pyObjectProxy.GetAttr(handlePropertyName).As(); + Assert.AreEqual(pyObject.DangerousGetAddressOrNull(), proxiedHandle); } // regression for https://github.com/pythonnet/pythonnet/issues/451 diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs index 498119d1e..a0f9b63eb 100644 --- a/src/embed_tests/TestDomainReload.cs +++ b/src/embed_tests/TestDomainReload.cs @@ -99,8 +99,7 @@ from Python.EmbeddingTest.Domain import MyClass { Debug.Assert(obj.AsManagedObject(type).GetType() == type); // We only needs its Python handle - PyRuntime.XIncref(obj); - return obj.Handle; + return new NewReference(obj).DangerousMoveToPointer(); } } } diff --git a/src/embed_tests/TestFinalizer.cs b/src/embed_tests/TestFinalizer.cs index 40ab03395..b748a2244 100644 --- a/src/embed_tests/TestFinalizer.cs +++ b/src/embed_tests/TestFinalizer.cs @@ -212,7 +212,9 @@ public void ValidateRefCount() Assert.AreEqual(ptr, e.Handle); Assert.AreEqual(2, e.ImpactedObjects.Count); // Fix for this test, don't do this on general environment +#pragma warning disable CS0618 // Type or member is obsolete Runtime.Runtime.XIncref(e.Reference); +#pragma warning restore CS0618 // Type or member is obsolete return false; }; Finalizer.Instance.IncorrectRefCntResolver += handler; @@ -234,8 +236,9 @@ private static IntPtr CreateStringGarbage() { PyString s1 = new PyString("test_string"); // s2 steal a reference from s1 - PyString s2 = new PyString(StolenReference.DangerousFromPointer(s1.Handle)); - return s1.Handle; + IntPtr address = s1.Reference.DangerousGetAddress(); + PyString s2 = new (StolenReference.DangerousFromPointer(address)); + return address; } } } diff --git a/src/embed_tests/TestNativeTypeOffset.cs b/src/embed_tests/TestNativeTypeOffset.cs index 2d31fe506..d692c24e6 100644 --- a/src/embed_tests/TestNativeTypeOffset.cs +++ b/src/embed_tests/TestNativeTypeOffset.cs @@ -33,7 +33,8 @@ public void LoadNativeTypeOffsetClass() { PyObject sys = Py.Import("sys"); // We can safely ignore the "m" abi flag - var abiflags = sys.GetAttr("abiflags", "".ToPython()).ToString().Replace("m", ""); + var abiflags = sys.HasAttr("abiflags") ? sys.GetAttr("abiflags").ToString() : ""; + abiflags = abiflags.Replace("m", ""); if (!string.IsNullOrEmpty(abiflags)) { string typeName = "Python.Runtime.NativeTypeOffset, Python.Runtime"; diff --git a/src/embed_tests/TestPythonException.cs b/src/embed_tests/TestPythonException.cs index a7cf05c83..a248b6a1f 100644 --- a/src/embed_tests/TestPythonException.cs +++ b/src/embed_tests/TestPythonException.cs @@ -161,7 +161,7 @@ def __init__(self, val): using var tbObj = tbPtr.MoveToPyObject(); // the type returned from PyErr_NormalizeException should not be the same type since a new // exception was raised by initializing the exception - Assert.AreNotEqual(type.Handle, typeObj.Handle); + Assert.IsFalse(PythonReferenceComparer.Instance.Equals(type, typeObj)); // the message should now be the string from the throw exception during normalization Assert.AreEqual("invalid literal for int() with base 10: 'dummy string'", strObj.ToString()); } diff --git a/src/runtime/InternString.cs b/src/runtime/InternString.cs index b6d9a0e4a..decb3981d 100644 --- a/src/runtime/InternString.cs +++ b/src/runtime/InternString.cs @@ -42,7 +42,7 @@ public static void Initialize() Debug.Assert(name == op.As()); SetIntern(name, op); var field = type.GetField("f" + name, PyIdentifierFieldFlags)!; - field.SetValue(null, op.rawPtr); + field.SetValue(null, op.DangerousGetAddressOrNull()); } } @@ -76,7 +76,7 @@ public static bool TryGetInterned(BorrowedReference op, out string s) private static void SetIntern(string s, PyString op) { _string2interns.Add(s, op); - _intern2strings.Add(op.rawPtr, s); + _intern2strings.Add(op.Reference.DangerousGetAddress(), s); } } } diff --git a/src/runtime/PythonTypes/PyObject.cs b/src/runtime/PythonTypes/PyObject.cs index 3d48e22ed..ce86753eb 100644 --- a/src/runtime/PythonTypes/PyObject.cs +++ b/src/runtime/PythonTypes/PyObject.cs @@ -27,7 +27,7 @@ public partial class PyObject : DynamicObject, IDisposable, ISerializable public StackTrace Traceback { get; } = new StackTrace(1); #endif - protected internal IntPtr rawPtr = IntPtr.Zero; + protected IntPtr rawPtr = IntPtr.Zero; internal readonly int run = Runtime.GetRun(); internal BorrowedReference obj => new (rawPtr); @@ -252,6 +252,8 @@ internal void Leak() rawPtr = IntPtr.Zero; } + internal IntPtr DangerousGetAddressOrNull() => rawPtr; + internal void CheckRun() { if (run != Runtime.GetRun()) diff --git a/src/runtime/Types/ReflectedClrType.cs b/src/runtime/Types/ReflectedClrType.cs index 2e8f95924..e92a28018 100644 --- a/src/runtime/Types/ReflectedClrType.cs +++ b/src/runtime/Types/ReflectedClrType.cs @@ -117,6 +117,6 @@ static ReflectedClrType AllocateClass(Type clrType) return new ReflectedClrType(type.Steal()); } - public override bool Equals(PyObject? other) => other != null && rawPtr == other.rawPtr; + public override bool Equals(PyObject? other) => rawPtr == other?.DangerousGetAddressOrNull(); public override int GetHashCode() => rawPtr.GetHashCode(); } diff --git a/src/runtime/Util/PythonReferenceComparer.cs b/src/runtime/Util/PythonReferenceComparer.cs index dd78f912d..63c35df57 100644 --- a/src/runtime/Util/PythonReferenceComparer.cs +++ b/src/runtime/Util/PythonReferenceComparer.cs @@ -13,10 +13,10 @@ public sealed class PythonReferenceComparer : IEqualityComparer public static PythonReferenceComparer Instance { get; } = new PythonReferenceComparer(); public bool Equals(PyObject? x, PyObject? y) { - return x?.rawPtr == y?.rawPtr; + return x?.DangerousGetAddressOrNull() == y?.DangerousGetAddressOrNull(); } - public int GetHashCode(PyObject obj) => obj.rawPtr.GetHashCode(); + public int GetHashCode(PyObject obj) => obj.DangerousGetAddressOrNull().GetHashCode(); private PythonReferenceComparer() { } } From ce3afa64c85e5bf88a54d581eae4719810f85e19 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Tue, 28 Jun 2022 14:45:43 +0200 Subject: [PATCH 03/34] Fix broken prefix and debug leftover Additionally, fixes a type hint and makes sure that the new default behaviour is to use the environment variable if given. --- pythonnet/__init__.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pythonnet/__init__.py b/pythonnet/__init__.py index fa6ed45cf..9876a0bec 100644 --- a/pythonnet/__init__.py +++ b/pythonnet/__init__.py @@ -3,7 +3,7 @@ from typing import Dict, Optional, Union import clr_loader -__all__ = ["set_runtime", "set_default_runtime", "load"] +__all__ = ["set_runtime", "set_runtime_from_env", "load"] _RUNTIME: Optional[clr_loader.Runtime] = None _LOADER_ASSEMBLY: Optional[clr_loader.wrappers.Assembly] = None @@ -30,7 +30,7 @@ def set_runtime(runtime: Union[clr_loader.Runtime, str], **params: str) -> None: def _get_params_from_env(prefix: str) -> Dict[str, str]: from os import environ - full_prefix = f"PYTHONNET_{prefix.upper()}" + full_prefix = f"PYTHONNET_{prefix.upper()}_" len_ = len(full_prefix) env_vars = { @@ -63,8 +63,8 @@ def _create_runtime_from_spec( raise RuntimeError(f"Invalid runtime name: '{spec}'") -def set_default_runtime() -> None: - """Set up the default runtime +def set_runtime_from_env() -> None: + """Set up the runtime using the environment This will use the environment variable PYTHONNET_RUNTIME to decide the runtime to use, which may be one of netfx, coreclr or mono. The parameters @@ -80,16 +80,13 @@ def set_default_runtime() -> None: """ from os import environ - print("Set default RUNTIME") - raise RuntimeError("Shouldn't be called here") - spec = environ.get("PYTHONNET_RUNTIME", "default") runtime = _create_runtime_from_spec(spec) set_runtime(runtime) def load( - runtime: Union[clr_loader.Runtime, str] = "default", **params: Dict[str, str] + runtime: Union[clr_loader.Runtime, str, None] = None, **params: str ) -> None: """Load Python.NET in the specified runtime @@ -102,7 +99,10 @@ def load( return if _RUNTIME is None: - set_runtime(runtime, **params) + if runtime is None: + set_runtime_from_env() + else: + set_runtime(runtime, **params) if _RUNTIME is None: raise RuntimeError("No valid runtime selected") From 14aae2ebc8de4fd5f9eaac520573d89bbfc89a3e Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Wed, 6 Jul 2022 08:32:51 +0200 Subject: [PATCH 04/34] Ensure that version.txt is always read from repo root Allows the project to be referenced in other .NET projects without adjusting its project file (#1853). --- Directory.Build.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Build.props b/Directory.Build.props index 8c5b53685..965610f91 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -6,7 +6,7 @@ Python.NET 10.0 false - $([System.IO.File]::ReadAllText("version.txt")) + $([System.IO.File]::ReadAllText($(MSBuildThisFileDirectory)version.txt)) $(FullVersion.Split('-', 2)[0]) $(FullVersion.Split('-', 2)[1]) From 60463a31b28b645114e17f9f55a8283d9ca74257 Mon Sep 17 00:00:00 2001 From: Tim Gates Date: Sun, 17 Jul 2022 09:09:05 +1000 Subject: [PATCH 05/34] docs: Fix a few typos There are small typos in: - pythonnet/__init__.py - tests/test_import.py Fixes: - Should read `splitted` rather than `splited`. - Should read `loaded` rather than `laoded`. Signed-off-by: Tim Gates --- pythonnet/__init__.py | 2 +- tests/test_import.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pythonnet/__init__.py b/pythonnet/__init__.py index 9876a0bec..8f3478713 100644 --- a/pythonnet/__init__.py +++ b/pythonnet/__init__.py @@ -121,7 +121,7 @@ def load( def unload() -> None: - """Explicitly unload a laoded runtime and shut down Python.NET""" + """Explicitly unload a loaded runtime and shut down Python.NET""" global _RUNTIME, _LOADER_ASSEMBLY if _LOADER_ASSEMBLY is not None: diff --git a/tests/test_import.py b/tests/test_import.py index 25877be15..877eacd84 100644 --- a/tests/test_import.py +++ b/tests/test_import.py @@ -15,7 +15,7 @@ def test_relative_missing_import(): def test_import_all_on_second_time(): """Test import all attributes after a normal import without '*'. - Due to import * only allowed at module level, the test body splited + Due to import * only allowed at module level, the test body splitted to a module file.""" from . import importtest del sys.modules[importtest.__name__] From 2c233efbeb1c78efb80e0b8066c5d2b5ce2eed0e Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Thu, 29 Sep 2022 17:13:43 +0200 Subject: [PATCH 06/34] Bump master version to 3.1.0-dev --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index 4a36342fc..0f9d6b15d 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -3.0.0 +3.1.0-dev From eb62787ff39792cb4b37b02a399bc8fe0e5b4a18 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Thu, 29 Sep 2022 17:14:14 +0200 Subject: [PATCH 07/34] Run documentation build from master --- .github/workflows/docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 21402a42e..5b782c8b4 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -24,7 +24,7 @@ jobs: path: doc/build/html/ deploy: - if: github.ref == 'refs/heads/release' + if: github.ref == 'refs/heads/master' runs-on: ubuntu-latest permissions: contents: read From a3b7789b6a575913941b74fe101e6529b0b70b9f Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Thu, 29 Sep 2022 17:20:49 +0200 Subject: [PATCH 08/34] Roll changelog over --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f0f212e6..caadec60d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,15 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. ### Added +### Changed + +### Fixed + + +## [3.0.0][] - 2022-09-29 + +### Added + - Ability to instantiate new .NET arrays using `Array[T](dim1, dim2, ...)` syntax - Python operator method will call C# operator method for supported binary and unary operators ([#1324][p1324]). - Add GetPythonThreadID and Interrupt methods in PythonEngine From 0119f1caa60462b89754f4ac052fa1b2e0e32cc3 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Fri, 30 Sep 2022 00:06:52 +0200 Subject: [PATCH 09/34] Specify supported Python version in project metadata --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 39c7c14fb..91f386fc6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,8 @@ dependencies = [ "clr_loader>=0.2.2,<0.3.0" ] +requires-python = ">=3.7, <3.11" + classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", From bf984f09b36f74b564528de82e5a33d6a6389a00 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Fri, 30 Sep 2022 02:09:28 +0200 Subject: [PATCH 10/34] Merge backports-2.5 changelog in and fix links --- CHANGELOG.md | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index caadec60d..5d33ee327 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. ### Fixed -## [3.0.0][] - 2022-09-29 +## [3.0.0](https://github.com/pythonnet/pythonnet/releases/tag/v3.0.0) - 2022-09-29 ### Added @@ -143,7 +143,24 @@ There is no need to specify it. - support for .NET Framework 4.0-4.6; Mono before 5.4. Python.NET now requires .NET Standard 2.0 (see [the matrix](https://docs.microsoft.com/en-us/dotnet/standard/net-standard#net-implementation-support)) -## [2.5.0][] - 2020-06-14 +## [2.5.2](https://github.com/pythonnet/pythonnet/releases/tag/v2.5.2) - 2021-02-05 + +Bugfix release. + +### Fixed +- Fix `object[]` parameters taking precedence when should not in overload resolution +- Empty parameter names (as can be generated from F#) do not cause crashes + +## [2.5.1](https://github.com/pythonnet/pythonnet/releases/tag/v2.5.1) - 2020-06-18 + +Bugfix release. + +### Fixed + +- Fix incorrect dereference of wrapper object in `tp_repr`, which may result in a program crash +- Fix incorrect dereference in params array handling + +## [2.5.0](https://github.com/pythonnet/pythonnet/releases/tag/v2.5.0) - 2020-06-14 This version improves performance on benchmarks significantly compared to 2.3. From b5c222ca7bda1e9d37c788286356a71db1f6e124 Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Thu, 13 Oct 2022 12:06:26 -0700 Subject: [PATCH 11/34] a few debug helper properties --- src/runtime/Finalizer.cs | 2 ++ src/runtime/PythonTypes/PyObject.cs | 17 ++++++++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/runtime/Finalizer.cs b/src/runtime/Finalizer.cs index f4b465ecb..713564f08 100644 --- a/src/runtime/Finalizer.cs +++ b/src/runtime/Finalizer.cs @@ -364,6 +364,8 @@ struct PendingFinalization { public IntPtr PyObj; public BorrowedReference Ref => new(PyObj); + public ManagedType? Managed => ManagedType.GetManagedObject(Ref); + public nint RefCount => Runtime.Refcount(Ref); public int RuntimeRun; #if TRACE_ALLOC public string StackTrace; diff --git a/src/runtime/PythonTypes/PyObject.cs b/src/runtime/PythonTypes/PyObject.cs index ce86753eb..bda2d9c02 100644 --- a/src/runtime/PythonTypes/PyObject.cs +++ b/src/runtime/PythonTypes/PyObject.cs @@ -1051,9 +1051,20 @@ public PyList Dir() return Runtime.GetManagedString(strval.BorrowOrThrow()); } - string? DebuggerDisplay => DebugUtil.HaveInterpreterLock() - ? this.ToString() - : $"pyobj at 0x{this.rawPtr:X} (get Py.GIL to see more info)"; + ManagedType? InternalManagedObject => ManagedType.GetManagedObject(this.Reference); + + string? DebuggerDisplay + { + get + { + if (DebugUtil.HaveInterpreterLock()) + return this.ToString(); + var obj = this.InternalManagedObject; + return obj is { } + ? obj.ToString() + : $"pyobj at 0x{this.rawPtr:X} (get Py.GIL to see more info)"; + } + } /// From 6838ee1ce2866d04d4a516801b1f2f7bc1aa2d55 Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Thu, 13 Oct 2022 13:36:17 -0700 Subject: [PATCH 12/34] fixed name collision for generated delegate dispatchers --- src/runtime/Util/CodeGenerator.cs | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/runtime/Util/CodeGenerator.cs b/src/runtime/Util/CodeGenerator.cs index 35a637113..6e0859da0 100644 --- a/src/runtime/Util/CodeGenerator.cs +++ b/src/runtime/Util/CodeGenerator.cs @@ -1,5 +1,7 @@ using System; using System.Collections.Generic; +using System.Globalization; +using System.Linq; using System.Reflection; using System.Reflection.Emit; using System.Threading; @@ -17,13 +19,15 @@ internal class CodeGenerator private readonly AssemblyBuilder aBuilder; private readonly ModuleBuilder mBuilder; + const string NamePrefix = "__Python_Runtime_Generated_"; + internal CodeGenerator() { - var aname = new AssemblyName { Name = "__CodeGenerator_Assembly" }; + var aname = new AssemblyName { Name = GetUniqueAssemblyName(NamePrefix + "Assembly") }; var aa = AssemblyBuilderAccess.Run; aBuilder = Thread.GetDomain().DefineDynamicAssembly(aname, aa); - mBuilder = aBuilder.DefineDynamicModule("__CodeGenerator_Module"); + mBuilder = aBuilder.DefineDynamicModule(NamePrefix + "Module"); } /// @@ -77,5 +81,20 @@ internal static void GenerateMarshalByRefsBack(ILGenerator il, IReadOnlyList(AppDomain.CurrentDomain + .GetAssemblies() + .Select(a => a.GetName().Name)); + for (int i = 0; i < int.MaxValue; i++) + { + string candidate = name + i.ToString(CultureInfo.InvariantCulture); + if (!taken.Contains(candidate)) + return candidate; + } + + throw new NotSupportedException("Too many assemblies"); + } } } From 5a28fd41db0bd1a78477717dbb91f1a10703a074 Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Thu, 13 Oct 2022 12:07:39 -0700 Subject: [PATCH 13/34] delete target object from event handler collections when it has no more event handlers fixes https://github.com/pythonnet/pythonnet/issues/1972 --- CHANGELOG.md | 2 + src/embed_tests/Events.cs | 67 ++++++++++++++++++++++ src/runtime/Util/EventHandlerCollection.cs | 4 ++ 3 files changed, 73 insertions(+) create mode 100644 src/embed_tests/Events.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d33ee327..9781f289c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. ### Fixed +- Fixed objects leaking when Python attached event handlers to them even if they were later removed + ## [3.0.0](https://github.com/pythonnet/pythonnet/releases/tag/v3.0.0) - 2022-09-29 diff --git a/src/embed_tests/Events.cs b/src/embed_tests/Events.cs new file mode 100644 index 000000000..c216f4214 --- /dev/null +++ b/src/embed_tests/Events.cs @@ -0,0 +1,67 @@ +using System; +using System.Diagnostics; +using System.Threading; + +using NUnit.Framework; + +using Python.Runtime; + +namespace Python.EmbeddingTest; + +public class Events +{ + [OneTimeSetUp] + public void SetUp() + { + PythonEngine.Initialize(); + } + + [OneTimeTearDown] + public void Dispose() + { + PythonEngine.Shutdown(); + } + + [Test] + public void UsingDoesNotLeak() + { + using var scope = Py.CreateScope(); + scope.Exec(@" +import gc + +from Python.EmbeddingTest import ClassWithEventHandler + +def event_handler(): + pass + +for _ in range(2000): + example = ClassWithEventHandler() + example.LeakEvent += event_handler + example.LeakEvent -= event_handler + del example + +gc.collect() +"); + Runtime.Runtime.TryCollectingGarbage(10); + Assert.AreEqual(0, ClassWithEventHandler.alive); + } +} + +public class ClassWithEventHandler +{ + internal static int alive; + + public event EventHandler LeakEvent; + private Array arr; // dummy array to exacerbate memory leak + + public ClassWithEventHandler() + { + Interlocked.Increment(ref alive); + this.arr = new int[800]; + } + + ~ClassWithEventHandler() + { + Interlocked.Decrement(ref alive); + } +} diff --git a/src/runtime/Util/EventHandlerCollection.cs b/src/runtime/Util/EventHandlerCollection.cs index 551893799..0cd03d0fd 100644 --- a/src/runtime/Util/EventHandlerCollection.cs +++ b/src/runtime/Util/EventHandlerCollection.cs @@ -99,6 +99,10 @@ internal bool RemoveEventHandler(BorrowedReference target, BorrowedReference han continue; } list.RemoveAt(i); + if (list.Count == 0) + { + Remove(key); + } return true; } From c422abdba471a114f04519c08abc7bba71163afb Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Sun, 18 Sep 2022 16:00:02 +0200 Subject: [PATCH 14/34] Add Python 3.11 type offsets --- src/runtime/Native/TypeOffset311.cs | 141 ++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 src/runtime/Native/TypeOffset311.cs diff --git a/src/runtime/Native/TypeOffset311.cs b/src/runtime/Native/TypeOffset311.cs new file mode 100644 index 000000000..7691236c9 --- /dev/null +++ b/src/runtime/Native/TypeOffset311.cs @@ -0,0 +1,141 @@ + +// Auto-generated by geninterop.py. +// DO NOT MODIFY BY HAND. + +// Python 3.11: ABI flags: '' + +// ReSharper disable InconsistentNaming +// ReSharper disable IdentifierTypo + +using System; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.InteropServices; + +using Python.Runtime.Native; + +namespace Python.Runtime +{ + [SuppressMessage("Style", "IDE1006:Naming Styles", + Justification = "Following CPython", + Scope = "type")] + + [StructLayout(LayoutKind.Sequential)] + internal class TypeOffset311 : GeneratedTypeOffsets, ITypeOffsets + { + public TypeOffset311() { } + // Auto-generated from PyHeapTypeObject in Python.h + public int ob_refcnt { get; private set; } + public int ob_type { get; private set; } + public int ob_size { get; private set; } + public int tp_name { get; private set; } + public int tp_basicsize { get; private set; } + public int tp_itemsize { get; private set; } + public int tp_dealloc { get; private set; } + public int tp_vectorcall_offset { get; private set; } + public int tp_getattr { get; private set; } + public int tp_setattr { get; private set; } + public int tp_as_async { get; private set; } + public int tp_repr { get; private set; } + public int tp_as_number { get; private set; } + public int tp_as_sequence { get; private set; } + public int tp_as_mapping { get; private set; } + public int tp_hash { get; private set; } + public int tp_call { get; private set; } + public int tp_str { get; private set; } + public int tp_getattro { get; private set; } + public int tp_setattro { get; private set; } + public int tp_as_buffer { get; private set; } + public int tp_flags { get; private set; } + public int tp_doc { get; private set; } + public int tp_traverse { get; private set; } + public int tp_clear { get; private set; } + public int tp_richcompare { get; private set; } + public int tp_weaklistoffset { get; private set; } + public int tp_iter { get; private set; } + public int tp_iternext { get; private set; } + public int tp_methods { get; private set; } + public int tp_members { get; private set; } + public int tp_getset { get; private set; } + public int tp_base { get; private set; } + public int tp_dict { get; private set; } + public int tp_descr_get { get; private set; } + public int tp_descr_set { get; private set; } + public int tp_dictoffset { get; private set; } + public int tp_init { get; private set; } + public int tp_alloc { get; private set; } + public int tp_new { get; private set; } + public int tp_free { get; private set; } + public int tp_is_gc { get; private set; } + public int tp_bases { get; private set; } + public int tp_mro { get; private set; } + public int tp_cache { get; private set; } + public int tp_subclasses { get; private set; } + public int tp_weaklist { get; private set; } + public int tp_del { get; private set; } + public int tp_version_tag { get; private set; } + public int tp_finalize { get; private set; } + public int tp_vectorcall { get; private set; } + public int am_await { get; private set; } + public int am_aiter { get; private set; } + public int am_anext { get; private set; } + public int am_send { get; private set; } + public int nb_add { get; private set; } + public int nb_subtract { get; private set; } + public int nb_multiply { get; private set; } + public int nb_remainder { get; private set; } + public int nb_divmod { get; private set; } + public int nb_power { get; private set; } + public int nb_negative { get; private set; } + public int nb_positive { get; private set; } + public int nb_absolute { get; private set; } + public int nb_bool { get; private set; } + public int nb_invert { get; private set; } + public int nb_lshift { get; private set; } + public int nb_rshift { get; private set; } + public int nb_and { get; private set; } + public int nb_xor { get; private set; } + public int nb_or { get; private set; } + public int nb_int { get; private set; } + public int nb_reserved { get; private set; } + public int nb_float { get; private set; } + public int nb_inplace_add { get; private set; } + public int nb_inplace_subtract { get; private set; } + public int nb_inplace_multiply { get; private set; } + public int nb_inplace_remainder { get; private set; } + public int nb_inplace_power { get; private set; } + public int nb_inplace_lshift { get; private set; } + public int nb_inplace_rshift { get; private set; } + public int nb_inplace_and { get; private set; } + public int nb_inplace_xor { get; private set; } + public int nb_inplace_or { get; private set; } + public int nb_floor_divide { get; private set; } + public int nb_true_divide { get; private set; } + public int nb_inplace_floor_divide { get; private set; } + public int nb_inplace_true_divide { get; private set; } + public int nb_index { get; private set; } + public int nb_matrix_multiply { get; private set; } + public int nb_inplace_matrix_multiply { get; private set; } + public int mp_length { get; private set; } + public int mp_subscript { get; private set; } + public int mp_ass_subscript { get; private set; } + public int sq_length { get; private set; } + public int sq_concat { get; private set; } + public int sq_repeat { get; private set; } + public int sq_item { get; private set; } + public int was_sq_slice { get; private set; } + public int sq_ass_item { get; private set; } + public int was_sq_ass_slice { get; private set; } + public int sq_contains { get; private set; } + public int sq_inplace_concat { get; private set; } + public int sq_inplace_repeat { get; private set; } + public int bf_getbuffer { get; private set; } + public int bf_releasebuffer { get; private set; } + public int name { get; private set; } + public int ht_slots { get; private set; } + public int qualname { get; private set; } + public int ht_cached_keys { get; private set; } + public int ht_module { get; private set; } + public int _ht_tpname { get; private set; } + } +} + From d6c024c8fb891024cbe7045268a0ca74351c547b Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Mon, 24 Oct 2022 09:53:59 -0700 Subject: [PATCH 15/34] explicit functions for exact type checks --- src/runtime/Runtime.cs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/runtime/Runtime.cs b/src/runtime/Runtime.cs index 6238119ff..7110f3cb0 100644 --- a/src/runtime/Runtime.cs +++ b/src/runtime/Runtime.cs @@ -1094,8 +1094,13 @@ internal static nint PyBuffer_SizeFromFormat(string format) internal static bool PyInt_Check(BorrowedReference ob) => PyObject_TypeCheck(ob, PyLongType); + internal static bool PyInt_CheckExact(BorrowedReference ob) + => PyObject_TypeCheckExact(ob, PyLongType); + internal static bool PyBool_Check(BorrowedReference ob) => PyObject_TypeCheck(ob, PyBoolType); + internal static bool PyBool_CheckExact(BorrowedReference ob) + => PyObject_TypeCheckExact(ob, PyBoolType); internal static NewReference PyInt_FromInt32(int value) => PyLong_FromLongLong(value); @@ -1141,6 +1146,8 @@ internal static NewReference PyLong_FromString(string value, int radix) internal static bool PyFloat_Check(BorrowedReference ob) => PyObject_TypeCheck(ob, PyFloatType); + internal static bool PyFloat_CheckExact(BorrowedReference ob) + => PyObject_TypeCheckExact(ob, PyFloatType); /// /// Return value: New reference. @@ -1282,9 +1289,9 @@ internal static bool PyFloat_Check(BorrowedReference ob) // Python string API //==================================================================== internal static bool PyString_Check(BorrowedReference ob) - { - return PyObject_TYPE(ob) == PyStringType; - } + => PyObject_TypeCheck(ob, PyStringType); + internal static bool PyString_CheckExact(BorrowedReference ob) + => PyObject_TypeCheckExact(ob, PyStringType); internal static NewReference PyString_FromString(string value) { @@ -1643,6 +1650,8 @@ internal static bool PyType_IsSubtype(BorrowedReference t1, BorrowedReference t2 return Delegates.PyType_IsSubtype(t1, t2); } + internal static bool PyObject_TypeCheckExact(BorrowedReference ob, BorrowedReference tp) + => PyObject_TYPE(ob) == tp; internal static bool PyObject_TypeCheck(BorrowedReference ob, BorrowedReference tp) { BorrowedReference t = PyObject_TYPE(ob); From 782a0e5e01bd118043497e7327d09c355f101631 Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Mon, 24 Oct 2022 09:56:23 -0700 Subject: [PATCH 16/34] allow decoders to override conversion of types derived from primitives when target type is System.Object useful to be able to change what numpy.float64 is converted to related to https://github.com/pythonnet/pythonnet/issues/1957 this is an alternative to https://github.com/pythonnet/pythonnet/pull/1958 --- src/embed_tests/Codecs.cs | 13 +++++++++++++ src/runtime/Converter.cs | 24 ++++++++++++++++++++---- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/src/embed_tests/Codecs.cs b/src/embed_tests/Codecs.cs index be416bc15..9b764d43f 100644 --- a/src/embed_tests/Codecs.cs +++ b/src/embed_tests/Codecs.cs @@ -355,6 +355,19 @@ from datetime import datetime scope.Exec("Codecs.AcceptsDateTime(datetime(2021, 1, 22))"); } + [Test] + public void FloatDerivedDecoded() + { + using var scope = Py.CreateScope(); + scope.Exec(@"class FloatDerived(float): pass"); + using var floatDerived = scope.Eval("FloatDerived"); + var decoder = new DecoderReturningPredefinedValue(floatDerived, 42); + PyObjectConversions.RegisterDecoder(decoder); + using var result = scope.Eval("FloatDerived()"); + object decoded = result.As(); + Assert.AreEqual(42, decoded); + } + [Test] public void ExceptionDecodedNoInstance() { diff --git a/src/runtime/Converter.cs b/src/runtime/Converter.cs index 3c46e9034..73bbd4a3a 100644 --- a/src/runtime/Converter.cs +++ b/src/runtime/Converter.cs @@ -361,28 +361,44 @@ internal static bool ToManagedValue(BorrowedReference value, Type obType, // conversions (Python string -> managed string). if (obType == objectType) { - if (Runtime.PyString_Check(value)) + if (Runtime.PyString_CheckExact(value)) { return ToPrimitive(value, stringType, out result, setError); } - if (Runtime.PyBool_Check(value)) + if (Runtime.PyBool_CheckExact(value)) { return ToPrimitive(value, boolType, out result, setError); } - if (Runtime.PyFloat_Check(value)) + if (Runtime.PyFloat_CheckExact(value)) { return ToPrimitive(value, doubleType, out result, setError); } - // give custom codecs a chance to take over conversion of ints and sequences + // give custom codecs a chance to take over conversion + // of ints, sequences, and types derived from primitives BorrowedReference pyType = Runtime.PyObject_TYPE(value); if (PyObjectConversions.TryDecode(value, pyType, obType, out result)) { return true; } + if (Runtime.PyString_Check(value)) + { + return ToPrimitive(value, stringType, out result, setError); + } + + if (Runtime.PyBool_Check(value)) + { + return ToPrimitive(value, boolType, out result, setError); + } + + if (Runtime.PyFloat_Check(value)) + { + return ToPrimitive(value, doubleType, out result, setError); + } + if (Runtime.PyInt_Check(value)) { result = new PyInt(value); From 2b5291000e0fae50bdf3db3e9a7a38157b319ba6 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Sun, 18 Sep 2022 16:03:40 +0200 Subject: [PATCH 17/34] Add Python 3.11 to metadata and workflows --- .github/workflows/main.yml | 2 +- pyproject.toml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2cc793621..ea93ce18c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ jobs: fail-fast: false matrix: os: [windows, ubuntu, macos] - python: ["3.7", "3.8", "3.9", "3.10"] + python: ["3.7", "3.8", "3.9", "3.10", "3.11"] platform: [x64, x86] exclude: - os: ubuntu diff --git a/pyproject.toml b/pyproject.toml index 91f386fc6..d0512cc45 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ classifiers = [ "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Operating System :: Microsoft :: Windows", "Operating System :: POSIX :: Linux", "Operating System :: MacOS :: MacOS X", From c0b4eb285ef3af1fdf90b0461e14c26b9969cf7a Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Mon, 19 Sep 2022 11:09:01 +0200 Subject: [PATCH 18/34] Improve geninterop script to handle new case in 3.11 --- tools/geninterop/geninterop.py | 144 +++++++++++++++++++-------------- 1 file changed, 83 insertions(+), 61 deletions(-) mode change 100644 => 100755 tools/geninterop/geninterop.py diff --git a/tools/geninterop/geninterop.py b/tools/geninterop/geninterop.py old mode 100644 new mode 100755 index 0c80c1904..78e4d45c2 --- a/tools/geninterop/geninterop.py +++ b/tools/geninterop/geninterop.py @@ -13,40 +13,26 @@ - clang """ -from __future__ import print_function - -import logging import os +import shutil import sys import sysconfig import subprocess -if sys.version_info.major > 2: - from io import StringIO -else: - from StringIO import StringIO - +from io import StringIO +from pathlib import Path from pycparser import c_ast, c_parser -_log = logging.getLogger() -logging.basicConfig(level=logging.DEBUG) - -PY_MAJOR = sys.version_info[0] -PY_MINOR = sys.version_info[1] - # rename some members from their C name when generating the C# _typeoffset_member_renames = { "ht_name": "name", - "ht_qualname": "qualname" + "ht_qualname": "qualname", + "getitem": "spec_cache_getitem", } def _check_output(*args, **kwargs): - """Check output wrapper for py2/py3 compatibility""" - output = subprocess.check_output(*args, **kwargs) - if PY_MAJOR == 2: - return output - return output.decode("ascii") + return subprocess.check_output(*args, **kwargs, encoding="utf8") class AstParser(object): @@ -92,7 +78,7 @@ def visit(self, node): self.visit_identifier(node) def visit_ast(self, ast): - for name, node in ast.children(): + for _name, node in ast.children(): self.visit(node) def visit_typedef(self, typedef): @@ -113,7 +99,7 @@ def visit_struct(self, struct): self.visit(decl) self._struct_members_stack.pop(0) self._struct_stack.pop(0) - elif self._ptr_decl_depth: + elif self._ptr_decl_depth or self._struct_members_stack: # the struct is empty, but add it as a member to the current # struct as the current member maybe a pointer to it. self._add_struct_member(struct.name) @@ -141,7 +127,8 @@ def _add_struct_member(self, type_name): current_struct = self._struct_stack[0] member_name = self._struct_members_stack[0] struct_members = self._struct_members.setdefault( - self._get_struct_name(current_struct), []) + self._get_struct_name(current_struct), [] + ) # get the node associated with this type node = None @@ -179,7 +166,6 @@ def _get_struct_name(self, node): class Writer(object): - def __init__(self): self._stream = StringIO() @@ -193,34 +179,47 @@ def to_string(self): return self._stream.getvalue() -def preprocess_python_headers(): +def preprocess_python_headers(*, cc=None, include_py=None): """Return Python.h pre-processed, ready for parsing. Requires clang. """ - fake_libc_include = os.path.join(os.path.dirname(__file__), - "fake_libc_include") + this_path = Path(__file__).parent + + fake_libc_include = this_path / "fake_libc_include" include_dirs = [fake_libc_include] - include_py = sysconfig.get_config_var("INCLUDEPY") + if cc is None: + cc = shutil.which("clang") + if cc is None: + cc = shutil.which("gcc") + if cc is None: + raise RuntimeError("No suitable C compiler found, need clang or gcc") + + if include_py is None: + include_py = sysconfig.get_config_var("INCLUDEPY") + include_py = Path(include_py) + include_dirs.append(include_py) - include_args = [c for p in include_dirs for c in ["-I", p]] + include_args = [c for p in include_dirs for c in ["-I", str(p)]] + # fmt: off defines = [ "-D", "__attribute__(x)=", "-D", "__inline__=inline", "-D", "__asm__=;#pragma asm", "-D", "__int64=long long", - "-D", "_POSIX_THREADS" + "-D", "_POSIX_THREADS", ] - if os.name == 'nt': + if sys.platform == "win32": defines.extend([ "-D", "__inline=inline", "-D", "__ptr32=", "-D", "__ptr64=", "-D", "__declspec(x)=", ]) + #fmt: on if hasattr(sys, "abiflags"): if "d" in sys.abiflags: @@ -228,8 +227,8 @@ def preprocess_python_headers(): if "u" in sys.abiflags: defines.extend(("-D", "PYTHON_WITH_WIDE_UNICODE")) - python_h = os.path.join(include_py, "Python.h") - cmd = ["clang", "-pthread"] + include_args + defines + ["-E", python_h] + python_h = include_py / "Python.h" + cmd = [cc, "-pthread"] + include_args + defines + ["-E", str(python_h)] # normalize as the parser doesn't like windows line endings. lines = [] @@ -240,16 +239,13 @@ def preprocess_python_headers(): return "\n".join(lines) - -def gen_interop_head(writer): +def gen_interop_head(writer, version, abi_flags): filename = os.path.basename(__file__) - abi_flags = getattr(sys, "abiflags", "").replace("m", "") - py_ver = "{0}.{1}".format(PY_MAJOR, PY_MINOR) - class_definition = """ -// Auto-generated by %s. + class_definition = f""" +// Auto-generated by {filename}. // DO NOT MODIFY BY HAND. -// Python %s: ABI flags: '%s' +// Python {".".join(version[:2])}: ABI flags: '{abi_flags}' // ReSharper disable InconsistentNaming // ReSharper disable IdentifierTypo @@ -261,7 +257,7 @@ def gen_interop_head(writer): using Python.Runtime.Native; namespace Python.Runtime -{""" % (filename, py_ver, abi_flags) +{{""" writer.extend(class_definition) @@ -271,25 +267,24 @@ def gen_interop_tail(writer): writer.extend(tail) -def gen_heap_type_members(parser, writer, type_name = None): +def gen_heap_type_members(parser, writer, type_name): """Generate the TypeOffset C# class""" members = parser.get_struct_members("PyHeapTypeObject") - type_name = type_name or "TypeOffset{0}{1}".format(PY_MAJOR, PY_MINOR) - class_definition = """ + class_definition = f""" [SuppressMessage("Style", "IDE1006:Naming Styles", Justification = "Following CPython", Scope = "type")] [StructLayout(LayoutKind.Sequential)] - internal class {0} : GeneratedTypeOffsets, ITypeOffsets + internal class {type_name} : GeneratedTypeOffsets, ITypeOffsets {{ - public {0}() {{ }} + public {type_name}() {{ }} // Auto-generated from PyHeapTypeObject in Python.h -""".format(type_name) +""" # All the members are sizeof(void*) so we don't need to do any # extra work to determine the size based on the type. - for name, tpy in members: + for name, _type in members: name = _typeoffset_member_renames.get(name, name) class_definition += " public int %s { get; private set; }\n" % name @@ -304,17 +299,18 @@ def gen_structure_code(parser, writer, type_name, indent): return False out = writer.append out(indent, "[StructLayout(LayoutKind.Sequential)]") - out(indent, "internal struct %s" % type_name) + out(indent, f"internal struct {type_name}") out(indent, "{") - for name, tpy in members: - out(indent + 1, "public IntPtr %s;" % name) + for name, _type in members: + out(indent + 1, f"public IntPtr {name};") out(indent, "}") out() return True -def main(): + +def main(*, cc=None, include_py=None, version=None, out=None): # preprocess Python.h and build the AST - python_h = preprocess_python_headers() + python_h = preprocess_python_headers(cc=cc, include_py=include_py) parser = c_parser.CParser() ast = parser.parse(python_h) @@ -323,21 +319,47 @@ def main(): ast_parser.visit(ast) writer = Writer() + + if include_py and not version: + raise RuntimeError("If the include path is overridden, version must be " + "defined" + ) + + if version: + version = version.split('.') + else: + version = sys.version_info + # generate the C# code - offsets_type_name = "NativeTypeOffset" if len(sys.argv) > 1 else None - gen_interop_head(writer) + abi_flags = getattr(sys, "abiflags", "").replace("m", "") + gen_interop_head(writer, version, abi_flags) - gen_heap_type_members(ast_parser, writer, type_name = offsets_type_name) + type_name = f"TypeOffset{version[0]}{version[1]}{abi_flags}" + gen_heap_type_members(ast_parser, writer, type_name) gen_interop_tail(writer) interop_cs = writer.to_string() - if len(sys.argv) > 1: - with open(sys.argv[1], "w") as fh: - fh.write(interop_cs) - else: + if not out or out == "-": print(interop_cs) + else: + with open(out, "w") as fh: + fh.write(interop_cs) if __name__ == "__main__": - sys.exit(main()) + import argparse + + a = argparse.ArgumentParser("Interop file generator for Python.NET") + a.add_argument("--cc", help="C compiler to use, either clang or gcc") + a.add_argument("--include-py", help="Include path of Python") + a.add_argument("--version", help="Python version") + a.add_argument("--out", help="Output path", default="-") + args = a.parse_args() + + sys.exit(main( + cc=args.cc, + include_py=args.include_py, + out=args.out, + version=args.version + )) From cc8606856278a511a2965c8224032cab40c6958d Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Mon, 19 Sep 2022 11:09:18 +0200 Subject: [PATCH 19/34] Fix offsets for 3.11 --- src/runtime/Native/TypeOffset311.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/Native/TypeOffset311.cs b/src/runtime/Native/TypeOffset311.cs index 7691236c9..de5afacb9 100644 --- a/src/runtime/Native/TypeOffset311.cs +++ b/src/runtime/Native/TypeOffset311.cs @@ -136,6 +136,6 @@ public TypeOffset311() { } public int ht_cached_keys { get; private set; } public int ht_module { get; private set; } public int _ht_tpname { get; private set; } + public int spec_cache_getitem { get; private set; } } } - From 5636262b27f883c484369c89ecdbcd54afa97a56 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Sat, 15 Oct 2022 21:41:55 +0200 Subject: [PATCH 20/34] Update requires-python --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d0512cc45..52f1adb18 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ dependencies = [ "clr_loader>=0.2.2,<0.3.0" ] -requires-python = ">=3.7, <3.11" +requires-python = ">=3.7, <3.12" classifiers = [ "Development Status :: 5 - Production/Stable", From 8668579834623a1ef196840b8f2a93407f52701b Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Sat, 15 Oct 2022 22:13:30 +0200 Subject: [PATCH 21/34] Define slots before initialization --- src/runtime/PythonTypes/PyType.cs | 1 + src/runtime/TypeManager.cs | 23 +++++++++++++---------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/runtime/PythonTypes/PyType.cs b/src/runtime/PythonTypes/PyType.cs index 260800592..af796a5c5 100644 --- a/src/runtime/PythonTypes/PyType.cs +++ b/src/runtime/PythonTypes/PyType.cs @@ -155,6 +155,7 @@ private static StolenReference FromSpec(TypeSpec spec, PyTuple? bases = null) using var nativeSpec = new NativeTypeSpec(spec); var basesRef = bases is null ? default : bases.Reference; var result = Runtime.PyType_FromSpecWithBases(in nativeSpec, basesRef); + // Runtime.PyErr_Print(); return result.StealOrThrow(); } } diff --git a/src/runtime/TypeManager.cs b/src/runtime/TypeManager.cs index 217b4820e..6170f820f 100644 --- a/src/runtime/TypeManager.cs +++ b/src/runtime/TypeManager.cs @@ -475,17 +475,20 @@ internal static PyType CreateMetatypeWithGCHandleOffset() int size = Util.ReadInt32(Runtime.PyTypeType, TypeOffset.tp_basicsize) + IntPtr.Size // tp_clr_inst_offset ; - var result = new PyType(new TypeSpec("clr._internal.GCOffsetBase", basicSize: size, - new TypeSpec.Slot[] - { - - }, - TypeFlags.Default | TypeFlags.HeapType | TypeFlags.HaveGC), - bases: new PyTuple(new[] { py_type })); - SetRequiredSlots(result, seen: new HashSet()); - - Runtime.PyType_Modified(result); + var slots = new[] { + new TypeSpec.Slot(TypeSlotID.tp_traverse, subtype_traverse), + new TypeSpec.Slot(TypeSlotID.tp_clear, subtype_clear) + }; + var result = new PyType( + new TypeSpec( + "clr._internal.GCOffsetBase", + basicSize: size, + slots: slots, + TypeFlags.Default | TypeFlags.HeapType | TypeFlags.HaveGC + ), + bases: new PyTuple(new[] { py_type }) + ); return result; } From 218610ea33ef22ec1836f4fd43c3223676de5e51 Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Fri, 28 Oct 2022 16:00:56 -0700 Subject: [PATCH 22/34] fixed positive PyInt converted to negative BigInteger BigInteger constructor uses the sign bit in the first byte. Since we explicitly handle the sign, the fix is to prepend a zero byte to the number, which does not change it, but ensures sign bit is zero. fixes https://github.com/pythonnet/pythonnet/issues/1990 --- CHANGELOG.md | 1 + src/embed_tests/TestPyInt.cs | 17 ++++++++++++----- src/runtime/PythonTypes/PyInt.cs | 2 +- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9781f289c..71c36b412 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. ### Fixed - Fixed objects leaking when Python attached event handlers to them even if they were later removed +- Fixed `PyInt` conversion to `BigInteger` and `System.String` produced incorrect result for values between 128 and 255. ## [3.0.0](https://github.com/pythonnet/pythonnet/releases/tag/v3.0.0) - 2022-09-29 diff --git a/src/embed_tests/TestPyInt.cs b/src/embed_tests/TestPyInt.cs index 822fe0715..c147e074b 100644 --- a/src/embed_tests/TestPyInt.cs +++ b/src/embed_tests/TestPyInt.cs @@ -191,16 +191,23 @@ public void ToBigInteger() { 0, 1, 2, 0x10, + 0x79, + 0x80, + 0x81, + 0xFF, 0x123, + 0x8000, 0x1234, + 0x8001, + 0x4000, + 0xFF, }; simpleValues = simpleValues.Concat(simpleValues.Select(v => -v)).ToArray(); - foreach (var val in simpleValues) - { - var pyInt = new PyInt(val); - Assert.AreEqual((BigInteger)val, pyInt.ToBigInteger()); - } + var expected = simpleValues.Select(v => new BigInteger(v)).ToArray(); + var actual = simpleValues.Select(v => new PyInt(v).ToBigInteger()).ToArray(); + + CollectionAssert.AreEqual(expected, actual); } [Test] diff --git a/src/runtime/PythonTypes/PyInt.cs b/src/runtime/PythonTypes/PyInt.cs index 6b3dbf210..e71462b74 100644 --- a/src/runtime/PythonTypes/PyInt.cs +++ b/src/runtime/PythonTypes/PyInt.cs @@ -212,7 +212,7 @@ public BigInteger ToBigInteger() offset++; neg = true; } - byte[] littleEndianBytes = new byte[(hex.Length - offset + 1) / 2]; + byte[] littleEndianBytes = new byte[(hex.Length - offset + 1) / 2 + 1]; for (; offset < hex.Length; offset++) { int littleEndianHexIndex = hex.Length - 1 - offset; From ddf5a701d2542c81b6bf0dcc6ad4a5a440713408 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Sun, 30 Oct 2022 21:36:03 +0100 Subject: [PATCH 23/34] Only clear dict if tp_dictoffset > 0 --- src/runtime/Types/ManagedType.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/runtime/Types/ManagedType.cs b/src/runtime/Types/ManagedType.cs index 2ed9d7970..97a19497c 100644 --- a/src/runtime/Types/ManagedType.cs +++ b/src/runtime/Types/ManagedType.cs @@ -148,8 +148,9 @@ protected static void ClearObjectDict(BorrowedReference ob) { BorrowedReference type = Runtime.PyObject_TYPE(ob); int instanceDictOffset = Util.ReadInt32(type, TypeOffset.tp_dictoffset); - Debug.Assert(instanceDictOffset > 0); - Runtime.Py_CLEAR(ob, instanceDictOffset); + // Debug.Assert(instanceDictOffset > 0); + if (instanceDictOffset > 0) + Runtime.Py_CLEAR(ob, instanceDictOffset); } protected static BorrowedReference GetObjectDict(BorrowedReference ob) From d3b56ffeeb35091365676362665460268d087a87 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Sun, 30 Oct 2022 21:40:26 +0100 Subject: [PATCH 24/34] Ensure that sub-processes in tests use the same runtime settings --- tests/conftest.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index e61e3680e..1ac20e1dd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -79,9 +79,10 @@ def pytest_configure(config): ["dotnet", "publish", "-f", fw, "-o", str(bin_path), str(test_proj_path)] ) - from pythonnet import load - - load(runtime_opt, **runtime_params) + import os + os.environ["PYTHONNET_RUNTIME"] = runtime_opt + for k, v in runtime_params.items(): + os.environ[f"PYTHONNET_{runtime_opt.upper()}_{k.upper()}"] = v import clr From a6efeaee77404b75e557bae5cf795cbdd704bd00 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Sun, 30 Oct 2022 21:40:42 +0100 Subject: [PATCH 25/34] Update MaxSupportedVersion --- src/runtime/PythonEngine.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/PythonEngine.cs b/src/runtime/PythonEngine.cs index e5879ae67..ddf597c4f 100644 --- a/src/runtime/PythonEngine.cs +++ b/src/runtime/PythonEngine.cs @@ -128,7 +128,7 @@ public static string PythonPath } public static Version MinSupportedVersion => new(3, 7); - public static Version MaxSupportedVersion => new(3, 10, int.MaxValue, int.MaxValue); + public static Version MaxSupportedVersion => new(3, 11, int.MaxValue, int.MaxValue); public static bool IsSupportedVersion(Version version) => version >= MinSupportedVersion && version <= MaxSupportedVersion; public static string Version From da082ac487190aed5d325a346b0a6c268c870020 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Sun, 30 Oct 2022 23:16:57 +0100 Subject: [PATCH 26/34] Enforce tp_traverse/clear in AllocateTypeObject --- src/runtime/TypeManager.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/runtime/TypeManager.cs b/src/runtime/TypeManager.cs index 6170f820f..e0a78ba49 100644 --- a/src/runtime/TypeManager.cs +++ b/src/runtime/TypeManager.cs @@ -620,6 +620,11 @@ internal static PyType AllocateTypeObject(string name, PyType metatype) Util.WriteRef(type, TypeOffset.name, new NewReference(temp).Steal()); Util.WriteRef(type, TypeOffset.qualname, temp.Steal()); + // Ensure that tp_traverse and tp_clear are always set, since their + // existence is enforced in newer Python versions in PyType_Ready + Util.WriteIntPtr(type, TypeOffset.tp_traverse, subtype_traverse); + Util.WriteIntPtr(type, TypeOffset.tp_clear, subtype_clear); + InheritSubstructs(type.Reference.DangerousGetAddress()); return type; From e9283e3ec35184c7c4464540756f3d20c1301bfd Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Mon, 31 Oct 2022 14:51:18 +0100 Subject: [PATCH 27/34] Ensure that Python is initialized before probing properties --- src/embed_tests/TestPythonEngineProperties.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/embed_tests/TestPythonEngineProperties.cs b/src/embed_tests/TestPythonEngineProperties.cs index ca9164a1d..bbaff047f 100644 --- a/src/embed_tests/TestPythonEngineProperties.cs +++ b/src/embed_tests/TestPythonEngineProperties.cs @@ -9,6 +9,7 @@ public class TestPythonEngineProperties [Test] public static void GetBuildinfoDoesntCrash() { + PythonEngine.Initialize(); using (Py.GIL()) { string s = PythonEngine.BuildInfo; @@ -21,6 +22,7 @@ public static void GetBuildinfoDoesntCrash() [Test] public static void GetCompilerDoesntCrash() { + PythonEngine.Initialize(); using (Py.GIL()) { string s = PythonEngine.Compiler; @@ -34,6 +36,7 @@ public static void GetCompilerDoesntCrash() [Test] public static void GetCopyrightDoesntCrash() { + PythonEngine.Initialize(); using (Py.GIL()) { string s = PythonEngine.Copyright; @@ -46,6 +49,7 @@ public static void GetCopyrightDoesntCrash() [Test] public static void GetPlatformDoesntCrash() { + PythonEngine.Initialize(); using (Py.GIL()) { string s = PythonEngine.Platform; @@ -58,6 +62,7 @@ public static void GetPlatformDoesntCrash() [Test] public static void GetVersionDoesntCrash() { + PythonEngine.Initialize(); using (Py.GIL()) { string s = PythonEngine.Version; From cc97b8a49646e952a774e105f77d00f21761d864 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Tue, 1 Nov 2022 14:46:39 +0100 Subject: [PATCH 28/34] Add an Action variant of TryUsingDll --- src/runtime/Runtime.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/runtime/Runtime.cs b/src/runtime/Runtime.cs index 6238119ff..26e83a5f9 100644 --- a/src/runtime/Runtime.cs +++ b/src/runtime/Runtime.cs @@ -672,6 +672,9 @@ internal static unsafe nint Refcount(BorrowedReference op) [Pure] internal static int Refcount32(BorrowedReference op) => checked((int)Refcount(op)); + internal static void TryUsingDll(Action op) => + TryUsingDll(() => { op(); return 0; }); + /// /// Call specified function, and handle PythonDLL-related failures. /// From 096f50a21e235b6a8a15c011d0a390468901daa2 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Wed, 2 Nov 2022 06:36:34 +0100 Subject: [PATCH 29/34] Adjust code a bit and skip PythonHome tests for empty strings --- src/embed_tests/TestPythonEngineProperties.cs | 49 ++++++++++++------- src/runtime/PythonEngine.cs | 15 ++++-- 2 files changed, 41 insertions(+), 23 deletions(-) diff --git a/src/embed_tests/TestPythonEngineProperties.cs b/src/embed_tests/TestPythonEngineProperties.cs index bbaff047f..be91d7f45 100644 --- a/src/embed_tests/TestPythonEngineProperties.cs +++ b/src/embed_tests/TestPythonEngineProperties.cs @@ -96,9 +96,6 @@ public static void GetProgramNameDefault() /// Test default behavior of PYTHONHOME. If ENVVAR is set it will /// return the same value. If not, returns EmptyString. /// - /// - /// AppVeyor.yml has been update to tests with ENVVAR set. - /// [Test] public static void GetPythonHomeDefault() { @@ -114,22 +111,19 @@ public static void GetPythonHomeDefault() [Test] public void SetPythonHome() { - // We needs to ensure that engine was started and shutdown at least once before setting dummy home. - // Otherwise engine will not run with dummy path with random problem. - if (!PythonEngine.IsInitialized) - { - PythonEngine.Initialize(); - } - + PythonEngine.Initialize(); + var pythonHomeBackup = PythonEngine.PythonHome; PythonEngine.Shutdown(); - var pythonHomeBackup = PythonEngine.PythonHome; + if (pythonHomeBackup == "") + Assert.Inconclusive("Can't reset PythonHome to empty string, skipping"); var pythonHome = "/dummypath/"; PythonEngine.PythonHome = pythonHome; PythonEngine.Initialize(); + Assert.AreEqual(pythonHome, PythonEngine.PythonHome); PythonEngine.Shutdown(); // Restoring valid pythonhome. @@ -139,15 +133,12 @@ public void SetPythonHome() [Test] public void SetPythonHomeTwice() { - // We needs to ensure that engine was started and shutdown at least once before setting dummy home. - // Otherwise engine will not run with dummy path with random problem. - if (!PythonEngine.IsInitialized) - { - PythonEngine.Initialize(); - } + PythonEngine.Initialize(); + var pythonHomeBackup = PythonEngine.PythonHome; PythonEngine.Shutdown(); - var pythonHomeBackup = PythonEngine.PythonHome; + if (pythonHomeBackup == "") + Assert.Inconclusive("Can't reset PythonHome to empty string, skipping"); var pythonHome = "/dummypath/"; @@ -161,6 +152,26 @@ public void SetPythonHomeTwice() PythonEngine.PythonHome = pythonHomeBackup; } + [Test] + [Ignore("Currently buggy in Python")] + public void SetPythonHomeEmptyString() + { + PythonEngine.Initialize(); + + var backup = PythonEngine.PythonHome; + if (backup == "") + { + PythonEngine.Shutdown(); + Assert.Inconclusive("Can't reset PythonHome to empty string, skipping"); + } + PythonEngine.PythonHome = ""; + + Assert.AreEqual("", PythonEngine.PythonHome); + + PythonEngine.PythonHome = backup; + PythonEngine.Shutdown(); + } + [Test] public void SetProgramName() { @@ -207,7 +218,7 @@ public void SetPythonPath() // The list sys.path is initialized with this value on interpreter startup; // it can be (and usually is) modified later to change the search path for loading modules. // See https://docs.python.org/3/c-api/init.html#c.Py_GetPath - // After PythonPath is set, then PythonEngine.PythonPath will correctly return the full search path. + // After PythonPath is set, then PythonEngine.PythonPath will correctly return the full search path. PythonEngine.Shutdown(); diff --git a/src/runtime/PythonEngine.cs b/src/runtime/PythonEngine.cs index ddf597c4f..4ed45b9e9 100644 --- a/src/runtime/PythonEngine.cs +++ b/src/runtime/PythonEngine.cs @@ -47,6 +47,14 @@ public static bool IsInitialized get { return initialized; } } + private static void EnsureInitialized() + { + if (!IsInitialized) + throw new InvalidOperationException( + "Python must be initialized for this operation" + ); + } + /// Set to true to enable GIL debugging assistance. public static bool DebugGIL { get; set; } = false; @@ -96,6 +104,7 @@ public static string PythonHome { get { + EnsureInitialized(); IntPtr p = Runtime.TryUsingDll(() => Runtime.Py_GetPythonHome()); return UcsMarshaler.PtrToPy3UnicodePy2String(p) ?? ""; } @@ -103,10 +112,8 @@ public static string PythonHome { // this value is null in the beginning Marshal.FreeHGlobal(_pythonHome); - _pythonHome = Runtime.TryUsingDll( - () => UcsMarshaler.Py3UnicodePy2StringtoPtr(value) - ); - Runtime.Py_SetPythonHome(_pythonHome); + _pythonHome = UcsMarshaler.Py3UnicodePy2StringtoPtr(value); + Runtime.TryUsingDll(() => Runtime.Py_SetPythonHome(_pythonHome)); } } From 461c9c708ce36a054c363700f232a1b5df09ed2e Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Wed, 2 Nov 2022 07:59:03 +0100 Subject: [PATCH 30/34] Add tests for deriving a Python subclass from a generic interface --- src/testing/interfacetest.cs | 23 ++++++++++++++++++++++- tests/test_subclass.py | 23 ++++++++++++++++++++++- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/src/testing/interfacetest.cs b/src/testing/interfacetest.cs index 7c5d937b9..a1a7106b5 100644 --- a/src/testing/interfacetest.cs +++ b/src/testing/interfacetest.cs @@ -79,7 +79,7 @@ private interface IPrivate { } } - + public interface IOutArg { string MyMethod_Out(string name, out int index); @@ -93,4 +93,25 @@ public static int CallMyMethod_Out(IOutArg myInterface) return index; } } + + public interface IGenericInterface + { + public T Get(T x); + } + + public class SpecificInterfaceUser + { + public SpecificInterfaceUser(IGenericInterface some, int x) + { + some.Get(x); + } + } + + public class GenericInterfaceUser + { + public GenericInterfaceUser(IGenericInterface some, T x) + { + some.Get(x); + } + } } diff --git a/tests/test_subclass.py b/tests/test_subclass.py index fa82c3663..a51e89da3 100644 --- a/tests/test_subclass.py +++ b/tests/test_subclass.py @@ -9,7 +9,7 @@ import System import pytest from Python.Test import (IInterfaceTest, SubClassTest, EventArgsTest, - FunctionsTest) + FunctionsTest, IGenericInterface) from System.Collections.Generic import List @@ -29,6 +29,17 @@ def bar(self, x, i): return InterfaceTestClass +def interface_generic_class_fixture(subnamespace): + + class GenericInterfaceImpl(IGenericInterface[int]): + __namespace__ = "Python.Test." + subnamespace + + def Get(self, x): + return x + + return GenericInterfaceImpl + + def derived_class_fixture(subnamespace): """Delay creation of class until test starts.""" @@ -306,3 +317,13 @@ class Derived(BaseClass): import gc gc.collect() + +def test_generic_interface(): + from System import Int32 + from Python.Test import GenericInterfaceUser, SpecificInterfaceUser + + GenericInterfaceImpl = interface_generic_class_fixture(test_generic_interface.__name__) + + obj = GenericInterfaceImpl() + SpecificInterfaceUser(obj, Int32(0)) + GenericInterfaceUser[Int32](obj, Int32(0)) From 15e2e9596ccf6c23ea1bc27db4977055455c170d Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Wed, 2 Nov 2022 08:29:02 +0100 Subject: [PATCH 31/34] Set PYTHONHOME for tests --- .github/workflows/main.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ea93ce18c..93963c70a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -54,15 +54,17 @@ jobs: run: | pip install -v . - - name: Set Python DLL path (non Windows) + - name: Set Python DLL path and PYTHONHOME (non Windows) if: ${{ matrix.os != 'windows' }} run: | echo PYTHONNET_PYDLL=$(python -m find_libpython) >> $GITHUB_ENV + echo PYTHONHOME=$(python -c 'import sys; print(sys.prefix)') >> $GITHUB_ENV - - name: Set Python DLL path (Windows) + - name: Set Python DLL path and PYTHONHOME (Windows) if: ${{ matrix.os == 'windows' }} run: | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append -InputObject "PYTHONNET_PYDLL=$(python -m find_libpython)" + Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append -InputObject "PYTHONHOME=$(python -c 'import sys; print(sys.prefix)')" - name: Embedding tests run: dotnet test --runtime any-${{ matrix.platform }} --logger "console;verbosity=detailed" src/embed_tests/ From cc364c359beac37b218cd302af3b600f29e17b35 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Wed, 2 Nov 2022 09:08:32 +0100 Subject: [PATCH 32/34] Update changelog and add Python 3.11 to metadata --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 71c36b412..8ac8e6c06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. ### Added +- Support for Python 3.11 + ### Changed ### Fixed From 5c1e02f0b5d38def43e8547b5ca701d724f80c80 Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Wed, 2 Nov 2022 11:15:04 -0700 Subject: [PATCH 33/34] fixed resolution of generic methods in Python implementations since RuntimeMethodHandle does not encode generic arguments, I had to supply RuntimeTypeHandle of the declaring type to be able to get fully specified method --- src/runtime/Types/ClassDerived.cs | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/src/runtime/Types/ClassDerived.cs b/src/runtime/Types/ClassDerived.cs index 02288faee..cf6d9b16b 100644 --- a/src/runtime/Types/ClassDerived.cs +++ b/src/runtime/Types/ClassDerived.cs @@ -436,6 +436,7 @@ private static void AddVirtualMethod(MethodInfo method, Type baseType, TypeBuild il.Emit(OpCodes.Ldloc_0); il.Emit(OpCodes.Ldtoken, method); + il.Emit(OpCodes.Ldtoken, method.DeclaringType); #pragma warning disable CS0618 // PythonDerivedType is for internal use only if (method.ReturnType == typeof(void)) { @@ -505,6 +506,7 @@ private static void AddPythonMethod(string methodName, PyObject func, TypeBuilde il.DeclareLocal(typeof(object[])); il.DeclareLocal(typeof(RuntimeMethodHandle)); + il.DeclareLocal(typeof(RuntimeTypeHandle)); // this il.Emit(OpCodes.Ldarg_0); @@ -546,6 +548,11 @@ private static void AddPythonMethod(string methodName, PyObject func, TypeBuilde il.Emit(OpCodes.Ldloca_S, 1); il.Emit(OpCodes.Initobj, typeof(RuntimeMethodHandle)); il.Emit(OpCodes.Ldloc_1); + + // type handle is also not required + il.Emit(OpCodes.Ldloca_S, 2); + il.Emit(OpCodes.Initobj, typeof(RuntimeTypeHandle)); + il.Emit(OpCodes.Ldloc_2); #pragma warning disable CS0618 // PythonDerivedType is for internal use only // invoke the method @@ -698,7 +705,7 @@ public class PythonDerivedType /// class) it calls it, otherwise it calls the base method. /// public static T? InvokeMethod(IPythonDerivedType obj, string methodName, string origMethodName, - object[] args, RuntimeMethodHandle methodHandle) + object[] args, RuntimeMethodHandle methodHandle, RuntimeTypeHandle declaringTypeHandle) { var self = GetPyObj(obj); @@ -724,7 +731,10 @@ public class PythonDerivedType } PyObject py_result = method.Invoke(pyargs); - PyTuple? result_tuple = MarshalByRefsBack(args, methodHandle, py_result, outsOffset: 1); + var clrMethod = methodHandle != default + ? MethodBase.GetMethodFromHandle(methodHandle, declaringTypeHandle) + : null; + PyTuple? result_tuple = MarshalByRefsBack(args, clrMethod, py_result, outsOffset: 1); return result_tuple is not null ? result_tuple[0].As() : py_result.As(); @@ -754,7 +764,7 @@ public class PythonDerivedType } public static void InvokeMethodVoid(IPythonDerivedType obj, string methodName, string origMethodName, - object?[] args, RuntimeMethodHandle methodHandle) + object?[] args, RuntimeMethodHandle methodHandle, RuntimeTypeHandle declaringTypeHandle) { var self = GetPyObj(obj); if (null != self.Ref) @@ -779,7 +789,10 @@ public static void InvokeMethodVoid(IPythonDerivedType obj, string methodName, s } PyObject py_result = method.Invoke(pyargs); - MarshalByRefsBack(args, methodHandle, py_result, outsOffset: 0); + var clrMethod = methodHandle != default + ? MethodBase.GetMethodFromHandle(methodHandle, declaringTypeHandle) + : null; + MarshalByRefsBack(args, clrMethod, py_result, outsOffset: 0); return; } } @@ -811,12 +824,11 @@ public static void InvokeMethodVoid(IPythonDerivedType obj, string methodName, s /// as a tuple of new values for those arguments, and updates corresponding /// elements of array. /// - private static PyTuple? MarshalByRefsBack(object?[] args, RuntimeMethodHandle methodHandle, PyObject pyResult, int outsOffset) + private static PyTuple? MarshalByRefsBack(object?[] args, MethodBase? method, PyObject pyResult, int outsOffset) { - if (methodHandle == default) return null; + if (method is null) return null; - var originalMethod = MethodBase.GetMethodFromHandle(methodHandle); - var parameters = originalMethod.GetParameters(); + var parameters = method.GetParameters(); PyTuple? outs = null; int byrefIndex = 0; for (int i = 0; i < parameters.Length; ++i) From e28dd80c3a8fb60bd62671b765ff41c98ac5b1e9 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Thu, 3 Nov 2022 00:21:39 +0100 Subject: [PATCH 34/34] Release 3.0.1 --- CHANGELOG.md | 11 +++++++++++ version.txt | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ac8e6c06..13bf09c2c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,14 +9,25 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. ### Added +### Changed + +### Fixed + +## [3.0.1](https://github.com/pythonnet/pythonnet/releases/tag/v3.0.1) - 2022-11-03 + +### Added + - Support for Python 3.11 ### Changed +- Allow decoders to override conversion of types derived from primitive types + ### Fixed - Fixed objects leaking when Python attached event handlers to them even if they were later removed - Fixed `PyInt` conversion to `BigInteger` and `System.String` produced incorrect result for values between 128 and 255. +- Fixed implementing a generic interface with a Python class ## [3.0.0](https://github.com/pythonnet/pythonnet/releases/tag/v3.0.0) - 2022-09-29 diff --git a/version.txt b/version.txt index 0f9d6b15d..cb2b00e4f 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -3.1.0-dev +3.0.1 pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy