Python for .NET

From 73bf84ed7d37a748da1d81c057753ee6b0508863 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Fri, 3 May 2019 09:37:43 +0200 Subject: [PATCH 001/112] Back to dev version for 2.4.1 --- CHANGELOG.md | 8 ++++++++ setup.py | 2 +- src/SharedAssemblyInfo.cs | 2 +- src/clrmodule/ClrModule.cs | 2 +- src/clrmodule/clrmodule.15.csproj | 2 +- src/console/Console.15.csproj | 2 +- src/embed_tests/Python.EmbeddingTest.15.csproj | 2 +- src/runtime/Python.Runtime.15.csproj | 2 +- src/runtime/resources/clr.py | 2 +- src/testing/Python.Test.15.csproj | 2 +- 10 files changed, 17 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 289b864cb..f2d276e51 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,14 @@ This project adheres to [Semantic Versioning][]. This document follows the conventions laid out in [Keep a CHANGELOG][]. +## [unreleased][] + +### Added + +### Changed + +### Fixed + ## [2.4.0][] ### Added diff --git a/setup.py b/setup.py index 7b12997d9..beb930afb 100644 --- a/setup.py +++ b/setup.py @@ -622,7 +622,7 @@ def run(self): setup( name="pythonnet", - version="2.4.0", + version="2.4.1-dev", description=".Net and Mono integration for Python", url="https://pythonnet.github.io/", license="MIT", diff --git a/src/SharedAssemblyInfo.cs b/src/SharedAssemblyInfo.cs index 669a43746..dc72b0bdf 100644 --- a/src/SharedAssemblyInfo.cs +++ b/src/SharedAssemblyInfo.cs @@ -25,4 +25,4 @@ // Version Information. Keeping it simple. May need to revisit for Nuget // See: https://codingforsmarties.wordpress.com/2016/01/21/how-to-version-assemblies-destined-for-nuget/ // AssemblyVersion can only be numeric -[assembly: AssemblyVersion("2.4.0")] +[assembly: AssemblyVersion("2.4.1")] diff --git a/src/clrmodule/ClrModule.cs b/src/clrmodule/ClrModule.cs index 3bb3a533b..7fc654fd6 100644 --- a/src/clrmodule/ClrModule.cs +++ b/src/clrmodule/ClrModule.cs @@ -53,7 +53,7 @@ public static void initclr() { #if USE_PYTHON_RUNTIME_VERSION // Has no effect until SNK works. Keep updated anyways. - Version = new Version("2.4.0"), + Version = new Version("2.4.1"), #endif CultureInfo = CultureInfo.InvariantCulture }; diff --git a/src/clrmodule/clrmodule.15.csproj b/src/clrmodule/clrmodule.15.csproj index 2585ffdd2..326620c00 100644 --- a/src/clrmodule/clrmodule.15.csproj +++ b/src/clrmodule/clrmodule.15.csproj @@ -9,7 +9,7 @@ clrmodule clrmodule clrmodule - 2.4.0 + 2.4.1 false false false diff --git a/src/console/Console.15.csproj b/src/console/Console.15.csproj index ec5008036..4e765fea4 100644 --- a/src/console/Console.15.csproj +++ b/src/console/Console.15.csproj @@ -8,7 +8,7 @@ nPython Python.Runtime nPython - 2.4.0 + 2.4.1 false false false diff --git a/src/embed_tests/Python.EmbeddingTest.15.csproj b/src/embed_tests/Python.EmbeddingTest.15.csproj index a741a589e..4f6b2de46 100644 --- a/src/embed_tests/Python.EmbeddingTest.15.csproj +++ b/src/embed_tests/Python.EmbeddingTest.15.csproj @@ -10,7 +10,7 @@ Python.EmbeddingTest Python.EmbeddingTest Python.EmbeddingTest - 2.4.0 + 2.4.1 false false false diff --git a/src/runtime/Python.Runtime.15.csproj b/src/runtime/Python.Runtime.15.csproj index 29177b78c..fb0020356 100644 --- a/src/runtime/Python.Runtime.15.csproj +++ b/src/runtime/Python.Runtime.15.csproj @@ -8,7 +8,7 @@ Python.Runtime Python.Runtime Python.Runtime - 2.4.0 + 2.4.1 false false false diff --git a/src/runtime/resources/clr.py b/src/runtime/resources/clr.py index ddb0d94e8..45265226a 100644 --- a/src/runtime/resources/clr.py +++ b/src/runtime/resources/clr.py @@ -2,7 +2,7 @@ Code in this module gets loaded into the main clr module. """ -__version__ = "2.4.0" +__version__ = "2.4.1" class clrproperty(object): diff --git a/src/testing/Python.Test.15.csproj b/src/testing/Python.Test.15.csproj index da20ed2ef..8c23fe4b5 100644 --- a/src/testing/Python.Test.15.csproj +++ b/src/testing/Python.Test.15.csproj @@ -7,7 +7,7 @@ Python.Test Python.Test Python.Test - 2.4.0 + 2.4.1 bin\ false $(OutputPath)\$(AssemblyName).xml From f544adc4076b1c5530896a19e2a3fbf7b238c754 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Sat, 4 May 2019 10:19:34 +0200 Subject: [PATCH 002/112] Drop official 3.4 compatibility Closes #817 --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index beb930afb..8528753b0 100644 --- a/setup.py +++ b/setup.py @@ -647,7 +647,6 @@ def run(self): "Programming Language :: Python :: 2", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", From af419b10e191f6c7673524b4a2dc5e8dab7643dc Mon Sep 17 00:00:00 2001 From: Marco Rossi Date: Sat, 1 Jun 2019 15:31:47 +0200 Subject: [PATCH 003/112] Add shield of conda-forge package to README (#877) --- README.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index e59ad94f6..5366649ae 100644 --- a/README.rst +++ b/README.rst @@ -5,7 +5,7 @@ pythonnet - Python for .NET |appveyor shield| |travis shield| |codecov shield| -|license shield| |pypi package version| |python supported shield| +|license shield| |pypi package version| |conda-forge version| |python supported shield| |stackexchange shield| Python for .NET is a package that gives Python programmers nearly @@ -111,3 +111,5 @@ https://github.com/pythonnet/pythonnet/wiki :target: https://pypi.python.org/pypi/pythonnet .. |stackexchange shield| image:: https://img.shields.io/badge/StackOverflow-python.net-blue.svg :target: http://stackoverflow.com/questions/tagged/python.net +.. |conda-forge version| image:: https://img.shields.io/conda/vn/conda-forge/pythonnet.svg + :target: https://anaconda.org/conda-forge/pythonnet From 0d6194dce15ae5444697d0be80243a9d5f699504 Mon Sep 17 00:00:00 2001 From: inna-w <40801200+inna-w@users.noreply.github.com> Date: Mon, 3 Jun 2019 22:16:51 +0300 Subject: [PATCH 004/112] Generate NuGet package during build (#875) * Generate NuGet package during build * Comment out PackageReleaseNotes field * Update CHANGELOG and AUTHORS --- AUTHORS.md | 5 +++-- CHANGELOG.md | 2 ++ appveyor.yml | 1 + setup.py | 1 + src/runtime/Python.Runtime.15.csproj | 21 ++++++++++++++------- 5 files changed, 21 insertions(+), 9 deletions(-) diff --git a/AUTHORS.md b/AUTHORS.md index 27aae63f4..ba954b47d 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -27,9 +27,10 @@ - David Lechner ([@dlech](https://github.com/dlech)) - Dmitriy Se ([@dmitriyse](https://github.com/dmitriyse)) - He-chien Tsai ([@t3476](https://github.com/t3476)) -- Ivan Cronyn ([@cronan](https://github.com/cronan)) +- Inna Wiesel ([@inna-w](https://github.com/inna-w)) +- Ivan Cronyn ([@cronan](https://github.com/cronan)) - Jan Krivanek ([@jakrivan](https://github.com/jakrivan)) -- Jeff Reback ([@jreback](https://github.com/jreback)) +- Jeff Reback ([@jreback](https://github.com/jreback)) - Joe Frayne ([@jfrayne](https://github.com/jfrayne)) - John Burnett ([@johnburnett](https://github.com/johnburnett)) - John Wilkes ([@jbw3](https://github.com/jbw3)) diff --git a/CHANGELOG.md b/CHANGELOG.md index f2d276e51..b5531bf47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. ### Added +- Added automatic NuGet package generation in appveyor and local builds + ### Changed ### Fixed diff --git a/appveyor.yml b/appveyor.yml index 74b9a9c8e..445f9bb5a 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -72,6 +72,7 @@ on_finish: artifacts: - path: dist\* + - path: '.\src\runtime\bin\*.nupkg' notifications: - provider: Slack diff --git a/setup.py b/setup.py index 8528753b0..53c7f3f67 100644 --- a/setup.py +++ b/setup.py @@ -334,6 +334,7 @@ def build_extension(self, ext): ), '/p:PythonBuildDir="{}"'.format(os.path.abspath(dest_dir)), '/p:PythonInteropFile="{}"'.format(os.path.basename(interop_file)), + "/p:PackageId=pythonnet_py{0}{1}_{2}".format(PY_MAJOR, PY_MINOR, ARCH), "/verbosity:{}".format(VERBOSITY), ] diff --git a/src/runtime/Python.Runtime.15.csproj b/src/runtime/Python.Runtime.15.csproj index fb0020356..a4d1773f7 100644 --- a/src/runtime/Python.Runtime.15.csproj +++ b/src/runtime/Python.Runtime.15.csproj @@ -7,14 +7,21 @@ net45 Python.Runtime Python.Runtime - Python.Runtime + pythonnet 2.4.1 - false - false - false - false - false - false + true + false + Python for .NET + Copyright (c) 2006-2019 the contributors of the 'Python for .NET' project + Python and CLR (.NET and Mono) cross-platform language interop + pythonnet + https://github.com/pythonnet/pythonnet/blob/master/LICENSE + https://github.com/pythonnet/pythonnet + git + + python interop dynamic dlr Mono pinvoke + https://raw.githubusercontent.com/pythonnet/pythonnet/master/src/console/python-clear.ico + https://pythonnet.github.io/ bin\ false $(OutputPath)\$(AssemblyName).xml From 43c972d50fa2526f5f9ad73c591f6170c0538854 Mon Sep 17 00:00:00 2001 From: Victor Date: Mon, 17 Jun 2019 12:31:11 -0700 Subject: [PATCH 005/112] Enable C# parameters of type `object` accept any argument, passed from Python (#853) * added a regression test for #881 https://github.com/pythonnet/pythonnet/issues/811 * when converting to object, wrap values of unknown type into PyObject instead of failing This enables overload resolution with object parameters to behave the same way PyObject parameters behave - e.g. allow any Python object to be passed as PyObject as fallback. Resolves #811 --- CHANGELOG.md | 1 + src/embed_tests/Python.EmbeddingTest.csproj | 1 + src/embed_tests/TestInstanceWrapping.cs | 58 +++++++++++++++++++++ src/runtime/converter.cs | 9 ++-- 4 files changed, 63 insertions(+), 6 deletions(-) create mode 100644 src/embed_tests/TestInstanceWrapping.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index b5531bf47..d8683622f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,7 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. - Reattach python exception traceback information (#545) - PythonEngine.Intialize will now call `Py_InitializeEx` with a default value of 0, so signals will not be configured by default on embedding. This is different from the previous behaviour, where `Py_Initialize` was called instead, which sets initSigs to 1. ([#449][i449]) - Refactored MethodBinder.Bind in preparation to make it extensible (#829) +- When calling C# from Python, enable passing argument of any type to a parameter of C# type `object` by wrapping it into `PyObject` instance. ([#881][i881]) - Look for installed Windows 10 sdk's during installation instead of relying on specific versions. ### Fixed diff --git a/src/embed_tests/Python.EmbeddingTest.csproj b/src/embed_tests/Python.EmbeddingTest.csproj index faa55fa27..d351709a4 100644 --- a/src/embed_tests/Python.EmbeddingTest.csproj +++ b/src/embed_tests/Python.EmbeddingTest.csproj @@ -89,6 +89,7 @@ + diff --git a/src/embed_tests/TestInstanceWrapping.cs b/src/embed_tests/TestInstanceWrapping.cs new file mode 100644 index 000000000..ec275d67a --- /dev/null +++ b/src/embed_tests/TestInstanceWrapping.cs @@ -0,0 +1,58 @@ +using System; +using System.Globalization; +using NUnit.Framework; +using Python.Runtime; + +namespace Python.EmbeddingTest +{ + public class TestInstanceWrapping + { + [OneTimeSetUp] + public void SetUp() + { + PythonEngine.Initialize(); + } + + [OneTimeTearDown] + public void Dispose() + { + PythonEngine.Shutdown(); + } + + // regression test for https://github.com/pythonnet/pythonnet/issues/811 + [Test] + public void OverloadResolution_UnknownToObject() + { + var overloaded = new Overloaded(); + using (Py.GIL()) + { + var o = overloaded.ToPython(); + + dynamic callWithSelf = PythonEngine.Eval("lambda o: o.ObjOrClass(KeyError())"); + callWithSelf(o); + Assert.AreEqual(Overloaded.Object, overloaded.Value); + } + } + + class Base {} + class Derived: Base { } + + class Overloaded: Derived + { + public int Value { get; set; } + public void IntOrStr(int arg) => this.Value = arg; + public void IntOrStr(string arg) => + this.Value = int.Parse(arg, NumberStyles.Integer, CultureInfo.InvariantCulture); + + public const int Object = 1; + public const int ConcreteClass = 2; + public void ObjOrClass(object _) => this.Value = Object; + public void ObjOrClass(Overloaded _) => this.Value = ConcreteClass; + + public const int Base = ConcreteClass + 1; + public const int Derived = Base + 1; + public void BaseOrDerived(Base _) => this.Value = Base; + public void BaseOrDerived(Derived _) => this.Value = Derived; + } + } +} diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 11c67bf82..1883dc32b 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -382,12 +382,9 @@ internal static bool ToManagedValue(IntPtr value, Type obType, return ToArray(value, typeof(object[]), out result, setError); } - if (setError) - { - Exceptions.SetError(Exceptions.TypeError, "value cannot be converted to Object"); - } - - return false; + Runtime.XIncref(value); // PyObject() assumes ownership + result = new PyObject(value); + return true; } // Conversion to 'Type' is done using the same mappings as above for objects. From e4e16a422a466b8ed5524f568859c3aba1f9d2ff Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Thu, 20 Jun 2019 17:59:28 +0200 Subject: [PATCH 006/112] Revert "Enable C# parameters of type `object` accept any argument, passed from Python (#853)" (#882) This reverts commit 43c972d50fa2526f5f9ad73c591f6170c0538854. --- CHANGELOG.md | 1 - src/embed_tests/Python.EmbeddingTest.csproj | 1 - src/embed_tests/TestInstanceWrapping.cs | 58 --------------------- src/runtime/converter.cs | 9 ++-- 4 files changed, 6 insertions(+), 63 deletions(-) delete mode 100644 src/embed_tests/TestInstanceWrapping.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index d8683622f..b5531bf47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,7 +41,6 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. - Reattach python exception traceback information (#545) - PythonEngine.Intialize will now call `Py_InitializeEx` with a default value of 0, so signals will not be configured by default on embedding. This is different from the previous behaviour, where `Py_Initialize` was called instead, which sets initSigs to 1. ([#449][i449]) - Refactored MethodBinder.Bind in preparation to make it extensible (#829) -- When calling C# from Python, enable passing argument of any type to a parameter of C# type `object` by wrapping it into `PyObject` instance. ([#881][i881]) - Look for installed Windows 10 sdk's during installation instead of relying on specific versions. ### Fixed diff --git a/src/embed_tests/Python.EmbeddingTest.csproj b/src/embed_tests/Python.EmbeddingTest.csproj index d351709a4..faa55fa27 100644 --- a/src/embed_tests/Python.EmbeddingTest.csproj +++ b/src/embed_tests/Python.EmbeddingTest.csproj @@ -89,7 +89,6 @@ - diff --git a/src/embed_tests/TestInstanceWrapping.cs b/src/embed_tests/TestInstanceWrapping.cs deleted file mode 100644 index ec275d67a..000000000 --- a/src/embed_tests/TestInstanceWrapping.cs +++ /dev/null @@ -1,58 +0,0 @@ -using System; -using System.Globalization; -using NUnit.Framework; -using Python.Runtime; - -namespace Python.EmbeddingTest -{ - public class TestInstanceWrapping - { - [OneTimeSetUp] - public void SetUp() - { - PythonEngine.Initialize(); - } - - [OneTimeTearDown] - public void Dispose() - { - PythonEngine.Shutdown(); - } - - // regression test for https://github.com/pythonnet/pythonnet/issues/811 - [Test] - public void OverloadResolution_UnknownToObject() - { - var overloaded = new Overloaded(); - using (Py.GIL()) - { - var o = overloaded.ToPython(); - - dynamic callWithSelf = PythonEngine.Eval("lambda o: o.ObjOrClass(KeyError())"); - callWithSelf(o); - Assert.AreEqual(Overloaded.Object, overloaded.Value); - } - } - - class Base {} - class Derived: Base { } - - class Overloaded: Derived - { - public int Value { get; set; } - public void IntOrStr(int arg) => this.Value = arg; - public void IntOrStr(string arg) => - this.Value = int.Parse(arg, NumberStyles.Integer, CultureInfo.InvariantCulture); - - public const int Object = 1; - public const int ConcreteClass = 2; - public void ObjOrClass(object _) => this.Value = Object; - public void ObjOrClass(Overloaded _) => this.Value = ConcreteClass; - - public const int Base = ConcreteClass + 1; - public const int Derived = Base + 1; - public void BaseOrDerived(Base _) => this.Value = Base; - public void BaseOrDerived(Derived _) => this.Value = Derived; - } - } -} diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 1883dc32b..11c67bf82 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -382,9 +382,12 @@ internal static bool ToManagedValue(IntPtr value, Type obType, return ToArray(value, typeof(object[]), out result, setError); } - Runtime.XIncref(value); // PyObject() assumes ownership - result = new PyObject(value); - return true; + if (setError) + { + Exceptions.SetError(Exceptions.TypeError, "value cannot be converted to Object"); + } + + return false; } // Conversion to 'Type' is done using the same mappings as above for objects. From 2bc514f6c5454bc9ff2b94709f5cda400c97442e Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Thu, 20 Jun 2019 19:29:20 +0200 Subject: [PATCH 007/112] Fix the failing test (#888) Warn instead of fail on issues with the garbage collector itself. --- src/embed_tests/TestFinalizer.cs | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/src/embed_tests/TestFinalizer.cs b/src/embed_tests/TestFinalizer.cs index 53838f315..650ee5686 100644 --- a/src/embed_tests/TestFinalizer.cs +++ b/src/embed_tests/TestFinalizer.cs @@ -45,7 +45,7 @@ public void CollectBasicObject() called = true; }; - Assert.IsFalse(called); + Assert.IsFalse(called, "The event handler was called before it was installed"); Finalizer.Instance.CollectOnce += handler; WeakReference shortWeak; @@ -55,13 +55,25 @@ public void CollectBasicObject() } FullGCCollect(); // The object has been resurrected - Assert.IsFalse(shortWeak.IsAlive); - Assert.IsTrue(longWeak.IsAlive); + Warn.If( + shortWeak.IsAlive, + "The referenced object is alive although it should have been collected", + shortWeak + ); + Assert.IsTrue( + longWeak.IsAlive, + "The reference object is not alive although it should still be", + longWeak + ); { var garbage = Finalizer.Instance.GetCollectedObjects(); - Assert.NotZero(garbage.Count); - Assert.IsTrue(garbage.Any(T => ReferenceEquals(T.Target, longWeak.Target))); + Assert.NotZero(garbage.Count, "There should still be garbage around"); + Warn.Unless( + garbage.Any(T => ReferenceEquals(T.Target, longWeak.Target)), + $"The {nameof(longWeak)} reference doesn't show up in the garbage list", + garbage + ); } try { @@ -71,7 +83,7 @@ public void CollectBasicObject() { Finalizer.Instance.CollectOnce -= handler; } - Assert.IsTrue(called); + Assert.IsTrue(called, "The event handler was not called during finalization"); Assert.GreaterOrEqual(objectCount, 1); } From fc7d8a466885ced4690327081695f708190e998a Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Mon, 24 Jun 2019 18:07:55 +0200 Subject: [PATCH 008/112] Support ARM architectures again (#887) --- src/runtime/runtime.cs | 4 ++++ src/runtime/typemanager.cs | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 7623200e0..294ecaf48 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -229,6 +229,8 @@ public enum MachineType { i386, x86_64, + armv7l, + armv8, Other }; @@ -247,6 +249,8 @@ public enum MachineType ["amd64"] = MachineType.x86_64, ["x64"] = MachineType.x86_64, ["em64t"] = MachineType.x86_64, + ["armv7l"] = MachineType.armv7l, + ["armv8"] = MachineType.armv8, }; ///

diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index d19c8737f..a260e8dfa 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -510,6 +510,10 @@ public static NativeCode Active return I386; case Runtime.MachineType.x86_64: return X86_64; + case Runtime.MachineType.armv7l: + return Armv7l; + case Runtime.MachineType.armv8: + return Armv8; default: throw new NotImplementedException($"No support for {Runtime.MachineName}"); } @@ -546,6 +550,34 @@ public static NativeCode Active /// ///

public static readonly NativeCode I386 = X86_64; + + public static readonly NativeCode Armv7l = new NativeCode() + { + Return0 = 0, + Return1 = 0x08, + Code = new byte[] + { + 0xe3, 0xa0, 0x00, 0x00, // mov r0, #0 + 0xe1, 0x2f, 0xff, 0x1e, // bx lr + + 0xe3, 0xa0, 0x00, 0x01, // mov r0, #1 + 0xe1, 0x2f, 0xff, 0x1e, // bx lr + } + }; + + public static readonly NativeCode Armv8 = new NativeCode() + { + Return0 = 0, + Return1 = 0x08, + Code = new byte[] + { + 0x52, 0x80, 0x00, 0x00, // mov w0, #0x0 + 0xd6, 0x5f, 0x03, 0xc0, // ret + + 0x52, 0x80, 0x00, 0x20, // mov w0, #0x1 + 0xd6, 0x5f, 0x03, 0xc0, // ret + } + }; } ///

From 1dd2ee1b02449b85eeee6120c88a4092dc78851a Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Tue, 25 Jun 2019 23:00:34 +0200 Subject: [PATCH 009/112] Get the correct library loading functions at runtime --- src/embed_tests/TestRuntime.cs | 7 +- src/runtime/Python.Runtime.csproj | 2 + src/runtime/platform/LibraryLoader.cs | 153 ++++++++++++++++++++++++++ src/runtime/platform/Types.cs | 22 ++++ src/runtime/runtime.cs | 128 ++------------------- src/runtime/typemanager.cs | 22 ++-- 6 files changed, 203 insertions(+), 131 deletions(-) create mode 100644 src/runtime/platform/LibraryLoader.cs create mode 100644 src/runtime/platform/Types.cs diff --git a/src/embed_tests/TestRuntime.cs b/src/embed_tests/TestRuntime.cs index ac1fa1ac0..25b70fac5 100644 --- a/src/embed_tests/TestRuntime.cs +++ b/src/embed_tests/TestRuntime.cs @@ -1,6 +1,7 @@ using System; using NUnit.Framework; using Python.Runtime; +using Python.Runtime.Platform; namespace Python.EmbeddingTest { @@ -26,10 +27,10 @@ public static void PlatformCache() { Runtime.Runtime.Initialize(); - Assert.That(Runtime.Runtime.Machine, Is.Not.EqualTo(Runtime.Runtime.MachineType.Other)); + Assert.That(Runtime.Runtime.Machine, Is.Not.EqualTo(MachineType.Other)); Assert.That(!string.IsNullOrEmpty(Runtime.Runtime.MachineName)); - Assert.That(Runtime.Runtime.OperatingSystem, Is.Not.EqualTo(Runtime.Runtime.OperatingSystemType.Other)); + Assert.That(Runtime.Runtime.OperatingSystem, Is.Not.EqualTo(OperatingSystemType.Other)); Assert.That(!string.IsNullOrEmpty(Runtime.Runtime.OperatingSystemName)); // Don't shut down the runtime: if the python engine was initialized @@ -39,7 +40,7 @@ public static void PlatformCache() [Test] public static void Py_IsInitializedValue() { - Runtime.Runtime.Py_Finalize(); + Runtime.Runtime.Py_Finalize(); Assert.AreEqual(0, Runtime.Runtime.Py_IsInitialized()); Runtime.Runtime.Py_Initialize(); Assert.AreEqual(1, Runtime.Runtime.Py_IsInitialized()); diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index 19f776c77..c4d63695f 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -140,6 +140,8 @@ + + diff --git a/src/runtime/platform/LibraryLoader.cs b/src/runtime/platform/LibraryLoader.cs new file mode 100644 index 000000000..c0157b04b --- /dev/null +++ b/src/runtime/platform/LibraryLoader.cs @@ -0,0 +1,153 @@ +using System; +using System.Runtime.InteropServices; + +namespace Python.Runtime.Platform +{ + interface ILibraryLoader + { + IntPtr Load(string dllToLoad); + + IntPtr GetFunction(IntPtr hModule, string procedureName); + + bool Free(IntPtr hModule); + } + + static class LibraryLoader + { + public static ILibraryLoader Get(OperatingSystemType os) + { + switch (os) + { + case OperatingSystemType.Windows: + return new WindowsLoader(); + case OperatingSystemType.Darwin: + return new DarwinLoader(); + case OperatingSystemType.Linux: + return new LinuxLoader(); + default: + throw new Exception($"This operating system ({os}) is not supported"); + } + } + } + + class LinuxLoader : ILibraryLoader + { + private static int RTLD_NOW = 0x2; + private static int RTLD_GLOBAL = 0x100; + private static IntPtr RTLD_DEFAULT = IntPtr.Zero; + private const string NativeDll = "libdl.so"; + + public IntPtr Load(string fileName) + { + return dlopen($"lib{fileName}.so", RTLD_NOW | RTLD_GLOBAL); + } + + public bool Free(IntPtr handle) + { + dlclose(handle); + return true; + } + + public IntPtr GetFunction(IntPtr dllHandle, string name) + { + // look in the exe if dllHandle is NULL + if (dllHandle == IntPtr.Zero) + { + dllHandle = RTLD_DEFAULT; + } + + // clear previous errors if any + dlerror(); + IntPtr res = dlsym(dllHandle, name); + IntPtr errPtr = dlerror(); + if (errPtr != IntPtr.Zero) + { + throw new Exception("dlsym: " + Marshal.PtrToStringAnsi(errPtr)); + } + return res; + } + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] + public static extern IntPtr dlopen(String fileName, int flags); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] + private static extern IntPtr dlsym(IntPtr handle, String symbol); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] + private static extern int dlclose(IntPtr handle); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] + private static extern IntPtr dlerror(); + } + + class DarwinLoader : ILibraryLoader + { + private static int RTLD_NOW = 0x2; + private static int RTLD_GLOBAL = 0x8; + private const string NativeDll = "/usr/lib/libSystem.dylib"; + private static IntPtr RTLD_DEFAULT = new IntPtr(-2); + + public IntPtr Load(string fileName) + { + return dlopen($"lib{fileName}.dylib", RTLD_NOW | RTLD_GLOBAL); + } + + public bool Free(IntPtr handle) + { + dlclose(handle); + return true; + } + + public IntPtr GetFunction(IntPtr dllHandle, string name) + { + // look in the exe if dllHandle is NULL + if (dllHandle == IntPtr.Zero) + { + dllHandle = RTLD_DEFAULT; + } + + // clear previous errors if any + dlerror(); + IntPtr res = dlsym(dllHandle, name); + IntPtr errPtr = dlerror(); + if (errPtr != IntPtr.Zero) + { + throw new Exception("dlsym: " + Marshal.PtrToStringAnsi(errPtr)); + } + return res; + } + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] + public static extern IntPtr dlopen(String fileName, int flags); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] + private static extern IntPtr dlsym(IntPtr handle, String symbol); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] + private static extern int dlclose(IntPtr handle); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] + private static extern IntPtr dlerror(); + } + + class WindowsLoader : ILibraryLoader + { + private const string NativeDll = "kernel32.dll"; + + [DllImport(NativeDll)] + static extern IntPtr LoadLibrary(string dllToLoad); + + public IntPtr Load(string dllToLoad) => WindowsLoader.LoadLibrary(dllToLoad); + + [DllImport(NativeDll)] + static extern IntPtr GetProcAddress(IntPtr hModule, string procedureName); + + public IntPtr GetFunction(IntPtr hModule, string procedureName) => WindowsLoader.GetProcAddress(hModule, procedureName); + + + [DllImport(NativeDll)] + static extern bool FreeLibrary(IntPtr hModule); + + public bool Free(IntPtr hModule) => WindowsLoader.FreeLibrary(hModule); + } +} diff --git a/src/runtime/platform/Types.cs b/src/runtime/platform/Types.cs new file mode 100644 index 000000000..bdc51af39 --- /dev/null +++ b/src/runtime/platform/Types.cs @@ -0,0 +1,22 @@ +namespace Python.Runtime.Platform +{ + public enum MachineType + { + i386, + x86_64, + armv7l, + armv8, + Other + }; + + ///

+ /// Operating system type as reported by Python. + ///

+ public enum OperatingSystemType + { + Windows, + Darwin, + Linux, + Other + } +} diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 294ecaf48..ec5bddfd0 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -7,96 +7,7 @@ namespace Python.Runtime { - [SuppressUnmanagedCodeSecurity] - internal static class NativeMethods - { -#if MONO_LINUX || MONO_OSX -#if NETSTANDARD - private static int RTLD_NOW = 0x2; -#if MONO_LINUX - private static int RTLD_GLOBAL = 0x100; - private static IntPtr RTLD_DEFAULT = IntPtr.Zero; - private const string NativeDll = "libdl.so"; - public static IntPtr LoadLibrary(string fileName) - { - return dlopen($"lib{fileName}.so", RTLD_NOW | RTLD_GLOBAL); - } -#elif MONO_OSX - private static int RTLD_GLOBAL = 0x8; - private const string NativeDll = "/usr/lib/libSystem.dylib"; - private static IntPtr RTLD_DEFAULT = new IntPtr(-2); - - public static IntPtr LoadLibrary(string fileName) - { - return dlopen($"lib{fileName}.dylib", RTLD_NOW | RTLD_GLOBAL); - } -#endif -#else - private static int RTLD_NOW = 0x2; - private static int RTLD_SHARED = 0x20; -#if MONO_OSX - private static IntPtr RTLD_DEFAULT = new IntPtr(-2); - private const string NativeDll = "__Internal"; -#elif MONO_LINUX - private static IntPtr RTLD_DEFAULT = IntPtr.Zero; - private const string NativeDll = "libdl.so"; -#endif - - public static IntPtr LoadLibrary(string fileName) - { - return dlopen(fileName, RTLD_NOW | RTLD_SHARED); - } -#endif - - - public static void FreeLibrary(IntPtr handle) - { - dlclose(handle); - } - - public static IntPtr GetProcAddress(IntPtr dllHandle, string name) - { - // look in the exe if dllHandle is NULL - if (dllHandle == IntPtr.Zero) - { - dllHandle = RTLD_DEFAULT; - } - - // clear previous errors if any - dlerror(); - IntPtr res = dlsym(dllHandle, name); - IntPtr errPtr = dlerror(); - if (errPtr != IntPtr.Zero) - { - throw new Exception("dlsym: " + Marshal.PtrToStringAnsi(errPtr)); - } - return res; - } - - [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] - public static extern IntPtr dlopen(String fileName, int flags); - - [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] - private static extern IntPtr dlsym(IntPtr handle, String symbol); - - [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int dlclose(IntPtr handle); - - [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr dlerror(); -#else // Windows - private const string NativeDll = "kernel32.dll"; - - [DllImport(NativeDll)] - public static extern IntPtr LoadLibrary(string dllToLoad); - - [DllImport(NativeDll)] - public static extern IntPtr GetProcAddress(IntPtr hModule, string procedureName); - - [DllImport(NativeDll)] - public static extern bool FreeLibrary(IntPtr hModule); -#endif - } + using Python.Runtime.Platform; ///

/// Encapsulates the low-level Python C API. Note that it is @@ -197,17 +108,6 @@ public class Runtime // .NET core: System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(OSPlatform.Windows) internal static bool IsWindows = Environment.OSVersion.Platform == PlatformID.Win32NT; - ///

- /// Operating system type as reported by Python. - ///

- public enum OperatingSystemType - { - Windows, - Darwin, - Linux, - Other - } - static readonly Dictionary OperatingSystemTypeMapping = new Dictionary() { { "Windows", OperatingSystemType.Windows }, @@ -225,14 +125,6 @@ public enum OperatingSystemType ///

public static string OperatingSystemName { get; private set; } - public enum MachineType - { - i386, - x86_64, - armv7l, - armv8, - Other - }; ///

/// Map lower-case version of the python machine name to the processor @@ -397,24 +289,24 @@ internal static void Initialize(bool initSigs = false) Error = new IntPtr(-1); + // Initialize data about the platform we're running on. We need + // this for the type manager and potentially other details. Must + // happen after caching the python types, above. + InitializePlatformData(); + IntPtr dllLocal = IntPtr.Zero; + var loader = LibraryLoader.Get(OperatingSystem); if (_PythonDll != "__Internal") { - dllLocal = NativeMethods.LoadLibrary(_PythonDll); + dllLocal = loader.Load(_PythonDll); } - _PyObject_NextNotImplemented = NativeMethods.GetProcAddress(dllLocal, "_PyObject_NextNotImplemented"); + _PyObject_NextNotImplemented = loader.GetFunction(dllLocal, "_PyObject_NextNotImplemented"); -#if !(MONO_LINUX || MONO_OSX) if (dllLocal != IntPtr.Zero) { - NativeMethods.FreeLibrary(dllLocal); + loader.Free(dllLocal); } -#endif - // Initialize data about the platform we're running on. We need - // this for the type manager and potentially other details. Must - // happen after caching the python types, above. - InitializePlatformData(); // Initialize modules that depend on the runtime class. AssemblyManager.Initialize(); diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index a260e8dfa..00a8f0a89 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -6,6 +6,8 @@ namespace Python.Runtime { + using Python.Runtime.Platform; + ///

/// The TypeManager class is responsible for building binary-compatible /// Python type objects that are implemented in managed code. @@ -504,15 +506,15 @@ public static NativeCode Active { get { - switch(Runtime.Machine) + switch (Runtime.Machine) { - case Runtime.MachineType.i386: + case MachineType.i386: return I386; - case Runtime.MachineType.x86_64: + case MachineType.x86_64: return X86_64; - case Runtime.MachineType.armv7l: + case MachineType.armv7l: return Armv7l; - case Runtime.MachineType.armv8: + case MachineType.armv8: return Armv8; default: throw new NotImplementedException($"No support for {Runtime.MachineName}"); @@ -635,9 +637,9 @@ int MAP_ANONYMOUS { switch (Runtime.OperatingSystem) { - case Runtime.OperatingSystemType.Darwin: + case OperatingSystemType.Darwin: return 0x1000; - case Runtime.OperatingSystemType.Linux: + case OperatingSystemType.Linux: return 0x20; default: throw new NotImplementedException($"mmap is not supported on {Runtime.OperatingSystemName}"); @@ -668,10 +670,10 @@ internal static IMemoryMapper CreateMemoryMapper() { switch (Runtime.OperatingSystem) { - case Runtime.OperatingSystemType.Darwin: - case Runtime.OperatingSystemType.Linux: + case OperatingSystemType.Darwin: + case OperatingSystemType.Linux: return new UnixMemoryMapper(); - case Runtime.OperatingSystemType.Windows: + case OperatingSystemType.Windows: return new WindowsMemoryMapper(); default: throw new NotImplementedException($"No support for {Runtime.OperatingSystemName}"); From 537ee5fae147c78d0221133f4db4de3371ebc319 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Wed, 26 Jun 2019 08:04:00 +0200 Subject: [PATCH 010/112] Implement error handling, move using statements --- src/runtime/platform/LibraryLoader.cs | 115 +++++++++++++++++++------- src/runtime/runtime.cs | 2 +- src/runtime/typemanager.cs | 2 +- 3 files changed, 87 insertions(+), 32 deletions(-) diff --git a/src/runtime/platform/LibraryLoader.cs b/src/runtime/platform/LibraryLoader.cs index c0157b04b..a6d88cd19 100644 --- a/src/runtime/platform/LibraryLoader.cs +++ b/src/runtime/platform/LibraryLoader.cs @@ -1,4 +1,5 @@ using System; +using System.ComponentModel; using System.Runtime.InteropServices; namespace Python.Runtime.Platform @@ -9,7 +10,7 @@ interface ILibraryLoader IntPtr GetFunction(IntPtr hModule, string procedureName); - bool Free(IntPtr hModule); + void Free(IntPtr hModule); } static class LibraryLoader @@ -25,7 +26,7 @@ public static ILibraryLoader Get(OperatingSystemType os) case OperatingSystemType.Linux: return new LinuxLoader(); default: - throw new Exception($"This operating system ({os}) is not supported"); + throw new PlatformNotSupportedException($"This operating system ({os}) is not supported"); } } } @@ -37,15 +38,23 @@ class LinuxLoader : ILibraryLoader private static IntPtr RTLD_DEFAULT = IntPtr.Zero; private const string NativeDll = "libdl.so"; - public IntPtr Load(string fileName) + public IntPtr Load(string dllToLoad) { - return dlopen($"lib{fileName}.so", RTLD_NOW | RTLD_GLOBAL); + var filename = $"lib{dllToLoad}.so"; + ClearError(); + var res = dlopen(filename, RTLD_NOW | RTLD_GLOBAL); + if (res == IntPtr.Zero) + { + var err = GetError(); + throw new DllNotFoundException($"Could not load {filename} with flags RTLD_NOW | RTLD_GLOBAL: {err}"); + } + + return res; } - public bool Free(IntPtr handle) + public void Free(IntPtr handle) { dlclose(handle); - return true; } public IntPtr GetFunction(IntPtr dllHandle, string name) @@ -56,22 +65,35 @@ public IntPtr GetFunction(IntPtr dllHandle, string name) dllHandle = RTLD_DEFAULT; } - // clear previous errors if any - dlerror(); + ClearError(); IntPtr res = dlsym(dllHandle, name); - IntPtr errPtr = dlerror(); - if (errPtr != IntPtr.Zero) + if (res == IntPtr.Zero) { - throw new Exception("dlsym: " + Marshal.PtrToStringAnsi(errPtr)); + var err = GetError(); + throw new MissingMethodException($"Failed to load symbol {name}: {err}"); } return res; } + void ClearError() + { + dlerror(); + } + + string GetError() + { + var res = dlerror(); + if (res != IntPtr.Zero) + return Marshal.PtrToStringAnsi(res); + else + return null; + } + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] - public static extern IntPtr dlopen(String fileName, int flags); + public static extern IntPtr dlopen(string fileName, int flags); [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] - private static extern IntPtr dlsym(IntPtr handle, String symbol); + private static extern IntPtr dlsym(IntPtr handle, string symbol); [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] private static extern int dlclose(IntPtr handle); @@ -87,15 +109,23 @@ class DarwinLoader : ILibraryLoader private const string NativeDll = "/usr/lib/libSystem.dylib"; private static IntPtr RTLD_DEFAULT = new IntPtr(-2); - public IntPtr Load(string fileName) + public IntPtr Load(string dllToLoad) { - return dlopen($"lib{fileName}.dylib", RTLD_NOW | RTLD_GLOBAL); + var filename = $"lib{dllToLoad}.dylib"; + ClearError(); + var res = dlopen(filename, RTLD_NOW | RTLD_GLOBAL); + if (res == IntPtr.Zero) + { + var err = GetError(); + throw new DllNotFoundException($"Could not load {filename} with flags RTLD_NOW | RTLD_GLOBAL: {err}"); + } + + return res; } - public bool Free(IntPtr handle) + public void Free(IntPtr handle) { dlclose(handle); - return true; } public IntPtr GetFunction(IntPtr dllHandle, string name) @@ -106,17 +136,30 @@ public IntPtr GetFunction(IntPtr dllHandle, string name) dllHandle = RTLD_DEFAULT; } - // clear previous errors if any - dlerror(); + ClearError(); IntPtr res = dlsym(dllHandle, name); - IntPtr errPtr = dlerror(); - if (errPtr != IntPtr.Zero) + if (res == IntPtr.Zero) { - throw new Exception("dlsym: " + Marshal.PtrToStringAnsi(errPtr)); + var err = GetError(); + throw new MissingMethodException($"Failed to load symbol {name}: {err}"); } return res; } + void ClearError() + { + dlerror(); + } + + string GetError() + { + var res = dlerror(); + if (res != IntPtr.Zero) + return Marshal.PtrToStringAnsi(res); + else + return null; + } + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] public static extern IntPtr dlopen(String fileName, int flags); @@ -134,20 +177,32 @@ class WindowsLoader : ILibraryLoader { private const string NativeDll = "kernel32.dll"; - [DllImport(NativeDll)] - static extern IntPtr LoadLibrary(string dllToLoad); - public IntPtr Load(string dllToLoad) => WindowsLoader.LoadLibrary(dllToLoad); + public IntPtr Load(string dllToLoad) + { + var res = WindowsLoader.LoadLibrary(dllToLoad); + if (res == IntPtr.Zero) + throw new DllNotFoundException($"Could not load {dllToLoad}", new Win32Exception()); + return res; + } + + public IntPtr GetFunction(IntPtr hModule, string procedureName) + { + var res = WindowsLoader.GetProcAddress(hModule, procedureName); + if (res == IntPtr.Zero) + throw new MissingMethodException($"Failed to load symbol {procedureName}", new Win32Exception()); + return res; + } - [DllImport(NativeDll)] - static extern IntPtr GetProcAddress(IntPtr hModule, string procedureName); + public void Free(IntPtr hModule) => WindowsLoader.FreeLibrary(hModule); - public IntPtr GetFunction(IntPtr hModule, string procedureName) => WindowsLoader.GetProcAddress(hModule, procedureName); + [DllImport(NativeDll, SetLastError = true)] + static extern IntPtr LoadLibrary(string dllToLoad); + [DllImport(NativeDll, SetLastError = true)] + static extern IntPtr GetProcAddress(IntPtr hModule, string procedureName); [DllImport(NativeDll)] static extern bool FreeLibrary(IntPtr hModule); - - public bool Free(IntPtr hModule) => WindowsLoader.FreeLibrary(hModule); } } diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index ec5bddfd0..a347651d0 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -4,10 +4,10 @@ using System.Text; using System.Threading; using System.Collections.Generic; +using Python.Runtime.Platform; namespace Python.Runtime { - using Python.Runtime.Platform; ///

/// Encapsulates the low-level Python C API. Note that it is diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 00a8f0a89..127e82eaa 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -3,10 +3,10 @@ using System.Collections.Generic; using System.Reflection; using System.Runtime.InteropServices; +using Python.Runtime.Platform; namespace Python.Runtime { - using Python.Runtime.Platform; ///

/// The TypeManager class is responsible for building binary-compatible From a8a94264164406e7166038d7fa8b03ac40eb8b71 Mon Sep 17 00:00:00 2001 From: Victor Date: Wed, 26 Jun 2019 10:55:55 -0700 Subject: [PATCH 011/112] Bump C# language version to 7.3 (#896) * Bump C# language version to 7.3 #860 * Switch to .NET SDK 2.2 * Use xenial image for travis CI --- .travis.yml | 20 ++++++++++---------- src/runtime/Python.Runtime.15.csproj | 2 +- src/runtime/Python.Runtime.csproj | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1dadbad1d..46f47489d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,4 @@ -dist: trusty +dist: xenial sudo: false language: python @@ -12,16 +12,16 @@ matrix: addons: &xplat-addons apt: sources: - - sourceline: deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-ubuntu-trusty-prod trusty main + - sourceline: deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-ubuntu-xenial-prod xenial main key_url: https://packages.microsoft.com/keys/microsoft.asc - - sourceline: deb http://download.mono-project.com/repo/ubuntu trusty main + - sourceline: deb http://download.mono-project.com/repo/ubuntu xenial main key_url: http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xA6A19B38D3D831EF packages: - mono-devel - ca-certificates-mono - - dotnet-hostfxr-2.0.0 - - dotnet-runtime-2.0.0 - - dotnet-sdk-2.0.0 + - dotnet-hostfxr-2.2 + - dotnet-runtime-2.2 + - dotnet-sdk-2.2 - python: 3.5 env: *xplat-env @@ -45,9 +45,9 @@ matrix: packages: - mono-devel - ca-certificates-mono - - dotnet-hostfxr-2.0.0 - - dotnet-runtime-2.0.0 - - dotnet-sdk-2.0.0 + - dotnet-hostfxr-2.2 + - dotnet-runtime-2.2 + - dotnet-sdk-2.2 # --------------------- Classic builds ------------------------ - python: 2.7 @@ -84,7 +84,7 @@ env: addons: apt: sources: - - sourceline: deb http://download.mono-project.com/repo/ubuntu trusty main + - sourceline: deb http://download.mono-project.com/repo/ubuntu xenial main key_url: http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xA6A19B38D3D831EF packages: - mono-devel diff --git a/src/runtime/Python.Runtime.15.csproj b/src/runtime/Python.Runtime.15.csproj index a4d1773f7..122132513 100644 --- a/src/runtime/Python.Runtime.15.csproj +++ b/src/runtime/Python.Runtime.15.csproj @@ -30,7 +30,7 @@ ..\..\ $(SolutionDir)\bin\ $(PythonBuildDir)\$(TargetFramework)\ - 6 + 7.3 True ..\pythonnet.snk $(PYTHONNET_DEFINE_CONSTANTS) diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index c4d63695f..ac6b59150 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -15,7 +15,7 @@ ..\..\ $(SolutionDir)\bin\ Properties - 6 + 7.3 true false ..\pythonnet.snk From df0574db552020397728648ee3ec576aa3a1c2a8 Mon Sep 17 00:00:00 2001 From: Victor Date: Thu, 27 Jun 2019 22:36:30 -0700 Subject: [PATCH 012/112] Improve "No method matches given arguments" message with more details (#900) * generate more useful message, when a .NET overload can't be found, that matches Python parameter types * provide detailed error message, when an overload can't be found when calling C# from Python Related: #811, #265, #782 --- CHANGELOG.md | 2 ++ src/embed_tests/TestCallbacks.cs | 35 ++++++++++++++++++++++++++++++++ src/runtime/methodbinder.cs | 31 +++++++++++++++++++++++++--- 3 files changed, 65 insertions(+), 3 deletions(-) create mode 100644 src/embed_tests/TestCallbacks.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index b5531bf47..e5a990922 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. ### Changed +- Added argument types information to "No method matches given arguments" message + ### Fixed ## [2.4.0][] diff --git a/src/embed_tests/TestCallbacks.cs b/src/embed_tests/TestCallbacks.cs new file mode 100644 index 000000000..220b0a86a --- /dev/null +++ b/src/embed_tests/TestCallbacks.cs @@ -0,0 +1,35 @@ +using System; + +using NUnit.Framework; +using Python.Runtime; + +namespace Python.EmbeddingTest { + using Runtime = Python.Runtime.Runtime; + + public class TestCallbacks { + [OneTimeSetUp] + public void SetUp() { + PythonEngine.Initialize(); + } + + [OneTimeTearDown] + public void Dispose() { + PythonEngine.Shutdown(); + } + + [Test] + public void TestNoOverloadException() { + int passed = 0; + var aFunctionThatCallsIntoPython = new Action(value => passed = value); + using (Py.GIL()) { + dynamic callWith42 = PythonEngine.Eval("lambda f: f([42])"); + var error = Assert.Throws(() => callWith42(aFunctionThatCallsIntoPython.ToPython())); + Assert.AreEqual("TypeError", error.PythonTypeName); + string expectedArgTypes = Runtime.IsPython2 + ? "()" + : "()"; + StringAssert.EndsWith(expectedArgTypes, error.Message); + } + } + } +} diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index 7471d5d7c..95b953555 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -1,6 +1,7 @@ using System; using System.Collections; using System.Reflection; +using System.Text; namespace Python.Runtime { @@ -555,12 +556,36 @@ internal virtual IntPtr Invoke(IntPtr inst, IntPtr args, IntPtr kw, MethodBase i if (binding == null) { - var value = "No method matches given arguments"; + var value = new StringBuilder("No method matches given arguments"); if (methodinfo != null && methodinfo.Length > 0) { - value += $" for {methodinfo[0].Name}"; + value.Append($" for {methodinfo[0].Name}"); } - Exceptions.SetError(Exceptions.TypeError, value); + + long argCount = Runtime.PyTuple_Size(args); + value.Append(": ("); + for(long argIndex = 0; argIndex < argCount; argIndex++) { + var arg = Runtime.PyTuple_GetItem(args, argIndex); + if (arg != IntPtr.Zero) { + var type = Runtime.PyObject_Type(arg); + if (type != IntPtr.Zero) { + try { + var description = Runtime.PyObject_Unicode(type); + if (description != IntPtr.Zero) { + value.Append(Runtime.GetManagedString(description)); + Runtime.XDecref(description); + } + } finally { + Runtime.XDecref(type); + } + } + } + + if (argIndex + 1 < argCount) + value.Append(", "); + } + value.Append(')'); + Exceptions.SetError(Exceptions.TypeError, value.ToString()); return IntPtr.Zero; } From 93968d25728b7282937067ebb966202da8c21a69 Mon Sep 17 00:00:00 2001 From: chrisjbremner Date: Wed, 3 Jul 2019 14:36:44 -0700 Subject: [PATCH 013/112] Safe wheel import (#905) Only allow wheel commands if wheel is installed --- AUTHORS.md | 1 + CHANGELOG.md | 1 + setup.py | 47 +++++++++++++++++++++++++++-------------------- 3 files changed, 29 insertions(+), 20 deletions(-) diff --git a/AUTHORS.md b/AUTHORS.md index ba954b47d..a45cf6d78 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -19,6 +19,7 @@ - Callum Noble ([@callumnoble](https://github.com/callumnoble)) - Christian Heimes ([@tiran](https://github.com/tiran)) - Christoph Gohlke ([@cgohlke](https://github.com/cgohlke)) +- Christopher Bremner ([@chrisjbremner](https://github.com/chrisjbremner)) - Christopher Pow ([@christopherpow](https://github.com/christopherpow)) - Daniel Fernandez ([@fdanny](https://github.com/fdanny)) - Daniel Santana ([@dgsantana](https://github.com/dgsantana)) diff --git a/CHANGELOG.md b/CHANGELOG.md index e5a990922..ada979147 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. ### Changed - Added argument types information to "No method matches given arguments" message +- Moved wheel import in setup.py inside of a try/except to prevent pip collection failures ### Fixed diff --git a/setup.py b/setup.py index 53c7f3f67..c6e4007e6 100644 --- a/setup.py +++ b/setup.py @@ -15,10 +15,14 @@ import sysconfig from distutils import spawn from distutils.command import install, build, build_ext, install_data, install_lib -from wheel import bdist_wheel from setuptools import Extension, setup +try: + from wheel import bdist_wheel +except ImportError: + bdist_wheel = None + # Allow config/verbosity to be set from cli # http://stackoverflow.com/a/4792601/5208670 CONFIG = "Release" # Release or Debug @@ -594,21 +598,21 @@ def run(self): _update_xlat_devtools() return install.install.run(self) +if bdist_wheel: + class BDistWheelPythonnet(bdist_wheel.bdist_wheel): + user_options = bdist_wheel.bdist_wheel.user_options + [("xplat", None, None)] -class BDistWheelPythonnet(bdist_wheel.bdist_wheel): - user_options = bdist_wheel.bdist_wheel.user_options + [("xplat", None, None)] + def initialize_options(self): + bdist_wheel.bdist_wheel.initialize_options(self) + self.xplat = None - def initialize_options(self): - bdist_wheel.bdist_wheel.initialize_options(self) - self.xplat = None + def finalize_options(self): + bdist_wheel.bdist_wheel.finalize_options(self) - def finalize_options(self): - bdist_wheel.bdist_wheel.finalize_options(self) - - def run(self): - if self.xplat: - _update_xlat_devtools() - return bdist_wheel.bdist_wheel.run(self) + def run(self): + if self.xplat: + _update_xlat_devtools() + return bdist_wheel.bdist_wheel.run(self) ############################################################################### @@ -621,6 +625,15 @@ def run(self): if not os.path.exists(_get_interop_filename()): setup_requires.append("pycparser") +cmdclass={ + "install": InstallPythonnet, + "build_ext": BuildExtPythonnet, + "install_lib": InstallLibPythonnet, + "install_data": InstallDataPythonnet, +} +if bdist_wheel: + cmdclass["bdist_wheel"] = BDistWheelPythonnet + setup( name="pythonnet", version="2.4.1-dev", @@ -633,13 +646,7 @@ def run(self): long_description=_get_long_description(), ext_modules=[Extension("clr", sources=list(_get_source_files()))], data_files=[("{install_platlib}", ["{build_lib}/Python.Runtime.dll"])], - cmdclass={ - "install": InstallPythonnet, - "build_ext": BuildExtPythonnet, - "install_lib": InstallLibPythonnet, - "install_data": InstallDataPythonnet, - "bdist_wheel": BDistWheelPythonnet, - }, + cmdclass=cmdclass, classifiers=[ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", From 6f635a42a97400bf5e284c4d821a75708393ac70 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Thu, 1 Aug 2019 17:04:07 +0200 Subject: [PATCH 014/112] Simplify travis config and pin mono to 5.20 (#927) --- .travis.yml | 94 +++++++++-------------------------------------------- 1 file changed, 16 insertions(+), 78 deletions(-) diff --git a/.travis.yml b/.travis.yml index 46f47489d..9689c0422 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,80 +1,17 @@ dist: xenial sudo: false language: python - -matrix: - include: -# --------------------- XPLAT builds ------------------------ - - python: 2.7 - env: &xplat-env - - BUILD_OPTS=--xplat - - NUNIT_PATH=~/.nuget/packages/nunit.consolerunner/3.*/tools/nunit3-console.exe - addons: &xplat-addons - apt: - sources: - - sourceline: deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-ubuntu-xenial-prod xenial main - key_url: https://packages.microsoft.com/keys/microsoft.asc - - sourceline: deb http://download.mono-project.com/repo/ubuntu xenial main - key_url: http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xA6A19B38D3D831EF - packages: - - mono-devel - - ca-certificates-mono - - dotnet-hostfxr-2.2 - - dotnet-runtime-2.2 - - dotnet-sdk-2.2 - - - python: 3.5 - env: *xplat-env - addons: *xplat-addons - - - python: 3.6 - env: *xplat-env - addons: *xplat-addons - - - python: 3.7 - env: *xplat-env - dist: xenial - sudo: true - addons: &xplat-addons-xenial - apt: - sources: - - sourceline: deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-ubuntu-xenial-prod xenial main - key_url: https://packages.microsoft.com/keys/microsoft.asc - - sourceline: deb https://download.mono-project.com/repo/ubuntu stable-xenial main - key_url: http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xA6A19B38D3D831EF - packages: - - mono-devel - - ca-certificates-mono - - dotnet-hostfxr-2.2 - - dotnet-runtime-2.2 - - dotnet-sdk-2.2 - -# --------------------- Classic builds ------------------------ - - python: 2.7 - env: &classic-env - - BUILD_OPTS= - - NUNIT_PATH=./packages/NUnit.*/tools/nunit3-console.exe - - - python: 3.5 - env: *classic-env - - - python: 3.6 - env: *classic-env - - - python: 3.7 - env: *classic-env - dist: xenial - sudo: true - addons: - apt: - sources: - - sourceline: deb http://download.mono-project.com/repo/ubuntu xenial main - key_url: http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xA6A19B38D3D831EF - packages: - - mono-devel - - ca-certificates-mono +python: + - 2.7 + - 3.5 + - 3.6 + - 3.7 env: + matrix: + - BUILD_OPTS=--xplat NUNIT_PATH="~/.nuget/packages/nunit.consolerunner/3.*/tools/nunit3-console.exe" RUN_TESTS=dotnet EMBED_TESTS_PATH=netcoreapp2.0_publish/ + - BUILD_OPTS="" NUNIT_PATH="./packages/NUnit.*/tools/nunit3-console.exe" RUN_TESTS="mono $NUNIT_PATH" EMBED_TESTS_PATH="" + global: - LD_PRELOAD=/lib/x86_64-linux-gnu/libSegFault.so - SEGFAULT_SIGNALS=all @@ -84,11 +21,16 @@ env: addons: apt: sources: - - sourceline: deb http://download.mono-project.com/repo/ubuntu xenial main + - sourceline: deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-ubuntu-xenial-prod xenial main + key_url: https://packages.microsoft.com/keys/microsoft.asc + - sourceline: deb http://download.mono-project.com/repo/ubuntu stable-xenial/snapshots/5.20 main key_url: http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xA6A19B38D3D831EF packages: - mono-devel - ca-certificates-mono + - dotnet-hostfxr-2.2 + - dotnet-runtime-2.2 + - dotnet-sdk-2.2 before_install: # Set-up dll path for embedded tests @@ -102,13 +44,9 @@ install: script: - python -m pytest - - mono $NUNIT_PATH src/embed_tests/bin/Python.EmbeddingTest.dll - - if [[ $BUILD_OPTS == --xplat ]]; then dotnet src/embed_tests/bin/netcoreapp2.0_publish/Python.EmbeddingTest.dll; fi + - $RUN_TESTS src/embed_tests/bin/$EMBED_TESTS_PATH/Python.EmbeddingTest.dll after_script: - # Uncomment if need to geninterop, ie. py37 final - # - python tools/geninterop/geninterop.py - # Waiting on mono-coverage, SharpCover or xr.Baboon - coverage xml -i - codecov --file coverage.xml --flags setup_linux From 1ce630e5b0c2fce208267591d6502a5d4d0e2a0a Mon Sep 17 00:00:00 2001 From: Ivan Cronyn Date: Fri, 2 Aug 2019 11:19:10 +0100 Subject: [PATCH 015/112] Removes imports deprecated in Python3 (#925) --- CHANGELOG.md | 1 + src/runtime/runtime.cs | 6 ++---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ada979147..b0d525b36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. - Added argument types information to "No method matches given arguments" message - Moved wheel import in setup.py inside of a try/except to prevent pip collection failures +- Removes PyLong_GetMax and PyClass_New when targetting Python3 ### Fixed diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index a347651d0..a6bfca431 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -769,8 +769,10 @@ public static extern int Py_Main( [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyCFunction_Call(IntPtr func, IntPtr args, IntPtr kw); +#if PYTHON2 [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyClass_New(IntPtr bases, IntPtr dict, IntPtr name); +#endif [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyInstance_New(IntPtr cls, IntPtr args, IntPtr kw); @@ -1012,10 +1014,6 @@ internal static IntPtr PyInt_FromInt64(long value) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyLong_FromString")] internal static extern IntPtr PyInt_FromString(string value, IntPtr end, int radix); - - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, - EntryPoint = "PyLong_GetMax")] - internal static extern int PyInt_GetMax(); #elif PYTHON2 [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] private static extern IntPtr PyInt_FromLong(IntPtr value); From f20bcf6498f336b684b8b5d515aaee5e3ebb24f4 Mon Sep 17 00:00:00 2001 From: ftreurni Date: Fri, 2 Aug 2019 12:21:48 +0200 Subject: [PATCH 016/112] Fix so that Runtime.PyModuleType is retrieved via Python.dll (#904) (#929) --- AUTHORS.md | 1 + CHANGELOG.md | 5 ++++- src/runtime/runtime.cs | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/AUTHORS.md b/AUTHORS.md index a45cf6d78..6c2817aeb 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -27,6 +27,7 @@ - David Lassonde ([@lassond](https://github.com/lassond)) - David Lechner ([@dlech](https://github.com/dlech)) - Dmitriy Se ([@dmitriyse](https://github.com/dmitriyse)) +- Florian Treurniet ([@ftreurni](https://github.com/ftreurni)) - He-chien Tsai ([@t3476](https://github.com/t3476)) - Inna Wiesel ([@inna-w](https://github.com/inna-w)) - Ivan Cronyn ([@cronan](https://github.com/cronan)) diff --git a/CHANGELOG.md b/CHANGELOG.md index b0d525b36..705b33b7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,9 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. ### Fixed +- Fixed runtime that fails loading when using pythonnet in an environment + together with Nuitka + ## [2.4.0][] ### Added @@ -61,7 +64,7 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. - Fixed conversion of 'float' and 'double' values ([#486][i486]) - Fixed 'clrmethod' for python 2 ([#492][i492]) - Fixed double calling of constructor when deriving from .NET class ([#495][i495]) -- Fixed `clr.GetClrType` when iterating over `System` members ([#607][p607]) +- Fixed `clr.GetClrType` when iterating over `System` members ([#607][p607]) - Fixed `LockRecursionException` when loading assemblies ([#627][i627]) - Fixed errors breaking .NET Remoting on method invoke ([#276][i276]) - Fixed PyObject.GetHashCode ([#676][i676]) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index a6bfca431..75f11492f 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -205,7 +205,6 @@ internal static void Initialize(bool initSigs = false) PyNotImplemented = PyObject_GetAttrString(op, "NotImplemented"); PyBaseObjectType = PyObject_GetAttrString(op, "object"); - PyModuleType = PyObject_Type(op); PyNone = PyObject_GetAttrString(op, "None"); PyTrue = PyObject_GetAttrString(op, "True"); PyFalse = PyObject_GetAttrString(op, "False"); @@ -302,6 +301,7 @@ internal static void Initialize(bool initSigs = false) dllLocal = loader.Load(_PythonDll); } _PyObject_NextNotImplemented = loader.GetFunction(dllLocal, "_PyObject_NextNotImplemented"); + PyModuleType = loader.GetFunction(dllLocal, "PyModule_Type"); if (dllLocal != IntPtr.Zero) { From c97a380bd38c28a055b7228028f01f5e8d1e5d8f Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Fri, 2 Aug 2019 14:10:09 +0200 Subject: [PATCH 017/112] Readd .NET implementations of GC slots again (#913) Since I failed to provide properly working implementations of Return0 and Return1 for ARM, I'll just add the old behaviour back for all platforms but x86 and amd64. --- src/runtime/typemanager.cs | 75 ++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 44 deletions(-) diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 127e82eaa..9a98e9ebb 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -512,12 +512,8 @@ public static NativeCode Active return I386; case MachineType.x86_64: return X86_64; - case MachineType.armv7l: - return Armv7l; - case MachineType.armv8: - return Armv8; default: - throw new NotImplementedException($"No support for {Runtime.MachineName}"); + return null; } } } @@ -552,34 +548,6 @@ public static NativeCode Active /// ///

public static readonly NativeCode I386 = X86_64; - - public static readonly NativeCode Armv7l = new NativeCode() - { - Return0 = 0, - Return1 = 0x08, - Code = new byte[] - { - 0xe3, 0xa0, 0x00, 0x00, // mov r0, #0 - 0xe1, 0x2f, 0xff, 0x1e, // bx lr - - 0xe3, 0xa0, 0x00, 0x01, // mov r0, #1 - 0xe1, 0x2f, 0xff, 0x1e, // bx lr - } - }; - - public static readonly NativeCode Armv8 = new NativeCode() - { - Return0 = 0, - Return1 = 0x08, - Code = new byte[] - { - 0x52, 0x80, 0x00, 0x00, // mov w0, #0x0 - 0xd6, 0x5f, 0x03, 0xc0, // ret - - 0x52, 0x80, 0x00, 0x20, // mov w0, #0x1 - 0xd6, 0x5f, 0x03, 0xc0, // ret - } - }; } ///

@@ -702,7 +670,7 @@ internal static void InitializeNativeCodePage() Marshal.Copy(NativeCode.Active.Code, 0, NativeCodePage, codeLength); mapper.SetReadExec(NativeCodePage, codeLength); } -#endregion + #endregion ///

/// Given a newly allocated Python type object and a managed Type that @@ -745,21 +713,40 @@ internal static void InitializeSlots(IntPtr type, Type impl) impl = impl.BaseType; } - // See the TestDomainReload test: there was a crash related to - // the gc-related slots. They always return 0 or 1 because we don't - // really support gc: + var native = NativeCode.Active; + + // The garbage collection related slots always have to return 1 or 0 + // since .NET objects don't take part in Python's gc: // tp_traverse (returns 0) // tp_clear (returns 0) // tp_is_gc (returns 1) - // We can't do without: python really wants those slots to exist. - // We can't implement those in C# because the application domain - // can be shut down and the memory released. - InitializeNativeCodePage(); - InitializeSlot(type, NativeCodePage + NativeCode.Active.Return0, "tp_traverse"); - InitializeSlot(type, NativeCodePage + NativeCode.Active.Return0, "tp_clear"); - InitializeSlot(type, NativeCodePage + NativeCode.Active.Return1, "tp_is_gc"); + // These have to be defined, though, so by default we fill these with + // static C# functions from this class. + + var ret0 = Interop.GetThunk(((Func)Return0).Method); + var ret1 = Interop.GetThunk(((Func)Return1).Method); + + if (native != null) + { + // If we want to support domain reload, the C# implementation + // cannot be used as the assembly may get released before + // CPython calls these functions. Instead, for amd64 and x86 we + // load them into a separate code page that is leaked + // intentionally. + InitializeNativeCodePage(); + ret1 = NativeCodePage + native.Return1; + ret0 = NativeCodePage + native.Return0; + } + + InitializeSlot(type, ret0, "tp_traverse"); + InitializeSlot(type, ret0, "tp_clear"); + InitializeSlot(type, ret1, "tp_is_gc"); } + static int Return1(IntPtr _) => 1; + + static int Return0(IntPtr _) => 0; + ///

/// Helper for InitializeSlots. /// From 51a186858096577aef6a860656523d3194eee9a8 Mon Sep 17 00:00:00 2001 From: jmlidbetter <53430310+jmlidbetter@users.noreply.github.com> Date: Mon, 5 Aug 2019 17:25:37 +0100 Subject: [PATCH 018/112] Adds support to convert iterators to arrays (#928) --- AUTHORS.md | 1 + CHANGELOG.md | 1 + src/runtime/converter.cs | 37 ++++++++++++++++++++----------------- src/tests/test_array.py | 31 +++++++++++++++++++++++++++++++ 4 files changed, 53 insertions(+), 17 deletions(-) diff --git a/AUTHORS.md b/AUTHORS.md index 6c2817aeb..39c2eb180 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -34,6 +34,7 @@ - Jan Krivanek ([@jakrivan](https://github.com/jakrivan)) - Jeff Reback ([@jreback](https://github.com/jreback)) - Joe Frayne ([@jfrayne](https://github.com/jfrayne)) +- Joe Lidbetter ([@jmlidbetter](https://github.com/jmlidbetter)) - John Burnett ([@johnburnett](https://github.com/johnburnett)) - John Wilkes ([@jbw3](https://github.com/jbw3)) - Luke Stratman ([@lstratman](https://github.com/lstratman)) diff --git a/CHANGELOG.md b/CHANGELOG.md index 705b33b7d..941045aef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. - Added argument types information to "No method matches given arguments" message - Moved wheel import in setup.py inside of a try/except to prevent pip collection failures - Removes PyLong_GetMax and PyClass_New when targetting Python3 +- Added support for converting python iterators to C# arrays ### Fixed diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 11c67bf82..5d8769a73 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -837,17 +837,20 @@ private static void SetConversionError(IntPtr value, Type target) ///

/// Convert a Python value to a correctly typed managed array instance. - /// The Python value must support the Python sequence protocol and the + /// The Python value must support the Python iterator protocol or and the /// items in the sequence must be convertible to the target array type. ///

private static bool ToArray(IntPtr value, Type obType, out object result, bool setError) { Type elementType = obType.GetElementType(); - var size = Runtime.PySequence_Size(value); result = null; - if (size < 0) - { + bool IsSeqObj = Runtime.PySequence_Check(value); + var len = IsSeqObj ? Runtime.PySequence_Size(value) : -1; + + IntPtr IterObject = Runtime.PyObject_GetIter(value); + + if(IterObject==IntPtr.Zero) { if (setError) { SetConversionError(value, obType); @@ -855,21 +858,17 @@ private static bool ToArray(IntPtr value, Type obType, out object result, bool s return false; } - Array items = Array.CreateInstance(elementType, size); + Array items; + + var listType = typeof(List<>); + var constructedListType = listType.MakeGenericType(elementType); + IList list = IsSeqObj ? (IList) Activator.CreateInstance(constructedListType, new Object[] {(int) len}) : + (IList) Activator.CreateInstance(constructedListType); + IntPtr item; - // XXX - is there a better way to unwrap this if it is a real array? - for (var i = 0; i < size; i++) + while ((item = Runtime.PyIter_Next(IterObject)) != IntPtr.Zero) { object obj = null; - IntPtr item = Runtime.PySequence_GetItem(value, i); - if (item == IntPtr.Zero) - { - if (setError) - { - SetConversionError(value, obType); - return false; - } - } if (!Converter.ToManaged(item, elementType, out obj, true)) { @@ -877,10 +876,14 @@ private static bool ToArray(IntPtr value, Type obType, out object result, bool s return false; } - items.SetValue(obj, i); + list.Add(obj); Runtime.XDecref(item); } + Runtime.XDecref(IterObject); + items = Array.CreateInstance(elementType, list.Count); + list.CopyTo(items, 0); + result = items; return true; } diff --git a/src/tests/test_array.py b/src/tests/test_array.py index 7ccadddff..b492a66d3 100644 --- a/src/tests/test_array.py +++ b/src/tests/test_array.py @@ -1337,3 +1337,34 @@ def test_array_abuse(): with pytest.raises(TypeError): desc = Test.PublicArrayTest.__dict__['__setitem__'] desc(0, 0, 0) + + +@pytest.mark.skipif(PY2, reason="Only applies in Python 3") +def test_iterator_to_array(): + from System import Array, String + + d = {"a": 1, "b": 2, "c": 3} + keys_iterator = iter(d.keys()) + arr = Array[String](keys_iterator) + + Array.Sort(arr) + + assert arr[0] == "a" + assert arr[1] == "b" + assert arr[2] == "c" + + +@pytest.mark.skipif(PY2, reason="Only applies in Python 3") +def test_dict_keys_to_array(): + from System import Array, String + + d = {"a": 1, "b": 2, "c": 3} + d_keys = d.keys() + arr = Array[String](d_keys) + + Array.Sort(arr) + + assert arr[0] == "a" + assert arr[1] == "b" + assert arr[2] == "c" + From f1da55e5d13f7c82d2eb62e211afd93d574d5fe8 Mon Sep 17 00:00:00 2001 From: jmlidbetter <53430310+jmlidbetter@users.noreply.github.com> Date: Mon, 26 Aug 2019 13:13:38 +0100 Subject: [PATCH 019/112] Fixes bug where there is casting on delegates -- this is explicitly in the NET standard documentation for GetDelegateForFunctionPointer (#936) --- CHANGELOG.md | 2 ++ src/runtime/nativecall.cs | 8 +++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 941045aef..a545f335c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,11 +17,13 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. - Moved wheel import in setup.py inside of a try/except to prevent pip collection failures - Removes PyLong_GetMax and PyClass_New when targetting Python3 - Added support for converting python iterators to C# arrays +- Changed usage of obselete function GetDelegateForFunctionPointer(IntPtr, Type) to GetDelegateForFunctionPointer(IntPtr) ### Fixed - Fixed runtime that fails loading when using pythonnet in an environment together with Nuitka +- Fixes bug where delegates get casts (dotnetcore) ## [2.4.0][] diff --git a/src/runtime/nativecall.cs b/src/runtime/nativecall.cs index b5bf25dd7..4a7bf05c8 100644 --- a/src/runtime/nativecall.cs +++ b/src/runtime/nativecall.cs @@ -32,19 +32,21 @@ internal class NativeCall public static void Void_Call_1(IntPtr fp, IntPtr a1) { - ((Void_1_Delegate)Marshal.GetDelegateForFunctionPointer(fp, typeof(Void_1_Delegate)))(a1); + var d = Marshal.GetDelegateForFunctionPointer(fp); + d(a1); } public static IntPtr Call_3(IntPtr fp, IntPtr a1, IntPtr a2, IntPtr a3) { - var d = (Interop.TernaryFunc)Marshal.GetDelegateForFunctionPointer(fp, typeof(Interop.TernaryFunc)); + var d = Marshal.GetDelegateForFunctionPointer(fp); return d(a1, a2, a3); } public static int Int_Call_3(IntPtr fp, IntPtr a1, IntPtr a2, IntPtr a3) { - return ((Int_3_Delegate)Marshal.GetDelegateForFunctionPointer(fp, typeof(Int_3_Delegate)))(a1, a2, a3); + var d = Marshal.GetDelegateForFunctionPointer(fp); + return d(a1, a2, a3); } #else private static AssemblyBuilder aBuilder; From 1bcbeb5d0e9bcc7f0304994a44a7485d6126873b Mon Sep 17 00:00:00 2001 From: Joe Savage Date: Thu, 12 Sep 2019 03:26:06 -0500 Subject: [PATCH 020/112] Feature/named arg support (#953) * Add support for named arguments (#849) * Remove kwarg check since it breaks the python-derived CLR class use-case * Add named parameter test cases * Update changelog and authors * Add default params tests --- AUTHORS.md | 1 + CHANGELOG.md | 1 + src/runtime/methodbinder.cs | 118 +++++++++++++++++++++----- src/testing/methodtest.cs | 33 ++++++++ src/tests/test_method.py | 162 ++++++++++++++++++++++++++++++++++++ 5 files changed, 296 insertions(+), 19 deletions(-) diff --git a/AUTHORS.md b/AUTHORS.md index 39c2eb180..9e13ca569 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -35,6 +35,7 @@ - Jeff Reback ([@jreback](https://github.com/jreback)) - Joe Frayne ([@jfrayne](https://github.com/jfrayne)) - Joe Lidbetter ([@jmlidbetter](https://github.com/jmlidbetter)) +- Joe Savage ([@s4v4g3](https://github.com/s4v4g3)) - John Burnett ([@johnburnett](https://github.com/johnburnett)) - John Wilkes ([@jbw3](https://github.com/jbw3)) - Luke Stratman ([@lstratman](https://github.com/lstratman)) diff --git a/CHANGELOG.md b/CHANGELOG.md index a545f335c..5cb0ea96c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. - Removes PyLong_GetMax and PyClass_New when targetting Python3 - Added support for converting python iterators to C# arrays - Changed usage of obselete function GetDelegateForFunctionPointer(IntPtr, Type) to GetDelegateForFunctionPointer(IntPtr) +- Added support for kwarg parameters when calling .NET methods from Python ### Fixed diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index 95b953555..8a7fc1930 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -2,6 +2,8 @@ using System.Collections; using System.Reflection; using System.Text; +using System.Collections.Generic; +using System.Linq; namespace Python.Runtime { @@ -280,6 +282,22 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth { // loop to find match, return invoker w/ or /wo error MethodBase[] _methods = null; + + var kwargDict = new Dictionary(); + if (kw != IntPtr.Zero) + { + var pynkwargs = (int)Runtime.PyDict_Size(kw); + IntPtr keylist = Runtime.PyDict_Keys(kw); + IntPtr valueList = Runtime.PyDict_Values(kw); + for (int i = 0; i < pynkwargs; ++i) + { + var keyStr = Runtime.GetManagedString(Runtime.PyList_GetItem(keylist, i)); + kwargDict[keyStr] = Runtime.PyList_GetItem(valueList, i); + } + Runtime.XDecref(keylist); + Runtime.XDecref(valueList); + } + var pynargs = (int)Runtime.PyTuple_Size(args); var isGeneric = false; if (info != null) @@ -303,11 +321,12 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth ArrayList defaultArgList; bool paramsArray; - if (!MatchesArgumentCount(pynargs, pi, out paramsArray, out defaultArgList)) { + if (!MatchesArgumentCount(pynargs, pi, kwargDict, out paramsArray, out defaultArgList)) + { continue; } var outs = 0; - var margs = TryConvertArguments(pi, paramsArray, args, pynargs, defaultArgList, + var margs = TryConvertArguments(pi, paramsArray, args, pynargs, kwargDict, defaultArgList, needsResolution: _methods.Length > 1, outs: out outs); @@ -351,19 +370,21 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth } ///

- /// Attempts to convert Python argument tuple into an array of managed objects, - /// that can be passed to a method. + /// Attempts to convert Python positional argument tuple and keyword argument table + /// into an array of managed objects, that can be passed to a method. ///

/// Information about expected parameters /// true, if the last parameter is a params array. /// A pointer to the Python argument tuple /// Number of arguments, passed by Python + /// Dictionary of keyword argument name to python object pointer /// A list of default values for omitted parameters /// true, if overloading resolution is required /// Returns number of output parameters /// An array of .NET arguments, that can be passed to a method. static object[] TryConvertArguments(ParameterInfo[] pi, bool paramsArray, IntPtr args, int pyArgCount, + Dictionary kwargDict, ArrayList defaultArgList, bool needsResolution, out int outs) @@ -374,7 +395,10 @@ static object[] TryConvertArguments(ParameterInfo[] pi, bool paramsArray, for (int paramIndex = 0; paramIndex < pi.Length; paramIndex++) { - if (paramIndex >= pyArgCount) + var parameter = pi[paramIndex]; + bool hasNamedParam = kwargDict.ContainsKey(parameter.Name); + + if (paramIndex >= pyArgCount && !hasNamedParam) { if (defaultArgList != null) { @@ -384,12 +408,19 @@ static object[] TryConvertArguments(ParameterInfo[] pi, bool paramsArray, continue; } - var parameter = pi[paramIndex]; - IntPtr op = (arrayStart == paramIndex) - // map remaining Python arguments to a tuple since - // the managed function accepts it - hopefully :] - ? Runtime.PyTuple_GetSlice(args, arrayStart, pyArgCount) - : Runtime.PyTuple_GetItem(args, paramIndex); + IntPtr op; + if (hasNamedParam) + { + op = kwargDict[parameter.Name]; + } + else + { + op = (arrayStart == paramIndex) + // map remaining Python arguments to a tuple since + // the managed function accepts it - hopefully :] + ? Runtime.PyTuple_GetSlice(args, arrayStart, pyArgCount) + : Runtime.PyTuple_GetItem(args, paramIndex); + } bool isOut; if (!TryConvertArgument(op, parameter.ParameterType, needsResolution, out margs[paramIndex], out isOut)) @@ -505,7 +536,8 @@ static Type TryComputeClrArgumentType(Type parameterType, IntPtr argument, bool return clrtype; } - static bool MatchesArgumentCount(int argumentCount, ParameterInfo[] parameters, + static bool MatchesArgumentCount(int positionalArgumentCount, ParameterInfo[] parameters, + Dictionary kwargDict, out bool paramsArray, out ArrayList defaultArgList) { @@ -513,21 +545,40 @@ static bool MatchesArgumentCount(int argumentCount, ParameterInfo[] parameters, var match = false; paramsArray = false; - if (argumentCount == parameters.Length) + if (positionalArgumentCount == parameters.Length) { match = true; - } else if (argumentCount < parameters.Length) + } + else if (positionalArgumentCount < parameters.Length) { + // every parameter past 'positionalArgumentCount' must have either + // a corresponding keyword argument or a default parameter match = true; defaultArgList = new ArrayList(); - for (var v = argumentCount; v < parameters.Length; v++) { - if (parameters[v].DefaultValue == DBNull.Value) { + for (var v = positionalArgumentCount; v < parameters.Length; v++) + { + if (kwargDict.ContainsKey(parameters[v].Name)) + { + // we have a keyword argument for this parameter, + // no need to check for a default parameter, but put a null + // placeholder in defaultArgList + defaultArgList.Add(null); + } + else if (parameters[v].IsOptional) + { + // IsOptional will be true if the parameter has a default value, + // or if the parameter has the [Optional] attribute specified. + // The GetDefaultValue() extension method will return the value + // to be passed in as the parameter value + defaultArgList.Add(parameters[v].GetDefaultValue()); + } + else + { match = false; - } else { - defaultArgList.Add(parameters[v].DefaultValue); } } - } else if (argumentCount > parameters.Length && parameters.Length > 0 && + } + else if (positionalArgumentCount > parameters.Length && parameters.Length > 0 && Attribute.IsDefined(parameters[parameters.Length - 1], typeof(ParamArrayAttribute))) { // This is a `foo(params object[] bar)` style method @@ -722,4 +773,33 @@ internal Binding(MethodBase info, object inst, object[] args, int outs) this.outs = outs; } } + + + static internal class ParameterInfoExtensions + { + public static object GetDefaultValue(this ParameterInfo parameterInfo) + { + // parameterInfo.HasDefaultValue is preferable but doesn't exist in .NET 4.0 + bool hasDefaultValue = (parameterInfo.Attributes & ParameterAttributes.HasDefault) == + ParameterAttributes.HasDefault; + + if (hasDefaultValue) + { + return parameterInfo.DefaultValue; + } + else + { + // [OptionalAttribute] was specified for the parameter. + // See https://stackoverflow.com/questions/3416216/optionalattribute-parameters-default-value + // for rules on determining the value to pass to the parameter + var type = parameterInfo.ParameterType; + if (type == typeof(object)) + return Type.Missing; + else if (type.IsValueType) + return Activator.CreateInstance(type); + else + return null; + } + } + } } diff --git a/src/testing/methodtest.cs b/src/testing/methodtest.cs index cf653f9f9..91836b727 100644 --- a/src/testing/methodtest.cs +++ b/src/testing/methodtest.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using System.Runtime.InteropServices; namespace Python.Test { @@ -651,6 +652,38 @@ public static string Casesensitive() { return "Casesensitive"; } + + public static string DefaultParams(int a=0, int b=0, int c=0, int d=0) + { + return string.Format("{0}{1}{2}{3}", a, b, c, d); + } + + public static string OptionalParams([Optional]int a, [Optional]int b, [Optional]int c, [Optional] int d) + { + return string.Format("{0}{1}{2}{3}", a, b, c, d); + } + + public static bool OptionalParams_TestMissing([Optional]object a) + { + return a == Type.Missing; + } + + public static bool OptionalParams_TestReferenceType([Optional]string a) + { + return a == null; + } + + public static string OptionalAndDefaultParams([Optional]int a, [Optional]int b, int c=0, int d=0) + { + return string.Format("{0}{1}{2}{3}", a, b, c, d); + } + + public static string OptionalAndDefaultParams2([Optional]int a, [Optional]int b, [Optional, DefaultParameterValue(1)]int c, int d = 2) + { + return string.Format("{0}{1}{2}{3}", a, b, c, d); + } + + } diff --git a/src/tests/test_method.py b/src/tests/test_method.py index ad678611b..34f460d59 100644 --- a/src/tests/test_method.py +++ b/src/tests/test_method.py @@ -776,6 +776,9 @@ def test_no_object_in_param(): res = MethodTest.TestOverloadedNoObject(5) assert res == "Got int" + + res = MethodTest.TestOverloadedNoObject(i=7) + assert res == "Got int" with pytest.raises(TypeError): MethodTest.TestOverloadedNoObject("test") @@ -787,9 +790,15 @@ def test_object_in_param(): res = MethodTest.TestOverloadedObject(5) assert res == "Got int" + + res = MethodTest.TestOverloadedObject(i=7) + assert res == "Got int" res = MethodTest.TestOverloadedObject("test") assert res == "Got object" + + res = MethodTest.TestOverloadedObject(o="test") + assert res == "Got object" def test_object_in_multiparam(): @@ -813,6 +822,42 @@ def test_object_in_multiparam(): res = MethodTest.TestOverloadedObjectTwo(7.24, 7.24) assert res == "Got object-object" + res = MethodTest.TestOverloadedObjectTwo(a=5, b=5) + assert res == "Got int-int" + + res = MethodTest.TestOverloadedObjectTwo(5, b=5) + assert res == "Got int-int" + + res = MethodTest.TestOverloadedObjectTwo(a=5, b="foo") + assert res == "Got int-string" + + res = MethodTest.TestOverloadedObjectTwo(5, b="foo") + assert res == "Got int-string" + + res = MethodTest.TestOverloadedObjectTwo(a="foo", b=7.24) + assert res == "Got string-object" + + res = MethodTest.TestOverloadedObjectTwo("foo", b=7.24) + assert res == "Got string-object" + + res = MethodTest.TestOverloadedObjectTwo(a="foo", b="bar") + assert res == "Got string-string" + + res = MethodTest.TestOverloadedObjectTwo("foo", b="bar") + assert res == "Got string-string" + + res = MethodTest.TestOverloadedObjectTwo(a="foo", b=5) + assert res == "Got string-int" + + res = MethodTest.TestOverloadedObjectTwo("foo", b=5) + assert res == "Got string-int" + + res = MethodTest.TestOverloadedObjectTwo(a=7.24, b=7.24) + assert res == "Got object-object" + + res = MethodTest.TestOverloadedObjectTwo(7.24, b=7.24) + assert res == "Got object-object" + def test_object_in_multiparam_exception(): """Test method with object multiparams behaves""" @@ -966,3 +1011,120 @@ def test_getting_overloaded_constructor_binding_does_not_leak_ref_count(): # simple test refCount = sys.getrefcount(PlainOldClass.Overloads[int]) assert refCount == 1 + + +def test_default_params(): + # all positional parameters + res = MethodTest.DefaultParams(1,2,3,4) + assert res == "1234" + + res = MethodTest.DefaultParams(1, 2, 3) + assert res == "1230" + + res = MethodTest.DefaultParams(1, 2) + assert res == "1200" + + res = MethodTest.DefaultParams(1) + assert res == "1000" + + res = MethodTest.DefaultParams(a=2) + assert res == "2000" + + res = MethodTest.DefaultParams(b=3) + assert res == "0300" + + res = MethodTest.DefaultParams(c=4) + assert res == "0040" + + res = MethodTest.DefaultParams(d=7) + assert res == "0007" + + res = MethodTest.DefaultParams(a=2, c=5) + assert res == "2050" + + res = MethodTest.DefaultParams(1, d=7, c=3) + assert res == "1037" + + with pytest.raises(TypeError): + MethodTest.DefaultParams(1,2,3,4,5) + +def test_optional_params(): + res = MethodTest.OptionalParams(1, 2, 3, 4) + assert res == "1234" + + res = MethodTest.OptionalParams(1, 2, 3) + assert res == "1230" + + res = MethodTest.OptionalParams(1, 2) + assert res == "1200" + + res = MethodTest.OptionalParams(1) + assert res == "1000" + + res = MethodTest.OptionalParams(a=2) + assert res == "2000" + + res = MethodTest.OptionalParams(b=3) + assert res == "0300" + + res = MethodTest.OptionalParams(c=4) + assert res == "0040" + + res = MethodTest.OptionalParams(d=7) + assert res == "0007" + + res = MethodTest.OptionalParams(a=2, c=5) + assert res == "2050" + + res = MethodTest.OptionalParams(1, d=7, c=3) + assert res == "1037" + + res = MethodTest.OptionalParams_TestMissing() + assert res == True + + res = MethodTest.OptionalParams_TestMissing(None) + assert res == False + + res = MethodTest.OptionalParams_TestMissing(a = None) + assert res == False + + res = MethodTest.OptionalParams_TestMissing(a='hi') + assert res == False + + res = MethodTest.OptionalParams_TestReferenceType() + assert res == True + + res = MethodTest.OptionalParams_TestReferenceType(None) + assert res == True + + res = MethodTest.OptionalParams_TestReferenceType(a=None) + assert res == True + + res = MethodTest.OptionalParams_TestReferenceType('hi') + assert res == False + + res = MethodTest.OptionalParams_TestReferenceType(a='hi') + assert res == False + +def test_optional_and_default_params(): + + res = MethodTest.OptionalAndDefaultParams() + assert res == "0000" + + res = MethodTest.OptionalAndDefaultParams(1) + assert res == "1000" + + res = MethodTest.OptionalAndDefaultParams(1, c=4) + assert res == "1040" + + res = MethodTest.OptionalAndDefaultParams(b=4, c=7) + assert res == "0470" + + res = MethodTest.OptionalAndDefaultParams2() + assert res == "0012" + + res = MethodTest.OptionalAndDefaultParams2(a=1,b=2,c=3,d=4) + assert res == "1234" + + res = MethodTest.OptionalAndDefaultParams2(b=2, c=3) + assert res == "0232" From 46c7597eb18f3586de76a66c3a5ec3d185b64010 Mon Sep 17 00:00:00 2001 From: Mark Visser Date: Fri, 20 Sep 2019 10:01:24 -0400 Subject: [PATCH 021/112] Update readme with resources (#955) Added resources section with mailing list and chat --- README.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.rst b/README.rst index 5366649ae..84bf93d84 100644 --- a/README.rst +++ b/README.rst @@ -113,3 +113,8 @@ https://github.com/pythonnet/pythonnet/wiki :target: http://stackoverflow.com/questions/tagged/python.net .. |conda-forge version| image:: https://img.shields.io/conda/vn/conda-forge/pythonnet.svg :target: https://anaconda.org/conda-forge/pythonnet + +Resources +--------- +Mailing list: https://mail.python.org/mailman/listinfo/pythondotnet +Chat: https://gitter.im/pythonnet/pythonnet From 60e6045f6873495170502f0247205a9c923fc1fe Mon Sep 17 00:00:00 2001 From: jmlidbetter <53430310+jmlidbetter@users.noreply.github.com> Date: Tue, 1 Oct 2019 16:13:34 +0100 Subject: [PATCH 022/112] Detect py arch (#961) * Gets size of C long from Is32Bit and IsWindows --- CHANGELOG.md | 1 + src/runtime/converter.cs | 15 ++++++++++++++- src/runtime/runtime.cs | 34 ++++++++++++++++++++++++++++++---- 3 files changed, 45 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5cb0ea96c..b5b11cd77 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. - Fixed runtime that fails loading when using pythonnet in an environment together with Nuitka - Fixes bug where delegates get casts (dotnetcore) +- Determine size of interpreter longs at runtime ## [2.4.0][] diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 5d8769a73..e7e047419 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -728,7 +728,20 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo } goto type_error; } - uint ui = (uint)Runtime.PyLong_AsUnsignedLong(op); + + uint ui; + try + { + ui = Convert.ToUInt32(Runtime.PyLong_AsUnsignedLong(op)); + } catch (OverflowException) + { + // Probably wasn't an overflow in python but was in C# (e.g. if cpython + // longs are 64 bit then 0xFFFFFFFF + 1 will not overflow in + // PyLong_AsUnsignedLong) + Runtime.XDecref(op); + goto overflow; + } + if (Exceptions.ErrorOccurred()) { diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 75f11492f..4985a57f5 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -1036,8 +1036,21 @@ internal static bool PyLong_Check(IntPtr ob) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyLong_FromLong(long value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyLong_FromUnsignedLong(uint value); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, + EntryPoint = "PyLong_FromUnsignedLong")] + internal static extern IntPtr PyLong_FromUnsignedLong32(uint value); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, + EntryPoint = "PyLong_FromUnsignedLong")] + internal static extern IntPtr PyLong_FromUnsignedLong64(ulong value); + + internal static IntPtr PyLong_FromUnsignedLong(object value) + { + if(Is32Bit || IsWindows) + return PyLong_FromUnsignedLong32(Convert.ToUInt32(value)); + else + return PyLong_FromUnsignedLong64(Convert.ToUInt64(value)); + } [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyLong_FromDouble(double value); @@ -1054,8 +1067,21 @@ internal static bool PyLong_Check(IntPtr ob) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyLong_AsLong(IntPtr value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern uint PyLong_AsUnsignedLong(IntPtr value); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, + EntryPoint = "PyLong_AsUnsignedLong")] + internal static extern uint PyLong_AsUnsignedLong32(IntPtr value); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, + EntryPoint = "PyLong_AsUnsignedLong")] + internal static extern ulong PyLong_AsUnsignedLong64(IntPtr value); + + internal static object PyLong_AsUnsignedLong(IntPtr value) + { + if (Is32Bit || IsWindows) + return PyLong_AsUnsignedLong32(value); + else + return PyLong_AsUnsignedLong64(value); + } [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern long PyLong_AsLongLong(IntPtr value); From 4a9457fed54d2eeee3860a10f2b8c59f48aef043 Mon Sep 17 00:00:00 2001 From: Mohamed Koubaa Date: Fri, 18 Oct 2019 03:07:57 -0500 Subject: [PATCH 023/112] Provide hook to implement __repr__ (#808) Provide hook to implement __repr__ --- AUTHORS.md | 1 + CHANGELOG.md | 1 + src/runtime/classbase.cs | 38 ++++++++++++ src/runtime/exceptions.cs | 23 +++++++ src/testing/Python.Test.csproj | 3 +- src/testing/ReprTest.cs | 108 +++++++++++++++++++++++++++++++++ src/tests/test_exceptions.py | 7 +-- src/tests/test_repr.py | 68 +++++++++++++++++++++ src/tests/tests.pyproj | 1 + 9 files changed, 244 insertions(+), 6 deletions(-) create mode 100644 src/testing/ReprTest.cs create mode 100644 src/tests/test_repr.py diff --git a/AUTHORS.md b/AUTHORS.md index 9e13ca569..9253c7e55 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -41,6 +41,7 @@ - Luke Stratman ([@lstratman](https://github.com/lstratman)) - Konstantin Posudevskiy ([@konstantin-posudevskiy](https://github.com/konstantin-posudevskiy)) - Matthias Dittrich ([@matthid](https://github.com/matthid)) +- Mohamed Koubaa ([@koubaa](https://github.com/koubaa)) - Patrick Stewart ([@patstew](https://github.com/patstew)) - Raphael Nestler ([@rnestler](https://github.com/rnestler)) - Rickard Holmberg ([@rickardraysearch](https://github.com/rickardraysearch)) diff --git a/CHANGELOG.md b/CHANGELOG.md index b5b11cd77..e24a46904 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,7 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. - Added PyObject finalizer support, Python objects referred by C# can be auto collect now ([#692][p692]). - Added detailed comments about aproaches and dangers to handle multi-app-domains ([#625][p625]) - Python 3.7 support, builds and testing added. Defaults changed from Python 3.6 to 3.7 ([#698][p698]) +- Added support for C# types to provide `__repr__` ([#680][p680]) ### Changed diff --git a/src/runtime/classbase.cs b/src/runtime/classbase.cs index 5846fa82a..41636c404 100644 --- a/src/runtime/classbase.cs +++ b/src/runtime/classbase.cs @@ -246,6 +246,44 @@ public static IntPtr tp_str(IntPtr ob) } } + public static IntPtr tp_repr(IntPtr ob) + { + var co = GetManagedObject(ob) as CLRObject; + if (co == null) + { + return Exceptions.RaiseTypeError("invalid object"); + } + try + { + //if __repr__ is defined, use it + var instType = co.inst.GetType(); + System.Reflection.MethodInfo methodInfo = instType.GetMethod("__repr__"); + if (methodInfo != null && methodInfo.IsPublic) + { + var reprString = methodInfo.Invoke(co.inst, null) as string; + return Runtime.PyString_FromString(reprString); + } + + //otherwise use the standard object.__repr__(inst) + IntPtr args = Runtime.PyTuple_New(1); + Runtime.PyTuple_SetItem(args, 0, ob); + IntPtr reprFunc = Runtime.PyObject_GetAttrString(Runtime.PyBaseObjectType, "__repr__"); + var output = Runtime.PyObject_Call(reprFunc, args, IntPtr.Zero); + Runtime.XDecref(args); + Runtime.XDecref(reprFunc); + return output; + } + catch (Exception e) + { + if (e.InnerException != null) + { + e = e.InnerException; + } + Exceptions.SetError(e); + return IntPtr.Zero; + } + } + ///

/// Standard dealloc implementation for instances of reflected types. diff --git a/src/runtime/exceptions.cs b/src/runtime/exceptions.cs index 8bed0abfd..31c367eb2 100644 --- a/src/runtime/exceptions.cs +++ b/src/runtime/exceptions.cs @@ -36,6 +36,29 @@ internal static Exception ToException(IntPtr ob) return e; } + ///

+ /// Exception __repr__ implementation + ///

+ public new static IntPtr tp_repr(IntPtr ob) + { + Exception e = ToException(ob); + if (e == null) + { + return Exceptions.RaiseTypeError("invalid object"); + } + string name = e.GetType().Name; + string message; + if (e.Message != String.Empty) + { + message = String.Format("{0}('{1}')", name, e.Message); + } + else + { + message = String.Format("{0}()", name); + } + return Runtime.PyUnicode_FromString(message); + } + ///

/// Exception __str__ implementation ///

diff --git a/src/testing/Python.Test.csproj b/src/testing/Python.Test.csproj index 27639ed5a..6bf5c2d22 100644 --- a/src/testing/Python.Test.csproj +++ b/src/testing/Python.Test.csproj @@ -91,6 +91,7 @@ + @@ -111,4 +112,4 @@ - \ No newline at end of file + diff --git a/src/testing/ReprTest.cs b/src/testing/ReprTest.cs new file mode 100644 index 000000000..48e93683a --- /dev/null +++ b/src/testing/ReprTest.cs @@ -0,0 +1,108 @@ +using System; +using System.Text; + +namespace Python.Test +{ + ///

+ /// Supports repr unit tests. + ///

+ public class ReprTest + { + public class Point + { + public Point(double x, double y) + { + X = x; + Y = y; + } + + public double X { get; set; } + public double Y { get; set; } + + public override string ToString() + { + return base.ToString() + ": X=" + X.ToString() + ", Y=" + Y.ToString(); + } + + public string __repr__() + { + return "Point(" + X.ToString() + "," + Y.ToString() + ")"; + } + } + + public class Foo + { + public string __repr__() + { + return "I implement __repr__() but not ToString()!"; + } + } + + public class Bar + { + public override string ToString() + { + return "I implement ToString() but not __repr__()!"; + } + } + + public class BazBase + { + public override string ToString() + { + return "Base class implementing ToString()!"; + } + } + + public class BazMiddle : BazBase + { + public override string ToString() + { + return "Middle class implementing ToString()!"; + } + } + + //implements ToString via BazMiddle + public class Baz : BazMiddle + { + + } + + public class Quux + { + public string ToString(string format) + { + return "I implement ToString() with an argument!"; + } + } + + public class QuuzBase + { + protected string __repr__() + { + return "I implement __repr__ but it isn't public!"; + } + } + + public class Quuz : QuuzBase + { + + } + + public class Corge + { + public string __repr__(int i) + { + return "__repr__ implemention with input parameter!"; + } + } + + public class Grault + { + public int __repr__() + { + return "__repr__ implemention with wrong return type!".Length; + } + } + } +} diff --git a/src/tests/test_exceptions.py b/src/tests/test_exceptions.py index a10d9a183..c2f18d443 100644 --- a/src/tests/test_exceptions.py +++ b/src/tests/test_exceptions.py @@ -282,11 +282,8 @@ def test_python_compat_of_managed_exceptions(): assert e.args == (msg,) assert isinstance(e.args, tuple) - if PY3: - strexp = "OverflowException('Simple message" - assert repr(e)[:len(strexp)] == strexp - elif PY2: - assert repr(e) == "OverflowException(u'Simple message',)" + strexp = "OverflowException('Simple message" + assert repr(e)[:len(strexp)] == strexp def test_exception_is_instance_of_system_object(): diff --git a/src/tests/test_repr.py b/src/tests/test_repr.py new file mode 100644 index 000000000..d120b0c4c --- /dev/null +++ b/src/tests/test_repr.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- + +"""Test __repr__ output""" + +import System +import pytest +from Python.Test import ReprTest + +def test_basic(): + """Test Point class which implements both ToString and __repr__ without inheritance""" + ob = ReprTest.Point(1,2) + # point implements ToString() and __repr__() + assert ob.__repr__() == "Point(1,2)" + assert str(ob) == "Python.Test.ReprTest+Point: X=1, Y=2" + +def test_system_string(): + """Test system string""" + ob = System.String("hello") + assert str(ob) == "hello" + assert " + From f2e6f6f89dde86b2043d6945fdffc2858eed2348 Mon Sep 17 00:00:00 2001 From: Alex Helms Date: Wed, 23 Oct 2019 01:32:44 -0700 Subject: [PATCH 024/112] Add function to set Py_NoSiteFlag global variable to 1 (#971) * Add function to set Py_NoSiteFlag global variable to 1. * Add myself to authors, update changelog. --- AUTHORS.md | 1 + CHANGELOG.md | 1 + src/runtime/pythonengine.cs | 10 ++++++++++ src/runtime/runtime.cs | 26 ++++++++++++++++++++++++++ 4 files changed, 38 insertions(+) diff --git a/AUTHORS.md b/AUTHORS.md index 9253c7e55..efa04c8f0 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -12,6 +12,7 @@ ## Contributors +- Alex Helms ([@alexhelms](https://github.com/alexhelms)) - Alexandre Catarino([@AlexCatarino](https://github.com/AlexCatarino)) - Arvid JB ([@ArvidJB](https://github.com/ArvidJB)) - Benoît Hudson ([@benoithudson](https://github.com/benoithudson)) diff --git a/CHANGELOG.md b/CHANGELOG.md index e24a46904..c7cad9567 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. ### Added - Added automatic NuGet package generation in appveyor and local builds +- Added function that sets Py_NoSiteFlag to 1. ### Changed diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index c1b663d22..700543839 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -130,6 +130,16 @@ public static string Compiler get { return Marshal.PtrToStringAnsi(Runtime.Py_GetCompiler()); } } + ///

+ /// Set the NoSiteFlag to disable loading the site module. + /// Must be called before Initialize. + /// https://docs.python.org/3/c-api/init.html#c.Py_NoSiteFlag + ///

+ public static void SetNoSiteFlag() + { + Runtime.SetNoSiteFlag(); + } + public static int RunSimpleString(string code) { return Runtime.PyRun_SimpleString(code); diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 4985a57f5..a1f9a38aa 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -414,6 +414,8 @@ internal static int AtExit() internal static IntPtr PyNoneType; internal static IntPtr PyTypeType; + internal static IntPtr Py_NoSiteFlag; + #if PYTHON3 internal static IntPtr PyBytesType; #endif @@ -1884,5 +1886,29 @@ internal static IntPtr PyMem_Realloc(IntPtr ptr, long size) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int Py_MakePendingCalls(); + + internal static void SetNoSiteFlag() + { + var loader = LibraryLoader.Get(OperatingSystem); + + IntPtr dllLocal; + if (_PythonDll != "__Internal") + { + dllLocal = loader.Load(_PythonDll); + } + + try + { + Py_NoSiteFlag = loader.GetFunction(dllLocal, "Py_NoSiteFlag"); + Marshal.WriteInt32(Py_NoSiteFlag, 1); + } + finally + { + if (dllLocal != IntPtr.Zero) + { + loader.Free(dllLocal); + } + } + } } } From 146353855b6ed2414f3c5fd1b4bf231c2a985da8 Mon Sep 17 00:00:00 2001 From: Victor Date: Wed, 23 Oct 2019 06:05:17 -0700 Subject: [PATCH 025/112] Adds performance tests, that compare to published NuGet (#975) Add performance tests that compare to published NuGet --- .editorconfig | 11 + pythonnet.15.sln | 191 +++++++++++++++++- .../BaselineComparisonBenchmarkBase.cs | 69 +++++++ src/perf_tests/BaselineComparisonConfig.cs | 47 +++++ src/perf_tests/BenchmarkTests.cs | 63 ++++++ src/perf_tests/Python.PerformanceTests.csproj | 34 ++++ src/perf_tests/PythonCallingNetBenchmark.cs | 46 +++++ 7 files changed, 454 insertions(+), 7 deletions(-) create mode 100644 src/perf_tests/BaselineComparisonBenchmarkBase.cs create mode 100644 src/perf_tests/BaselineComparisonConfig.cs create mode 100644 src/perf_tests/BenchmarkTests.cs create mode 100644 src/perf_tests/Python.PerformanceTests.csproj create mode 100644 src/perf_tests/PythonCallingNetBenchmark.cs diff --git a/.editorconfig b/.editorconfig index 2e7c58ffe..9e10931d0 100644 --- a/.editorconfig +++ b/.editorconfig @@ -19,6 +19,17 @@ indent_size = 2 [*.{csproj,pyproj,config}] indent_size = 2 +# .NET formatting settings +[*.{cs,vb}] +dotnet_sort_system_directives_first = true +dotnet_separate_import_directive_groups = true + +[*.cs] +csharp_new_line_before_open_brace = true +csharp_new_line_before_else = true +csharp_new_line_before_catch = true +csharp_new_line_before_finally = true + # Solution [*.sln] indent_style = tab diff --git a/pythonnet.15.sln b/pythonnet.15.sln index f2015e480..096dfbe9a 100644 --- a/pythonnet.15.sln +++ b/pythonnet.15.sln @@ -1,189 +1,366 @@ Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 15 -VisualStudioVersion = 15.0.26730.3 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.29102.190 MinimumVisualStudioVersion = 10.0.40219.1 -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.Runtime.15", "src/runtime/Python.Runtime.15.csproj", "{2759F4FF-716B-4828-916F-50FA86613DFC}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.Runtime.15", "src\runtime\Python.Runtime.15.csproj", "{2759F4FF-716B-4828-916F-50FA86613DFC}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.EmbeddingTest.15", "src/embed_tests/Python.EmbeddingTest.15.csproj", "{66B8D01A-9906-452A-B09E-BF75EA76468F}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.EmbeddingTest.15", "src\embed_tests\Python.EmbeddingTest.15.csproj", "{66B8D01A-9906-452A-B09E-BF75EA76468F}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "clrmodule.15", "src/clrmodule/clrmodule.15.csproj", "{E08678D4-9A52-4AD5-B63D-8EBC7399981B}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "clrmodule.15", "src\clrmodule\clrmodule.15.csproj", "{E08678D4-9A52-4AD5-B63D-8EBC7399981B}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Console.15", "src/console/Console.15.csproj", "{CDAD305F-8E72-492C-A314-64CF58D472A0}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Console.15", "src\console\Console.15.csproj", "{CDAD305F-8E72-492C-A314-64CF58D472A0}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.Test.15", "src/testing/Python.Test.15.csproj", "{F94B547A-E97E-4500-8D53-B4D64D076E5F}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.Test.15", "src\testing\Python.Test.15.csproj", "{F94B547A-E97E-4500-8D53-B4D64D076E5F}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.PerformanceTests", "src\perf_tests\Python.PerformanceTests.csproj", "{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Repo", "Repo", "{441A0123-F4C6-4EE4-9AEE-315FD79BE2D5}" + ProjectSection(SolutionItems) = preProject + .editorconfig = .editorconfig + EndProjectSection EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + DebugMono|Any CPU = DebugMono|Any CPU DebugMono|x64 = DebugMono|x64 DebugMono|x86 = DebugMono|x86 + DebugMonoPY3|Any CPU = DebugMonoPY3|Any CPU DebugMonoPY3|x64 = DebugMonoPY3|x64 DebugMonoPY3|x86 = DebugMonoPY3|x86 + DebugWin|Any CPU = DebugWin|Any CPU DebugWin|x64 = DebugWin|x64 DebugWin|x86 = DebugWin|x86 + DebugWinPY3|Any CPU = DebugWinPY3|Any CPU DebugWinPY3|x64 = DebugWinPY3|x64 DebugWinPY3|x86 = DebugWinPY3|x86 + Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 + ReleaseMono|Any CPU = ReleaseMono|Any CPU ReleaseMono|x64 = ReleaseMono|x64 ReleaseMono|x86 = ReleaseMono|x86 + ReleaseMonoPY3|Any CPU = ReleaseMonoPY3|Any CPU ReleaseMonoPY3|x64 = ReleaseMonoPY3|x64 ReleaseMonoPY3|x86 = ReleaseMonoPY3|x86 + ReleaseWin|Any CPU = ReleaseWin|Any CPU ReleaseWin|x64 = ReleaseWin|x64 ReleaseWin|x86 = ReleaseWin|x86 + ReleaseWinPY3|Any CPU = ReleaseWinPY3|Any CPU ReleaseWinPY3|x64 = ReleaseWinPY3|x64 ReleaseWinPY3|x86 = ReleaseWinPY3|x86 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution + {2759F4FF-716B-4828-916F-50FA86613DFC}.Debug|Any CPU.ActiveCfg = DebugWinPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.Debug|Any CPU.Build.0 = DebugWinPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.Debug|x64.ActiveCfg = DebugWinPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.Debug|x64.Build.0 = DebugWinPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.Debug|x86.ActiveCfg = DebugWinPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.Debug|x86.Build.0 = DebugWinPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugMono|Any CPU.ActiveCfg = DebugMono|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugMono|Any CPU.Build.0 = DebugMono|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugMono|x64.ActiveCfg = DebugMono|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugMono|x64.Build.0 = DebugMono|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugMono|x86.ActiveCfg = DebugMono|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugMono|x86.Build.0 = DebugMono|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugMonoPY3|Any CPU.ActiveCfg = DebugMonoPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugMonoPY3|Any CPU.Build.0 = DebugMonoPY3|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugMonoPY3|x64.ActiveCfg = DebugMonoPY3|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugMonoPY3|x64.Build.0 = DebugMonoPY3|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugMonoPY3|x86.ActiveCfg = DebugMonoPY3|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugMonoPY3|x86.Build.0 = DebugMonoPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugWin|Any CPU.ActiveCfg = DebugWin|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugWin|Any CPU.Build.0 = DebugWin|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugWin|x64.ActiveCfg = DebugWin|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugWin|x64.Build.0 = DebugWin|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugWin|x86.ActiveCfg = DebugWin|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugWin|x86.Build.0 = DebugWin|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugWinPY3|Any CPU.ActiveCfg = DebugWinPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugWinPY3|Any CPU.Build.0 = DebugWinPY3|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugWinPY3|x64.ActiveCfg = DebugWinPY3|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugWinPY3|x64.Build.0 = DebugWinPY3|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugWinPY3|x86.ActiveCfg = DebugWinPY3|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugWinPY3|x86.Build.0 = DebugWinPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.Release|Any CPU.ActiveCfg = ReleaseWinPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.Release|Any CPU.Build.0 = ReleaseWinPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.Release|x64.ActiveCfg = ReleaseWinPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.Release|x64.Build.0 = ReleaseWinPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.Release|x86.ActiveCfg = ReleaseWinPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.Release|x86.Build.0 = ReleaseWinPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseMono|Any CPU.ActiveCfg = ReleaseMono|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseMono|Any CPU.Build.0 = ReleaseMono|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseMono|x64.ActiveCfg = ReleaseMono|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseMono|x64.Build.0 = ReleaseMono|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseMono|x86.ActiveCfg = ReleaseMono|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseMono|x86.Build.0 = ReleaseMono|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseMonoPY3|Any CPU.ActiveCfg = ReleaseMonoPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseMonoPY3|Any CPU.Build.0 = ReleaseMonoPY3|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseMonoPY3|x64.ActiveCfg = ReleaseMonoPY3|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseMonoPY3|x64.Build.0 = ReleaseMonoPY3|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseMonoPY3|x86.ActiveCfg = ReleaseMonoPY3|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseMonoPY3|x86.Build.0 = ReleaseMonoPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseWin|Any CPU.ActiveCfg = ReleaseWin|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseWin|Any CPU.Build.0 = ReleaseWin|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseWin|x64.ActiveCfg = ReleaseWin|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseWin|x64.Build.0 = ReleaseWin|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseWin|x86.ActiveCfg = ReleaseWin|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseWin|x86.Build.0 = ReleaseWin|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseWinPY3|Any CPU.ActiveCfg = ReleaseWinPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseWinPY3|Any CPU.Build.0 = ReleaseWinPY3|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseWinPY3|x64.ActiveCfg = ReleaseWinPY3|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseWinPY3|x64.Build.0 = ReleaseWinPY3|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseWinPY3|x86.ActiveCfg = ReleaseWinPY3|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseWinPY3|x86.Build.0 = ReleaseWinPY3|Any CPU + {66B8D01A-9906-452A-B09E-BF75EA76468F}.Debug|Any CPU.ActiveCfg = ReleaseWinPY3|x86 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.Debug|Any CPU.Build.0 = ReleaseWinPY3|x86 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.Debug|x64.ActiveCfg = DebugWinPY3|x64 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.Debug|x64.Build.0 = DebugWinPY3|x64 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.Debug|x86.ActiveCfg = DebugWinPY3|x86 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.Debug|x86.Build.0 = DebugWinPY3|x86 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugMono|Any CPU.ActiveCfg = DebugMono|x86 {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugMono|x64.ActiveCfg = DebugMono|x64 {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugMono|x64.Build.0 = DebugMono|x64 {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugMono|x86.ActiveCfg = DebugMono|x86 {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugMono|x86.Build.0 = DebugMono|x86 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugMonoPY3|Any CPU.ActiveCfg = DebugMonoPY3|x86 {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugMonoPY3|x64.ActiveCfg = DebugMonoPY3|x64 {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugMonoPY3|x64.Build.0 = DebugMonoPY3|x64 {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugMonoPY3|x86.ActiveCfg = DebugMonoPY3|x86 {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugMonoPY3|x86.Build.0 = DebugMonoPY3|x86 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugWin|Any CPU.ActiveCfg = DebugWin|x86 {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugWin|x64.ActiveCfg = DebugWin|x64 {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugWin|x64.Build.0 = DebugWin|x64 {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugWin|x86.ActiveCfg = DebugWin|x86 {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugWin|x86.Build.0 = DebugWin|x86 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugWinPY3|Any CPU.ActiveCfg = DebugWinPY3|x86 {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugWinPY3|x64.ActiveCfg = DebugWinPY3|x64 {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugWinPY3|x64.Build.0 = DebugWinPY3|x64 {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugWinPY3|x86.ActiveCfg = DebugWinPY3|x86 {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugWinPY3|x86.Build.0 = DebugWinPY3|x86 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.Release|Any CPU.ActiveCfg = ReleaseWinPY3|x86 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.Release|Any CPU.Build.0 = ReleaseWinPY3|x86 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.Release|x64.ActiveCfg = ReleaseWinPY3|x64 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.Release|x64.Build.0 = ReleaseWinPY3|x64 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.Release|x86.ActiveCfg = ReleaseWinPY3|x86 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.Release|x86.Build.0 = ReleaseWinPY3|x86 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseMono|Any CPU.ActiveCfg = ReleaseMono|x86 {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseMono|x64.ActiveCfg = ReleaseMono|x64 {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseMono|x64.Build.0 = ReleaseMono|x64 {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseMono|x86.ActiveCfg = ReleaseMono|x86 {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseMono|x86.Build.0 = ReleaseMono|x86 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseMonoPY3|Any CPU.ActiveCfg = ReleaseMonoPY3|x86 {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseMonoPY3|x64.ActiveCfg = ReleaseMonoPY3|x64 {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseMonoPY3|x64.Build.0 = ReleaseMonoPY3|x64 {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseMonoPY3|x86.ActiveCfg = ReleaseMonoPY3|x86 {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseMonoPY3|x86.Build.0 = ReleaseMonoPY3|x86 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseWin|Any CPU.ActiveCfg = ReleaseWin|x86 {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseWin|x64.ActiveCfg = ReleaseWin|x64 {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseWin|x64.Build.0 = ReleaseWin|x64 {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseWin|x86.ActiveCfg = ReleaseWin|x86 {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseWin|x86.Build.0 = ReleaseWin|x86 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseWinPY3|Any CPU.ActiveCfg = ReleaseWinPY3|x86 {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseWinPY3|x64.ActiveCfg = ReleaseWinPY3|x64 {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseWinPY3|x64.Build.0 = ReleaseWinPY3|x64 {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseWinPY3|x86.ActiveCfg = ReleaseWinPY3|x86 {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseWinPY3|x86.Build.0 = ReleaseWinPY3|x86 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.Debug|Any CPU.ActiveCfg = ReleaseWinPY3|x86 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.Debug|Any CPU.Build.0 = ReleaseWinPY3|x86 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.Debug|x64.ActiveCfg = DebugWinPY3|x64 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.Debug|x64.Build.0 = DebugWinPY3|x64 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.Debug|x86.ActiveCfg = DebugWinPY3|x86 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.Debug|x86.Build.0 = DebugWinPY3|x86 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.DebugMono|Any CPU.ActiveCfg = DebugMono|x86 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.DebugMono|x64.ActiveCfg = DebugMono|x64 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.DebugMono|x86.ActiveCfg = DebugMono|x86 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.DebugMonoPY3|Any CPU.ActiveCfg = DebugMonoPY3|x86 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.DebugMonoPY3|x64.ActiveCfg = DebugMonoPY3|x64 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.DebugMonoPY3|x86.ActiveCfg = DebugMonoPY3|x86 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.DebugWin|Any CPU.ActiveCfg = DebugWin|x86 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.DebugWin|x64.ActiveCfg = DebugWin|x64 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.DebugWin|x64.Build.0 = DebugWin|x64 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.DebugWin|x86.ActiveCfg = DebugWin|x86 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.DebugWin|x86.Build.0 = DebugWin|x86 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.DebugWinPY3|Any CPU.ActiveCfg = DebugWinPY3|x86 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.DebugWinPY3|x64.ActiveCfg = DebugWinPY3|x64 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.DebugWinPY3|x64.Build.0 = DebugWinPY3|x64 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.DebugWinPY3|x86.ActiveCfg = DebugWinPY3|x86 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.DebugWinPY3|x86.Build.0 = DebugWinPY3|x86 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.Release|Any CPU.ActiveCfg = ReleaseWinPY3|x86 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.Release|Any CPU.Build.0 = ReleaseWinPY3|x86 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.Release|x64.ActiveCfg = ReleaseWinPY3|x64 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.Release|x64.Build.0 = ReleaseWinPY3|x64 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.Release|x86.ActiveCfg = ReleaseWinPY3|x86 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.Release|x86.Build.0 = ReleaseWinPY3|x86 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.ReleaseMono|Any CPU.ActiveCfg = ReleaseMono|x86 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.ReleaseMono|x64.ActiveCfg = ReleaseMono|x64 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.ReleaseMono|x86.ActiveCfg = ReleaseMono|x86 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.ReleaseMonoPY3|Any CPU.ActiveCfg = ReleaseMonoPY3|x86 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.ReleaseMonoPY3|x64.ActiveCfg = ReleaseMonoPY3|x64 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.ReleaseMonoPY3|x86.ActiveCfg = ReleaseMonoPY3|x86 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.ReleaseWin|Any CPU.ActiveCfg = ReleaseWin|x86 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.ReleaseWin|x64.ActiveCfg = ReleaseWin|x64 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.ReleaseWin|x64.Build.0 = ReleaseWin|x64 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.ReleaseWin|x86.ActiveCfg = ReleaseWin|x86 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.ReleaseWin|x86.Build.0 = ReleaseWin|x86 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.ReleaseWinPY3|Any CPU.ActiveCfg = ReleaseWinPY3|x86 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.ReleaseWinPY3|x64.ActiveCfg = ReleaseWinPY3|x64 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.ReleaseWinPY3|x64.Build.0 = ReleaseWinPY3|x64 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.ReleaseWinPY3|x86.ActiveCfg = ReleaseWinPY3|x86 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.ReleaseWinPY3|x86.Build.0 = ReleaseWinPY3|x86 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.Debug|Any CPU.ActiveCfg = ReleaseWinPY3|x86 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.Debug|Any CPU.Build.0 = ReleaseWinPY3|x86 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.Debug|x64.ActiveCfg = DebugWinPY3|x64 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.Debug|x64.Build.0 = DebugWinPY3|x64 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.Debug|x86.ActiveCfg = DebugWinPY3|x86 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.Debug|x86.Build.0 = DebugWinPY3|x86 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugMono|Any CPU.ActiveCfg = DebugMono|x86 {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugMono|x64.ActiveCfg = DebugMono|x64 {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugMono|x64.Build.0 = DebugMono|x64 {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugMono|x86.ActiveCfg = DebugMono|x86 {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugMono|x86.Build.0 = DebugMono|x86 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugMonoPY3|Any CPU.ActiveCfg = DebugMonoPY3|x86 {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugMonoPY3|x64.ActiveCfg = DebugMonoPY3|x64 {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugMonoPY3|x64.Build.0 = DebugMonoPY3|x64 {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugMonoPY3|x86.ActiveCfg = DebugMonoPY3|x86 {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugMonoPY3|x86.Build.0 = DebugMonoPY3|x86 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugWin|Any CPU.ActiveCfg = DebugWin|x86 {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugWin|x64.ActiveCfg = DebugWin|x64 {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugWin|x64.Build.0 = DebugWin|x64 {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugWin|x86.ActiveCfg = DebugWin|x86 {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugWin|x86.Build.0 = DebugWin|x86 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugWinPY3|Any CPU.ActiveCfg = DebugWinPY3|x86 {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugWinPY3|x64.ActiveCfg = DebugWinPY3|x64 {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugWinPY3|x64.Build.0 = DebugWinPY3|x64 {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugWinPY3|x86.ActiveCfg = DebugWinPY3|x86 {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugWinPY3|x86.Build.0 = DebugWinPY3|x86 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.Release|Any CPU.ActiveCfg = ReleaseWinPY3|x86 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.Release|Any CPU.Build.0 = ReleaseWinPY3|x86 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.Release|x64.ActiveCfg = ReleaseWinPY3|x64 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.Release|x64.Build.0 = ReleaseWinPY3|x64 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.Release|x86.ActiveCfg = ReleaseWinPY3|x86 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.Release|x86.Build.0 = ReleaseWinPY3|x86 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseMono|Any CPU.ActiveCfg = ReleaseMono|x86 {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseMono|x64.ActiveCfg = ReleaseMono|x64 {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseMono|x64.Build.0 = ReleaseMono|x64 {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseMono|x86.ActiveCfg = ReleaseMono|x86 {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseMono|x86.Build.0 = ReleaseMono|x86 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseMonoPY3|Any CPU.ActiveCfg = ReleaseMonoPY3|x86 {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseMonoPY3|x64.ActiveCfg = ReleaseMonoPY3|x64 {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseMonoPY3|x64.Build.0 = ReleaseMonoPY3|x64 {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseMonoPY3|x86.ActiveCfg = ReleaseMonoPY3|x86 {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseMonoPY3|x86.Build.0 = ReleaseMonoPY3|x86 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseWin|Any CPU.ActiveCfg = ReleaseWin|x86 {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseWin|x64.ActiveCfg = ReleaseWin|x64 {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseWin|x64.Build.0 = ReleaseWin|x64 {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseWin|x86.ActiveCfg = ReleaseWin|x86 {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseWin|x86.Build.0 = ReleaseWin|x86 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseWinPY3|Any CPU.ActiveCfg = ReleaseWinPY3|x86 {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseWinPY3|x64.ActiveCfg = ReleaseWinPY3|x64 {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseWinPY3|x64.Build.0 = ReleaseWinPY3|x64 {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseWinPY3|x86.ActiveCfg = ReleaseWinPY3|x86 {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseWinPY3|x86.Build.0 = ReleaseWinPY3|x86 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.Debug|Any CPU.ActiveCfg = ReleaseWinPY3|x86 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.Debug|Any CPU.Build.0 = ReleaseWinPY3|x86 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.Debug|x64.ActiveCfg = DebugWinPY3|x64 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.Debug|x64.Build.0 = DebugWinPY3|x64 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.Debug|x86.ActiveCfg = DebugWinPY3|x86 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.Debug|x86.Build.0 = DebugWinPY3|x86 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugMono|Any CPU.ActiveCfg = DebugMono|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugMono|x64.ActiveCfg = DebugMono|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugMono|x64.Build.0 = DebugMono|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugMono|x86.ActiveCfg = DebugMono|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugMono|x86.Build.0 = DebugMono|x86 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugMonoPY3|Any CPU.ActiveCfg = DebugMonoPY3|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugMonoPY3|x64.ActiveCfg = DebugMonoPY3|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugMonoPY3|x64.Build.0 = DebugMonoPY3|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugMonoPY3|x86.ActiveCfg = DebugMonoPY3|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugMonoPY3|x86.Build.0 = DebugMonoPY3|x86 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugWin|Any CPU.ActiveCfg = DebugWin|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugWin|x64.ActiveCfg = DebugWin|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugWin|x64.Build.0 = DebugWin|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugWin|x86.ActiveCfg = DebugWin|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugWin|x86.Build.0 = DebugWin|x86 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugWinPY3|Any CPU.ActiveCfg = DebugWinPY3|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugWinPY3|x64.ActiveCfg = DebugWinPY3|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugWinPY3|x64.Build.0 = DebugWinPY3|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugWinPY3|x86.ActiveCfg = DebugWinPY3|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugWinPY3|x86.Build.0 = DebugWinPY3|x86 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.Release|Any CPU.ActiveCfg = ReleaseWinPY3|x86 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.Release|Any CPU.Build.0 = ReleaseWinPY3|x86 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.Release|x64.ActiveCfg = ReleaseWinPY3|x64 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.Release|x64.Build.0 = ReleaseWinPY3|x64 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.Release|x86.ActiveCfg = ReleaseWinPY3|x86 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.Release|x86.Build.0 = ReleaseWinPY3|x86 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseMono|Any CPU.ActiveCfg = ReleaseMono|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseMono|x64.ActiveCfg = ReleaseMono|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseMono|x64.Build.0 = ReleaseMono|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseMono|x86.ActiveCfg = ReleaseMono|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseMono|x86.Build.0 = ReleaseMono|x86 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseMonoPY3|Any CPU.ActiveCfg = ReleaseMonoPY3|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseMonoPY3|x64.ActiveCfg = ReleaseMonoPY3|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseMonoPY3|x64.Build.0 = ReleaseMonoPY3|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseMonoPY3|x86.ActiveCfg = ReleaseMonoPY3|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseMonoPY3|x86.Build.0 = ReleaseMonoPY3|x86 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseWin|Any CPU.ActiveCfg = ReleaseWin|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseWin|x64.ActiveCfg = ReleaseWin|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseWin|x64.Build.0 = ReleaseWin|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseWin|x86.ActiveCfg = ReleaseWin|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseWin|x86.Build.0 = ReleaseWin|x86 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseWinPY3|Any CPU.ActiveCfg = ReleaseWinPY3|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseWinPY3|x64.ActiveCfg = ReleaseWinPY3|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseWinPY3|x64.Build.0 = ReleaseWinPY3|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseWinPY3|x86.ActiveCfg = ReleaseWinPY3|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseWinPY3|x86.Build.0 = ReleaseWinPY3|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|Any CPU.ActiveCfg = DebugMono|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|Any CPU.Build.0 = DebugMono|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|x64.ActiveCfg = DebugMono|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|x64.Build.0 = DebugMono|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|x86.ActiveCfg = DebugMono|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|x86.Build.0 = DebugMono|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMono|Any CPU.ActiveCfg = DebugMono|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMono|x64.ActiveCfg = DebugMono|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMono|x86.ActiveCfg = DebugMono|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMonoPY3|Any CPU.ActiveCfg = DebugMonoPY3|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMonoPY3|x64.ActiveCfg = DebugMonoPY3|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMonoPY3|x86.ActiveCfg = DebugMonoPY3|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|Any CPU.ActiveCfg = DebugWin|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|Any CPU.Build.0 = DebugWin|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|x64.ActiveCfg = DebugWin|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|x64.Build.0 = DebugWin|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|x86.ActiveCfg = DebugWin|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|x86.Build.0 = DebugWin|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|Any CPU.ActiveCfg = DebugWinPY3|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|Any CPU.Build.0 = DebugWinPY3|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|x64.ActiveCfg = DebugWinPY3|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|x64.Build.0 = DebugWinPY3|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|x86.ActiveCfg = DebugWinPY3|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|x86.Build.0 = DebugWinPY3|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|Any CPU.ActiveCfg = ReleaseMono|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|Any CPU.Build.0 = ReleaseMono|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|x64.ActiveCfg = ReleaseMono|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|x64.Build.0 = ReleaseMono|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|x86.ActiveCfg = ReleaseMono|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|x86.Build.0 = ReleaseMono|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMono|Any CPU.ActiveCfg = ReleaseMono|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMono|x64.ActiveCfg = ReleaseMono|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMono|x86.ActiveCfg = ReleaseMono|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMonoPY3|Any CPU.ActiveCfg = ReleaseMonoPY3|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMonoPY3|x64.ActiveCfg = ReleaseMonoPY3|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMonoPY3|x86.ActiveCfg = ReleaseMonoPY3|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|Any CPU.ActiveCfg = ReleaseWin|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|Any CPU.Build.0 = ReleaseWin|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|x64.ActiveCfg = ReleaseWin|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|x64.Build.0 = ReleaseWin|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|x86.ActiveCfg = ReleaseWin|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|x86.Build.0 = ReleaseWin|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|Any CPU.ActiveCfg = ReleaseWinPY3|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|Any CPU.Build.0 = ReleaseWinPY3|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|x64.ActiveCfg = ReleaseWinPY3|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|x64.Build.0 = ReleaseWinPY3|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|x86.ActiveCfg = ReleaseWinPY3|Any CPU + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|x86.Build.0 = ReleaseWinPY3|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/src/perf_tests/BaselineComparisonBenchmarkBase.cs b/src/perf_tests/BaselineComparisonBenchmarkBase.cs new file mode 100644 index 000000000..2388e3982 --- /dev/null +++ b/src/perf_tests/BaselineComparisonBenchmarkBase.cs @@ -0,0 +1,69 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; + +using Python.Runtime; + +namespace Python.PerformanceTests +{ + public class BaselineComparisonBenchmarkBase + { + public BaselineComparisonBenchmarkBase() + { + Console.WriteLine($"CWD: {Environment.CurrentDirectory}"); + Console.WriteLine($"Using Python.Runtime from {typeof(PythonEngine).Assembly.Location} {typeof(PythonEngine).Assembly.GetName()}"); + + try { + PythonEngine.Initialize(); + Console.WriteLine("Python Initialized"); + if (PythonEngine.BeginAllowThreads() == IntPtr.Zero) + throw new PythonException(); + Console.WriteLine("Threading enabled"); + } + catch (Exception e) { + Console.WriteLine(e); + throw; + } + } + + static BaselineComparisonBenchmarkBase() + { + string pythonRuntimeDll = Environment.GetEnvironmentVariable(BaselineComparisonConfig.EnvironmentVariableName); + if (string.IsNullOrEmpty(pythonRuntimeDll)) + { + throw new ArgumentException( + "Required environment variable is missing", + BaselineComparisonConfig.EnvironmentVariableName); + } + + Console.WriteLine("Preloading " + pythonRuntimeDll); + Assembly.LoadFrom(pythonRuntimeDll); + foreach (var assembly in AppDomain.CurrentDomain.GetAssemblies()) { + if (assembly.FullName.StartsWith("Python.Runtime")) + Console.WriteLine(assembly.Location); + foreach(var dependency in assembly.GetReferencedAssemblies()) + if (dependency.FullName.Contains("Python.Runtime")) { + Console.WriteLine($"{assembly} -> {dependency}"); + } + } + + AppDomain.CurrentDomain.AssemblyResolve += CurrentDomainOnAssemblyResolve; + } + + static Assembly CurrentDomainOnAssemblyResolve(object sender, ResolveEventArgs args) { + if (!args.Name.StartsWith("Python.Runtime")) + return null; + + var preloaded = AppDomain.CurrentDomain.GetAssemblies() + .FirstOrDefault(a => a.GetName().Name == "Python.Runtime"); + if (preloaded != null) return preloaded; + + string pythonRuntimeDll = Environment.GetEnvironmentVariable(BaselineComparisonConfig.EnvironmentVariableName); + if (string.IsNullOrEmpty(pythonRuntimeDll)) + return null; + + return Assembly.LoadFrom(pythonRuntimeDll); + } + } +} diff --git a/src/perf_tests/BaselineComparisonConfig.cs b/src/perf_tests/BaselineComparisonConfig.cs new file mode 100644 index 000000000..06d529ff9 --- /dev/null +++ b/src/perf_tests/BaselineComparisonConfig.cs @@ -0,0 +1,47 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Reflection; + +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Jobs; + +namespace Python.PerformanceTests +{ + public class BaselineComparisonConfig : ManualConfig + { + public const string EnvironmentVariableName = "PythonRuntimeDLL"; + + public BaselineComparisonConfig() + { + this.Options |= ConfigOptions.DisableOptimizationsValidator; + + string deploymentRoot = BenchmarkTests.DeploymentRoot; + + var baseJob = Job.Default; + this.Add(baseJob + .WithId("baseline") + .WithEnvironmentVariable(EnvironmentVariableName, + Path.Combine(deploymentRoot, "baseline", "Python.Runtime.dll")) + .WithBaseline(true)); + this.Add(baseJob + .WithId("new") + .WithEnvironmentVariable(EnvironmentVariableName, + Path.Combine(deploymentRoot, "new", "Python.Runtime.dll"))); + } + + static BaselineComparisonConfig() { + AppDomain.CurrentDomain.AssemblyResolve += CurrentDomainOnAssemblyResolve; + } + + static Assembly CurrentDomainOnAssemblyResolve(object sender, ResolveEventArgs args) { + Console.WriteLine(args.Name); + if (!args.Name.StartsWith("Python.Runtime")) + return null; + string pythonRuntimeDll = Environment.GetEnvironmentVariable(EnvironmentVariableName); + if (string.IsNullOrEmpty(pythonRuntimeDll)) + pythonRuntimeDll = Path.Combine(BenchmarkTests.DeploymentRoot, "baseline", "Python.Runtime.dll"); + return Assembly.LoadFrom(pythonRuntimeDll); + } + } +} diff --git a/src/perf_tests/BenchmarkTests.cs b/src/perf_tests/BenchmarkTests.cs new file mode 100644 index 000000000..12ba6c900 --- /dev/null +++ b/src/perf_tests/BenchmarkTests.cs @@ -0,0 +1,63 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Reflection; + +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; +using NUnit.Framework; + +namespace Python.PerformanceTests +{ + public class BenchmarkTests + { + Summary summary; + + [OneTimeSetUp] + public void SetUp() + { + Environment.CurrentDirectory = Path.Combine(DeploymentRoot, "new"); + this.summary = BenchmarkRunner.Run(); + Assert.IsNotEmpty(this.summary.Reports); + Assert.IsTrue(this.summary.Reports.All(r => r.Success)); + } + + [Test] + public void ReadInt64Property() + { + double optimisticPerfRatio = GetOptimisticPerfRatio(this.summary.Reports); + Assert.LessOrEqual(optimisticPerfRatio, 0.68); + } + + [Test] + public void WriteInt64Property() + { + double optimisticPerfRatio = GetOptimisticPerfRatio(this.summary.Reports); + Assert.LessOrEqual(optimisticPerfRatio, 0.66); + } + + static double GetOptimisticPerfRatio( + IReadOnlyList reports, + [CallerMemberName] string methodName = null) + { + reports = reports.Where(r => r.BenchmarkCase.Descriptor.WorkloadMethod.Name == methodName).ToArray(); + if (reports.Count == 0) + throw new ArgumentException( + message: $"No reports found for {methodName}. " + + "You have to match test method name to benchmark method name or " + + "pass benchmark method name explicitly", + paramName: nameof(methodName)); + + var baseline = reports.Single(r => r.BenchmarkCase.Job.ResolvedId == "baseline").ResultStatistics; + var @new = reports.Single(r => r.BenchmarkCase.Job.ResolvedId != "baseline").ResultStatistics; + + double newTimeOptimistic = @new.Mean - (@new.StandardDeviation + baseline.StandardDeviation) * 0.5; + + return newTimeOptimistic / baseline.Mean; + } + + public static string DeploymentRoot => Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); + } +} diff --git a/src/perf_tests/Python.PerformanceTests.csproj b/src/perf_tests/Python.PerformanceTests.csproj new file mode 100644 index 000000000..33949fdc1 --- /dev/null +++ b/src/perf_tests/Python.PerformanceTests.csproj @@ -0,0 +1,34 @@ + + + + net461 + DebugMono;DebugMonoPY3;ReleaseMono;ReleaseMonoPY3;DebugWin;DebugWinPY3;ReleaseWin;ReleaseWinPY3 + + false + + + + + + + + + compile + + + + + + + + + + + + + + + + + + diff --git a/src/perf_tests/PythonCallingNetBenchmark.cs b/src/perf_tests/PythonCallingNetBenchmark.cs new file mode 100644 index 000000000..4e9461d2e --- /dev/null +++ b/src/perf_tests/PythonCallingNetBenchmark.cs @@ -0,0 +1,46 @@ +using System; +using System.Collections.Generic; +using System.Text; + +using BenchmarkDotNet.Attributes; +using Python.Runtime; + +namespace Python.PerformanceTests +{ + [Config(typeof(BaselineComparisonConfig))] + public class PythonCallingNetBenchmark: BaselineComparisonBenchmarkBase + { + [Benchmark] + public void ReadInt64Property() + { + using (Py.GIL()) + { + var locals = new PyDict(); + locals.SetItem("a", new NetObject().ToPython()); + PythonEngine.Exec($@" +s = 0 +for i in range(300000): + s += a.{nameof(NetObject.LongProperty)} +", locals: locals.Handle); + } + } + + [Benchmark] + public void WriteInt64Property() { + using (Py.GIL()) { + var locals = new PyDict(); + locals.SetItem("a", new NetObject().ToPython()); + PythonEngine.Exec($@" +s = 0 +for i in range(300000): + a.{nameof(NetObject.LongProperty)} += i +", locals: locals.Handle); + } + } + } + + class NetObject + { + public long LongProperty { get; set; } = 42; + } +} From e1931262c8d0f18e6bd55cb122510be417733794 Mon Sep 17 00:00:00 2001 From: Ivan Cronyn Date: Wed, 13 Nov 2019 13:46:22 +0000 Subject: [PATCH 026/112] Adds support for the Jetson Nano (#986) --- CHANGELOG.md | 1 + src/runtime/platform/Types.cs | 1 + src/runtime/runtime.cs | 1 + 3 files changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c7cad9567..5c999d668 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. - Added automatic NuGet package generation in appveyor and local builds - Added function that sets Py_NoSiteFlag to 1. +- Added support for Jetson Nano. ### Changed diff --git a/src/runtime/platform/Types.cs b/src/runtime/platform/Types.cs index bdc51af39..62be0e421 100644 --- a/src/runtime/platform/Types.cs +++ b/src/runtime/platform/Types.cs @@ -6,6 +6,7 @@ public enum MachineType x86_64, armv7l, armv8, + aarch64, Other }; diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index a1f9a38aa..f97821d13 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -143,6 +143,7 @@ public class Runtime ["em64t"] = MachineType.x86_64, ["armv7l"] = MachineType.armv7l, ["armv8"] = MachineType.armv8, + ["aarch64"] = MachineType.aarch64, }; ///

From 2736094ab87dcde26e63e1fddbb30a7a2453c961 Mon Sep 17 00:00:00 2001 From: matham Date: Tue, 19 Nov 2019 01:19:29 -0500 Subject: [PATCH 027/112] Add CI support for py3.8. (#988) * Add CI support for py3.8 * Add interop38.cs * Add PYTHON38 * Add support for 3.8 * Bump 3.7 to 3.8 * Allow failures for py3.8 because it's a Python 3.8.0 bug * Add note about py3.8.0 to readme --- README.rst | 8 ++ appveyor.yml | 6 ++ src/runtime/Python.Runtime.15.csproj | 2 +- src/runtime/Python.Runtime.csproj | 27 ++--- src/runtime/interop38.cs | 152 +++++++++++++++++++++++++++ src/runtime/runtime.cs | 5 +- 6 files changed, 185 insertions(+), 15 deletions(-) create mode 100644 src/runtime/interop38.cs diff --git a/README.rst b/README.rst index 84bf93d84..ee6573d84 100644 --- a/README.rst +++ b/README.rst @@ -95,6 +95,14 @@ projects using pythonnet can be found in the Wiki: https://github.com/pythonnet/pythonnet/wiki +Python 3.8.0 support +-------------------- + +Some features are disabled in Python 3.8.0 because of +`this bug in Python `_. The error is +``System.EntryPointNotFoundException : Unable to find an entry point named +'Py_CompileString' in DLL 'python38'``. This will be fixed in Python 3.8.1. + .. |Join the chat at https://gitter.im/pythonnet/pythonnet| image:: https://badges.gitter.im/pythonnet/pythonnet.svg :target: https://gitter.im/pythonnet/pythonnet?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge .. |appveyor shield| image:: https://img.shields.io/appveyor/ci/pythonnet/pythonnet/master.svg?label=AppVeyor diff --git a/appveyor.yml b/appveyor.yml index 445f9bb5a..20d8ed991 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -23,16 +23,22 @@ environment: BUILD_OPTS: --xplat - PYTHON_VERSION: 3.7 BUILD_OPTS: --xplat + - PYTHON_VERSION: 3.8 + BUILD_OPTS: --xplat - PYTHON_VERSION: 2.7 - PYTHON_VERSION: 3.5 - PYTHON_VERSION: 3.6 - PYTHON_VERSION: 3.7 + - PYTHON_VERSION: 3.8 matrix: allow_failures: - PYTHON_VERSION: 3.4 BUILD_OPTS: --xplat - PYTHON_VERSION: 3.4 + - PYTHON_VERSION: 3.8 + BUILD_OPTS: --xplat + - PYTHON_VERSION: 3.8 init: # Update Environment Variables based on matrix/platform diff --git a/src/runtime/Python.Runtime.15.csproj b/src/runtime/Python.Runtime.15.csproj index 122132513..c31d4bf91 100644 --- a/src/runtime/Python.Runtime.15.csproj +++ b/src/runtime/Python.Runtime.15.csproj @@ -42,7 +42,7 @@ $(PYTHONNET_PY2_VERSION) PYTHON27 $(PYTHONNET_PY3_VERSION) - PYTHON37 + PYTHON38 $(PYTHONNET_WIN_DEFINE_CONSTANTS) UCS2 $(PYTHONNET_MONO_DEFINE_CONSTANTS) diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index ac6b59150..02656e51e 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -22,11 +22,11 @@ - PYTHON2;PYTHON27;UCS4 @@ -34,7 +34,7 @@ pdbonly - PYTHON3;PYTHON37;UCS4 + PYTHON3;PYTHON38;UCS4 true pdbonly @@ -46,7 +46,7 @@ true - PYTHON3;PYTHON37;UCS4;TRACE;DEBUG + PYTHON3;PYTHON38;UCS4;TRACE;DEBUG false full @@ -56,7 +56,7 @@ pdbonly - PYTHON3;PYTHON37;UCS2 + PYTHON3;PYTHON38;UCS2 true pdbonly @@ -68,7 +68,7 @@ true - PYTHON3;PYTHON37;UCS2;TRACE;DEBUG + PYTHON3;PYTHON38;UCS2;TRACE;DEBUG false full @@ -140,8 +140,8 @@ - - + + @@ -151,7 +151,8 @@ - + + @@ -170,4 +171,4 @@ - + diff --git a/src/runtime/interop38.cs b/src/runtime/interop38.cs new file mode 100644 index 000000000..8f2e32afe --- /dev/null +++ b/src/runtime/interop38.cs @@ -0,0 +1,152 @@ + +// Auto-generated by geninterop.py. +// DO NOT MODIFIY BY HAND. + + +#if PYTHON38 +using System; +using System.Collections; +using System.Collections.Specialized; +using System.Runtime.InteropServices; +using System.Reflection; +using System.Text; + +namespace Python.Runtime +{ + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] + internal class TypeOffset + { + static TypeOffset() + { + Type type = typeof(TypeOffset); + FieldInfo[] fi = type.GetFields(); + int size = IntPtr.Size; + for (int i = 0; i < fi.Length; i++) + { + fi[i].SetValue(null, i * size); + } + } + + public static int magic() + { + return ob_size; + } + + // Auto-generated from PyHeapTypeObject in Python.h + public static int ob_refcnt = 0; + public static int ob_type = 0; + public static int ob_size = 0; + public static int tp_name = 0; + public static int tp_basicsize = 0; + public static int tp_itemsize = 0; + public static int tp_dealloc = 0; + public static int tp_vectorcall_offset = 0; + public static int tp_getattr = 0; + public static int tp_setattr = 0; + public static int tp_as_async = 0; + public static int tp_repr = 0; + public static int tp_as_number = 0; + public static int tp_as_sequence = 0; + public static int tp_as_mapping = 0; + public static int tp_hash = 0; + public static int tp_call = 0; + public static int tp_str = 0; + public static int tp_getattro = 0; + public static int tp_setattro = 0; + public static int tp_as_buffer = 0; + public static int tp_flags = 0; + public static int tp_doc = 0; + public static int tp_traverse = 0; + public static int tp_clear = 0; + public static int tp_richcompare = 0; + public static int tp_weaklistoffset = 0; + public static int tp_iter = 0; + public static int tp_iternext = 0; + public static int tp_methods = 0; + public static int tp_members = 0; + public static int tp_getset = 0; + public static int tp_base = 0; + public static int tp_dict = 0; + public static int tp_descr_get = 0; + public static int tp_descr_set = 0; + public static int tp_dictoffset = 0; + public static int tp_init = 0; + public static int tp_alloc = 0; + public static int tp_new = 0; + public static int tp_free = 0; + public static int tp_is_gc = 0; + public static int tp_bases = 0; + public static int tp_mro = 0; + public static int tp_cache = 0; + public static int tp_subclasses = 0; + public static int tp_weaklist = 0; + public static int tp_del = 0; + public static int tp_version_tag = 0; + public static int tp_finalize = 0; + public static int tp_vectorcall = 0; + public static int am_await = 0; + public static int am_aiter = 0; + public static int am_anext = 0; + public static int nb_add = 0; + public static int nb_subtract = 0; + public static int nb_multiply = 0; + public static int nb_remainder = 0; + public static int nb_divmod = 0; + public static int nb_power = 0; + public static int nb_negative = 0; + public static int nb_positive = 0; + public static int nb_absolute = 0; + public static int nb_bool = 0; + public static int nb_invert = 0; + public static int nb_lshift = 0; + public static int nb_rshift = 0; + public static int nb_and = 0; + public static int nb_xor = 0; + public static int nb_or = 0; + public static int nb_int = 0; + public static int nb_reserved = 0; + public static int nb_float = 0; + public static int nb_inplace_add = 0; + public static int nb_inplace_subtract = 0; + public static int nb_inplace_multiply = 0; + public static int nb_inplace_remainder = 0; + public static int nb_inplace_power = 0; + public static int nb_inplace_lshift = 0; + public static int nb_inplace_rshift = 0; + public static int nb_inplace_and = 0; + public static int nb_inplace_xor = 0; + public static int nb_inplace_or = 0; + public static int nb_floor_divide = 0; + public static int nb_true_divide = 0; + public static int nb_inplace_floor_divide = 0; + public static int nb_inplace_true_divide = 0; + public static int nb_index = 0; + public static int nb_matrix_multiply = 0; + public static int nb_inplace_matrix_multiply = 0; + public static int mp_length = 0; + public static int mp_subscript = 0; + public static int mp_ass_subscript = 0; + public static int sq_length = 0; + public static int sq_concat = 0; + public static int sq_repeat = 0; + public static int sq_item = 0; + public static int was_sq_slice = 0; + public static int sq_ass_item = 0; + public static int was_sq_ass_slice = 0; + public static int sq_contains = 0; + public static int sq_inplace_concat = 0; + public static int sq_inplace_repeat = 0; + public static int bf_getbuffer = 0; + public static int bf_releasebuffer = 0; + public static int name = 0; + public static int ht_slots = 0; + public static int qualname = 0; + public static int ht_cached_keys = 0; + + /* here are optional user slots, followed by the members. */ + public static int members = 0; + } +} + +#endif + diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index f97821d13..7a78cd6e1 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -64,8 +64,11 @@ public class Runtime #elif PYTHON37 internal const string _pyversion = "3.7"; internal const string _pyver = "37"; +#elif PYTHON38 + internal const string _pyversion = "3.8"; + internal const string _pyver = "38"; #else -#error You must define one of PYTHON34 to PYTHON37 or PYTHON27 +#error You must define one of PYTHON34 to PYTHON38 or PYTHON27 #endif #if MONO_LINUX || MONO_OSX // Linux/macOS use dotted version string From 5f2e2e2f81d0d4439f04d6ee322bae304d382f21 Mon Sep 17 00:00:00 2001 From: Benoit Hudson Date: Thu, 21 Nov 2019 04:10:58 -0500 Subject: [PATCH 028/112] Split from PR 958: restoring the __import__ after shutdown. (#993) When C# shuts down we should restore Python to its original state. --- src/runtime/importhook.cs | 67 +++++++++++++++++++++++++++++++-------- 1 file changed, 54 insertions(+), 13 deletions(-) diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index 7e4a208f5..06ba7a56d 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -26,25 +26,64 @@ internal static void InitializeModuleDef() #endif ///

- /// Initialization performed on startup of the Python runtime. + /// Get a New reference to the builtins module. ///

- internal static void Initialize() + static IntPtr GetNewRefToBuiltins() { - // Initialize the Python <--> CLR module hook. We replace the - // built-in Python __import__ with our own. This isn't ideal, - // but it provides the most "Pythonic" way of dealing with CLR - // modules (Python doesn't provide a way to emulate packages). - IntPtr dict = Runtime.PyImport_GetModuleDict(); + if (Runtime.IsPython3) + { + return Runtime.PyImport_ImportModule("builtins"); + } + else + { + // dict is a borrowed ref, no need to decref + IntPtr dict = Runtime.PyImport_GetModuleDict(); - IntPtr mod = Runtime.IsPython3 - ? Runtime.PyImport_ImportModule("builtins") - : Runtime.PyDict_GetItemString(dict, "__builtin__"); + // GetItemString is a borrowed ref; incref to get a new ref + IntPtr builtins = Runtime.PyDict_GetItemString(dict, "__builtin__"); + Runtime.XIncref(builtins); + return builtins; + } + } - py_import = Runtime.PyObject_GetAttrString(mod, "__import__"); + ///

+ /// Initialize just the __import__ hook itself. + ///

+ static void InitImport() + { + // We replace the built-in Python __import__ with our own: first + // look in CLR modules, then if we don't find any call the default + // Python __import__. + IntPtr builtins = GetNewRefToBuiltins(); + py_import = Runtime.PyObject_GetAttrString(builtins, "__import__"); hook = new MethodWrapper(typeof(ImportHook), "__import__", "TernaryFunc"); - Runtime.PyObject_SetAttrString(mod, "__import__", hook.ptr); + Runtime.PyObject_SetAttrString(builtins, "__import__", hook.ptr); Runtime.XDecref(hook.ptr); + Runtime.XDecref(builtins); + } + + ///

+ /// Restore the __import__ hook. + ///

+ static void RestoreImport() + { + IntPtr builtins = GetNewRefToBuiltins(); + + Runtime.PyObject_SetAttrString(builtins, "__import__", py_import); + Runtime.XDecref(py_import); + py_import = IntPtr.Zero; + + Runtime.XDecref(builtins); + } + + ///

+ /// Initialization performed on startup of the Python runtime. + ///

+ internal static void Initialize() + { + InitImport(); + // Initialize the clr module and tell Python about it. root = new CLRModule(); #if PYTHON3 @@ -62,6 +101,7 @@ internal static void Initialize() Runtime.XIncref(root.pyHandle); // we are using the module two times py_clr_module = root.pyHandle; // Alias handle for PY2/PY3 #endif + IntPtr dict = Runtime.PyImport_GetModuleDict(); Runtime.PyDict_SetItemString(dict, "CLR", py_clr_module); Runtime.PyDict_SetItemString(dict, "clr", py_clr_module); } @@ -74,9 +114,10 @@ internal static void Shutdown() { if (Runtime.Py_IsInitialized() != 0) { + RestoreImport(); + Runtime.XDecref(py_clr_module); Runtime.XDecref(root.pyHandle); - Runtime.XDecref(py_import); } } From 627cac0cb91eea4bfa6a36b25e523500a792c88c Mon Sep 17 00:00:00 2001 From: Jeff17Robbins Date: Wed, 27 Nov 2019 20:57:19 -0500 Subject: [PATCH 029/112] Capture function pointer declarations in structs - Update geninterop.py to visit function pointers - Support running on Windows 10 - Update interop38.cs --- AUTHORS.md | 1 + src/runtime/interop38.cs | 4 ++-- tools/geninterop/geninterop.py | 14 +++++++++++++- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/AUTHORS.md b/AUTHORS.md index efa04c8f0..e42a456ae 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -34,6 +34,7 @@ - Ivan Cronyn ([@cronan](https://github.com/cronan)) - Jan Krivanek ([@jakrivan](https://github.com/jakrivan)) - Jeff Reback ([@jreback](https://github.com/jreback)) +- Jeff Robbins ([@jeff17robbins](https://github.com/jeff17robbins)) - Joe Frayne ([@jfrayne](https://github.com/jfrayne)) - Joe Lidbetter ([@jmlidbetter](https://github.com/jmlidbetter)) - Joe Savage ([@s4v4g3](https://github.com/s4v4g3)) diff --git a/src/runtime/interop38.cs b/src/runtime/interop38.cs index 8f2e32afe..9126bca6a 100644 --- a/src/runtime/interop38.cs +++ b/src/runtime/interop38.cs @@ -1,6 +1,6 @@ // Auto-generated by geninterop.py. -// DO NOT MODIFIY BY HAND. +// DO NOT MODIFY BY HAND. #if PYTHON38 @@ -84,6 +84,7 @@ public static int magic() public static int tp_version_tag = 0; public static int tp_finalize = 0; public static int tp_vectorcall = 0; + public static int tp_print = 0; public static int am_await = 0; public static int am_aiter = 0; public static int am_anext = 0; @@ -149,4 +150,3 @@ public static int magic() } #endif - diff --git a/tools/geninterop/geninterop.py b/tools/geninterop/geninterop.py index f8ef8e561..1f4751939 100644 --- a/tools/geninterop/geninterop.py +++ b/tools/geninterop/geninterop.py @@ -76,6 +76,8 @@ def visit(self, node): self.visit_struct(node) elif isinstance(node, c_ast.Decl): self.visit_decl(node) + elif isinstance(node, c_ast.FuncDecl): + self.visit_funcdecl(node) elif isinstance(node, c_ast.PtrDecl): self.visit_ptrdecl(node) elif isinstance(node, c_ast.IdentifierType): @@ -110,6 +112,9 @@ def visit_struct(self, struct): def visit_decl(self, decl): self.visit(decl.type) + def visit_funcdecl(self, funcdecl): + self.visit(funcdecl.type) + def visit_ptrdecl(self, ptrdecl): self.__ptr_decl_depth += 1 self.visit(ptrdecl.type) @@ -177,6 +182,13 @@ def preprocess_python_headers(): "-D", "_POSIX_THREADS" ] + if os.name == 'nt': + defines.extend([ + "-D", "__inline=inline", + "-D", "__ptr32=", + "-D", "__declspec(x)=", + ]) + if hasattr(sys, "abiflags"): if "d" in sys.abiflags: defines.extend(("-D", "PYTHON_WITH_PYDEBUG")) @@ -216,7 +228,7 @@ def gen_interop_code(members): defines_str = " && ".join(defines) class_definition = """ // Auto-generated by %s. -// DO NOT MODIFIY BY HAND. +// DO NOT MODIFY BY HAND. #if %s From abbe870db593ba63efc5e9999b2474169d87e141 Mon Sep 17 00:00:00 2001 From: amos402 Date: Mon, 2 Dec 2019 19:53:15 +0800 Subject: [PATCH 030/112] Internal implement Py_CompileString for compat with CPython 3.8.0 (#1000) --- src/runtime/pythonengine.cs | 6 +++--- src/runtime/runtime.cs | 30 +++++++++++++++++++++++++++++- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index 700543839..5073067d3 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -503,7 +503,7 @@ public static PyObject ReloadModule(PyObject module) /// public static PyObject ModuleFromString(string name, string code) { - IntPtr c = Runtime.Py_CompileString(code, "none", (IntPtr)257); + IntPtr c = Runtime.Py_CompileString(code, "none", (int)RunFlagType.File); Runtime.CheckExceptionOccurred(); IntPtr m = Runtime.PyImport_ExecCodeModule(name, c); Runtime.CheckExceptionOccurred(); @@ -512,7 +512,7 @@ public static PyObject ModuleFromString(string name, string code) public static PyObject Compile(string code, string filename = "", RunFlagType mode = RunFlagType.File) { - var flag = (IntPtr)mode; + var flag = (int)mode; IntPtr ptr = Runtime.Py_CompileString(code, filename, flag); Runtime.CheckExceptionOccurred(); return new PyObject(ptr); @@ -610,7 +610,7 @@ internal static PyObject RunString(string code, IntPtr? globals, IntPtr? locals, } } - public enum RunFlagType + public enum RunFlagType : int { Single = 256, File = 257, /* Py_file_input */ diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 7a78cd6e1..66f5ed123 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -763,8 +763,36 @@ public static extern int Py_Main( [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyEval_EvalCode(IntPtr co, IntPtr globals, IntPtr locals); +#if PYTHON2 + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr Py_CompileString(string code, string file, int start); +#else + ///

+ /// Return value: New reference. + /// This is a simplified interface to Py_CompileStringFlags() below, leaving flags set to NULL. + ///

+ internal static IntPtr Py_CompileString(string str, string file, int start) + { + return Py_CompileStringFlags(str, file, start, IntPtr.Zero); + } + + ///

+ /// Return value: New reference. + /// This is a simplified interface to Py_CompileStringExFlags() below, with optimize set to -1. + ///

+ internal static IntPtr Py_CompileStringFlags(string str, string file, int start, IntPtr flags) + { + return Py_CompileStringExFlags(str, file, start, flags, -1); + } + + ///

+ /// Return value: New reference. + /// Like Py_CompileStringObject(), but filename is a byte string decoded from the filesystem encoding(os.fsdecode()). + ///

+ /// [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr Py_CompileString(string code, string file, IntPtr tok); + internal static extern IntPtr Py_CompileStringExFlags(string str, string file, int start, IntPtr flags, int optimize); +#endif [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyImport_ExecCodeModule(string name, IntPtr code); From d1044c3a54f1a359785251836868d835dbf97c6b Mon Sep 17 00:00:00 2001 From: amos402 Date: Mon, 2 Dec 2019 20:23:47 +0800 Subject: [PATCH 031/112] Remove unnecessary `CopySlot` calls (#1004) --- src/runtime/typemanager.cs | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 9a98e9ebb..e9a8818f0 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -309,17 +309,11 @@ internal static IntPtr CreateMetaType(Type impl) Marshal.WriteIntPtr(type, TypeOffset.tp_base, py_type); Runtime.XIncref(py_type); - // Copy gc and other type slots from the base Python metatype. - - CopySlot(py_type, type, TypeOffset.tp_basicsize); - CopySlot(py_type, type, TypeOffset.tp_itemsize); - - CopySlot(py_type, type, TypeOffset.tp_dictoffset); - CopySlot(py_type, type, TypeOffset.tp_weaklistoffset); - - CopySlot(py_type, type, TypeOffset.tp_traverse); - CopySlot(py_type, type, TypeOffset.tp_clear); - CopySlot(py_type, type, TypeOffset.tp_is_gc); + // Slots will inherit from TypeType, it's not neccesary for setting them. + // Inheried slots: + // tp_basicsize, tp_itemsize, + // tp_dictoffset, tp_weaklistoffset, + // tp_traverse, tp_clear, tp_is_gc, etc. // Override type slots with those of the managed implementation. From 5f56ebc8f0785f6581c10d5beb9c1a8fffe6c650 Mon Sep 17 00:00:00 2001 From: amos402 Date: Mon, 2 Dec 2019 20:31:05 +0800 Subject: [PATCH 032/112] Fix refcnt errors (split from #958) (#1001) * Add exception helper * Fixed refcnt error in ExtensionType.FinalizeObject * Fixed typename leaking * Fix refcnt error by using `using` --- .editorconfig | 2 +- src/runtime/extensiontype.cs | 3 ++- src/runtime/metatype.cs | 1 + src/runtime/pythonexception.cs | 17 +++++++++++++++++ src/runtime/runtime.cs | 4 ++++ src/runtime/typemanager.cs | 8 ++------ 6 files changed, 27 insertions(+), 8 deletions(-) diff --git a/.editorconfig b/.editorconfig index 9e10931d0..d64f74bc1 100644 --- a/.editorconfig +++ b/.editorconfig @@ -25,7 +25,7 @@ dotnet_sort_system_directives_first = true dotnet_separate_import_directive_groups = true [*.cs] -csharp_new_line_before_open_brace = true +csharp_new_line_before_open_brace = all csharp_new_line_before_else = true csharp_new_line_before_catch = true csharp_new_line_before_finally = true diff --git a/src/runtime/extensiontype.cs b/src/runtime/extensiontype.cs index 693a46f42..6585180c1 100644 --- a/src/runtime/extensiontype.cs +++ b/src/runtime/extensiontype.cs @@ -38,6 +38,7 @@ public ExtensionType() Runtime.PyObject_GC_UnTrack(py); + // Steals a ref to tpHandle. tpHandle = tp; pyHandle = py; gcHandle = gc; @@ -50,7 +51,7 @@ public ExtensionType() public static void FinalizeObject(ManagedType self) { Runtime.PyObject_GC_Del(self.pyHandle); - Runtime.XDecref(self.tpHandle); + // Not necessary for decref of `tpHandle`. self.gcHandle.Free(); } diff --git a/src/runtime/metatype.cs b/src/runtime/metatype.cs index 8853c2d5e..5af2e1a7e 100644 --- a/src/runtime/metatype.cs +++ b/src/runtime/metatype.cs @@ -266,6 +266,7 @@ private static IntPtr DoInstanceCheck(IntPtr tp, IntPtr args, bool checkType) return Runtime.PyFalse; } + Runtime.XIncref(args); using (var argsObj = new PyList(args)) { if (argsObj.Length() != 1) diff --git a/src/runtime/pythonexception.cs b/src/runtime/pythonexception.cs index 295a63b3d..8a6a24799 100644 --- a/src/runtime/pythonexception.cs +++ b/src/runtime/pythonexception.cs @@ -1,4 +1,5 @@ using System; +using System.Runtime.CompilerServices; namespace Python.Runtime { @@ -190,5 +191,21 @@ public static bool Matches(IntPtr ob) { return Runtime.PyErr_ExceptionMatches(ob) != 0; } + + public static void ThrowIfIsNull(IntPtr ob) + { + if (ob == IntPtr.Zero) + { + throw new PythonException(); + } + } + + public static void ThrowIfIsNotZero(int value) + { + if (value != 0) + { + throw new PythonException(); + } + } } } diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 66f5ed123..449d22435 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -1368,6 +1368,10 @@ internal static IntPtr PyUnicode_FromStringAndSize(IntPtr value, long size) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] private static extern IntPtr PyUnicode_FromStringAndSize(IntPtr value, IntPtr size); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr PyUnicode_AsUTF8(IntPtr unicode); + #elif PYTHON2 internal static IntPtr PyString_FromStringAndSize(string value, long size) { diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index e9a8818f0..4427305e6 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -409,12 +409,8 @@ internal static IntPtr AllocateTypeObject(string name) // the Python version of the type name - otherwise we'd have to // allocate the tp_name and would have no way to free it. #if PYTHON3 - // For python3 we leak two objects. One for the ASCII representation - // required for tp_name, and another for the Unicode representation - // for ht_name. - IntPtr temp = Runtime.PyBytes_FromString(name); - IntPtr raw = Runtime.PyBytes_AS_STRING(temp); - temp = Runtime.PyUnicode_FromString(name); + IntPtr temp = Runtime.PyUnicode_FromString(name); + IntPtr raw = Runtime.PyUnicode_AsUTF8(temp); #elif PYTHON2 IntPtr temp = Runtime.PyString_FromString(name); IntPtr raw = Runtime.PyString_AsString(temp); From ba5127aaf687d8bb816423cd31a4bc7698b1ebc7 Mon Sep 17 00:00:00 2001 From: Alex Earl Date: Mon, 16 Dec 2019 14:33:00 -0700 Subject: [PATCH 033/112] Add mp_length slot for .NET classes implementing ICollection/ICollection (#994) - Add mp_length slot implementation for .NET types - Check if the object implement ICollection or ICollection - Add tests for explicit and non-explicit interface implementation --- AUTHORS.md | 1 + CHANGELOG.md | 1 + src/runtime/Python.Runtime.csproj | 1 + src/runtime/arrayobject.cs | 11 -- src/runtime/slots/mp_length.cs | 50 +++++++++ src/runtime/typemanager.cs | 15 +++ src/testing/Python.Test.csproj | 1 + src/testing/mp_lengthtest.cs | 171 ++++++++++++++++++++++++++++++ src/tests/test_mp_length.py | 49 +++++++++ 9 files changed, 289 insertions(+), 11 deletions(-) create mode 100644 src/runtime/slots/mp_length.cs create mode 100644 src/testing/mp_lengthtest.cs create mode 100644 src/tests/test_mp_length.py diff --git a/AUTHORS.md b/AUTHORS.md index e42a456ae..26285bf6a 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -12,6 +12,7 @@ ## Contributors +- Alex Earl ([@slide](https://github.com/slide)) - Alex Helms ([@alexhelms](https://github.com/alexhelms)) - Alexandre Catarino([@AlexCatarino](https://github.com/AlexCatarino)) - Arvid JB ([@ArvidJB](https://github.com/ArvidJB)) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c999d668..1fd2b1dcf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. - Added automatic NuGet package generation in appveyor and local builds - Added function that sets Py_NoSiteFlag to 1. - Added support for Jetson Nano. +- Added support for __len__ for .NET classes that implement ICollection ### Changed diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index 02656e51e..0c2f912de 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -142,6 +142,7 @@ + diff --git a/src/runtime/arrayobject.cs b/src/runtime/arrayobject.cs index c37295704..1ef318473 100644 --- a/src/runtime/arrayobject.cs +++ b/src/runtime/arrayobject.cs @@ -244,16 +244,5 @@ public static int sq_contains(IntPtr ob, IntPtr v) return 0; } - - - ///

- /// Implements __len__ for array types. - ///

- public static int mp_length(IntPtr ob) - { - var self = (CLRObject)GetManagedObject(ob); - var items = self.inst as Array; - return items.Length; - } } } diff --git a/src/runtime/slots/mp_length.cs b/src/runtime/slots/mp_length.cs new file mode 100644 index 000000000..b0a2e8d79 --- /dev/null +++ b/src/runtime/slots/mp_length.cs @@ -0,0 +1,50 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; + +namespace Python.Runtime.Slots +{ + internal static class mp_length_slot + { + ///

+ /// Implements __len__ for classes that implement ICollection + /// (this includes any IList implementer or Array subclass) + ///

+ public static int mp_length(IntPtr ob) + { + var co = ManagedType.GetManagedObject(ob) as CLRObject; + if (co == null) + { + Exceptions.RaiseTypeError("invalid object"); + } + + // first look for ICollection implementation directly + if (co.inst is ICollection c) + { + return c.Count; + } + + Type clrType = co.inst.GetType(); + + // now look for things that implement ICollection directly (non-explicitly) + PropertyInfo p = clrType.GetProperty("Count"); + if (p != null && clrType.GetInterfaces().Any(x => x.IsGenericType && x.GetGenericTypeDefinition() == typeof(ICollection<>))) + { + return (int)p.GetValue(co.inst, null); + } + + // finally look for things that implement the interface explicitly + var iface = clrType.GetInterfaces().FirstOrDefault(x => x.IsGenericType && x.GetGenericTypeDefinition() == typeof(ICollection<>)); + if (iface != null) + { + p = iface.GetProperty(nameof(ICollection.Count)); + return (int)p.GetValue(co.inst, null); + } + + Exceptions.SetError(Exceptions.TypeError, $"object of type '{clrType.Name}' has no len()"); + return -1; + } + } +} diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 4427305e6..97e6032cd 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -1,9 +1,11 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Linq; using System.Reflection; using System.Runtime.InteropServices; using Python.Runtime.Platform; +using Python.Runtime.Slots; namespace Python.Runtime { @@ -153,6 +155,13 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) Marshal.WriteIntPtr(type, TypeOffset.tp_itemsize, IntPtr.Zero); Marshal.WriteIntPtr(type, TypeOffset.tp_dictoffset, (IntPtr)tp_dictoffset); + // add a __len__ slot for inheritors of ICollection and ICollection<> + if (typeof(ICollection).IsAssignableFrom(clrType) || clrType.GetInterfaces().Any(x => x.IsGenericType && x.GetGenericTypeDefinition() == typeof(ICollection<>))) + { + InitializeSlot(type, TypeOffset.mp_length, typeof(mp_length_slot).GetMethod(nameof(mp_length_slot.mp_length))); + } + + // we want to do this after the slot stuff above in case the class itself implements a slot method InitializeSlots(type, impl.GetType()); if (base_ != IntPtr.Zero) @@ -193,6 +202,12 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) return type; } + static void InitializeSlot(IntPtr type, int slotOffset, MethodInfo method) + { + IntPtr thunk = Interop.GetThunk(method); + Marshal.WriteIntPtr(type, slotOffset, thunk); + } + internal static IntPtr CreateSubType(IntPtr py_name, IntPtr py_base_type, IntPtr py_dict) { // Utility to create a subtype of a managed type with the ability for the diff --git a/src/testing/Python.Test.csproj b/src/testing/Python.Test.csproj index 6bf5c2d22..515fd928c 100644 --- a/src/testing/Python.Test.csproj +++ b/src/testing/Python.Test.csproj @@ -92,6 +92,7 @@ + diff --git a/src/testing/mp_lengthtest.cs b/src/testing/mp_lengthtest.cs new file mode 100644 index 000000000..a4f3e8c25 --- /dev/null +++ b/src/testing/mp_lengthtest.cs @@ -0,0 +1,171 @@ +using System; +using System.Collections; +using System.Collections.Generic; + +namespace Python.Test +{ + public class MpLengthCollectionTest : ICollection + { + private readonly List items; + + public MpLengthCollectionTest() + { + SyncRoot = new object(); + items = new List + { + 1, + 2, + 3 + }; + } + + public int Count => items.Count; + + public object SyncRoot { get; private set; } + + public bool IsSynchronized => false; + + public void CopyTo(Array array, int index) + { + throw new NotImplementedException(); + } + + public IEnumerator GetEnumerator() + { + throw new NotImplementedException(); + } + } + + public class MpLengthExplicitCollectionTest : ICollection + { + private readonly List items; + private readonly object syncRoot; + + public MpLengthExplicitCollectionTest() + { + syncRoot = new object(); + items = new List + { + 9, + 10 + }; + } + int ICollection.Count => items.Count; + + object ICollection.SyncRoot => syncRoot; + + bool ICollection.IsSynchronized => false; + + void ICollection.CopyTo(Array array, int index) + { + throw new NotImplementedException(); + } + + IEnumerator IEnumerable.GetEnumerator() + { + throw new NotImplementedException(); + } + } + + public class MpLengthGenericCollectionTest : ICollection + { + private readonly List items; + + public MpLengthGenericCollectionTest() { + SyncRoot = new object(); + items = new List(); + } + + public int Count => items.Count; + + public object SyncRoot { get; private set; } + + public bool IsSynchronized => false; + + public bool IsReadOnly => false; + + public void Add(T item) + { + items.Add(item); + } + + public void Clear() + { + items.Clear(); + } + + public bool Contains(T item) + { + return items.Contains(item); + } + + public void CopyTo(T[] array, int arrayIndex) + { + items.CopyTo(array, arrayIndex); + } + + public IEnumerator GetEnumerator() + { + return ((IEnumerable)items).GetEnumerator(); + } + + public bool Remove(T item) + { + return items.Remove(item); + } + + IEnumerator IEnumerable.GetEnumerator() + { + return items.GetEnumerator(); + } + } + + public class MpLengthExplicitGenericCollectionTest : ICollection + { + private readonly List items; + + public MpLengthExplicitGenericCollectionTest() + { + items = new List(); + } + + int ICollection.Count => items.Count; + + bool ICollection.IsReadOnly => false; + + public void Add(T item) + { + items.Add(item); + } + + void ICollection.Clear() + { + items.Clear(); + } + + bool ICollection.Contains(T item) + { + return items.Contains(item); + } + + void ICollection.CopyTo(T[] array, int arrayIndex) + { + items.CopyTo(array, arrayIndex); + } + + IEnumerator IEnumerable.GetEnumerator() + { + return items.GetEnumerator(); + } + + IEnumerator IEnumerable.GetEnumerator() + { + return ((IEnumerable)items).GetEnumerator(); + } + + bool ICollection.Remove(T item) + { + return items.Remove(item); + } + } +} diff --git a/src/tests/test_mp_length.py b/src/tests/test_mp_length.py new file mode 100644 index 000000000..c96ac77d1 --- /dev/null +++ b/src/tests/test_mp_length.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- + +"""Test __len__ for .NET classes implementing ICollection/ICollection.""" + +import System +import pytest +from Python.Test import MpLengthCollectionTest, MpLengthExplicitCollectionTest, MpLengthGenericCollectionTest, MpLengthExplicitGenericCollectionTest + +def test_simple___len__(): + """Test __len__ for simple ICollection implementers""" + import System + import System.Collections.Generic + l = System.Collections.Generic.List[int]() + assert len(l) == 0 + l.Add(5) + l.Add(6) + assert len(l) == 2 + + d = System.Collections.Generic.Dictionary[int, int]() + assert len(d) == 0 + d.Add(4, 5) + assert len(d) == 1 + + a = System.Array[int]([0,1,2,3]) + assert len(a) == 4 + +def test_custom_collection___len__(): + """Test __len__ for custom collection class""" + s = MpLengthCollectionTest() + assert len(s) == 3 + +def test_custom_collection_explicit___len__(): + """Test __len__ for custom collection class that explicitly implements ICollection""" + s = MpLengthExplicitCollectionTest() + assert len(s) == 2 + +def test_custom_generic_collection___len__(): + """Test __len__ for custom generic collection class""" + s = MpLengthGenericCollectionTest[int]() + s.Add(1) + s.Add(2) + assert len(s) == 2 + +def test_custom_generic_collection_explicit___len__(): + """Test __len__ for custom generic collection that explicity implements ICollection""" + s = MpLengthExplicitGenericCollectionTest[int]() + s.Add(1) + s.Add(10) + assert len(s) == 2 From c1190f4397f365667d8246503ac936d3945bdc2b Mon Sep 17 00:00:00 2001 From: amos402 Date: Wed, 18 Dec 2019 16:03:28 +0800 Subject: [PATCH 034/112] Release method wrapper(split from #958) (#1002) * Add exception helper * Release memory on ImportHook.Shutdown * Release ModuleDef when py_clr_module released * Completely ModuleDef initialization --- src/runtime/importhook.cs | 44 ++++++++++++++++++++++++++++++------ src/runtime/interop.cs | 2 +- src/runtime/methodwrapper.cs | 17 ++++++++++++++ src/runtime/moduleobject.cs | 5 ++++ src/runtime/runtime.cs | 1 - 5 files changed, 60 insertions(+), 9 deletions(-) diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index 06ba7a56d..94f0f7c94 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -23,6 +23,16 @@ internal static void InitializeModuleDef() module_def = ModuleDefOffset.AllocModuleDef("clr"); } } + + internal static void ReleaseModuleDef() + { + if (module_def == IntPtr.Zero) + { + return; + } + ModuleDefOffset.FreeModuleDef(module_def); + module_def = IntPtr.Zero; + } #endif ///

@@ -56,9 +66,12 @@ static void InitImport() // Python __import__. IntPtr builtins = GetNewRefToBuiltins(); py_import = Runtime.PyObject_GetAttrString(builtins, "__import__"); + PythonException.ThrowIfIsNull(py_import); + hook = new MethodWrapper(typeof(ImportHook), "__import__", "TernaryFunc"); - Runtime.PyObject_SetAttrString(builtins, "__import__", hook.ptr); - Runtime.XDecref(hook.ptr); + int res = Runtime.PyObject_SetAttrString(builtins, "__import__", hook.ptr); + PythonException.ThrowIfIsNotZero(res); + Runtime.XDecref(builtins); } @@ -69,10 +82,14 @@ static void RestoreImport() { IntPtr builtins = GetNewRefToBuiltins(); - Runtime.PyObject_SetAttrString(builtins, "__import__", py_import); + int res = Runtime.PyObject_SetAttrString(builtins, "__import__", py_import); + PythonException.ThrowIfIsNotZero(res); Runtime.XDecref(py_import); py_import = IntPtr.Zero; + hook.Release(); + hook = null; + Runtime.XDecref(builtins); } @@ -112,13 +129,26 @@ internal static void Initialize() ///

internal static void Shutdown() { - if (Runtime.Py_IsInitialized() != 0) + if (Runtime.Py_IsInitialized() == 0) { - RestoreImport(); + return; + } - Runtime.XDecref(py_clr_module); - Runtime.XDecref(root.pyHandle); + RestoreImport(); + + bool shouldFreeDef = Runtime.Refcount(py_clr_module) == 1; + Runtime.XDecref(py_clr_module); + py_clr_module = IntPtr.Zero; +#if PYTHON3 + if (shouldFreeDef) + { + ReleaseModuleDef(); } +#endif + + Runtime.XDecref(root.pyHandle); + root = null; + CLRModule.Reset(); } ///

diff --git a/src/runtime/interop.cs b/src/runtime/interop.cs index 4ae4b61e0..2e29601fd 100644 --- a/src/runtime/interop.cs +++ b/src/runtime/interop.cs @@ -227,7 +227,7 @@ public static IntPtr AllocModuleDef(string modulename) byte[] ascii = Encoding.ASCII.GetBytes(modulename); int size = name + ascii.Length + 1; IntPtr ptr = Marshal.AllocHGlobal(size); - for (int i = 0; i < m_free; i += IntPtr.Size) + for (int i = 0; i <= m_free; i += IntPtr.Size) Marshal.WriteIntPtr(ptr, i, IntPtr.Zero); Marshal.Copy(ascii, 0, (IntPtr)(ptr + name), ascii.Length); Marshal.WriteIntPtr(ptr, m_name, (IntPtr)(ptr + name)); diff --git a/src/runtime/methodwrapper.cs b/src/runtime/methodwrapper.cs index 2f3ce3ef2..ba92e99d4 100644 --- a/src/runtime/methodwrapper.cs +++ b/src/runtime/methodwrapper.cs @@ -12,6 +12,7 @@ internal class MethodWrapper { public IntPtr mdef; public IntPtr ptr; + private bool _disposed = false; public MethodWrapper(Type type, string name, string funcType = null) { @@ -31,5 +32,21 @@ public IntPtr Call(IntPtr args, IntPtr kw) { return Runtime.PyCFunction_Call(ptr, args, kw); } + + public void Release() + { + if (_disposed) + { + return; + } + _disposed = true; + bool freeDef = Runtime.Refcount(ptr) == 1; + Runtime.XDecref(ptr); + if (freeDef && mdef != IntPtr.Zero) + { + Runtime.PyMem_Free(mdef); + mdef = IntPtr.Zero; + } + } } } diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index 7a45c6c81..544f69c81 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -316,6 +316,11 @@ internal class CLRModule : ModuleObject internal static bool _SuppressDocs = false; internal static bool _SuppressOverloads = false; + static CLRModule() + { + Reset(); + } + public CLRModule() : base("clr") { _namespace = string.Empty; diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 449d22435..130d90c0a 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -187,7 +187,6 @@ internal static void Initialize(bool initSigs = false) IsFinalizing = false; - CLRModule.Reset(); GenericUtil.Reset(); PyScopeManager.Reset(); ClassManager.Reset(); From e896aa6714b9d736bea58fae7c9cbf0fd01337a2 Mon Sep 17 00:00:00 2001 From: amos402 Date: Thu, 19 Dec 2019 02:49:46 +0800 Subject: [PATCH 035/112] * Decref the members of Runtime * Unified GetBuiltins method --- src/runtime/importhook.cs | 25 +----- src/runtime/runtime.cs | 170 ++++++++++++++++++++++++++------------ 2 files changed, 120 insertions(+), 75 deletions(-) diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index 94f0f7c94..aa3bbab6d 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -35,27 +35,6 @@ internal static void ReleaseModuleDef() } #endif - ///

- /// Get a New reference to the builtins module. - ///

- static IntPtr GetNewRefToBuiltins() - { - if (Runtime.IsPython3) - { - return Runtime.PyImport_ImportModule("builtins"); - } - else - { - // dict is a borrowed ref, no need to decref - IntPtr dict = Runtime.PyImport_GetModuleDict(); - - // GetItemString is a borrowed ref; incref to get a new ref - IntPtr builtins = Runtime.PyDict_GetItemString(dict, "__builtin__"); - Runtime.XIncref(builtins); - return builtins; - } - } - ///

/// Initialize just the __import__ hook itself. ///

@@ -64,7 +43,7 @@ static void InitImport() // We replace the built-in Python __import__ with our own: first // look in CLR modules, then if we don't find any call the default // Python __import__. - IntPtr builtins = GetNewRefToBuiltins(); + IntPtr builtins = Runtime.GetBuiltins(); py_import = Runtime.PyObject_GetAttrString(builtins, "__import__"); PythonException.ThrowIfIsNull(py_import); @@ -80,7 +59,7 @@ static void InitImport() ///

static void RestoreImport() { - IntPtr builtins = GetNewRefToBuiltins(); + IntPtr builtins = Runtime.GetBuiltins(); int res = Runtime.PyObject_SetAttrString(builtins, "__import__", py_import); PythonException.ThrowIfIsNotZero(res); diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 130d90c0a..2443e3edc 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -169,6 +169,8 @@ public class Runtime ///

internal static readonly Encoding PyEncoding = _UCS == 2 ? Encoding.Unicode : Encoding.UTF32; + private static PyReferenceCollection _pyRefs = new PyReferenceCollection(); + ///

/// Initialize the runtime... ///

@@ -194,99 +196,94 @@ internal static void Initialize(bool initSigs = false) TypeManager.Reset(); IntPtr op; - IntPtr dict; - if (IsPython3) - { - op = PyImport_ImportModule("builtins"); - dict = PyObject_GetAttrString(op, "__dict__"); - } - else // Python2 { - dict = PyImport_GetModuleDict(); - op = PyDict_GetItemString(dict, "__builtin__"); - } - PyNotImplemented = PyObject_GetAttrString(op, "NotImplemented"); - PyBaseObjectType = PyObject_GetAttrString(op, "object"); + var builtins = GetBuiltins(); + SetPyMember(ref PyNotImplemented, PyObject_GetAttrString(builtins, "NotImplemented")); - PyNone = PyObject_GetAttrString(op, "None"); - PyTrue = PyObject_GetAttrString(op, "True"); - PyFalse = PyObject_GetAttrString(op, "False"); + SetPyMember(ref PyBaseObjectType, PyObject_GetAttrString(builtins, "object")); - PyBoolType = PyObject_Type(PyTrue); - PyNoneType = PyObject_Type(PyNone); - PyTypeType = PyObject_Type(PyNoneType); + SetPyMember(ref PyNone, PyObject_GetAttrString(builtins, "None")); + SetPyMember(ref PyTrue, PyObject_GetAttrString(builtins, "True")); + SetPyMember(ref PyFalse, PyObject_GetAttrString(builtins, "False")); - op = PyObject_GetAttrString(dict, "keys"); - PyMethodType = PyObject_Type(op); - XDecref(op); + SetPyMember(ref PyBoolType, PyObject_Type(PyTrue)); + SetPyMember(ref PyNoneType, PyObject_Type(PyNone)); + SetPyMember(ref PyTypeType, PyObject_Type(PyNoneType)); - // For some arcane reason, builtins.__dict__.__setitem__ is *not* - // a wrapper_descriptor, even though dict.__setitem__ is. - // - // object.__init__ seems safe, though. - op = PyObject_GetAttrString(PyBaseObjectType, "__init__"); - PyWrapperDescriptorType = PyObject_Type(op); - XDecref(op); + op = PyObject_GetAttrString(builtins, "len"); + SetPyMember(ref PyMethodType, PyObject_Type(op)); + XDecref(op); -#if PYTHON3 - XDecref(dict); -#endif + // For some arcane reason, builtins.__dict__.__setitem__ is *not* + // a wrapper_descriptor, even though dict.__setitem__ is. + // + // object.__init__ seems safe, though. + op = PyObject_GetAttrString(PyBaseObjectType, "__init__"); + SetPyMember(ref PyWrapperDescriptorType, PyObject_Type(op)); + XDecref(op); + + SetPyMember(ref PySuper_Type, PyObject_GetAttrString(builtins, "super")); + + XDecref(builtins); + } op = PyString_FromString("string"); - PyStringType = PyObject_Type(op); + SetPyMember(ref PyStringType, PyObject_Type(op)); XDecref(op); op = PyUnicode_FromString("unicode"); - PyUnicodeType = PyObject_Type(op); + SetPyMember(ref PyUnicodeType, PyObject_Type(op)); XDecref(op); #if PYTHON3 op = PyBytes_FromString("bytes"); - PyBytesType = PyObject_Type(op); + SetPyMember(ref PyBytesType, PyObject_Type(op)); XDecref(op); #endif op = PyTuple_New(0); - PyTupleType = PyObject_Type(op); + SetPyMember(ref PyTupleType, PyObject_Type(op)); XDecref(op); op = PyList_New(0); - PyListType = PyObject_Type(op); + SetPyMember(ref PyListType, PyObject_Type(op)); XDecref(op); op = PyDict_New(); - PyDictType = PyObject_Type(op); + SetPyMember(ref PyDictType, PyObject_Type(op)); XDecref(op); op = PyInt_FromInt32(0); - PyIntType = PyObject_Type(op); + SetPyMember(ref PyIntType, PyObject_Type(op)); XDecref(op); op = PyLong_FromLong(0); - PyLongType = PyObject_Type(op); + SetPyMember(ref PyLongType, PyObject_Type(op)); XDecref(op); op = PyFloat_FromDouble(0); - PyFloatType = PyObject_Type(op); + SetPyMember(ref PyFloatType, PyObject_Type(op)); XDecref(op); -#if PYTHON3 +#if !PYTHON2 PyClassType = IntPtr.Zero; PyInstanceType = IntPtr.Zero; -#elif PYTHON2 - IntPtr s = PyString_FromString("_temp"); - IntPtr d = PyDict_New(); +#else + { + IntPtr s = PyString_FromString("_temp"); + IntPtr d = PyDict_New(); - IntPtr c = PyClass_New(IntPtr.Zero, d, s); - PyClassType = PyObject_Type(c); + IntPtr c = PyClass_New(IntPtr.Zero, d, s); + SetPyMember(ref PyClassType, PyObject_Type(c)); - IntPtr i = PyInstance_New(c, IntPtr.Zero, IntPtr.Zero); - PyInstanceType = PyObject_Type(i); + IntPtr i = PyInstance_New(c, IntPtr.Zero, IntPtr.Zero); + SetPyMember(ref PyInstanceType, PyObject_Type(i)); - XDecref(s); - XDecref(i); - XDecref(c); - XDecref(d); + XDecref(s); + XDecref(i); + XDecref(c); + XDecref(d); + } #endif Error = new IntPtr(-1); @@ -380,6 +377,9 @@ internal static void Shutdown() Exceptions.Shutdown(); ImportHook.Shutdown(); Finalizer.Shutdown(); + // TOOD: PyCLRMetaType's release operation still in #958 + PyCLRMetaType = IntPtr.Zero; + ResetPyMembers(); Py_Finalize(); } @@ -393,6 +393,19 @@ internal static int AtExit() return 0; } + private static void SetPyMember(ref IntPtr obj, IntPtr value) + { + // XXX: For current usages, value should not be null. + PythonException.ThrowIfIsNull(value); + obj = value; + _pyRefs.Add(ref obj); + } + + private static void ResetPyMembers() + { + _pyRefs.Release(); + } + internal static IntPtr Py_single_input = (IntPtr)256; internal static IntPtr Py_file_input = (IntPtr)257; internal static IntPtr Py_eval_input = (IntPtr)258; @@ -401,6 +414,7 @@ internal static int AtExit() internal static IntPtr PyModuleType; internal static IntPtr PyClassType; internal static IntPtr PyInstanceType; + internal static IntPtr PySuper_Type; internal static IntPtr PyCLRMetaType; internal static IntPtr PyMethodType; internal static IntPtr PyWrapperDescriptorType; @@ -1746,6 +1760,9 @@ internal static bool PyIter_Check(IntPtr pointer) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyImport_Import(IntPtr name); + ///

+ /// Return value: New reference. + ///

[DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyImport_ImportModule(string name); @@ -1945,5 +1962,54 @@ internal static void SetNoSiteFlag() } } } + + ///

+ /// Return value: New reference. + ///

+ internal static IntPtr GetBuiltins() + { + return IsPython3 ? PyImport_ImportModule("builtins") + : PyImport_ImportModule("__builtin__"); + } + } + + + class PyReferenceCollection + { + public List _objects { get; private set; } + + public PyReferenceCollection() + { + _objects = new List(); + } + + ///

+ /// Record obj's address to release the obj in the future, + /// obj must alive before calling Release. + ///

+ public void Add(ref IntPtr obj) + { + unsafe + { + fixed (void* p = &obj) + { + _objects.Add((IntPtr)p); + } + } + } + + public void Release() + { + foreach (var objRef in _objects) + { + unsafe + { + var p = (void**)objRef; + Runtime.XDecref((IntPtr)(*p)); + *p = null; + } + } + _objects.Clear(); + } } } From f0e9c380687663cbc9fa5dbd52f0e461416cce0f Mon Sep 17 00:00:00 2001 From: amos402 Date: Thu, 19 Dec 2019 07:00:18 +0800 Subject: [PATCH 036/112] Transfer the ownership of method's thunk to caller(split from #958) (#1003) * Add exception helper * Make the caller of `Interop.GetThunk` handle thunk's lifecycle(unfinished) * Use Marshal.GetFunctionPointerForDelegate instead of Marshal + Thunk --- src/runtime/interop.cs | 43 ++++++++++++++++++++++++------------ src/runtime/methodwrapper.cs | 6 +++-- src/runtime/typemanager.cs | 17 ++++++++------ 3 files changed, 43 insertions(+), 23 deletions(-) diff --git a/src/runtime/interop.cs b/src/runtime/interop.cs index 2e29601fd..ca3c35bfd 100644 --- a/src/runtime/interop.cs +++ b/src/runtime/interop.cs @@ -4,6 +4,7 @@ using System.Runtime.InteropServices; using System.Reflection; using System.Text; +using System.Collections.Generic; namespace Python.Runtime { @@ -334,7 +335,7 @@ internal class TypeFlags internal class Interop { - private static ArrayList keepAlive; + private static List keepAlive; private static Hashtable pmap; static Interop() @@ -351,8 +352,7 @@ static Interop() p[item.Name] = item; } - keepAlive = new ArrayList(); - Marshal.AllocHGlobal(IntPtr.Size); + keepAlive = new List(); pmap = new Hashtable(); pmap["tp_dealloc"] = p["DestructorFunc"]; @@ -449,7 +449,7 @@ internal static Type GetPrototype(string name) return pmap[name] as Type; } - internal static IntPtr GetThunk(MethodInfo method, string funcType = null) + internal static ThunkInfo GetThunk(MethodInfo method, string funcType = null) { Type dt; if (funcType != null) @@ -457,18 +457,15 @@ internal static IntPtr GetThunk(MethodInfo method, string funcType = null) else dt = GetPrototype(method.Name); - if (dt != null) + if (dt == null) { - IntPtr tmp = Marshal.AllocHGlobal(IntPtr.Size); - Delegate d = Delegate.CreateDelegate(dt, method); - Thunk cb = new Thunk(d); - Marshal.StructureToPtr(cb, tmp, false); - IntPtr fp = Marshal.ReadIntPtr(tmp, 0); - Marshal.FreeHGlobal(tmp); - keepAlive.Add(d); - return fp; + return ThunkInfo.Empty; } - return IntPtr.Zero; + Delegate d = Delegate.CreateDelegate(dt, method); + var info = new ThunkInfo(d); + // TODO: remove keepAlive when #958 merged, let the lifecycle of ThunkInfo transfer to caller. + keepAlive.Add(info); + return info; } [UnmanagedFunctionPointer(CallingConvention.Cdecl)] @@ -522,4 +519,22 @@ public Thunk(Delegate d) fn = d; } } + + internal class ThunkInfo + { + public readonly Delegate Target; + public readonly IntPtr Address; + + public static readonly ThunkInfo Empty = new ThunkInfo(null); + + public ThunkInfo(Delegate target) + { + if (target == null) + { + return; + } + Target = target; + Address = Marshal.GetFunctionPointerForDelegate(target); + } + } } diff --git a/src/runtime/methodwrapper.cs b/src/runtime/methodwrapper.cs index ba92e99d4..bc7500dab 100644 --- a/src/runtime/methodwrapper.cs +++ b/src/runtime/methodwrapper.cs @@ -14,17 +14,19 @@ internal class MethodWrapper public IntPtr ptr; private bool _disposed = false; + private ThunkInfo _thunk; + public MethodWrapper(Type type, string name, string funcType = null) { // Turn the managed method into a function pointer - IntPtr fp = Interop.GetThunk(type.GetMethod(name), funcType); + _thunk = Interop.GetThunk(type.GetMethod(name), funcType); // Allocate and initialize a PyMethodDef structure to represent // the managed method, then create a PyCFunction. mdef = Runtime.PyMem_Malloc(4 * IntPtr.Size); - TypeManager.WriteMethodDef(mdef, name, fp, 0x0003); + TypeManager.WriteMethodDef(mdef, name, _thunk.Address, 0x0003); ptr = Runtime.PyCFunction_NewEx(mdef, IntPtr.Zero, IntPtr.Zero); } diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 97e6032cd..bb920b74f 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -204,8 +204,8 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) static void InitializeSlot(IntPtr type, int slotOffset, MethodInfo method) { - IntPtr thunk = Interop.GetThunk(method); - Marshal.WriteIntPtr(type, slotOffset, thunk); + var thunk = Interop.GetThunk(method); + Marshal.WriteIntPtr(type, slotOffset, thunk.Address); } internal static IntPtr CreateSubType(IntPtr py_name, IntPtr py_base_type, IntPtr py_dict) @@ -344,16 +344,18 @@ internal static IntPtr CreateMetaType(Type impl) // 4 int-ptrs in size. IntPtr mdef = Runtime.PyMem_Malloc(3 * 4 * IntPtr.Size); IntPtr mdefStart = mdef; + ThunkInfo thunkInfo = Interop.GetThunk(typeof(MetaType).GetMethod("__instancecheck__"), "BinaryFunc"); mdef = WriteMethodDef( mdef, "__instancecheck__", - Interop.GetThunk(typeof(MetaType).GetMethod("__instancecheck__"), "BinaryFunc") + thunkInfo.Address ); + thunkInfo = Interop.GetThunk(typeof(MetaType).GetMethod("__subclasscheck__"), "BinaryFunc"); mdef = WriteMethodDef( mdef, "__subclasscheck__", - Interop.GetThunk(typeof(MetaType).GetMethod("__subclasscheck__"), "BinaryFunc") + thunkInfo.Address ); // FIXME: mdef is not used @@ -710,7 +712,8 @@ internal static void InitializeSlots(IntPtr type, Type impl) continue; } - InitializeSlot(type, Interop.GetThunk(method), name); + var thunkInfo = Interop.GetThunk(method); + InitializeSlot(type, thunkInfo.Address, name); seen.Add(name); } @@ -728,8 +731,8 @@ internal static void InitializeSlots(IntPtr type, Type impl) // These have to be defined, though, so by default we fill these with // static C# functions from this class. - var ret0 = Interop.GetThunk(((Func)Return0).Method); - var ret1 = Interop.GetThunk(((Func)Return1).Method); + var ret0 = Interop.GetThunk(((Func)Return0).Method).Address; + var ret1 = Interop.GetThunk(((Func)Return1).Method).Address; if (native != null) { From ab0cb02fb8306f15beb9b734a8fbf56b2708ad26 Mon Sep 17 00:00:00 2001 From: amos402 Date: Sat, 21 Dec 2019 11:56:30 +0800 Subject: [PATCH 037/112] Add explicit release action --- src/runtime/runtime.cs | 121 ++++++++++++++++++++++------------------- 1 file changed, 64 insertions(+), 57 deletions(-) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 2443e3edc..7748bafa9 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -198,20 +198,29 @@ internal static void Initialize(bool initSigs = false) IntPtr op; { var builtins = GetBuiltins(); - SetPyMember(ref PyNotImplemented, PyObject_GetAttrString(builtins, "NotImplemented")); - - SetPyMember(ref PyBaseObjectType, PyObject_GetAttrString(builtins, "object")); - - SetPyMember(ref PyNone, PyObject_GetAttrString(builtins, "None")); - SetPyMember(ref PyTrue, PyObject_GetAttrString(builtins, "True")); - SetPyMember(ref PyFalse, PyObject_GetAttrString(builtins, "False")); - - SetPyMember(ref PyBoolType, PyObject_Type(PyTrue)); - SetPyMember(ref PyNoneType, PyObject_Type(PyNone)); - SetPyMember(ref PyTypeType, PyObject_Type(PyNoneType)); + SetPyMember(ref PyNotImplemented, PyObject_GetAttrString(builtins, "NotImplemented"), + () => PyNotImplemented = IntPtr.Zero); + + SetPyMember(ref PyBaseObjectType, PyObject_GetAttrString(builtins, "object"), + () => PyBaseObjectType = IntPtr.Zero); + + SetPyMember(ref PyNone, PyObject_GetAttrString(builtins, "None"), + () => PyNone = IntPtr.Zero); + SetPyMember(ref PyTrue, PyObject_GetAttrString(builtins, "True"), + () => PyTrue = IntPtr.Zero); + SetPyMember(ref PyFalse, PyObject_GetAttrString(builtins, "False"), + () => PyFalse = IntPtr.Zero); + + SetPyMember(ref PyBoolType, PyObject_Type(PyTrue), + () => PyBoolType = IntPtr.Zero); + SetPyMember(ref PyNoneType, PyObject_Type(PyNone), + () => PyNoneType = IntPtr.Zero); + SetPyMember(ref PyTypeType, PyObject_Type(PyNoneType), + () => PyTypeType = IntPtr.Zero); op = PyObject_GetAttrString(builtins, "len"); - SetPyMember(ref PyMethodType, PyObject_Type(op)); + SetPyMember(ref PyMethodType, PyObject_Type(op), + () => PyMethodType = IntPtr.Zero); XDecref(op); // For some arcane reason, builtins.__dict__.__setitem__ is *not* @@ -219,50 +228,61 @@ internal static void Initialize(bool initSigs = false) // // object.__init__ seems safe, though. op = PyObject_GetAttrString(PyBaseObjectType, "__init__"); - SetPyMember(ref PyWrapperDescriptorType, PyObject_Type(op)); + SetPyMember(ref PyWrapperDescriptorType, PyObject_Type(op), + () => PyWrapperDescriptorType = IntPtr.Zero); XDecref(op); - SetPyMember(ref PySuper_Type, PyObject_GetAttrString(builtins, "super")); + SetPyMember(ref PySuper_Type, PyObject_GetAttrString(builtins, "super"), + () => PySuper_Type = IntPtr.Zero); XDecref(builtins); } op = PyString_FromString("string"); - SetPyMember(ref PyStringType, PyObject_Type(op)); + SetPyMember(ref PyStringType, PyObject_Type(op), + () => PyStringType = IntPtr.Zero); XDecref(op); op = PyUnicode_FromString("unicode"); - SetPyMember(ref PyUnicodeType, PyObject_Type(op)); + SetPyMember(ref PyUnicodeType, PyObject_Type(op), + () => PyUnicodeType = IntPtr.Zero); XDecref(op); #if PYTHON3 op = PyBytes_FromString("bytes"); - SetPyMember(ref PyBytesType, PyObject_Type(op)); + SetPyMember(ref PyBytesType, PyObject_Type(op), + () => PyBytesType = IntPtr.Zero); XDecref(op); #endif op = PyTuple_New(0); - SetPyMember(ref PyTupleType, PyObject_Type(op)); + SetPyMember(ref PyTupleType, PyObject_Type(op), + () => PyTupleType = IntPtr.Zero); XDecref(op); op = PyList_New(0); - SetPyMember(ref PyListType, PyObject_Type(op)); + SetPyMember(ref PyListType, PyObject_Type(op), + () => PyListType = IntPtr.Zero); XDecref(op); op = PyDict_New(); - SetPyMember(ref PyDictType, PyObject_Type(op)); + SetPyMember(ref PyDictType, PyObject_Type(op), + () => PyDictType = IntPtr.Zero); XDecref(op); op = PyInt_FromInt32(0); - SetPyMember(ref PyIntType, PyObject_Type(op)); + SetPyMember(ref PyIntType, PyObject_Type(op), + () => PyIntType = IntPtr.Zero); XDecref(op); op = PyLong_FromLong(0); - SetPyMember(ref PyLongType, PyObject_Type(op)); + SetPyMember(ref PyLongType, PyObject_Type(op), + () => PyLongType = IntPtr.Zero); XDecref(op); op = PyFloat_FromDouble(0); - SetPyMember(ref PyFloatType, PyObject_Type(op)); + SetPyMember(ref PyFloatType, PyObject_Type(op), + () => PyFloatType = IntPtr.Zero); XDecref(op); #if !PYTHON2 @@ -274,10 +294,12 @@ internal static void Initialize(bool initSigs = false) IntPtr d = PyDict_New(); IntPtr c = PyClass_New(IntPtr.Zero, d, s); - SetPyMember(ref PyClassType, PyObject_Type(c)); + SetPyMember(ref PyClassType, PyObject_Type(c), + () => PyClassType = IntPtr.Zero); IntPtr i = PyInstance_New(c, IntPtr.Zero, IntPtr.Zero); - SetPyMember(ref PyInstanceType, PyObject_Type(i)); + SetPyMember(ref PyInstanceType, PyObject_Type(i), + () => PyInstanceType = IntPtr.Zero); XDecref(s); XDecref(i); @@ -393,12 +415,12 @@ internal static int AtExit() return 0; } - private static void SetPyMember(ref IntPtr obj, IntPtr value) + private static void SetPyMember(ref IntPtr obj, IntPtr value, Action onRelease) { // XXX: For current usages, value should not be null. PythonException.ThrowIfIsNull(value); obj = value; - _pyRefs.Add(ref obj); + _pyRefs.Add(value, onRelease); } private static void ResetPyMembers() @@ -977,7 +999,7 @@ internal static int PyObject_Compare(IntPtr value1, IntPtr value2) internal static long PyObject_Size(IntPtr pointer) { - return (long) _PyObject_Size(pointer); + return (long)_PyObject_Size(pointer); } [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyObject_Size")] @@ -1093,7 +1115,7 @@ internal static bool PyLong_Check(IntPtr ob) internal static IntPtr PyLong_FromUnsignedLong(object value) { - if(Is32Bit || IsWindows) + if (Is32Bit || IsWindows) return PyLong_FromUnsignedLong32(Convert.ToUInt32(value)); else return PyLong_FromUnsignedLong64(Convert.ToUInt64(value)); @@ -1283,7 +1305,7 @@ internal static int PySequence_DelSlice(IntPtr pointer, long i1, long i2) internal static long PySequence_Size(IntPtr pointer) { - return (long) _PySequence_Size(pointer); + return (long)_PySequence_Size(pointer); } [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PySequence_Size")] @@ -1308,7 +1330,7 @@ internal static IntPtr PySequence_Repeat(IntPtr pointer, long count) internal static long PySequence_Count(IntPtr pointer, IntPtr value) { - return (long) _PySequence_Count(pointer, value); + return (long)_PySequence_Count(pointer, value); } [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PySequence_Count")] @@ -1351,7 +1373,7 @@ internal static IntPtr PyString_FromString(string value) internal static long PyBytes_Size(IntPtr op) { - return (long) _PyBytes_Size(op); + return (long)_PyBytes_Size(op); } [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyBytes_Size")] @@ -1582,7 +1604,7 @@ internal static bool PyDict_Check(IntPtr ob) internal static long PyDict_Size(IntPtr pointer) { - return (long) _PyDict_Size(pointer); + return (long)_PyDict_Size(pointer); } [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyDict_Size")] @@ -1660,7 +1682,7 @@ internal static int PyList_SetSlice(IntPtr pointer, long start, long end, IntPtr internal static long PyList_Size(IntPtr pointer) { - return (long) _PyList_Size(pointer); + return (long)_PyList_Size(pointer); } [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyList_Size")] @@ -1709,7 +1731,7 @@ internal static IntPtr PyTuple_GetSlice(IntPtr pointer, long start, long end) internal static long PyTuple_Size(IntPtr pointer) { - return (long) _PyTuple_Size(pointer); + return (long)_PyTuple_Size(pointer); } [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyTuple_Size")] @@ -1976,40 +1998,25 @@ internal static IntPtr GetBuiltins() class PyReferenceCollection { - public List _objects { get; private set; } - - public PyReferenceCollection() - { - _objects = new List(); - } + private List> _actions = new List>(); ///

/// Record obj's address to release the obj in the future, /// obj must alive before calling Release. ///

- public void Add(ref IntPtr obj) + public void Add(IntPtr ob, Action onRelease) { - unsafe - { - fixed (void* p = &obj) - { - _objects.Add((IntPtr)p); - } - } + _actions.Add(new KeyValuePair(ob, onRelease)); } public void Release() { - foreach (var objRef in _objects) + foreach (var item in _actions) { - unsafe - { - var p = (void**)objRef; - Runtime.XDecref((IntPtr)(*p)); - *p = null; - } + Runtime.XDecref(item.Key); + item.Value?.Invoke(); } - _objects.Clear(); + _actions.Clear(); } } } From 4e5fbe040ac5569bd297d5d87a2960304343a043 Mon Sep 17 00:00:00 2001 From: Tim Gates Date: Tue, 31 Dec 2019 19:31:15 +1100 Subject: [PATCH 038/112] Fix simple typo: heirarchy -> hierarchy (#1026) Closes #1025 --- src/tests/test_repr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/test_repr.py b/src/tests/test_repr.py index d120b0c4c..5131f5d88 100644 --- a/src/tests/test_repr.py +++ b/src/tests/test_repr.py @@ -26,7 +26,7 @@ def test_str_only(): assert " Date: Wed, 11 Sep 2019 21:59:27 -0700 Subject: [PATCH 039/112] enable expanding set of marshaling conversions via PyObjectConversions added sample TupleCodec (only supporting ValueTuple) --- src/embed_tests/Codecs.cs | 86 ++++++++ .../Python.EmbeddingTest.15.csproj | 3 +- src/embed_tests/Python.EmbeddingTest.csproj | 3 +- src/embed_tests/packages.config | 5 +- src/runtime/Codecs/TupleCodecs.cs | 129 ++++++++++++ src/runtime/Python.Runtime.csproj | 6 +- src/runtime/converter.cs | 24 ++- src/runtime/converterextensions.cs | 185 ++++++++++++++++++ src/runtime/pythonengine.cs | 2 + src/runtime/runtime.cs | 9 + 10 files changed, 444 insertions(+), 8 deletions(-) create mode 100644 src/embed_tests/Codecs.cs create mode 100644 src/runtime/Codecs/TupleCodecs.cs create mode 100644 src/runtime/converterextensions.cs diff --git a/src/embed_tests/Codecs.cs b/src/embed_tests/Codecs.cs new file mode 100644 index 000000000..600215cf0 --- /dev/null +++ b/src/embed_tests/Codecs.cs @@ -0,0 +1,86 @@ +namespace Python.EmbeddingTest { + using System; + using System.Collections.Generic; + using System.Text; + using NUnit.Framework; + using Python.Runtime; + using Python.Runtime.Codecs; + + public class Codecs { + [SetUp] + public void SetUp() { + PythonEngine.Initialize(); + } + + [TearDown] + public void Dispose() { + PythonEngine.Shutdown(); + } + + [Test] + public void ConversionsGeneric() { + ConversionsGeneric, ValueTuple>(); + } + + static void ConversionsGeneric() { + TupleCodec.Register(); + var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); + T restored = default; + using (Py.GIL()) + using (var scope = Py.CreateScope()) { + void Accept(T value) => restored = value; + var accept = new Action(Accept).ToPython(); + scope.Set(nameof(tuple), tuple); + scope.Set(nameof(accept), accept); + scope.Exec($"{nameof(accept)}({nameof(tuple)})"); + Assert.AreEqual(expected: tuple, actual: restored); + } + } + + [Test] + public void ConversionsObject() { + ConversionsObject, ValueTuple>(); + } + static void ConversionsObject() { + TupleCodec.Register(); + var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); + T restored = default; + using (Py.GIL()) + using (var scope = Py.CreateScope()) { + void Accept(object value) => restored = (T)value; + var accept = new Action

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.