diff --git a/.travis.yml b/.travis.yml index 025229d04..4869b81de 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,6 @@ matrix: - python: 2.7 env: &xplat-env - BUILD_OPTS=--xplat - - NUNIT_PATH=~/.nuget/packages/nunit.consolerunner/3.*/tools/nunit3-console.exe addons: &xplat-addons apt: sources: @@ -45,7 +44,6 @@ matrix: - python: 2.7 env: &classic-env - BUILD_OPTS= - - NUNIT_PATH=./packages/NUnit.*/tools/nunit3-console.exe - python: 3.3 env: *classic-env @@ -97,7 +95,7 @@ install: script: - python -m pytest - - mono $NUNIT_PATH src/embed_tests/bin/Python.EmbeddingTest.dll + - mono src/embed_tests/bin/Python.EmbeddingTest.exe - if [[ $BUILD_OPTS == --xplat ]]; then dotnet src/embed_tests/bin/netcoreapp2.0_publish/Python.EmbeddingTest.dll; fi after_script: diff --git a/CHANGELOG.md b/CHANGELOG.md index 380c6f6d3..aad78e95e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,11 +8,14 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. ## [unreleased][] ### Added +- Added tool for debugging floating bugs. Stable tests are executed in the loop. ~100 cycles is enough to pop up any bugs. + Usage: Python.EmbeddingTest.exe --loop --where="cat != Unstable" - Added support for embedding python into dotnet core 2.0 (NetStandard 2.0) - Added new build system (pythonnet.15.sln) based on dotnetcore-sdk/xplat(crossplatform msbuild). Currently there two side-by-side build systems that produces the same output (net40) from the same sources. After a some transition time, current (mono/ msbuild 14.0) build system will be removed. -- NUnit upgraded to 3.7 (eliminates travis-ci random bug) +- Python.EmbeddingTest (net40) now tested through built-in NUnitLite in Travis-CI. (Solves NUnit vs Mono stability problems.) +- NUnit upgraded to 3.8.1, Python.EmbeddingTest now executable with the NUnitLite self-tester. - Added `clr.GetClrType` (#432, #433) - Allowed passing `None` for nullable args (#460) - Added keyword arguments based on C# syntax for calling CPython methods (#461) @@ -20,7 +23,11 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. ### Changed ### Fixed - +- Fixed memory leaks, caused by non-working PyObject, PythonException, DelecateManager->Dispatcher finalizers. + Set environment variable PYTHONNET_GC_DEBUG=1 to force full GC on each PythonEngine.Shutdown, this allows + to esily detect GC related bugs. +- Fixed secondary PythonEngine.Initialize call, all sensitive static variables now reseted. + This is a hidden bug. Once python cleaning up enough memory, objects from previous engine run becomes corrupted. - Fixed Visual Studio 2017 compat (#434) for setup.py - Fixed crash on exit of the Python interpreter if a python class derived from a .NET class has a `__namespace__` or `__assembly__` diff --git a/NuGet.config b/NuGet.config index 719fbc83c..f270fb328 100644 --- a/NuGet.config +++ b/NuGet.config @@ -1,7 +1,7 @@ - + \ No newline at end of file diff --git a/ci/appveyor_run_tests.ps1 b/ci/appveyor_run_tests.ps1 index b45440fbe..2811c5bed 100644 --- a/ci/appveyor_run_tests.ps1 +++ b/ci/appveyor_run_tests.ps1 @@ -20,7 +20,7 @@ else{ $PY = Get-Command python # Can't use ".\build\*\Python.EmbeddingTest.dll". Missing framework files. -$CS_TESTS = ".\src\embed_tests\bin\Python.EmbeddingTest.dll" +$CS_TESTS = ".\src\embed_tests\bin\Python.EmbeddingTest.exe" $RUNTIME_DIR = ".\src\runtime\bin\" # Run python tests with C# coverage diff --git a/src/embed_tests/Program.cs b/src/embed_tests/Program.cs index b4439e3e4..a18894367 100644 --- a/src/embed_tests/Program.cs +++ b/src/embed_tests/Program.cs @@ -1,8 +1,12 @@ using System; - +using System.Collections; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; using NUnit.Common; using NUnitLite; +using Python.Runtime; namespace Python.EmbeddingTest { @@ -10,6 +14,36 @@ public class Program { public static int Main(string[] args) { + if (args.Contains("--loop")) + { + args = args.Where(x => x != "--loop").ToArray(); + int result; + int runNumber = 0; + string pathEnv = Environment.GetEnvironmentVariable("PATH"); + do + { + Console.ForegroundColor = ConsoleColor.Green; + Console.Error.WriteLine($"----- Run = {++runNumber}, MemUsage = {Process.GetCurrentProcess().WorkingSet64 / 1024 / 1024} Mb -----"); + Console.ForegroundColor = ConsoleColor.Gray; + + result = new AutoRun(typeof(Program).Assembly).Execute( + args, + new ExtendedTextWrapper(Console.Out), + Console.In); + + // Python does not see Environment.SetEnvironmentVariable changes. + // So we needs restore PATH variable in a pythonic way. + using (new PythonEngine()) + { + dynamic os = PythonEngine.ImportModule("os"); + os.environ["PATH"] = new PyString(pathEnv); + } + } + while (!Console.KeyAvailable); + + return result; + } + return new AutoRun(typeof(Program).Assembly).Execute( args, new ExtendedTextWrapper(Console.Out), diff --git a/src/embed_tests/Python.EmbeddingTest.15.csproj b/src/embed_tests/Python.EmbeddingTest.15.csproj index 92d55a7e0..24bead01f 100644 --- a/src/embed_tests/Python.EmbeddingTest.15.csproj +++ b/src/embed_tests/Python.EmbeddingTest.15.csproj @@ -5,7 +5,7 @@ net40;netcoreapp2.0 x64;x86 DebugMono;DebugMonoPY3;ReleaseMono;ReleaseMonoPY3;DebugWin;DebugWinPY3;ReleaseWin;ReleaseWinPY3 - Exe + Exe false Python.EmbeddingTest Python.EmbeddingTest @@ -76,10 +76,10 @@ - + - + diff --git a/src/embed_tests/Python.EmbeddingTest.csproj b/src/embed_tests/Python.EmbeddingTest.csproj index fe02b0526..bbb35a8e0 100644 --- a/src/embed_tests/Python.EmbeddingTest.csproj +++ b/src/embed_tests/Python.EmbeddingTest.csproj @@ -4,7 +4,7 @@ Debug AnyCPU {4165C59D-2822-499F-A6DB-EACA4C331EB5} - Library + Exe Python.EmbeddingTest Python.EmbeddingTest bin\Python.EmbeddingTest.xml @@ -70,8 +70,11 @@ - - ..\..\packages\NUnit.3.7.1\lib\net40\nunit.framework.dll + + ..\..\packages\NUnit.3.8.1\lib\net40\nunit.framework.dll + + + ..\..\packages\NUnitLite.3.8.1\lib\net40\nunitlite.dll @@ -84,6 +87,7 @@ + @@ -102,6 +106,8 @@ + + diff --git a/src/embed_tests/TestExample.cs b/src/embed_tests/TestExample.cs index 671f9e33d..7265ffa44 100644 --- a/src/embed_tests/TestExample.cs +++ b/src/embed_tests/TestExample.cs @@ -5,6 +5,7 @@ namespace Python.EmbeddingTest { + [Category("Unstable")] public class TestExample { [OneTimeSetUp] diff --git a/src/embed_tests/TestFinalizer.cs b/src/embed_tests/TestFinalizer.cs new file mode 100644 index 000000000..a7b6432a5 --- /dev/null +++ b/src/embed_tests/TestFinalizer.cs @@ -0,0 +1,130 @@ +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using NUnit.Framework; +using Python.Runtime; + +namespace Python.EmbeddingTest +{ + public class TestFinalizer + { + [OneTimeSetUp] + public void SetUp() + { + PythonEngine.Initialize(); + } + + [OneTimeTearDown] + public void Dispose() + { + PythonEngine.Shutdown(); + } + + [Test] + public void TestClrObjectFullRelease() + { + var gs = PythonEngine.BeginAllowThreads(); + + WeakReference weakRef; + try + { + var weakRefCreateTask = Task.Factory.StartNew(() => + { + using (Py.GIL()) + { + byte[] testObject = new byte[100]; + var testObjectWeakReference = new WeakReference(testObject); + + dynamic pyList = new PyList(); + pyList.append(testObject); + return testObjectWeakReference; + } + }); + + weakRef = weakRefCreateTask.Result; + } + finally + { + PythonEngine.EndAllowThreads(gs); + } + + // Triggering C# finalizer (PyList ref should be scheduled to decref). + GC.Collect(); + GC.WaitForPendingFinalizers(); + + // Forcing dec reference thread to wakeup and decref PyList. + PythonEngine.EndAllowThreads(PythonEngine.BeginAllowThreads()); + Thread.Sleep(200); + PythonEngine.CurrentRefDecrementer.WaitForPendingDecReferences(); + + // Now python free up GCHandle on CLRObject and subsequent GC should fully remove testObject. + GC.Collect(); + GC.WaitForPendingFinalizers(); + + Assert.IsFalse(weakRef.IsAlive, "Clr object should be collected."); + } + + [Test] + [Ignore("For debug only")] + public void TestExceptionMemoryLeak() + { + dynamic pymodule; + PyObject gc; + dynamic testmethod; + + var ts = PythonEngine.BeginAllowThreads(); + IntPtr pylock = PythonEngine.AcquireLock(); + +#if NETCOREAPP + const string s = "../../fixtures"; +#else + const string s = "../fixtures"; +#endif + string testPath = Path.Combine(TestContext.CurrentContext.TestDirectory, s); + + IntPtr str = Runtime.Runtime.PyString_FromString(testPath); + IntPtr path = Runtime.Runtime.PySys_GetObject("path"); + Runtime.Runtime.PyList_Append(path, str); + + { + PyObject sys = PythonEngine.ImportModule("sys"); + gc = PythonEngine.ImportModule("gc"); + + pymodule = PythonEngine.ImportModule("MemoryLeakTest.pyraise"); + testmethod = pymodule.test_raise_exception; + } + + PythonEngine.ReleaseLock(pylock); + + double floatarg1 = 5.1f; + dynamic res = null; + { + for (int i = 1; i <= 10000000; i++) + { + using (Py.GIL()) + { + try + { + res = testmethod(Py.kw("number", floatarg1), Py.kw("astring", "bbc")); + } + catch (Exception e) + { + if (i % 10000 == 0) + { + TestContext.Progress.WriteLine(e.Message); + } + } + } + + if (i % 10000 == 0) + { + GC.Collect(); + } + } + } + + PythonEngine.EndAllowThreads(ts); + } + } +} diff --git a/src/embed_tests/TestPyAnsiString.cs b/src/embed_tests/TestPyAnsiString.cs index 9ba7d6cc6..b4a965ff7 100644 --- a/src/embed_tests/TestPyAnsiString.cs +++ b/src/embed_tests/TestPyAnsiString.cs @@ -63,6 +63,7 @@ public void TestCtorPtr() const string expected = "foo"; var t = new PyAnsiString(expected); + Runtime.Runtime.XIncref(t.Handle); var actual = new PyAnsiString(t.Handle); Assert.AreEqual(expected, actual.ToString()); diff --git a/src/embed_tests/TestPyLong.cs b/src/embed_tests/TestPyLong.cs index fe3e13ef5..903256f12 100644 --- a/src/embed_tests/TestPyLong.cs +++ b/src/embed_tests/TestPyLong.cs @@ -102,6 +102,7 @@ public void TestCtorDouble() public void TestCtorPtr() { var i = new PyLong(5); + Runtime.Runtime.XIncref(i.Handle); var a = new PyLong(i.Handle); Assert.AreEqual(5, a.ToInt32()); } diff --git a/src/embed_tests/TestPyString.cs b/src/embed_tests/TestPyString.cs index 9d1cdb0e9..0de436e35 100644 --- a/src/embed_tests/TestPyString.cs +++ b/src/embed_tests/TestPyString.cs @@ -64,6 +64,7 @@ public void TestCtorPtr() const string expected = "foo"; var t = new PyString(expected); + Runtime.Runtime.XIncref(t.Handle); var actual = new PyString(t.Handle); Assert.AreEqual(expected, actual.ToString()); diff --git a/src/embed_tests/TestPythonEngineProperties.cs b/src/embed_tests/TestPythonEngineProperties.cs index 01c6ae7e3..d95942577 100644 --- a/src/embed_tests/TestPythonEngineProperties.cs +++ b/src/embed_tests/TestPythonEngineProperties.cs @@ -1,4 +1,5 @@ using System; +using System.Diagnostics; using NUnit.Framework; using Python.Runtime; @@ -109,18 +110,41 @@ public static void GetPythonHomeDefault() [Test] public void SetPythonHome() { + // We needs to ensure that engine was started and shutdown at least once before setting dummy home. + // Otherwise engine will not run with dummy path with random problem. + if (!PythonEngine.IsInitialized) + { + PythonEngine.Initialize(); + } + + PythonEngine.Shutdown(); + + var pythonHomeBackup = PythonEngine.PythonHome; + var pythonHome = "/dummypath/"; PythonEngine.PythonHome = pythonHome; PythonEngine.Initialize(); - Assert.AreEqual(pythonHome, PythonEngine.PythonHome); PythonEngine.Shutdown(); + + // Restoring valid pythonhome. + PythonEngine.PythonHome = pythonHomeBackup; } [Test] public void SetPythonHomeTwice() { + // We needs to ensure that engine was started and shutdown at least once before setting dummy home. + // Otherwise engine will not run with dummy path with random problem. + if (!PythonEngine.IsInitialized) + { + PythonEngine.Initialize(); + } + PythonEngine.Shutdown(); + + var pythonHomeBackup = PythonEngine.PythonHome; + var pythonHome = "/dummypath/"; PythonEngine.PythonHome = "/dummypath2/"; @@ -129,11 +153,20 @@ public void SetPythonHomeTwice() Assert.AreEqual(pythonHome, PythonEngine.PythonHome); PythonEngine.Shutdown(); + + PythonEngine.PythonHome = pythonHomeBackup; } [Test] public void SetProgramName() { + if (PythonEngine.IsInitialized) + { + PythonEngine.Shutdown(); + } + + var programNameBackup = PythonEngine.ProgramName; + var programName = "FooBar"; PythonEngine.ProgramName = programName; @@ -141,6 +174,8 @@ public void SetProgramName() Assert.AreEqual(programName, PythonEngine.ProgramName); PythonEngine.Shutdown(); + + PythonEngine.ProgramName = programNameBackup; } [Test] @@ -156,7 +191,7 @@ public void SetPythonPath() string path = PythonEngine.PythonPath; PythonEngine.Shutdown(); - PythonEngine.ProgramName = path; + PythonEngine.PythonPath = path; PythonEngine.Initialize(); Assert.AreEqual(path, PythonEngine.PythonPath); @@ -171,7 +206,6 @@ public void SetPythonPathExceptionOn27() Assert.Pass(); } - // Get previous path to avoid crashing Python PythonEngine.Initialize(); string path = PythonEngine.PythonPath; PythonEngine.Shutdown(); diff --git a/src/embed_tests/TestRuntime.cs b/src/embed_tests/TestRuntime.cs index 2e0598da7..9dfb04c9c 100644 --- a/src/embed_tests/TestRuntime.cs +++ b/src/embed_tests/TestRuntime.cs @@ -1,4 +1,4 @@ -using System; +using System; using NUnit.Framework; using Python.Runtime; @@ -6,10 +6,21 @@ namespace Python.EmbeddingTest { public class TestRuntime { + [OneTimeSetUp] + public void SetUp() + { + // We needs to ensure that no any engines are running. + if (PythonEngine.IsInitialized) + { + PythonEngine.Shutdown(); + } + } + [Test] public static void Py_IsInitializedValue() { - Runtime.Runtime.Py_Finalize(); // In case another test left it on. + // We know for sure that all engines are shut down. + // Runtime.Runtime.Py_Finalize(); Assert.AreEqual(0, Runtime.Runtime.Py_IsInitialized()); Runtime.Runtime.Py_Initialize(); Assert.AreEqual(1, Runtime.Runtime.Py_IsInitialized()); diff --git a/src/embed_tests/TestsSuite.cs b/src/embed_tests/TestsSuite.cs new file mode 100644 index 000000000..44ce2d4b8 --- /dev/null +++ b/src/embed_tests/TestsSuite.cs @@ -0,0 +1,18 @@ +using NUnit.Framework; +using Python.Runtime; + +namespace Python.EmbeddingTest +{ + [SetUpFixture] + public class TestsSuite + { + [OneTimeTearDown] + public void FinalCleanup() + { + if (PythonEngine.IsInitialized) + { + PythonEngine.Shutdown(); + } + } + } +} diff --git a/src/embed_tests/fixtures/MemoryLeakTest/pyraise.py b/src/embed_tests/fixtures/MemoryLeakTest/pyraise.py new file mode 100644 index 000000000..444594283 --- /dev/null +++ b/src/embed_tests/fixtures/MemoryLeakTest/pyraise.py @@ -0,0 +1,8 @@ +def test_raise_exception(number=3, astring='abc'): + raise ValueError("testing for memory leak") + return astring * int(number) + +def test_raise_exception2(number, astring): + #raise ValueError("testing for memory leak") + #astring * int(number) + return "test" diff --git a/src/embed_tests/packages.config b/src/embed_tests/packages.config index 8c175f441..aa03916a2 100644 --- a/src/embed_tests/packages.config +++ b/src/embed_tests/packages.config @@ -1,5 +1,6 @@ - + + diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index 82825a626..b36f228b2 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -129,6 +129,7 @@ + diff --git a/src/runtime/assemblymanager.cs b/src/runtime/assemblymanager.cs index 06a4449a2..34d08f0ec 100644 --- a/src/runtime/assemblymanager.cs +++ b/src/runtime/assemblymanager.cs @@ -17,13 +17,14 @@ internal class AssemblyManager { // modified from event handlers below, potentially triggered from different .NET threads // therefore this should be a ConcurrentDictionary - private static ConcurrentDictionary> namespaces; + private static ConcurrentDictionary> namespaces = + new ConcurrentDictionary>(); //private static Dictionary> generics; private static AssemblyLoadEventHandler lhandler; private static ResolveEventHandler rhandler; // updated only under GIL? - private static Dictionary probed; + private static Dictionary probed = new Dictionary(32); // modified from event handlers below, potentially triggered from different .NET threads private static AssemblyList assemblies; @@ -40,9 +41,6 @@ private AssemblyManager() /// internal static void Initialize() { - namespaces = new ConcurrentDictionary>(); - probed = new Dictionary(32); - //generics = new Dictionary>(); assemblies = new AssemblyList(16); pypath = new List(16); diff --git a/src/runtime/classderived.cs b/src/runtime/classderived.cs index 16d3b99db..ec3734ea5 100644 --- a/src/runtime/classderived.cs +++ b/src/runtime/classderived.cs @@ -1,8 +1,9 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using System.Reflection; using System.Reflection.Emit; +using System.Resources; using System.Runtime.InteropServices; using System.Threading.Tasks; @@ -32,6 +33,12 @@ static ClassDerivedObject() moduleBuilders = new Dictionary, ModuleBuilder>(); } + public static void Reset() + { + assemblyBuilders = new Dictionary(); + moduleBuilders = new Dictionary, ModuleBuilder>(); + } + internal ClassDerivedObject(Type tp) : base(tp) { } diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index 6a9d40ebd..0b084a49d 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -34,6 +34,11 @@ static ClassManager() dtype = typeof(MulticastDelegate); } + public static void Reset() + { + cache = new Dictionary(128); + } + ///

/// Return the ClassBase-derived instance that implements a particular /// reflected managed type, creating it if it doesn't yet exist. diff --git a/src/runtime/debughelper.cs b/src/runtime/debughelper.cs index 2a91a74b4..3fe9ee5bb 100644 --- a/src/runtime/debughelper.cs +++ b/src/runtime/debughelper.cs @@ -116,10 +116,10 @@ internal static void debug(string msg) Console.WriteLine(" {0}", msg); } - ///

+ ///

/// Helper function to inspect/compare managed to native conversions. - /// Especially useful when debugging CustomMarshaler. - ///

+ /// Especially useful when debugging CustomMarshaler. + ///

/// [Conditional("DEBUG")] public static void PrintHexBytes(byte[] bytes) diff --git a/src/runtime/delegatemanager.cs b/src/runtime/delegatemanager.cs index 7632816d1..54256ac17 100644 --- a/src/runtime/delegatemanager.cs +++ b/src/runtime/delegatemanager.cs @@ -185,9 +185,12 @@ public class Dispatcher { public IntPtr target; public Type dtype; + private readonly PyReferenceDecrementer _referenceDecrementer; public Dispatcher(IntPtr target, Type dtype) { + _referenceDecrementer = PythonEngine.CurrentRefDecrementer; + Runtime.XIncref(target); this.target = target; this.dtype = dtype; @@ -195,18 +198,7 @@ public Dispatcher(IntPtr target, Type dtype) ~Dispatcher() { - // We needs to disable Finalizers until it's valid implementation. - // Current implementation can produce low probability floating bugs. - return; - - // Note: the managed GC thread can run and try to free one of - // these *after* the Python runtime has been finalized! - if (Runtime.Py_IsInitialized() > 0) - { - IntPtr gs = PythonEngine.AcquireLock(); - Runtime.XDecref(target); - PythonEngine.ReleaseLock(gs); - } + _referenceDecrementer?.ScheduleDecRef(target); } public object Dispatch(ArrayList args) diff --git a/src/runtime/genericutil.cs b/src/runtime/genericutil.cs index 9772d082f..3a230e12c 100644 --- a/src/runtime/genericutil.cs +++ b/src/runtime/genericutil.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Resources; namespace Python.Runtime { @@ -20,6 +21,11 @@ static GenericUtil() mapping = new Dictionary>>(); } + public static void Reset() + { + mapping = new Dictionary>>(); + } + ///

/// Register a generic type that appears in a given namespace. ///

diff --git a/src/runtime/interop.cs b/src/runtime/interop.cs index 4ae4b61e0..872d5fc2c 100644 --- a/src/runtime/interop.cs +++ b/src/runtime/interop.cs @@ -352,7 +352,6 @@ static Interop() } keepAlive = new ArrayList(); - Marshal.AllocHGlobal(IntPtr.Size); pmap = new Hashtable(); pmap["tp_dealloc"] = p["DestructorFunc"]; diff --git a/src/runtime/methodbinding.cs b/src/runtime/methodbinding.cs index 07090a92c..b58c092de 100644 --- a/src/runtime/methodbinding.cs +++ b/src/runtime/methodbinding.cs @@ -21,11 +21,12 @@ public MethodBinding(MethodObject m, IntPtr target, IntPtr targetType) Runtime.XIncref(target); this.target = target; - Runtime.XIncref(targetType); if (targetType == IntPtr.Zero) { targetType = Runtime.PyObject_Type(target); } + + Runtime.XIncref(targetType); this.targetType = targetType; this.info = null; diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index e683026f9..0ec9900dd 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -328,6 +328,17 @@ public CLRModule() : base("clr") } } + public static void Reset() + { + hacked = false; + interactive_preload = true; + preload = false; + + // XXX Test performance of new features // + _SuppressDocs = false; + _SuppressOverloads = false; + } + ///

/// The initializing of the preload hook has to happen as late as /// possible since sys.ps1 is created after the CLR module is diff --git a/src/runtime/pyiter.cs b/src/runtime/pyiter.cs index ee07bcecf..5fa6ec70c 100644 --- a/src/runtime/pyiter.cs +++ b/src/runtime/pyiter.cs @@ -40,14 +40,20 @@ public PyIter(PyObject iterable) } } - protected override void Dispose(bool disposing) + protected internal override void Dispose(bool disposing) { - if (null != _current) + try { - _current.Dispose(); - _current = null; + if (null != _current) + { + _current.Dispose(disposing); + _current = null; + } + } + finally + { + base.Dispose(disposing); } - base.Dispose(disposing); } public bool MoveNext() diff --git a/src/runtime/pyobject.cs b/src/runtime/pyobject.cs index 0d186bf4e..843325c5b 100644 --- a/src/runtime/pyobject.cs +++ b/src/runtime/pyobject.cs @@ -16,6 +16,7 @@ public class PyObject : DynamicObject, IEnumerable, IDisposable { protected internal IntPtr obj = IntPtr.Zero; private bool disposed = false; + private readonly PyReferenceDecrementer _referenceDecrementer; ///

/// PyObject Constructor @@ -26,7 +27,7 @@ public class PyObject : DynamicObject, IEnumerable, IDisposable /// and the reference will be DECREFed when the PyObject is garbage /// collected or explicitly disposed. /// - public PyObject(IntPtr ptr) + public PyObject(IntPtr ptr): this() { obj = ptr; } @@ -36,6 +37,7 @@ public PyObject(IntPtr ptr) protected PyObject() { + _referenceDecrementer = PythonEngine.CurrentRefDecrementer; } // Ensure that encapsulated Python object is decref'ed appropriately @@ -43,11 +45,7 @@ protected PyObject() ~PyObject() { - // We needs to disable Finalizers until it's valid implementation. - // Current implementation can produce low probability floating bugs. - return; - - Dispose(); + Dispose(false); } @@ -134,25 +132,40 @@ public T As() /// to Python objects will not be released until a managed garbage /// collection occurs. /// - protected virtual void Dispose(bool disposing) + protected internal virtual void Dispose(bool disposing) { if (!disposed) { - if (Runtime.Py_IsInitialized() > 0 && !Runtime.IsFinalizing) + disposed = true; + IntPtr objToDispose = obj; + obj = IntPtr.Zero; + + if (disposing) { - IntPtr gs = PythonEngine.AcquireLock(); - Runtime.XDecref(obj); - obj = IntPtr.Zero; - PythonEngine.ReleaseLock(gs); + if (Runtime.Py_IsInitialized() > 0 && !Runtime.IsFinalizing) + { + IntPtr gs = PythonEngine.AcquireLock(); + try + { + Runtime.XDecref(objToDispose); + } + finally + { + PythonEngine.ReleaseLock(gs); + } + } + } + else + { + _referenceDecrementer?.ScheduleDecRef(objToDispose); } - disposed = true; } } public void Dispose() { - Dispose(true); GC.SuppressFinalize(this); + Dispose(true); } diff --git a/src/runtime/pyreferencedecrementer.cs b/src/runtime/pyreferencedecrementer.cs new file mode 100644 index 000000000..674ebb5f9 --- /dev/null +++ b/src/runtime/pyreferencedecrementer.cs @@ -0,0 +1,179 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.Threading; +using System.Threading.Tasks; + +namespace Python.Runtime +{ + ///

+ /// Background python objects reference decrementer. + /// !!! By some unknown reasons python references decrements have been performed in the GC thread - affects GC. !!! + /// This component contains background thread with the queue of IntPtr python object references. + /// Finalizers should schedule reference decrement operation in this component to avoid problem with GC. + /// Each PythonEngine Init/Shutdown have it's own PyReferenceDecrementer, so PyObject should also have a field with the + /// reference to the correct PyReferenceDecrementer. + ///

+ internal class PyReferenceDecrementer : IDisposable + { + private static readonly DedicatedThreadTaskScheduler DedicatedThreadTaskScheduler = new DedicatedThreadTaskScheduler(); + + private readonly BlockingCollection _asyncDecRefQueue = new BlockingCollection(); + + private CancellationTokenSource _cts; + private CancellationToken _ct; + private Task _backgroundWorkerTask; + + public PyReferenceDecrementer() + { + InitDecRefThread(); + } + + public void ScheduleDecRef(IntPtr pyRef) + { + // ReSharper disable once MethodSupportsCancellation + _asyncDecRefQueue.Add(pyRef); + } + + internal void WaitForPendingDecReferences() + { + ShutdownDecRefThread(); + InitDecRefThread(); + } + + private void ShutdownDecRefThread() + { + _cts?.Cancel(); + try + { + IntPtr ts = IntPtr.Zero; + if (Runtime.PyGILState_GetThisThreadState() != IntPtr.Zero) + { + ts = Runtime.PyEval_SaveThread(); + } + try + { + // ReSharper disable once MethodSupportsCancellation + _backgroundWorkerTask.Wait(); + } + catch (AggregateException ex) + { + if (!(ex.InnerException is OperationCanceledException)) + { + throw; + } + } + finally + { + if (ts != IntPtr.Zero) + { + Runtime.PyEval_RestoreThread(ts); + } + } + } + catch + { + // Just stopping background thread. + } + + _cts = null; + _ct = default(CancellationToken); + + _backgroundWorkerTask = null; + } + + private void InitDecRefThread() + { + _cts = new CancellationTokenSource(); + _ct = _cts.Token; + + _backgroundWorkerTask = Task.Factory.StartNew(WorkerThread, _ct, TaskCreationOptions.LongRunning, + DedicatedThreadTaskScheduler); + } + + private void WorkerThread() + { + while (true) + { + IntPtr refToDecrease = _asyncDecRefQueue.Take(_ct); ; + + try + { + + IntPtr gs = PythonEngine.AcquireLock(); + try + { + do + { + Runtime.XDecref(refToDecrease); + } while (_asyncDecRefQueue.TryTake(out refToDecrease)); + } + finally + { + PythonEngine.ReleaseLock(gs); + } + } + catch + { + // Nothing to do in this case. + } + } + } + + public void Dispose() + { + ShutdownDecRefThread(); + } + } + + + ///

+ /// Scheduler that uses only one thread for all scheduled task. + ///

+ internal class DedicatedThreadTaskScheduler : TaskScheduler + { + private readonly BlockingCollection _queuedTasks = new BlockingCollection(); + + + ///

+ /// Initializes a new instance of the class. + ///

+ public DedicatedThreadTaskScheduler() + { + var thread = new Thread(WorkerThreadProc); + thread.IsBackground = true; + thread.Start(); + } + + /// + protected override IEnumerable GetScheduledTasks() + { + return _queuedTasks; + } + + /// + protected override void QueueTask(Task task) + { + _queuedTasks.Add(task); + } + + /// + protected override bool TryExecuteTaskInline(Task task, bool taskWasPreviouslyQueued) + { + return false; + } + + private void WorkerThreadProc() + { + for (;;) + { + Task dequeuedTask = _queuedTasks.Take(); + + // This is synchronous execution. + bool taskExecuted = TryExecuteTask(dequeuedTask); + Debug.Assert(taskExecuted, "DedicatedThread task have some problem."); + } + } + } +} diff --git a/src/runtime/pyscope.cs b/src/runtime/pyscope.cs index 67f93c6e2..88161e677 100644 --- a/src/runtime/pyscope.cs +++ b/src/runtime/pyscope.cs @@ -24,12 +24,14 @@ public class PyGILAttribute : Attribute [PyGIL] public class PyScope : DynamicObject, IDisposable { + private readonly PyReferenceDecrementer _referenceDecrementer; + public readonly string Name; ///

/// the python Module object the scope associated with. ///

- internal readonly IntPtr obj; + internal IntPtr obj; ///

/// the variable dict of the scope. @@ -57,6 +59,8 @@ public class PyScope : DynamicObject, IDisposable /// internal PyScope(IntPtr ptr, PyScopeManager manager) { + _referenceDecrementer = PythonEngine.CurrentRefDecrementer; + if (!Runtime.PyType_IsSubtype(Runtime.PyObject_TYPE(ptr), Runtime.PyModuleType)) { throw new PyScopeException("object is not a module"); @@ -514,33 +518,68 @@ private void Check() } } - public void Dispose() + protected virtual void Dispose(bool disposing) { - if (_isDisposed) + if (!_isDisposed) { - return; + IntPtr objToDispose = obj; + if (disposing) + { + try + { + OnDispose?.Invoke(this); + } + finally + { + _isDisposed = true; + obj = IntPtr.Zero; + } + + if (Runtime.Py_IsInitialized() > 0 && !Runtime.IsFinalizing) + { + IntPtr gs = PythonEngine.AcquireLock(); + try + { + Runtime.XDecref(objToDispose); + } + finally + { + PythonEngine.ReleaseLock(gs); + } + } + } + else + { + _isDisposed = true; + obj = IntPtr.Zero; + _referenceDecrementer?.ScheduleDecRef(objToDispose); + } } - _isDisposed = true; - Runtime.XDecref(obj); - this.OnDispose?.Invoke(this); } - ~PyScope() + public void Dispose() { - // We needs to disable Finalizers until it's valid implementation. - // Current implementation can produce low probability floating bugs. - return; + GC.SuppressFinalize(this); + Dispose(true); + } - Dispose(); + ~PyScope() + { + Dispose(false); } } public class PyScopeManager { - public readonly static PyScopeManager Global = new PyScopeManager(); + public static PyScopeManager Global; private Dictionary NamedScopes = new Dictionary(); + internal static void Reset() + { + Global = new PyScopeManager(); + } + internal PyScope NewScope(string name) { if (name == null) diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index a23c7ac79..e0593140c 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -1,9 +1,10 @@ -using System; +using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Reflection; using System.Runtime.InteropServices; +using System.Threading; namespace Python.Runtime { @@ -17,6 +18,12 @@ public class PythonEngine : IDisposable private static IntPtr _pythonHome = IntPtr.Zero; private static IntPtr _programName = IntPtr.Zero; private static IntPtr _pythonPath = IntPtr.Zero; + private static bool _isGcDebugEnabled; + + static PythonEngine() + { + _isGcDebugEnabled = !String.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("PYTHONNET_GC_DEBUG")); + } public PythonEngine() { @@ -130,6 +137,8 @@ public static string Compiler get { return Marshal.PtrToStringAnsi(Runtime.Py_GetCompiler()); } } + internal static PyReferenceDecrementer CurrentRefDecrementer { get; private set; } + public static int RunSimpleString(string code) { return Runtime.PyRun_SimpleString(code); @@ -215,6 +224,8 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true) } key.Dispose(); } + + CurrentRefDecrementer = new PyReferenceDecrementer(); } finally { @@ -294,13 +305,36 @@ public static void Shutdown() { if (initialized) { + + if (_isGcDebugEnabled) + { + GC.Collect(); + GC.WaitForPendingFinalizers(); + Thread.Sleep(100); + } + + CurrentRefDecrementer?.Dispose(); + CurrentRefDecrementer = null; + + if (_isGcDebugEnabled) + { + GC.Collect(); + GC.WaitForPendingFinalizers(); + Thread.Sleep(100); + } + + PyScopeManager.Global.Clear(); - Marshal.FreeHGlobal(_pythonHome); - _pythonHome = IntPtr.Zero; - Marshal.FreeHGlobal(_programName); - _programName = IntPtr.Zero; - Marshal.FreeHGlobal(_pythonPath); - _pythonPath = IntPtr.Zero; + // We should not release memory for variables that can be used without initialized python engine. + // It's assumed that Py_GetPythonHome returns valid string without engine initialize. Py_GetPythonHome will always return the + // same pointer that was passed before to Py_SetPythonHome and stored in the _pythonHome. + + ////Marshal.FreeHGlobal(_pythonHome); + ////_pythonHome = IntPtr.Zero; + ////Marshal.FreeHGlobal(_programName); + ////_programName = IntPtr.Zero; + ////Marshal.FreeHGlobal(_pythonPath); + ////_pythonPath = IntPtr.Zero; Runtime.Shutdown(); initialized = false; @@ -562,15 +596,24 @@ internal GILState() state = PythonEngine.AcquireLock(); } + protected virtual void Dispose(bool disposing) + { + //ReleaseLock is thread bound and if it's called in finalizer thread it can wrongly release lock. + if (disposing) + { + PythonEngine.ReleaseLock(state); + } + } + public void Dispose() { - PythonEngine.ReleaseLock(state); GC.SuppressFinalize(this); + Dispose(true); } ~GILState() { - Dispose(); + Dispose(false); } } diff --git a/src/runtime/pythonexception.cs b/src/runtime/pythonexception.cs index ded7fbeb5..dcba80bd6 100644 --- a/src/runtime/pythonexception.cs +++ b/src/runtime/pythonexception.cs @@ -8,6 +8,7 @@ namespace Python.Runtime ///

public class PythonException : System.Exception { + private readonly PyReferenceDecrementer _referenceDecrementer; private IntPtr _pyType = IntPtr.Zero; private IntPtr _pyValue = IntPtr.Zero; private IntPtr _pyTB = IntPtr.Zero; @@ -18,11 +19,14 @@ public class PythonException : System.Exception public PythonException() { + _referenceDecrementer = PythonEngine.CurrentRefDecrementer; IntPtr gs = PythonEngine.AcquireLock(); Runtime.PyErr_Fetch(ref _pyType, ref _pyValue, ref _pyTB); - Runtime.XIncref(_pyType); - Runtime.XIncref(_pyValue); - Runtime.XIncref(_pyTB); + + // Those references already owned by the caller + ////Runtime.XIncref(_pyType); + ////Runtime.XIncref(_pyValue); + ////Runtime.XIncref(_pyTB); if (_pyType != IntPtr.Zero && _pyValue != IntPtr.Zero) { string type; @@ -45,7 +49,13 @@ public PythonException() } if (_pyTB != IntPtr.Zero) { - PyObject tb_module = PythonEngine.ImportModule("traceback"); + + if (_tbModule == null) + { + _tbModule = PythonEngine.ImportModule("traceback"); + } + PyObject tb_module = _tbModule; + Runtime.XIncref(_pyTB); using (var pyTB = new PyObject(_pyTB)) { @@ -55,16 +65,15 @@ public PythonException() PythonEngine.ReleaseLock(gs); } + private static PyObject _tbModule; + // Ensure that encapsulated Python objects are decref'ed appropriately // when the managed exception wrapper is garbage-collected. ~PythonException() { - // We needs to disable Finalizers until it's valid implementation. - // Current implementation can produce low probability floating bugs. - return; - Dispose(); + Dispose(false); } ///

@@ -153,23 +162,64 @@ public string PythonTypeName /// See GH#397 and GH#400. /// public void Dispose() + { + GC.SuppressFinalize(this); + Dispose(true); + } + + protected void Dispose(bool disposing) { if (!disposed) { - if (Runtime.Py_IsInitialized() > 0 && !Runtime.IsFinalizing) + disposed = true; + + IntPtr pyTypeToDispose = _pyType; + _pyType = IntPtr.Zero; + + IntPtr pyValueToDispose = _pyValue; + _pyValue = IntPtr.Zero; + + IntPtr pyTBToDispose = _pyTB; + _pyTB = IntPtr.Zero; + + if (disposing) { - IntPtr gs = PythonEngine.AcquireLock(); - Runtime.XDecref(_pyType); - Runtime.XDecref(_pyValue); - // XXX Do we ever get TraceBack? // - if (_pyTB != IntPtr.Zero) + if (Runtime.Py_IsInitialized() > 0 && !Runtime.IsFinalizing) { - Runtime.XDecref(_pyTB); + IntPtr gs = PythonEngine.AcquireLock(); + try + { + Runtime.XDecref(pyTypeToDispose); + Runtime.XDecref(pyValueToDispose); + // XXX Do we ever get TraceBack? // + if (pyTBToDispose != IntPtr.Zero) + { + Runtime.XDecref(pyTBToDispose); + } + } + finally + { + PythonEngine.ReleaseLock(gs); + } + } + } + else + { + if (pyTypeToDispose != IntPtr.Zero) + { + _referenceDecrementer?.ScheduleDecRef(pyTypeToDispose); + } + + if (pyValueToDispose != IntPtr.Zero) + { + _referenceDecrementer?.ScheduleDecRef(pyValueToDispose); + } + + if (pyTBToDispose != IntPtr.Zero) + { + _referenceDecrementer?.ScheduleDecRef(pyTBToDispose); } - PythonEngine.ReleaseLock(gs); } - GC.SuppressFinalize(this); - disposed = true; } } diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index abd0661a4..164b1bfaa 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -221,6 +221,15 @@ internal static void Initialize() PyEval_InitThreads(); } + IsFinalizing = false; + + CLRModule.Reset(); + GenericUtil.Reset(); + PyScopeManager.Reset(); + ClassManager.Reset(); + ClassDerivedObject.Reset(); + TypeManager.Reset(); + IntPtr op; IntPtr dict; if (IsPython3) diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 6570ee083..ececcc79c 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections; using System.Collections.Generic; using System.Reflection; @@ -21,6 +21,10 @@ static TypeManager() cache = new Dictionary(128); } + public static void Reset() + { + cache = new Dictionary(128); + } ///

/// Given a managed Type derived from ExtensionType, get the handle to pFad - Phonifier reborn

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.

Alternative Proxies:

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.