diff --git a/.travis.yml b/.travis.yml index 1ffe7754b..e664a4696 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,8 @@ env: matrix: - BUILD_OPTS=--xplat NUNIT_PATH="~/.nuget/packages/nunit.consolerunner/3.*/tools/nunit3-console.exe" RUN_TESTS=dotnet EMBED_TESTS_PATH=netcoreapp2.0_publish/ PERF_TESTS_PATH=net461/ - BUILD_OPTS="" NUNIT_PATH="./packages/NUnit.*/tools/nunit3-console.exe" RUN_TESTS="mono $NUNIT_PATH" EMBED_TESTS_PATH="" PERF_TESTS_PATH="" + - PYTHONNET_SHUTDOWN_MODE="Soft" BUILD_OPTS=--xplat NUNIT_PATH="~/.nuget/packages/nunit.consolerunner/3.*/tools/nunit3-console.exe" RUN_TESTS=dotnet EMBED_TESTS_PATH=netcoreapp2.0_publish/ PERF_TESTS_PATH=net461/ + - PYTHONNET_SHUTDOWN_MODE="Soft" BUILD_OPTS="" NUNIT_PATH="./packages/NUnit.*/tools/nunit3-console.exe" RUN_TESTS="mono $NUNIT_PATH" EMBED_TESTS_PATH="" PERF_TESTS_PATH="" global: - LD_PRELOAD=/lib/x86_64-linux-gnu/libSegFault.so diff --git a/AUTHORS.md b/AUTHORS.md index ce6a79513..eeafd98e4 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -35,6 +35,7 @@ - David Lassonde ([@lassond](https://github.com/lassond)) - David Lechner ([@dlech](https://github.com/dlech)) - Dmitriy Se ([@dmitriyse](https://github.com/dmitriyse)) +- Félix Bourbonnais ([@BadSingleton](https://github.com/BadSingleton)) - Florian Treurniet ([@ftreurni](https://github.com/ftreurni)) - He-chien Tsai ([@t3476](https://github.com/t3476)) - Inna Wiesel ([@inna-w](https://github.com/inna-w)) @@ -66,6 +67,7 @@ - William Sardar ([@williamsardar])(https://github.com/williamsardar) - Xavier Dupré ([@sdpython](https://github.com/sdpython)) - Zane Purvis ([@zanedp](https://github.com/zanedp)) +- ([@amos402]https://github.com/amos402) - ([@bltribble](https://github.com/bltribble)) - ([@civilx64](https://github.com/civilx64)) - ([@GSPP](https://github.com/GSPP)) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4417fbaf5..5bdf5e32b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -113,6 +113,7 @@ This version improves performance on benchmarks significantly compared to 2.3. - PythonEngine.Intialize will now call `Py_InitializeEx` with a default value of 0, so signals will not be configured by default on embedding. This is different from the previous behaviour, where `Py_Initialize` was called instead, which sets initSigs to 1. ([#449][i449]) - Refactored MethodBinder.Bind in preparation to make it extensible (#829) - Look for installed Windows 10 sdk's during installation instead of relying on specific versions. +- Remove `LoadLibrary` call. ([#880][p880]) ### Fixed diff --git a/appveyor.yml b/appveyor.yml index f64fbf0f6..d45ab5b36 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -16,7 +16,6 @@ environment: matrix: - PYTHON_VERSION: 3.8 - BUILD_OPTS: --xplat - PYTHON_VERSION: 3.7 BUILD_OPTS: --xplat - PYTHON_VERSION: 3.6 @@ -24,7 +23,10 @@ environment: - PYTHON_VERSION: 3.8 - PYTHON_VERSION: 3.7 - PYTHON_VERSION: 3.6 - + - PYTHON_VERSION: 3.7 + PYTHONNET_SHUTDOWN_MODE: Soft + - PYTHON_VERSION: 3.8 + PYTHONNET_SHUTDOWN_MODE: Soft init: # Update Environment Variables based on matrix/platform - set PY_VER=%PYTHON_VERSION:.=% diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs index b162d4eb0..3556df0f6 100644 --- a/src/embed_tests/TestDomainReload.cs +++ b/src/embed_tests/TestDomainReload.cs @@ -1,9 +1,12 @@ using System; -using System.CodeDom.Compiler; +using System.Collections.Generic; +using System.Diagnostics; using System.Reflection; +using System.Runtime.InteropServices; using NUnit.Framework; using Python.Runtime; +using PyRuntime = Python.Runtime.Runtime; // // This test case is disabled on .NET Standard because it doesn't have all the // APIs we use. We could work around that, but .NET Core doesn't implement @@ -16,6 +19,12 @@ namespace Python.EmbeddingTest { class TestDomainReload { + abstract class CrossCaller : MarshalByRefObject + { + public abstract ValueType Execute(ValueType arg); + } + + /// /// Test that the python runtime can survive a C# domain reload without crashing. /// @@ -50,173 +59,411 @@ class TestDomainReload [Test] public static void DomainReloadAndGC() { - // We're set up to run in the directory that includes the bin directory. - System.IO.Directory.SetCurrentDirectory(AppDomain.CurrentDomain.BaseDirectory); - - Assembly pythonRunner1 = BuildAssembly("test1"); - RunAssemblyAndUnload(pythonRunner1, "test1"); - - // Verify that python is not initialized even though we ran it. - Assert.That(Runtime.Runtime.Py_IsInitialized(), Is.Zero); - - // This caused a crash because objects allocated in pythonRunner1 - // still existed in memory, but the code to do python GC on those - // objects is gone. - Assembly pythonRunner2 = BuildAssembly("test2"); - RunAssemblyAndUnload(pythonRunner2, "test2"); - } - - // - // The code we'll test. All that really matters is - // using GIL { Python.Exec(pyScript); } - // but the rest is useful for debugging. - // - // What matters in the python code is gc.collect and clr.AddReference. - // - // Note that the language version is 2.0, so no $"foo{bar}" syntax. - // - const string TestCode = @" - using Python.Runtime; - using System; - class PythonRunner { - public static void RunPython() { - AppDomain.CurrentDomain.DomainUnload += OnDomainUnload; - string name = AppDomain.CurrentDomain.FriendlyName; - Console.WriteLine(string.Format(""[{0} in .NET] In PythonRunner.RunPython"", name)); - using (Py.GIL()) { - try { - var pyScript = string.Format(""import clr\n"" - + ""print('[{0} in python] imported clr')\n"" - + ""clr.AddReference('System')\n"" - + ""print('[{0} in python] allocated a clr object')\n"" - + ""import gc\n"" - + ""gc.collect()\n"" - + ""print('[{0} in python] collected garbage')\n"", - name); - PythonEngine.Exec(pyScript); - } catch(Exception e) { - Console.WriteLine(string.Format(""[{0} in .NET] Caught exception: {1}"", name, e)); + Assert.IsFalse(PythonEngine.IsInitialized); + RunAssemblyAndUnload("test1"); + Assert.That(PyRuntime.Py_IsInitialized() != 0, + "On soft-shutdown mode, Python runtime should still running"); + + RunAssemblyAndUnload("test2"); + Assert.That(PyRuntime.Py_IsInitialized() != 0, + "On soft-shutdown mode, Python runtime should still running"); + + if (PythonEngine.DefaultShutdownMode == ShutdownMode.Normal) + { + // The default mode is a normal mode, + // it should shutdown the Python VM avoiding influence other tests. + PyRuntime.PyGILState_Ensure(); + PyRuntime.Py_Finalize(); + } + } + + #region CrossDomainObject + + class CrossDomainObjectStep1 : CrossCaller + { + public override ValueType Execute(ValueType arg) + { + try + { + // Create a C# user-defined object in Python. Asssing some values. + Type type = typeof(Python.EmbeddingTest.Domain.MyClass); + string code = string.Format(@" +import clr +clr.AddReference('{0}') + +from Python.EmbeddingTest.Domain import MyClass +obj = MyClass() +obj.Method() +obj.StaticMethod() +obj.Property = 1 +obj.Field = 10 +", Assembly.GetExecutingAssembly().FullName); + + using (Py.GIL()) + using (var scope = Py.CreateScope()) + { + scope.Exec(code); + using (PyObject obj = scope.Get("obj")) + { + Debug.Assert(obj.AsManagedObject(type).GetType() == type); + // We only needs its Python handle + PyRuntime.XIncref(obj.Handle); + return obj.Handle; } } } - static void OnDomainUnload(object sender, EventArgs e) { - System.Console.WriteLine(string.Format(""[{0} in .NET] unloading"", AppDomain.CurrentDomain.FriendlyName)); + catch (Exception e) + { + Debug.WriteLine(e); + throw; } - }"; + } + } + class CrossDomainObjectStep2 : CrossCaller + { + public override ValueType Execute(ValueType arg) + { + // handle refering a clr object created in previous domain, + // it should had been deserialized and became callable agian. + IntPtr handle = (IntPtr)arg; + try + { + using (Py.GIL()) + { + IntPtr tp = Runtime.Runtime.PyObject_TYPE(handle); + IntPtr tp_clear = Marshal.ReadIntPtr(tp, TypeOffset.tp_clear); + Assert.That(tp_clear, Is.Not.Null); + + using (PyObject obj = new PyObject(handle)) + { + obj.InvokeMethod("Method"); + obj.InvokeMethod("StaticMethod"); + + using (var scope = Py.CreateScope()) + { + scope.Set("obj", obj); + scope.Exec(@" +obj.Method() +obj.StaticMethod() +obj.Property += 1 +obj.Field += 10 +"); + } + var clrObj = obj.As(); + Assert.AreEqual(clrObj.Property, 2); + Assert.AreEqual(clrObj.Field, 20); + } + } + } + catch (Exception e) + { + Debug.WriteLine(e); + throw; + } + return 0; + } + } + /// - /// Build an assembly out of the source code above. - /// - /// This creates a file .dll in order - /// to support the statement "proxy.theAssembly = assembly" below. - /// That statement needs a file, can't run via memory. + /// Create a C# custom object in a domain, in python code. + /// Unload the domain, create a new domain. + /// Make sure the C# custom object created in the previous domain has been re-created /// - static Assembly BuildAssembly(string assemblyName) + [Test] + public static void CrossDomainObject() { - var provider = CodeDomProvider.CreateProvider("CSharp"); + RunDomainReloadSteps(); + } + + #endregion - var compilerparams = new CompilerParameters(); - compilerparams.ReferencedAssemblies.Add("Python.Runtime.dll"); - compilerparams.GenerateExecutable = false; - compilerparams.GenerateInMemory = false; - compilerparams.IncludeDebugInformation = false; - compilerparams.OutputAssembly = assemblyName; + #region TestClassReference - var results = provider.CompileAssemblyFromSource(compilerparams, TestCode); - if (results.Errors.HasErrors) + class ReloadClassRefStep1 : CrossCaller + { + public override ValueType Execute(ValueType arg) { - var errors = new System.Text.StringBuilder("Compiler Errors:\n"); - foreach (CompilerError error in results.Errors) + const string code = @" +from Python.EmbeddingTest.Domain import MyClass + +def test_obj_call(): + obj = MyClass() + obj.Method() + obj.StaticMethod() + obj.Property = 1 + obj.Field = 10 + +test_obj_call() +"; + const string name = "test_domain_reload_mod"; + using (Py.GIL()) { - errors.AppendFormat("Line {0},{1}\t: {2}\n", - error.Line, error.Column, error.ErrorText); + // Create a new module + IntPtr module = PyRuntime.PyModule_New(name); + Assert.That(module != IntPtr.Zero); + IntPtr globals = PyRuntime.PyObject_GetAttrString(module, "__dict__"); + Assert.That(globals != IntPtr.Zero); + try + { + // import builtins + // module.__dict__[__builtins__] = builtins + int res = PyRuntime.PyDict_SetItemString(globals, "__builtins__", + PyRuntime.PyEval_GetBuiltins()); + PythonException.ThrowIfIsNotZero(res); + + // Execute the code in the module's scope + PythonEngine.Exec(code, globals); + // import sys + // modules = sys.modules + IntPtr modules = PyRuntime.PyImport_GetModuleDict(); + // modules[name] = module + res = PyRuntime.PyDict_SetItemString(modules, name, module); + PythonException.ThrowIfIsNotZero(res); + } + catch + { + PyRuntime.XDecref(module); + throw; + } + finally + { + PyRuntime.XDecref(globals); + } + return module; } - throw new Exception(errors.ToString()); } - else + } + + class ReloadClassRefStep2 : CrossCaller + { + public override ValueType Execute(ValueType arg) { - return results.CompiledAssembly; + var module = (IntPtr)arg; + using (Py.GIL()) + { + var test_obj_call = PyRuntime.PyObject_GetAttrString(module, "test_obj_call"); + PythonException.ThrowIfIsNull(test_obj_call); + var args = PyRuntime.PyTuple_New(0); + var res = PyRuntime.PyObject_CallObject(test_obj_call, args); + PythonException.ThrowIfIsNull(res); + + PyRuntime.XDecref(args); + PyRuntime.XDecref(res); + } + return 0; } } + + [Test] /// - /// This is a magic incantation required to run code in an application - /// domain other than the current one. + /// Create a new Python module, define a function in it. + /// Unload the domain, load a new one. + /// Make sure the function (and module) still exists. /// - class Proxy : MarshalByRefObject + public void TestClassReference() { - Assembly theAssembly = null; + RunDomainReloadSteps(); + } + + #endregion + + #region Tempary tests - public void InitAssembly(string assemblyPath) + // https://github.com/pythonnet/pythonnet/pull/1074#issuecomment-596139665 + [Test] + public void CrossReleaseBuiltinType() + { + void ExecTest() { - theAssembly = Assembly.LoadFile(System.IO.Path.GetFullPath(assemblyPath)); + try + { + PythonEngine.Initialize(); + var numRef = CreateNumReference(); + Assert.True(numRef.IsAlive); + PythonEngine.Shutdown(); // <- "run" 1 ends + PythonEngine.Initialize(); // <- "run" 2 starts + + GC.Collect(); + GC.WaitForPendingFinalizers(); // <- this will put former `num` into Finalizer queue + Finalizer.Instance.Collect(forceDispose: true); + // ^- this will call PyObject.Dispose, which will call XDecref on `num.Handle`, + // but Python interpreter from "run" 1 is long gone, so it will corrupt memory instead. + Assert.False(numRef.IsAlive); + } + finally + { + PythonEngine.Shutdown(); + } } - public void RunPython() + var errorArgs = new List(); + void ErrorHandler(object sender, Finalizer.ErrorArgs e) { - Console.WriteLine("[Proxy] Entering RunPython"); + errorArgs.Add(e); + } + Finalizer.Instance.ErrorHandler += ErrorHandler; + try + { + for (int i = 0; i < 10; i++) + { + ExecTest(); + } + } + finally + { + Finalizer.Instance.ErrorHandler -= ErrorHandler; + } + Assert.AreEqual(errorArgs.Count, 0); + } - // Call into the new assembly. Will execute Python code - var pythonrunner = theAssembly.GetType("PythonRunner"); - var runPythonMethod = pythonrunner.GetMethod("RunPython"); - runPythonMethod.Invoke(null, new object[] { }); + [Test] + public void CrossReleaseCustomType() + { + void ExecTest() + { + try + { + PythonEngine.Initialize(); + var objRef = CreateConcreateObject(); + Assert.True(objRef.IsAlive); + PythonEngine.Shutdown(); // <- "run" 1 ends + PythonEngine.Initialize(); // <- "run" 2 starts + GC.Collect(); + GC.WaitForPendingFinalizers(); + Finalizer.Instance.Collect(forceDispose: true); + Assert.False(objRef.IsAlive); + } + finally + { + PythonEngine.Shutdown(); + } + } + + var errorArgs = new List(); + void ErrorHandler(object sender, Finalizer.ErrorArgs e) + { + errorArgs.Add(e); + } + Finalizer.Instance.ErrorHandler += ErrorHandler; + try + { + for (int i = 0; i < 10; i++) + { + ExecTest(); + } + } + finally + { + Finalizer.Instance.ErrorHandler -= ErrorHandler; + } + Assert.AreEqual(errorArgs.Count, 0); + } + + private static WeakReference CreateNumReference() + { + var num = 3216757418.ToPython(); + Assert.AreEqual(num.Refcount, 1); + WeakReference numRef = new WeakReference(num, false); + return numRef; + } + + private static WeakReference CreateConcreateObject() + { + var obj = new Domain.MyClass().ToPython(); + Assert.AreEqual(obj.Refcount, 1); + WeakReference numRef = new WeakReference(obj, false); + return numRef; + } + #endregion Tempary tests + + /// + /// This is a magic incantation required to run code in an application + /// domain other than the current one. + /// + class Proxy : MarshalByRefObject + { + public void RunPython() + { + Console.WriteLine("[Proxy] Entering RunPython"); + PythonRunner.RunPython(); Console.WriteLine("[Proxy] Leaving RunPython"); } + + public object Call(string methodName, params object[] args) + { + var pythonrunner = typeof(PythonRunner); + var method = pythonrunner.GetMethod(methodName); + return method.Invoke(null, args); + } + } + + static T CreateInstanceInstanceAndUnwrap(AppDomain domain) + { + Type type = typeof(T); + var theProxy = (T)domain.CreateInstanceAndUnwrap( + type.Assembly.FullName, + type.FullName); + return theProxy; } /// /// Create a domain, run the assembly in it (the RunPython function), /// and unload the domain. /// - static void RunAssemblyAndUnload(Assembly assembly, string assemblyName) + static void RunAssemblyAndUnload(string domainName) { - Console.WriteLine($"[Program.Main] === creating domain for assembly {assembly.FullName}"); - - // Create the domain. Make sure to set PrivateBinPath to a relative - // path from the CWD (namely, 'bin'). - // See https://stackoverflow.com/questions/24760543/createinstanceandunwrap-in-another-domain - var currentDomain = AppDomain.CurrentDomain; - var domainsetup = new AppDomainSetup() - { - ApplicationBase = currentDomain.SetupInformation.ApplicationBase, - ConfigurationFile = currentDomain.SetupInformation.ConfigurationFile, - LoaderOptimization = LoaderOptimization.SingleDomain, - PrivateBinPath = "." - }; - var domain = AppDomain.CreateDomain( - $"My Domain {assemblyName}", - currentDomain.Evidence, - domainsetup); + Console.WriteLine($"[Program.Main] === creating domain {domainName}"); + AppDomain domain = CreateDomain(domainName); // Create a Proxy object in the new domain, where we want the // assembly (and Python .NET) to reside - Type type = typeof(Proxy); - System.IO.Directory.SetCurrentDirectory(AppDomain.CurrentDomain.BaseDirectory); - var theProxy = (Proxy)domain.CreateInstanceAndUnwrap( - type.Assembly.FullName, - type.FullName); + var theProxy = CreateInstanceInstanceAndUnwrap(domain); + theProxy.Call("InitPython", ShutdownMode.Soft); // From now on use the Proxy to call into the new assembly - theProxy.InitAssembly(assemblyName); theProxy.RunPython(); - Console.WriteLine($"[Program.Main] Before Domain Unload on {assembly.FullName}"); + theProxy.Call("ShutdownPython"); + Console.WriteLine($"[Program.Main] Before Domain Unload on {domainName}"); AppDomain.Unload(domain); - Console.WriteLine($"[Program.Main] After Domain Unload on {assembly.FullName}"); + Console.WriteLine($"[Program.Main] After Domain Unload on {domainName}"); // Validate that the assembly does not exist anymore try { Console.WriteLine($"[Program.Main] The Proxy object is valid ({theProxy}). Unexpected domain unload behavior"); + Assert.Fail($"{theProxy} should be invlaid now"); } - catch (Exception) + catch (AppDomainUnloadedException) { Console.WriteLine("[Program.Main] The Proxy object is not valid anymore, domain unload complete."); } } + private static AppDomain CreateDomain(string name) + { + // Create the domain. Make sure to set PrivateBinPath to a relative + // path from the CWD (namely, 'bin'). + // See https://stackoverflow.com/questions/24760543/createinstanceandunwrap-in-another-domain + var currentDomain = AppDomain.CurrentDomain; + var domainsetup = new AppDomainSetup() + { + ApplicationBase = currentDomain.SetupInformation.ApplicationBase, + ConfigurationFile = currentDomain.SetupInformation.ConfigurationFile, + LoaderOptimization = LoaderOptimization.SingleDomain, + PrivateBinPath = "." + }; + var domain = AppDomain.CreateDomain( + $"My Domain {name}", + currentDomain.Evidence, + domainsetup); + return domain; + } + /// /// Resolves the assembly. Why doesn't this just work normally? /// @@ -234,6 +481,141 @@ static Assembly ResolveAssembly(object sender, ResolveEventArgs args) return null; } + + static void RunDomainReloadSteps() where T1 : CrossCaller where T2 : CrossCaller + { + ValueType arg = null; + Type type = typeof(Proxy); + { + AppDomain domain = CreateDomain("test_domain_reload_1"); + try + { + var theProxy = CreateInstanceInstanceAndUnwrap(domain); + theProxy.Call("InitPython", ShutdownMode.Reload); + + var caller = CreateInstanceInstanceAndUnwrap(domain); + arg = caller.Execute(arg); + + theProxy.Call("ShutdownPython"); + } + finally + { + AppDomain.Unload(domain); + } + } + + { + AppDomain domain = CreateDomain("test_domain_reload_2"); + try + { + var theProxy = CreateInstanceInstanceAndUnwrap(domain); + theProxy.Call("InitPython", ShutdownMode.Reload); + + var caller = CreateInstanceInstanceAndUnwrap(domain); + caller.Execute(arg); + theProxy.Call("ShutdownPythonCompletely"); + } + finally + { + AppDomain.Unload(domain); + } + } + if (PythonEngine.DefaultShutdownMode == ShutdownMode.Normal) + { + Assert.IsTrue(PyRuntime.Py_IsInitialized() == 0); + } + } + } + + + // + // The code we'll test. All that really matters is + // using GIL { Python.Exec(pyScript); } + // but the rest is useful for debugging. + // + // What matters in the python code is gc.collect and clr.AddReference. + // + // Note that the language version is 2.0, so no $"foo{bar}" syntax. + // + static class PythonRunner + { + public static void RunPython() + { + AppDomain.CurrentDomain.DomainUnload += OnDomainUnload; + string name = AppDomain.CurrentDomain.FriendlyName; + Console.WriteLine("[{0} in .NET] In PythonRunner.RunPython", name); + using (Py.GIL()) + { + try + { + var pyScript = string.Format("import clr\n" + + "print('[{0} in python] imported clr')\n" + + "clr.AddReference('System')\n" + + "print('[{0} in python] allocated a clr object')\n" + + "import gc\n" + + "gc.collect()\n" + + "print('[{0} in python] collected garbage')\n", + name); + PythonEngine.Exec(pyScript); + } + catch (Exception e) + { + Console.WriteLine(string.Format("[{0} in .NET] Caught exception: {1}", name, e)); + throw; + } + } + } + + + private static IntPtr _state; + + public static void InitPython(ShutdownMode mode) + { + PythonEngine.Initialize(mode: mode); + _state = PythonEngine.BeginAllowThreads(); + } + + public static void ShutdownPython() + { + PythonEngine.EndAllowThreads(_state); + PythonEngine.Shutdown(); + } + + public static void ShutdownPythonCompletely() + { + PythonEngine.EndAllowThreads(_state); + // XXX: Reload mode will reserve clr objects after `Runtime.Shutdown`, + // if it used a another mode(the default mode) in other tests, + // when other tests trying to access these reserved objects, it may cause Domain exception, + // thus it needs to reduct to Soft mode to make sure all clr objects remove from Python. + var defaultMode = PythonEngine.DefaultShutdownMode; + if (defaultMode != ShutdownMode.Reload) + { + PythonEngine.ShutdownMode = defaultMode; + } + PythonEngine.Shutdown(); + } + + static void OnDomainUnload(object sender, EventArgs e) + { + Console.WriteLine(string.Format("[{0} in .NET] unloading", AppDomain.CurrentDomain.FriendlyName)); + } } + } + + +namespace Python.EmbeddingTest.Domain +{ + [Serializable] + public class MyClass + { + public int Property { get; set; } + public int Field; + public void Method() { } + public static void StaticMethod() { } + } +} + + #endif diff --git a/src/embed_tests/TestFinalizer.cs b/src/embed_tests/TestFinalizer.cs index 2d8c996bf..a54bc7a96 100644 --- a/src/embed_tests/TestFinalizer.cs +++ b/src/embed_tests/TestFinalizer.cs @@ -1,6 +1,9 @@ using NUnit.Framework; using Python.Runtime; using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Diagnostics; using System.Linq; using System.Threading; @@ -25,10 +28,22 @@ public void TearDown() PythonEngine.Shutdown(); } - private static void FullGCCollect() + private static bool FullGCCollect() { GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced); - GC.WaitForPendingFinalizers(); + try + { + return GC.WaitForFullGCComplete() == GCNotificationStatus.Succeeded; + } + catch (NotImplementedException) + { + // Some clr runtime didn't implement GC.WaitForFullGCComplete yet. + return false; + } + finally + { + GC.WaitForPendingFinalizers(); + } } [Test] @@ -88,23 +103,33 @@ public void CollectBasicObject() } [Test] + [Ignore("Ignore temporarily")] public void CollectOnShutdown() { - MakeAGarbage(out var shortWeak, out var longWeak); - FullGCCollect(); - var garbage = Finalizer.Instance.GetCollectedObjects(); - Assert.IsNotEmpty(garbage); + IntPtr op = MakeAGarbage(out var shortWeak, out var longWeak); + int hash = shortWeak.Target.GetHashCode(); + List garbage; + if (!FullGCCollect()) + { + Assert.IsTrue(WaitForCollected(op, hash, 10000)); + } + Assert.IsFalse(shortWeak.IsAlive); + garbage = Finalizer.Instance.GetCollectedObjects(); + Assert.IsNotEmpty(garbage, "The garbage object should be collected"); + Assert.IsTrue(garbage.Any(r => ReferenceEquals(r.Target, longWeak.Target)), + "Garbage should contains the collected object"); + PythonEngine.Shutdown(); garbage = Finalizer.Instance.GetCollectedObjects(); Assert.IsEmpty(garbage); } - private static void MakeAGarbage(out WeakReference shortWeak, out WeakReference longWeak) + private static IntPtr MakeAGarbage(out WeakReference shortWeak, out WeakReference longWeak) { PyLong obj = new PyLong(1024); shortWeak = new WeakReference(obj); longWeak = new WeakReference(obj, true); - obj = null; + return obj.Handle; } private static long CompareWithFinalizerOn(PyObject pyCollect, bool enbale) @@ -226,7 +251,7 @@ public void ValidateRefCount() { if (!Finalizer.Instance.RefCountValidationEnabled) { - Assert.Pass("Only run with FINALIZER_CHECK"); + Assert.Ignore("Only run with FINALIZER_CHECK"); } IntPtr ptr = IntPtr.Zero; bool called = false; @@ -261,5 +286,28 @@ private static IntPtr CreateStringGarbage() return s1.Handle; } + private static bool WaitForCollected(IntPtr op, int hash, int milliseconds) + { + var stopwatch = Stopwatch.StartNew(); + do + { + var garbage = Finalizer.Instance.GetCollectedObjects(); + foreach (var item in garbage) + { + // The validation is not 100% precise, + // but it's rare that two conditions satisfied but they're still not the same object. + if (item.Target.GetHashCode() != hash) + { + continue; + } + var obj = (IPyDisposable)item.Target; + if (obj.GetTrackedHandles().Contains(op)) + { + return true; + } + } + } while (stopwatch.ElapsedMilliseconds < milliseconds); + return false; + } } } diff --git a/src/embed_tests/TestPyScope.cs b/src/embed_tests/TestPyScope.cs index 701e698ec..a94b8ce28 100644 --- a/src/embed_tests/TestPyScope.cs +++ b/src/embed_tests/TestPyScope.cs @@ -338,8 +338,8 @@ public void TestThread() //add function to the scope //can be call many times, more efficient than ast ps.Exec( - "import threading\n" + - "lock = threading.Lock()\n" + + "import threading\n"+ + "lock = threading.Lock()\n"+ "def update():\n" + " global res, th_cnt\n" + " with lock:\n" + diff --git a/src/embed_tests/TestRuntime.cs b/src/embed_tests/TestRuntime.cs index 4129d3df3..38878205c 100644 --- a/src/embed_tests/TestRuntime.cs +++ b/src/embed_tests/TestRuntime.cs @@ -28,16 +28,22 @@ public static void PlatformCache() { Runtime.Runtime.Initialize(); - Assert.That(Runtime.Runtime.Machine, Is.Not.EqualTo(MachineType.Other)); - Assert.That(Runtime.Runtime.OperatingSystem, Is.Not.EqualTo(OperatingSystemType.Other)); + Assert.That(NativeCodePageHelper.Machine, Is.Not.EqualTo(MachineType.Other)); + Assert.That(!string.IsNullOrEmpty(NativeCodePageHelper.MachineName)); - // Don't shut down the runtime: if the python engine was initialized - // but not shut down by another test, we'd end up in a bad state. - } + Assert.That(NativeCodePageHelper.OperatingSystem, Is.Not.EqualTo(OperatingSystemType.Other)); + Assert.That(!string.IsNullOrEmpty(NativeCodePageHelper.OperatingSystemName)); + + Runtime.Runtime.Shutdown(); + } [Test] public static void Py_IsInitializedValue() { + if (Runtime.Runtime.Py_IsInitialized() == 1) + { + Runtime.Runtime.PyGILState_Ensure(); + } Runtime.Runtime.Py_Finalize(); Assert.AreEqual(0, Runtime.Runtime.Py_IsInitialized()); Runtime.Runtime.Py_Initialize(); diff --git a/src/embed_tests/TestTypeManager.cs b/src/embed_tests/TestTypeManager.cs index 931c44236..43155e1bf 100644 --- a/src/embed_tests/TestTypeManager.cs +++ b/src/embed_tests/TestTypeManager.cs @@ -1,5 +1,6 @@ using NUnit.Framework; using Python.Runtime; +using Python.Runtime.Platform; using System.Runtime.InteropServices; namespace Python.EmbeddingTest @@ -15,22 +16,21 @@ public static void Init() [TearDown] public static void Fini() { - // Don't shut down the runtime: if the python engine was initialized - // but not shut down by another test, we'd end up in a bad state. + Runtime.Runtime.Shutdown(); } [Test] public static void TestNativeCode() { - Assert.That(() => { var _ = TypeManager.NativeCode.Active; }, Throws.Nothing); - Assert.That(TypeManager.NativeCode.Active.Code.Length, Is.GreaterThan(0)); + Assert.That(() => { var _ = NativeCodePageHelper.NativeCode.Active; }, Throws.Nothing); + Assert.That(NativeCodePageHelper.NativeCode.Active.Code.Length, Is.GreaterThan(0)); } [Test] public static void TestMemoryMapping() { - Assert.That(() => { var _ = TypeManager.CreateMemoryMapper(); }, Throws.Nothing); - var mapper = TypeManager.CreateMemoryMapper(); + Assert.That(() => { var _ = NativeCodePageHelper.CreateMemoryMapper(); }, Throws.Nothing); + var mapper = NativeCodePageHelper.CreateMemoryMapper(); // Allocate a read-write page. int len = 12; diff --git a/src/embed_tests/pyimport.cs b/src/embed_tests/pyimport.cs index d78b030ea..ebb4fabd0 100644 --- a/src/embed_tests/pyimport.cs +++ b/src/embed_tests/pyimport.cs @@ -40,6 +40,7 @@ public void SetUp() IntPtr str = Runtime.Runtime.PyString_FromString(testPath); BorrowedReference path = Runtime.Runtime.PySys_GetObject("path"); Runtime.Runtime.PyList_Append(path, str); + Runtime.Runtime.XDecref(str); } [TearDown] diff --git a/src/embed_tests/pyinitialize.cs b/src/embed_tests/pyinitialize.cs index ea1d8d023..c774680dd 100644 --- a/src/embed_tests/pyinitialize.cs +++ b/src/embed_tests/pyinitialize.cs @@ -24,9 +24,11 @@ public static void StartAndStopTwice() public static void LoadDefaultArgs() { using (new PythonEngine()) - using (var argv = new PyList(Runtime.Runtime.PySys_GetObject("argv"))) { - Assert.AreNotEqual(0, argv.Length()); + using(var argv = new PyList(Runtime.Runtime.PySys_GetObject("argv"))) + { + Assert.AreNotEqual(0, argv.Length()); + } } } @@ -35,10 +37,12 @@ public static void LoadSpecificArgs() { var args = new[] { "test1", "test2" }; using (new PythonEngine(args)) - using (var argv = new PyList(Runtime.Runtime.PySys_GetObject("argv"))) { - Assert.AreEqual(args[0], argv[0].ToString()); - Assert.AreEqual(args[1], argv[1].ToString()); + using (var argv = new PyList(Runtime.Runtime.PySys_GetObject("argv"))) + { + Assert.AreEqual(args[0], argv[0].ToString()); + Assert.AreEqual(args[1], argv[1].ToString()); + } } } @@ -134,5 +138,46 @@ public void ShutdownHandlers() // Wrong: (4 * 2) + 1 + 1 + 1 = 11 Assert.That(shutdown_count, Is.EqualTo(12)); } + + [Test] + public static void TestRunExitFuncs() + { + if (Runtime.Runtime.GetDefaultShutdownMode() == ShutdownMode.Normal) + { + // If the runtime using the normal mode, + // callback registered by atexit will be called after we release the clr information, + // thus there's no chance we can check it here. + Assert.Ignore("Skip on normal mode"); + } + Runtime.Runtime.Initialize(); + PyObject atexit; + try + { + atexit = Py.Import("atexit"); + } + catch (PythonException e) + { + string msg = e.ToString(); + Runtime.Runtime.Shutdown(); + + if (e.IsMatches(Exceptions.ImportError)) + { + Assert.Ignore("no atexit module"); + } + else + { + Assert.Fail(msg); + } + return; + } + bool called = false; + Action callback = () => + { + called = true; + }; + atexit.InvokeMethod("register", callback.ToPython()); + Runtime.Runtime.Shutdown(); + Assert.True(called); + } } } diff --git a/src/runtime/BorrowedReference.cs b/src/runtime/BorrowedReference.cs index a3bf29056..8ae382e77 100644 --- a/src/runtime/BorrowedReference.cs +++ b/src/runtime/BorrowedReference.cs @@ -10,6 +10,7 @@ readonly ref struct BorrowedReference readonly IntPtr pointer; public bool IsNull => this.pointer == IntPtr.Zero; + /// Gets a raw pointer to the Python object public IntPtr DangerousGetAddress() => this.IsNull ? throw new NullReferenceException() : this.pointer; diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index 8afa60f4f..08dc1d860 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -1,185 +1,188 @@ - - - - Debug - AnyCPU - {097B4AC0-74E9-4C58-BCF8-C69746EC8271} - Library - Python.Runtime - Python.Runtime - bin\Python.Runtime.xml - bin\ - v4.0 - - 1591 - ..\..\ - $(SolutionDir)\bin\ - Properties - 7.3 - true - false - ..\pythonnet.snk - - - - - - PYTHON2;PYTHON27;UCS4 - true - pdbonly - - - PYTHON3;PYTHON38;UCS4 - true - pdbonly - - - true - PYTHON2;PYTHON27;UCS4;TRACE;DEBUG - false - full - - - true - PYTHON3;PYTHON38;UCS4;TRACE;DEBUG - false - full - - - PYTHON2;PYTHON27;UCS2 - true - pdbonly - - - PYTHON3;PYTHON38;UCS2 - true - pdbonly - - - true - PYTHON2;PYTHON27;UCS2;TRACE;DEBUG - false - full - - - true - PYTHON3;PYTHON38;UCS2;TRACE;DEBUG - false - full - - - - - - - - - - - - - - Properties\SharedAssemblyInfo.cs - - - - + + + + Debug + AnyCPU + {097B4AC0-74E9-4C58-BCF8-C69746EC8271} + Library + Python.Runtime + Python.Runtime + bin\Python.Runtime.xml + bin\ + v4.0 + + 1591 + ..\..\ + $(SolutionDir)\bin\ + Properties + 7.3 + true + false + ..\pythonnet.snk + + + + + + PYTHON2;PYTHON27;UCS4 + true + pdbonly + + + PYTHON3;PYTHON38;UCS4 + true + pdbonly + + + true + PYTHON2;PYTHON27;UCS4;TRACE;DEBUG + false + full + + + true + PYTHON3;PYTHON38;UCS4;TRACE;DEBUG + false + full + + + PYTHON2;PYTHON27;UCS2 + true + pdbonly + + + PYTHON3;PYTHON38;UCS2 + true + pdbonly + + + true + PYTHON2;PYTHON27;UCS2;TRACE;DEBUG + false + full + + + true + PYTHON3;PYTHON38;UCS2;TRACE;DEBUG + false + full + + + + + + + + + + + + + + Properties\SharedAssemblyInfo.cs + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - clr.py - - - - - $(TargetPath) - $(TargetDir)$(TargetName).pdb - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + clr.py + + + + + $(TargetPath) + $(TargetDir)$(TargetName).pdb + + + + + diff --git a/src/runtime/arrayobject.cs b/src/runtime/arrayobject.cs index 364d91969..0db84dd90 100644 --- a/src/runtime/arrayobject.cs +++ b/src/runtime/arrayobject.cs @@ -8,6 +8,7 @@ namespace Python.Runtime /// the same as a ClassObject, except that it provides sequence semantics /// to support natural array usage (indexing) from Python. /// + [Serializable] internal class ArrayObject : ClassBase { internal ArrayObject(Type tp) : base(tp) diff --git a/src/runtime/classbase.cs b/src/runtime/classbase.cs index 972380928..66153fbe1 100644 --- a/src/runtime/classbase.cs +++ b/src/runtime/classbase.cs @@ -1,6 +1,8 @@ using System; using System.Collections; +using System.Diagnostics; using System.Runtime.InteropServices; +using System.Runtime.Serialization; namespace Python.Runtime { @@ -12,6 +14,7 @@ namespace Python.Runtime /// concrete subclasses provide slot implementations appropriate for /// each variety of reflected type. /// + [Serializable] internal class ClassBase : ManagedType { internal Indexer indexer; @@ -28,13 +31,6 @@ internal virtual bool CanSubclass() return !type.IsEnum; } - /// - /// Implements __init__ for reflected classes and value types. - /// - public static int tp_init(IntPtr ob, IntPtr args, IntPtr kw) - { - return 0; - } /// /// Default implementation of [] semantics for reflected types. @@ -292,15 +288,44 @@ public static IntPtr tp_repr(IntPtr ob) public static void tp_dealloc(IntPtr ob) { ManagedType self = GetManagedObject(ob); - IntPtr dict = Marshal.ReadIntPtr(ob, ObjectOffset.TypeDictOffset(self.tpHandle)); - if (dict != IntPtr.Zero) - { - Runtime.XDecref(dict); - } + tp_clear(ob); Runtime.PyObject_GC_UnTrack(self.pyHandle); Runtime.PyObject_GC_Del(self.pyHandle); - Runtime.XDecref(self.tpHandle); - self.gcHandle.Free(); + self.FreeGCHandle(); + } + + public static int tp_clear(IntPtr ob) + { + ManagedType self = GetManagedObject(ob); + if (!self.IsTypeObject()) + { + ClearObjectDict(ob); + } + self.tpHandle = IntPtr.Zero; + return 0; + } + + protected override void OnSave(InterDomainContext context) + { + base.OnSave(context); + if (pyHandle != tpHandle) + { + IntPtr dict = GetObjectDict(pyHandle); + Runtime.XIncref(dict); + context.Storage.AddValue("dict", dict); + } + } + + protected override void OnLoad(InterDomainContext context) + { + base.OnLoad(context); + if (pyHandle != tpHandle) + { + IntPtr dict = context.Storage.GetValue("dict"); + SetObjectDict(pyHandle, dict); + } + gcHandle = AllocGCHandle(); + Marshal.WriteIntPtr(pyHandle, TypeOffset.magic(), (IntPtr)gcHandle); } diff --git a/src/runtime/classderived.cs b/src/runtime/classderived.cs index af16b1359..e55e89240 100644 --- a/src/runtime/classderived.cs +++ b/src/runtime/classderived.cs @@ -22,6 +22,7 @@ public interface IPythonDerivedType { } + [Serializable] internal class ClassDerivedObject : ClassObject { private static Dictionary assemblyBuilders; @@ -99,6 +100,10 @@ internal static IntPtr ToPython(IPythonDerivedType obj) // collected while Python still has a reference to it. if (Runtime.Refcount(self.pyHandle) == 1) { + +#if PYTHON_WITH_PYDEBUG + Runtime._Py_NewReference(self.pyHandle); +#endif GCHandle gc = GCHandle.Alloc(self, GCHandleType.Normal); Marshal.WriteIntPtr(self.pyHandle, ObjectOffset.magic(self.tpHandle), (IntPtr)gc); self.gcHandle.Free(); @@ -130,7 +135,7 @@ internal static Type CreateDerivedType(string name, if (null == assemblyName) { - assemblyName = Assembly.GetExecutingAssembly().FullName; + assemblyName = "Python.Runtime.Dynamic"; } ModuleBuilder moduleBuilder = GetModuleBuilder(assemblyName, moduleName); diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index 08918adc1..15f3d821d 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -4,6 +4,7 @@ using System.Reflection; using System.Runtime.InteropServices; using System.Security; +using System.Linq; namespace Python.Runtime { @@ -18,7 +19,7 @@ namespace Python.Runtime internal class ClassManager { private static Dictionary cache; - private static Type dtype; + private static readonly Type dtype; private ClassManager() { @@ -26,7 +27,6 @@ private ClassManager() static ClassManager() { - cache = new Dictionary(128); // SEE: https://msdn.microsoft.com/en-us/library/96b1ayy4(v=vs.100).aspx // ""All delegates inherit from MulticastDelegate, which inherits from Delegate."" // Was Delegate, which caused a null MethodInfo returned from GetMethode("Invoke") @@ -39,6 +39,76 @@ public static void Reset() cache = new Dictionary(128); } + internal static void DisposePythonWrappersForClrTypes() + { + var visited = new HashSet(); + var visitedHandle = GCHandle.Alloc(visited); + var visitedPtr = (IntPtr)visitedHandle; + try + { + foreach (var cls in cache.Values) + { + // XXX: Force to release instance's managed resources + // but not dealloc itself immediately. + // These managed resources should preserve vacant shells + // since others may still referencing it. + cls.CallTypeTraverse(TraverseTypeClear, visitedPtr); + cls.CallTypeClear(); + cls.DecrRefCount(); + } + } + finally + { + visitedHandle.Free(); + } + cache.Clear(); + } + + private static int TraverseTypeClear(IntPtr ob, IntPtr arg) + { + var visited = (HashSet)GCHandle.FromIntPtr(arg).Target; + if (!visited.Add(ob)) + { + return 0; + } + var clrObj = ManagedType.GetManagedObject(ob); + if (clrObj != null) + { + clrObj.CallTypeTraverse(TraverseTypeClear, arg); + clrObj.CallTypeClear(); + } + return 0; + } + + internal static void SaveRuntimeData(RuntimeDataStorage storage) + { + var contexts = storage.AddValue("contexts", + new Dictionary()); + storage.AddValue("cache", cache); + foreach (var cls in cache.Values) + { + // This incref is for cache to hold the cls, + // thus no need for decreasing it at RestoreRuntimeData. + Runtime.XIncref(cls.pyHandle); + var context = contexts[cls.pyHandle] = new InterDomainContext(); + cls.Save(context); + } + } + + internal static Dictionary RestoreRuntimeData(RuntimeDataStorage storage) + { + cache = storage.GetValue>("cache"); + var contexts = storage.GetValue >("contexts"); + var loadedObjs = new Dictionary(); + foreach (var cls in cache.Values) + { + var context = contexts[cls.pyHandle]; + cls.Load(context); + loadedObjs.Add(cls, context); + } + return loadedObjs; + } + /// /// Return the ClassBase-derived instance that implements a particular /// reflected managed type, creating it if it doesn't yet exist. @@ -134,7 +204,6 @@ private static void InitClassBase(Type type, ClassBase impl) IntPtr tp = TypeManager.GetTypeHandle(impl, type); - impl.tpHandle = tp; // Finally, initialize the class __dict__ and return the object. IntPtr dict = Marshal.ReadIntPtr(tp, TypeOffset.tp_dict); @@ -146,6 +215,8 @@ private static void InitClassBase(Type type, ClassBase impl) var item = (ManagedType)iter.Value; var name = (string)iter.Key; Runtime.PyDict_SetItemString(dict, name, item.pyHandle); + // Decref the item now that it's been used. + item.DecrRefCount(); } // If class has constructors, generate an __doc__ attribute. @@ -180,6 +251,7 @@ private static void InitClassBase(Type type, ClassBase impl) // TODO: deprecate __overloads__ soon... Runtime.PyDict_SetItemString(dict, "__overloads__", ctors.pyHandle); Runtime.PyDict_SetItemString(dict, "Overloads", ctors.pyHandle); + ctors.DecrRefCount(); } // don't generate the docstring if one was already set from a DocStringAttribute. @@ -195,7 +267,7 @@ private static void InitClassBase(Type type, ClassBase impl) private static ClassInfo GetClassInfo(Type type) { - var ci = new ClassInfo(type); + var ci = new ClassInfo(); var methods = new Hashtable(); ArrayList list; MethodInfo meth; @@ -410,18 +482,22 @@ private static ClassInfo GetClassInfo(Type type) return ci; } - } - - - internal class ClassInfo - { - public Indexer indexer; - public Hashtable members; - - internal ClassInfo(Type t) + + /// + /// This class owns references to PyObjects in the `members` member. + /// The caller has responsibility to DECREF them. + /// + private class ClassInfo { - members = new Hashtable(); - indexer = null; + public Indexer indexer; + public Hashtable members; + + internal ClassInfo() + { + members = new Hashtable(); + indexer = null; + } } } + } diff --git a/src/runtime/classobject.cs b/src/runtime/classobject.cs index d7624ed6e..18816781f 100644 --- a/src/runtime/classobject.cs +++ b/src/runtime/classobject.cs @@ -9,6 +9,7 @@ namespace Python.Runtime /// Python type objects. Each of those type objects is associated with /// an instance of ClassObject, which provides its implementation. /// + [Serializable] internal class ClassObject : ClassBase { internal ConstructorBinder binder; diff --git a/src/runtime/clrobject.cs b/src/runtime/clrobject.cs index 5c7ad7891..0b62fecba 100644 --- a/src/runtime/clrobject.cs +++ b/src/runtime/clrobject.cs @@ -3,12 +3,14 @@ namespace Python.Runtime { + [Serializable] internal class CLRObject : ManagedType { internal object inst; internal CLRObject(object ob, IntPtr tp) { + System.Diagnostics.Debug.Assert(tp != IntPtr.Zero); IntPtr py = Runtime.PyType_GenericAlloc(tp, 0); long flags = Util.ReadCLong(tp, TypeOffset.tp_flags); @@ -22,11 +24,10 @@ internal CLRObject(object ob, IntPtr tp) } } - GCHandle gc = GCHandle.Alloc(this); + GCHandle gc = AllocGCHandle(TrackTypes.Wrapper); Marshal.WriteIntPtr(py, ObjectOffset.magic(tp), (IntPtr)gc); tpHandle = tp; pyHandle = py; - gcHandle = gc; inst = ob; // Fix the BaseException args (and __cause__ in case of Python 3) @@ -34,6 +35,9 @@ internal CLRObject(object ob, IntPtr tp) Exceptions.SetArgsAndCause(py); } + protected CLRObject() + { + } static CLRObject GetInstance(object ob, IntPtr pyType) { @@ -68,5 +72,30 @@ internal static IntPtr GetInstHandle(object ob) CLRObject co = GetInstance(ob); return co.pyHandle; } + + internal static CLRObject Restore(object ob, IntPtr pyHandle, InterDomainContext context) + { + CLRObject co = new CLRObject() + { + inst = ob, + pyHandle = pyHandle, + tpHandle = Runtime.PyObject_TYPE(pyHandle) + }; + co.Load(context); + return co; + } + + protected override void OnSave(InterDomainContext context) + { + base.OnSave(context); + Runtime.XIncref(pyHandle); + } + + protected override void OnLoad(InterDomainContext context) + { + base.OnLoad(context); + GCHandle gc = AllocGCHandle(TrackTypes.Wrapper); + Marshal.WriteIntPtr(pyHandle, ObjectOffset.magic(tpHandle), (IntPtr)gc); + } } } diff --git a/src/runtime/constructorbinder.cs b/src/runtime/constructorbinder.cs index 973707f02..0cda3a3d9 100644 --- a/src/runtime/constructorbinder.cs +++ b/src/runtime/constructorbinder.cs @@ -11,6 +11,7 @@ namespace Python.Runtime /// standard MethodBinder because of a difference in invoking constructors /// using reflection (which is seems to be a CLR bug). /// + [Serializable] internal class ConstructorBinder : MethodBinder { private Type _containingType; diff --git a/src/runtime/constructorbinding.cs b/src/runtime/constructorbinding.cs index 3908628b9..0c81c0a93 100644 --- a/src/runtime/constructorbinding.cs +++ b/src/runtime/constructorbinding.cs @@ -19,18 +19,20 @@ namespace Python.Runtime /// and creating the BoundContructor object which contains ContructorInfo object. /// 3) In tp_call, if ctorInfo is not null, ctorBinder.InvokeRaw() is called. /// + [Serializable] internal class ConstructorBinding : ExtensionType { private Type type; // The managed Type being wrapped in a ClassObject private IntPtr pyTypeHndl; // The python type tells GetInstHandle which Type to create. private ConstructorBinder ctorBinder; + + [NonSerialized] private IntPtr repr; public ConstructorBinding(Type type, IntPtr pyTypeHndl, ConstructorBinder ctorBinder) { this.type = type; - Runtime.XIncref(pyTypeHndl); - this.pyTypeHndl = pyTypeHndl; + this.pyTypeHndl = pyTypeHndl; // steal a type reference this.ctorBinder = ctorBinder; repr = IntPtr.Zero; } @@ -144,8 +146,25 @@ public static IntPtr tp_repr(IntPtr ob) { var self = (ConstructorBinding)GetManagedObject(ob); Runtime.XDecref(self.repr); - Runtime.XDecref(self.pyTypeHndl); - ExtensionType.FinalizeObject(self); + self.Dealloc(); + } + + public static int tp_clear(IntPtr ob) + { + var self = (ConstructorBinding)GetManagedObject(ob); + Runtime.Py_CLEAR(ref self.repr); + return 0; + } + + public static int tp_traverse(IntPtr ob, IntPtr visit, IntPtr arg) + { + var self = (ConstructorBinding)GetManagedObject(ob); + int res = PyVisit(self.pyTypeHndl, visit, arg); + if (res != 0) return res; + + res = PyVisit(self.repr, visit, arg); + if (res != 0) return res; + return 0; } } @@ -157,6 +176,7 @@ public static IntPtr tp_repr(IntPtr ob) /// An earlier implementation hung the __call__ on the ContructorBinding class and /// returned an Incref()ed self.pyHandle from the __get__ function. /// + [Serializable] internal class BoundContructor : ExtensionType { private Type type; // The managed Type being wrapped in a ClassObject @@ -168,8 +188,7 @@ internal class BoundContructor : ExtensionType public BoundContructor(Type type, IntPtr pyTypeHndl, ConstructorBinder ctorBinder, ConstructorInfo ci) { this.type = type; - Runtime.XIncref(pyTypeHndl); - this.pyTypeHndl = pyTypeHndl; + this.pyTypeHndl = pyTypeHndl; // steal a type reference this.ctorBinder = ctorBinder; ctorInfo = ci; repr = IntPtr.Zero; @@ -230,8 +249,25 @@ public static IntPtr tp_repr(IntPtr ob) { var self = (BoundContructor)GetManagedObject(ob); Runtime.XDecref(self.repr); - Runtime.XDecref(self.pyTypeHndl); - FinalizeObject(self); + self.Dealloc(); + } + + public static int tp_clear(IntPtr ob) + { + var self = (BoundContructor)GetManagedObject(ob); + Runtime.Py_CLEAR(ref self.repr); + return 0; + } + + public static int tp_traverse(IntPtr ob, IntPtr visit, IntPtr arg) + { + var self = (BoundContructor)GetManagedObject(ob); + int res = PyVisit(self.pyTypeHndl, visit, arg); + if (res != 0) return res; + + res = PyVisit(self.repr, visit, arg); + if (res != 0) return res; + return 0; } } } diff --git a/src/runtime/delegateobject.cs b/src/runtime/delegateobject.cs index c9aad9898..c5078740f 100644 --- a/src/runtime/delegateobject.cs +++ b/src/runtime/delegateobject.cs @@ -8,6 +8,7 @@ namespace Python.Runtime /// Each of those type objects is associated an instance of this class, /// which provides its implementation. /// + [Serializable] internal class DelegateObject : ClassBase { private MethodBinder binder; diff --git a/src/runtime/eventbinding.cs b/src/runtime/eventbinding.cs index b8b4c82ad..581095185 100644 --- a/src/runtime/eventbinding.cs +++ b/src/runtime/eventbinding.cs @@ -5,6 +5,7 @@ namespace Python.Runtime /// /// Implements a Python event binding type, similar to a method binding. /// + [Serializable] internal class EventBinding : ExtensionType { private EventObject e; @@ -118,7 +119,14 @@ public static IntPtr tp_repr(IntPtr ob) { var self = (EventBinding)GetManagedObject(ob); Runtime.XDecref(self.target); - ExtensionType.FinalizeObject(self); + self.Dealloc(); + } + + public static int tp_clear(IntPtr ob) + { + var self = (EventBinding)GetManagedObject(ob); + Runtime.Py_CLEAR(ref self.target); + return 0; } } } diff --git a/src/runtime/eventobject.cs b/src/runtime/eventobject.cs index 5f18c4609..0f2796a14 100644 --- a/src/runtime/eventobject.cs +++ b/src/runtime/eventobject.cs @@ -7,6 +7,7 @@ namespace Python.Runtime /// /// Implements a Python descriptor type that provides access to CLR events. /// + [Serializable] internal class EventObject : ExtensionType { internal string name; @@ -202,7 +203,7 @@ public static IntPtr tp_repr(IntPtr ob) { Runtime.XDecref(self.unbound.pyHandle); } - FinalizeObject(self); + self.Dealloc(); } } diff --git a/src/runtime/exceptions.cs b/src/runtime/exceptions.cs index f4cb519a6..58506bfbb 100644 --- a/src/runtime/exceptions.cs +++ b/src/runtime/exceptions.cs @@ -15,6 +15,7 @@ namespace Python.Runtime /// it subclasses System.Object. Instead TypeManager.CreateType() uses /// Python's exception.Exception class as base class for System.Exception. /// + [Serializable] internal class ExceptionClassObject : ClassObject { internal ExceptionClassObject(Type tp) : base(tp) @@ -89,15 +90,11 @@ internal static Exception ToException(IntPtr ob) /// /// Readability of the Exceptions class improvements as we look toward version 2.7 ... /// - public class Exceptions + public static class Exceptions { internal static IntPtr warnings_module; internal static IntPtr exceptions_module; - private Exceptions() - { - } - /// /// Initialization performed on startup of the Python runtime. /// @@ -132,21 +129,23 @@ internal static void Initialize() /// internal static void Shutdown() { - if (Runtime.Py_IsInitialized() != 0) + if (Runtime.Py_IsInitialized() == 0) + { + return; + } + Type type = typeof(Exceptions); + foreach (FieldInfo fi in type.GetFields(BindingFlags.Public | BindingFlags.Static)) { - Type type = typeof(Exceptions); - foreach (FieldInfo fi in type.GetFields(BindingFlags.Public | BindingFlags.Static)) + var op = (IntPtr)fi.GetValue(type); + if (op == IntPtr.Zero) { - var op = (IntPtr)fi.GetValue(type); - if (op != IntPtr.Zero) - { - Runtime.XDecref(op); - } + continue; } - Runtime.XDecref(exceptions_module); - Runtime.PyObject_HasAttrString(warnings_module, "xx"); - Runtime.XDecref(warnings_module); + Runtime.XDecref(op); + fi.SetValue(null, IntPtr.Zero); } + Runtime.Py_CLEAR(ref exceptions_module); + Runtime.Py_CLEAR(ref warnings_module); } /// diff --git a/src/runtime/extensiontype.cs b/src/runtime/extensiontype.cs index 6585180c1..a5f0f1219 100644 --- a/src/runtime/extensiontype.cs +++ b/src/runtime/extensiontype.cs @@ -8,6 +8,7 @@ namespace Python.Runtime /// type object, such as the types that represent CLR methods, fields, /// etc. Instances implemented by this class do not support sub-typing. /// + [Serializable] internal abstract class ExtensionType : ManagedType { public ExtensionType() @@ -28,20 +29,24 @@ public ExtensionType() IntPtr py = Runtime.PyType_GenericAlloc(tp, 0); - GCHandle gc = GCHandle.Alloc(this); - Marshal.WriteIntPtr(py, ObjectOffset.magic(tp), (IntPtr)gc); + // Steals a ref to tpHandle. + tpHandle = tp; + pyHandle = py; + + SetupGc(); + } + + void SetupGc () + { + GCHandle gc = AllocGCHandle(TrackTypes.Extension); + Marshal.WriteIntPtr(pyHandle, ObjectOffset.magic(tpHandle), (IntPtr)gc); // We have to support gc because the type machinery makes it very // hard not to - but we really don't have a need for it in most // concrete extension types, so untrack the object to save calls // from Python into the managed runtime that are pure overhead. - Runtime.PyObject_GC_UnTrack(py); - - // Steals a ref to tpHandle. - tpHandle = tp; - pyHandle = py; - gcHandle = gc; + Runtime.PyObject_GC_UnTrack(pyHandle); } @@ -50,11 +55,16 @@ public ExtensionType() /// public static void FinalizeObject(ManagedType self) { + ClearObjectDict(self.pyHandle); Runtime.PyObject_GC_Del(self.pyHandle); // Not necessary for decref of `tpHandle`. - self.gcHandle.Free(); + self.FreeGCHandle(); } + protected void Dealloc() + { + FinalizeObject(this); + } /// /// Type __setattr__ implementation. @@ -89,8 +99,14 @@ public static void tp_dealloc(IntPtr ob) { // Clean up a Python instance of this extension type. This // frees the allocated Python object and decrefs the type. - ManagedType self = GetManagedObject(ob); - FinalizeObject(self); + var self = (ExtensionType)GetManagedObject(ob); + self.Dealloc(); + } + + protected override void OnLoad(InterDomainContext context) + { + base.OnLoad(context); + SetupGc(); } } } diff --git a/src/runtime/fieldobject.cs b/src/runtime/fieldobject.cs index 7c9a466d5..86b93dd1b 100644 --- a/src/runtime/fieldobject.cs +++ b/src/runtime/fieldobject.cs @@ -6,6 +6,7 @@ namespace Python.Runtime /// /// Implements a Python descriptor type that provides access to CLR fields. /// + [Serializable] internal class FieldObject : ExtensionType { private FieldInfo info; @@ -55,6 +56,11 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) try { var co = (CLRObject)GetManagedObject(ob); + if (co == null) + { + Exceptions.SetError(Exceptions.TypeError, "instance is not a clr object"); + return IntPtr.Zero; + } result = info.GetValue(co.inst); return Converter.ToPython(result, info.FieldType); } @@ -115,6 +121,11 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) if (!is_static) { var co = (CLRObject)GetManagedObject(ob); + if (co == null) + { + Exceptions.SetError(Exceptions.TypeError, "instance is not a clr object"); + return -1; + } info.SetValue(co.inst, newval); } else diff --git a/src/runtime/finalizer.cs b/src/runtime/finalizer.cs index ba562cc26..70b69345b 100644 --- a/src/runtime/finalizer.cs +++ b/src/runtime/finalizer.cs @@ -57,7 +57,7 @@ public IncorrectRefCountException(IntPtr ptr) IntPtr pyname = Runtime.PyObject_Unicode(PyPtr); string name = Runtime.GetManagedString(pyname); Runtime.XDecref(pyname); - _message = $"{name} may has a incorrect ref count"; + _message = $"<{name}> may has a incorrect ref count"; } } diff --git a/src/runtime/generictype.cs b/src/runtime/generictype.cs index eeae801d2..76d2e9a5d 100644 --- a/src/runtime/generictype.cs +++ b/src/runtime/generictype.cs @@ -8,6 +8,7 @@ namespace Python.Runtime /// generic types. Both are essentially factories for creating closed /// types based on the required generic type parameters. /// + [Serializable] internal class GenericType : ClassBase { internal GenericType(Type tp) : base(tp) diff --git a/src/runtime/genericutil.cs b/src/runtime/genericutil.cs index 3a230e12c..df78d9899 100644 --- a/src/runtime/genericutil.cs +++ b/src/runtime/genericutil.cs @@ -16,11 +16,6 @@ private GenericUtil() { } - static GenericUtil() - { - mapping = new Dictionary>>(); - } - public static void Reset() { mapping = new Dictionary>>(); diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index 96a8b7ebe..8cf57c85d 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -7,7 +7,7 @@ namespace Python.Runtime /// /// Implements the "import hook" used to integrate Python with the CLR. /// - internal class ImportHook + internal static class ImportHook { private static IntPtr py_import; private static CLRModule root; @@ -122,6 +122,24 @@ internal static void Shutdown() CLRModule.Reset(); } + internal static void SaveRuntimeData(RuntimeDataStorage storage) + { + // Increment the reference counts here so that the objects don't + // get freed in Shutdown. + Runtime.XIncref(py_clr_module); + Runtime.XIncref(root.pyHandle); + storage.AddValue("py_clr_module", py_clr_module); + storage.AddValue("root", root.pyHandle); + } + + internal static void RestoreRuntimeData(RuntimeDataStorage storage) + { + InitImport(); + storage.GetValue("py_clr_module", out py_clr_module); + var rootHandle = storage.GetValue("root"); + root = (CLRModule)ManagedType.GetManagedObject(rootHandle); + } + /// /// Return the clr python module (new reference) /// diff --git a/src/runtime/indexer.cs b/src/runtime/indexer.cs index 71f7e7aa1..0772b57c6 100644 --- a/src/runtime/indexer.cs +++ b/src/runtime/indexer.cs @@ -6,6 +6,7 @@ namespace Python.Runtime /// /// Bundles the information required to support an indexer property. /// + [Serializable] internal class Indexer { public MethodBinder GetterBinder; diff --git a/src/runtime/interfaceobject.cs b/src/runtime/interfaceobject.cs index 74396f50c..a2fa86479 100644 --- a/src/runtime/interfaceobject.cs +++ b/src/runtime/interfaceobject.cs @@ -10,6 +10,7 @@ namespace Python.Runtime /// Each of those type objects is associated with an instance of this /// class, which provides the implementation for the Python type. /// + [Serializable] internal class InterfaceObject : ClassBase { internal ConstructorInfo ctor; diff --git a/src/runtime/interop.cs b/src/runtime/interop.cs index 95f3e5b9f..1caabab17 100644 --- a/src/runtime/interop.cs +++ b/src/runtime/interop.cs @@ -5,7 +5,6 @@ using System.Runtime.InteropServices; using System.Reflection; using System.Text; -using System.Collections.Generic; namespace Python.Runtime { @@ -69,39 +68,83 @@ public ModulePropertyAttribute() } } + internal static partial class TypeOffset + { + static TypeOffset() + { + Type type = typeof(TypeOffset); + FieldInfo[] fields = type.GetFields(); + int size = IntPtr.Size; + for (int i = 0; i < fields.Length; i++) + { + int offset = i * size; + FieldInfo fi = fields[i]; + fi.SetValue(null, offset); + } + } + + public static int magic() => ManagedDataOffsets.Magic; + } + internal static class ManagedDataOffsets { + public static int Magic { get; private set; } + public static readonly Dictionary NameMapping = new Dictionary(); + + static class DataOffsets + { + public static readonly int ob_data; + public static readonly int ob_dict; + + static DataOffsets() + { + FieldInfo[] fields = typeof(DataOffsets).GetFields(BindingFlags.Static | BindingFlags.Public); + for (int i = 0; i < fields.Length; i++) + { + fields[i].SetValue(null, -(i * IntPtr.Size) - IntPtr.Size); + } + } + } + static ManagedDataOffsets() { - FieldInfo[] fi = typeof(ManagedDataOffsets).GetFields(BindingFlags.Static | BindingFlags.Public); - for (int i = 0; i < fi.Length; i++) + Type type = typeof(TypeOffset); + foreach (FieldInfo fi in type.GetFields()) { - fi[i].SetValue(null, -(i * IntPtr.Size) - IntPtr.Size); + NameMapping[fi.Name] = (int)fi.GetValue(null); } + // XXX: Use the members after PyHeapTypeObject as magic slot + Magic = TypeOffset.members; - size = fi.Length * IntPtr.Size; + FieldInfo[] fields = typeof(DataOffsets).GetFields(BindingFlags.Static | BindingFlags.Public); + size = fields.Length * IntPtr.Size; } - public static readonly int ob_data; - public static readonly int ob_dict; + public static int GetSlotOffset(string name) + { + return NameMapping[name]; + } private static int BaseOffset(IntPtr type) { Debug.Assert(type != IntPtr.Zero); int typeSize = Marshal.ReadInt32(type, TypeOffset.tp_basicsize); - Debug.Assert(typeSize > 0 && typeSize <= ExceptionOffset.Size()); + Debug.Assert(typeSize > 0); return typeSize; } + public static int DataOffset(IntPtr type) { - return BaseOffset(type) + ob_data; + return BaseOffset(type) + DataOffsets.ob_data; } public static int DictOffset(IntPtr type) { - return BaseOffset(type) + ob_dict; + return BaseOffset(type) + DataOffsets.ob_dict; } + public static int ob_data => DataOffsets.ob_data; + public static int ob_dict => DataOffsets.ob_dict; public static int Size { get { return size; } } private static readonly int size; @@ -312,6 +355,7 @@ public static void FreeModuleDef(IntPtr ptr) public static int name = 0; } + /// /// TypeFlags(): The actual bit values for the Type Flags stored /// in a class. @@ -320,34 +364,34 @@ public static void FreeModuleDef(IntPtr ptr) /// internal class TypeFlags { - public static int HeapType = (1 << 9); - public static int BaseType = (1 << 10); - public static int Ready = (1 << 12); - public static int Readying = (1 << 13); - public static int HaveGC = (1 << 14); + public const int HeapType = (1 << 9); + public const int BaseType = (1 << 10); + public const int Ready = (1 << 12); + public const int Readying = (1 << 13); + public const int HaveGC = (1 << 14); // 15 and 16 are reserved for stackless - public static int HaveStacklessExtension = 0; + public const int HaveStacklessExtension = 0; /* XXX Reusing reserved constants */ - public static int Managed = (1 << 15); // PythonNet specific - public static int Subclass = (1 << 16); // PythonNet specific - public static int HaveIndex = (1 << 17); + public const int Managed = (1 << 15); // PythonNet specific + public const int Subclass = (1 << 16); // PythonNet specific + public const int HaveIndex = (1 << 17); /* Objects support nb_index in PyNumberMethods */ - public static int HaveVersionTag = (1 << 18); - public static int ValidVersionTag = (1 << 19); - public static int IsAbstract = (1 << 20); - public static int HaveNewBuffer = (1 << 21); + public const int HaveVersionTag = (1 << 18); + public const int ValidVersionTag = (1 << 19); + public const int IsAbstract = (1 << 20); + public const int HaveNewBuffer = (1 << 21); // TODO: Implement FastSubclass functions - public static int IntSubclass = (1 << 23); - public static int LongSubclass = (1 << 24); - public static int ListSubclass = (1 << 25); - public static int TupleSubclass = (1 << 26); - public static int StringSubclass = (1 << 27); - public static int UnicodeSubclass = (1 << 28); - public static int DictSubclass = (1 << 29); - public static int BaseExceptionSubclass = (1 << 30); - public static int TypeSubclass = (1 << 31); - - public static int Default = ( + public const int IntSubclass = (1 << 23); + public const int LongSubclass = (1 << 24); + public const int ListSubclass = (1 << 25); + public const int TupleSubclass = (1 << 26); + public const int StringSubclass = (1 << 27); + public const int UnicodeSubclass = (1 << 28); + public const int DictSubclass = (1 << 29); + public const int BaseExceptionSubclass = (1 << 30); + public const int TypeSubclass = (1 << 31); + + public const int Default = ( HaveStacklessExtension | HaveVersionTag); } @@ -360,7 +404,6 @@ internal class TypeFlags internal class Interop { - private static List keepAlive; private static Hashtable pmap; static Interop() @@ -377,7 +420,6 @@ static Interop() p[item.Name] = item; } - keepAlive = new List(); pmap = new Hashtable(); pmap["tp_dealloc"] = p["DestructorFunc"]; @@ -482,11 +524,10 @@ internal static ThunkInfo GetThunk(MethodInfo method, string funcType = null) } Delegate d = Delegate.CreateDelegate(dt, method); var info = new ThunkInfo(d); - // TODO: remove keepAlive when #958 merged, let the lifecycle of ThunkInfo transfer to caller. - keepAlive.Add(info); return info; } + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] public delegate IntPtr UnaryFunc(IntPtr ob); @@ -528,17 +569,6 @@ internal static ThunkInfo GetThunk(MethodInfo method, string funcType = null) } - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal struct Thunk - { - public Delegate fn; - - public Thunk(Delegate d) - { - fn = d; - } - } - internal class ThunkInfo { public readonly Delegate Target; @@ -556,4 +586,29 @@ public ThunkInfo(Delegate target) Address = Marshal.GetFunctionPointerForDelegate(target); } } + + [StructLayout(LayoutKind.Sequential)] + struct PyGC_Node + { + public IntPtr gc_next; + public IntPtr gc_prev; + public IntPtr gc_refs; + } + + [StructLayout(LayoutKind.Sequential)] + struct PyGC_Head + { + public PyGC_Node gc; + } + + + [StructLayout(LayoutKind.Sequential)] + struct PyMethodDef + { + public IntPtr ml_name; + public IntPtr ml_meth; + public int ml_flags; + public IntPtr ml_doc; + } + } diff --git a/src/runtime/interop36.cs b/src/runtime/interop36.cs index c46bcc2f5..d68539d56 100644 --- a/src/runtime/interop36.cs +++ b/src/runtime/interop36.cs @@ -1,5 +1,6 @@ + // Auto-generated by geninterop.py. -// DO NOT MODIFIY BY HAND. +// DO NOT MODIFY BY HAND. #if PYTHON36 @@ -12,25 +13,10 @@ namespace Python.Runtime { - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal class TypeOffset - { - static TypeOffset() - { - Type type = typeof(TypeOffset); - FieldInfo[] fi = type.GetFields(); - int size = IntPtr.Size; - for (int i = 0; i < fi.Length; i++) - { - fi[i].SetValue(null, i * size); - } - } - - public static int magic() - { - return ob_size; - } + [StructLayout(LayoutKind.Sequential)] + internal static partial class TypeOffset + { // Auto-generated from PyHeapTypeObject in Python.h public static int ob_refcnt = 0; public static int ob_type = 0; @@ -144,6 +130,96 @@ public static int magic() /* here are optional user slots, followed by the members. */ public static int members = 0; } -} + [StructLayout(LayoutKind.Sequential)] + internal struct PyNumberMethods + { + public IntPtr nb_add; + public IntPtr nb_subtract; + public IntPtr nb_multiply; + public IntPtr nb_remainder; + public IntPtr nb_divmod; + public IntPtr nb_power; + public IntPtr nb_negative; + public IntPtr nb_positive; + public IntPtr nb_absolute; + public IntPtr nb_bool; + public IntPtr nb_invert; + public IntPtr nb_lshift; + public IntPtr nb_rshift; + public IntPtr nb_and; + public IntPtr nb_xor; + public IntPtr nb_or; + public IntPtr nb_int; + public IntPtr nb_reserved; + public IntPtr nb_float; + public IntPtr nb_inplace_add; + public IntPtr nb_inplace_subtract; + public IntPtr nb_inplace_multiply; + public IntPtr nb_inplace_remainder; + public IntPtr nb_inplace_power; + public IntPtr nb_inplace_lshift; + public IntPtr nb_inplace_rshift; + public IntPtr nb_inplace_and; + public IntPtr nb_inplace_xor; + public IntPtr nb_inplace_or; + public IntPtr nb_floor_divide; + public IntPtr nb_true_divide; + public IntPtr nb_inplace_floor_divide; + public IntPtr nb_inplace_true_divide; + public IntPtr nb_index; + public IntPtr nb_matrix_multiply; + public IntPtr nb_inplace_matrix_multiply; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PySequenceMethods + { + public IntPtr sq_length; + public IntPtr sq_concat; + public IntPtr sq_repeat; + public IntPtr sq_item; + public IntPtr was_sq_slice; + public IntPtr sq_ass_item; + public IntPtr was_sq_ass_slice; + public IntPtr sq_contains; + public IntPtr sq_inplace_concat; + public IntPtr sq_inplace_repeat; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PyMappingMethods + { + public IntPtr mp_length; + public IntPtr mp_subscript; + public IntPtr mp_ass_subscript; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PyAsyncMethods + { + public IntPtr am_await; + public IntPtr am_aiter; + public IntPtr am_anext; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PyBufferProcs + { + public IntPtr bf_getbuffer; + public IntPtr bf_releasebuffer; + } + + internal static partial class SlotTypes + { + public static readonly Type[] Types = { + typeof(PyNumberMethods), + typeof(PySequenceMethods), + typeof(PyMappingMethods), + typeof(PyAsyncMethods), + typeof(PyBufferProcs), + }; + } + +} #endif diff --git a/src/runtime/interop37.cs b/src/runtime/interop37.cs index d5fc76ad3..c85d06525 100644 --- a/src/runtime/interop37.cs +++ b/src/runtime/interop37.cs @@ -1,5 +1,6 @@ + // Auto-generated by geninterop.py. -// DO NOT MODIFIY BY HAND. +// DO NOT MODIFY BY HAND. #if PYTHON37 @@ -12,25 +13,10 @@ namespace Python.Runtime { - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal class TypeOffset - { - static TypeOffset() - { - Type type = typeof(TypeOffset); - FieldInfo[] fi = type.GetFields(); - int size = IntPtr.Size; - for (int i = 0; i < fi.Length; i++) - { - fi[i].SetValue(null, i * size); - } - } - - public static int magic() - { - return ob_size; - } + [StructLayout(LayoutKind.Sequential)] + internal static partial class TypeOffset + { // Auto-generated from PyHeapTypeObject in Python.h public static int ob_refcnt = 0; public static int ob_type = 0; @@ -144,6 +130,96 @@ public static int magic() /* here are optional user slots, followed by the members. */ public static int members = 0; } -} + [StructLayout(LayoutKind.Sequential)] + internal struct PyNumberMethods + { + public IntPtr nb_add; + public IntPtr nb_subtract; + public IntPtr nb_multiply; + public IntPtr nb_remainder; + public IntPtr nb_divmod; + public IntPtr nb_power; + public IntPtr nb_negative; + public IntPtr nb_positive; + public IntPtr nb_absolute; + public IntPtr nb_bool; + public IntPtr nb_invert; + public IntPtr nb_lshift; + public IntPtr nb_rshift; + public IntPtr nb_and; + public IntPtr nb_xor; + public IntPtr nb_or; + public IntPtr nb_int; + public IntPtr nb_reserved; + public IntPtr nb_float; + public IntPtr nb_inplace_add; + public IntPtr nb_inplace_subtract; + public IntPtr nb_inplace_multiply; + public IntPtr nb_inplace_remainder; + public IntPtr nb_inplace_power; + public IntPtr nb_inplace_lshift; + public IntPtr nb_inplace_rshift; + public IntPtr nb_inplace_and; + public IntPtr nb_inplace_xor; + public IntPtr nb_inplace_or; + public IntPtr nb_floor_divide; + public IntPtr nb_true_divide; + public IntPtr nb_inplace_floor_divide; + public IntPtr nb_inplace_true_divide; + public IntPtr nb_index; + public IntPtr nb_matrix_multiply; + public IntPtr nb_inplace_matrix_multiply; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PySequenceMethods + { + public IntPtr sq_length; + public IntPtr sq_concat; + public IntPtr sq_repeat; + public IntPtr sq_item; + public IntPtr was_sq_slice; + public IntPtr sq_ass_item; + public IntPtr was_sq_ass_slice; + public IntPtr sq_contains; + public IntPtr sq_inplace_concat; + public IntPtr sq_inplace_repeat; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PyMappingMethods + { + public IntPtr mp_length; + public IntPtr mp_subscript; + public IntPtr mp_ass_subscript; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PyAsyncMethods + { + public IntPtr am_await; + public IntPtr am_aiter; + public IntPtr am_anext; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PyBufferProcs + { + public IntPtr bf_getbuffer; + public IntPtr bf_releasebuffer; + } + + internal static partial class SlotTypes + { + public static readonly Type[] Types = { + typeof(PyNumberMethods), + typeof(PySequenceMethods), + typeof(PyMappingMethods), + typeof(PyAsyncMethods), + typeof(PyBufferProcs), + }; + } + +} #endif diff --git a/src/runtime/interop38.cs b/src/runtime/interop38.cs index 9126bca6a..a87573e90 100644 --- a/src/runtime/interop38.cs +++ b/src/runtime/interop38.cs @@ -13,25 +13,10 @@ namespace Python.Runtime { - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal class TypeOffset - { - static TypeOffset() - { - Type type = typeof(TypeOffset); - FieldInfo[] fi = type.GetFields(); - int size = IntPtr.Size; - for (int i = 0; i < fi.Length; i++) - { - fi[i].SetValue(null, i * size); - } - } - - public static int magic() - { - return ob_size; - } + [StructLayout(LayoutKind.Sequential)] + internal static partial class TypeOffset + { // Auto-generated from PyHeapTypeObject in Python.h public static int ob_refcnt = 0; public static int ob_type = 0; @@ -147,6 +132,96 @@ public static int magic() /* here are optional user slots, followed by the members. */ public static int members = 0; } -} + [StructLayout(LayoutKind.Sequential)] + internal struct PyNumberMethods + { + public IntPtr nb_add; + public IntPtr nb_subtract; + public IntPtr nb_multiply; + public IntPtr nb_remainder; + public IntPtr nb_divmod; + public IntPtr nb_power; + public IntPtr nb_negative; + public IntPtr nb_positive; + public IntPtr nb_absolute; + public IntPtr nb_bool; + public IntPtr nb_invert; + public IntPtr nb_lshift; + public IntPtr nb_rshift; + public IntPtr nb_and; + public IntPtr nb_xor; + public IntPtr nb_or; + public IntPtr nb_int; + public IntPtr nb_reserved; + public IntPtr nb_float; + public IntPtr nb_inplace_add; + public IntPtr nb_inplace_subtract; + public IntPtr nb_inplace_multiply; + public IntPtr nb_inplace_remainder; + public IntPtr nb_inplace_power; + public IntPtr nb_inplace_lshift; + public IntPtr nb_inplace_rshift; + public IntPtr nb_inplace_and; + public IntPtr nb_inplace_xor; + public IntPtr nb_inplace_or; + public IntPtr nb_floor_divide; + public IntPtr nb_true_divide; + public IntPtr nb_inplace_floor_divide; + public IntPtr nb_inplace_true_divide; + public IntPtr nb_index; + public IntPtr nb_matrix_multiply; + public IntPtr nb_inplace_matrix_multiply; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PySequenceMethods + { + public IntPtr sq_length; + public IntPtr sq_concat; + public IntPtr sq_repeat; + public IntPtr sq_item; + public IntPtr was_sq_slice; + public IntPtr sq_ass_item; + public IntPtr was_sq_ass_slice; + public IntPtr sq_contains; + public IntPtr sq_inplace_concat; + public IntPtr sq_inplace_repeat; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PyMappingMethods + { + public IntPtr mp_length; + public IntPtr mp_subscript; + public IntPtr mp_ass_subscript; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PyAsyncMethods + { + public IntPtr am_await; + public IntPtr am_aiter; + public IntPtr am_anext; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct PyBufferProcs + { + public IntPtr bf_getbuffer; + public IntPtr bf_releasebuffer; + } + + internal static partial class SlotTypes + { + public static readonly Type[] Types = { + typeof(PyNumberMethods), + typeof(PySequenceMethods), + typeof(PyMappingMethods), + typeof(PyAsyncMethods), + typeof(PyBufferProcs), + }; + } + +} #endif diff --git a/src/runtime/managedtype.cs b/src/runtime/managedtype.cs index afa3bf2d7..bc2805d80 100644 --- a/src/runtime/managedtype.cs +++ b/src/runtime/managedtype.cs @@ -1,5 +1,8 @@ using System; +using System.Collections.Generic; +using System.Diagnostics; using System.Runtime.InteropServices; +using System.Linq; namespace Python.Runtime { @@ -8,12 +11,69 @@ namespace Python.Runtime /// code. It defines the common fields that associate CLR and Python /// objects and common utilities to convert between those identities. /// + [Serializable] internal abstract class ManagedType { + internal enum TrackTypes + { + Untrack, + Extension, + Wrapper, + } + + [NonSerialized] internal GCHandle gcHandle; // Native handle + internal IntPtr pyHandle; // PyObject * internal IntPtr tpHandle; // PyType * + private static readonly Dictionary _managedObjs = new Dictionary(); + + internal void IncrRefCount() + { + Runtime.XIncref(pyHandle); + } + + internal void DecrRefCount() + { + Runtime.XDecref(pyHandle); + } + + internal long RefCount + { + get + { + var gs = Runtime.PyGILState_Ensure(); + try + { + return Runtime.Refcount(pyHandle); + } + finally + { + Runtime.PyGILState_Release(gs); + } + } + } + + internal GCHandle AllocGCHandle(TrackTypes track = TrackTypes.Untrack) + { + gcHandle = GCHandle.Alloc(this); + if (track != TrackTypes.Untrack) + { + _managedObjs.Add(this, track); + } + return gcHandle; + } + + internal void FreeGCHandle() + { + _managedObjs.Remove(this); + if (gcHandle.IsAllocated) + { + gcHandle.Free(); + gcHandle = default; + } + } /// /// Given a Python object, return the associated managed object or null. @@ -94,5 +154,108 @@ internal static bool IsManagedType(IntPtr ob) } return false; } + + public bool IsTypeObject() + { + return pyHandle == tpHandle; + } + + internal static IDictionary GetManagedObjects() + { + return _managedObjs; + } + + internal static void ClearTrackedObjects() + { + _managedObjs.Clear(); + } + + internal static int PyVisit(IntPtr ob, IntPtr visit, IntPtr arg) + { + if (ob == IntPtr.Zero) + { + return 0; + } + var visitFunc = (Interop.ObjObjFunc)Marshal.GetDelegateForFunctionPointer(visit, typeof(Interop.ObjObjFunc)); + return visitFunc(ob, arg); + } + + /// + /// Wrapper for calling tp_clear + /// + internal void CallTypeClear() + { + if (tpHandle == IntPtr.Zero || pyHandle == IntPtr.Zero) + { + return; + } + var clearPtr = Marshal.ReadIntPtr(tpHandle, TypeOffset.tp_clear); + if (clearPtr == IntPtr.Zero) + { + return; + } + var clearFunc = (Interop.InquiryFunc)Marshal.GetDelegateForFunctionPointer(clearPtr, typeof(Interop.InquiryFunc)); + clearFunc(pyHandle); + } + + /// + /// Wrapper for calling tp_traverse + /// + internal void CallTypeTraverse(Interop.ObjObjFunc visitproc, IntPtr arg) + { + if (tpHandle == IntPtr.Zero || pyHandle == IntPtr.Zero) + { + return; + } + var traversePtr = Marshal.ReadIntPtr(tpHandle, TypeOffset.tp_traverse); + if (traversePtr == IntPtr.Zero) + { + return; + } + var traverseFunc = (Interop.ObjObjArgFunc)Marshal.GetDelegateForFunctionPointer(traversePtr, typeof(Interop.ObjObjArgFunc)); + var visiPtr = Marshal.GetFunctionPointerForDelegate(visitproc); + traverseFunc(pyHandle, visiPtr, arg); + } + + protected void TypeClear() + { + ClearObjectDict(pyHandle); + } + + internal void Save(InterDomainContext context) + { + OnSave(context); + } + + internal void Load(InterDomainContext context) + { + OnLoad(context); + } + + protected virtual void OnSave(InterDomainContext context) { } + protected virtual void OnLoad(InterDomainContext context) { } + + protected static void ClearObjectDict(IntPtr ob) + { + IntPtr dict = GetObjectDict(ob); + if (dict == IntPtr.Zero) + { + return; + } + SetObjectDict(ob, IntPtr.Zero); + Runtime.XDecref(dict); + } + + protected static IntPtr GetObjectDict(IntPtr ob) + { + IntPtr type = Runtime.PyObject_TYPE(ob); + return Marshal.ReadIntPtr(ob, ObjectOffset.TypeDictOffset(type)); + } + + protected static void SetObjectDict(IntPtr ob, IntPtr value) + { + IntPtr type = Runtime.PyObject_TYPE(ob); + Marshal.WriteIntPtr(ob, ObjectOffset.TypeDictOffset(type), value); + } } } diff --git a/src/runtime/metatype.cs b/src/runtime/metatype.cs index 5af2e1a7e..f7afd5d6d 100644 --- a/src/runtime/metatype.cs +++ b/src/runtime/metatype.cs @@ -1,4 +1,6 @@ using System; +using System.Collections; +using System.IO; using System.Runtime.InteropServices; namespace Python.Runtime @@ -11,17 +13,55 @@ namespace Python.Runtime internal class MetaType : ManagedType { private static IntPtr PyCLRMetaType; + private static SlotsHolder _metaSlotsHodler; + internal static readonly string[] CustomMethods = new string[] + { + "__instancecheck__", + "__subclasscheck__", + }; /// /// Metatype initialization. This bootstraps the CLR metatype to life. /// public static IntPtr Initialize() { - PyCLRMetaType = TypeManager.CreateMetaType(typeof(MetaType)); + PyCLRMetaType = TypeManager.CreateMetaType(typeof(MetaType), out _metaSlotsHodler); return PyCLRMetaType; } + public static void Release() + { + if (Runtime.Refcount(PyCLRMetaType) > 1) + { + _metaSlotsHodler.ResetSlots(); + } + Runtime.Py_CLEAR(ref PyCLRMetaType); + _metaSlotsHodler = null; + } + + internal static void SaveRuntimeData(RuntimeDataStorage storage) + { + Runtime.XIncref(PyCLRMetaType); + storage.PushValue(PyCLRMetaType); + } + + internal static IntPtr RestoreRuntimeData(RuntimeDataStorage storage) + { + PyCLRMetaType = storage.PopValue(); + _metaSlotsHodler = new SlotsHolder(PyCLRMetaType); + TypeManager.InitializeSlots(PyCLRMetaType, typeof(MetaType), _metaSlotsHodler); + + IntPtr mdef = Marshal.ReadIntPtr(PyCLRMetaType, TypeOffset.tp_methods); + foreach (var methodName in CustomMethods) + { + var mi = typeof(MetaType).GetMethod(methodName); + ThunkInfo thunkInfo = Interop.GetThunk(mi, "BinaryFunc"); + _metaSlotsHodler.KeeapAlive(thunkInfo); + mdef = TypeManager.WriteMethodDef(mdef, methodName, thunkInfo.Address); + } + return PyCLRMetaType; + } /// /// Metatype __new__ implementation. This is called to create a new diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index e2d8581b3..c0cc4c75c 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -13,6 +13,7 @@ namespace Python.Runtime /// a set of Python arguments. This is also used as a base class for the /// ConstructorBinder, a minor variation used to invoke constructors. /// + [Serializable] internal class MethodBinder { public ArrayList list; diff --git a/src/runtime/methodbinding.cs b/src/runtime/methodbinding.cs index f402f91f8..011d8217d 100644 --- a/src/runtime/methodbinding.cs +++ b/src/runtime/methodbinding.cs @@ -9,6 +9,7 @@ namespace Python.Runtime /// standard Python method bindings, but the same type is used to bind /// both static and instance methods. /// + [Serializable] internal class MethodBinding : ExtensionType { internal MethodInfo info; @@ -21,11 +22,15 @@ public MethodBinding(MethodObject m, IntPtr target, IntPtr targetType) Runtime.XIncref(target); this.target = target; - Runtime.XIncref(targetType); if (targetType == IntPtr.Zero) { targetType = Runtime.PyObject_Type(target); } + else + { + Runtime.XIncref(targetType); + } + this.targetType = targetType; this.info = null; @@ -36,6 +41,12 @@ public MethodBinding(MethodObject m, IntPtr target) : this(m, target, IntPtr.Zer { } + private void ClearMembers() + { + Runtime.Py_CLEAR(ref target); + Runtime.Py_CLEAR(ref targetType); + } + /// /// Implement binding of generic methods using the subscript syntax []. /// @@ -237,9 +248,22 @@ public static IntPtr tp_repr(IntPtr ob) public new static void tp_dealloc(IntPtr ob) { var self = (MethodBinding)GetManagedObject(ob); - Runtime.XDecref(self.target); - Runtime.XDecref(self.targetType); - FinalizeObject(self); + self.ClearMembers(); + self.Dealloc(); + } + + public static int tp_clear(IntPtr ob) + { + var self = (MethodBinding)GetManagedObject(ob); + self.ClearMembers(); + return 0; + } + + protected override void OnSave(InterDomainContext context) + { + base.OnSave(context); + Runtime.XIncref(target); + Runtime.XIncref(targetType); } } } diff --git a/src/runtime/methodobject.cs b/src/runtime/methodobject.cs index 8df9c8029..eb3ce8a18 100644 --- a/src/runtime/methodobject.cs +++ b/src/runtime/methodobject.cs @@ -10,6 +10,7 @@ namespace Python.Runtime /// /// TODO: ForbidPythonThreadsAttribute per method info /// + [Serializable] internal class MethodObject : ExtensionType { internal MethodInfo[] info; @@ -17,6 +18,7 @@ internal class MethodObject : ExtensionType internal MethodBinding unbound; internal MethodBinder binder; internal bool is_static = false; + internal IntPtr doc; internal Type type; @@ -109,6 +111,16 @@ internal bool IsStatic() return is_static; } + private void ClearMembers() + { + Runtime.Py_CLEAR(ref doc); + if (unbound != null) + { + Runtime.XDecref(unbound.pyHandle); + unbound = null; + } + } + /// /// Descriptor __getattribute__ implementation. /// @@ -196,12 +208,27 @@ public static IntPtr tp_repr(IntPtr ob) public new static void tp_dealloc(IntPtr ob) { var self = (MethodObject)GetManagedObject(ob); - Runtime.XDecref(self.doc); - if (self.unbound != null) + self.ClearMembers(); + ClearObjectDict(ob); + self.Dealloc(); + } + + public static int tp_clear(IntPtr ob) + { + var self = (MethodObject)GetManagedObject(ob); + self.ClearMembers(); + ClearObjectDict(ob); + return 0; + } + + protected override void OnSave(InterDomainContext context) + { + base.OnSave(context); + if (unbound != null) { - Runtime.XDecref(self.unbound.pyHandle); + Runtime.XIncref(unbound.pyHandle); } - ExtensionType.FinalizeObject(self); + Runtime.XIncref(doc); } } } diff --git a/src/runtime/methodwrapper.cs b/src/runtime/methodwrapper.cs index bc7500dab..4caefccb6 100644 --- a/src/runtime/methodwrapper.cs +++ b/src/runtime/methodwrapper.cs @@ -12,9 +12,10 @@ internal class MethodWrapper { public IntPtr mdef; public IntPtr ptr; + private ThunkInfo _thunk; + private bool _disposed = false; - private ThunkInfo _thunk; public MethodWrapper(Type type, string name, string funcType = null) { @@ -30,6 +31,7 @@ public MethodWrapper(Type type, string name, string funcType = null) ptr = Runtime.PyCFunction_NewEx(mdef, IntPtr.Zero, IntPtr.Zero); } + public IntPtr Call(IntPtr args, IntPtr kw) { return Runtime.PyCFunction_Call(ptr, args, kw); diff --git a/src/runtime/modulefunctionobject.cs b/src/runtime/modulefunctionobject.cs index 8f8692af9..e7a2c515a 100644 --- a/src/runtime/modulefunctionobject.cs +++ b/src/runtime/modulefunctionobject.cs @@ -7,6 +7,7 @@ namespace Python.Runtime /// /// Module level functions /// + [Serializable] internal class ModuleFunctionObject : MethodObject { public ModuleFunctionObject(Type type, string name, MethodInfo[] info, bool allow_threads) diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index ea5bbbfa0..6313975da 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -11,9 +11,11 @@ namespace Python.Runtime /// Implements a Python type that provides access to CLR namespaces. The /// type behaves like a Python module, and can contain other sub-modules. /// + [Serializable] internal class ModuleObject : ExtensionType { private Dictionary cache; + internal string moduleName; internal IntPtr dict; protected string _namespace; @@ -54,7 +56,8 @@ public ModuleObject(string name) Runtime.XDecref(pyfilename); Runtime.XDecref(pydocstring); - Marshal.WriteIntPtr(pyHandle, ObjectOffset.TypeDictOffset(tpHandle), dict); + Runtime.XIncref(dict); + SetObjectDict(pyHandle, dict); InitializeModuleMembers(); } @@ -100,6 +103,7 @@ public ManagedType GetAttribute(string name, bool guess) { m = new ModuleObject(qname); StoreAttribute(name, m); + m.DecrRefCount(); return m; } @@ -125,6 +129,7 @@ public ManagedType GetAttribute(string name, bool guess) { m = new ModuleObject(qname); StoreAttribute(name, m); + m.DecrRefCount(); return m; } @@ -172,7 +177,11 @@ static void ImportWarning(Exception exception) /// private void StoreAttribute(string name, ManagedType ob) { - Runtime.PyDict_SetItemString(dict, name, ob.pyHandle); + if (Runtime.PyDict_SetItemString(dict, name, ob.pyHandle) != 0) + { + throw new PythonException(); + } + ob.IncrRefCount(); cache[name] = ob; } @@ -229,6 +238,7 @@ internal void InitializeModuleMembers() mi[0] = method; var m = new ModuleFunctionObject(type, name, mi, allow_threads); StoreAttribute(name, m); + m.DecrRefCount(); } } @@ -241,6 +251,7 @@ internal void InitializeModuleMembers() string name = property.Name; var p = new ModulePropertyObject(property); StoreAttribute(name, p); + p.DecrRefCount(); } } type = type.BaseType; @@ -309,6 +320,58 @@ public static IntPtr tp_repr(IntPtr ob) var self = (ModuleObject)GetManagedObject(ob); return Runtime.PyString_FromString($""); } + + public new static void tp_dealloc(IntPtr ob) + { + var self = (ModuleObject)GetManagedObject(ob); + tp_clear(ob); + self.Dealloc(); + } + + public static int tp_traverse(IntPtr ob, IntPtr visit, IntPtr arg) + { + var self = (ModuleObject)GetManagedObject(ob); + int res = PyVisit(self.dict, visit, arg); + if (res != 0) return res; + foreach (var attr in self.cache.Values) + { + res = PyVisit(attr.pyHandle, visit, arg); + if (res != 0) return res; + } + return 0; + } + + public static int tp_clear(IntPtr ob) + { + var self = (ModuleObject)GetManagedObject(ob); + Runtime.Py_CLEAR(ref self.dict); + ClearObjectDict(ob); + foreach (var attr in self.cache.Values) + { + Runtime.XDecref(attr.pyHandle); + } + self.cache.Clear(); + return 0; + } + + protected override void OnSave(InterDomainContext context) + { + base.OnSave(context); + System.Diagnostics.Debug.Assert(dict == GetObjectDict(pyHandle)); + foreach (var attr in cache.Values) + { + Runtime.XIncref(attr.pyHandle); + } + // Decref twice in tp_clear, equilibrate them. + Runtime.XIncref(dict); + Runtime.XIncref(dict); + } + + protected override void OnLoad(InterDomainContext context) + { + base.OnLoad(context); + SetObjectDict(pyHandle, dict); + } } /// @@ -316,6 +379,7 @@ public static IntPtr tp_repr(IntPtr ob) /// to import assemblies. It has a fixed module name "clr" and doesn't /// provide a namespace. /// + [Serializable] internal class CLRModule : ModuleObject { protected static bool hacked = false; diff --git a/src/runtime/nativecall.cs b/src/runtime/nativecall.cs index 4a7bf05c8..ec0bf338c 100644 --- a/src/runtime/nativecall.cs +++ b/src/runtime/nativecall.cs @@ -23,7 +23,6 @@ namespace Python.Runtime /// internal class NativeCall { -#if NETSTANDARD [UnmanagedFunctionPointer(CallingConvention.Cdecl)] private delegate void Void_1_Delegate(IntPtr a1); @@ -32,148 +31,27 @@ internal class NativeCall public static void Void_Call_1(IntPtr fp, IntPtr a1) { - var d = Marshal.GetDelegateForFunctionPointer(fp); + var d = GetDelegate(fp); d(a1); } public static IntPtr Call_3(IntPtr fp, IntPtr a1, IntPtr a2, IntPtr a3) { - var d = Marshal.GetDelegateForFunctionPointer(fp); + var d = GetDelegate(fp); return d(a1, a2, a3); } public static int Int_Call_3(IntPtr fp, IntPtr a1, IntPtr a2, IntPtr a3) { - var d = Marshal.GetDelegateForFunctionPointer(fp); + var d = GetDelegate(fp); return d(a1, a2, a3); } -#else - private static AssemblyBuilder aBuilder; - private static ModuleBuilder mBuilder; - public static INativeCall Impl; - - static NativeCall() - { - // The static constructor is responsible for generating the - // assembly and the methods that implement the IJW thunks. - // - // To do this, we actually use reflection on the INativeCall - // interface (defined below) and generate the required thunk - // code based on the method signatures. - - var aname = new AssemblyName { Name = "e__NativeCall_Assembly" }; - var aa = AssemblyBuilderAccess.Run; - - aBuilder = Thread.GetDomain().DefineDynamicAssembly(aname, aa); - mBuilder = aBuilder.DefineDynamicModule("e__NativeCall_Module"); - - var ta = TypeAttributes.Public; - TypeBuilder tBuilder = mBuilder.DefineType("e__NativeCall", ta); - - Type iType = typeof(INativeCall); - tBuilder.AddInterfaceImplementation(iType); - - // Use reflection to loop over the INativeCall interface methods, - // calling GenerateThunk to create a managed thunk for each one. - - foreach (MethodInfo method in iType.GetMethods()) - { - GenerateThunk(tBuilder, method); - } - - Type theType = tBuilder.CreateType(); - - Impl = (INativeCall)Activator.CreateInstance(theType); - } - - private static void GenerateThunk(TypeBuilder tb, MethodInfo method) - { - ParameterInfo[] pi = method.GetParameters(); - int count = pi.Length; - int argc = count - 1; - - var args = new Type[count]; - for (var i = 0; i < count; i++) - { - args[i] = pi[i].ParameterType; - } - - MethodBuilder mb = tb.DefineMethod( - method.Name, - MethodAttributes.Public | - MethodAttributes.Virtual, - method.ReturnType, - args - ); - - // Build the method signature for the actual native function. - // This is essentially the signature of the wrapper method - // minus the first argument (the passed in function pointer). - - var nargs = new Type[argc]; - for (var i = 1; i < count; i++) - { - nargs[i - 1] = args[i]; - } - - // IL generation: the (implicit) first argument of the method - // is the 'this' pointer and the second is the function pointer. - // This code pushes the real args onto the stack, followed by - // the function pointer, then the calli opcode to make the call. - - ILGenerator il = mb.GetILGenerator(); - - for (var i = 0; i < argc; i++) - { - il.Emit(OpCodes.Ldarg_S, i + 2); - } - - il.Emit(OpCodes.Ldarg_1); - - il.EmitCalli(OpCodes.Calli, - CallingConvention.Cdecl, - method.ReturnType, - nargs - ); - - il.Emit(OpCodes.Ret); - - tb.DefineMethodOverride(mb, method); - } - - - public static void Void_Call_1(IntPtr fp, IntPtr a1) - { - Impl.Void_Call_1(fp, a1); - } - - public static IntPtr Call_3(IntPtr fp, IntPtr a1, IntPtr a2, IntPtr a3) - { - return Impl.Call_3(fp, a1, a2, a3); - } - - public static int Int_Call_3(IntPtr fp, IntPtr a1, IntPtr a2, IntPtr a3) + private static T GetDelegate(IntPtr fp) where T: Delegate { - return Impl.Int_Call_3(fp, a1, a2, a3); + // Use Marshal.GetDelegateForFunctionPointer<> directly after upgrade the framework + return (T)Marshal.GetDelegateForFunctionPointer(fp, typeof(T)); } -#endif - } - -#if !NETSTANDARD - /// - /// Defines native call signatures to be generated by NativeCall. - /// - public interface INativeCall - { - void Void_Call_0(IntPtr funcPtr); - - void Void_Call_1(IntPtr funcPtr, IntPtr arg1); - - int Int_Call_3(IntPtr funcPtr, IntPtr t, IntPtr n, IntPtr v); - - IntPtr Call_3(IntPtr funcPtr, IntPtr a1, IntPtr a2, IntPtr a3); } -#endif } diff --git a/src/runtime/overload.cs b/src/runtime/overload.cs index 67868a1b1..e9fa91d3b 100644 --- a/src/runtime/overload.cs +++ b/src/runtime/overload.cs @@ -65,7 +65,7 @@ public static IntPtr tp_repr(IntPtr op) { var self = (OverloadMapper)GetManagedObject(ob); Runtime.XDecref(self.target); - FinalizeObject(self); + self.Dealloc(); } } } diff --git a/src/runtime/platform/NativeCodePage.cs b/src/runtime/platform/NativeCodePage.cs new file mode 100644 index 000000000..ab2ee3bcf --- /dev/null +++ b/src/runtime/platform/NativeCodePage.cs @@ -0,0 +1,291 @@ +using System; +using System.Collections.Generic; +using System.Runtime.InteropServices; + +namespace Python.Runtime.Platform +{ + class NativeCodePageHelper + { + /// + /// Gets the operating system as reported by python's platform.system(). + /// + public static OperatingSystemType OperatingSystem { get; private set; } + + /// + /// Gets the operating system as reported by python's platform.system(). + /// + [Obsolete] + public static string OperatingSystemName => PythonEngine.Platform; + + /// + /// Gets the machine architecture as reported by python's platform.machine(). + /// + public static MachineType Machine { get; private set; }/* set in Initialize using python's platform.machine */ + + /// + /// Gets the machine architecture as reported by python's platform.machine(). + /// + [Obsolete] + public static string MachineName { get; private set; } + + /// + /// Initialized by InitializeNativeCodePage. + /// + /// This points to a page of memory allocated using mmap or VirtualAlloc + /// (depending on the system), and marked read and execute (not write). + /// Very much on purpose, the page is *not* released on a shutdown and + /// is instead leaked. See the TestDomainReload test case. + /// + /// The contents of the page are two native functions: one that returns 0, + /// one that returns 1. + /// + /// If python didn't keep its gc list through a Py_Finalize we could remove + /// this entire section. + /// + internal static IntPtr NativeCodePage = IntPtr.Zero; + + + static readonly Dictionary OperatingSystemTypeMapping = new Dictionary() + { + { "Windows", OperatingSystemType.Windows }, + { "Darwin", OperatingSystemType.Darwin }, + { "Linux", OperatingSystemType.Linux }, + }; + + /// + /// Map lower-case version of the python machine name to the processor + /// type. There are aliases, e.g. x86_64 and amd64 are two names for + /// the same thing. Make sure to lower-case the search string, because + /// capitalization can differ. + /// + static readonly Dictionary MachineTypeMapping = new Dictionary() + { + ["i386"] = MachineType.i386, + ["i686"] = MachineType.i386, + ["x86"] = MachineType.i386, + ["x86_64"] = MachineType.x86_64, + ["amd64"] = MachineType.x86_64, + ["x64"] = MachineType.x86_64, + ["em64t"] = MachineType.x86_64, + ["armv7l"] = MachineType.armv7l, + ["armv8"] = MachineType.armv8, + ["aarch64"] = MachineType.aarch64, + }; + + /// + /// Structure to describe native code. + /// + /// Use NativeCode.Active to get the native code for the current platform. + /// + /// Generate the code by creating the following C code: + /// + /// int Return0() { return 0; } + /// int Return1() { return 1; } + /// + /// Then compiling on the target platform, e.g. with gcc or clang: + /// cc -c -fomit-frame-pointer -O2 foo.c + /// And then analyzing the resulting functions with a hex editor, e.g.: + /// objdump -disassemble foo.o + /// + internal class NativeCode + { + /// + /// The code, as a string of bytes. + /// + public byte[] Code { get; private set; } + + /// + /// Where does the "return 0" function start? + /// + public int Return0 { get; private set; } + + /// + /// Where does the "return 1" function start? + /// + public int Return1 { get; private set; } + + public static NativeCode Active + { + get + { + switch (Machine) + { + case MachineType.i386: + return I386; + case MachineType.x86_64: + return X86_64; + default: + return null; + } + } + } + + /// + /// Code for x86_64. See the class comment for how it was generated. + /// + public static readonly NativeCode X86_64 = new NativeCode() + { + Return0 = 0x10, + Return1 = 0, + Code = new byte[] + { + // First Return1: + 0xb8, 0x01, 0x00, 0x00, 0x00, // movl $1, %eax + 0xc3, // ret + + // Now some padding so that Return0 can be 16-byte-aligned. + // I put Return1 first so there's not as much padding to type in. + 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, // nop + + // Now Return0. + 0x31, 0xc0, // xorl %eax, %eax + 0xc3, // ret + } + }; + + /// + /// Code for X86. + /// + /// It's bitwise identical to X86_64, so we just point to it. + /// + /// + public static readonly NativeCode I386 = X86_64; + } + + /// + /// Platform-dependent mmap and mprotect. + /// + internal interface IMemoryMapper + { + /// + /// Map at least numBytes of memory. Mark the page read-write (but not exec). + /// + IntPtr MapWriteable(int numBytes); + + /// + /// Sets the mapped memory to be read-exec (but not write). + /// + void SetReadExec(IntPtr mappedMemory, int numBytes); + } + + class WindowsMemoryMapper : IMemoryMapper + { + const UInt32 MEM_COMMIT = 0x1000; + const UInt32 MEM_RESERVE = 0x2000; + const UInt32 PAGE_READWRITE = 0x04; + const UInt32 PAGE_EXECUTE_READ = 0x20; + + [DllImport("kernel32.dll")] + static extern IntPtr VirtualAlloc(IntPtr lpAddress, IntPtr dwSize, UInt32 flAllocationType, UInt32 flProtect); + + [DllImport("kernel32.dll")] + static extern bool VirtualProtect(IntPtr lpAddress, IntPtr dwSize, UInt32 flNewProtect, out UInt32 lpflOldProtect); + + public IntPtr MapWriteable(int numBytes) + { + return VirtualAlloc(IntPtr.Zero, new IntPtr(numBytes), + MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); + } + + public void SetReadExec(IntPtr mappedMemory, int numBytes) + { + UInt32 _; + VirtualProtect(mappedMemory, new IntPtr(numBytes), PAGE_EXECUTE_READ, out _); + } + } + + class UnixMemoryMapper : IMemoryMapper + { + const int PROT_READ = 0x1; + const int PROT_WRITE = 0x2; + const int PROT_EXEC = 0x4; + + const int MAP_PRIVATE = 0x2; + int MAP_ANONYMOUS + { + get + { + switch (OperatingSystem) + { + case OperatingSystemType.Darwin: + return 0x1000; + case OperatingSystemType.Linux: + return 0x20; + default: + throw new NotImplementedException($"mmap is not supported on {OperatingSystemName}"); + } + } + } + + [DllImport("libc")] + static extern IntPtr mmap(IntPtr addr, IntPtr len, int prot, int flags, int fd, IntPtr offset); + + [DllImport("libc")] + static extern int mprotect(IntPtr addr, IntPtr len, int prot); + + public IntPtr MapWriteable(int numBytes) + { + // MAP_PRIVATE must be set on linux, even though MAP_ANON implies it. + // It doesn't hurt on darwin, so just do it. + return mmap(IntPtr.Zero, new IntPtr(numBytes), PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, IntPtr.Zero); + } + + public void SetReadExec(IntPtr mappedMemory, int numBytes) + { + mprotect(mappedMemory, new IntPtr(numBytes), PROT_READ | PROT_EXEC); + } + } + + /// + /// Initializes the data about platforms. + /// + /// This must be the last step when initializing the runtime: + /// GetManagedString needs to have the cached values for types. + /// But it must run before initializing anything outside the runtime + /// because those rely on the platform data. + /// + public static void InitializePlatformData() + { + MachineName = SystemInfo.GetArchitecture(); + Machine = SystemInfo.GetMachineType(); + OperatingSystem = SystemInfo.GetSystemType(); + } + + internal static IMemoryMapper CreateMemoryMapper() + { + switch (OperatingSystem) + { + case OperatingSystemType.Darwin: + case OperatingSystemType.Linux: + return new UnixMemoryMapper(); + case OperatingSystemType.Windows: + return new WindowsMemoryMapper(); + default: + throw new NotImplementedException($"No support for {OperatingSystemName}"); + } + } + + /// + /// Initializes the native code page. + /// + /// Safe to call if we already initialized (this function is idempotent). + /// + /// + internal static void InitializeNativeCodePage() + { + // Do nothing if we already initialized. + if (NativeCodePage != IntPtr.Zero) + { + return; + } + + // Allocate the page, write the native code into it, then set it + // to be executable. + IMemoryMapper mapper = CreateMemoryMapper(); + int codeLength = NativeCode.Active.Code.Length; + NativeCodePage = mapper.MapWriteable(codeLength); + Marshal.Copy(NativeCode.Active.Code, 0, NativeCodePage, codeLength); + mapper.SetReadExec(NativeCodePage, codeLength); + } + } +} diff --git a/src/runtime/platform/Types.cs b/src/runtime/platform/Types.cs index 62be0e421..15235da5a 100644 --- a/src/runtime/platform/Types.cs +++ b/src/runtime/platform/Types.cs @@ -1,3 +1,6 @@ +using System; +using System.Runtime.InteropServices; + namespace Python.Runtime.Platform { public enum MachineType @@ -20,4 +23,170 @@ public enum OperatingSystemType Linux, Other } + + + static class SystemInfo + { + public static MachineType GetMachineType() + { + return Runtime.IsWindows ? GetMachineType_Windows() : GetMachineType_Unix(); + } + + public static string GetArchitecture() + { + return Runtime.IsWindows ? GetArchName_Windows() : GetArchName_Unix(); + } + + public static OperatingSystemType GetSystemType() + { + if (Runtime.IsWindows) + { + return OperatingSystemType.Windows; + } + switch (PythonEngine.Platform) + { + case "linux": + return OperatingSystemType.Linux; + + case "darwin": + return OperatingSystemType.Darwin; + + default: + return OperatingSystemType.Other; + } + } + + #region WINDOWS + + static string GetArchName_Windows() + { + // https://docs.microsoft.com/en-us/windows/win32/winprog64/wow64-implementation-details + return Environment.GetEnvironmentVariable("PROCESSOR_ARCHITECTURE"); + } + + static MachineType GetMachineType_Windows() + { + if (Runtime.Is32Bit) + { + return MachineType.i386; + } + switch (GetArchName_Windows()) + { + case "AMD64": + return MachineType.x86_64; + case "ARM64": + return MachineType.aarch64; + default: + return MachineType.Other; + } + } + + #endregion + + #region UNIX + + + [StructLayout(LayoutKind.Sequential)] + unsafe struct utsname_linux + { + const int NameLength = 65; + + /* Name of the implementation of the operating system. */ + public fixed byte sysname[NameLength]; + + /* Name of this node on the network. */ + public fixed byte nodename[NameLength]; + + /* Current release level of this implementation. */ + public fixed byte release[NameLength]; + + /* Current version level of this release. */ + public fixed byte version[NameLength]; + + /* Name of the hardware type the system is running on. */ + public fixed byte machine[NameLength]; + + // GNU extension + fixed byte domainname[NameLength]; /* NIS or YP domain name */ + } + + [StructLayout(LayoutKind.Sequential)] + unsafe struct utsname_darwin + { + const int NameLength = 256; + + /* Name of the implementation of the operating system. */ + public fixed byte sysname[NameLength]; + + /* Name of this node on the network. */ + public fixed byte nodename[NameLength]; + + /* Current release level of this implementation. */ + public fixed byte release[NameLength]; + + /* Current version level of this release. */ + public fixed byte version[NameLength]; + + /* Name of the hardware type the system is running on. */ + public fixed byte machine[NameLength]; + } + + [DllImport("libc")] + static extern int uname(IntPtr buf); + + + static unsafe string GetArchName_Unix() + { + switch (GetSystemType()) + { + case OperatingSystemType.Linux: + { + var buf = stackalloc utsname_linux[1]; + if (uname((IntPtr)buf) != 0) + { + return null; + } + return Marshal.PtrToStringAnsi((IntPtr)buf->machine); + } + + case OperatingSystemType.Darwin: + { + var buf = stackalloc utsname_darwin[1]; + if (uname((IntPtr)buf) != 0) + { + return null; + } + return Marshal.PtrToStringAnsi((IntPtr)buf->machine); + } + + default: + return null; + } + } + + static unsafe MachineType GetMachineType_Unix() + { + switch (GetArchName_Unix()) + { + case "x86_64": + case "em64t": + return Runtime.Is32Bit ? MachineType.i386 : MachineType.x86_64; + case "i386": + case "i686": + return MachineType.i386; + + case "armv7l": + return MachineType.armv7l; + case "armv8": + return Runtime.Is32Bit ? MachineType.armv7l : MachineType.armv8; + case "aarch64": + return Runtime.Is32Bit ? MachineType.armv7l : MachineType.aarch64; + + default: + return MachineType.Other; + } + } + + #endregion + } } diff --git a/src/runtime/propertyobject.cs b/src/runtime/propertyobject.cs index f2c97f163..ac1d077f9 100644 --- a/src/runtime/propertyobject.cs +++ b/src/runtime/propertyobject.cs @@ -7,6 +7,7 @@ namespace Python.Runtime /// /// Implements a Python descriptor type that manages CLR properties. /// + [Serializable] internal class PropertyObject : ExtensionType { private PropertyInfo info; diff --git a/src/runtime/pylist.cs b/src/runtime/pylist.cs index dcb0ea78f..7f5566401 100644 --- a/src/runtime/pylist.cs +++ b/src/runtime/pylist.cs @@ -93,7 +93,6 @@ public PyList(PyObject[] items) : base(FromArray(items)) { } - /// /// IsListType Method /// diff --git a/src/runtime/pyobject.cs b/src/runtime/pyobject.cs index a7f5e8305..9328312ce 100644 --- a/src/runtime/pyobject.cs +++ b/src/runtime/pyobject.cs @@ -20,7 +20,8 @@ public interface IPyDisposable : IDisposable /// PY3: https://docs.python.org/3/c-api/object.html /// for details. /// - public class PyObject : DynamicObject, IEnumerable, IPyDisposable + [Serializable] + public partial class PyObject : DynamicObject, IEnumerable, IPyDisposable { #if TRACE_ALLOC /// diff --git a/src/runtime/pyscope.cs b/src/runtime/pyscope.cs index 9b9c78bae..fee78b40a 100644 --- a/src/runtime/pyscope.cs +++ b/src/runtime/pyscope.cs @@ -278,10 +278,9 @@ public PyObject Eval(string code, PyDict locals = null) { Check(); IntPtr _locals = locals == null ? variables : locals.obj; - var flag = (IntPtr)Runtime.Py_eval_input; NewReference reference = Runtime.PyRun_String( - code, flag, variables, _locals + code, RunFlagType.Eval, variables, _locals ); PythonException.ThrowIfIsNull(reference); return reference.MoveToPyObject(); @@ -317,9 +316,8 @@ public void Exec(string code, PyDict locals = null) private void Exec(string code, IntPtr _globals, IntPtr _locals) { - var flag = (IntPtr)Runtime.Py_file_input; NewReference reference = Runtime.PyRun_String( - code, flag, _globals, _locals + code, RunFlagType.File, _globals, _locals ); PythonException.ThrowIfIsNull(reference); reference.Dispose(); diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index 38ef2ee48..1d688ef9a 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -12,6 +12,14 @@ namespace Python.Runtime /// public class PythonEngine : IDisposable { + public static ShutdownMode ShutdownMode + { + get => Runtime.ShutdownMode; + set => Runtime.ShutdownMode = value; + } + + public static ShutdownMode DefaultShutdownMode => Runtime.GetDefaultShutdownMode(); + private static DelegateManager delegateManager; private static bool initialized; private static IntPtr _pythonHome = IntPtr.Zero; @@ -147,9 +155,9 @@ public static void Initialize() Initialize(setSysArgv: true); } - public static void Initialize(bool setSysArgv = true, bool initSigs = false) + public static void Initialize(bool setSysArgv = true, bool initSigs = false, ShutdownMode mode = ShutdownMode.Default) { - Initialize(Enumerable.Empty(), setSysArgv: setSysArgv, initSigs: initSigs); + Initialize(Enumerable.Empty(), setSysArgv: setSysArgv, initSigs: initSigs, mode); } /// @@ -162,82 +170,85 @@ public static void Initialize(bool setSysArgv = true, bool initSigs = false) /// interpreter lock (GIL) to call this method. /// initSigs can be set to 1 to do default python signal configuration. This will override the way signals are handled by the application. /// - public static void Initialize(IEnumerable args, bool setSysArgv = true, bool initSigs = false) + public static void Initialize(IEnumerable args, bool setSysArgv = true, bool initSigs = false, ShutdownMode mode = ShutdownMode.Default) { - if (!initialized) + if (initialized) { - // Creating the delegateManager MUST happen before Runtime.Initialize - // is called. If it happens afterwards, DelegateManager's CodeGenerator - // throws an exception in its ctor. This exception is eaten somehow - // during an initial "import clr", and the world ends shortly thereafter. - // This is probably masking some bad mojo happening somewhere in Runtime.Initialize(). - delegateManager = new DelegateManager(); - Runtime.Initialize(initSigs); - initialized = true; - Exceptions.Clear(); - - // Make sure we clean up properly on app domain unload. - AppDomain.CurrentDomain.DomainUnload += OnDomainUnload; - - // Remember to shut down the runtime. - AddShutdownHandler(Runtime.Shutdown); - - // The global scope gets used implicitly quite early on, remember - // to clear it out when we shut down. - AddShutdownHandler(PyScopeManager.Global.Clear); - - if (setSysArgv) - { - Py.SetArgv(args); - } + return; + } + // Creating the delegateManager MUST happen before Runtime.Initialize + // is called. If it happens afterwards, DelegateManager's CodeGenerator + // throws an exception in its ctor. This exception is eaten somehow + // during an initial "import clr", and the world ends shortly thereafter. + // This is probably masking some bad mojo happening somewhere in Runtime.Initialize(). + delegateManager = new DelegateManager(); + Runtime.Initialize(initSigs, mode); + initialized = true; + Exceptions.Clear(); + + // Make sure we clean up properly on app domain unload. + AppDomain.CurrentDomain.DomainUnload += OnDomainUnload; + + // The global scope gets used implicitly quite early on, remember + // to clear it out when we shut down. + AddShutdownHandler(PyScopeManager.Global.Clear); + + if (setSysArgv) + { + Py.SetArgv(args); + } + if (mode == ShutdownMode.Normal) + { + // TOOD: Check if this can be remove completely or not. // register the atexit callback (this doesn't use Py_AtExit as the C atexit // callbacks are called after python is fully finalized but the python ones // are called while the python engine is still running). - string code = - "import atexit, clr\n" + - "atexit.register(clr._AtExit)\n"; - PythonEngine.Exec(code); + //string code = + // "import atexit, clr\n" + + // "atexit.register(clr._AtExit)\n"; + //PythonEngine.Exec(code); + } - // Load the clr.py resource into the clr module - IntPtr clr = Python.Runtime.ImportHook.GetCLRModule(); - IntPtr clr_dict = Runtime.PyModule_GetDict(clr); + // Load the clr.py resource into the clr module + IntPtr clr = Python.Runtime.ImportHook.GetCLRModule(); + IntPtr clr_dict = Runtime.PyModule_GetDict(clr); - var locals = new PyDict(); - try + var locals = new PyDict(); + try + { + IntPtr module = Runtime.PyImport_AddModule("clr._extras"); + IntPtr module_globals = Runtime.PyModule_GetDict(module); + IntPtr builtins = Runtime.PyEval_GetBuiltins(); + Runtime.PyDict_SetItemString(module_globals, "__builtins__", builtins); + + Assembly assembly = Assembly.GetExecutingAssembly(); + using (Stream stream = assembly.GetManifestResourceStream("clr.py")) + using (var reader = new StreamReader(stream)) { - IntPtr module = Runtime.PyImport_AddModule("clr._extras"); - IntPtr module_globals = Runtime.PyModule_GetDict(module); - IntPtr builtins = Runtime.PyEval_GetBuiltins(); - Runtime.PyDict_SetItemString(module_globals, "__builtins__", builtins); - - Assembly assembly = Assembly.GetExecutingAssembly(); - using (Stream stream = assembly.GetManifestResourceStream("clr.py")) - using (var reader = new StreamReader(stream)) - { - // add the contents of clr.py to the module - string clr_py = reader.ReadToEnd(); - Exec(clr_py, module_globals, locals.Handle); - } + // add the contents of clr.py to the module + string clr_py = reader.ReadToEnd(); + Exec(clr_py, module_globals, locals.Handle); + } - // add the imported module to the clr module, and copy the API functions - // and decorators into the main clr module. - Runtime.PyDict_SetItemString(clr_dict, "_extras", module); - foreach (PyObject key in locals.Keys()) + // add the imported module to the clr module, and copy the API functions + // and decorators into the main clr module. + Runtime.PyDict_SetItemString(clr_dict, "_extras", module); + using (var keys = locals.Keys()) + foreach (PyObject key in keys) + { + if (!key.ToString().StartsWith("_") || key.ToString().Equals("__version__")) { - if (!key.ToString().StartsWith("_") || key.ToString().Equals("__version__")) - { - PyObject value = locals[key]; - Runtime.PyDict_SetItem(clr_dict, key.Handle, value.Handle); - value.Dispose(); - } - key.Dispose(); + PyObject value = locals[key]; + Runtime.PyDict_SetItem(clr_dict, key.Handle, value.Handle); + value.Dispose(); } + key.Dispose(); } - finally - { - locals.Dispose(); - } + } + finally + { + locals.Dispose(); } } @@ -305,22 +316,37 @@ public static IntPtr InitExt() /// Python runtime can no longer be used in the current process /// after calling the Shutdown method. /// - public static void Shutdown() + /// The ShutdownMode to use when shutting down the Runtime + public static void Shutdown(ShutdownMode mode) { - if (initialized) + if (!initialized) { - PyScopeManager.Global.Clear(); - - // If the shutdown handlers trigger a domain unload, - // don't call shutdown again. - AppDomain.CurrentDomain.DomainUnload -= OnDomainUnload; + return; + } + // If the shutdown handlers trigger a domain unload, + // don't call shutdown again. + AppDomain.CurrentDomain.DomainUnload -= OnDomainUnload; - ExecuteShutdownHandlers(); + PyScopeManager.Global.Clear(); + ExecuteShutdownHandlers(); + // Remember to shut down the runtime. + Runtime.Shutdown(mode); + PyObjectConversions.Reset(); - PyObjectConversions.Reset(); + initialized = false; + } - initialized = false; - } + /// + /// Shutdown Method + /// + /// + /// Shutdown and release resources held by the Python runtime. The + /// Python runtime can no longer be used in the current process + /// after calling the Shutdown method. + /// + public static void Shutdown() + { + Shutdown(Runtime.ShutdownMode); } /// @@ -585,7 +611,7 @@ internal static PyObject RunString(string code, IntPtr? globals, IntPtr? locals, try { NewReference result = Runtime.PyRun_String( - code, (IntPtr)flag, globals.Value, locals.Value + code, flag, globals.Value, locals.Value ); PythonException.ThrowIfIsNull(result); return result.MoveToPyObject(); diff --git a/src/runtime/pythonexception.cs b/src/runtime/pythonexception.cs index b69ac34e4..893bd9491 100644 --- a/src/runtime/pythonexception.cs +++ b/src/runtime/pythonexception.cs @@ -187,6 +187,11 @@ public string Format() return res; } + public bool IsMatches(IntPtr exc) + { + return Runtime.PyErr_GivenExceptionMatches(PyType, exc) != 0; + } + /// /// Dispose Method /// diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 2d523758b..83d404f9d 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -7,6 +7,7 @@ using System.Threading; using System.Collections.Generic; using Python.Runtime.Platform; +using System.Linq; namespace Python.Runtime { @@ -86,7 +87,9 @@ public class Runtime internal static object IsFinalizingLock = new object(); internal static bool IsFinalizing; - internal static bool Is32Bit => IntPtr.Size == 4; + private static bool _isInitialized = false; + + internal static readonly bool Is32Bit = IntPtr.Size == 4; // .NET core: System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(OSPlatform.Windows) internal static bool IsWindows = Environment.OSVersion.Platform == PlatformID.Win32NT; @@ -94,49 +97,6 @@ public class Runtime internal static Version InteropVersion { get; } = System.Reflection.Assembly.GetExecutingAssembly().GetName().Version; - static readonly Dictionary OperatingSystemTypeMapping = new Dictionary() - { - { "Windows", OperatingSystemType.Windows }, - { "Darwin", OperatingSystemType.Darwin }, - { "Linux", OperatingSystemType.Linux }, - }; - - [Obsolete] - public static string OperatingSystemName => OperatingSystem.ToString(); - - [Obsolete] - public static string MachineName => Machine.ToString(); - - /// - /// Gets the operating system as reported by python's platform.system(). - /// - public static OperatingSystemType OperatingSystem { get; private set; } - - /// - /// Map lower-case version of the python machine name to the processor - /// type. There are aliases, e.g. x86_64 and amd64 are two names for - /// the same thing. Make sure to lower-case the search string, because - /// capitalization can differ. - /// - static readonly Dictionary MachineTypeMapping = new Dictionary() - { - ["i386"] = MachineType.i386, - ["i686"] = MachineType.i386, - ["x86"] = MachineType.i386, - ["x86_64"] = MachineType.x86_64, - ["amd64"] = MachineType.x86_64, - ["x64"] = MachineType.x86_64, - ["em64t"] = MachineType.x86_64, - ["armv7l"] = MachineType.armv7l, - ["armv8"] = MachineType.armv8, - ["aarch64"] = MachineType.aarch64, - }; - - /// - /// Gets the machine architecture as reported by python's platform.machine(). - /// - public static MachineType Machine { get; private set; }/* set in Initialize using python's platform.machine */ - public static int MainManagedThreadId { get; private set; } /// @@ -144,6 +104,7 @@ public class Runtime /// internal static readonly Encoding PyEncoding = _UCS == 2 ? Encoding.Unicode : Encoding.UTF32; + public static ShutdownMode ShutdownMode { get; internal set; } private static PyReferenceCollection _pyRefs = new PyReferenceCollection(); internal static Version PyVersion @@ -162,18 +123,44 @@ internal static Version PyVersion /// /// Initialize the runtime... /// - internal static void Initialize(bool initSigs = false) + /// Always call this method from the Main thread. After the + /// first call to this method, the main thread has acquired the GIL. + internal static void Initialize(bool initSigs = false, ShutdownMode mode = ShutdownMode.Default) { + if (_isInitialized) + { + return; + } + _isInitialized = true; + + if (mode == ShutdownMode.Default) + { + mode = GetDefaultShutdownMode(); + } + ShutdownMode = mode; + if (Py_IsInitialized() == 0) { Py_InitializeEx(initSigs ? 1 : 0); - MainManagedThreadId = Thread.CurrentThread.ManagedThreadId; + if (PyEval_ThreadsInitialized() == 0) + { + PyEval_InitThreads(); + } + // XXX: Reload mode may reduct to Soft mode, + // so even on Reload mode it still needs to save the RuntimeState + if (mode == ShutdownMode.Soft || mode == ShutdownMode.Reload) + { + RuntimeState.Save(); + } } - - if (PyEval_ThreadsInitialized() == 0) + else { - PyEval_InitThreads(); + // If we're coming back from a domain reload or a soft shutdown, + // we have previously released the thread state. Restore the main + // thread state here. + PyGILState_Ensure(); } + MainManagedThreadId = Thread.CurrentThread.ManagedThreadId; IsFinalizing = false; @@ -181,8 +168,43 @@ internal static void Initialize(bool initSigs = false) PyScopeManager.Reset(); ClassManager.Reset(); ClassDerivedObject.Reset(); - TypeManager.Reset(); + TypeManager.Initialize(); + InitPyMembers(); + + // Initialize data about the platform we're running on. We need + // this for the type manager and potentially other details. Must + // happen after caching the python types, above. + NativeCodePageHelper.InitializePlatformData(); + + // Initialize modules that depend on the runtime class. + AssemblyManager.Initialize(); + if (mode == ShutdownMode.Reload && RuntimeData.HasStashData()) + { + RuntimeData.RestoreRuntimeData(); + } + else + { + PyCLRMetaType = MetaType.Initialize(); // Steal a reference + ImportHook.Initialize(); + } + Exceptions.Initialize(); + + // Need to add the runtime directory to sys.path so that we + // can find built-in assemblies like System.Data, et. al. + string rtdir = RuntimeEnvironment.GetRuntimeDirectory(); + IntPtr path = PySys_GetObject("path").DangerousGetAddress(); + IntPtr item = PyString_FromString(rtdir); + if (PySequence_Contains(path, item) == 0) + { + PyList_Append(new BorrowedReference(path), item); + } + XDecref(item); + AssemblyManager.UpdatePath(); + } + + private static void InitPyMembers() + { IntPtr op; { var builtins = GetBuiltins(); @@ -276,129 +298,141 @@ internal static void Initialize(bool initSigs = false) Error = new IntPtr(-1); - // Initialize data about the platform we're running on. We need - // this for the type manager and potentially other details. Must - // happen after caching the python types, above. - InitializePlatformData(); + _PyObject_NextNotImplemented = Get_PyObject_NextNotImplemented(); + { + IntPtr sys = PyImport_ImportModule("sys"); + PyModuleType = PyObject_Type(sys); + XDecref(sys); + } + } - IntPtr dllLocal = IntPtr.Zero; - var loader = LibraryLoader.Get(OperatingSystem); + private static IntPtr Get_PyObject_NextNotImplemented() + { + IntPtr pyType = SlotHelper.CreateObjectType(); + IntPtr iternext = Marshal.ReadIntPtr(pyType, TypeOffset.tp_iternext); + Runtime.XDecref(pyType); + return iternext; + } - if (_PythonDll != "__Internal") + /// + /// Tries to downgrade the shutdown mode, if possible. + /// The only possibles downgrades are: + /// Soft -> Normal + /// Reload -> Soft + /// Reload -> Normal + /// + /// The desired shutdown mode + /// The `mode` parameter if the downgrade is supported, the ShutdownMode + /// set at initialization otherwise. + static ShutdownMode TryDowngradeShutdown(ShutdownMode mode) + { + if ( + mode == Runtime.ShutdownMode + || mode == ShutdownMode.Normal + || (mode == ShutdownMode.Soft && Runtime.ShutdownMode == ShutdownMode.Reload) + ) { - dllLocal = loader.Load(_PythonDll); + return mode; } - _PyObject_NextNotImplemented = loader.GetFunction(dllLocal, "_PyObject_NextNotImplemented"); - PyModuleType = loader.GetFunction(dllLocal, "PyModule_Type"); - - if (dllLocal != IntPtr.Zero) + else // we can't downgrade { - loader.Free(dllLocal); + return Runtime.ShutdownMode; } - - // Initialize modules that depend on the runtime class. - AssemblyManager.Initialize(); - PyCLRMetaType = MetaType.Initialize(); - Exceptions.Initialize(); - ImportHook.Initialize(); - - // Need to add the runtime directory to sys.path so that we - // can find built-in assemblies like System.Data, et. al. - string rtdir = RuntimeEnvironment.GetRuntimeDirectory(); - BorrowedReference path = PySys_GetObject("path"); - IntPtr item = PyString_FromString(rtdir); - PyList_Append(path, item); - XDecref(item); - AssemblyManager.UpdatePath(); } - /// - /// Initializes the data about platforms. - /// - /// This must be the last step when initializing the runtime: - /// GetManagedString needs to have the cached values for types. - /// But it must run before initializing anything outside the runtime - /// because those rely on the platform data. - /// - private static void InitializePlatformData() + internal static void Shutdown(ShutdownMode mode) { -#if !NETSTANDARD - IntPtr op; - IntPtr fn; - IntPtr platformModule = PyImport_ImportModule("platform"); - IntPtr emptyTuple = PyTuple_New(0); - - fn = PyObject_GetAttrString(platformModule, "system"); - op = PyObject_Call(fn, emptyTuple, IntPtr.Zero); - string operatingSystemName = GetManagedString(op); - XDecref(op); - XDecref(fn); + if (Py_IsInitialized() == 0 || !_isInitialized) + { + return; + } + _isInitialized = false; - fn = PyObject_GetAttrString(platformModule, "machine"); - op = PyObject_Call(fn, emptyTuple, IntPtr.Zero); - string machineName = GetManagedString(op); - XDecref(op); - XDecref(fn); + // If the shutdown mode specified is not the the same as the one specified + // during Initialization, we need to validate it; we can only downgrade, + // not upgrade the shutdown mode. + mode = TryDowngradeShutdown(mode); - XDecref(emptyTuple); - XDecref(platformModule); + var state = PyGILState_Ensure(); - // Now convert the strings into enum values so we can do switch - // statements rather than constant parsing. - OperatingSystemType OSType; - if (!OperatingSystemTypeMapping.TryGetValue(operatingSystemName, out OSType)) + if (mode == ShutdownMode.Soft) + { + RunExitFuncs(); + } + if (mode == ShutdownMode.Reload) { - OSType = OperatingSystemType.Other; + RuntimeData.Stash(); } - OperatingSystem = OSType; + AssemblyManager.Shutdown(); + ImportHook.Shutdown(); + + ClearClrModules(); + RemoveClrRootModule(); + + MoveClrInstancesOnwershipToPython(); + ClassManager.DisposePythonWrappersForClrTypes(); + TypeManager.RemoveTypes(); - MachineType MType; - if (!MachineTypeMapping.TryGetValue(machineName.ToLower(), out MType)) + MetaType.Release(); + PyCLRMetaType = IntPtr.Zero; + + Exceptions.Shutdown(); + Finalizer.Shutdown(); + + if (mode != ShutdownMode.Normal) { - MType = MachineType.Other; + PyGC_Collect(); + if (mode == ShutdownMode.Soft) + { + RuntimeState.Restore(); + } + ResetPyMembers(); + GC.Collect(); + try + { + GC.WaitForFullGCComplete(); + } + catch (NotImplementedException) + { + // Some clr runtime didn't implement GC.WaitForFullGCComplete yet. + } + GC.WaitForPendingFinalizers(); + PyGILState_Release(state); + // Then release the GIL for good, if there is somehting to release + // Use the unchecked version as the checked version calls `abort()` + // if the current state is NULL. + if (_PyThreadState_UncheckedGet() != IntPtr.Zero) + { + PyEval_SaveThread(); + } + } - Machine = MType; -#else - if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) - OperatingSystem = OperatingSystemType.Linux; - else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) - OperatingSystem = OperatingSystemType.Darwin; - else if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - OperatingSystem = OperatingSystemType.Windows; else - OperatingSystem = OperatingSystemType.Other; - - switch (RuntimeInformation.ProcessArchitecture) { - case Architecture.X86: - Machine = MachineType.i386; - break; - case Architecture.X64: - Machine = MachineType.x86_64; - break; - case Architecture.Arm: - Machine = MachineType.armv7l; - break; - case Architecture.Arm64: - Machine = MachineType.aarch64; - break; - default: - Machine = MachineType.Other; - break; + ResetPyMembers(); + Py_Finalize(); } -#endif } internal static void Shutdown() { - AssemblyManager.Shutdown(); - Exceptions.Shutdown(); - ImportHook.Shutdown(); - Finalizer.Shutdown(); - // TOOD: PyCLRMetaType's release operation still in #958 - PyCLRMetaType = IntPtr.Zero; - ResetPyMembers(); - Py_Finalize(); + var mode = ShutdownMode; + Shutdown(mode); + } + + internal static ShutdownMode GetDefaultShutdownMode() + { + string modeEvn = Environment.GetEnvironmentVariable("PYTHONNET_SHUTDOWN_MODE"); + if (modeEvn == null) + { + return ShutdownMode.Normal; + } + ShutdownMode mode; + if (Enum.TryParse(modeEvn, true, out mode)) + { + return mode; + } + return ShutdownMode.Normal; } // called *without* the GIL acquired by clr._AtExit @@ -411,6 +445,37 @@ internal static int AtExit() return 0; } + private static void RunExitFuncs() + { + PyObject atexit; + try + { + atexit = Py.Import("atexit"); + } + catch (PythonException e) + { + if (!e.IsMatches(Exceptions.ImportError)) + { + throw; + } + e.Dispose(); + // The runtime may not provided `atexit` module. + return; + } + using (atexit) + { + try + { + atexit.InvokeMethod("_run_exitfuncs").Dispose(); + } + catch (PythonException e) + { + Console.Error.WriteLine(e); + e.Dispose(); + } + } + } + private static void SetPyMember(ref IntPtr obj, IntPtr value, Action onRelease) { // XXX: For current usages, value should not be null. @@ -424,9 +489,75 @@ private static void ResetPyMembers() _pyRefs.Release(); } - internal static IntPtr Py_single_input = (IntPtr)256; - internal static IntPtr Py_file_input = (IntPtr)257; - internal static IntPtr Py_eval_input = (IntPtr)258; + private static void ClearClrModules() + { + var modules = PyImport_GetModuleDict(); + var items = PyDict_Items(modules); + long length = PyList_Size(items); + for (long i = 0; i < length; i++) + { + var item = PyList_GetItem(items, i); + var name = PyTuple_GetItem(item.DangerousGetAddress(), 0); + var module = PyTuple_GetItem(item.DangerousGetAddress(), 1); + if (ManagedType.IsManagedType(module)) + { + PyDict_DelItem(modules, name); + } + } + items.Dispose(); + } + + private static void RemoveClrRootModule() + { + var modules = PyImport_GetModuleDict(); + PyDictTryDelItem(modules, "CLR"); + PyDictTryDelItem(modules, "clr"); + PyDictTryDelItem(modules, "clr._extra"); + } + + private static void PyDictTryDelItem(IntPtr dict, string key) + { + if (PyDict_DelItemString(dict, key) == 0) + { + return; + } + if (!PythonException.Matches(Exceptions.KeyError)) + { + throw new PythonException(); + } + PyErr_Clear(); + } + + private static void MoveClrInstancesOnwershipToPython() + { + var objs = ManagedType.GetManagedObjects(); + var copyObjs = objs.ToArray(); + foreach (var entry in copyObjs) + { + ManagedType obj = entry.Key; + if (!objs.ContainsKey(obj)) + { + System.Diagnostics.Debug.Assert(obj.gcHandle == default); + continue; + } + if (entry.Value == ManagedType.TrackTypes.Extension) + { + obj.CallTypeClear(); + // obj's tp_type will degenerate to a pure Python type after TypeManager.RemoveTypes(), + // thus just be safe to give it back to GC chain. + if (!_PyObject_GC_IS_TRACKED(obj.pyHandle)) + { + PyObject_GC_Track(obj.pyHandle); + } + } + if (obj.gcHandle.IsAllocated) + { + obj.gcHandle.Free(); + } + obj.gcHandle = default; + } + ManagedType.ClearTrackedObjects(); + } internal static IntPtr PyBaseObjectType; internal static IntPtr PyModuleType; @@ -647,7 +778,7 @@ internal static unsafe void XDecref(IntPtr op) { return; } - NativeCall.Impl.Void_Call_1(new IntPtr(f), op); + NativeCall.Void_Call_1(new IntPtr(f), op); } } #endif @@ -656,7 +787,11 @@ internal static unsafe void XDecref(IntPtr op) [Pure] internal static unsafe long Refcount(IntPtr op) { +#if PYTHON_WITH_PYDEBUG + var p = (void*)(op + TypeOffset.ob_refcnt); +#else var p = (void*)op; +#endif if ((void*)0 == p) { return 0; @@ -704,6 +839,9 @@ internal static unsafe long Refcount(IntPtr op) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyThreadState_Get(); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr _PyThreadState_UncheckedGet(); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyThread_get_key_value(IntPtr key); @@ -802,7 +940,7 @@ public static extern int Py_Main( internal static extern int PyRun_SimpleString(string code); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern NewReference PyRun_String([MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(Utf8Marshaler))] string code, IntPtr st, IntPtr globals, IntPtr locals); + internal static extern NewReference PyRun_String([MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(Utf8Marshaler))] string code, RunFlagType st, IntPtr globals, IntPtr locals); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyEval_EvalCode(IntPtr co, IntPtr globals, IntPtr locals); @@ -851,6 +989,7 @@ internal static IntPtr Py_CompileStringFlags(string str, string file, int start, //==================================================================== /// + /// Return value: Borrowed reference. /// A macro-like method to get the type of a Python object. This is /// designed to be lean and mean in IL & avoid managed <-> unmanaged /// transitions. Note that this does not incref the type object. @@ -907,6 +1046,9 @@ internal static bool PyObject_IsIterable(IntPtr pointer) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyObject_GetAttrString(IntPtr pointer, string name); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr PyObject_GetAttrString(IntPtr pointer, IntPtr name); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyObject_SetAttrString(IntPtr pointer, string name, IntPtr value); @@ -1004,6 +1146,11 @@ internal static long PyObject_Size(IntPtr pointer) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyObject_Dir(IntPtr pointer); +#if PYTHON_WITH_PYDEBUG + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern void _Py_NewReference(IntPtr ob); +#endif + //==================================================================== // Python buffer API //==================================================================== @@ -1152,6 +1299,19 @@ internal static bool PyFloat_Check(IntPtr ob) return PyObject_TYPE(ob) == PyFloatType; } + /// + /// Return value: New reference. + /// Create a Python integer from the pointer p. The pointer value can be retrieved from the resulting value using PyLong_AsVoidPtr(). + /// + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr PyLong_FromVoidPtr(IntPtr p); + + /// + /// Convert a Python integer pylong to a C void pointer. If pylong cannot be converted, an OverflowError will be raised. This is only assured to produce a usable void pointer for values created with PyLong_FromVoidPtr(). + /// + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr PyLong_AsVoidPtr(IntPtr ob); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyFloat_FromDouble(double value); @@ -1484,18 +1644,34 @@ internal static bool PyDict_Check(IntPtr ob) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyDict_New(); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyDict_Next(IntPtr p, out IntPtr ppos, out IntPtr pkey, out IntPtr pvalue); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyDictProxy_New(IntPtr dict); + /// + /// Return value: Borrowed reference. + /// Return NULL if the key key is not present, but without setting an exception. + /// [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyDict_GetItem(IntPtr pointer, IntPtr key); + /// + /// Return value: Borrowed reference. + /// [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyDict_GetItemString(IntPtr pointer, string key); + /// + /// Return 0 on success or -1 on failure. + /// [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyDict_SetItem(IntPtr pointer, IntPtr key, IntPtr value); + /// + /// Return 0 on success or -1 on failure. + /// [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyDict_SetItemString(IntPtr pointer, string key, IntPtr value); @@ -1535,6 +1711,21 @@ internal static long PyDict_Size(IntPtr pointer) internal static extern IntPtr _PyDict_Size(IntPtr pointer); + /// + /// Return value: New reference. + /// + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr PySet_New(IntPtr iterable); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PySet_Add(IntPtr set, IntPtr key); + + /// + /// Return 1 if found, 0 if not found, and -1 if an error is encountered. + /// + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PySet_Contains(IntPtr anyset, IntPtr key); + //==================================================================== // Python list API //==================================================================== @@ -1693,7 +1884,11 @@ internal static bool PyIter_Check(IntPtr pointer) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern string PyModule_GetFilename(IntPtr module); +#if PYTHON_WITH_PYDEBUG + [DllImport(_PythonDll, EntryPoint = "PyModule_Create2TraceRefs", CallingConvention = CallingConvention.Cdecl)] +#else [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] +#endif internal static extern IntPtr PyModule_Create2(IntPtr module, int apiver); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] @@ -1721,6 +1916,10 @@ internal static extern void PySys_SetArgvEx( int updatepath ); + /// + /// Return value: Borrowed reference. + /// Return the object name from the sys module or NULL if it does not exist, without setting an exception. + /// [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern BorrowedReference PySys_GetObject(string name); @@ -1765,6 +1964,9 @@ internal static IntPtr PyType_GenericAlloc(IntPtr type, long n) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] private static extern IntPtr PyType_GenericAlloc(IntPtr type, IntPtr n); + /// + /// Finalize a type object. This should be called on all type objects to finish their initialization. This function is responsible for adding inherited slots from a type’s base class. Return 0 on success, or return -1 and sets an exception on error. + /// [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyType_Ready(IntPtr type); @@ -1789,6 +1991,8 @@ internal static IntPtr PyType_GenericAlloc(IntPtr type, long n) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void PyObject_GC_UnTrack(IntPtr tp); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern void _PyObject_Dump(IntPtr ob); //==================================================================== // Python memory API @@ -1864,6 +2068,74 @@ internal static IntPtr PyMem_Realloc(IntPtr ptr, long size) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyCell_Set(BorrowedReference cell, IntPtr value); + //==================================================================== + // Python GC API + //==================================================================== + + internal const int _PyGC_REFS_SHIFT = 1; + internal const long _PyGC_REFS_UNTRACKED = -2; + internal const long _PyGC_REFS_REACHABLE = -3; + internal const long _PyGC_REFS_TENTATIVELY_UNREACHABLE = -4; + + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr PyGC_Collect(); + + internal static IntPtr _Py_AS_GC(IntPtr ob) + { + // XXX: PyGC_Head has a force alignment depend on platform. + // See PyGC_Head in objimpl.h for more details. + return Is32Bit ? ob - 16 : ob - 24; + } + + internal static IntPtr _Py_FROM_GC(IntPtr gc) + { + return Is32Bit ? gc + 16 : gc + 24; + } + + internal static IntPtr _PyGCHead_REFS(IntPtr gc) + { + unsafe + { + var pGC = (PyGC_Head*)gc; + var refs = pGC->gc.gc_refs; + if (Is32Bit) + { + return new IntPtr(refs.ToInt32() >> _PyGC_REFS_SHIFT); + } + return new IntPtr(refs.ToInt64() >> _PyGC_REFS_SHIFT); + } + } + + internal static IntPtr _PyGC_REFS(IntPtr ob) + { + return _PyGCHead_REFS(_Py_AS_GC(ob)); + } + + internal static bool _PyObject_GC_IS_TRACKED(IntPtr ob) + { + return (long)_PyGC_REFS(ob) != _PyGC_REFS_UNTRACKED; + } + + internal static void Py_CLEAR(ref IntPtr ob) + { + XDecref(ob); + ob = IntPtr.Zero; + } + + //==================================================================== + // Python Capsules API + //==================================================================== + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern NewReference PyCapsule_New(IntPtr pointer, string name, IntPtr destructor); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr PyCapsule_GetPointer(BorrowedReference capsule, string name); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyCapsule_SetPointer(BorrowedReference capsule, IntPtr pointer); + //==================================================================== // Miscellaneous //==================================================================== @@ -1882,12 +2154,16 @@ internal static IntPtr PyMem_Realloc(IntPtr ptr, long size) internal static void SetNoSiteFlag() { - var loader = LibraryLoader.Get(OperatingSystem); - - IntPtr dllLocal = _PythonDll != "__Internal" - ? loader.Load(_PythonDll) - : IntPtr.Zero; - + var loader = LibraryLoader.Get(NativeCodePageHelper.OperatingSystem); + IntPtr dllLocal; + if (_PythonDll != "__Internal") + { + dllLocal = loader.Load(_PythonDll); + if (dllLocal == IntPtr.Zero) + { + throw new Exception($"Cannot load {_PythonDll}"); + } + } try { Py_NoSiteFlag = loader.GetFunction(dllLocal, "Py_NoSiteFlag"); @@ -1912,6 +2188,15 @@ internal static IntPtr GetBuiltins() } + public enum ShutdownMode + { + Default, + Normal, + Soft, + Reload, + } + + class PyReferenceCollection { private List> _actions = new List>(); diff --git a/src/runtime/runtime_data.cs b/src/runtime/runtime_data.cs new file mode 100644 index 000000000..060573db4 --- /dev/null +++ b/src/runtime/runtime_data.cs @@ -0,0 +1,447 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; +using System.Runtime.Serialization; +using System.Runtime.Serialization.Formatters.Binary; + +using static Python.Runtime.Runtime; + +namespace Python.Runtime +{ + public static class RuntimeData + { + private static Type _formatterType; + public static Type FormatterType + { + get => _formatterType; + set + { + if (!typeof(IFormatter).IsAssignableFrom(value)) + { + throw new ArgumentException("Not a type implemented IFormatter"); + } + _formatterType = value; + } + } + + public static ICLRObjectStorer WrappersStorer { get; set; } + + /// + /// Clears the old "clr_data" entry if a previous one is present. + /// + static void ClearCLRData () + { + BorrowedReference capsule = PySys_GetObject("clr_data"); + if (!capsule.IsNull) + { + IntPtr oldData = PyCapsule_GetPointer(capsule, null); + PyMem_Free(oldData); + PyCapsule_SetPointer(capsule, IntPtr.Zero); + } + } + + internal static void Stash() + { + var metaStorage = new RuntimeDataStorage(); + MetaType.SaveRuntimeData(metaStorage); + + var importStorage = new RuntimeDataStorage(); + ImportHook.SaveRuntimeData(importStorage); + + var typeStorage = new RuntimeDataStorage(); + TypeManager.SaveRuntimeData(typeStorage); + + var clsStorage = new RuntimeDataStorage(); + ClassManager.SaveRuntimeData(clsStorage); + + var moduleStorage = new RuntimeDataStorage(); + SaveRuntimeDataModules(moduleStorage); + + var objStorage = new RuntimeDataStorage(); + SaveRuntimeDataObjects(objStorage); + + var runtimeStorage = new RuntimeDataStorage(); + runtimeStorage.AddValue("meta", metaStorage); + runtimeStorage.AddValue("import", importStorage); + runtimeStorage.AddValue("types", typeStorage); + runtimeStorage.AddValue("classes", clsStorage); + runtimeStorage.AddValue("modules", moduleStorage); + runtimeStorage.AddValue("objs", objStorage); + + IFormatter formatter = CreateFormatter(); + var ms = new MemoryStream(); + formatter.Serialize(ms, runtimeStorage); + + Debug.Assert(ms.Length <= int.MaxValue); + byte[] data = ms.GetBuffer(); + // TODO: use buffer api instead + IntPtr mem = PyMem_Malloc(ms.Length + IntPtr.Size); + Marshal.WriteIntPtr(mem, (IntPtr)ms.Length); + Marshal.Copy(data, 0, mem + IntPtr.Size, (int)ms.Length); + + ClearCLRData(); + NewReference capsule = PyCapsule_New(mem, null, IntPtr.Zero); + PySys_SetObject("clr_data", capsule.DangerousGetAddress()); + // Let the dictionary own the reference + capsule.Dispose(); + } + + internal static void RestoreRuntimeData() + { + try + { + RestoreRuntimeDataImpl(); + } + finally + { + ClearStash(); + } + } + + private static void RestoreRuntimeDataImpl() + { + BorrowedReference capsule = PySys_GetObject("clr_data"); + if (capsule.IsNull) + { + return; + } + IntPtr mem = PyCapsule_GetPointer(capsule, null); + int length = (int)Marshal.ReadIntPtr(mem); + byte[] data = new byte[length]; + Marshal.Copy(mem + IntPtr.Size, data, 0, length); + var ms = new MemoryStream(data); + var formatter = CreateFormatter(); + var storage = (RuntimeDataStorage)formatter.Deserialize(ms); + + var objs = RestoreRuntimeDataObjects(storage.GetStorage("objs")); + RestoreRuntimeDataModules(storage.GetStorage("modules")); + var clsObjs = ClassManager.RestoreRuntimeData(storage.GetStorage("classes")); + TypeManager.RestoreRuntimeData(storage.GetStorage("types")); + ImportHook.RestoreRuntimeData(storage.GetStorage("import")); + PyCLRMetaType = MetaType.RestoreRuntimeData(storage.GetStorage("meta")); + + foreach (var item in objs) + { + item.Value.ExecutePostActions(); + XDecref(item.Key.pyHandle); + } + foreach (var item in clsObjs) + { + item.Value.ExecutePostActions(); + } + } + + public static bool HasStashData() + { + return !PySys_GetObject("clr_data").IsNull; + } + + public static void ClearStash() + { + PySys_SetObject("clr_data", IntPtr.Zero); + } + + static bool CheckSerializable (object o) + { + Type type = o.GetType(); + do + { + if (!type.IsSerializable) + { + return false; + } + } while ((type = type.BaseType) != null); + return true; + } + + private static void SaveRuntimeDataObjects(RuntimeDataStorage storage) + { + var objs = ManagedType.GetManagedObjects(); + var extensionObjs = new List(); + var wrappers = new Dictionary>(); + var serializeObjs = new CLRWrapperCollection(); + var contexts = new Dictionary(); + foreach (var entry in objs) + { + var obj = entry.Key; + XIncref(obj.pyHandle); + switch (entry.Value) + { + case ManagedType.TrackTypes.Extension: + Debug.Assert(CheckSerializable(obj)); + var context = new InterDomainContext(); + contexts[obj.pyHandle] = context; + obj.Save(context); + extensionObjs.Add(obj); + break; + case ManagedType.TrackTypes.Wrapper: + // Wrapper must be the CLRObject + var clrObj = (CLRObject)obj; + object inst = clrObj.inst; + CLRMappedItem item; + List mappedObjs; + if (!serializeObjs.TryGetValue(inst, out item)) + { + item = new CLRMappedItem(inst) + { + Handles = new List() + }; + serializeObjs.Add(item); + + Debug.Assert(!wrappers.ContainsKey(inst)); + mappedObjs = new List(); + wrappers.Add(inst, mappedObjs); + } + else + { + mappedObjs = wrappers[inst]; + } + item.Handles.Add(clrObj.pyHandle); + mappedObjs.Add(clrObj); + break; + default: + break; + } + } + + var wrapperStorage = new RuntimeDataStorage(); + WrappersStorer?.Store(serializeObjs, wrapperStorage); + + var internalStores = new List(); + foreach (var item in serializeObjs) + { + if (!item.Stored) + { + if (!CheckSerializable(item.Instance)) + { + continue; + } + internalStores.AddRange(wrappers[item.Instance]); + } + foreach (var clrObj in wrappers[item.Instance]) + { + XIncref(clrObj.pyHandle); + var context = new InterDomainContext(); + contexts[clrObj.pyHandle] = context; + clrObj.Save(context); + } + } + storage.AddValue("internalStores", internalStores); + storage.AddValue("extensions", extensionObjs); + storage.AddValue("wrappers", wrapperStorage); + storage.AddValue("contexts", contexts); + } + + private static Dictionary RestoreRuntimeDataObjects(RuntimeDataStorage storage) + { + var extensions = storage.GetValue>("extensions"); + var internalStores = storage.GetValue>("internalStores"); + var contexts = storage.GetValue >("contexts"); + var storedObjs = new Dictionary(); + foreach (var obj in Enumerable.Union(extensions, internalStores)) + { + var context = contexts[obj.pyHandle]; + obj.Load(context); + storedObjs.Add(obj, context); + } + if (WrappersStorer != null) + { + var wrapperStorage = storage.GetStorage("wrappers"); + var handle2Obj = WrappersStorer.Restore(wrapperStorage); + foreach (var item in handle2Obj) + { + object obj = item.Instance; + foreach (var handle in item.Handles) + { + var context = contexts[handle]; + var co = CLRObject.Restore(obj, handle, context); + storedObjs.Add(co, context); + } + } + } + return storedObjs; + } + + private static void SaveRuntimeDataModules(RuntimeDataStorage storage) + { + var pyModules = PyImport_GetModuleDict(); + var items = PyDict_Items(pyModules); + long length = PyList_Size(items); + var modules = new Dictionary(); ; + for (long i = 0; i < length; i++) + { + var item = PyList_GetItem(items, i); + var name = PyTuple_GetItem(item.DangerousGetAddress(), 0); + var module = PyTuple_GetItem(item.DangerousGetAddress(), 1); + if (ManagedType.IsManagedType(module)) + { + XIncref(name); + XIncref(module); + modules.Add(name, module); + } + } + items.Dispose(); + storage.AddValue("modules", modules); + } + + private static void RestoreRuntimeDataModules(RuntimeDataStorage storage) + { + var modules = storage.GetValue>("modules"); + var pyMoudles = PyImport_GetModuleDict(); + foreach (var item in modules) + { + int res = PyDict_SetItem(pyMoudles, item.Key, item.Value); + PythonException.ThrowIfIsNotZero(res); + XDecref(item.Key); + XDecref(item.Value); + } + modules.Clear(); + } + + private static IFormatter CreateFormatter() + { + return FormatterType != null ? + (IFormatter)Activator.CreateInstance(FormatterType) + : new BinaryFormatter(); + } + } + + + [Serializable] + public class RuntimeDataStorage + { + private Stack _stack; + private Dictionary _namedValues; + + public T AddValue(string name, T value) + { + if (_namedValues == null) + { + _namedValues = new Dictionary(); + } + _namedValues.Add(name, value); + return value; + } + + public object GetValue(string name) + { + return _namedValues[name]; + } + + public T GetValue(string name) + { + return (T)GetValue(name); + } + + public T GetValue(string name, out T value) + { + value = GetValue(name); + return value; + } + + public RuntimeDataStorage GetStorage(string name) + { + return GetValue(name); + } + + public T PushValue(T value) + { + if (_stack == null) + { + _stack = new Stack(); + } + _stack.Push(value); + return value; + } + + public object PopValue() + { + return _stack.Pop(); + } + + public T PopValue() + { + return (T)PopValue(); + } + + public T PopValue(out T value) + { + return value = (T)PopValue(); + } + } + + + [Serializable] + class InterDomainContext + { + private RuntimeDataStorage _storage; + public RuntimeDataStorage Storage => _storage ?? (_storage = new RuntimeDataStorage()); + + /// + /// Actions after loaded. + /// + [NonSerialized] + private List _postActions; + public List PostActions => _postActions ?? (_postActions = new List()); + + public void AddPostAction(Action action) + { + PostActions.Add(action); + } + + public void ExecutePostActions() + { + if (_postActions == null) + { + return; + } + foreach (var action in _postActions) + { + action(); + } + } + } + + public class CLRMappedItem + { + public object Instance { get; private set; } + public IList Handles { get; set; } + public bool Stored { get; set; } + + public CLRMappedItem(object instance) + { + Instance = instance; + } + } + + + public interface ICLRObjectStorer + { + ICollection Store(CLRWrapperCollection wrappers, RuntimeDataStorage storage); + CLRWrapperCollection Restore(RuntimeDataStorage storage); + } + + + public class CLRWrapperCollection : KeyedCollection + { + public bool TryGetValue(object key, out CLRMappedItem value) + { + if (Dictionary == null) + { + value = null; + return false; + } + return Dictionary.TryGetValue(key, out value); + } + + protected override object GetKeyForItem(CLRMappedItem item) + { + return item.Instance; + } + } +} diff --git a/src/runtime/runtime_state.cs b/src/runtime/runtime_state.cs new file mode 100644 index 000000000..69acbcd31 --- /dev/null +++ b/src/runtime/runtime_state.cs @@ -0,0 +1,238 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Runtime.InteropServices; + +using static Python.Runtime.Runtime; + +namespace Python.Runtime +{ + class RuntimeState + { + public static bool ShouldRestoreObjects { get; set; } = false; + public static bool UseDummyGC { get; set; } = false; + + public static void Save() + { + if (!PySys_GetObject("dummy_gc").IsNull) + { + throw new Exception("Runtime State set already"); + } + + IntPtr objs = IntPtr.Zero; + if (ShouldRestoreObjects) + { + objs = PySet_New(IntPtr.Zero); + foreach (var obj in PyGCGetObjects()) + { + AddObjPtrToSet(objs, obj); + } + } + + var modules = PySet_New(IntPtr.Zero); + foreach (var name in GetModuleNames()) + { + int res = PySet_Add(modules, name); + PythonException.ThrowIfIsNotZero(res); + } + + + var dummyGCHead = PyMem_Malloc(Marshal.SizeOf(typeof(PyGC_Head))); + unsafe + { + var head = (PyGC_Head*)dummyGCHead; + head->gc.gc_next = dummyGCHead; + head->gc.gc_prev = dummyGCHead; + head->gc.gc_refs = IntPtr.Zero; + } + { + var pyDummyGC = PyLong_FromVoidPtr(dummyGCHead); + int res = PySys_SetObject("dummy_gc", pyDummyGC); + PythonException.ThrowIfIsNotZero(res); + XDecref(pyDummyGC); + + try + { + res = PySys_SetObject("initial_modules", modules); + PythonException.ThrowIfIsNotZero(res); + } + finally + { + XDecref(modules); + } + + if (ShouldRestoreObjects) + { + AddObjPtrToSet(objs, modules); + try + { + res = PySys_SetObject("initial_objs", objs); + PythonException.ThrowIfIsNotZero(res); + } + finally + { + XDecref(objs); + } + } + } + } + + public static void Restore() + { + var dummyGCAddr = PySys_GetObject("dummy_gc").DangerousGetAddress(); + if (dummyGCAddr == IntPtr.Zero) + { + throw new InvalidOperationException("Runtime state have not set"); + } + var dummyGC = PyLong_AsVoidPtr(dummyGCAddr); + ResotreModules(dummyGC); + if (ShouldRestoreObjects) + { + RestoreObjects(dummyGC); + } + } + + private static void ResotreModules(IntPtr dummyGC) + { + var intialModules = PySys_GetObject("initial_modules"); + Debug.Assert(!intialModules.IsNull); + var modules = PyImport_GetModuleDict(); + foreach (var name in GetModuleNames()) + { + if (PySet_Contains(intialModules.DangerousGetAddress(), name) == 1) + { + continue; + } + var module = PyDict_GetItem(modules, name); + + if (UseDummyGC && _PyObject_GC_IS_TRACKED(module)) + { + ExchangeGCChain(module, dummyGC); + } + if (PyDict_DelItem(modules, name) != 0) + { + PyErr_Print(); + } + } + } + + private static void RestoreObjects(IntPtr dummyGC) + { + if (!UseDummyGC) + { + throw new Exception("To prevent crash by _PyObject_GC_UNTRACK in Python internal, UseDummyGC should be enabled when using ResotreObjects"); + } + IntPtr intialObjs = PySys_GetObject("initial_objs").DangerousGetAddress(); + Debug.Assert(intialObjs != IntPtr.Zero); + foreach (var obj in PyGCGetObjects()) + { + var p = PyLong_FromVoidPtr(obj); + try + { + if (PySet_Contains(intialObjs, p) == 1) + { + continue; + } + } + finally + { + XDecref(p); + } + Debug.Assert(_PyObject_GC_IS_TRACKED(obj), "A GC object must be tracked"); + ExchangeGCChain(obj, dummyGC); + } + } + + public static IEnumerable PyGCGetObjects() + { + var gc = PyImport_ImportModule("gc"); + PythonException.ThrowIfIsNull(gc); + var get_objects = PyObject_GetAttrString(gc, "get_objects"); + var objs = PyObject_CallObject(get_objects, IntPtr.Zero); + var length = PyList_Size(new BorrowedReference(objs)); + for (long i = 0; i < length; i++) + { + var obj = PyList_GetItem(new BorrowedReference(objs), i); + yield return obj.DangerousGetAddress(); + } + XDecref(objs); + XDecref(gc); + } + + public static IEnumerable GetModuleNames() + { + var modules = PyImport_GetModuleDict(); + var names = PyDict_Keys(modules); + var length = PyList_Size(new BorrowedReference(names)); + for (int i = 0; i < length; i++) + { + var name = PyList_GetItem(new BorrowedReference(names), i); + yield return name.DangerousGetAddress(); + } + XDecref(names); + } + + private static void AddObjPtrToSet(IntPtr set, IntPtr obj) + { + var p = PyLong_FromVoidPtr(obj); + XIncref(obj); + try + { + int res = PySet_Add(set, p); + PythonException.ThrowIfIsNotZero(res); + } + finally + { + XDecref(p); + } + } + /// + /// Exchange gc to a dummy gc prevent nullptr error in _PyObject_GC_UnTrack macro. + /// + private static void ExchangeGCChain(IntPtr obj, IntPtr gc) + { + var head = _Py_AS_GC(obj); + if ((long)_PyGCHead_REFS(head) == _PyGC_REFS_UNTRACKED) + { + throw new ArgumentException("GC object untracked"); + } + unsafe + { + var g = (PyGC_Head*)head; + var newGCGen = (PyGC_Head*)gc; + + ((PyGC_Head*)g->gc.gc_prev)->gc.gc_next = g->gc.gc_next; + ((PyGC_Head*)g->gc.gc_next)->gc.gc_prev = g->gc.gc_prev; + + g->gc.gc_next = gc; + g->gc.gc_prev = newGCGen->gc.gc_prev; + ((PyGC_Head*)g->gc.gc_prev)->gc.gc_next = head; + newGCGen->gc.gc_prev = head; + } + } + + private static IEnumerable IterGCNodes(IntPtr gc) + { + var node = GetNextGCNode(gc); + while (node != gc) + { + var next = GetNextGCNode(node); + yield return node; + node = next; + } + } + + private static IEnumerable IterObjects(IntPtr gc) + { + foreach (var node in IterGCNodes(gc)) + { + yield return _Py_FROM_GC(node); + } + } + + private static unsafe IntPtr GetNextGCNode(IntPtr node) + { + return ((PyGC_Head*)node)->gc.gc_next; + } + } +} diff --git a/src/runtime/slots/mp_length.cs b/src/runtime/slots/mp_length.cs index b0a2e8d79..42448a2e9 100644 --- a/src/runtime/slots/mp_length.cs +++ b/src/runtime/slots/mp_length.cs @@ -1,6 +1,7 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Reflection; @@ -8,11 +9,41 @@ namespace Python.Runtime.Slots { internal static class mp_length_slot { + private static MethodInfo _lengthMethod; + public static MethodInfo Method + { + get + { + if (_lengthMethod != null) + { + return _lengthMethod; + } + _lengthMethod = typeof(mp_length_slot).GetMethod( + nameof(mp_length_slot.mp_length), + BindingFlags.Static | BindingFlags.NonPublic); + Debug.Assert(_lengthMethod != null); + return _lengthMethod; + } + } + + public static bool CanAssgin(Type clrType) + { + if (typeof(ICollection).IsAssignableFrom(clrType)) + { + return true; + } + if (clrType.GetInterfaces().Any(x => x.IsGenericType && x.GetGenericTypeDefinition() == typeof(ICollection<>))) + { + return true; + } + return false; + } + /// /// Implements __len__ for classes that implement ICollection /// (this includes any IList implementer or Array subclass) /// - public static int mp_length(IntPtr ob) + private static int mp_length(IntPtr ob) { var co = ManagedType.GetManagedObject(ob) as CLRObject; if (co == null) diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index a64f91e75..c00247ca4 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -4,7 +4,7 @@ using System.Linq; using System.Reflection; using System.Runtime.InteropServices; -using Python.Runtime.Platform; +using System.Diagnostics; using Python.Runtime.Slots; namespace Python.Runtime @@ -16,21 +16,79 @@ namespace Python.Runtime /// internal class TypeManager { - private static BindingFlags tbFlags; - private static Dictionary cache; + internal static IntPtr subtype_traverse; + internal static IntPtr subtype_clear; - static TypeManager() + private const BindingFlags tbFlags = BindingFlags.Public | BindingFlags.Static; + private static Dictionary cache = new Dictionary(); + private static readonly Dictionary _slotsHolders = new Dictionary(); + private static Dictionary _slotsImpls = new Dictionary(); + + // Slots which must be set + private static readonly string[] _requiredSlots = new string[] + { + "tp_traverse", + "tp_clear", + }; + + internal static void Initialize() { - tbFlags = BindingFlags.Public | BindingFlags.Static; - cache = new Dictionary(128); + Debug.Assert(cache.Count == 0, "Cache should be empty", + "Some errors may occurred on last shutdown"); + IntPtr type = SlotHelper.CreateObjectType(); + subtype_traverse = Marshal.ReadIntPtr(type, TypeOffset.tp_traverse); + subtype_clear = Marshal.ReadIntPtr(type, TypeOffset.tp_clear); + Runtime.XDecref(type); } - public static void Reset() + internal static void RemoveTypes() { - cache = new Dictionary(128); + foreach (var tpHandle in cache.Values) + { + SlotsHolder holder; + if (_slotsHolders.TryGetValue(tpHandle, out holder)) + { + // If refcount > 1, it needs to reset the managed slot, + // otherwise it can dealloc without any trick. + if (Runtime.Refcount(tpHandle) > 1) + { + holder.ResetSlots(); + } + } + Runtime.XDecref(tpHandle); + } + cache.Clear(); + _slotsImpls.Clear(); + _slotsHolders.Clear(); + } + + internal static void SaveRuntimeData(RuntimeDataStorage storage) + { + foreach (var tpHandle in cache.Values) + { + Runtime.XIncref(tpHandle); + } + storage.AddValue("cache", cache); + storage.AddValue("slots", _slotsImpls); + } + + internal static void RestoreRuntimeData(RuntimeDataStorage storage) + { + Debug.Assert(cache == null || cache.Count == 0); + storage.GetValue("slots", out _slotsImpls); + storage.GetValue("cache", out cache); + foreach (var entry in cache) + { + Type type = entry.Key; + IntPtr handle = entry.Value; + SlotsHolder holder = CreateSolotsHolder(handle); + InitializeSlots(handle, _slotsImpls[type], holder); + // FIXME: mp_length_slot.CanAssgin(clrType) + } } /// + /// Return value: Borrowed reference. /// Given a managed Type derived from ExtensionType, get the handle to /// a Python type object that delegates its implementation to the Type /// object. These Python type instances are used to implement internal @@ -48,11 +106,13 @@ internal static IntPtr GetTypeHandle(Type type) } handle = CreateType(type); cache[type] = handle; + _slotsImpls.Add(type, type); return handle; } /// + /// Return value: Borrowed reference. /// Get the handle of a Python type that reflects the given CLR type. /// The given ManagedType instance is a managed object that implements /// the appropriate semantics in Python for the reflected managed type. @@ -67,6 +127,7 @@ internal static IntPtr GetTypeHandle(ManagedType obj, Type type) } handle = CreateType(obj, type); cache[type] = handle; + _slotsImpls.Add(type, obj.GetType()); return handle; } @@ -90,20 +151,24 @@ internal static IntPtr CreateType(Type impl) var offset = (IntPtr)ObjectOffset.TypeDictOffset(type); Marshal.WriteIntPtr(type, TypeOffset.tp_dictoffset, offset); - InitializeSlots(type, impl); + SlotsHolder slotsHolder = CreateSolotsHolder(type); + InitializeSlots(type, impl, slotsHolder); int flags = TypeFlags.Default | TypeFlags.Managed | TypeFlags.HeapType | TypeFlags.HaveGC; Util.WriteCLong(type, TypeOffset.tp_flags, flags); - Runtime.PyType_Ready(type); + if (Runtime.PyType_Ready(type) != 0) + { + throw new PythonException(); + } IntPtr dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); IntPtr mod = Runtime.PyString_FromString("CLR"); Runtime.PyDict_SetItemString(dict, "__module__", mod); + Runtime.XDecref(mod); InitMethods(type, impl); - return type; } @@ -155,21 +220,22 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) Marshal.WriteIntPtr(type, TypeOffset.tp_itemsize, IntPtr.Zero); Marshal.WriteIntPtr(type, TypeOffset.tp_dictoffset, (IntPtr)tp_dictoffset); - // add a __len__ slot for inheritors of ICollection and ICollection<> - if (typeof(ICollection).IsAssignableFrom(clrType) || clrType.GetInterfaces().Any(x => x.IsGenericType && x.GetGenericTypeDefinition() == typeof(ICollection<>))) + // we want to do this after the slot stuff above in case the class itself implements a slot method + SlotsHolder slotsHolder = CreateSolotsHolder(type); + InitializeSlots(type, impl.GetType(), slotsHolder); + + if (Marshal.ReadIntPtr(type, TypeOffset.mp_length) == IntPtr.Zero + && mp_length_slot.CanAssgin(clrType)) { - InitializeSlot(type, TypeOffset.mp_length, typeof(mp_length_slot).GetMethod(nameof(mp_length_slot.mp_length))); + InitializeSlot(type, TypeOffset.mp_length, mp_length_slot.Method, slotsHolder); } - // we want to do this after the slot stuff above in case the class itself implements a slot method - InitializeSlots(type, impl.GetType()); - if (!typeof(IEnumerable).IsAssignableFrom(clrType) && !typeof(IEnumerator).IsAssignableFrom(clrType)) - { - // The tp_iter slot should only be set for enumerable types. - Marshal.WriteIntPtr(type, TypeOffset.tp_iter, IntPtr.Zero); - } + { + // The tp_iter slot should only be set for enumerable types. + Marshal.WriteIntPtr(type, TypeOffset.tp_iter, IntPtr.Zero); + } // Only set mp_subscript and mp_ass_subscript for types with indexers @@ -199,31 +265,34 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) Runtime.XIncref(base_); } - int flags = TypeFlags.Default; - flags |= TypeFlags.Managed; - flags |= TypeFlags.HeapType; - flags |= TypeFlags.BaseType; - flags |= TypeFlags.HaveGC; + const int flags = TypeFlags.Default + | TypeFlags.Managed + | TypeFlags.HeapType + | TypeFlags.BaseType + | TypeFlags.HaveGC; Util.WriteCLong(type, TypeOffset.tp_flags, flags); // Leverage followup initialization from the Python runtime. Note // that the type of the new type must PyType_Type at the time we // call this, else PyType_Ready will skip some slot initialization. - Runtime.PyType_Ready(type); + if (Runtime.PyType_Ready(type) != 0) + { + throw new PythonException(); + } IntPtr dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); string mn = clrType.Namespace ?? ""; IntPtr mod = Runtime.PyString_FromString(mn); Runtime.PyDict_SetItemString(dict, "__module__", mod); + Runtime.XDecref(mod); // Hide the gchandle of the implementation in a magic type slot. - GCHandle gc = GCHandle.Alloc(impl); + GCHandle gc = impl.AllocGCHandle(); Marshal.WriteIntPtr(type, TypeOffset.magic(), (IntPtr)gc); // Set the handle attributes on the implementing instance. - impl.tpHandle = Runtime.PyCLRMetaType; - impl.gcHandle = gc; + impl.tpHandle = type; impl.pyHandle = type; //DebugUtil.DumpType(type); @@ -231,12 +300,6 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) return type; } - static void InitializeSlot(IntPtr type, int slotOffset, MethodInfo method) - { - var thunk = Interop.GetThunk(method); - Marshal.WriteIntPtr(type, slotOffset, thunk.Address); - } - internal static IntPtr CreateSubType(IntPtr py_name, IntPtr py_base_type, IntPtr py_dict) { // Utility to create a subtype of a managed type with the ability for the @@ -308,7 +371,7 @@ internal static IntPtr CreateSubType(IntPtr py_name, IntPtr py_base_type, IntPtr // derived class we want the python overrides in there instead if they exist. IntPtr cls_dict = Marshal.ReadIntPtr(py_type, TypeOffset.tp_dict); Runtime.PyDict_Update(cls_dict, py_dict); - + Runtime.XIncref(py_type); // Update the __classcell__ if it exists var cell = new BorrowedReference(Runtime.PyDict_GetItemString(cls_dict, "__classcell__")); if (!cell.IsNull) @@ -348,7 +411,25 @@ internal static IntPtr WriteMethodDefSentinel(IntPtr mdef) return WriteMethodDef(mdef, IntPtr.Zero, IntPtr.Zero, 0, IntPtr.Zero); } - internal static IntPtr CreateMetaType(Type impl) + internal static void FreeMethodDef(IntPtr mdef) + { + unsafe + { + var def = (PyMethodDef*)mdef; + if (def->ml_name != IntPtr.Zero) + { + Marshal.FreeHGlobal(def->ml_name); + def->ml_name = IntPtr.Zero; + } + if (def->ml_doc != IntPtr.Zero) + { + Marshal.FreeHGlobal(def->ml_doc); + def->ml_doc = IntPtr.Zero; + } + } + } + + internal static IntPtr CreateMetaType(Type impl, out SlotsHolder slotsHolder) { // The managed metatype is functionally little different than the // standard Python metatype (PyType_Type). It overrides certain of @@ -361,57 +442,90 @@ internal static IntPtr CreateMetaType(Type impl) Marshal.WriteIntPtr(type, TypeOffset.tp_base, py_type); Runtime.XIncref(py_type); + const int flags = TypeFlags.Default + | TypeFlags.Managed + | TypeFlags.HeapType + | TypeFlags.HaveGC; + Util.WriteCLong(type, TypeOffset.tp_flags, flags); + // Slots will inherit from TypeType, it's not neccesary for setting them. // Inheried slots: // tp_basicsize, tp_itemsize, // tp_dictoffset, tp_weaklistoffset, // tp_traverse, tp_clear, tp_is_gc, etc. + slotsHolder = SetupMetaSlots(impl, type); - // Override type slots with those of the managed implementation. + if (Runtime.PyType_Ready(type) != 0) + { + throw new PythonException(); + } - InitializeSlots(type, impl); + IntPtr dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); + IntPtr mod = Runtime.PyString_FromString("CLR"); + Runtime.PyDict_SetItemString(dict, "__module__", mod); - int flags = TypeFlags.Default; - flags |= TypeFlags.Managed; - flags |= TypeFlags.HeapType; - flags |= TypeFlags.HaveGC; - Util.WriteCLong(type, TypeOffset.tp_flags, flags); + //DebugUtil.DumpType(type); + + return type; + } - // We need space for 3 PyMethodDef structs, each of them - // 4 int-ptrs in size. - IntPtr mdef = Runtime.PyMem_Malloc(3 * 4 * IntPtr.Size); + internal static SlotsHolder SetupMetaSlots(Type impl, IntPtr type) + { + // Override type slots with those of the managed implementation. + SlotsHolder slotsHolder = new SlotsHolder(type); + InitializeSlots(type, impl, slotsHolder); + + // We need space for 3 PyMethodDef structs. + int mdefSize = (MetaType.CustomMethods.Length + 1) * Marshal.SizeOf(typeof(PyMethodDef)); + IntPtr mdef = Runtime.PyMem_Malloc(mdefSize); IntPtr mdefStart = mdef; - ThunkInfo thunkInfo = Interop.GetThunk(typeof(MetaType).GetMethod("__instancecheck__"), "BinaryFunc"); - mdef = WriteMethodDef( - mdef, - "__instancecheck__", - thunkInfo.Address - ); - - thunkInfo = Interop.GetThunk(typeof(MetaType).GetMethod("__subclasscheck__"), "BinaryFunc"); - mdef = WriteMethodDef( - mdef, - "__subclasscheck__", - thunkInfo.Address - ); - - // FIXME: mdef is not used + foreach (var methodName in MetaType.CustomMethods) + { + mdef = AddCustomMetaMethod(methodName, type, mdef, slotsHolder); + } mdef = WriteMethodDefSentinel(mdef); + Debug.Assert((long)(mdefStart + mdefSize) <= (long)mdef); Marshal.WriteIntPtr(type, TypeOffset.tp_methods, mdefStart); - Runtime.PyType_Ready(type); - - IntPtr dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); - IntPtr mod = Runtime.PyString_FromString("CLR"); - Runtime.PyDict_SetItemString(dict, "__module__", mod); + // XXX: Hard code with mode check. + if (Runtime.ShutdownMode != ShutdownMode.Reload) + { + slotsHolder.Set(TypeOffset.tp_methods, (t, offset) => + { + var p = Marshal.ReadIntPtr(t, offset); + Runtime.PyMem_Free(p); + Marshal.WriteIntPtr(t, offset, IntPtr.Zero); + }); + } + return slotsHolder; + } - //DebugUtil.DumpType(type); + private static IntPtr AddCustomMetaMethod(string name, IntPtr type, IntPtr mdef, SlotsHolder slotsHolder) + { + MethodInfo mi = typeof(MetaType).GetMethod(name); + ThunkInfo thunkInfo = Interop.GetThunk(mi, "BinaryFunc"); + slotsHolder.KeeapAlive(thunkInfo); - return type; + // XXX: Hard code with mode check. + if (Runtime.ShutdownMode != ShutdownMode.Reload) + { + IntPtr mdefAddr = mdef; + slotsHolder.AddDealloctor(() => + { + IntPtr tp_dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); + if (Runtime.PyDict_DelItemString(tp_dict, name) != 0) + { + Runtime.PyErr_Print(); + Debug.Fail($"Cannot remove {name} from metatype"); + } + FreeMethodDef(mdefAddr); + }); + } + mdef = WriteMethodDef(mdef, name, thunkInfo.Address); + return mdef; } - internal static IntPtr BasicSubType(string name, IntPtr base_, Type impl) { // Utility to create a subtype of a std Python type, but with @@ -440,9 +554,13 @@ internal static IntPtr BasicSubType(string name, IntPtr base_, Type impl) CopySlot(base_, type, TypeOffset.tp_clear); CopySlot(base_, type, TypeOffset.tp_is_gc); - InitializeSlots(type, impl); + SlotsHolder slotsHolder = CreateSolotsHolder(type); + InitializeSlots(type, impl, slotsHolder); - Runtime.PyType_Ready(type); + if (Runtime.PyType_Ready(type) != 0) + { + throw new PythonException(); + } IntPtr tp_dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); IntPtr mod = Runtime.PyString_FromString("CLR"); @@ -458,6 +576,10 @@ internal static IntPtr BasicSubType(string name, IntPtr base_, Type impl) internal static IntPtr AllocateTypeObject(string name) { IntPtr type = Runtime.PyType_GenericAlloc(Runtime.PyTypeType, 0); + // Clr type would not use __slots__, + // and the PyMemberDef after PyHeapTypeObject will have other uses(e.g. type handle), + // thus set the ob_size to 0 for avoiding slots iterations. + Marshal.WriteIntPtr(type, TypeOffset.ob_size, IntPtr.Zero); // Cheat a little: we'll set tp_name to the internal char * of // the Python version of the type name - otherwise we'd have to @@ -467,254 +589,28 @@ internal static IntPtr AllocateTypeObject(string name) Marshal.WriteIntPtr(type, TypeOffset.tp_name, raw); Marshal.WriteIntPtr(type, TypeOffset.name, temp); + Runtime.XIncref(temp); Marshal.WriteIntPtr(type, TypeOffset.qualname, temp); - - long ptr = type.ToInt64(); // 64-bit safe - - temp = new IntPtr(ptr + TypeOffset.nb_add); + temp = type + TypeOffset.nb_add; Marshal.WriteIntPtr(type, TypeOffset.tp_as_number, temp); - temp = new IntPtr(ptr + TypeOffset.sq_length); + temp = type + TypeOffset.sq_length; Marshal.WriteIntPtr(type, TypeOffset.tp_as_sequence, temp); - temp = new IntPtr(ptr + TypeOffset.mp_length); + temp = type + TypeOffset.mp_length; Marshal.WriteIntPtr(type, TypeOffset.tp_as_mapping, temp); - temp = new IntPtr(ptr + TypeOffset.bf_getbuffer); + temp = type + TypeOffset.bf_getbuffer; Marshal.WriteIntPtr(type, TypeOffset.tp_as_buffer, temp); return type; } - - #region Native Code Page - /// - /// Initialized by InitializeNativeCodePage. - /// - /// This points to a page of memory allocated using mmap or VirtualAlloc - /// (depending on the system), and marked read and execute (not write). - /// Very much on purpose, the page is *not* released on a shutdown and - /// is instead leaked. See the TestDomainReload test case. - /// - /// The contents of the page are two native functions: one that returns 0, - /// one that returns 1. - /// - /// If python didn't keep its gc list through a Py_Finalize we could remove - /// this entire section. - /// - internal static IntPtr NativeCodePage = IntPtr.Zero; - - /// - /// Structure to describe native code. - /// - /// Use NativeCode.Active to get the native code for the current platform. - /// - /// Generate the code by creating the following C code: - /// - /// int Return0() { return 0; } - /// int Return1() { return 1; } - /// - /// Then compiling on the target platform, e.g. with gcc or clang: - /// cc -c -fomit-frame-pointer -O2 foo.c - /// And then analyzing the resulting functions with a hex editor, e.g.: - /// objdump -disassemble foo.o - /// - internal class NativeCode - { - /// - /// The code, as a string of bytes. - /// - public byte[] Code { get; private set; } - - /// - /// Where does the "return 0" function start? - /// - public int Return0 { get; private set; } - - /// - /// Where does the "return 1" function start? - /// - public int Return1 { get; private set; } - - public static NativeCode Active - { - get - { - switch (Runtime.Machine) - { - case MachineType.i386: - return I386; - case MachineType.x86_64: - return X86_64; - default: - return null; - } - } - } - - /// - /// Code for x86_64. See the class comment for how it was generated. - /// - public static readonly NativeCode X86_64 = new NativeCode() - { - Return0 = 0x10, - Return1 = 0, - Code = new byte[] - { - // First Return1: - 0xb8, 0x01, 0x00, 0x00, 0x00, // movl $1, %eax - 0xc3, // ret - - // Now some padding so that Return0 can be 16-byte-aligned. - // I put Return1 first so there's not as much padding to type in. - 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, // nop - - // Now Return0. - 0x31, 0xc0, // xorl %eax, %eax - 0xc3, // ret - } - }; - - /// - /// Code for X86. - /// - /// It's bitwise identical to X86_64, so we just point to it. - /// - /// - public static readonly NativeCode I386 = X86_64; - } - - /// - /// Platform-dependent mmap and mprotect. - /// - internal interface IMemoryMapper - { - /// - /// Map at least numBytes of memory. Mark the page read-write (but not exec). - /// - IntPtr MapWriteable(int numBytes); - - /// - /// Sets the mapped memory to be read-exec (but not write). - /// - void SetReadExec(IntPtr mappedMemory, int numBytes); - } - - class WindowsMemoryMapper : IMemoryMapper - { - const UInt32 MEM_COMMIT = 0x1000; - const UInt32 MEM_RESERVE = 0x2000; - const UInt32 PAGE_READWRITE = 0x04; - const UInt32 PAGE_EXECUTE_READ = 0x20; - - [DllImport("kernel32.dll")] - static extern IntPtr VirtualAlloc(IntPtr lpAddress, IntPtr dwSize, UInt32 flAllocationType, UInt32 flProtect); - - [DllImport("kernel32.dll")] - static extern bool VirtualProtect(IntPtr lpAddress, IntPtr dwSize, UInt32 flNewProtect, out UInt32 lpflOldProtect); - - public IntPtr MapWriteable(int numBytes) - { - return VirtualAlloc(IntPtr.Zero, new IntPtr(numBytes), - MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); - } - - public void SetReadExec(IntPtr mappedMemory, int numBytes) - { - UInt32 _; - VirtualProtect(mappedMemory, new IntPtr(numBytes), PAGE_EXECUTE_READ, out _); - } - } - - class UnixMemoryMapper : IMemoryMapper - { - const int PROT_READ = 0x1; - const int PROT_WRITE = 0x2; - const int PROT_EXEC = 0x4; - - const int MAP_PRIVATE = 0x2; - int MAP_ANONYMOUS - { - get - { - switch (Runtime.OperatingSystem) - { - case OperatingSystemType.Darwin: - return 0x1000; - case OperatingSystemType.Linux: - return 0x20; - default: - throw new NotImplementedException( - $"mmap is not supported on {Runtime.OperatingSystem}" - ); - } - } - } - - [DllImport("libc")] - static extern IntPtr mmap(IntPtr addr, IntPtr len, int prot, int flags, int fd, IntPtr offset); - - [DllImport("libc")] - static extern int mprotect(IntPtr addr, IntPtr len, int prot); - - public IntPtr MapWriteable(int numBytes) - { - // MAP_PRIVATE must be set on linux, even though MAP_ANON implies it. - // It doesn't hurt on darwin, so just do it. - return mmap(IntPtr.Zero, new IntPtr(numBytes), PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, IntPtr.Zero); - } - - public void SetReadExec(IntPtr mappedMemory, int numBytes) - { - mprotect(mappedMemory, new IntPtr(numBytes), PROT_READ | PROT_EXEC); - } - } - - internal static IMemoryMapper CreateMemoryMapper() - { - switch (Runtime.OperatingSystem) - { - case OperatingSystemType.Darwin: - case OperatingSystemType.Linux: - return new UnixMemoryMapper(); - case OperatingSystemType.Windows: - return new WindowsMemoryMapper(); - default: - throw new NotImplementedException( - $"No support for {Runtime.OperatingSystem}" - ); - } - } - - /// - /// Initializes the native code page. - /// - /// Safe to call if we already initialized (this function is idempotent). - /// - /// - internal static void InitializeNativeCodePage() - { - // Do nothing if we already initialized. - if (NativeCodePage != IntPtr.Zero) - { - return; - } - - // Allocate the page, write the native code into it, then set it - // to be executable. - IMemoryMapper mapper = CreateMemoryMapper(); - int codeLength = NativeCode.Active.Code.Length; - NativeCodePage = mapper.MapWriteable(codeLength); - Marshal.Copy(NativeCode.Active.Code, 0, NativeCodePage, codeLength); - mapper.SetReadExec(NativeCodePage, codeLength); - } - #endregion - /// /// Given a newly allocated Python type object and a managed Type that /// provides the implementation for the type, connect the type slots of /// the Python object to the managed methods of the implementing Type. /// - internal static void InitializeSlots(IntPtr type, Type impl) + internal static void InitializeSlots(IntPtr type, Type impl, SlotsHolder slotsHolder = null) { // We work from the most-derived class up; make sure to get // the most-derived slot and not to override it with a base @@ -727,12 +623,7 @@ internal static void InitializeSlots(IntPtr type, Type impl) foreach (MethodInfo method in methods) { string name = method.Name; - if (!(name.StartsWith("tp_") || - name.StartsWith("nb_") || - name.StartsWith("sq_") || - name.StartsWith("mp_") || - name.StartsWith("bf_") - )) + if (!name.StartsWith("tp_") && !SlotTypes.IsSlotName(name)) { continue; } @@ -742,8 +633,7 @@ internal static void InitializeSlots(IntPtr type, Type impl) continue; } - var thunkInfo = Interop.GetThunk(method); - InitializeSlot(type, thunkInfo.Address, name); + InitializeSlot(type, Interop.GetThunk(method), name, slotsHolder); seen.Add(name); } @@ -751,40 +641,17 @@ internal static void InitializeSlots(IntPtr type, Type impl) impl = impl.BaseType; } - var native = NativeCode.Active; - - // The garbage collection related slots always have to return 1 or 0 - // since .NET objects don't take part in Python's gc: - // tp_traverse (returns 0) - // tp_clear (returns 0) - // tp_is_gc (returns 1) - // These have to be defined, though, so by default we fill these with - // static C# functions from this class. - - var ret0 = Interop.GetThunk(((Func)Return0).Method).Address; - var ret1 = Interop.GetThunk(((Func)Return1).Method).Address; - - if (native != null) + foreach (string slot in _requiredSlots) { - // If we want to support domain reload, the C# implementation - // cannot be used as the assembly may get released before - // CPython calls these functions. Instead, for amd64 and x86 we - // load them into a separate code page that is leaked - // intentionally. - InitializeNativeCodePage(); - ret1 = NativeCodePage + native.Return1; - ret0 = NativeCodePage + native.Return0; + if (seen.Contains(slot)) + { + continue; + } + var offset = ManagedDataOffsets.GetSlotOffset(slot); + Marshal.WriteIntPtr(type, offset, SlotsHolder.GetDefaultSlot(offset)); } - - InitializeSlot(type, ret0, "tp_traverse"); - InitializeSlot(type, ret0, "tp_clear"); - InitializeSlot(type, ret1, "tp_is_gc"); } - static int Return1(IntPtr _) => 1; - - static int Return0(IntPtr _) => 0; - /// /// Helper for InitializeSlots. /// @@ -795,13 +662,48 @@ internal static void InitializeSlots(IntPtr type, Type impl) /// Type being initialized. /// Function pointer. /// Name of the method. - static void InitializeSlot(IntPtr type, IntPtr slot, string name) + /// Can override the slot when it existed + static void InitializeSlot(IntPtr type, IntPtr slot, string name, bool canOverride = true) + { + var offset = ManagedDataOffsets.GetSlotOffset(name); + if (!canOverride && Marshal.ReadIntPtr(type, offset) != IntPtr.Zero) + { + return; + } + Marshal.WriteIntPtr(type, offset, slot); + } + + static void InitializeSlot(IntPtr type, ThunkInfo thunk, string name, SlotsHolder slotsHolder = null, bool canOverride = true) { Type typeOffset = typeof(TypeOffset); FieldInfo fi = typeOffset.GetField(name); var offset = (int)fi.GetValue(typeOffset); - Marshal.WriteIntPtr(type, offset, slot); + if (!canOverride && Marshal.ReadIntPtr(type, offset) != IntPtr.Zero) + { + return; + } + Marshal.WriteIntPtr(type, offset, thunk.Address); + if (slotsHolder != null) + { + slotsHolder.Set(offset, thunk); + } + } + + static void InitializeSlot(IntPtr type, int slotOffset, MethodInfo method, SlotsHolder slotsHolder = null) + { + var thunk = Interop.GetThunk(method); + Marshal.WriteIntPtr(type, slotOffset, thunk.Address); + if (slotsHolder != null) + { + slotsHolder.Set(slotOffset, thunk); + } + } + + static bool IsSlotSet(IntPtr type, string name) + { + int offset = ManagedDataOffsets.GetSlotOffset(name); + return Marshal.ReadIntPtr(type, offset) != IntPtr.Zero; } /// @@ -832,6 +734,7 @@ private static void InitMethods(IntPtr pytype, Type type) mi[0] = method; MethodObject m = new TypeMethod(type, method_name, mi); Runtime.PyDict_SetItemString(dict, method_name, m.pyHandle); + m.DecrRefCount(); addedMethods.Add(method_name); } } @@ -849,5 +752,204 @@ internal static void CopySlot(IntPtr from, IntPtr to, int offset) IntPtr fp = Marshal.ReadIntPtr(from, offset); Marshal.WriteIntPtr(to, offset, fp); } + + private static SlotsHolder CreateSolotsHolder(IntPtr type) + { + var holder = new SlotsHolder(type); + _slotsHolders.Add(type, holder); + return holder; + } + } + + + class SlotsHolder + { + public delegate void Resetor(IntPtr type, int offset); + + private readonly IntPtr _type; + private Dictionary _slots = new Dictionary(); + private List _keepalive = new List(); + private Dictionary _customResetors = new Dictionary(); + private List _deallocators = new List(); + private bool _alreadyReset = false; + + /// + /// Create slots holder for holding the delegate of slots and be able to reset them. + /// + /// Steals a reference to target type + public SlotsHolder(IntPtr type) + { + _type = type; + } + + public void Set(int offset, ThunkInfo thunk) + { + _slots[offset] = thunk; + } + + public void Set(int offset, Resetor resetor) + { + _customResetors[offset] = resetor; + } + + public void AddDealloctor(Action deallocate) + { + _deallocators.Add(deallocate); + } + + public void KeeapAlive(ThunkInfo thunk) + { + _keepalive.Add(thunk); + } + + public void ResetSlots() + { + if (_alreadyReset) + { + return; + } + _alreadyReset = true; +#if DEBUG + IntPtr tp_name = Marshal.ReadIntPtr(_type, TypeOffset.tp_name); + string typeName = Marshal.PtrToStringAnsi(tp_name); +#endif + foreach (var offset in _slots.Keys) + { + IntPtr ptr = GetDefaultSlot(offset); +#if DEBUG + //DebugUtil.Print($"Set slot<{TypeOffsetHelper.GetSlotNameByOffset(offset)}> to 0x{ptr.ToString("X")} at {typeName}<0x{_type}>"); +#endif + Marshal.WriteIntPtr(_type, offset, ptr); + } + + foreach (var action in _deallocators) + { + action(); + } + + foreach (var pair in _customResetors) + { + int offset = pair.Key; + var resetor = pair.Value; + resetor?.Invoke(_type, offset); + } + + _customResetors.Clear(); + _slots.Clear(); + _keepalive.Clear(); + _deallocators.Clear(); + + // Custom reset + IntPtr handlePtr = Marshal.ReadIntPtr(_type, TypeOffset.magic()); + if (handlePtr != IntPtr.Zero) + { + GCHandle handle = GCHandle.FromIntPtr(handlePtr); + if (handle.IsAllocated) + { + handle.Free(); + } + Marshal.WriteIntPtr(_type, TypeOffset.magic(), IntPtr.Zero); + } + } + + public static IntPtr GetDefaultSlot(int offset) + { + if (offset == TypeOffset.tp_clear) + { + return TypeManager.subtype_clear; + } + else if (offset == TypeOffset.tp_traverse) + { + return TypeManager.subtype_traverse; + } + else if (offset == TypeOffset.tp_dealloc) + { + // tp_free of PyTypeType is point to PyObejct_GC_Del. + return Marshal.ReadIntPtr(Runtime.PyTypeType, TypeOffset.tp_free); + } + else if (offset == TypeOffset.tp_free) + { + // PyObject_GC_Del + return Marshal.ReadIntPtr(Runtime.PyTypeType, TypeOffset.tp_free); + } + else if (offset == TypeOffset.tp_call) + { + return IntPtr.Zero; + } + else if (offset == TypeOffset.tp_new) + { + // PyType_GenericNew + return Marshal.ReadIntPtr(Runtime.PySuper_Type, TypeOffset.tp_new); + } + else if (offset == TypeOffset.tp_getattro) + { + // PyObject_GenericGetAttr + return Marshal.ReadIntPtr(Runtime.PyBaseObjectType, TypeOffset.tp_getattro); + } + else if (offset == TypeOffset.tp_setattro) + { + // PyObject_GenericSetAttr + return Marshal.ReadIntPtr(Runtime.PyBaseObjectType, TypeOffset.tp_setattro); + } + + return Marshal.ReadIntPtr(Runtime.PyTypeType, offset); + } + } + + + static class SlotHelper + { + public static IntPtr CreateObjectType() + { + IntPtr globals = Runtime.PyDict_New(); + if (Runtime.PyDict_SetItemString(globals, "__builtins__", Runtime.PyEval_GetBuiltins()) != 0) + { + Runtime.XDecref(globals); + throw new PythonException(); + } + const string code = "class A(object): pass"; + var resRef = Runtime.PyRun_String(code, RunFlagType.File, globals, globals); + IntPtr res = resRef.DangerousGetAddress(); + if (res == IntPtr.Zero) + { + try + { + throw new PythonException(); + } + finally + { + Runtime.XDecref(globals); + } + } + resRef.Dispose(); + IntPtr A = Runtime.PyDict_GetItemString(globals, "A"); + Debug.Assert(A != IntPtr.Zero); + Runtime.XIncref(A); + Runtime.XDecref(globals); + return A; + } + } + + + static partial class SlotTypes + { + private static Dictionary _nameMap = new Dictionary(); + + static SlotTypes() + { + foreach (var type in Types) + { + FieldInfo[] fields = type.GetFields(); + foreach (var fi in fields) + { + _nameMap[fi.Name] = type; + } + } + } + + public static bool IsSlotName(string name) + { + return _nameMap.ContainsKey(name); + } } } diff --git a/tools/geninterop/geninterop.py b/tools/geninterop/geninterop.py index 902296229..aacc4af65 100644 --- a/tools/geninterop/geninterop.py +++ b/tools/geninterop/geninterop.py @@ -21,6 +21,11 @@ import sysconfig import subprocess +if sys.version_info.major > 2: + from io import StringIO +else: + from StringIO import StringIO + from pycparser import c_ast, c_parser _log = logging.getLogger() @@ -55,15 +60,18 @@ def __init__(self): self.__struct_members_stack = [] self.__ptr_decl_depth = 0 self.__struct_members = {} + self.__decl_names = {} def get_struct_members(self, name): """return a list of (name, type) of struct members""" - if name in self.__typedefs: - node = self.__get_leaf_node(self.__typedefs[name]) - name = node.name - if name not in self.__struct_members: - raise Exception("Unknown struct '%s'" % name) - return self.__struct_members[name] + defs = self.__typedefs.get(name) + if defs is None: + return None + node = self.__get_leaf_node(defs) + name = node.name + if name is None: + name = defs.declname + return self.__struct_members.get(name) def visit(self, node): if isinstance(node, c_ast.FileAST): @@ -92,6 +100,7 @@ def visit_typedef(self, typedef): self.visit(typedef.type) def visit_typedecl(self, typedecl): + self.__decl_names[typedecl.type] = typedecl.declname self.visit(typedecl.type) def visit_struct(self, struct): @@ -160,7 +169,22 @@ def __get_leaf_node(self, node): return node def __get_struct_name(self, node): - return node.name or "_struct_%d" % id(node) + return node.name or self.__decl_names.get(node) or "_struct_%d" % id(node) + + +class Writer(object): + + def __init__(self): + self._stream = StringIO() + + def append(self, indent=0, code=""): + self._stream.write("%s%s\n" % (indent * " ", code)) + + def extend(self, s): + self._stream.write(s) + + def to_string(self): + return self._stream.getvalue() def preprocess_python_headers(): @@ -188,6 +212,7 @@ def preprocess_python_headers(): defines.extend([ "-D", "__inline=inline", "-D", "__ptr32=", + "-D", "__ptr64=", "-D", "__declspec(x)=", ]) @@ -211,9 +236,8 @@ def preprocess_python_headers(): return "\n".join(lines) -def gen_interop_code(members): - """Generate the TypeOffset C# class""" +def gen_interop_head(writer): defines = [ "PYTHON{0}{1}".format(PY_MAJOR, PY_MINOR) ] @@ -243,27 +267,26 @@ def gen_interop_code(members): namespace Python.Runtime { - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal class TypeOffset - { - static TypeOffset() - { - Type type = typeof(TypeOffset); - FieldInfo[] fi = type.GetFields(); - int size = IntPtr.Size; - for (int i = 0; i < fi.Length; i++) - { - fi[i].SetValue(null, i * size); - } - } - - public static int magic() - { - return ob_size; - } +""" % (filename, defines_str) + writer.extend(class_definition) + + +def gen_interop_tail(writer): + tail = """} +#endif +""" + writer.extend(tail) + +def gen_heap_type_members(parser, writer): + """Generate the TypeOffset C# class""" + members = parser.get_struct_members("PyHeapTypeObject") + class_definition = """ + [StructLayout(LayoutKind.Sequential)] + internal static partial class TypeOffset + { // Auto-generated from PyHeapTypeObject in Python.h -""" % (filename, defines_str) +""" # All the members are sizeof(void*) so we don't need to do any # extra work to determine the size based on the type. @@ -275,11 +298,36 @@ def gen_interop_code(members): /* here are optional user slots, followed by the members. */ public static int members = 0; } -} -#endif """ - return class_definition + writer.extend(class_definition) + + +def gen_structure_code(parser, writer, type_name, indent): + members = parser.get_struct_members(type_name) + if members is None: + return False + out = writer.append + out(indent, "[StructLayout(LayoutKind.Sequential)]") + out(indent, "internal struct %s" % type_name) + out(indent, "{") + for name, tpy in members: + out(indent + 1, "public IntPtr %s;" % name) + out(indent, "}") + out() + return True + + +def gen_supported_slot_record(writer, types, indent): + out = writer.append + out(indent, "internal static partial class SlotTypes") + out(indent, "{") + out(indent + 1, "public static readonly Type[] Types = {") + for name in types: + out(indent + 2, "typeof(%s)," % name) + out(indent + 1, "};") + out(indent, "}") + out() def main(): @@ -292,10 +340,29 @@ def main(): ast_parser = AstParser() ast_parser.visit(ast) + writer = Writer() # generate the C# code - members = ast_parser.get_struct_members("PyHeapTypeObject") - interop_cs = gen_interop_code(members) + gen_interop_head(writer) + + gen_heap_type_members(ast_parser, writer) + slots_types = [ + "PyNumberMethods", + "PySequenceMethods", + "PyMappingMethods", + "PyAsyncMethods", + "PyBufferProcs", + ] + supported_types = [] + indent = 1 + for type_name in slots_types: + if not gen_structure_code(ast_parser, writer, type_name, indent): + continue + supported_types.append(type_name) + gen_supported_slot_record(writer, supported_types, indent) + + gen_interop_tail(writer) + interop_cs = writer.to_string() if len(sys.argv) > 1: with open(sys.argv[1], "w") as fh: fh.write(interop_cs) pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy