diff --git a/AUTHORS.md b/AUTHORS.md index 3c39794e4..023b4d185 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -14,6 +14,7 @@ - Alexandre Catarino([@AlexCatarino](https://github.com/AlexCatarino)) - Arvid JB ([@ArvidJB](https://github.com/ArvidJB)) +- Benoît Hudson ([@benoithudson](https://github.com/benoithudson)) - Bradley Friedman ([@leith-bartrich](https://github.com/leith-bartrich)) - Callum Noble ([@callumnoble](https://github.com/callumnoble)) - Christian Heimes ([@tiran](https://github.com/tiran)) @@ -22,6 +23,7 @@ - Daniel Fernandez ([@fdanny](https://github.com/fdanny)) - Daniel Santana ([@dgsantana](https://github.com/dgsantana)) - Dave Hirschfeld ([@dhirschfeld](https://github.com/dhirschfeld)) +- David Lassonde ([@lassond](https://github.com/lassond)) - David Lechner ([@dlech](https://github.com/dlech)) - Dmitriy Se ([@dmitriyse](https://github.com/dmitriyse)) - He-chien Tsai ([@t3476](https://github.com/t3476)) @@ -39,6 +41,7 @@ - Sam Winstanley ([@swinstanley](https://github.com/swinstanley)) - Sean Freitag ([@cowboygneox](https://github.com/cowboygneox)) - Serge Weinstock ([@sweinst](https://github.com/sweinst)) +- Viktoria Kovescses ([@vkovec](https://github.com/vkovec)) - Ville M. Vainio ([@vivainio](https://github.com/vivainio)) - Virgil Dupras ([@hsoft](https://github.com/hsoft)) - Wenguang Yang ([@yagweb](https://github.com/yagweb)) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3dc668b0c..c165a6c4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,8 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. ### Fixed - Fixed Visual Studio 2017 compat ([#434][i434]) for setup.py +- Fixed crashes when integrating pythonnet in Unity3d ([#714][i714]), + related to unloading the Application Domain - Fixed crash on exit of the Python interpreter if a python class derived from a .NET class has a `__namespace__` or `__assembly__` attribute ([#481][i481]) diff --git a/appveyor.yml b/appveyor.yml index 6bebef490..b38fc48dd 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -28,6 +28,12 @@ environment: - PYTHON_VERSION: 3.5 - PYTHON_VERSION: 3.6 +matrix: + allow_failures: + - PYTHON_VERSION: 3.4 + BUILD_OPTS: --xplat + - PYTHON_VERSION: 3.4 + init: # Update Environment Variables based on matrix/platform - set PY_VER=%PYTHON_VERSION:.=% @@ -38,11 +44,13 @@ init: - set PATH=%PYTHON%;%PYTHON%\Scripts;%PATH% install: + # Upgrade setuptools to find MSVC. Otherwise you get "error: Microsoft Visual C++ 10.0 is required (Unable to find vcvarsall.bat)." - python -m pip install -U pip - - pip install --upgrade -r requirements.txt --quiet + - pip install --upgrade setuptools + - pip install --upgrade -r requirements.txt # Install OpenCover. Can't put on `packages.config`, not Mono compatible - - .\tools\nuget\nuget.exe install OpenCover -OutputDirectory packages -Verbosity quiet + - .\tools\nuget\nuget.exe install OpenCover -OutputDirectory packages build_script: # Create clean `sdist`. Only used for releases diff --git a/setup.py b/setup.py index 4ec2a2113..183ba4c3a 100644 --- a/setup.py +++ b/setup.py @@ -329,7 +329,7 @@ def _install_packages(self): self.debug_print("Updating NuGet: {0}".format(cmd)) subprocess.check_call(cmd, shell=use_shell) - cmd = "{0} restore pythonnet.sln -o packages".format(nuget) + cmd = "{0} restore pythonnet.sln -MSBuildVersion 14 -o packages".format(nuget) self.debug_print("Installing packages: {0}".format(cmd)) subprocess.check_call(cmd, shell=use_shell) diff --git a/src/embed_tests/Python.EmbeddingTest.15.csproj b/src/embed_tests/Python.EmbeddingTest.15.csproj index 92d55a7e0..a741a589e 100644 --- a/src/embed_tests/Python.EmbeddingTest.15.csproj +++ b/src/embed_tests/Python.EmbeddingTest.15.csproj @@ -29,6 +29,7 @@ XPLAT $(DefineConstants);$(CustomDefineConstants);$(BaseDefineConstants); $(DefineConstants);NETCOREAPP + $(DefineConstants);NETSTANDARD $(DefineConstants);TRACE;DEBUG $(NuGetPackageRoot)\microsoft.targetingpack.netframework.v4.5\1.0.1\lib\net45\ diff --git a/src/embed_tests/Python.EmbeddingTest.csproj b/src/embed_tests/Python.EmbeddingTest.csproj index 66e8c7165..e50053f07 100644 --- a/src/embed_tests/Python.EmbeddingTest.csproj +++ b/src/embed_tests/Python.EmbeddingTest.csproj @@ -86,6 +86,7 @@ + @@ -103,6 +104,7 @@ + @@ -122,4 +124,4 @@ - \ No newline at end of file + diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs new file mode 100644 index 000000000..092daa413 --- /dev/null +++ b/src/embed_tests/TestDomainReload.cs @@ -0,0 +1,270 @@ +using System; +using System.CodeDom.Compiler; +using System.Reflection; +using NUnit.Framework; +using Python.Runtime; + +// +// This test case is disabled on .NET Standard because it doesn't have all the +// APIs we use. We could work around that, but .NET Core doesn't implement +// domain creation, so it's not worth it. +// +// Unfortunately this means no continuous integration testing for this case. +// +#if !NETSTANDARD && !NETCOREAPP +namespace Python.EmbeddingTest +{ + class TestDomainReload + { + /// + /// Test that the python runtime can survive a C# domain reload without crashing. + /// + /// At the time this test was written, there was a very annoying + /// seemingly random crash bug when integrating pythonnet into Unity. + /// + /// The repro steps that David Lassonde, Viktoria Kovecses and + /// Benoit Hudson eventually worked out: + /// 1. Write a HelloWorld.cs script that uses Python.Runtime to access + /// some C# data from python: C# calls python, which calls C#. + /// 2. Execute the script (e.g. make it a MenuItem and click it). + /// 3. Touch HelloWorld.cs on disk, forcing Unity to recompile scripts. + /// 4. Wait several seconds for Unity to be done recompiling and + /// reloading the C# domain. + /// 5. Make python run the gc (e.g. by calling gc.collect()). + /// + /// The reason: + /// A. In step 2, Python.Runtime registers a bunch of new types with + /// their tp_traverse slot pointing to managed code, and allocates + /// some objects of those types. + /// B. In step 4, Unity unloads the C# domain. That frees the managed + /// code. But at the time of the crash investigation, pythonnet + /// leaked the python side of the objects allocated in step 1. + /// C. In step 5, python sees some pythonnet objects in its gc list of + /// potentially-leaked objects. It calls tp_traverse on those objects. + /// But tp_traverse was freed in step 3 => CRASH. + /// + /// This test distills what's going on without needing Unity around (we'd see + /// similar behaviour if we were using pythonnet on a .NET web server that did + /// a hot reload). + /// + [Test] + public static void DomainReloadAndGC() + { + // We're set up to run in the directory that includes the bin directory. + System.IO.Directory.SetCurrentDirectory(AppDomain.CurrentDomain.BaseDirectory); + + Assembly pythonRunner1 = BuildAssembly("test1"); + RunAssemblyAndUnload(pythonRunner1, "test1"); + + // Verify that python is not initialized even though we ran it. + Assert.That(Runtime.Runtime.Py_IsInitialized(), Is.Zero); + + // This caused a crash because objects allocated in pythonRunner1 + // still existed in memory, but the code to do python GC on those + // objects is gone. + Assembly pythonRunner2 = BuildAssembly("test2"); + RunAssemblyAndUnload(pythonRunner2, "test2"); + } + + /// + /// On travis, printing to stdout causes a race condition (which + /// becomes visible as either EPIPE or a deadlock). Set Verbose to true + /// to print-debug the example on your laptop, but keep it off for CI. + /// + /// Verbose is protected to quash a warning that it's never set. + /// + protected static bool Verbose = false; + + /// + /// On travis, printing to stdout causes a race condition (which + /// becomes visible as either EPIPE or a deadlock). Set Verbose to true + /// to print-debug the example on your laptop, but keep it off for CI. + /// + static void WriteLine(string line) { + if (!Verbose) { return; } + Console.WriteLine(line); + Console.Out.Flush(); + } + + /// + /// The code we'll test. All that really matters is + /// using GIL { Python.Exec(pyScript); } + /// but the rest is useful for debugging. + /// + /// What matters in the python code is gc.collect and clr.AddReference. + /// + /// On Windows, the language version is 2.0, so no $"foo{bar}" syntax. + /// + /// On travis, printing to stdout causes a race condition (which + /// becomes visible as either EPIPE or a deadlock). Set Verbose to true + /// to print-debug the example on your laptop, but keep it off for CI. + /// + const string TestCode = @" + using Python.Runtime; + using System; + class PythonRunner { + protected static bool Verbose = false; + static void WriteLine(string line) { + if (!Verbose) { return; } + Console.WriteLine(line); + Console.Out.Flush(); + } + public static void RunPython() { + AppDomain.CurrentDomain.DomainUnload += OnDomainUnload; + string name = AppDomain.CurrentDomain.FriendlyName; + WriteLine(string.Format(""[{0} in .NET] In PythonRunner.RunPython"", name)); + + using (Py.GIL()) { + try { + var pyScript = string.Format(""import clr\n"" + + ""print('[{0} in python] imported clr')\n"" + + ""clr.AddReference('System')\n"" + + ""print('[{0} in python] allocated a clr object')\n"" + + ""import gc\n"" + + ""gc.collect()\n"" + + ""print('[{0} in python] collected garbage')\n"", + name); + PythonEngine.Exec(pyScript); + } catch(Exception e) { + WriteLine(string.Format(""[{0} in .NET] Caught exception: {1}"", name, e)); + } + } + } + static void OnDomainUnload(object sender, EventArgs e) { + WriteLine(string.Format(""[{0} in .NET] unloading"", AppDomain.CurrentDomain.FriendlyName)); + } + }"; + + + /// + /// Build an assembly out of the source code above. + /// + /// This creates a file .dll in order + /// to support the statement "proxy.theAssembly = assembly" below. + /// That statement needs a file, can't run via memory. + /// + static Assembly BuildAssembly(string assemblyName) + { + var provider = CodeDomProvider.CreateProvider("CSharp"); + + var compilerparams = new CompilerParameters(); + compilerparams.ReferencedAssemblies.Add("Python.Runtime.dll"); + compilerparams.GenerateExecutable = false; + compilerparams.GenerateInMemory = false; + compilerparams.IncludeDebugInformation = false; + compilerparams.OutputAssembly = assemblyName; + + var results = provider.CompileAssemblyFromSource(compilerparams, TestCode); + if (results.Errors.HasErrors) + { + var errors = new System.Text.StringBuilder("Compiler Errors:\n"); + foreach (CompilerError error in results.Errors) + { + errors.AppendFormat("Line {0},{1}\t: {2}\n", + error.Line, error.Column, error.ErrorText); + } + throw new Exception(errors.ToString()); + } + else + { + return results.CompiledAssembly; + } + } + + /// + /// This is a magic incantation required to run code in an application + /// domain other than the current one. + /// + class Proxy : MarshalByRefObject + { + Assembly theAssembly = null; + + public void InitAssembly(string assemblyPath) + { + theAssembly = Assembly.LoadFile(System.IO.Path.GetFullPath(assemblyPath)); + } + + public void RunPython() + { + WriteLine("[Proxy] Entering RunPython"); + + // Call into the new assembly. Will execute Python code + var pythonrunner = theAssembly.GetType("PythonRunner"); + var runPythonMethod = pythonrunner.GetMethod("RunPython"); + runPythonMethod.Invoke(null, new object[] { }); + + WriteLine("[Proxy] Leaving RunPython"); + } + } + + /// + /// Create a domain, run the assembly in it (the RunPython function), + /// and unload the domain. + /// + static void RunAssemblyAndUnload(Assembly assembly, string assemblyName) + { + WriteLine($"[Program.Main] === creating domain for assembly {assembly.FullName}"); + + // Create the domain. Make sure to set PrivateBinPath to a relative + // path from the CWD (namely, 'bin'). + // See https://stackoverflow.com/questions/24760543/createinstanceandunwrap-in-another-domain + var currentDomain = AppDomain.CurrentDomain; + var domainsetup = new AppDomainSetup() + { + ApplicationBase = currentDomain.SetupInformation.ApplicationBase, + ConfigurationFile = currentDomain.SetupInformation.ConfigurationFile, + LoaderOptimization = LoaderOptimization.SingleDomain, + PrivateBinPath = "." + }; + var domain = AppDomain.CreateDomain( + $"My Domain {assemblyName}", + currentDomain.Evidence, + domainsetup); + + // Create a Proxy object in the new domain, where we want the + // assembly (and Python .NET) to reside + Type type = typeof(Proxy); + System.IO.Directory.SetCurrentDirectory(AppDomain.CurrentDomain.BaseDirectory); + var theProxy = (Proxy)domain.CreateInstanceAndUnwrap( + type.Assembly.FullName, + type.FullName); + + // From now on use the Proxy to call into the new assembly + theProxy.InitAssembly(assemblyName); + theProxy.RunPython(); + + WriteLine($"[Program.Main] Before Domain Unload on {assembly.FullName}"); + AppDomain.Unload(domain); + WriteLine($"[Program.Main] After Domain Unload on {assembly.FullName}"); + + // Validate that the assembly does not exist anymore + try + { + WriteLine($"[Program.Main] The Proxy object is valid ({theProxy}). Unexpected domain unload behavior"); + } + catch (Exception) + { + WriteLine("[Program.Main] The Proxy object is not valid anymore, domain unload complete."); + } + } + + /// + /// Resolves the assembly. Why doesn't this just work normally? + /// + static Assembly ResolveAssembly(object sender, ResolveEventArgs args) + { + var loadedAssemblies = AppDomain.CurrentDomain.GetAssemblies(); + + foreach (var assembly in loadedAssemblies) + { + if (assembly.FullName == args.Name) + { + return assembly; + } + } + + return null; + } + } +} +#endif diff --git a/src/embed_tests/TestRuntime.cs b/src/embed_tests/TestRuntime.cs index 2e0598da7..f26a1e4b4 100644 --- a/src/embed_tests/TestRuntime.cs +++ b/src/embed_tests/TestRuntime.cs @@ -6,6 +6,26 @@ namespace Python.EmbeddingTest { public class TestRuntime { + /// + /// Test the cache of the information from the platform module. + /// + /// Test fails on platforms we haven't implemented yet. + /// + [Test] + public static void PlatformCache() + { + Runtime.Runtime.Initialize(); + + Assert.That(Runtime.Runtime.Machine, Is.Not.EqualTo(Runtime.Runtime.MachineType.Other)); + Assert.That(!string.IsNullOrEmpty(Runtime.Runtime.MachineName)); + + Assert.That(Runtime.Runtime.OperatingSystem, Is.Not.EqualTo(Runtime.Runtime.OperatingSystemType.Other)); + Assert.That(!string.IsNullOrEmpty(Runtime.Runtime.OperatingSystemName)); + + // Don't shut down the runtime: if the python engine was initialized + // but not shut down by another test, we'd end up in a bad state. + } + [Test] public static void Py_IsInitializedValue() { diff --git a/src/embed_tests/TestTypeManager.cs b/src/embed_tests/TestTypeManager.cs new file mode 100644 index 000000000..a4ef86913 --- /dev/null +++ b/src/embed_tests/TestTypeManager.cs @@ -0,0 +1,65 @@ +using NUnit.Framework; +using Python.Runtime; +using System.Runtime.InteropServices; + +namespace Python.EmbeddingTest +{ + class TestTypeManager + { + [SetUp] + public static void Init() + { + Runtime.Runtime.Initialize(); + } + + [TearDown] + public static void Fini() + { + // Don't shut down the runtime: if the python engine was initialized + // but not shut down by another test, we'd end up in a bad state. + } + + [Test] + public static void TestNativeCode() + { + Assert.That(() => { var _ = TypeManager.NativeCode.Active; }, Throws.Nothing); + Assert.That(TypeManager.NativeCode.Active.Code.Length, Is.GreaterThan(0)); + } + + [Test] + public static void TestMemoryMapping() + { + Assert.That(() => { var _ = TypeManager.CreateMemoryMapper(); }, Throws.Nothing); + var mapper = TypeManager.CreateMemoryMapper(); + + // Allocate a read-write page. + int len = 12; + var page = mapper.MapWriteable(len); + Assert.That(() => { Marshal.WriteInt64(page, 17); }, Throws.Nothing); + Assert.That(Marshal.ReadInt64(page), Is.EqualTo(17)); + + // Mark it read-execute. We can still read, haven't changed any values. + mapper.SetReadExec(page, len); + Assert.That(Marshal.ReadInt64(page), Is.EqualTo(17)); + + // Test that we can't write to the protected page. + // + // We can't actually test access protection under Microsoft + // versions of .NET, because AccessViolationException is assumed to + // mean we're in a corrupted state: + // https://stackoverflow.com/questions/3469368/how-to-handle-accessviolationexception + // + // We can test under Mono but it throws NRE instead of AccessViolationException. + // + // We can't use compiler flags because we compile with MONO_LINUX + // while running on the Microsoft .NET Core during continuous + // integration tests. + if (System.Type.GetType ("Mono.Runtime") != null) + { + // Mono throws NRE instead of AccessViolationException for some reason. + Assert.That(() => { Marshal.WriteInt64(page, 73); }, Throws.TypeOf()); + Assert.That(Marshal.ReadInt64(page), Is.EqualTo(17)); + } + } + } +} diff --git a/src/embed_tests/dynamic.cs b/src/embed_tests/dynamic.cs index d75dc01d6..b05943c6d 100644 --- a/src/embed_tests/dynamic.cs +++ b/src/embed_tests/dynamic.cs @@ -12,13 +12,23 @@ public class DynamicTest [SetUp] public void SetUp() { - _gs = Py.GIL(); + try { + _gs = Py.GIL(); + } catch (Exception e) { + Console.WriteLine($"exception in SetUp: {e}"); + throw; + } } [TearDown] public void Dispose() { - _gs.Dispose(); + try { + _gs.Dispose(); + } catch(Exception e) { + Console.WriteLine($"exception in TearDown: {e}"); + throw; + } } /// diff --git a/src/runtime/classbase.cs b/src/runtime/classbase.cs index 4dd3b5364..5846fa82a 100644 --- a/src/runtime/classbase.cs +++ b/src/runtime/classbase.cs @@ -247,24 +247,6 @@ public static IntPtr tp_str(IntPtr ob) } - /// - /// Default implementations for required Python GC support. - /// - public static int tp_traverse(IntPtr ob, IntPtr func, IntPtr args) - { - return 0; - } - - public static int tp_clear(IntPtr ob) - { - return 0; - } - - public static int tp_is_gc(IntPtr type) - { - return 1; - } - /// /// Standard dealloc implementation for instances of reflected types. /// diff --git a/src/runtime/extensiontype.cs b/src/runtime/extensiontype.cs index 9569b0485..693a46f42 100644 --- a/src/runtime/extensiontype.cs +++ b/src/runtime/extensiontype.cs @@ -81,27 +81,6 @@ public static int tp_descr_set(IntPtr ds, IntPtr ob, IntPtr val) } - /// - /// Required Python GC support. - /// - public static int tp_traverse(IntPtr ob, IntPtr func, IntPtr args) - { - return 0; - } - - - public static int tp_clear(IntPtr ob) - { - return 0; - } - - - public static int tp_is_gc(IntPtr type) - { - return 1; - } - - /// /// Default dealloc implementation. /// diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index a23c7ac79..8ab44d041 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -220,9 +220,17 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true) { locals.Dispose(); } + + // Make sure we clean up properly on app domain unload. + AppDomain.CurrentDomain.DomainUnload += OnDomainUnload; } } + static void OnDomainUnload(object _, EventArgs __) + { + Shutdown(); + } + /// /// A helper to perform initialization from the context of an active /// CPython interpreter process - this bootstraps the managed runtime @@ -303,6 +311,8 @@ public static void Shutdown() _pythonPath = IntPtr.Zero; Runtime.Shutdown(); + + AppDomain.CurrentDomain.DomainUnload -= OnDomainUnload; initialized = false; } } diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index b4ddc7f7e..9423b0138 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -2,6 +2,7 @@ using System.Runtime.InteropServices; using System.Security; using System.Text; +using System.Collections.Generic; namespace Python.Runtime { @@ -21,7 +22,7 @@ public static IntPtr LoadLibrary(string fileName) } #elif MONO_OSX private static int RTLD_GLOBAL = 0x8; - private const string NativeDll = "/usr/lib/libSystem.dylib" + private const string NativeDll = "/usr/lib/libSystem.dylib"; private static IntPtr RTLD_DEFAULT = new IntPtr(-2); public static IntPtr LoadLibrary(string fileName) @@ -195,6 +196,64 @@ public class Runtime // .NET core: System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(OSPlatform.Windows) internal static bool IsWindows = Environment.OSVersion.Platform == PlatformID.Win32NT; + /// + /// Operating system type as reported by Python. + /// + public enum OperatingSystemType + { + Windows, + Darwin, + Linux, + Other + } + + static readonly Dictionary OperatingSystemTypeMapping = new Dictionary() + { + { "Windows", OperatingSystemType.Windows }, + { "Darwin", OperatingSystemType.Darwin }, + { "Linux", OperatingSystemType.Linux }, + }; + + /// + /// Gets the operating system as reported by python's platform.system(). + /// + public static OperatingSystemType OperatingSystem { get; private set; } + + /// + /// Gets the operating system as reported by python's platform.system(). + /// + public static string OperatingSystemName { get; private set; } + + public enum MachineType + { + i386, + x86_64, + Other + }; + + /// + /// Map lower-case version of the python machine name to the processor + /// type. There are aliases, e.g. x86_64 and amd64 are two names for + /// the same thing. Make sure to lower-case the search string, because + /// capitalization can differ. + /// + static readonly Dictionary MachineTypeMapping = new Dictionary() + { + { "i386", MachineType.i386 }, + { "x86_64", MachineType.x86_64 }, + { "amd64", MachineType.x86_64 }, + }; + + /// + /// Gets the machine architecture as reported by python's platform.machine(). + /// + public static MachineType Machine { get; private set; }/* set in Initialize using python's platform.machine */ + + /// + /// Gets the machine architecture as reported by python's platform.machine(). + /// + public static string MachineName { get; private set; } + internal static bool IsPython2 = pyversionnumber < 30; internal static bool IsPython3 = pyversionnumber >= 30; @@ -331,6 +390,10 @@ internal static void Initialize() NativeMethods.FreeLibrary(dllLocal); } #endif + // Initialize data about the platform we're running on. We need + // this for the type manager and potentially other details. Must + // happen after caching the python types, above. + InitializePlatformData(); // Initialize modules that depend on the runtime class. AssemblyManager.Initialize(); @@ -348,6 +411,53 @@ internal static void Initialize() AssemblyManager.UpdatePath(); } + /// + /// Initializes the data about platforms. + /// + /// This must be the last step when initializing the runtime: + /// GetManagedString needs to have the cached values for types. + /// But it must run before initializing anything outside the runtime + /// because those rely on the platform data. + /// + private static void InitializePlatformData() + { + IntPtr op; + IntPtr fn; + IntPtr platformModule = PyImport_ImportModule("platform"); + IntPtr emptyTuple = PyTuple_New(0); + + fn = PyObject_GetAttrString(platformModule, "system"); + op = PyObject_Call(fn, emptyTuple, IntPtr.Zero); + OperatingSystemName = GetManagedString(op); + XDecref(op); + XDecref(fn); + + fn = PyObject_GetAttrString(platformModule, "machine"); + op = PyObject_Call(fn, emptyTuple, IntPtr.Zero); + MachineName = GetManagedString(op); + XDecref(op); + XDecref(fn); + + XDecref(emptyTuple); + XDecref(platformModule); + + // Now convert the strings into enum values so we can do switch + // statements rather than constant parsing. + OperatingSystemType OSType; + if (!OperatingSystemTypeMapping.TryGetValue(OperatingSystemName, out OSType)) + { + OSType = OperatingSystemType.Other; + } + OperatingSystem = OSType; + + MachineType MType; + if (!MachineTypeMapping.TryGetValue(MachineName.ToLower(), out MType)) + { + MType = MachineType.Other; + } + Machine = MType; + } + internal static void Shutdown() { AssemblyManager.Shutdown(); diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 6570ee083..df2e71be0 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections; using System.Collections.Generic; using System.Reflection; @@ -447,6 +447,225 @@ internal static IntPtr AllocateTypeObject(string name) } + #region Native Code Page + /// + /// Initialized by InitializeNativeCodePage. + /// + /// This points to a page of memory allocated using mmap or VirtualAlloc + /// (depending on the system), and marked read and execute (not write). + /// Very much on purpose, the page is *not* released on a shutdown and + /// is instead leaked. See the TestDomainReload test case. + /// + /// The contents of the page are two native functions: one that returns 0, + /// one that returns 1. + /// + /// If python didn't keep its gc list through a Py_Finalize we could remove + /// this entire section. + /// + internal static IntPtr NativeCodePage = IntPtr.Zero; + + /// + /// Structure to describe native code. + /// + /// Use NativeCode.Active to get the native code for the current platform. + /// + /// Generate the code by creating the following C code: + /// + /// int Return0() { return 0; } + /// int Return1() { return 1; } + /// + /// Then compiling on the target platform, e.g. with gcc or clang: + /// cc -c -fomit-frame-pointer -O2 foo.c + /// And then analyzing the resulting functions with a hex editor, e.g.: + /// objdump -disassemble foo.o + /// + internal class NativeCode + { + /// + /// The code, as a string of bytes. + /// + public byte[] Code { get; private set; } + + /// + /// Where does the "return 0" function start? + /// + public int Return0 { get; private set; } + + /// + /// Where does the "return 1" function start? + /// + public int Return1 { get; private set; } + + public static NativeCode Active + { + get + { + switch(Runtime.Machine) + { + case Runtime.MachineType.i386: + return I386; + case Runtime.MachineType.x86_64: + return X86_64; + default: + throw new NotImplementedException($"No support for {Runtime.MachineName}"); + } + } + } + + /// + /// Code for x86_64. See the class comment for how it was generated. + /// + public static readonly NativeCode X86_64 = new NativeCode() + { + Return0 = 0x10, + Return1 = 0, + Code = new byte[] + { + // First Return1: + 0xb8, 0x01, 0x00, 0x00, 0x00, // movl $1, %eax + 0xc3, // ret + + // Now some padding so that Return0 can be 16-byte-aligned. + // I put Return1 first so there's not as much padding to type in. + 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, // nop + + // Now Return0. + 0x31, 0xc0, // xorl %eax, %eax + 0xc3, // ret + } + }; + + /// + /// Code for X86. + /// + /// It's bitwise identical to X86_64, so we just point to it. + /// + /// + public static readonly NativeCode I386 = X86_64; + } + + /// + /// Platform-dependent mmap and mprotect. + /// + internal interface IMemoryMapper + { + /// + /// Map at least numBytes of memory. Mark the page read-write (but not exec). + /// + IntPtr MapWriteable(int numBytes); + + /// + /// Sets the mapped memory to be read-exec (but not write). + /// + void SetReadExec(IntPtr mappedMemory, int numBytes); + } + + class WindowsMemoryMapper : IMemoryMapper + { + const UInt32 MEM_COMMIT = 0x1000; + const UInt32 MEM_RESERVE = 0x2000; + const UInt32 PAGE_READWRITE = 0x04; + const UInt32 PAGE_EXECUTE_READ = 0x20; + + [DllImport("kernel32.dll")] + static extern IntPtr VirtualAlloc(IntPtr lpAddress, IntPtr dwSize, UInt32 flAllocationType, UInt32 flProtect); + + [DllImport("kernel32.dll")] + static extern bool VirtualProtect(IntPtr lpAddress, IntPtr dwSize, UInt32 flNewProtect, out UInt32 lpflOldProtect); + + public IntPtr MapWriteable(int numBytes) + { + return VirtualAlloc(IntPtr.Zero, new IntPtr(numBytes), + MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); + } + + public void SetReadExec(IntPtr mappedMemory, int numBytes) + { + UInt32 _; + VirtualProtect(mappedMemory, new IntPtr(numBytes), PAGE_EXECUTE_READ, out _); + } + } + + class UnixMemoryMapper : IMemoryMapper + { + const int PROT_READ = 0x1; + const int PROT_WRITE = 0x2; + const int PROT_EXEC = 0x4; + + const int MAP_PRIVATE = 0x2; + int MAP_ANONYMOUS + { + get + { + switch (Runtime.OperatingSystem) + { + case Runtime.OperatingSystemType.Darwin: + return 0x1000; + case Runtime.OperatingSystemType.Linux: + return 0x20; + default: + throw new NotImplementedException($"mmap is not supported on {Runtime.OperatingSystemName}"); + } + } + } + + [DllImport("libc")] + static extern IntPtr mmap(IntPtr addr, IntPtr len, int prot, int flags, int fd, IntPtr offset); + + [DllImport("libc")] + static extern int mprotect(IntPtr addr, IntPtr len, int prot); + + public IntPtr MapWriteable(int numBytes) + { + // MAP_PRIVATE must be set on linux, even though MAP_ANON implies it. + // It doesn't hurt on darwin, so just do it. + return mmap(IntPtr.Zero, new IntPtr(numBytes), PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, IntPtr.Zero); + } + + public void SetReadExec(IntPtr mappedMemory, int numBytes) + { + mprotect(mappedMemory, new IntPtr(numBytes), PROT_READ | PROT_EXEC); + } + } + + internal static IMemoryMapper CreateMemoryMapper() + { + switch (Runtime.OperatingSystem) + { + case Runtime.OperatingSystemType.Darwin: + case Runtime.OperatingSystemType.Linux: + return new UnixMemoryMapper(); + case Runtime.OperatingSystemType.Windows: + return new WindowsMemoryMapper(); + default: + throw new NotImplementedException($"No support for {Runtime.OperatingSystemName}"); + } + } + + /// + /// Initializes the native code page. + /// + /// Safe to call if we already initialized (this function is idempotent). + /// + /// + internal static void InitializeNativeCodePage() + { + // Do nothing if we already initialized. + if (NativeCodePage != IntPtr.Zero) + { + return; + } + + // Allocate the page, write the native code into it, then set it + // to be executable. + IMemoryMapper mapper = CreateMemoryMapper(); + int codeLength = NativeCode.Active.Code.Length; + NativeCodePage = mapper.MapWriteable(codeLength); + Marshal.Copy(NativeCode.Active.Code, 0, NativeCodePage, codeLength); + mapper.SetReadExec(NativeCodePage, codeLength); + } +#endregion + /// /// Given a newly allocated Python type object and a managed Type that /// provides the implementation for the type, connect the type slots of @@ -454,8 +673,10 @@ internal static IntPtr AllocateTypeObject(string name) /// internal static void InitializeSlots(IntPtr type, Type impl) { - var seen = new Hashtable(8); - Type offsetType = typeof(TypeOffset); + // We work from the most-derived class up; make sure to get + // the most-derived slot and not to override it with a base + // class's slot. + var seen = new HashSet(); while (impl != null) { @@ -473,24 +694,52 @@ internal static void InitializeSlots(IntPtr type, Type impl) continue; } - if (seen[name] != null) + if (seen.Contains(name)) { continue; } - FieldInfo fi = offsetType.GetField(name); - var offset = (int)fi.GetValue(offsetType); - - IntPtr slot = Interop.GetThunk(method); - Marshal.WriteIntPtr(type, offset, slot); + InitializeSlot(type, Interop.GetThunk(method), name); - seen[name] = 1; + seen.Add(name); } impl = impl.BaseType; } + + // See the TestDomainReload test: there was a crash related to + // the gc-related slots. They always return 0 or 1 because we don't + // really support gc: + // tp_traverse (returns 0) + // tp_clear (returns 0) + // tp_is_gc (returns 1) + // We can't do without: python really wants those slots to exist. + // We can't implement those in C# because the application domain + // can be shut down and the memory released. + InitializeNativeCodePage(); + InitializeSlot(type, NativeCodePage + NativeCode.Active.Return0, "tp_traverse"); + InitializeSlot(type, NativeCodePage + NativeCode.Active.Return0, "tp_clear"); + InitializeSlot(type, NativeCodePage + NativeCode.Active.Return1, "tp_is_gc"); } + /// + /// Helper for InitializeSlots. + /// + /// Initializes one slot to point to a function pointer. + /// The function pointer might be a thunk for C#, or it may be + /// an address in the NativeCodePage. + /// + /// Type being initialized. + /// Function pointer. + /// Name of the method. + static void InitializeSlot(IntPtr type, IntPtr slot, string name) + { + Type typeOffset = typeof(TypeOffset); + FieldInfo fi = typeOffset.GetField(name); + var offset = (int)fi.GetValue(typeOffset); + + Marshal.WriteIntPtr(type, offset, slot); + } /// /// Given a newly allocated Python type object and a managed Type that pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy