diff --git a/CHANGELOG.md b/CHANGELOG.md index 8aba9e9b4..da8f94774 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,6 +50,7 @@ One must now either use enum members (e.g. `MyEnum.Option`), or use enum constru - .NET and Python exceptions are preserved when crossing Python/.NET boundary - BREAKING: custom encoders are no longer called for instances of `System.Type` - `PythonException.Restore` no longer clears `PythonException` instance. +- Replaced the old `__import__` hook hack with a PEP302-style Meta Path Loader ### Fixed diff --git a/src/embed_tests/TestPythonException.cs b/src/embed_tests/TestPythonException.cs index 0763bfb34..a4b28906c 100644 --- a/src/embed_tests/TestPythonException.cs +++ b/src/embed_tests/TestPythonException.cs @@ -178,6 +178,7 @@ public void TestPythonException_Normalize_ThrowsWhenErrorSet() var pythonException = PythonException.FetchCurrentRaw(); Exceptions.SetError(Exceptions.TypeError, "Another error"); Assert.Throws(() => pythonException.Normalize()); + Exceptions.Clear(); } } } diff --git a/src/embed_tests/pyimport.cs b/src/embed_tests/pyimport.cs index e98461cbb..de8a06bf8 100644 --- a/src/embed_tests/pyimport.cs +++ b/src/embed_tests/pyimport.cs @@ -37,7 +37,7 @@ public void SetUp() Assert.IsFalse(str == IntPtr.Zero); BorrowedReference path = Runtime.Runtime.PySys_GetObject("path"); Assert.IsFalse(path.IsNull); - Runtime.Runtime.PyList_Append(path, str); + Runtime.Runtime.PyList_Append(path, new BorrowedReference(str)); Runtime.Runtime.XDecref(str); } diff --git a/src/runtime/assemblymanager.cs b/src/runtime/assemblymanager.cs index 0387d2dfc..d44f5f666 100644 --- a/src/runtime/assemblymanager.cs +++ b/src/runtime/assemblymanager.cs @@ -37,7 +37,6 @@ internal class AssemblyManager // modified from event handlers below, potentially triggered from different .NET threads private static ConcurrentQueue assemblies; internal static List pypath; - private AssemblyManager() { } @@ -312,6 +311,15 @@ public static bool IsValidNamespace(string name) return !string.IsNullOrEmpty(name) && namespaces.ContainsKey(name); } + /// + /// Returns an IEnumerable containing the namepsaces exported + /// by loaded assemblies in the current app domain. + /// + public static IEnumerable GetNamespaces () + { + return namespaces.Keys; + } + /// /// Returns list of assemblies that declare types in a given namespace /// diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 80f31f058..7cee0890c 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -202,6 +202,16 @@ internal static IntPtr ToPython(object value, Type type) return ClassDerivedObject.ToPython(pyderived); } + // ModuleObjects are created in a way that their wrapping them as + // a CLRObject fails, the ClassObject has no tpHandle. Return the + // pyHandle as is, do not convert. + if (value is ModuleObject modobj) + { + var handle = modobj.pyHandle; + Runtime.XIncref(handle); + return handle; + } + // hmm - from Python, we almost never care what the declared // type is. we'd rather have the object bound to the actual // implementing class. diff --git a/src/runtime/extensiontype.cs b/src/runtime/extensiontype.cs index 78df805ee..0ebd7ec4c 100644 --- a/src/runtime/extensiontype.cs +++ b/src/runtime/extensiontype.cs @@ -83,7 +83,7 @@ public static int tp_setattro(IntPtr ob, IntPtr key, IntPtr val) { message = "readonly attribute"; } - Exceptions.SetError(Exceptions.TypeError, message); + Exceptions.SetError(Exceptions.AttributeError, message); return -1; } diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index 9ac492d21..1111adc28 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -9,63 +9,45 @@ namespace Python.Runtime /// internal static class ImportHook { - private static IntPtr py_import; private static CLRModule root; - private static MethodWrapper hook; private static IntPtr py_clr_module; static BorrowedReference ClrModuleReference => new BorrowedReference(py_clr_module); - /// - /// Initialize just the __import__ hook itself. - /// - static void InitImport() - { - // We replace the built-in Python __import__ with our own: first - // look in CLR modules, then if we don't find any call the default - // Python __import__. - IntPtr builtins = Runtime.GetBuiltins(); - py_import = Runtime.PyObject_GetAttr(builtins, PyIdentifier.__import__); - PythonException.ThrowIfIsNull(py_import); - - hook = new MethodWrapper(typeof(ImportHook), "__import__", "TernaryFunc"); - int res = Runtime.PyObject_SetAttr(builtins, PyIdentifier.__import__, hook.ptr); - PythonException.ThrowIfIsNotZero(res); - - Runtime.XDecref(builtins); - } + private const string LoaderCode = @" +import importlib.abc +import sys - /// - /// Restore the __import__ hook. - /// - static void RestoreImport() - { - IntPtr builtins = Runtime.GetBuiltins(); +class DotNetLoader(importlib.abc.Loader): - IntPtr existing = Runtime.PyObject_GetAttr(builtins, PyIdentifier.__import__); - Runtime.XDecref(existing); - if (existing != hook.ptr) - { - throw new NotSupportedException("Unable to restore original __import__."); - } + @classmethod + def exec_module(klass, mod): + # This method needs to exist. + pass - int res = Runtime.PyObject_SetAttr(builtins, PyIdentifier.__import__, py_import); - PythonException.ThrowIfIsNotZero(res); - Runtime.XDecref(py_import); - py_import = IntPtr.Zero; + @classmethod + def create_module(klass, spec): + import clr + return clr._load_clr_module(spec) - hook.Release(); - hook = null; +class DotNetFinder(importlib.abc.MetaPathFinder): - Runtime.XDecref(builtins); - } + @classmethod + def find_spec(klass, fullname, paths=None, target=None): + # Don't import, we might call ourselves recursively! + if 'clr' not in sys.modules: + return None + clr = sys.modules['clr'] + if clr._available_namespaces and fullname in clr._available_namespaces: + return importlib.machinery.ModuleSpec(fullname, DotNetLoader(), is_package=True) + return None + "; + const string availableNsKey = "_available_namespaces"; /// /// Initialization performed on startup of the Python runtime. /// internal static unsafe void Initialize() { - InitImport(); - // Initialize the clr module and tell Python about it. root = new CLRModule(); @@ -80,9 +62,10 @@ internal static unsafe void Initialize() BorrowedReference dict = Runtime.PyImport_GetModuleDict(); Runtime.PyDict_SetItemString(dict, "CLR", ClrModuleReference); Runtime.PyDict_SetItemString(dict, "clr", ClrModuleReference); + SetupNamespaceTracking(); + SetupImportHook(); } - /// /// Cleanup resources upon shutdown of the Python runtime. /// @@ -93,8 +76,7 @@ internal static void Shutdown() return; } - RestoreImport(); - + TeardownNameSpaceTracking(); Runtime.XDecref(py_clr_module); py_clr_module = IntPtr.Zero; @@ -115,187 +97,133 @@ internal static void SaveRuntimeData(RuntimeDataStorage storage) internal static void RestoreRuntimeData(RuntimeDataStorage storage) { - InitImport(); storage.GetValue("py_clr_module", out py_clr_module); var rootHandle = storage.GetValue("root"); root = (CLRModule)ManagedType.GetManagedObject(rootHandle); + BorrowedReference dict = Runtime.PyImport_GetModuleDict(); + Runtime.PyDict_SetItemString(dict, "clr", ClrModuleReference); + SetupNamespaceTracking(); } - /// - /// Return the clr python module (new reference) - /// - public static unsafe NewReference GetCLRModule(BorrowedReference fromList = default) + static void SetupImportHook() { - root.InitializePreload(); - - // update the module dictionary with the contents of the root dictionary - root.LoadNames(); - BorrowedReference py_mod_dict = Runtime.PyModule_GetDict(ClrModuleReference); - using (var clr_dict = Runtime.PyObject_GenericGetDict(root.ObjectReference)) + // Create the import hook module + var import_hook_module = Runtime.PyModule_New("clr.loader"); + + // Run the python code to create the module's classes. + var builtins = Runtime.PyEval_GetBuiltins(); + var exec = Runtime.PyDict_GetItemString(builtins, "exec"); + using var args = NewReference.DangerousFromPointer(Runtime.PyTuple_New(2)); + + var codeStr = NewReference.DangerousFromPointer(Runtime.PyString_FromString(LoaderCode)); + Runtime.PyTuple_SetItem(args, 0, codeStr); + var mod_dict = Runtime.PyModule_GetDict(import_hook_module); + // reference not stolen due to overload incref'ing for us. + Runtime.PyTuple_SetItem(args, 1, mod_dict); + Runtime.PyObject_Call(exec, args, default); + // Set as a sub-module of clr. + if(Runtime.PyModule_AddObject(ClrModuleReference, "loader", import_hook_module.DangerousGetAddress()) != 0) { - Runtime.PyDict_Update(py_mod_dict, clr_dict); + Runtime.XDecref(import_hook_module.DangerousGetAddress()); + throw PythonException.ThrowLastAsClrException(); } - // find any items from the from list and get them from the root if they're not - // already in the module dictionary - if (fromList != null) - { - if (Runtime.PyTuple_Check(fromList)) - { - using var mod_dict = new PyDict(py_mod_dict); - using var from = new PyTuple(fromList); - foreach (PyObject item in from) - { - if (mod_dict.HasKey(item)) - { - continue; - } - - var s = item.AsManagedObject(typeof(string)) as string; - if (s == null) - { - continue; - } - - ManagedType attr = root.GetAttribute(s, true); - if (attr == null) - { - continue; - } - - Runtime.XIncref(attr.pyHandle); - using (var obj = new PyObject(attr.pyHandle)) - { - mod_dict.SetItem(s, obj); - } - } - } - } - Runtime.XIncref(py_clr_module); - return NewReference.DangerousFromPointer(py_clr_module); + // Finally, add the hook to the meta path + var findercls = Runtime.PyDict_GetItemString(mod_dict, "DotNetFinder"); + var finderCtorArgs = NewReference.DangerousFromPointer(Runtime.PyTuple_New(0)); + var finder_inst = Runtime.PyObject_CallObject(findercls, finderCtorArgs); + var metapath = Runtime.PySys_GetObject("meta_path"); + Runtime.PyList_Append(metapath, finder_inst); } /// - /// The actual import hook that ties Python to the managed world. + /// Sets up the tracking of loaded namespaces. This makes available to + /// Python, as a Python object, the loaded namespaces. The set of loaded + /// namespaces is used during the import to verify if we can import a + /// CLR assembly as a module or not. The set is stored on the clr module. /// - public static IntPtr __import__(IntPtr self, IntPtr argsRaw, IntPtr kw) + static void SetupNamespaceTracking() { - var args = new BorrowedReference(argsRaw); - - // Replacement for the builtin __import__. The original import - // hook is saved as this.py_import. This version handles CLR - // import and defers to the normal builtin for everything else. - - var num_args = Runtime.PyTuple_Size(args); - if (num_args < 1) + using var newset = Runtime.PySet_New(default); + foreach (var ns in AssemblyManager.GetNamespaces()) { - return Exceptions.RaiseTypeError("__import__() takes at least 1 argument (0 given)"); - } - - BorrowedReference py_mod_name = Runtime.PyTuple_GetItem(args, 0); - if (py_mod_name.IsNull || - !Runtime.IsStringType(py_mod_name)) - { - return Exceptions.RaiseTypeError("string expected"); - } - - // Check whether the import is of the form 'from x import y'. - // This determines whether we return the head or tail module. - - BorrowedReference fromList = default; - var fromlist = false; - if (num_args >= 4) - { - fromList = Runtime.PyTuple_GetItem(args, 3); - if (fromList != null && - Runtime.PyObject_IsTrue(fromList) == 1) + using var pyNs = NewReference.DangerousFromPointer(Runtime.PyString_FromString(ns)); + if (Runtime.PySet_Add(newset, pyNs) != 0) { - fromlist = true; + throw PythonException.ThrowLastAsClrException(); } - } - - string mod_name = Runtime.GetManagedString(py_mod_name); - // Check these BEFORE the built-in import runs; may as well - // do the Incref()ed return here, since we've already found - // the module. - if (mod_name == "clr") - { - NewReference clr_module = GetCLRModule(fromList); - if (!clr_module.IsNull()) + if (Runtime.PyDict_SetItemString(root.DictRef, availableNsKey, newset) != 0) { - BorrowedReference sys_modules = Runtime.PyImport_GetModuleDict(); - if (!sys_modules.IsNull) - { - Runtime.PyDict_SetItemString(sys_modules, "clr", clr_module); - } + throw PythonException.ThrowLastAsClrException(); } - return clr_module.DangerousMoveToPointerOrNull(); } - string realname = mod_name; - - // 2010-08-15: Always seemed smart to let python try first... - // This shaves off a few tenths of a second on test_module.py - // and works around a quirk where 'sys' is found by the - // LoadImplicit() deprecation logic. - // Turns out that the AssemblyManager.ResolveHandler() checks to see if any - // Assembly's FullName.ToLower().StartsWith(name.ToLower()), which makes very - // little sense to me. - IntPtr res = Runtime.PyObject_Call(py_import, args.DangerousGetAddress(), kw); - if (res != IntPtr.Zero) + } + + /// + /// Removes the set of available namespaces from the clr module. + /// + static void TeardownNameSpaceTracking() + { + // If the C# runtime isn't loaded, then there are no namespaces available + Runtime.PyDict_SetItemString(root.dict, availableNsKey, Runtime.PyNone); + } + + public static void AddNamespace(string name) + { + var pyNs = Runtime.PyString_FromString(name); + try { - // There was no error. - if (fromlist && IsLoadAll(fromList)) + var nsSet = Runtime.PyDict_GetItemString(new BorrowedReference(root.dict), availableNsKey); + if (!(nsSet.IsNull || nsSet.DangerousGetAddress() == Runtime.PyNone)) { - var mod = ManagedType.GetManagedObject(res) as ModuleObject; - mod?.LoadNames(); + if (Runtime.PySet_Add(nsSet, new BorrowedReference(pyNs)) != 0) + { + throw PythonException.ThrowLastAsClrException(); + } } - return res; } - // There was an error - if (!Exceptions.ExceptionMatches(Exceptions.ImportError)) + finally { - // and it was NOT an ImportError; bail out here. - return IntPtr.Zero; + Runtime.XDecref(pyNs); } + } - if (mod_name == string.Empty) - { - // Most likely a missing relative import. - // For example site-packages\bs4\builder\__init__.py uses it to check if a package exists: - // from . import _html5lib - // We don't support them anyway - return IntPtr.Zero; - } - // Save the exception - var originalException = PythonException.FetchCurrentRaw(); - string[] names = realname.Split('.'); + /// + /// Because we use a proxy module for the clr module, we somtimes need + /// to force the py_clr_module to sync with the actual clr module's dict. + /// + internal static void UpdateCLRModuleDict() + { + root.InitializePreload(); - // See if sys.modules for this interpreter already has the - // requested module. If so, just return the existing module. - BorrowedReference modules = Runtime.PyImport_GetModuleDict(); - BorrowedReference module = Runtime.PyDict_GetItem(modules, py_mod_name); + // update the module dictionary with the contents of the root dictionary + root.LoadNames(); + BorrowedReference py_mod_dict = Runtime.PyModule_GetDict(ClrModuleReference); + using var clr_dict = Runtime.PyObject_GenericGetDict(root.ObjectReference); - if (module != null) - { - if (fromlist) - { - if (IsLoadAll(fromList)) - { - var mod = ManagedType.GetManagedObject(module) as ModuleObject; - mod?.LoadNames(); - } - return new NewReference(module).DangerousMoveToPointer(); - } + Runtime.PyDict_Update(py_mod_dict, clr_dict); + } - module = Runtime.PyDict_GetItemString(modules, names[0]); - return new NewReference(module, canBeNull: true).DangerousMoveToPointer(); - } - Exceptions.Clear(); + /// + /// Return the clr python module (new reference) + /// + public static unsafe NewReference GetCLRModule() + { + UpdateCLRModuleDict(); + Runtime.XIncref(py_clr_module); + return NewReference.DangerousFromPointer(py_clr_module); + } - // Traverse the qualified module name to get the named module - // and place references in sys.modules as we go. Note that if + /// + /// The hook to import a CLR module into Python. Returns a new reference + /// to the module. + /// + public static ModuleObject Import(string modname) + { + // Traverse the qualified module name to get the named module. + // Note that if // we are running in interactive mode we pre-load the names in // each module, which is often useful for introspection. If we // are not interactive, we stick to just-in-time creation of @@ -304,17 +232,18 @@ public static IntPtr __import__(IntPtr self, IntPtr argsRaw, IntPtr kw) // enable preloading in a non-interactive python processing by // setting clr.preload = True - ModuleObject head = mod_name == realname ? null : root; + ModuleObject head = null; ModuleObject tail = root; root.InitializePreload(); + string[] names = modname.Split('.'); foreach (string name in names) { ManagedType mt = tail.GetAttribute(name, true); if (!(mt is ModuleObject)) { - originalException.Restore(); - return IntPtr.Zero; + Exceptions.SetError(Exceptions.ImportError, $"'{name}' Is not a ModuleObject."); + throw PythonException.ThrowLastAsClrException(); } if (head == null) { @@ -325,22 +254,9 @@ public static IntPtr __import__(IntPtr self, IntPtr argsRaw, IntPtr kw) { tail.LoadNames(); } - - // Add the module to sys.modules - Runtime.PyDict_SetItemString(modules, tail.moduleName, tail.ObjectReference); - } - - { - var mod = fromlist ? tail : head; - - if (fromlist && IsLoadAll(fromList)) - { - mod.LoadNames(); - } - - Runtime.XIncref(mod.pyHandle); - return mod.pyHandle; } + tail.IncrRefCount(); + return tail; } private static bool IsLoadAll(BorrowedReference fromList) diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index dfb6fdf55..c2614b1d8 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -20,6 +20,12 @@ internal class ModuleObject : ExtensionType internal IntPtr dict; internal BorrowedReference DictRef => new BorrowedReference(dict); protected string _namespace; + private IntPtr __all__ = IntPtr.Zero; + + // Attributes to be set on the module according to PEP302 and 451 + // by the import machinery. + static readonly HashSet settableAttributes = + new HashSet {"__spec__", "__file__", "__name__", "__path__", "__loader__", "__package__"}; public ModuleObject(string name) { @@ -47,7 +53,7 @@ public ModuleObject(string name) var dictRef = Runtime.PyObject_GenericGetDict(ObjectReference); PythonException.ThrowIfIsNull(dictRef); dict = dictRef.DangerousMoveToPointer(); - + __all__ = Runtime.PyList_New(0); using var pyname = NewReference.DangerousFromPointer(Runtime.PyString_FromString(moduleName)); using var pyfilename = NewReference.DangerousFromPointer(Runtime.PyString_FromString(filename)); using var pydocstring = NewReference.DangerousFromPointer(Runtime.PyString_FromString(docstring)); @@ -181,7 +187,23 @@ public void LoadNames() { continue; } - GetAttribute(name, true); + + if(GetAttribute(name, true) != null) + { + // if it's a valid attribute, add it to __all__ + var pyname = Runtime.PyString_FromString(name); + try + { + if (Runtime.PyList_Append(new BorrowedReference(__all__), new BorrowedReference(pyname)) != 0) + { + throw PythonException.ThrowLastAsClrException(); + } + } + finally + { + Runtime.XDecref(pyname); + } + } } } @@ -263,6 +285,13 @@ public static IntPtr tp_getattro(IntPtr ob, IntPtr key) return self.dict; } + if (name == "__all__") + { + self.LoadNames(); + Runtime.XIncref(self.__all__); + return self.__all__; + } + ManagedType attr = null; try @@ -320,6 +349,25 @@ protected override void Clear() base.Clear(); } + /// + /// Override the setattr implementation. + /// This is needed because the import mechanics need + /// to set a few attributes + /// + [ForbidPythonThreads] + public new static int tp_setattro(IntPtr ob, IntPtr key, IntPtr val) + { + var managedKey = Runtime.GetManagedString(key); + if ((settableAttributes.Contains(managedKey)) || + (ManagedType.GetManagedObject(val)?.GetType() == typeof(ModuleObject)) ) + { + var self = (ModuleObject)ManagedType.GetManagedObject(ob); + return Runtime.PyDict_SetItem(self.dict, key, val); + } + + return ExtensionType.tp_setattro(ob, key, val); + } + protected override void OnSave(InterDomainContext context) { base.OnSave(context); @@ -461,6 +509,7 @@ public static bool SuppressOverloads public static Assembly AddReference(string name) { AssemblyManager.UpdatePath(); + var origNs = AssemblyManager.GetNamespaces(); Assembly assembly = null; assembly = AssemblyManager.FindLoadedAssembly(name); if (assembly == null) @@ -479,7 +528,16 @@ public static Assembly AddReference(string name) { throw new FileNotFoundException($"Unable to find assembly '{name}'."); } - + // Classes that are not in a namespace needs an extra nudge to be found. + ImportHook.UpdateCLRModuleDict(); + + // A bit heavyhanded, but we can't use the AssemblyManager's AssemblyLoadHandler + // method because it may be called from other threads, leading to deadlocks + // if it is called while Python code is executing. + var currNs = AssemblyManager.GetNamespaces().Except(origNs); + foreach(var ns in currNs){ + ImportHook.AddNamespace(ns); + } return assembly; } @@ -526,5 +584,23 @@ public static string[] ListAssemblies(bool verbose) } return names; } + + /// + /// Note: This should *not* be called directly. + /// The function that get/import a CLR assembly as a python module. + /// This function should only be called by the import machinery as seen + /// in importhook.cs + /// + /// A ModuleSpec Python object + /// A new reference to the imported module, as a PyObject. + [ModuleFunction] + [ForbidPythonThreads] + public static ModuleObject _load_clr_module(PyObject spec) + { + ModuleObject mod = null; + using var modname = spec.GetAttr("name"); + mod = ImportHook.Import(modname.ToString()); + return mod; + } } } diff --git a/src/runtime/native/TypeOffset.cs b/src/runtime/native/TypeOffset.cs index a73a9ae43..6e6da2d93 100644 --- a/src/runtime/native/TypeOffset.cs +++ b/src/runtime/native/TypeOffset.cs @@ -160,6 +160,7 @@ static void ValidateRequiredOffsetsPresent(PropertyInfo[] offsetProperties) "getPreload", "Initialize", "ListAssemblies", + "_load_clr_module", "Release", "Reset", "set_SuppressDocs", diff --git a/src/runtime/pylist.cs b/src/runtime/pylist.cs index 039f5e313..8f346524f 100644 --- a/src/runtime/pylist.cs +++ b/src/runtime/pylist.cs @@ -132,7 +132,7 @@ public static PyList AsList(PyObject value) /// public void Append(PyObject item) { - int r = Runtime.PyList_Append(this.Reference, item.obj); + int r = Runtime.PyList_Append(this.Reference, new BorrowedReference(item.obj)); if (r < 0) { throw PythonException.ThrowLastAsClrException(); diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 537e5348f..009412ea5 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -174,7 +174,7 @@ internal static void Initialize(bool initSigs = false, ShutdownMode mode = Shutd IntPtr item = PyString_FromString(rtdir); if (PySequence_Contains(path, item) == 0) { - PyList_Append(new BorrowedReference(path), item); + PyList_Append(new BorrowedReference(path), new BorrowedReference(item)); } XDecref(item); AssemblyManager.UpdatePath(); @@ -1087,6 +1087,8 @@ internal static IntPtr PyObject_GetAttr(IntPtr pointer, IntPtr name) internal static IntPtr PyObject_Call(IntPtr pointer, IntPtr args, IntPtr kw) => Delegates.PyObject_Call(pointer, args, kw); + internal static IntPtr PyObject_Call(BorrowedReference pointer, BorrowedReference args, BorrowedReference kw) + => Delegates.PyObject_Call(pointer.DangerousGetAddress(), args.DangerousGetAddress(), kw.DangerousGetAddressOrNull()); internal static NewReference PyObject_CallObject(BorrowedReference callable, BorrowedReference args) => Delegates.PyObject_CallObject(callable, args); @@ -1796,7 +1798,7 @@ internal static int PyList_Insert(BorrowedReference pointer, long index, IntPtr private static int PyList_Insert(BorrowedReference pointer, IntPtr index, IntPtr value) => Delegates.PyList_Insert(pointer, index, value); - internal static int PyList_Append(BorrowedReference pointer, IntPtr value) => Delegates.PyList_Append(pointer, value); + internal static int PyList_Append(BorrowedReference pointer, BorrowedReference value) => Delegates.PyList_Append(pointer, value); internal static int PyList_Reverse(BorrowedReference pointer) => Delegates.PyList_Reverse(pointer); @@ -1859,7 +1861,15 @@ internal static int PyTuple_SetItem(IntPtr pointer, long index, IntPtr value) { return PyTuple_SetItem(pointer, new IntPtr(index), value); } + internal static int PyTuple_SetItem(BorrowedReference pointer, long index, StolenReference value) + => PyTuple_SetItem(pointer.DangerousGetAddress(), new IntPtr(index), value.DangerousGetAddressOrNull()); + internal static int PyTuple_SetItem(BorrowedReference pointer, long index, BorrowedReference value) + { + var increfValue = value.DangerousGetAddress(); + Runtime.XIncref(increfValue); + return PyTuple_SetItem(pointer.DangerousGetAddress(), new IntPtr(index), increfValue); + } private static int PyTuple_SetItem(IntPtr pointer, IntPtr index, IntPtr value) => Delegates.PyTuple_SetItem(pointer, index, value); @@ -1919,6 +1929,22 @@ internal static string PyModule_GetFilename(IntPtr module) internal static IntPtr PyImport_Import(IntPtr name) => Delegates.PyImport_Import(name); + /// + /// We can't use a StolenReference here because the reference is stolen only on success. + /// + /// The module to add the object to. + /// The key that will refer to the object. + /// + /// The object to add to the module. The reference will be stolen only if the + /// method returns 0. + /// + /// Return -1 on error, 0 on success. + internal static int PyModule_AddObject(BorrowedReference module, string name, IntPtr stolenObject) + { + using var namePtr = new StrPtr(name, Encoding.UTF8); + return Delegates.PyModule_AddObject(module, namePtr, stolenObject); + } + /// /// Return value: New reference. /// @@ -2451,7 +2477,7 @@ static Delegates() PyList_GetItem = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyList_GetItem), GetUnmanagedDll(_PythonDll)); PyList_SetItem = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyList_SetItem), GetUnmanagedDll(_PythonDll)); PyList_Insert = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyList_Insert), GetUnmanagedDll(_PythonDll)); - PyList_Append = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyList_Append), GetUnmanagedDll(_PythonDll)); + PyList_Append = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyList_Append), GetUnmanagedDll(_PythonDll)); PyList_Reverse = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyList_Reverse), GetUnmanagedDll(_PythonDll)); PyList_Sort = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyList_Sort), GetUnmanagedDll(_PythonDll)); PyList_GetSlice = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyList_GetSlice), GetUnmanagedDll(_PythonDll)); @@ -2475,6 +2501,7 @@ static Delegates() { PyModule_Create2 = (delegate* unmanaged[Cdecl])GetFunctionByName("PyModule_Create2TraceRefs", GetUnmanagedDll(_PythonDll)); } + PyModule_AddObject = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyModule_AddObject), GetUnmanagedDll(_PythonDll)); PyImport_Import = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyImport_Import), GetUnmanagedDll(_PythonDll)); PyImport_ImportModule = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyImport_ImportModule), GetUnmanagedDll(_PythonDll)); PyImport_ReloadModule = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyImport_ReloadModule), GetUnmanagedDll(_PythonDll)); @@ -2746,7 +2773,7 @@ static Delegates() internal static delegate* unmanaged[Cdecl] PyList_GetItem { get; } internal static delegate* unmanaged[Cdecl] PyList_SetItem { get; } internal static delegate* unmanaged[Cdecl] PyList_Insert { get; } - internal static delegate* unmanaged[Cdecl] PyList_Append { get; } + internal static delegate* unmanaged[Cdecl] PyList_Append { get; } internal static delegate* unmanaged[Cdecl] PyList_Reverse { get; } internal static delegate* unmanaged[Cdecl] PyList_Sort { get; } internal static delegate* unmanaged[Cdecl] PyList_GetSlice { get; } @@ -2763,6 +2790,7 @@ static Delegates() internal static delegate* unmanaged[Cdecl] PyModule_GetDict { get; } internal static delegate* unmanaged[Cdecl] PyModule_GetFilename { get; } internal static delegate* unmanaged[Cdecl] PyModule_Create2 { get; } + internal static delegate* unmanaged[Cdecl] PyModule_AddObject { get; } internal static delegate* unmanaged[Cdecl] PyImport_Import { get; } internal static delegate* unmanaged[Cdecl] PyImport_ImportModule { get; } internal static delegate* unmanaged[Cdecl] PyImport_ReloadModule { get; } diff --git a/tests/domain_tests/TestRunner.cs b/tests/domain_tests/TestRunner.cs index a21297829..66fb4f894 100644 --- a/tests/domain_tests/TestRunner.cs +++ b/tests/domain_tests/TestRunner.cs @@ -30,6 +30,11 @@ namespace Python.DomainReloadTests /// which test case to run. That's because pytest assumes we'll run /// everything in one process, but we really want a clean process on each /// test case to test the init/reload/teardown parts of the domain reload. + /// + /// ### Debugging tips: ### + /// * Running pytest with the `-s` argument prevents stdout capture by pytest + /// * Add a sleep into the python test case before the crash/failure, then while + /// sleeping, attach the debugger to the Python.TestDomainReload.exe process. /// /// class TestRunner @@ -1092,6 +1097,29 @@ assert sys.my_obj is not None foo = sys.my_obj.Inner() print(foo) + ", + }, + new TestCase + { + // The C# code for this test doesn't matter; we're testing + // that the import hook behaves properly after a domain reload + Name = "import_after_reload", + DotNetBefore = "", + DotNetAfter = "", + PythonCode = @" +import sys + +def before_reload(): + import clr + import System + + +def after_reload(): + assert 'System' in sys.modules + assert 'clr' in sys.modules + import clr + import System + ", }, }; @@ -1264,7 +1292,13 @@ static string CreateAssembly(string name, string code, bool exe = false) } parameters.ReferencedAssemblies.Add(netstandard); parameters.ReferencedAssemblies.Add(PythonDllLocation); - CompilerResults results = provider.CompileAssemblyFromSource(parameters, code); + // Write code to file so it can debugged. + var sourcePath = Path.Combine(TestPath, name+"_source.cs"); + using(var file = new StreamWriter(sourcePath)) + { + file.Write(code); + } + CompilerResults results = provider.CompileAssemblyFromFile(parameters, sourcePath); if (results.NativeCompilerReturnValue != 0) { var stderr = System.Console.Error; diff --git a/tests/domain_tests/test_domain_reload.py b/tests/domain_tests/test_domain_reload.py index a7cd2fa4d..e7a82ded2 100644 --- a/tests/domain_tests/test_domain_reload.py +++ b/tests/domain_tests/test_domain_reload.py @@ -88,3 +88,6 @@ def test_in_to_ref_param(): def test_nested_type(): _run_test("nested_type") + +def test_import_after_reload(): + _run_test("import_after_reload") \ No newline at end of file pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy