diff --git a/AUTHORS.md b/AUTHORS.md index 26285bf6a..3ba87c02a 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -29,6 +29,7 @@ - David Lassonde ([@lassond](https://github.com/lassond)) - David Lechner ([@dlech](https://github.com/dlech)) - Dmitriy Se ([@dmitriyse](https://github.com/dmitriyse)) +- Félix Bourbonnais ([@BadSingleton](https://github.com/BadSingleton)) - Florian Treurniet ([@ftreurni](https://github.com/ftreurni)) - He-chien Tsai ([@t3476](https://github.com/t3476)) - Inna Wiesel ([@inna-w](https://github.com/inna-w)) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1fd2b1dcf..dc644de33 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. - Added support for converting python iterators to C# arrays - Changed usage of obselete function GetDelegateForFunctionPointer(IntPtr, Type) to GetDelegateForFunctionPointer(IntPtr) - Added support for kwarg parameters when calling .NET methods from Python +- Added cleanup logic for the managed types on shutdown. ### Fixed diff --git a/src/runtime/interop.cs b/src/runtime/interop.cs index ca3c35bfd..26d4b60d8 100644 --- a/src/runtime/interop.cs +++ b/src/runtime/interop.cs @@ -509,7 +509,7 @@ internal static ThunkInfo GetThunk(MethodInfo method, string funcType = null) } - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] + [StructLayout(LayoutKind.Sequential)] internal struct Thunk { public Delegate fn; @@ -537,4 +537,13 @@ public ThunkInfo(Delegate target) Address = Marshal.GetFunctionPointerForDelegate(target); } } + + [StructLayout(LayoutKind.Sequential)] + struct PyMethodDef + { + public IntPtr ml_name; + public IntPtr ml_meth; + public int ml_flags; + public IntPtr ml_doc; + } } diff --git a/src/runtime/metatype.cs b/src/runtime/metatype.cs index 5af2e1a7e..519c3994b 100644 --- a/src/runtime/metatype.cs +++ b/src/runtime/metatype.cs @@ -11,17 +11,30 @@ namespace Python.Runtime internal class MetaType : ManagedType { private static IntPtr PyCLRMetaType; - + private static SlotsHolder _metaSlotsHodler; /// /// Metatype initialization. This bootstraps the CLR metatype to life. /// public static IntPtr Initialize() { - PyCLRMetaType = TypeManager.CreateMetaType(typeof(MetaType)); + PyCLRMetaType = TypeManager.CreateMetaType(typeof(MetaType), out _metaSlotsHodler); return PyCLRMetaType; } + public static void Release() + { + if (PyCLRMetaType == IntPtr.Zero) + { + throw new ObjectDisposedException("PyCLRMetaType"); + } + if (Runtime.Refcount(PyCLRMetaType) > 1) + { + _metaSlotsHodler.ResetSlots(); + } + Runtime.Py_CLEAR(ref PyCLRMetaType); + _metaSlotsHodler = null; + } /// /// Metatype __new__ implementation. This is called to create a new diff --git a/src/runtime/methodwrapper.cs b/src/runtime/methodwrapper.cs index bc7500dab..9c235c5f5 100644 --- a/src/runtime/methodwrapper.cs +++ b/src/runtime/methodwrapper.cs @@ -13,7 +13,6 @@ internal class MethodWrapper public IntPtr mdef; public IntPtr ptr; private bool _disposed = false; - private ThunkInfo _thunk; public MethodWrapper(Type type, string name, string funcType = null) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 130d90c0a..ed06f40b0 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -211,6 +211,7 @@ internal static void Initialize(bool initSigs = false) PyNone = PyObject_GetAttrString(op, "None"); PyTrue = PyObject_GetAttrString(op, "True"); PyFalse = PyObject_GetAttrString(op, "False"); + PySuper_Type = PyObject_GetAttrString(op, "super"); PyBoolType = PyObject_Type(PyTrue); PyNoneType = PyObject_Type(PyNone); @@ -380,6 +381,11 @@ internal static void Shutdown() Exceptions.Shutdown(); ImportHook.Shutdown(); Finalizer.Shutdown(); + + TypeManager.RemoveTypes(); + MetaType.Release(); + PyCLRMetaType = IntPtr.Zero; + Py_Finalize(); } @@ -401,6 +407,7 @@ internal static int AtExit() internal static IntPtr PyModuleType; internal static IntPtr PyClassType; internal static IntPtr PyInstanceType; + internal static IntPtr PySuper_Type; internal static IntPtr PyCLRMetaType; internal static IntPtr PyMethodType; internal static IntPtr PyWrapperDescriptorType; @@ -1905,6 +1912,22 @@ internal static IntPtr PyMem_Realloc(IntPtr ptr, long size) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void PyErr_Print(); + internal static void Py_CLEAR(ref IntPtr ob) + { + XDecref(ob); + ob = IntPtr.Zero; + } + + //==================================================================== + // Python Capsules API + //==================================================================== + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr PyCapsule_New(IntPtr pointer, string name, IntPtr destructor); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr PyCapsule_GetPointer(IntPtr capsule, string name); + //==================================================================== // Miscellaneous diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index bb920b74f..f677c4c79 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -1,6 +1,7 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Reflection; using System.Runtime.InteropServices; @@ -16,18 +17,38 @@ namespace Python.Runtime /// internal class TypeManager { - private static BindingFlags tbFlags; - private static Dictionary cache; + private const BindingFlags tbFlags = BindingFlags.Public | BindingFlags.Static; + private static readonly Dictionary cache = new Dictionary(); + private static readonly Dictionary _slotsHolders = new Dictionary(); static TypeManager() { - tbFlags = BindingFlags.Public | BindingFlags.Static; - cache = new Dictionary(128); + } public static void Reset() { - cache = new Dictionary(128); + cache.Clear(); + } + + internal static void RemoveTypes() + { + foreach (var tpHandle in cache.Values) + { + SlotsHolder holder; + if (_slotsHolders.TryGetValue(tpHandle, out holder)) + { + // If refcount > 1, it needs to reset the managed slot, + // otherwise it can dealloc without any trick. + if (Runtime.Refcount(tpHandle) > 1) + { + holder.ResetSlots(); + } + } + Runtime.XDecref(tpHandle); + } + cache.Clear(); + _slotsHolders.Clear(); } /// @@ -79,6 +100,7 @@ internal static IntPtr GetTypeHandle(ManagedType obj, Type type) /// behavior needed and the desire to have the existing Python runtime /// do as much of the allocation and initialization work as possible. /// + /// Return value: Borrowed reference. internal static IntPtr CreateType(Type impl) { IntPtr type = AllocateTypeObject(impl.Name); @@ -90,20 +112,23 @@ internal static IntPtr CreateType(Type impl) var offset = (IntPtr)ObjectOffset.DictOffset(type); Marshal.WriteIntPtr(type, TypeOffset.tp_dictoffset, offset); - InitializeSlots(type, impl); + SlotsHolder slotsHolder = CreateSlotsHolder(type); + InitializeSlots(type, impl, slotsHolder); int flags = TypeFlags.Default | TypeFlags.Managed | TypeFlags.HeapType | TypeFlags.HaveGC; Util.WriteCLong(type, TypeOffset.tp_flags, flags); - Runtime.PyType_Ready(type); - + if (Runtime.PyType_Ready(type) != 0) + { + throw new PythonException(); + } IntPtr dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); IntPtr mod = Runtime.PyString_FromString("CLR"); Runtime.PyDict_SetItemString(dict, "__module__", mod); + Runtime.XDecref(mod); InitMethods(type, impl); - return type; } @@ -155,15 +180,18 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) Marshal.WriteIntPtr(type, TypeOffset.tp_itemsize, IntPtr.Zero); Marshal.WriteIntPtr(type, TypeOffset.tp_dictoffset, (IntPtr)tp_dictoffset); + // we want to do this after the slot stuff above in case the class itself implements a slot method + SlotsHolder slotsHolder = CreateSlotsHolder(type); + InitializeSlots(type, impl.GetType(), slotsHolder); + // add a __len__ slot for inheritors of ICollection and ICollection<> if (typeof(ICollection).IsAssignableFrom(clrType) || clrType.GetInterfaces().Any(x => x.IsGenericType && x.GetGenericTypeDefinition() == typeof(ICollection<>))) { - InitializeSlot(type, TypeOffset.mp_length, typeof(mp_length_slot).GetMethod(nameof(mp_length_slot.mp_length))); + var method = typeof(mp_length_slot).GetMethod(nameof(mp_length_slot.mp_length)); + var thunk = Interop.GetThunk(method); + InitializeSlot(type, thunk, "__len__", slotsHolder); } - // we want to do this after the slot stuff above in case the class itself implements a slot method - InitializeSlots(type, impl.GetType()); - if (base_ != IntPtr.Zero) { Marshal.WriteIntPtr(type, TypeOffset.tp_base, base_); @@ -181,12 +209,16 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) // that the type of the new type must PyType_Type at the time we // call this, else PyType_Ready will skip some slot initialization. - Runtime.PyType_Ready(type); + if (Runtime.PyType_Ready(type) != 0) + { + throw new PythonException(); + } IntPtr dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); string mn = clrType.Namespace ?? ""; IntPtr mod = Runtime.PyString_FromString(mn); Runtime.PyDict_SetItemString(dict, "__module__", mod); + Runtime.XDecref(mod); // Hide the gchandle of the implementation in a magic type slot. GCHandle gc = GCHandle.Alloc(impl); @@ -202,12 +234,6 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) return type; } - static void InitializeSlot(IntPtr type, int slotOffset, MethodInfo method) - { - var thunk = Interop.GetThunk(method); - Marshal.WriteIntPtr(type, slotOffset, thunk.Address); - } - internal static IntPtr CreateSubType(IntPtr py_name, IntPtr py_base_type, IntPtr py_dict) { // Utility to create a subtype of a managed type with the ability for the @@ -311,7 +337,47 @@ internal static IntPtr WriteMethodDefSentinel(IntPtr mdef) return WriteMethodDef(mdef, IntPtr.Zero, IntPtr.Zero, 0, IntPtr.Zero); } - internal static IntPtr CreateMetaType(Type impl) + internal static void FreeMethodDef(IntPtr mdef) + { + unsafe + { + var def = (PyMethodDef*)mdef; + if (def->ml_name != IntPtr.Zero) + { + Marshal.FreeHGlobal(def->ml_name); + def->ml_name = IntPtr.Zero; + } + if (def->ml_doc != IntPtr.Zero) + { + Marshal.FreeHGlobal(def->ml_doc); + def->ml_doc = IntPtr.Zero; + } + } + } + + /// + /// Adds a deallocator for a type's method. At deallocation, the deallocator will remove the + /// method from the type's Dict and deallocate the PyMethodDef object. + /// + /// The type to add the deallocator to. + /// The pointer to the PyMethodDef structure. + /// The name of the slot. + /// The SlotsHolder holding the deallocator/. + internal static void AddDeallocator(IntPtr t, IntPtr mdef, string name, SlotsHolder slotsHolder) + { + slotsHolder.AddDealloctor(() => + { + //IntPtr t = type; + IntPtr tp_dict = Marshal.ReadIntPtr(t, TypeOffset.tp_dict); + if (Runtime.PyDict_DelItemString(tp_dict, name) != 0) + { + Runtime.PyErr_Print(); + } + FreeMethodDef(mdef); + }); + } + + internal static IntPtr CreateMetaType(Type impl, out SlotsHolder slotsHolder) { // The managed metatype is functionally little different than the // standard Python metatype (PyType_Type). It overrides certain of @@ -331,8 +397,8 @@ internal static IntPtr CreateMetaType(Type impl) // tp_traverse, tp_clear, tp_is_gc, etc. // Override type slots with those of the managed implementation. - - InitializeSlots(type, impl); + slotsHolder = new SlotsHolder(type); + InitializeSlots(type, impl, slotsHolder); int flags = TypeFlags.Default; flags |= TypeFlags.Managed; @@ -343,27 +409,44 @@ internal static IntPtr CreateMetaType(Type impl) // We need space for 3 PyMethodDef structs, each of them // 4 int-ptrs in size. IntPtr mdef = Runtime.PyMem_Malloc(3 * 4 * IntPtr.Size); + Debug.Assert(4 * IntPtr.Size == Marshal.SizeOf(typeof(PyMethodDef))); IntPtr mdefStart = mdef; - ThunkInfo thunkInfo = Interop.GetThunk(typeof(MetaType).GetMethod("__instancecheck__"), "BinaryFunc"); + ThunkInfo thunk = Interop.GetThunk(typeof(MetaType).GetMethod("__instancecheck__"), "BinaryFunc"); + slotsHolder.KeepAlive(thunk.Target); + // Add deallocator before writing the method def, as after WriteMethodDef, mdef + // will not have the same value. + AddDeallocator(type, mdef, "__instancecheck__", slotsHolder); mdef = WriteMethodDef( mdef, "__instancecheck__", - thunkInfo.Address + thunk.Address ); - thunkInfo = Interop.GetThunk(typeof(MetaType).GetMethod("__subclasscheck__"), "BinaryFunc"); + thunk = Interop.GetThunk(typeof(MetaType).GetMethod("__subclasscheck__"), "BinaryFunc"); + slotsHolder.KeepAlive(thunk.Target); + AddDeallocator(type, mdef, "__subclasscheck__", slotsHolder); + mdef = WriteMethodDef( mdef, "__subclasscheck__", - thunkInfo.Address + thunk.Address ); - // FIXME: mdef is not used + // Pad the last field with zeroes to terminate the array mdef = WriteMethodDefSentinel(mdef); Marshal.WriteIntPtr(type, TypeOffset.tp_methods, mdefStart); + slotsHolder.Set(TypeOffset.tp_methods, (t, offset) => + { + var p = Marshal.ReadIntPtr(t, offset); + Runtime.PyMem_Free(p); + Marshal.WriteIntPtr(t, offset, IntPtr.Zero); + }); - Runtime.PyType_Ready(type); + if (Runtime.PyType_Ready(type) != 0) + { + throw new PythonException(); + } IntPtr dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); IntPtr mod = Runtime.PyString_FromString("CLR"); @@ -403,9 +486,13 @@ internal static IntPtr BasicSubType(string name, IntPtr base_, Type impl) CopySlot(base_, type, TypeOffset.tp_clear); CopySlot(base_, type, TypeOffset.tp_is_gc); - InitializeSlots(type, impl); + SlotsHolder slotsHolder = CreateSlotsHolder(type); + InitializeSlots(type, impl, slotsHolder); - Runtime.PyType_Ready(type); + if (Runtime.PyType_Ready(type) != 0) + { + throw new PythonException(); + } IntPtr tp_dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict); IntPtr mod = Runtime.PyString_FromString("CLR"); @@ -684,7 +771,7 @@ internal static void InitializeNativeCodePage() /// provides the implementation for the type, connect the type slots of /// the Python object to the managed methods of the implementing Type. /// - internal static void InitializeSlots(IntPtr type, Type impl) + internal static void InitializeSlots(IntPtr type, Type impl, SlotsHolder slotsHolder = null) { // We work from the most-derived class up; make sure to get // the most-derived slot and not to override it with a base @@ -731,9 +818,8 @@ internal static void InitializeSlots(IntPtr type, Type impl) // These have to be defined, though, so by default we fill these with // static C# functions from this class. - var ret0 = Interop.GetThunk(((Func)Return0).Method).Address; - var ret1 = Interop.GetThunk(((Func)Return1).Method).Address; - + IntPtr ret0 = IntPtr.Zero; + IntPtr ret1 = IntPtr.Zero; if (native != null) { // If we want to support domain reload, the C# implementation @@ -745,6 +831,11 @@ internal static void InitializeSlots(IntPtr type, Type impl) ret1 = NativeCodePage + native.Return1; ret0 = NativeCodePage + native.Return0; } + else + { + ret1 = Interop.GetThunk(((Func)Return1).Method).Address; + ret0 = Interop.GetThunk(((Func)Return0).Method).Address; + } InitializeSlot(type, ret0, "tp_traverse"); InitializeSlot(type, ret0, "tp_clear"); @@ -764,14 +855,46 @@ internal static void InitializeSlots(IntPtr type, Type impl) /// /// Type being initialized. /// Function pointer. - /// Name of the method. + /// Name of the slot to initialize static void InitializeSlot(IntPtr type, IntPtr slot, string name) + { + var offset = GetSlotOffset(name); + if (Marshal.ReadIntPtr(type, offset) != IntPtr.Zero) + { + return; + } + Marshal.WriteIntPtr(type, offset, slot); + } + + static void InitializeSlot(IntPtr type, ThunkInfo thunk, string name, SlotsHolder slotsHolder = null, bool canOverride = true) { Type typeOffset = typeof(TypeOffset); FieldInfo fi = typeOffset.GetField(name); var offset = (int)fi.GetValue(typeOffset); - Marshal.WriteIntPtr(type, offset, slot); + if (!canOverride && Marshal.ReadIntPtr(type, offset) != IntPtr.Zero) + { + return; + } + Marshal.WriteIntPtr(type, offset, thunk.Address); + if (slotsHolder != null) + { + slotsHolder.Add(offset, thunk); + } + } + + static int GetSlotOffset(string name) + { + Type typeOffset = typeof(TypeOffset); + FieldInfo fi = typeOffset.GetField(name); + var offset = (int)fi.GetValue(typeOffset); + return offset; + } + + static bool IsSlotSet(IntPtr type, string name) + { + int offset = GetSlotOffset(name); + return Marshal.ReadIntPtr(type, offset) != IntPtr.Zero; } /// @@ -819,5 +942,182 @@ internal static void CopySlot(IntPtr from, IntPtr to, int offset) IntPtr fp = Marshal.ReadIntPtr(from, offset); Marshal.WriteIntPtr(to, offset, fp); } + + private static SlotsHolder CreateSlotsHolder(IntPtr type) + { + var holder = new SlotsHolder(type); + _slotsHolders.Add(type, holder); + return holder; + } + } + + class SlotsHolder + { + /// + /// Delegate called to customize a (Python) Type slot reset + /// + /// The type that will have a slot reset + /// The offset of the slot + public delegate void ResetSlotAction(IntPtr type, int offset); + + private readonly IntPtr type; + private Dictionary slots = new Dictionary(); + private List keepalive = new List(); + private Dictionary customResetors = new Dictionary(); + private List deallocators = new List(); + private bool alreadyReset = false; + + /// + /// Create slots holder for holding the delegate of slots and be able to reset them. + /// + /// Steals a reference to target type + public SlotsHolder(IntPtr type) + { + this.type = type; + } + + public void Add(int offset, ThunkInfo thunk) + { + slots.Add(offset, thunk); + } + + public void Set(int offset, ResetSlotAction resetor) + { + customResetors[offset] = resetor; + } + + public void AddDealloctor(Action deallocate) + { + deallocators.Add(deallocate); + } + + /// + /// Add a delegate to keep it from being garbage collected. + /// + /// The delegate to add + public void KeepAlive(Delegate d) + { + keepalive.Add(d); + } + + public void ResetSlots() + { + if (alreadyReset) + { + return; + } + alreadyReset = true; + foreach (var offset in slots.Keys) + { + IntPtr ptr = GetDefaultSlot(offset); + //DebugUtil.Print($"Set slot<{TypeOffsetHelper.GetSlotNameByOffset(offset)}> to 0x{ptr.ToString("X")} at {typeName}<0x{_type}>"); + Marshal.WriteIntPtr(type, offset, ptr); + } + + foreach (var action in deallocators) + { + action(); + } + + foreach (var pair in customResetors) + { + int offset = pair.Key; + var resetor = pair.Value; + resetor?.Invoke(type, offset); + } + + customResetors.Clear(); + slots.Clear(); + keepalive.Clear(); + deallocators.Clear(); + + // Custom reset + IntPtr tp_base = Marshal.ReadIntPtr(type, TypeOffset.tp_base); + Runtime.XDecref(tp_base); + Marshal.WriteIntPtr(type, TypeOffset.tp_base, IntPtr.Zero); + + IntPtr tp_bases = Marshal.ReadIntPtr(type, TypeOffset.tp_bases); + Runtime.XDecref(tp_bases); + tp_bases = Runtime.PyTuple_New(0); + Marshal.WriteIntPtr(type, TypeOffset.tp_bases, tp_bases); + } + + /// + /// Returns the default C function pointer for the slot to reset. + /// + /// The offset of the slot. + /// The default C function pointer of the slot. + private static IntPtr GetDefaultSlot(int offset) + { + if (offset == TypeOffset.tp_clear + || offset == TypeOffset.tp_traverse) + { + return TypeManager.NativeCodePage + TypeManager.NativeCode.Active.Return0; + } + else if (offset == TypeOffset.tp_dealloc) + { + // tp_free of PyTypeType is point to PyObject_GC_Del. + return Marshal.ReadIntPtr(Runtime.PyTypeType, TypeOffset.tp_free); + } + else if (offset == TypeOffset.tp_free) + { + // PyObject_GC_Del + return Marshal.ReadIntPtr(Runtime.PyTypeType, TypeOffset.tp_free); + } + else if (offset == TypeOffset.tp_call) + { + return IntPtr.Zero; + } + else if (offset == TypeOffset.tp_new) + { + // PyType_GenericNew + return Marshal.ReadIntPtr(Runtime.PySuper_Type, TypeOffset.tp_new); + } + else if (offset == TypeOffset.tp_getattro) + { + // PyObject_GenericGetAttr + return Marshal.ReadIntPtr(Runtime.PyBaseObjectType, TypeOffset.tp_getattro); + } + else if (offset == TypeOffset.tp_setattro) + { + // PyObject_GenericSetAttr + return Marshal.ReadIntPtr(Runtime.PyBaseObjectType, TypeOffset.tp_setattro); + } + + return Marshal.ReadIntPtr(Runtime.PyTypeType, offset); + } + } + + + static class SlotHelper + { + public static IntPtr CreateObjectType() + { + IntPtr globals = Runtime.PyDict_New(); + if (Runtime.PyDict_SetItemString(globals, "__builtins__", Runtime.PyEval_GetBuiltins()) != 0) + { + Runtime.XDecref(globals); + throw new PythonException(); + } + const string code = "class A(object): pass"; + IntPtr res = Runtime.PyRun_String(code, (IntPtr)RunFlagType.File, globals, globals); + if (res == IntPtr.Zero) + { + try + { + throw new PythonException(); + } + finally + { + Runtime.XDecref(globals); + } + } + Runtime.XDecref(res); + IntPtr A = Runtime.PyDict_GetItemString(globals, "A"); + Debug.Assert(A != IntPtr.Zero); + Runtime.XIncref(A); + Runtime.XDecref(globals); + return A; + } } } pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy