diff --git a/pythonnet/__init__.py b/pythonnet/__init__.py index 692fd5700..4f162dcd5 100644 --- a/pythonnet/__init__.py +++ b/pythonnet/__init__.py @@ -63,7 +63,7 @@ def unload(): global _RUNTIME if _LOADER_ASSEMBLY is not None: func = _LOADER_ASSEMBLY["Python.Runtime.Loader.Shutdown"] - if func(b"") != 0: + if func(b"full_shutdown") != 0: raise RuntimeError("Failed to call Python.NET shutdown") if _RUNTIME is not None: diff --git a/src/embed_tests/Python.EmbeddingTest.csproj b/src/embed_tests/Python.EmbeddingTest.csproj index 67a7d3338..099d026fa 100644 --- a/src/embed_tests/Python.EmbeddingTest.csproj +++ b/src/embed_tests/Python.EmbeddingTest.csproj @@ -2,6 +2,7 @@ net472;netcoreapp3.1 + 9.0 ..\pythonnet.snk true diff --git a/src/embed_tests/TestClass.cs b/src/embed_tests/TestClass.cs new file mode 100644 index 000000000..cbcb92483 --- /dev/null +++ b/src/embed_tests/TestClass.cs @@ -0,0 +1,78 @@ +using System; +using System.Runtime.InteropServices; + +using NUnit.Framework; + +using Python.Runtime; + +using PyRuntime = Python.Runtime.Runtime; + +namespace Python.EmbeddingTest +{ + public class TestClass + { + public class MyClass + { + } + + [OneTimeSetUp] + public void SetUp() + { + PythonEngine.Initialize(); + } + + [OneTimeTearDown] + public void Dispose() + { + PythonEngine.Shutdown(); + } + + [Test] + public void WeakRefForClrObject() + { + var obj = new MyClass(); + using var scope = Py.CreateScope(); + scope.Set("clr_obj", obj); + scope.Exec(@" +import weakref +ref = weakref.ref(clr_obj) +"); + using PyObject pyobj = scope.Get("clr_obj"); + ValidateAttachedGCHandle(obj, pyobj.Handle); + } + + [Test] + public void WeakRefForSubClass() + { + using (var scope = Py.CreateScope()) + { + scope.Exec(@" +from Python.EmbeddingTest import TestClass +import weakref + +class Sub(TestClass.MyClass): + pass + +obj = Sub() +ref = weakref.ref(obj) +"); + using (PyObject pyobj = scope.Get("obj")) + { + IntPtr op = pyobj.Handle; + IntPtr type = PyRuntime.PyObject_TYPE(op); + IntPtr clrHandle = Marshal.ReadIntPtr(op, ObjectOffset.magic(type)); + var clobj = (CLRObject)GCHandle.FromIntPtr(clrHandle).Target; + Assert.IsTrue(clobj.inst is MyClass); + } + } + } + + private static void ValidateAttachedGCHandle(object obj, IntPtr op) + { + IntPtr type = PyRuntime.PyObject_TYPE(op); + IntPtr clrHandle = Marshal.ReadIntPtr(op, ObjectOffset.magic(type)); + var clobj = (CLRObject)GCHandle.FromIntPtr(clrHandle).Target; + Assert.True(ReferenceEquals(clobj.inst, obj)); + } + } +} diff --git a/src/runtime/classbase.cs b/src/runtime/classbase.cs index 8b96a96da..a29e3a9b2 100644 --- a/src/runtime/classbase.cs +++ b/src/runtime/classbase.cs @@ -1,9 +1,7 @@ using System; using System.Collections; using System.Collections.Generic; -using System.Diagnostics; using System.Runtime.InteropServices; -using System.Runtime.Serialization; namespace Python.Runtime { @@ -354,44 +352,39 @@ public static IntPtr tp_repr(IntPtr ob) public static void tp_dealloc(IntPtr ob) { ManagedType self = GetManagedObject(ob); - tp_clear(ob); - Runtime.PyObject_GC_UnTrack(self.pyHandle); - Runtime.PyObject_GC_Del(self.pyHandle); + if (Runtime.PyType_SUPPORTS_WEAKREFS(Runtime.PyObject_TYPE(ob))) + { + Runtime.PyObject_ClearWeakRefs(ob); + } + RemoveObjectDict(ob); + Runtime.Py_CLEAR(ref self.tpHandle); + Runtime.PyObject_GC_UnTrack(ob); + Runtime.PyObject_GC_Del(ob); self.FreeGCHandle(); } public static int tp_clear(IntPtr ob) { - ManagedType self = GetManagedObject(ob); - if (!self.IsTypeObject()) - { - ClearObjectDict(ob); - } - self.tpHandle = IntPtr.Zero; + ClearObjectDict(ob); return 0; } protected override void OnSave(InterDomainContext context) { base.OnSave(context); - if (pyHandle != tpHandle) - { - IntPtr dict = GetObjectDict(pyHandle); - Runtime.XIncref(dict); - context.Storage.AddValue("dict", dict); - } + IntPtr dict = GetObjectDict(pyHandle); + Runtime.XIncref(dict); + Runtime.XIncref(tpHandle); + context.Storage.AddValue("dict", dict); } protected override void OnLoad(InterDomainContext context) { base.OnLoad(context); - if (pyHandle != tpHandle) - { - IntPtr dict = context.Storage.GetValue("dict"); - SetObjectDict(pyHandle, dict); - } + IntPtr dict = context.Storage.GetValue("dict"); + SetObjectDict(pyHandle, dict); gcHandle = AllocGCHandle(); - Marshal.WriteIntPtr(pyHandle, TypeOffset.magic(), (IntPtr)gcHandle); + Marshal.WriteIntPtr(pyHandle, ObjectOffset.magic(tpHandle), (IntPtr)gcHandle); } diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index 1ee06e682..5cad4bd69 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -5,6 +5,7 @@ using System.Runtime.InteropServices; using System.Security; using System.Linq; +using System.Diagnostics; namespace Python.Runtime { @@ -270,7 +271,7 @@ private static void InitClassBase(Type type, ClassBase impl) // Finally, initialize the class __dict__ and return the object. var dict = new BorrowedReference(Marshal.ReadIntPtr(tp, TypeOffset.tp_dict)); - + Debug.Assert(!dict.IsNull); if (impl.dotNetMembers == null) { diff --git a/src/runtime/clrobject.cs b/src/runtime/clrobject.cs index 0a352b381..9c288b145 100644 --- a/src/runtime/clrobject.cs +++ b/src/runtime/clrobject.cs @@ -30,10 +30,6 @@ internal CLRObject(object ob, IntPtr tp) tpHandle = tp; pyHandle = py; inst = ob; - - // Fix the BaseException args (and __cause__ in case of Python 3) - // slot if wrapping a CLR exception - Exceptions.SetArgsAndCause(py); } protected CLRObject() @@ -49,7 +45,7 @@ static CLRObject GetInstance(object ob, IntPtr pyType) static CLRObject GetInstance(object ob) { ClassBase cc = ClassManager.GetClass(ob.GetType()); - return GetInstance(ob, cc.tpHandle); + return GetInstance(ob, cc.pyHandle); } internal static NewReference GetInstHandle(object ob, BorrowedReference pyType) @@ -67,7 +63,7 @@ internal static IntPtr GetInstHandle(object ob, IntPtr pyType) internal static IntPtr GetInstHandle(object ob, Type type) { ClassBase cc = ClassManager.GetClass(type); - CLRObject co = GetInstance(ob, cc.tpHandle); + CLRObject co = GetInstance(ob, cc.pyHandle); return co.pyHandle; } diff --git a/src/runtime/exceptions.cs b/src/runtime/exceptions.cs index afd0bc14e..51c7729e3 100644 --- a/src/runtime/exceptions.cs +++ b/src/runtime/exceptions.cs @@ -85,6 +85,13 @@ internal static Exception ToException(IntPtr ob) } return Runtime.PyUnicode_FromString(message); } + + public static int tp_init(IntPtr ob, IntPtr args, IntPtr kwds) + { + Exceptions.SetArgsAndCause(ob); + return 0; + } + } ///

@@ -180,16 +187,23 @@ internal static void SetArgsAndCause(IntPtr ob) args = Runtime.PyTuple_New(0); } - Marshal.WriteIntPtr(ob, ExceptionOffset.args, args); - + int baseOffset = OriginalObjectOffsets.Size; + Runtime.Py_SETREF(ob, baseOffset + ExceptionOffset.args, args); + if (e.InnerException != null) { - // Note: For an AggregateException, InnerException is only the first of the InnerExceptions. - IntPtr cause = CLRObject.GetInstHandle(e.InnerException); - Marshal.WriteIntPtr(ob, ExceptionOffset.cause, cause); + IntPtr cause = GetExceptHandle(e.InnerException); + Runtime.Py_SETREF(ob, baseOffset + ExceptionOffset.cause, cause); } } + internal static IntPtr GetExceptHandle(Exception e) + { + IntPtr op = CLRObject.GetInstHandle(e); + SetArgsAndCause(op); + return op; + } + ///

/// Shortcut for (pointer == NULL) -> throw PythonException ///

@@ -289,7 +303,7 @@ public static void SetError(Exception e) return; } - IntPtr op = CLRObject.GetInstHandle(e); + IntPtr op = GetExceptHandle(e); IntPtr etype = Runtime.PyObject_GetAttr(op, PyIdentifier.__class__); Runtime.PyErr_SetObject(new BorrowedReference(etype), new BorrowedReference(op)); Runtime.XDecref(etype); diff --git a/src/runtime/extensiontype.cs b/src/runtime/extensiontype.cs index a5f0f1219..e6c95ecaa 100644 --- a/src/runtime/extensiontype.cs +++ b/src/runtime/extensiontype.cs @@ -55,7 +55,7 @@ void SetupGc () ///

public static void FinalizeObject(ManagedType self) { - ClearObjectDict(self.pyHandle); + RemoveObjectDict(self.pyHandle); Runtime.PyObject_GC_Del(self.pyHandle); // Not necessary for decref of `tpHandle`. self.FreeGCHandle(); diff --git a/src/runtime/interop.cs b/src/runtime/interop.cs index 0f56f77d9..970ecb20d 100644 --- a/src/runtime/interop.cs +++ b/src/runtime/interop.cs @@ -68,27 +68,22 @@ public ModulePropertyAttribute() } } - internal static partial class TypeOffset - { - public static int magic() => ManagedDataOffsets.Magic; - } - internal static class ManagedDataOffsets { - public static int Magic { get; internal set; } public static readonly Dictionary NameMapping = new Dictionary(); static class DataOffsets { public static readonly int ob_data = 0; public static readonly int ob_dict = 0; + public static readonly int ob_weaklist = 0; static DataOffsets() { FieldInfo[] fields = typeof(DataOffsets).GetFields(BindingFlags.Static | BindingFlags.Public); for (int i = 0; i < fields.Length; i++) { - fields[i].SetValue(null, -(i * IntPtr.Size) - IntPtr.Size); + fields[i].SetValue(null, (i) * IntPtr.Size); } } } @@ -98,7 +93,7 @@ static ManagedDataOffsets() NameMapping = TypeOffset.GetOffsets(); FieldInfo[] fields = typeof(DataOffsets).GetFields(BindingFlags.Static | BindingFlags.Public); - size = fields.Length * IntPtr.Size; + Size = fields.Length * IntPtr.Size; } public static int GetSlotOffset(string name) @@ -106,29 +101,10 @@ public static int GetSlotOffset(string name) return NameMapping[name]; } - private static int BaseOffset(IntPtr type) - { - Debug.Assert(type != IntPtr.Zero); - int typeSize = Marshal.ReadInt32(type, TypeOffset.tp_basicsize); - Debug.Assert(typeSize > 0); - return typeSize; - } - - public static int DataOffset(IntPtr type) - { - return BaseOffset(type) + DataOffsets.ob_data; - } - - public static int DictOffset(IntPtr type) - { - return BaseOffset(type) + DataOffsets.ob_dict; - } - public static int ob_data => DataOffsets.ob_data; public static int ob_dict => DataOffsets.ob_dict; - public static int Size { get { return size; } } - - private static readonly int size; + public static int ob_weaklist => DataOffsets.ob_weaklist; + public static int Size { get; private set; } } internal static class OriginalObjectOffsets @@ -163,6 +139,26 @@ static OriginalObjectOffsets() public static int ob_type; } + + internal static class ManagedExceptionOffset + { + public static int ob_data { get; private set; } + public static int ob_dict { get; private set; } + public static int ob_weaklist { get; private set; } + + public static int Size { get; private set; } + + static ManagedExceptionOffset() + { + int baseOffset = OriginalObjectOffsets.Size + ExceptionOffset.Size(); + ob_data = baseOffset + ManagedDataOffsets.ob_data; + ob_dict = baseOffset + ManagedDataOffsets.ob_dict; + ob_weaklist = baseOffset + ManagedDataOffsets.ob_weaklist; + Size = ob_weaklist + IntPtr.Size; + } + } + + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] internal class ObjectOffset { @@ -175,27 +171,17 @@ static ObjectOffset() ob_refcnt = OriginalObjectOffsets.ob_refcnt; ob_type = OriginalObjectOffsets.ob_type; - size = OriginalObjectOffsets.Size + ManagedDataOffsets.Size; + Size = OriginalObjectOffsets.Size + ManagedDataOffsets.Size; } public static int magic(IntPtr type) { - return ManagedDataOffsets.DataOffset(type); + return (int)Marshal.ReadIntPtr(type, TypeOffset.Size); } public static int TypeDictOffset(IntPtr type) { - return ManagedDataOffsets.DictOffset(type); - } - - public static int Size(IntPtr pyType) - { - if (IsException(pyType)) - { - return ExceptionOffset.Size(); - } - - return size; + return (int)Marshal.ReadIntPtr(type, TypeOffset.tp_dictoffset); } #if PYTHON_WITH_PYDEBUG @@ -204,16 +190,8 @@ public static int Size(IntPtr pyType) #endif public static int ob_refcnt; public static int ob_type; - private static readonly int size; - private static bool IsException(IntPtr pyObjectPtr) - { - var pyObject = new BorrowedReference(pyObjectPtr); - var type = Runtime.PyObject_TYPE(pyObject); - return Runtime.PyType_IsSameAsOrSubtype(type, ofType: Exceptions.BaseException) - || Runtime.PyType_IsSameAsOrSubtype(type, ofType: Runtime.PyTypeType) - && Runtime.PyType_IsSubtype(pyObject, Exceptions.BaseException); - } + public static int Size { get; private set; } } [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] @@ -225,10 +203,9 @@ static ExceptionOffset() FieldInfo[] fi = type.GetFields(BindingFlags.Static | BindingFlags.Public); for (int i = 0; i < fi.Length; i++) { - fi[i].SetValue(null, (i * IntPtr.Size) + OriginalObjectOffsets.Size); + fi[i].SetValue(null, i * IntPtr.Size); } - - size = fi.Length * IntPtr.Size + OriginalObjectOffsets.Size + ManagedDataOffsets.Size; + size = fi.Length * IntPtr.Size; } public static int Size() { return size; } diff --git a/src/runtime/managedtype.cs b/src/runtime/managedtype.cs index 09e8a3be2..12aa7104f 100644 --- a/src/runtime/managedtype.cs +++ b/src/runtime/managedtype.cs @@ -26,7 +26,6 @@ internal enum TrackTypes internal IntPtr pyHandle; // PyObject * internal IntPtr tpHandle; // PyType * - internal BorrowedReference ObjectReference => new BorrowedReference(pyHandle); private static readonly Dictionary _managedObjs = new Dictionary(); @@ -84,29 +83,28 @@ internal static ManagedType GetManagedObject(BorrowedReference ob) /// internal static ManagedType GetManagedObject(IntPtr ob) { - if (ob != IntPtr.Zero) + if (ob == IntPtr.Zero) { - IntPtr tp = Runtime.PyObject_TYPE(ob); - if (tp == Runtime.PyTypeType || tp == Runtime.PyCLRMetaType) - { - tp = ob; - } - - var flags = Util.ReadCLong(tp, TypeOffset.tp_flags); - if ((flags & TypeFlags.Managed) != 0) - { - IntPtr op = tp == ob - ? Marshal.ReadIntPtr(tp, TypeOffset.magic()) - : Marshal.ReadIntPtr(ob, ObjectOffset.magic(tp)); - if (op == IntPtr.Zero) - { - return null; - } - var gc = (GCHandle)op; - return (ManagedType)gc.Target; - } + return null; + } + IntPtr tp = Runtime.PyObject_TYPE(ob); + var flags = Util.ReadCLong(tp, TypeOffset.tp_flags); + if ((flags & TypeFlags.Managed) == 0) + { + return null; + } + int offset = ObjectOffset.magic(tp); + if (offset == 0) + { + return null; } - return null; + IntPtr op = Marshal.ReadIntPtr(ob, offset); + if (op == IntPtr.Zero) + { + return null; + } + var gc = (GCHandle)op; + return (ManagedType)gc.Target; } ///

@@ -114,18 +112,12 @@ internal static ManagedType GetManagedObject(IntPtr ob) ///

internal static ManagedType GetManagedObjectType(IntPtr ob) { - if (ob != IntPtr.Zero) + if (ob == IntPtr.Zero) { - IntPtr tp = Runtime.PyObject_TYPE(ob); - var flags = Util.ReadCLong(tp, TypeOffset.tp_flags); - if ((flags & TypeFlags.Managed) != 0) - { - tp = Marshal.ReadIntPtr(tp, TypeOffset.magic()); - var gc = (GCHandle)tp; - return (ManagedType)gc.Target; - } + return null; } - return null; + IntPtr tp = Runtime.PyObject_TYPE(ob); + return GetManagedObject(tp); } @@ -147,11 +139,6 @@ internal static bool IsManagedType(IntPtr ob) if (ob != IntPtr.Zero) { IntPtr tp = Runtime.PyObject_TYPE(ob); - if (tp == Runtime.PyTypeType || tp == Runtime.PyCLRMetaType) - { - tp = ob; - } - var flags = Util.ReadCLong(tp, TypeOffset.tp_flags); if ((flags & TypeFlags.Managed) != 0) { @@ -161,11 +148,6 @@ internal static bool IsManagedType(IntPtr ob) return false; } - public bool IsTypeObject() - { - return pyHandle == tpHandle; - } - internal static IDictionary GetManagedObjects() { return _managedObjs; @@ -243,6 +225,16 @@ protected virtual void OnSave(InterDomainContext context) { } protected virtual void OnLoad(InterDomainContext context) { } protected static void ClearObjectDict(IntPtr ob) + { + IntPtr dict = GetObjectDict(ob); + if (dict == IntPtr.Zero) + { + return; + } + Runtime.PyDict_Clear(dict); + } + + protected static void RemoveObjectDict(IntPtr ob) { IntPtr dict = GetObjectDict(ob); if (dict == IntPtr.Zero) diff --git a/src/runtime/metatype.cs b/src/runtime/metatype.cs index 36b406c7b..3f4717190 100644 --- a/src/runtime/metatype.cs +++ b/src/runtime/metatype.cs @@ -1,5 +1,6 @@ using System; using System.Collections; +using System.Diagnostics; using System.IO; using System.Runtime.InteropServices; using System.Runtime.Serialization; @@ -154,38 +155,17 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) flags |= TypeFlags.Subclass; flags |= TypeFlags.HaveGC; Util.WriteCLong(type, TypeOffset.tp_flags, flags); - - TypeManager.CopySlot(base_type, type, TypeOffset.tp_dealloc); - - // Hmm - the standard subtype_traverse, clear look at ob_size to - // do things, so to allow gc to work correctly we need to move - // our hidden handle out of ob_size. Then, in theory we can - // comment this out and still not crash. - TypeManager.CopySlot(base_type, type, TypeOffset.tp_traverse); - TypeManager.CopySlot(base_type, type, TypeOffset.tp_clear); - - // for now, move up hidden handle... - IntPtr gc = Marshal.ReadIntPtr(base_type, TypeOffset.magic()); - Marshal.WriteIntPtr(type, TypeOffset.magic(), gc); - - return type; - } - - - public static IntPtr tp_alloc(IntPtr mt, int n) - { - IntPtr type = Runtime.PyType_GenericAlloc(mt, n); + unsafe + { + var baseTypeEx = ClrMetaTypeEx.FromType(base_type); + var typeEx = ClrMetaTypeEx.FromType(type); + typeEx->ClrHandleOffset = (IntPtr)(OriginalObjectOffsets.Size + ManagedDataOffsets.ob_data); + typeEx->ClrHandle = baseTypeEx->ClrHandle; + } return type; } - - public static void tp_free(IntPtr tp) - { - Runtime.PyObject_GC_Del(tp); - } - - ///

/// Metatype __call__ implementation. This is needed to ensure correct /// initialization (__init__ support), because the tp_call we inherit @@ -288,7 +268,7 @@ public static void tp_dealloc(IntPtr tp) var flags = Util.ReadCLong(tp, TypeOffset.tp_flags); if ((flags & TypeFlags.Subclass) == 0) { - IntPtr gc = Marshal.ReadIntPtr(tp, TypeOffset.magic()); + IntPtr gc = Marshal.ReadIntPtr(tp, ObjectOffset.magic(Runtime.PyObject_TYPE(tp))); ((GCHandle)gc).Free(); } @@ -304,6 +284,12 @@ public static void tp_dealloc(IntPtr tp) NativeCall.Void_Call_1(op, tp); } + public static int tp_clear(IntPtr ob) + { + ClearObjectDict(ob); + return 0; + } + private static IntPtr DoInstanceCheck(IntPtr tp, IntPtr args, bool checkType) { var cb = GetManagedObject(tp) as ClassBase; @@ -360,4 +346,18 @@ public static IntPtr __subclasscheck__(IntPtr tp, IntPtr args) return DoInstanceCheck(tp, args, true); } } + + + [StructLayout(LayoutKind.Sequential)] + struct ClrMetaTypeEx + { + public nint ClrHandleOffset; + public IntPtr ClrHandle; + + public static unsafe ClrMetaTypeEx* FromType(IntPtr type) + { + return (ClrMetaTypeEx*)(type + TypeOffset.Size); + } + } + } diff --git a/src/runtime/methodobject.cs b/src/runtime/methodobject.cs index 37c01f5c5..630f286da 100644 --- a/src/runtime/methodobject.cs +++ b/src/runtime/methodobject.cs @@ -217,7 +217,7 @@ public static IntPtr tp_repr(IntPtr ob) { var self = (MethodObject)GetManagedObject(ob); self.ClearMembers(); - ClearObjectDict(ob); + RemoveObjectDict(ob); self.Dealloc(); } diff --git a/src/runtime/native/ABI.cs b/src/runtime/native/ABI.cs index 3264531de..21890d57e 100644 --- a/src/runtime/native/ABI.cs +++ b/src/runtime/native/ABI.cs @@ -29,8 +29,6 @@ internal static void Initialize(Version version, BorrowedReference pyType) } var typeOffsets = (ITypeOffsets)Activator.CreateInstance(typeOffsetsClass); TypeOffset.Use(typeOffsets); - - ManagedDataOffsets.Magic = Marshal.ReadInt32(pyType.DangerousGetAddress(), TypeOffset.tp_basicsize); } } } diff --git a/src/runtime/native/ITypeOffsets.cs b/src/runtime/native/ITypeOffsets.cs index 485c041f8..438e96fe8 100644 --- a/src/runtime/native/ITypeOffsets.cs +++ b/src/runtime/native/ITypeOffsets.cs @@ -30,6 +30,7 @@ interface ITypeOffsets int nb_inplace_add { get; } int nb_inplace_subtract { get; } int ob_size { get; } + int ob_refcnt { get; } int ob_type { get; } int qualname { get; } int sq_contains { get; } @@ -49,6 +50,7 @@ interface ITypeOffsets int tp_descr_set { get; } int tp_dict { get; } int tp_dictoffset { get; } + int tp_init { get; } int tp_flags { get; } int tp_free { get; } int tp_getattro { get; } @@ -63,6 +65,7 @@ interface ITypeOffsets int tp_new { get; } int tp_repr { get; } int tp_richcompare { get; } + int tp_weaklistoffset { get; } int tp_setattro { get; } int tp_str { get; } int tp_traverse { get; } diff --git a/src/runtime/native/TypeOffset.cs b/src/runtime/native/TypeOffset.cs index 9f5ed671b..62bd90b4a 100644 --- a/src/runtime/native/TypeOffset.cs +++ b/src/runtime/native/TypeOffset.cs @@ -8,6 +8,7 @@ namespace Python.Runtime using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Reflection; + using System.Runtime.InteropServices; using Python.Runtime.Native; @@ -37,6 +38,7 @@ static partial class TypeOffset internal static int nb_inplace_add { get; private set; } internal static int nb_inplace_subtract { get; private set; } internal static int ob_size { get; private set; } + internal static int ob_refcnt { get; private set; } internal static int ob_type { get; private set; } internal static int qualname { get; private set; } internal static int sq_contains { get; private set; } @@ -56,6 +58,7 @@ static partial class TypeOffset internal static int tp_descr_set { get; private set; } internal static int tp_dict { get; private set; } internal static int tp_dictoffset { get; private set; } + internal static int tp_init { get; private set; } internal static int tp_flags { get; private set; } internal static int tp_free { get; private set; } internal static int tp_getattro { get; private set; } @@ -70,10 +73,13 @@ static partial class TypeOffset internal static int tp_new { get; private set; } internal static int tp_repr { get; private set; } internal static int tp_richcompare { get; private set; } + internal static int tp_weaklistoffset { get; private set; } internal static int tp_setattro { get; private set; } internal static int tp_str { get; private set; } internal static int tp_traverse { get; private set; } + internal static int Size; + internal static void Use(ITypeOffsets offsets) { if (offsets is null) throw new ArgumentNullException(nameof(offsets)); @@ -88,7 +94,7 @@ internal static void Use(ITypeOffsets offsets) int value = (int)sourceProperty.GetValue(offsets, null); offsetProperty.SetValue(obj: null, value: value, index: null); } - + Size = (int)Marshal.ReadIntPtr(Runtime.PyTypeType, tp_basicsize); ValidateUnusedTypeOffsetProperties(offsetProperties); ValidateRequiredOffsetsPresent(offsetProperties); } diff --git a/src/runtime/pyobject.cs b/src/runtime/pyobject.cs index 3c7f13ec6..aaffa29fe 100644 --- a/src/runtime/pyobject.cs +++ b/src/runtime/pyobject.cs @@ -119,7 +119,9 @@ public static PyObject FromManagedObject(object ob) Runtime.XIncref(Runtime.PyNone); return new PyObject(Runtime.PyNone); } - IntPtr op = CLRObject.GetInstHandle(ob); + IntPtr op = typeof(Exception).IsAssignableFrom(ob.GetType()) ? + Exceptions.GetExceptHandle((Exception)ob) + : CLRObject.GetInstHandle(ob); return new PyObject(op); } diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index ec7f5e446..cd581d381 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -348,8 +348,8 @@ internal static void Shutdown(ShutdownMode mode) ClearClrModules(); RemoveClrRootModule(); - MoveClrInstancesOnwershipToPython(); ClassManager.DisposePythonWrappersForClrTypes(); + MoveClrInstancesOnwershipToPython(); TypeManager.RemoveTypes(); MetaType.Release(); @@ -503,15 +503,19 @@ private static void PyDictTryDelItem(BorrowedReference dict, string key) private static void MoveClrInstancesOnwershipToPython() { var objs = ManagedType.GetManagedObjects(); - var copyObjs = objs.ToArray(); - foreach (var entry in copyObjs) + var copyObjs = new KeyValuePair[objs.Count]; { - ManagedType obj = entry.Key; - if (!objs.ContainsKey(obj)) + int i = 0; + foreach (var entry in objs) { - System.Diagnostics.Debug.Assert(obj.gcHandle == default); - continue; + ManagedType obj = entry.Key; + XIncref(obj.pyHandle); + copyObjs[i++] = entry; } + } + foreach (var entry in copyObjs) + { + ManagedType obj = entry.Key; if (entry.Value == ManagedType.TrackTypes.Extension) { obj.CallTypeClear(); @@ -522,11 +526,21 @@ private static void MoveClrInstancesOnwershipToPython() PyObject_GC_Track(obj.pyHandle); } } - if (obj.gcHandle.IsAllocated) + } + foreach (var entry in copyObjs) + { + ManagedType obj = entry.Key; + if (!objs.ContainsKey(obj)) { - obj.gcHandle.Free(); + System.Diagnostics.Debug.Assert(obj.gcHandle == default); + continue; + } + if (obj.RefCount > 1) + { + obj.FreeGCHandle(); + Marshal.WriteIntPtr(obj.pyHandle, ObjectOffset.magic(obj.tpHandle), IntPtr.Zero); } - obj.gcHandle = default; + XDecref(obj.pyHandle); } ManagedType.ClearTrackedObjects(); } @@ -2018,25 +2032,27 @@ internal static IntPtr PyType_GenericAlloc(IntPtr type, long n) internal static IntPtr PyObject_GenericGetAttr(IntPtr obj, IntPtr name) => Delegates.PyObject_GenericGetAttr(obj, name); - + internal static bool PyType_SUPPORTS_WEAKREFS(IntPtr type) + { + return Marshal.ReadIntPtr(type, TypeOffset.tp_weaklistoffset) != IntPtr.Zero; + } internal static int PyObject_GenericSetAttr(IntPtr obj, IntPtr name, IntPtr value) => Delegates.PyObject_GenericSetAttr(obj, name, value); - internal static BorrowedReference* _PyObject_GetDictPtr(BorrowedReference obj) => Delegates._PyObject_GetDictPtr(obj); - internal static void PyObject_GC_Del(IntPtr tp) => Delegates.PyObject_GC_Del(tp); + internal static IntPtr _PyObject_GC_Calloc(IntPtr basicsize) => Delegates._PyObject_GC_Calloc(basicsize); internal static void PyObject_GC_Track(IntPtr tp) => Delegates.PyObject_GC_Track(tp); - internal static void PyObject_GC_UnTrack(IntPtr tp) => Delegates.PyObject_GC_UnTrack(tp); - internal static void _PyObject_Dump(IntPtr ob) => Delegates._PyObject_Dump(ob); + internal static void PyObject_ClearWeakRefs(IntPtr obj) => Delegates.PyObject_ClearWeakRefs(obj); + //==================================================================== // Python memory API //==================================================================== @@ -2173,6 +2189,16 @@ internal static void Py_CLEAR(ref IntPtr ob) ob = IntPtr.Zero; } + internal static unsafe void Py_SETREF(IntPtr ob, int offset, IntPtr target) + { + var p = (void**)(ob + offset); + if (*p != null) + { + XDecref((IntPtr)(*p)); + } + *p = (void*)target; + } + //==================================================================== // Python Capsules API //==================================================================== @@ -2475,9 +2501,11 @@ static Delegates() PyObject_GenericSetAttr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GenericSetAttr), GetUnmanagedDll(_PythonDll)); _PyObject_GetDictPtr = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(_PyObject_GetDictPtr), GetUnmanagedDll(_PythonDll)); PyObject_GC_Del = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GC_Del), GetUnmanagedDll(_PythonDll)); + _PyObject_GC_Calloc = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(_PyObject_GC_Calloc), GetUnmanagedDll(_PythonDll)); PyObject_GC_Track = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GC_Track), GetUnmanagedDll(_PythonDll)); PyObject_GC_UnTrack = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_GC_UnTrack), GetUnmanagedDll(_PythonDll)); _PyObject_Dump = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(_PyObject_Dump), GetUnmanagedDll(_PythonDll)); + PyObject_ClearWeakRefs = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyObject_ClearWeakRefs), GetUnmanagedDll(_PythonDll)); PyMem_Malloc = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyMem_Malloc), GetUnmanagedDll(_PythonDll)); PyMem_Realloc = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyMem_Realloc), GetUnmanagedDll(_PythonDll)); PyMem_Free = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyMem_Free), GetUnmanagedDll(_PythonDll)); @@ -2740,9 +2768,11 @@ static Delegates() internal static delegate* unmanaged[Cdecl] PyObject_GenericSetAttr { get; } internal static delegate* unmanaged[Cdecl] _PyObject_GetDictPtr { get; } internal static delegate* unmanaged[Cdecl] PyObject_GC_Del { get; } + internal static delegate* unmanaged[Cdecl] _PyObject_GC_Calloc { get; } internal static delegate* unmanaged[Cdecl] PyObject_GC_Track { get; } internal static delegate* unmanaged[Cdecl] PyObject_GC_UnTrack { get; } internal static delegate* unmanaged[Cdecl] _PyObject_Dump { get; } + internal static delegate* unmanaged[Cdecl] PyObject_ClearWeakRefs { get; } internal static delegate* unmanaged[Cdecl] PyMem_Malloc { get; } internal static delegate* unmanaged[Cdecl] PyMem_Realloc { get; } internal static delegate* unmanaged[Cdecl] PyMem_Free { get; } diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index e0564b243..1975c2421 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -45,10 +45,10 @@ internal static void Initialize() internal static void RemoveTypes() { - foreach (var tpHandle in cache.Values) + foreach (var entry in _slotsHolders) { - SlotsHolder holder; - if (_slotsHolders.TryGetValue(tpHandle, out holder)) + IntPtr tpHandle = entry.Key; + SlotsHolder holder = entry.Value; { // If refcount > 1, it needs to reset the managed slot, // otherwise it can dealloc without any trick. @@ -160,13 +160,13 @@ internal static IntPtr GetTypeHandle(ManagedType obj, Type type) internal static IntPtr CreateType(Type impl) { IntPtr type = AllocateTypeObject(impl.Name, metatype: Runtime.PyTypeType); - int ob_size = ObjectOffset.Size(type); + int ob_size = ObjectOffset.Size; // Set tp_basicsize to the size of our managed instance objects. Marshal.WriteIntPtr(type, TypeOffset.tp_basicsize, (IntPtr)ob_size); - var offset = (IntPtr)ObjectOffset.TypeDictOffset(type); - Marshal.WriteIntPtr(type, TypeOffset.tp_dictoffset, offset); + var offset = OriginalObjectOffsets.Size + ManagedDataOffsets.ob_dict; + Marshal.WriteIntPtr(type, TypeOffset.tp_dictoffset, (IntPtr)offset); SlotsHolder slotsHolder = CreateSolotsHolder(type); InitializeSlots(type, impl, slotsHolder); @@ -186,6 +186,11 @@ internal static IntPtr CreateType(Type impl) mod.Dispose(); InitMethods(type, impl); + unsafe + { + var typeEx = ClrMetaTypeEx.FromType(type); + typeEx->ClrHandleOffset = (IntPtr)OriginalObjectOffsets.Size + ManagedDataOffsets.ob_data; + } // The type has been modified after PyType_Ready has been called // Refresh the type @@ -210,17 +215,25 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) } IntPtr base_ = IntPtr.Zero; - int ob_size = ObjectOffset.Size(Runtime.PyTypeType); - + int baseOffset = OriginalObjectOffsets.Size; + int ob_size, tp_dictoffset, tp_weaklistoffset, magicOffset; // XXX Hack, use a different base class for System.Exception // Python 2.5+ allows new style class exceptions but they *must* // subclass BaseException (or better Exception). if (typeof(Exception).IsAssignableFrom(clrType)) { - ob_size = ObjectOffset.Size(Exceptions.Exception); + tp_dictoffset = ManagedExceptionOffset.ob_dict; + tp_weaklistoffset = 0; + ob_size = ManagedExceptionOffset.Size; + magicOffset = ManagedExceptionOffset.ob_data; + } + else + { + tp_dictoffset = baseOffset + ManagedDataOffsets.ob_dict; + tp_weaklistoffset = baseOffset + ManagedDataOffsets.ob_weaklist; + ob_size = baseOffset + ManagedDataOffsets.Size; + magicOffset = baseOffset + ManagedDataOffsets.ob_data; } - - int tp_dictoffset = ob_size + ManagedDataOffsets.ob_dict; if (clrType == typeof(Exception)) { @@ -240,6 +253,7 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) Marshal.WriteIntPtr(type, TypeOffset.tp_basicsize, (IntPtr)ob_size); Marshal.WriteIntPtr(type, TypeOffset.tp_itemsize, IntPtr.Zero); Marshal.WriteIntPtr(type, TypeOffset.tp_dictoffset, (IntPtr)tp_dictoffset); + Marshal.WriteIntPtr(type, TypeOffset.tp_weaklistoffset, (IntPtr)tp_weaklistoffset); // we want to do this after the slot stuff above in case the class itself implements a slot method SlotsHolder slotsHolder = CreateSolotsHolder(type); @@ -311,10 +325,16 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) // Hide the gchandle of the implementation in a magic type slot. GCHandle gc = impl.AllocGCHandle(); - Marshal.WriteIntPtr(type, TypeOffset.magic(), (IntPtr)gc); + unsafe + { + var typePtr = ClrMetaTypeEx.FromType(type); + typePtr->ClrHandle = (IntPtr)gc; + typePtr->ClrHandleOffset = (IntPtr)magicOffset; + } // Set the handle attributes on the implementing instance. - impl.tpHandle = type; + impl.tpHandle = Runtime.PyCLRMetaType; + Runtime.XIncref(type); impl.pyHandle = type; //DebugUtil.DumpType(type); @@ -448,14 +468,19 @@ internal static IntPtr CreateMetaType(Type impl, out SlotsHolder slotsHolder) // the standard type slots, and has to subclass PyType_Type for // certain functions in the C runtime to work correctly with it. - IntPtr type = AllocateTypeObject("CLR Metatype", metatype: Runtime.PyTypeType); - IntPtr py_type = Runtime.PyTypeType; - Marshal.WriteIntPtr(type, TypeOffset.tp_base, py_type); + var heapTypeSize = (int)Marshal.ReadIntPtr(py_type, TypeOffset.tp_basicsize); + Debug.Assert(heapTypeSize == TypeOffset.Size); + int metaSize = heapTypeSize + Marshal.SizeOf(typeof(ClrMetaTypeEx)); + + IntPtr type = Runtime._PyObject_GC_Calloc(new IntPtr(metaSize)); Runtime.XIncref(py_type); + Marshal.WriteIntPtr(type, TypeOffset.tp_base, py_type); + Marshal.WriteIntPtr(type, TypeOffset.ob_refcnt, (IntPtr)1); + Marshal.WriteIntPtr(type, TypeOffset.tp_basicsize, (IntPtr)metaSize); + Marshal.WriteIntPtr(type, heapTypeSize, (IntPtr)(heapTypeSize + IntPtr.Size)); - int size = TypeOffset.magic() + IntPtr.Size; - Marshal.WriteIntPtr(type, TypeOffset.tp_basicsize, new IntPtr(size)); + SetupHeapType(type, "CLR Metatype"); const int flags = TypeFlags.Default | TypeFlags.Managed @@ -591,39 +616,13 @@ internal static IntPtr BasicSubType(string name, IntPtr base_, Type impl) return type; } - ///

/// Utility method to allocate a type object & do basic initialization. ///

internal static IntPtr AllocateTypeObject(string name, IntPtr metatype) { - IntPtr type = Runtime.PyType_GenericAlloc(metatype, 0); - // Clr type would not use __slots__, - // and the PyMemberDef after PyHeapTypeObject will have other uses(e.g. type handle), - // thus set the ob_size to 0 for avoiding slots iterations. - Marshal.WriteIntPtr(type, TypeOffset.ob_size, IntPtr.Zero); - - // Cheat a little: we'll set tp_name to the internal char * of - // the Python version of the type name - otherwise we'd have to - // allocate the tp_name and would have no way to free it. - IntPtr temp = Runtime.PyUnicode_FromString(name); - IntPtr raw = Runtime.PyUnicode_AsUTF8(temp); - Marshal.WriteIntPtr(type, TypeOffset.tp_name, raw); - Marshal.WriteIntPtr(type, TypeOffset.name, temp); - - Runtime.XIncref(temp); - Marshal.WriteIntPtr(type, TypeOffset.qualname, temp); - temp = type + TypeOffset.nb_add; - Marshal.WriteIntPtr(type, TypeOffset.tp_as_number, temp); - - temp = type + TypeOffset.sq_length; - Marshal.WriteIntPtr(type, TypeOffset.tp_as_sequence, temp); - - temp = type + TypeOffset.mp_length; - Marshal.WriteIntPtr(type, TypeOffset.tp_as_mapping, temp); - - temp = type + TypeOffset.bf_getbuffer; - Marshal.WriteIntPtr(type, TypeOffset.tp_as_buffer, temp); + IntPtr type = Runtime.PyType_GenericAlloc(Runtime.PyCLRMetaType, 0); + SetupHeapType(type, name); return type; } @@ -780,6 +779,36 @@ private static SlotsHolder CreateSolotsHolder(IntPtr type) _slotsHolders.Add(type, holder); return holder; } + + private static void SetupHeapType(IntPtr type, string name) + { + // Clr type would not use __slots__, + // and the PyMemberDef after PyHeapTypeObject will have other uses(e.g. type handle), + // thus set the ob_size to 0 for avoiding slots iterations. + Marshal.WriteIntPtr(type, TypeOffset.ob_size, IntPtr.Zero); + + // Cheat a little: we'll set tp_name to the internal char * of + // the Python version of the type name - otherwise we'd have to + // allocate the tp_name and would have no way to free it. + IntPtr temp = Runtime.PyUnicode_FromString(name); + IntPtr raw = Runtime.PyUnicode_AsUTF8(temp); + Marshal.WriteIntPtr(type, TypeOffset.tp_name, raw); + Marshal.WriteIntPtr(type, TypeOffset.name, temp); + + Runtime.XIncref(temp); + Marshal.WriteIntPtr(type, TypeOffset.qualname, temp); + temp = type + TypeOffset.nb_add; + Marshal.WriteIntPtr(type, TypeOffset.tp_as_number, temp); + + temp = type + TypeOffset.sq_length; + Marshal.WriteIntPtr(type, TypeOffset.tp_as_sequence, temp); + + temp = type + TypeOffset.mp_length; + Marshal.WriteIntPtr(type, TypeOffset.tp_as_mapping, temp); + + temp = type + TypeOffset.bf_getbuffer; + Marshal.WriteIntPtr(type, TypeOffset.tp_as_buffer, temp); + } } @@ -788,10 +817,10 @@ class SlotsHolder public delegate void Resetor(IntPtr type, int offset); private readonly IntPtr _type; - private Dictionary _slots = new Dictionary(); - private List _keepalive = new List(); - private Dictionary _customResetors = new Dictionary(); - private List _deallocators = new List(); + private Dictionary _slots; + private List _keepalive; + private Dictionary _customResetors; + private List _deallocators; private bool _alreadyReset = false; ///

@@ -805,21 +834,25 @@ public SlotsHolder(IntPtr type) public void Set(int offset, ThunkInfo thunk) { + if (_slots == null) _slots = new Dictionary(); _slots[offset] = thunk; } public void Set(int offset, Resetor resetor) { + if (_customResetors == null) _customResetors = new Dictionary(); _customResetors[offset] = resetor; } public void AddDealloctor(Action deallocate) { + if (_deallocators == null) _deallocators = new List(); _deallocators.Add(deallocate); } public void KeeapAlive(ThunkInfo thunk) { + if (_keepalive == null) _keepalive = new List(); _keepalive.Add(thunk); } @@ -830,47 +863,15 @@ public void ResetSlots() return; } _alreadyReset = true; -#if DEBUG - IntPtr tp_name = Marshal.ReadIntPtr(_type, TypeOffset.tp_name); - string typeName = Marshal.PtrToStringAnsi(tp_name); -#endif - foreach (var offset in _slots.Keys) - { - IntPtr ptr = GetDefaultSlot(offset); -#if DEBUG - //DebugUtil.Print($"Set slot<{TypeOffsetHelper.GetSlotNameByOffset(offset)}> to 0x{ptr.ToString("X")} at {typeName}<0x{_type}>"); -#endif - Marshal.WriteIntPtr(_type, offset, ptr); - } - - foreach (var action in _deallocators) - { - action(); - } + ResetDefaultSlots(); + InvokeDeallocActions(); + InvokeCustomResetors(); - foreach (var pair in _customResetors) + if (_keepalive != null) { - int offset = pair.Key; - var resetor = pair.Value; - resetor?.Invoke(_type, offset); - } - - _customResetors.Clear(); - _slots.Clear(); - _keepalive.Clear(); - _deallocators.Clear(); - - // Custom reset - IntPtr handlePtr = Marshal.ReadIntPtr(_type, TypeOffset.magic()); - if (handlePtr != IntPtr.Zero) - { - GCHandle handle = GCHandle.FromIntPtr(handlePtr); - if (handle.IsAllocated) - { - handle.Free(); - } - Marshal.WriteIntPtr(_type, TypeOffset.magic(), IntPtr.Zero); + _keepalive.Clear(); } + ReleaseGCHandle(); } public static IntPtr GetDefaultSlot(int offset) @@ -915,6 +916,65 @@ public static IntPtr GetDefaultSlot(int offset) return Marshal.ReadIntPtr(Runtime.PyTypeType, offset); } + + private void InvokeCustomResetors() + { + if (_customResetors == null) return; + foreach (var pair in _customResetors) + { + int offset = pair.Key; + var resetor = pair.Value; + resetor?.Invoke(_type, offset); + } + _customResetors.Clear(); + } + + private void InvokeDeallocActions() + { + if (_deallocators == null) return; + foreach (var action in _deallocators) + { + action(); + } + _deallocators.Clear(); + } + + private void ResetDefaultSlots() + { + if (_slots == null) return; +#if DEBUG + IntPtr tp_name = Marshal.ReadIntPtr(_type, TypeOffset.tp_name); + string typeName = Marshal.PtrToStringAnsi(tp_name); +#endif + foreach (var offset in _slots.Keys) + { + IntPtr ptr = GetDefaultSlot(offset); +#if DEBUG + //DebugUtil.Print($"Set slot<{TypeOffsetHelper.GetSlotNameByOffset(offset)}> to 0x{ptr.ToString("X")} at {typeName}<0x{_type}>"); +#endif + Marshal.WriteIntPtr(_type, offset, ptr); + } + _slots.Clear(); + } + + private void ReleaseGCHandle() + { + if (!ManagedType.IsManagedType(_type)) + { + return; + } + int offset = ObjectOffset.magic(Runtime.PyObject_TYPE(_type)); + IntPtr handlePtr = Marshal.ReadIntPtr(_type, offset); + if (handlePtr != IntPtr.Zero) + { + GCHandle handle = GCHandle.FromIntPtr(handlePtr); + if (handle.IsAllocated) + { + handle.Free(); + } + Marshal.WriteIntPtr(_type, offset, IntPtr.Zero); + } + } } pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.

Alternative Proxies: