diff --git a/AUTHORS.md b/AUTHORS.md index 69e7b5c4a..167fd496c 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -24,6 +24,7 @@ - Benoît Hudson ([@benoithudson](https://github.com/benoithudson)) - Bradley Friedman ([@leith-bartrich](https://github.com/leith-bartrich)) - Callum Noble ([@callumnoble](https://github.com/callumnoble)) +- Christabella Irwanto([@christabella](https://github.com/christabella)) - Christian Heimes ([@tiran](https://github.com/tiran)) - Christoph Gohlke ([@cgohlke](https://github.com/cgohlke)) - Christopher Bremner ([@chrisjbremner](https://github.com/chrisjbremner)) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53c45f419..a9a804e8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. ### Added - Ability to instantiate new .NET arrays using `Array[T](dim1, dim2, ...)` syntax +- Python operator method will call C# operator method for supported binary and unary operators ([#1324][p1324]). ### Changed - Drop support for Python 2, 3.4, and 3.5 diff --git a/src/embed_tests/TestOperator.cs b/src/embed_tests/TestOperator.cs new file mode 100644 index 000000000..ecdb0c1dc --- /dev/null +++ b/src/embed_tests/TestOperator.cs @@ -0,0 +1,332 @@ +using NUnit.Framework; + +using Python.Runtime; + +using System.Linq; +using System.Reflection; + +namespace Python.EmbeddingTest +{ + public class TestOperator + { + [OneTimeSetUp] + public void SetUp() + { + PythonEngine.Initialize(); + } + + [OneTimeTearDown] + public void Dispose() + { + PythonEngine.Shutdown(); + } + + public class OperableObject + { + public int Num { get; set; } + + public OperableObject(int num) + { + Num = num; + } + + public static OperableObject operator ~(OperableObject a) + { + return new OperableObject(~a.Num); + } + + public static OperableObject operator +(OperableObject a) + { + return new OperableObject(+a.Num); + } + + public static OperableObject operator -(OperableObject a) + { + return new OperableObject(-a.Num); + } + + public static OperableObject operator +(int a, OperableObject b) + { + return new OperableObject(a + b.Num); + } + public static OperableObject operator +(OperableObject a, OperableObject b) + { + return new OperableObject(a.Num + b.Num); + } + public static OperableObject operator +(OperableObject a, int b) + { + return new OperableObject(a.Num + b); + } + + public static OperableObject operator -(int a, OperableObject b) + { + return new OperableObject(a - b.Num); + } + public static OperableObject operator -(OperableObject a, OperableObject b) + { + return new OperableObject(a.Num - b.Num); + } + public static OperableObject operator -(OperableObject a, int b) + { + return new OperableObject(a.Num - b); + } + + public static OperableObject operator *(int a, OperableObject b) + { + return new OperableObject(a * b.Num); + } + public static OperableObject operator *(OperableObject a, OperableObject b) + { + return new OperableObject(a.Num * b.Num); + } + public static OperableObject operator *(OperableObject a, int b) + { + return new OperableObject(a.Num * b); + } + + public static OperableObject operator /(int a, OperableObject b) + { + return new OperableObject(a / b.Num); + } + public static OperableObject operator /(OperableObject a, OperableObject b) + { + return new OperableObject(a.Num / b.Num); + } + public static OperableObject operator /(OperableObject a, int b) + { + return new OperableObject(a.Num / b); + } + + public static OperableObject operator %(int a, OperableObject b) + { + return new OperableObject(a % b.Num); + } + public static OperableObject operator %(OperableObject a, OperableObject b) + { + return new OperableObject(a.Num % b.Num); + } + public static OperableObject operator %(OperableObject a, int b) + { + return new OperableObject(a.Num % b); + } + + public static OperableObject operator &(int a, OperableObject b) + { + return new OperableObject(a & b.Num); + } + public static OperableObject operator &(OperableObject a, OperableObject b) + { + return new OperableObject(a.Num & b.Num); + } + public static OperableObject operator &(OperableObject a, int b) + { + return new OperableObject(a.Num & b); + } + + public static OperableObject operator |(int a, OperableObject b) + { + return new OperableObject(a | b.Num); + } + public static OperableObject operator |(OperableObject a, OperableObject b) + { + return new OperableObject(a.Num | b.Num); + } + public static OperableObject operator |(OperableObject a, int b) + { + return new OperableObject(a.Num | b); + } + + public static OperableObject operator ^(int a, OperableObject b) + { + return new OperableObject(a ^ b.Num); + } + public static OperableObject operator ^(OperableObject a, OperableObject b) + { + return new OperableObject(a.Num ^ b.Num); + } + public static OperableObject operator ^(OperableObject a, int b) + { + return new OperableObject(a.Num ^ b); + } + + public static OperableObject operator <<(OperableObject a, int offset) + { + return new OperableObject(a.Num << offset); + } + + public static OperableObject operator >>(OperableObject a, int offset) + { + return new OperableObject(a.Num >> offset); + } + } + + [Test] + public void OperatorOverloads() + { + string name = string.Format("{0}.{1}", + typeof(OperableObject).DeclaringType.Name, + typeof(OperableObject).Name); + string module = MethodBase.GetCurrentMethod().DeclaringType.Namespace; + + PythonEngine.Exec($@" +from {module} import * +cls = {name} +a = cls(-2) +b = cls(10) +c = ~a +assert c.Num == ~a.Num + +c = +a +assert c.Num == +a.Num + +a = cls(2) +c = -a +assert c.Num == -a.Num + +c = a + b +assert c.Num == a.Num + b.Num + +c = a - b +assert c.Num == a.Num - b.Num + +c = a * b +assert c.Num == a.Num * b.Num + +c = a / b +assert c.Num == a.Num // b.Num + +c = a % b +assert c.Num == a.Num % b.Num + +c = a & b +assert c.Num == a.Num & b.Num + +c = a | b +assert c.Num == a.Num | b.Num + +c = a ^ b +assert c.Num == a.Num ^ b.Num +"); + } + + [Test] + public void OperatorOverloadMissingArgument() + { + string name = string.Format("{0}.{1}", + typeof(OperableObject).DeclaringType.Name, + typeof(OperableObject).Name); + string module = MethodBase.GetCurrentMethod().DeclaringType.Namespace; + + Assert.Throws(() => + PythonEngine.Exec($@" +from {module} import * +cls = {name} +a = cls(2) +b = cls(10) +a.op_Addition() +")); + } + + [Test] + public void ForwardOperatorOverloads() + { + string name = string.Format("{0}.{1}", + typeof(OperableObject).DeclaringType.Name, + typeof(OperableObject).Name); + string module = MethodBase.GetCurrentMethod().DeclaringType.Namespace; + + PythonEngine.Exec($@" +from {module} import * +cls = {name} +a = cls(2) +b = 10 +c = a + b +assert c.Num == a.Num + b + +c = a - b +assert c.Num == a.Num - b + +c = a * b +assert c.Num == a.Num * b + +c = a / b +assert c.Num == a.Num // b + +c = a % b +assert c.Num == a.Num % b + +c = a & b +assert c.Num == a.Num & b + +c = a | b +assert c.Num == a.Num | b + +c = a ^ b +assert c.Num == a.Num ^ b +"); + } + + + [Test] + public void ReverseOperatorOverloads() + { + string name = string.Format("{0}.{1}", + typeof(OperableObject).DeclaringType.Name, + typeof(OperableObject).Name); + string module = MethodBase.GetCurrentMethod().DeclaringType.Namespace; + + PythonEngine.Exec($@" +from {module} import * +cls = {name} +a = 2 +b = cls(10) + +c = a + b +assert c.Num == a + b.Num + +c = a - b +assert c.Num == a - b.Num + +c = a * b +assert c.Num == a * b.Num + +c = a / b +assert c.Num == a // b.Num + +c = a % b +assert c.Num == a % b.Num + +c = a & b +assert c.Num == a & b.Num + +c = a | b +assert c.Num == a | b.Num + +c = a ^ b +assert c.Num == a ^ b.Num +"); + + } + [Test] + public void ShiftOperatorOverloads() + { + string name = string.Format("{0}.{1}", + typeof(OperableObject).DeclaringType.Name, + typeof(OperableObject).Name); + string module = MethodBase.GetCurrentMethod().DeclaringType.Namespace; + + PythonEngine.Exec($@" +from {module} import * +cls = {name} +a = cls(2) +b = cls(10) + +c = a << b.Num +assert c.Num == a.Num << b.Num + +c = a >> b.Num +assert c.Num == a.Num >> b.Num +"); + } + } +} diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index c8bed6bc4..db4146722 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -470,6 +470,19 @@ private static ClassInfo GetClassInfo(Type type) ob = new MethodObject(type, name, mlist); ci.members[name] = ob; + if (mlist.Any(OperatorMethod.IsOperatorMethod)) + { + string pyName = OperatorMethod.GetPyMethodName(name); + string pyNameReverse = OperatorMethod.ReversePyMethodName(pyName); + MethodInfo[] forwardMethods, reverseMethods; + OperatorMethod.FilterMethods(mlist, out forwardMethods, out reverseMethods); + // Only methods where the left operand is the declaring type. + if (forwardMethods.Length > 0) + ci.members[pyName] = new MethodObject(type, name, forwardMethods); + // Only methods where only the right operand is the declaring type. + if (reverseMethods.Length > 0) + ci.members[pyNameReverse] = new MethodObject(type, name, reverseMethods); + } } if (ci.indexer == null && type.IsClass) diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index 2cf548f48..47883f0e6 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -342,20 +342,59 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth bool paramsArray; int kwargsMatched; int defaultsNeeded; - - if (!MatchesArgumentCount(pynargs, pi, kwargDict, out paramsArray, out defaultArgList, out kwargsMatched, out defaultsNeeded)) + bool isOperator = OperatorMethod.IsOperatorMethod(mi); + int clrnargs = pi.Length; + // Binary operator methods will have 2 CLR args but only one Python arg + // (unary operators will have 1 less each), since Python operator methods are bound. + isOperator = isOperator && pynargs == clrnargs - 1; + if (!MatchesArgumentCount(pynargs, pi, kwargDict, out paramsArray, out defaultArgList, out kwargsMatched, out defaultsNeeded) && !isOperator) { continue; } + // Preprocessing pi to remove either the first or second argument. + bool isReverse = isOperator && OperatorMethod.IsReverse((MethodInfo)mi); // Only cast if isOperator. + if (isOperator && !isReverse) { + // The first Python arg is the right operand, while the bound instance is the left. + // We need to skip the first (left operand) CLR argument. + pi = pi.Skip(1).ToArray(); + } + else if (isOperator && isReverse) { + // The first Python arg is the left operand. + // We need to take the first CLR argument. + pi = pi.Take(1).ToArray(); + } var outs = 0; var margs = TryConvertArguments(pi, paramsArray, args, pynargs, kwargDict, defaultArgList, - needsResolution: _methods.Length > 1, + needsResolution: _methods.Length > 1, // If there's more than one possible match. outs: out outs); - if (margs == null) { continue; } + if (isOperator) + { + if (inst != IntPtr.Zero) + { + if (ManagedType.GetManagedObject(inst) is CLRObject co) + { + bool isUnary = pynargs == 0; + // Postprocessing to extend margs. + var margsTemp = isUnary ? new object[1] : new object[2]; + // If reverse, the bound instance is the right operand. + int boundOperandIndex = isReverse ? 1 : 0; + // If reverse, the passed instance is the left operand. + int passedOperandIndex = isReverse ? 0 : 1; + margsTemp[boundOperandIndex] = co.inst; + if (!isUnary) + { + margsTemp[passedOperandIndex] = margs[0]; + } + margs = margsTemp; + } + else { break; } + } + } + var matchedMethod = new MatchedMethod(kwargsMatched, defaultsNeeded, margs, outs, mi); argMatchedMethods.Add(matchedMethod); @@ -543,6 +582,15 @@ static object[] TryConvertArguments(ParameterInfo[] pi, bool paramsArray, return margs; } + /// + /// Try to convert a Python argument object to a managed CLR type. + /// + /// Pointer to the object at a particular parameter. + /// That parameter's managed type. + /// There are multiple overloading methods that need resolution. + /// Converted argument. + /// Whether the CLR type is passed by reference. + /// static bool TryConvertArgument(IntPtr op, Type parameterType, bool needsResolution, out object arg, out bool isOut) { @@ -633,7 +681,17 @@ static Type TryComputeClrArgumentType(Type parameterType, IntPtr argument, bool return clrtype; } - + /// + /// Check whether the number of Python and .NET arguments match, and compute additional arg information. + /// + /// Number of positional args passed from Python. + /// Parameters of the specified .NET method. + /// Keyword args passed from Python. + /// True if the final param of the .NET method is an array (`params` keyword). + /// List of default values for arguments. + /// Number of kwargs from Python that are also present in the .NET method. + /// Number of non-null defaultsArgs. + /// static bool MatchesArgumentCount(int positionalArgumentCount, ParameterInfo[] parameters, Dictionary kwargDict, out bool paramsArray, @@ -644,19 +702,18 @@ static bool MatchesArgumentCount(int positionalArgumentCount, ParameterInfo[] pa defaultArgList = null; var match = false; paramsArray = parameters.Length > 0 ? Attribute.IsDefined(parameters[parameters.Length - 1], typeof(ParamArrayAttribute)) : false; - var kwargCount = kwargDict.Count; kwargsMatched = 0; defaultsNeeded = 0; - if (positionalArgumentCount == parameters.Length && kwargDict.Count == 0) { match = true; } else if (positionalArgumentCount < parameters.Length && (!paramsArray || positionalArgumentCount == parameters.Length - 1)) { - // every parameter past 'positionalArgumentCount' must have either - // a corresponding keyword argument or a default parameter match = true; + // every parameter past 'positionalArgumentCount' must have either + // a corresponding keyword arg or a default param, unless the method + // method accepts a params array (which cannot have a default value) defaultArgList = new ArrayList(); for (var v = positionalArgumentCount; v < parameters.Length; v++) { diff --git a/src/runtime/native/ITypeOffsets.cs b/src/runtime/native/ITypeOffsets.cs index 31344c66d..485c041f8 100644 --- a/src/runtime/native/ITypeOffsets.cs +++ b/src/runtime/native/ITypeOffsets.cs @@ -14,7 +14,19 @@ interface ITypeOffsets int mp_length { get; } int mp_subscript { get; } int name { get; } + int nb_positive { get; } + int nb_negative { get; } int nb_add { get; } + int nb_subtract { get; } + int nb_multiply { get; } + int nb_true_divide { get; } + int nb_and { get; } + int nb_or { get; } + int nb_xor { get; } + int nb_lshift { get; } + int nb_rshift { get; } + int nb_remainder { get; } + int nb_invert { get; } int nb_inplace_add { get; } int nb_inplace_subtract { get; } int ob_size { get; } diff --git a/src/runtime/native/TypeOffset.cs b/src/runtime/native/TypeOffset.cs index bca191565..4c1bcefa0 100644 --- a/src/runtime/native/TypeOffset.cs +++ b/src/runtime/native/TypeOffset.cs @@ -21,7 +21,19 @@ static partial class TypeOffset internal static int mp_length { get; private set; } internal static int mp_subscript { get; private set; } internal static int name { get; private set; } + internal static int nb_positive { get; private set; } + internal static int nb_negative { get; private set; } internal static int nb_add { get; private set; } + internal static int nb_subtract { get; private set; } + internal static int nb_multiply { get; private set; } + internal static int nb_true_divide { get; private set; } + internal static int nb_and { get; private set; } + internal static int nb_or { get; private set; } + internal static int nb_xor { get; private set; } + internal static int nb_lshift { get; private set; } + internal static int nb_rshift { get; private set; } + internal static int nb_remainder { get; private set; } + internal static int nb_invert { get; private set; } internal static int nb_inplace_add { get; private set; } internal static int nb_inplace_subtract { get; private set; } internal static int ob_size { get; private set; } diff --git a/src/runtime/operatormethod.cs b/src/runtime/operatormethod.cs new file mode 100644 index 000000000..1e0244510 --- /dev/null +++ b/src/runtime/operatormethod.cs @@ -0,0 +1,172 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Reflection; +using System.Runtime.InteropServices; +using System.Text; + +namespace Python.Runtime +{ + internal static class OperatorMethod + { + /// + /// Maps the compiled method name in .NET CIL (e.g. op_Addition) to + /// the equivalent Python operator (e.g. __add__) as well as the offset + /// that identifies that operator's slot (e.g. nb_add) in heap space. + /// + public static Dictionary OpMethodMap { get; private set; } + public readonly struct SlotDefinition + { + public SlotDefinition(string methodName, int typeOffset) + { + MethodName = methodName; + TypeOffset = typeOffset; + } + public string MethodName { get; } + public int TypeOffset { get; } + } + private static PyObject _opType; + + static OperatorMethod() + { + // .NET operator method names are documented at: + // https://docs.microsoft.com/en-us/dotnet/standard/design-guidelines/operator-overloads + // Python operator methods and slots are documented at: + // https://docs.python.org/3/c-api/typeobj.html + // TODO: Rich compare, inplace operator support + OpMethodMap = new Dictionary + { + ["op_Addition"] = new SlotDefinition("__add__", TypeOffset.nb_add), + ["op_Subtraction"] = new SlotDefinition("__sub__", TypeOffset.nb_subtract), + ["op_Multiply"] = new SlotDefinition("__mul__", TypeOffset.nb_multiply), + ["op_Division"] = new SlotDefinition("__truediv__", TypeOffset.nb_true_divide), + ["op_Modulus"] = new SlotDefinition("__mod__", TypeOffset.nb_remainder), + ["op_BitwiseAnd"] = new SlotDefinition("__and__", TypeOffset.nb_and), + ["op_BitwiseOr"] = new SlotDefinition("__or__", TypeOffset.nb_or), + ["op_ExclusiveOr"] = new SlotDefinition("__xor__", TypeOffset.nb_xor), + ["op_LeftShift"] = new SlotDefinition("__lshift__", TypeOffset.nb_lshift), + ["op_RightShift"] = new SlotDefinition("__rshift__", TypeOffset.nb_rshift), + ["op_OnesComplement"] = new SlotDefinition("__invert__", TypeOffset.nb_invert), + ["op_UnaryNegation"] = new SlotDefinition("__neg__", TypeOffset.nb_negative), + ["op_UnaryPlus"] = new SlotDefinition("__pos__", TypeOffset.nb_positive), + }; + } + + public static void Initialize() + { + _opType = GetOperatorType(); + } + + public static void Shutdown() + { + if (_opType != null) + { + _opType.Dispose(); + _opType = null; + } + } + + public static bool IsOperatorMethod(MethodBase method) + { + if (!method.IsSpecialName) + { + return false; + } + return OpMethodMap.ContainsKey(method.Name); + } + /// + /// For the operator methods of a CLR type, set the special slots of the + /// corresponding Python type's operator methods. + /// + /// + /// + public static void FixupSlots(IntPtr pyType, Type clrType) + { + const BindingFlags flags = BindingFlags.Public | BindingFlags.Static; + Debug.Assert(_opType != null); + foreach (var method in clrType.GetMethods(flags)) + { + if (!IsOperatorMethod(method)) + { + continue; + } + int offset = OpMethodMap[method.Name].TypeOffset; + // Copy the default implementation of e.g. the nb_add slot, + // which simply calls __add__ on the type. + IntPtr func = Marshal.ReadIntPtr(_opType.Handle, offset); + // Write the slot definition of the target Python type, so + // that we can later modify __add___ and it will be called + // when used with a Python operator. + // https://tenthousandmeters.com/blog/python-behind-the-scenes-6-how-python-object-system-works/ + Marshal.WriteIntPtr(pyType, offset, func); + + } + } + + public static string GetPyMethodName(string clrName) + { + return OpMethodMap[clrName].MethodName; + } + + private static string GenerateDummyCode() + { + StringBuilder sb = new StringBuilder(); + sb.AppendLine("class OperatorMethod(object):"); + foreach (var item in OpMethodMap.Values) + { + string def = string.Format(" def {0}(self, other): pass", item.MethodName); + sb.AppendLine(def); + } + return sb.ToString(); + } + + private static PyObject GetOperatorType() + { + using (PyDict locals = new PyDict()) + { + // A hack way for getting typeobject.c::slotdefs + string code = GenerateDummyCode(); + // The resulting OperatorMethod class is stored in a PyDict. + PythonEngine.Exec(code, null, locals.Handle); + // Return the class itself, which is a type. + return locals.GetItem("OperatorMethod"); + } + } + + public static string ReversePyMethodName(string pyName) + { + return pyName.Insert(2, "r"); + } + + /// + /// Check if the method is performing a reverse operation. + /// + /// The operator method. + /// + public static bool IsReverse(MethodInfo method) + { + Type declaringType = method.DeclaringType; + Type leftOperandType = method.GetParameters()[0].ParameterType; + return leftOperandType != declaringType; + } + + public static void FilterMethods(MethodInfo[] methods, out MethodInfo[] forwardMethods, out MethodInfo[] reverseMethods) + { + List forwardMethodsList = new List(); + List reverseMethodsList = new List(); + foreach (var method in methods) + { + if (IsReverse(method)) + { + reverseMethodsList.Add(method); + } else + { + forwardMethodsList.Add(method); + } + + } + forwardMethods = forwardMethodsList.ToArray(); + reverseMethods = reverseMethodsList.ToArray(); + } + } +} diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index f80db04b6..1e8db8278 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -160,6 +160,7 @@ internal static void Initialize(bool initSigs = false, ShutdownMode mode = Shutd // Initialize modules that depend on the runtime class. AssemblyManager.Initialize(); + OperatorMethod.Initialize(); if (mode == ShutdownMode.Reload && RuntimeData.HasStashData()) { RuntimeData.RestoreRuntimeData(); @@ -345,6 +346,7 @@ internal static void Shutdown(ShutdownMode mode) RuntimeData.Stash(); } AssemblyManager.Shutdown(); + OperatorMethod.Shutdown(); ImportHook.Shutdown(); ClearClrModules(); diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 49a46cb72..31682c519 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -273,6 +273,7 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) | TypeFlags.HaveGC; Util.WriteCLong(type, TypeOffset.tp_flags, flags); + OperatorMethod.FixupSlots(type, clrType); // Leverage followup initialization from the Python runtime. Note // that the type of the new type must PyType_Type at the time we // call this, else PyType_Ready will skip some slot initialization. pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy