diff --git a/.github/workflows/ARM.yml b/.github/workflows/ARM.yml index af257bcb8..eef0e666d 100644 --- a/.github/workflows/ARM.yml +++ b/.github/workflows/ARM.yml @@ -27,25 +27,25 @@ jobs: - name: Install dependencies run: | - pip install --upgrade -r requirements.txt - pip install pytest numpy # for tests + pip3.8 install -r requirements.txt + pip3.8 install pytest numpy # for tests - name: Build and Install run: | - pip install -v . + pip3.8 install -v . - name: Set Python DLL path (non Windows) run: | - echo PYTHONNET_PYDLL=$(python -m find_libpython) >> $GITHUB_ENV + echo PYTHONNET_PYDLL=$(python3.8 -m find_libpython) >> $GITHUB_ENV - name: Embedding tests run: dotnet test --logger "console;verbosity=detailed" src/embed_tests/ - name: Python Tests (Mono) - run: python -m pytest --runtime mono + run: python3.8 -m pytest --runtime mono - name: Python Tests (.NET Core) - run: python -m pytest --runtime coreclr + run: python3.8 -m pytest --runtime coreclr - name: Python tests run from .NET run: dotnet test src/python_tests_runner/ diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 5b782c8b4..30163cd14 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -6,9 +6,9 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v4 - name: Doxygen Action - uses: mattnotmitt/doxygen-action@1.9.4 + uses: mattnotmitt/doxygen-action@1.12.0 with: working-directory: "doc/" @@ -19,7 +19,7 @@ jobs: - name: Upload artifact # Automatically uploads an artifact from the './_site' directory by default - uses: actions/upload-pages-artifact@v1 + uses: actions/upload-pages-artifact@v3 with: path: doc/build/html/ @@ -37,4 +37,4 @@ jobs: steps: - name: Deploy to GitHub Pages id: deployment - uses: actions/deploy-pages@v1 + uses: actions/deploy-pages@v4 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 93963c70a..8485189e1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -9,25 +9,43 @@ on: jobs: build-test: name: Build and Test - runs-on: ${{ matrix.os }}-latest + runs-on: ${{ matrix.os.instance }} timeout-minutes: 15 strategy: fail-fast: false matrix: - os: [windows, ubuntu, macos] - python: ["3.7", "3.8", "3.9", "3.10", "3.11"] - platform: [x64, x86] - exclude: - - os: ubuntu - platform: x86 - - os: macos + os: + - category: windows platform: x86 + instance: windows-latest + + - category: windows + platform: x64 + instance: windows-latest + + - category: ubuntu + platform: x64 + instance: ubuntu-22.04 + + - category: macos + platform: x64 + instance: macos-13 + + python: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] + + # This fails in pytest with: + # CSC : error CS4023: /platform:anycpu32bitpreferred can only be used with /t:exe, /t:winexe and /t:appcontainerexe [D:\a\pythonnet\pythonnet\src\runtime\Python.Runtime.csproj] + exclude: + - os: + category: windows + platform: x86 + python: "3.13" steps: - name: Set Environment on macOS uses: maxim-lobanov/setup-xamarin@v1 - if: ${{ matrix.os == 'macos' }} + if: ${{ matrix.os.category == 'macos' }} with: mono-version: latest @@ -43,7 +61,7 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python }} - architecture: ${{ matrix.platform }} + architecture: ${{ matrix.os.platform }} - name: Install dependencies run: | @@ -55,42 +73,44 @@ jobs: pip install -v . - name: Set Python DLL path and PYTHONHOME (non Windows) - if: ${{ matrix.os != 'windows' }} + if: ${{ matrix.os.category != 'windows' }} run: | echo PYTHONNET_PYDLL=$(python -m find_libpython) >> $GITHUB_ENV echo PYTHONHOME=$(python -c 'import sys; print(sys.prefix)') >> $GITHUB_ENV - name: Set Python DLL path and PYTHONHOME (Windows) - if: ${{ matrix.os == 'windows' }} + if: ${{ matrix.os.category == 'windows' }} run: | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append -InputObject "PYTHONNET_PYDLL=$(python -m find_libpython)" Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append -InputObject "PYTHONHOME=$(python -c 'import sys; print(sys.prefix)')" - name: Embedding tests - run: dotnet test --runtime any-${{ matrix.platform }} --logger "console;verbosity=detailed" src/embed_tests/ + if: ${{ matrix.python != '3.13' }} + run: dotnet test --runtime any-${{ matrix.os.platform }} --logger "console;verbosity=detailed" src/embed_tests/ env: MONO_THREADS_SUSPEND: preemptive # https://github.com/mono/mono/issues/21466 - name: Python Tests (Mono) - if: ${{ matrix.os != 'windows' }} + if: ${{ matrix.os.category != 'windows' }} run: pytest --runtime mono # TODO: Run these tests on Windows x86 - name: Python Tests (.NET Core) - if: ${{ matrix.platform == 'x64' }} + if: ${{ matrix.os.platform == 'x64' }} run: pytest --runtime coreclr - name: Python Tests (.NET Framework) - if: ${{ matrix.os == 'windows' }} + if: ${{ matrix.os.category == 'windows' }} run: pytest --runtime netfx - name: Python tests run from .NET - run: dotnet test --runtime any-${{ matrix.platform }} src/python_tests_runner/ + if: ${{ matrix.python != '3.13' }} + run: dotnet test --runtime any-${{ matrix.os.platform }} src/python_tests_runner/ - name: Perf tests - if: ${{ (matrix.python == '3.8') && (matrix.platform == 'x64') }} + if: ${{ (matrix.python == '3.8') && (matrix.os.platform == 'x64') }} run: | pip install --force --no-deps --target src/perf_tests/baseline/ pythonnet==2.5.2 - dotnet test --configuration Release --runtime any-${{ matrix.platform }} --logger "console;verbosity=detailed" src/perf_tests/ + dotnet test --configuration Release --runtime any-${{ matrix.os.platform }} --logger "console;verbosity=detailed" src/perf_tests/ # TODO: Run mono tests on Windows? diff --git a/AUTHORS.md b/AUTHORS.md index 92f1a4a97..7ea639059 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -38,6 +38,7 @@ - Dmitriy Se ([@dmitriyse](https://github.com/dmitriyse)) - Félix Bourbonnais ([@BadSingleton](https://github.com/BadSingleton)) - Florian Treurniet ([@ftreurni](https://github.com/ftreurni)) +- Frank Witscher ([@Frawak](https://github.com/Frawak)) - He-chien Tsai ([@t3476](https://github.com/t3476)) - Inna Wiesel ([@inna-w](https://github.com/inna-w)) - Ivan Cronyn ([@cronan](https://github.com/cronan)) @@ -58,6 +59,7 @@ - Peter Kese ([@pkese](https://github.com/pkese)) - Raphael Nestler ([@rnestler](https://github.com/rnestler)) - Rickard Holmberg ([@rickardraysearch](https://github.com/rickardraysearch)) +- Roberto Pastor Muela ([@RobPasMue](https://github.com/RobPasMue)) - Sam Winstanley ([@swinstanley](https://github.com/swinstanley)) - Sean Freitag ([@cowboygneox](https://github.com/cowboygneox)) - Serge Weinstock ([@sweinst](https://github.com/sweinst)) @@ -85,3 +87,5 @@ - ([@alxnull](https://github.com/alxnull)) - ([@gpetrou](https://github.com/gpetrou)) - Ehsan Iran-Nejad ([@eirannejad](https://github.com/eirannejad)) +- ([@legomanww](https://github.com/legomanww)) +- ([@gertdreyer](https://github.com/gertdreyer)) diff --git a/CHANGELOG.md b/CHANGELOG.md index 13bf09c2c..df68fbb39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,14 +5,71 @@ project adheres to [Semantic Versioning][]. This document follows the conventions laid out in [Keep a CHANGELOG][]. -## [Unreleased][] +## Unreleased ### Added +- Support `del obj[...]` for types derived from `IList` and `IDictionary` + +### Changed +### Fixed + +- Fixed crash when trying to `del clrObj[...]` for non-arrays +- ci: properly exclude job (#2542) + +## [3.0.5](https://github.com/pythonnet/pythonnet/releases/tag/v3.0.5) - 2024-12-13 + +### Added + +- Support for Python 3.13 (#2454) + + +## [3.0.4](https://github.com/pythonnet/pythonnet/releases/tag/v3.0.4) - 2024-09-19 + +### Added + +- Added `ToPythonAs()` extension method to allow for explicit conversion + using a specific type. ([#2311][i2311]) +- Added `IComparable` and `IEquatable` implementations to `PyInt`, `PyFloat`, + and `PyString` to compare with primitive .NET types like `long`. + ### Changed +- Added a `FormatterFactory` member in RuntimeData to create formatters with + parameters. For compatibility, the `FormatterType` member is still present + and has precedence when defining both `FormatterFactory` and `FormatterType` +- Added a post-serialization and a pre-deserialization step callbacks to + extend (de)serialization process +- Added an API to stash serialized data on Python capsules + ### Fixed +- Fixed RecursionError for reverse operators on C# operable types from python. See #2240 +- Fixed crash when .NET event has no `AddMethod` +- Fixed probing for assemblies in `sys.path` failing when a path in `sys.path` + has invalid characters. See #2376 +- Fixed possible access violation exception on shutdown. See ([#1977][i1977]) + +## [3.0.3](https://github.com/pythonnet/pythonnet/releases/tag/v3.0.3) - 2023-10-11 + +### Added + +- Support for Python 3.12 + +### Changed + +- Use enum name in `repr` + +## [3.0.2](https://github.com/pythonnet/pythonnet/releases/tag/v3.0.2) - 2023-08-29 + +### Fixed + +- Fixed error occuring when inheriting a class containing a virtual generic method +- Make a second call to `pythonnet.load` a no-op, as it was intended +- Added support for multiple inheritance when inheriting from a class and/or multiple interfaces +- Fixed error occuring when calling `GetBuffer` for anything other than `PyBUF.SIMPLE` +- Bumped `clr_loader` dependency to incorporate patches + ## [3.0.1](https://github.com/pythonnet/pythonnet/releases/tag/v3.0.1) - 2022-11-03 ### Added @@ -821,7 +878,7 @@ This version improves performance on benchmarks significantly compared to 2.3. [semantic versioning]: http://semver.org/ -[unreleased]: ../../compare/v2.3.0...HEAD +[unreleased]: ../../compare/v3.0.1...HEAD [2.3.0]: ../../compare/v2.2.2...v2.3.0 @@ -938,3 +995,5 @@ This version improves performance on benchmarks significantly compared to 2.3. [i238]: https://github.com/pythonnet/pythonnet/issues/238 [i1481]: https://github.com/pythonnet/pythonnet/issues/1481 [i1672]: https://github.com/pythonnet/pythonnet/pull/1672 +[i2311]: https://github.com/pythonnet/pythonnet/issues/2311 +[i1977]: https://github.com/pythonnet/pythonnet/issues/1977 diff --git a/doc/requirements.txt b/doc/requirements.txt index 64018840c..8ef3b7159 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -7,3 +7,6 @@ pygments>=2.7 # C# via doxygen breathe git+https://github.com/rogerbarton/sphinx-csharp.git + +# Dependency of pythonnet, needed for autodocs +clr_loader diff --git a/doc/source/dotnet.rst b/doc/source/dotnet.rst index d02a6f0cb..ed2b741a7 100644 --- a/doc/source/dotnet.rst +++ b/doc/source/dotnet.rst @@ -17,7 +17,7 @@ to: - Reference ``Python.Runtime.dll`` (e.g. via a ``PackageReference``) - Call ``PythonEngine.Initialize()`` to initialize Python -- Call ``PythonEngine.ImportModule(name)`` to import a module +- Call ``var mod = PyModule.Import(name)`` to import a module as ``mod`` The module you import can either start working with your managed app environment at the time its imported, or you can explicitly lookup and @@ -42,16 +42,10 @@ application. Before interacting with any of the objects or APIs provided by the ``Python.Runtime`` namespace, calling code must have acquired the Python -global interpreter lock by calling the ``PythonEngine.AcquireLock`` -method. The only exception to this rule is the -``PythonEngine.Initialize`` method, which may be called at startup -without having acquired the GIL. - -When finished using Python APIs, managed code must call a corresponding -``PythonEngine.ReleaseLock`` to release the GIL and allow other threads -to use Python. - -A ``using`` statement may be used to acquire and release the GIL: +global interpreter lock by ``using'' ``Py.GIL()``. The only exception to +this rule is the ``PythonEngine.Initialize`` method, which may be called +at startup without having acquired the GIL. The GIL is released again +by disposing the return value of `Py.GIL()`: .. code:: csharp @@ -59,11 +53,28 @@ A ``using`` statement may be used to acquire and release the GIL: { PythonEngine.Exec("doStuff()"); } + + // or + { + using var _ = Py.GIL() + PythonEngine.Exec("doStuff()"); + } + + // or + var gil = Py.GIL(); + try + { + PythonEngine.Exec("doStuff()"); + } + finally + { + gil.Dispose(); + } -The AcquireLock and ReleaseLock methods are thin wrappers over the -unmanaged ``PyGILState_Ensure`` and ``PyGILState_Release`` functions -from the Python API, and the documentation for those APIs applies to the -managed versions. +The ``Py.GIL()'' object is a thin wrapper over the unmanaged +``PyGILState_Ensure`` (on construction) and ``PyGILState_Release`` (on +disposal) functions from the Python API, and the documentation for those +APIs applies to the managed versions. Passing C# Objects to the Python Engine --------------------------------------- @@ -99,7 +110,7 @@ Code executed from the scope will have access to the variable: using (Py.GIL()) { // create a Python scope - using (PyScope scope = Py.CreateScope()) + using (PyModule scope = Py.CreateScope()) { // convert the Person object to a PyObject PyObject pyPerson = person.ToPython(); diff --git a/pyproject.toml b/pyproject.toml index 52f1adb18..968998e8d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,10 +10,10 @@ license = {text = "MIT"} readme = "README.rst" dependencies = [ - "clr_loader>=0.2.2,<0.3.0" + "clr_loader>=0.2.7,<0.3.0" ] -requires-python = ">=3.7, <3.12" +requires-python = ">=3.7, <3.14" classifiers = [ "Development Status :: 5 - Production/Stable", @@ -26,6 +26,8 @@ classifiers = [ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Operating System :: Microsoft :: Windows", "Operating System :: POSIX :: Linux", "Operating System :: MacOS :: MacOS X", @@ -33,6 +35,15 @@ classifiers = [ dynamic = ["version"] +[dependency-groups] +dev = [ + "pytest >= 6", + "find_libpython >= 0.3.0", + "numpy >=2 ; python_version >= '3.10'", + "numpy <2 ; python_version < '3.10'", + "psutil" +] + [[project.authors]] name = "The Contributors of the Python.NET Project" email = "pythonnet@python.org" @@ -44,12 +55,14 @@ Sources = "https://github.com/pythonnet/pythonnet" [tool.setuptools] zip-safe = false py-modules = ["clr"] +license-files = [] [tool.setuptools.dynamic.version] file = "version.txt" [tool.setuptools.packages.find] include = ["pythonnet*"] +exclude = [".gitignore"] [tool.pytest.ini_options] xfail_strict = true diff --git a/pythonnet.sln b/pythonnet.sln index d1a47892e..cf684c0e1 100644 --- a/pythonnet.sln +++ b/pythonnet.sln @@ -4,8 +4,6 @@ VisualStudioVersion = 17.0.31912.275 MinimumVisualStudioVersion = 15.0.26124.0 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.Runtime", "src\runtime\Python.Runtime.csproj", "{4E8C8FE2-0FB8-4517-B2D9-5FB2D5FC849B}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Console", "src\console\Console.csproj", "{E6B01706-00BA-4144-9029-186AC42FBE9A}" -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.EmbeddingTest", "src\embed_tests\Python.EmbeddingTest.csproj", "{819E089B-4770-400E-93C6-4F7A35F0EA12}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.Test", "src\testing\Python.Test.csproj", "{14EF9518-5BB7-4F83-8686-015BD2CC788E}" @@ -22,6 +20,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Repo", "Repo", "{441A0123-F LICENSE = LICENSE README.rst = README.rst version.txt = version.txt + shell.nix = shell.nix EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "CI", "CI", "{D301657F-5EAF-4534-B280-B858D651B2E5}" diff --git a/pythonnet/__init__.py b/pythonnet/__init__.py index 9d7b6c20a..5c1ca108a 100644 --- a/pythonnet/__init__.py +++ b/pythonnet/__init__.py @@ -120,7 +120,9 @@ def load(runtime: Union[clr_loader.Runtime, str, None] = None, **params: str) -> The same parameters as for `set_runtime` can be used. By default, `set_default_runtime` is called if no environment has been set yet and no - parameters are passed.""" + parameters are passed. + + After a successful call, further invocations will return immediately.""" global _LOADED, _LOADER_ASSEMBLY if _LOADED: @@ -142,6 +144,8 @@ def load(runtime: Union[clr_loader.Runtime, str, None] = None, **params: str) -> if func(b"") != 0: raise RuntimeError("Failed to initialize Python.Runtime.dll") + + _LOADED = True import atexit diff --git a/requirements.txt b/requirements.txt index 8e911ef5a..33f38cca7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,8 +8,7 @@ codecov wheel pycparser -setuptools -clr-loader +clr-loader==0.2.* # Discover libpython -find_libpython \ No newline at end of file +find_libpython==0.3.* diff --git a/shell.nix b/shell.nix new file mode 100644 index 000000000..fe653deb7 --- /dev/null +++ b/shell.nix @@ -0,0 +1,10 @@ +{ pkgs ? import {}}: +let + fhs = pkgs.buildFHSUserEnv { + name = "my-fhs-environment"; + + targetPkgs = _: [ + pkgs.python3 + ]; + }; +in fhs.env diff --git a/src/console/Console.csproj b/src/console/Console.csproj deleted file mode 100644 index bcbc1292b..000000000 --- a/src/console/Console.csproj +++ /dev/null @@ -1,27 +0,0 @@ - - - net472;net6.0 - x64;x86 - Exe - nPython - Python.Runtime - nPython - python-clear.ico - - - - - - Python.Runtime.dll - - - - - - - - runtime; build; native; contentfiles; analyzers; buildtransitive - all - - - diff --git a/src/console/python-clear.ico b/src/console/python-clear.ico deleted file mode 100644 index b37050cce..000000000 Binary files a/src/console/python-clear.ico and /dev/null differ diff --git a/src/console/python-clear.png b/src/console/python-clear.png deleted file mode 100644 index d67b5b8d4..000000000 Binary files a/src/console/python-clear.png and /dev/null differ diff --git a/src/console/pythonconsole.cs b/src/console/pythonconsole.cs deleted file mode 100644 index bf17848f7..000000000 --- a/src/console/pythonconsole.cs +++ /dev/null @@ -1,82 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Reflection; -using Python.Runtime; - -namespace Python.Runtime -{ - /// - /// Example of Embedding Python inside of a .NET program. - /// - /// - /// It has similar functionality to doing `import clr` from within Python, but this does it - /// the other way around; That is, it loads Python inside of .NET program. - /// See https://github.com/pythonnet/pythonnet/issues/358 for more info. - /// - public sealed class PythonConsole - { -#if NET40 - private static AssemblyLoader assemblyLoader = new AssemblyLoader(); -#endif - private PythonConsole() - { - } - - [STAThread] - public static int Main(string[] args) - { - // Only .NET Framework is capable to safely inject python.runtime.dll into resources. -#if NET40 - // reference the static assemblyLoader to stop it being optimized away - AssemblyLoader a = assemblyLoader; -#endif - string[] cmd = Environment.GetCommandLineArgs(); - PythonEngine.Initialize(); - - int i = Runtime.Py_Main(cmd.Length, cmd); - PythonEngine.Shutdown(); - - return i; - } - -#if NET40 - // Register a callback function to load embedded assemblies. - // (Python.Runtime.dll is included as a resource) - private sealed class AssemblyLoader - { - private Dictionary loadedAssemblies; - - public AssemblyLoader() - { - loadedAssemblies = new Dictionary(); - - AppDomain.CurrentDomain.AssemblyResolve += (sender, args) => - { - string shortName = args.Name.Split(',')[0]; - string resourceName = $"{shortName}.dll"; - - if (loadedAssemblies.ContainsKey(resourceName)) - { - return loadedAssemblies[resourceName]; - } - - // looks for the assembly from the resources and load it - using (Stream stream = Assembly.GetExecutingAssembly().GetManifestResourceStream(resourceName)) - { - if (stream != null) - { - var assemblyData = new byte[stream.Length]; - stream.Read(assemblyData, 0, assemblyData.Length); - Assembly assembly = Assembly.Load(assemblyData); - loadedAssemblies[resourceName] = assembly; - return assembly; - } - } - return null; - }; - } - } -#endif - } -} diff --git a/src/embed_tests/Codecs.cs b/src/embed_tests/Codecs.cs index 9b764d43f..c8b8ecb6e 100644 --- a/src/embed_tests/Codecs.cs +++ b/src/embed_tests/Codecs.cs @@ -371,11 +371,23 @@ public void FloatDerivedDecoded() [Test] public void ExceptionDecodedNoInstance() { - PyObjectConversions.RegisterDecoder(new InstancelessExceptionDecoder()); - using var scope = Py.CreateScope(); - var error = Assert.Throws(() => PythonEngine.Exec( - $"[].__iter__().__next__()")); - Assert.AreEqual(TestExceptionMessage, error.Message); + if (Runtime.PyVersion < new Version(3, 12)) + { + PyObjectConversions.RegisterDecoder(new InstancelessExceptionDecoder()); + using var scope = Py.CreateScope(); + + var error = Assert.Throws(() => + PythonEngine.Exec($"[].__iter__().__next__()") + ); + Assert.AreEqual(TestExceptionMessage, error.Message); + } + else + { + Assert.Ignore( + "This test does not work for Python 3.12, see " + + "https://github.com/python/cpython/issues/101578" + ); + } } public static void AcceptsDateTime(DateTime v) {} diff --git a/src/embed_tests/Modules.cs b/src/embed_tests/Modules.cs index a88ab8552..6cab4dd07 100644 --- a/src/embed_tests/Modules.cs +++ b/src/embed_tests/Modules.cs @@ -51,7 +51,7 @@ public void TestEval() ps.Set("a", 1); var result = ps.Eval("a + 2"); Assert.AreEqual(3, result); - } + } } /// @@ -169,6 +169,62 @@ public void TestScopeClass() } } + /// + /// Create a class in the scope, the class can read variables in the scope. + /// Its methods can write the variables with the help of 'global' keyword. + /// + [Test] + public void TestCreateVirtualPackageStructure() + { + using (Py.GIL()) + { + using var _p1 = PyModule.FromString("test", ""); + // Sub-module + using var _p2 = PyModule.FromString("test.scope", + "class Class1():\n" + + " def __init__(self, value):\n" + + " self.value = value\n" + + " def call(self, arg):\n" + + " return self.value + bb + arg\n" + // use scope variables + " def update(self, arg):\n" + + " global bb\n" + + " bb = self.value + arg\n", // update scope variable + "test" + ); + + dynamic ps2 = Py.Import("test.scope"); + ps2.bb = 100; + + dynamic obj1 = ps2.Class1(20); + var result = obj1.call(10).As(); + Assert.AreEqual(130, result); + + obj1.update(10); + result = ps2.Get("bb"); + Assert.AreEqual(30, result); + } + } + + /// + /// Test setting the file attribute via a FromString parameter + /// + [Test] + public void TestCreateModuleWithFilename() + { + using var _gil = Py.GIL(); + + using var mod = PyModule.FromString("mod", ""); + using var modWithoutName = PyModule.FromString("mod_without_name", "", " "); + using var modNullName = PyModule.FromString("mod_null_name", "", null); + + using var modWithName = PyModule.FromString("mod_with_name", "", "some_filename"); + + Assert.AreEqual("none", mod.Get("__file__")); + Assert.AreEqual("none", modWithoutName.Get("__file__")); + Assert.AreEqual("none", modNullName.Get("__file__")); + Assert.AreEqual("some_filename", modWithName.Get("__file__")); + } + /// /// Import a python module into the session. /// Equivalent to the Python "import" statement. @@ -194,7 +250,7 @@ public void TestImportModule() } /// - /// Create a scope and import variables from a scope, + /// Create a scope and import variables from a scope, /// exec Python statements in the scope then discard it. /// [Test] @@ -218,7 +274,7 @@ public void TestImportScope() } /// - /// Create a scope and import variables from a scope, + /// Create a scope and import variables from a scope, /// exec Python statements in the scope then discard it. /// [Test] @@ -241,7 +297,7 @@ public void TestImportAllFromScope() } /// - /// Create a scope and import variables from a scope, + /// Create a scope and import variables from a scope, /// call the function imported. /// [Test] @@ -286,7 +342,7 @@ public void TestImportScopeFunction() public void TestVariables() { using (Py.GIL()) - { + { (ps.Variables() as dynamic)["ee"] = new PyInt(200); var a0 = ps.Get("ee"); Assert.AreEqual(200, a0); @@ -326,8 +382,8 @@ public void TestThread() _ps.res = 0; _ps.bb = 100; _ps.th_cnt = 0; - //add function to the scope - //can be call many times, more efficient than ast + //add function to the scope + //can be call many times, more efficient than ast ps.Exec( "import threading\n"+ "lock = threading.Lock()\n"+ diff --git a/src/embed_tests/TestConverter.cs b/src/embed_tests/TestConverter.cs index 0686d528b..a59b9c97b 100644 --- a/src/embed_tests/TestConverter.cs +++ b/src/embed_tests/TestConverter.cs @@ -185,6 +185,15 @@ public void RawPyObjectProxy() Assert.AreEqual(pyObject.DangerousGetAddressOrNull(), proxiedHandle); } + [Test] + public void GenericToPython() + { + int i = 42; + var pyObject = i.ToPythonAs(); + var type = pyObject.GetPythonType(); + Assert.AreEqual(nameof(IConvertible), type.Name); + } + // regression for https://github.com/pythonnet/pythonnet/issues/451 [Test] public void CanGetListFromDerivedClass() diff --git a/src/embed_tests/TestOperator.cs b/src/embed_tests/TestOperator.cs index a5713274a..6bfb81bdb 100644 --- a/src/embed_tests/TestOperator.cs +++ b/src/embed_tests/TestOperator.cs @@ -15,6 +15,7 @@ public class TestOperator public void SetUp() { PythonEngine.Initialize(); + OwnIntCodec.Setup(); } [OneTimeTearDown] @@ -23,6 +24,125 @@ public void Dispose() PythonEngine.Shutdown(); } + // Mock Integer class to test math ops on non-native dotnet types + public readonly struct OwnInt + { + private readonly int _value; + + public int Num => _value; + + public OwnInt() + { + _value = 0; + } + + public OwnInt(int value) + { + _value = value; + } + + public override int GetHashCode() + { + return unchecked(65535 + _value.GetHashCode()); + } + + public override bool Equals(object obj) + { + return obj is OwnInt @object && + Num == @object.Num; + } + + public static OwnInt operator -(OwnInt p1, OwnInt p2) + { + return new OwnInt(p1._value - p2._value); + } + + public static OwnInt operator +(OwnInt p1, OwnInt p2) + { + return new OwnInt(p1._value + p2._value); + } + + public static OwnInt operator *(OwnInt p1, OwnInt p2) + { + return new OwnInt(p1._value * p2._value); + } + + public static OwnInt operator /(OwnInt p1, OwnInt p2) + { + return new OwnInt(p1._value / p2._value); + } + + public static OwnInt operator %(OwnInt p1, OwnInt p2) + { + return new OwnInt(p1._value % p2._value); + } + + public static OwnInt operator ^(OwnInt p1, OwnInt p2) + { + return new OwnInt(p1._value ^ p2._value); + } + + public static bool operator <(OwnInt p1, OwnInt p2) + { + return p1._value < p2._value; + } + + public static bool operator >(OwnInt p1, OwnInt p2) + { + return p1._value > p2._value; + } + + public static bool operator ==(OwnInt p1, OwnInt p2) + { + return p1._value == p2._value; + } + + public static bool operator !=(OwnInt p1, OwnInt p2) + { + return p1._value != p2._value; + } + + public static OwnInt operator |(OwnInt p1, OwnInt p2) + { + return new OwnInt(p1._value | p2._value); + } + + public static OwnInt operator &(OwnInt p1, OwnInt p2) + { + return new OwnInt(p1._value & p2._value); + } + + public static bool operator <=(OwnInt p1, OwnInt p2) + { + return p1._value <= p2._value; + } + + public static bool operator >=(OwnInt p1, OwnInt p2) + { + return p1._value >= p2._value; + } + } + + // Codec for mock class above. + public class OwnIntCodec : IPyObjectDecoder + { + public static void Setup() + { + PyObjectConversions.RegisterDecoder(new OwnIntCodec()); + } + + public bool CanDecode(PyType objectType, Type targetType) + { + return objectType.Name == "int" && targetType == typeof(OwnInt); + } + + public bool TryDecode(PyObject pyObj, out T value) + { + value = (T)(object)new OwnInt(pyObj.As()); + return true; + } + } + public class OperableObject { public int Num { get; set; } @@ -524,6 +644,121 @@ public void ShiftOperatorOverloads() c = a >> b.Num assert c.Num == a.Num >> b.Num +"); + } + + [Test] + public void ReverseOperatorWithCodec() + { + string name = string.Format("{0}.{1}", + typeof(OwnInt).DeclaringType.Name, + typeof(OwnInt).Name); + string module = MethodBase.GetCurrentMethod().DeclaringType.Namespace; + + PythonEngine.Exec($@" +from {module} import * +cls = {name} +a = 2 +b = cls(10) + +c = a + b +assert c.Num == a + b.Num + +c = a - b +assert c.Num == a - b.Num + +c = a * b +assert c.Num == a * b.Num + +c = a / b +assert c.Num == a // b.Num + +c = a % b +assert c.Num == a % b.Num + +c = a & b +assert c.Num == a & b.Num + +c = a | b +assert c.Num == a | b.Num + +c = a ^ b +assert c.Num == a ^ b.Num + +c = a == b +assert c == (a == b.Num) + +c = a != b +assert c == (a != b.Num) + +c = a <= b +assert c == (a <= b.Num) + +c = a >= b +assert c == (a >= b.Num) + +c = a < b +assert c == (a < b.Num) + +c = a > b +assert c == (a > b.Num) +"); + } + + [Test] + public void ForwardOperatorWithCodec() + { + string name = string.Format("{0}.{1}", + typeof(OwnInt).DeclaringType.Name, + typeof(OwnInt).Name); + string module = MethodBase.GetCurrentMethod().DeclaringType.Namespace; + + PythonEngine.Exec($@" +from {module} import * +cls = {name} +a = cls(2) +b = 10 +c = a + b +assert c.Num == a.Num + b + +c = a - b +assert c.Num == a.Num - b + +c = a * b +assert c.Num == a.Num * b + +c = a / b +assert c.Num == a.Num // b + +c = a % b +assert c.Num == a.Num % b + +c = a & b +assert c.Num == a.Num & b + +c = a | b +assert c.Num == a.Num | b + +c = a ^ b +assert c.Num == a.Num ^ b + +c = a == b +assert c == (a.Num == b) + +c = a != b +assert c == (a.Num != b) + +c = a <= b +assert c == (a.Num <= b) + +c = a >= b +assert c == (a.Num >= b) + +c = a < b +assert c == (a.Num < b) + +c = a > b +assert c == (a.Num > b) "); } } diff --git a/src/embed_tests/TestPyBuffer.cs b/src/embed_tests/TestPyBuffer.cs index a1bcc161d..1b4e28d12 100644 --- a/src/embed_tests/TestPyBuffer.cs +++ b/src/embed_tests/TestPyBuffer.cs @@ -110,6 +110,26 @@ public void Finalization() Assert.AreEqual(1, arr.Refcount); } + [Test] + public void MultidimensionalNumPyArray() + { + var ndarray = np.arange(24).reshape(1,2,3,4).T; + PyObject ndim = ndarray.ndim; + PyObject shape = ndarray.shape; + PyObject strides = ndarray.strides; + PyObject contiguous = ndarray.flags["C_CONTIGUOUS"]; + + using PyBuffer buf = ndarray.GetBuffer(PyBUF.STRIDED); + + Assert.Multiple(() => + { + Assert.That(buf.Dimensions, Is.EqualTo(ndim.As())); + Assert.That(buf.Shape, Is.EqualTo(shape.As())); + Assert.That(buf.Strides, Is.EqualTo(strides.As())); + Assert.That(buf.IsContiguous(BufferOrderStyle.C), Is.EqualTo(contiguous.As())); + }); + } + [MethodImpl(MethodImplOptions.NoInlining)] static void MakeBufAndLeak(PyObject bufProvider) { @@ -121,5 +141,21 @@ static PyObject ByteArrayFromAsciiString(string str) using var scope = Py.CreateScope(); return Runtime.Runtime.PyByteArray_FromStringAndSize(str).MoveToPyObject(); } + + dynamic np + { + get + { + try + { + return Py.Import("numpy"); + } + catch (PythonException) + { + Assert.Inconclusive("Numpy or dependency not installed"); + return null; + } + } + } } } diff --git a/src/embed_tests/TestPyFloat.cs b/src/embed_tests/TestPyFloat.cs index 36531cb6a..89e29e5fd 100644 --- a/src/embed_tests/TestPyFloat.cs +++ b/src/embed_tests/TestPyFloat.cs @@ -126,5 +126,32 @@ public void AsFloatBad() StringAssert.StartsWith("could not convert string to float", ex.Message); Assert.IsNull(a); } + + [Test] + public void CompareTo() + { + var v = new PyFloat(42); + + Assert.AreEqual(0, v.CompareTo(42f)); + Assert.AreEqual(0, v.CompareTo(42d)); + + Assert.AreEqual(1, v.CompareTo(41f)); + Assert.AreEqual(1, v.CompareTo(41d)); + + Assert.AreEqual(-1, v.CompareTo(43f)); + Assert.AreEqual(-1, v.CompareTo(43d)); + } + + [Test] + public void Equals() + { + var v = new PyFloat(42); + + Assert.IsTrue(v.Equals(42f)); + Assert.IsTrue(v.Equals(42d)); + + Assert.IsFalse(v.Equals(41f)); + Assert.IsFalse(v.Equals(41d)); + } } } diff --git a/src/embed_tests/TestPyInt.cs b/src/embed_tests/TestPyInt.cs index c147e074b..d2767e664 100644 --- a/src/embed_tests/TestPyInt.cs +++ b/src/embed_tests/TestPyInt.cs @@ -210,6 +210,76 @@ public void ToBigInteger() CollectionAssert.AreEqual(expected, actual); } + [Test] + public void CompareTo() + { + var v = new PyInt(42); + + #region Signed + Assert.AreEqual(0, v.CompareTo(42L)); + Assert.AreEqual(0, v.CompareTo(42)); + Assert.AreEqual(0, v.CompareTo((short)42)); + Assert.AreEqual(0, v.CompareTo((sbyte)42)); + + Assert.AreEqual(1, v.CompareTo(41L)); + Assert.AreEqual(1, v.CompareTo(41)); + Assert.AreEqual(1, v.CompareTo((short)41)); + Assert.AreEqual(1, v.CompareTo((sbyte)41)); + + Assert.AreEqual(-1, v.CompareTo(43L)); + Assert.AreEqual(-1, v.CompareTo(43)); + Assert.AreEqual(-1, v.CompareTo((short)43)); + Assert.AreEqual(-1, v.CompareTo((sbyte)43)); + #endregion Signed + + #region Unsigned + Assert.AreEqual(0, v.CompareTo(42UL)); + Assert.AreEqual(0, v.CompareTo(42U)); + Assert.AreEqual(0, v.CompareTo((ushort)42)); + Assert.AreEqual(0, v.CompareTo((byte)42)); + + Assert.AreEqual(1, v.CompareTo(41UL)); + Assert.AreEqual(1, v.CompareTo(41U)); + Assert.AreEqual(1, v.CompareTo((ushort)41)); + Assert.AreEqual(1, v.CompareTo((byte)41)); + + Assert.AreEqual(-1, v.CompareTo(43UL)); + Assert.AreEqual(-1, v.CompareTo(43U)); + Assert.AreEqual(-1, v.CompareTo((ushort)43)); + Assert.AreEqual(-1, v.CompareTo((byte)43)); + #endregion Unsigned + } + + [Test] + public void Equals() + { + var v = new PyInt(42); + + #region Signed + Assert.True(v.Equals(42L)); + Assert.True(v.Equals(42)); + Assert.True(v.Equals((short)42)); + Assert.True(v.Equals((sbyte)42)); + + Assert.False(v.Equals(41L)); + Assert.False(v.Equals(41)); + Assert.False(v.Equals((short)41)); + Assert.False(v.Equals((sbyte)41)); + #endregion Signed + + #region Unsigned + Assert.True(v.Equals(42UL)); + Assert.True(v.Equals(42U)); + Assert.True(v.Equals((ushort)42)); + Assert.True(v.Equals((byte)42)); + + Assert.False(v.Equals(41UL)); + Assert.False(v.Equals(41U)); + Assert.False(v.Equals((ushort)41)); + Assert.False(v.Equals((byte)41)); + #endregion Unsigned + } + [Test] public void ToBigIntegerLarge() { diff --git a/src/embed_tests/TestPyString.cs b/src/embed_tests/TestPyString.cs index b12e08c23..35c6339ee 100644 --- a/src/embed_tests/TestPyString.cs +++ b/src/embed_tests/TestPyString.cs @@ -112,5 +112,24 @@ public void TestUnicodeSurrogate() Assert.AreEqual(4, actual.Length()); Assert.AreEqual(expected, actual.ToString()); } + + [Test] + public void CompareTo() + { + var a = new PyString("foo"); + + Assert.AreEqual(0, a.CompareTo("foo")); + Assert.AreEqual("foo".CompareTo("bar"), a.CompareTo("bar")); + Assert.AreEqual("foo".CompareTo("foz"), a.CompareTo("foz")); + } + + [Test] + public void Equals() + { + var a = new PyString("foo"); + + Assert.True(a.Equals("foo")); + Assert.False(a.Equals("bar")); + } } } diff --git a/src/embed_tests/TestPyType.cs b/src/embed_tests/TestPyType.cs index 34645747d..d98dfda2e 100644 --- a/src/embed_tests/TestPyType.cs +++ b/src/embed_tests/TestPyType.cs @@ -28,7 +28,8 @@ public void CanCreateHeapType() const string name = "nÁmæ"; const string docStr = "dÁcæ"; - using var doc = new StrPtr(docStr, Encoding.UTF8); + using var doc = new StrPtr(docStr); + var spec = new TypeSpec( name: name, basicSize: Util.ReadInt32(Runtime.Runtime.PyBaseObjectType, TypeOffset.tp_basicsize), diff --git a/src/python_tests_runner/PythonTestRunner.cs b/src/python_tests_runner/PythonTestRunner.cs index 05298997b..f97cc5aec 100644 --- a/src/python_tests_runner/PythonTestRunner.cs +++ b/src/python_tests_runner/PythonTestRunner.cs @@ -35,6 +35,7 @@ static IEnumerable PythonTestCases() // Add the test that you want to debug here. yield return new[] { "test_indexer", "test_boolean_indexer" }; yield return new[] { "test_delegate", "test_bool_delegate" }; + yield return new[] { "test_subclass", "test_implement_interface_and_class" }; } /// diff --git a/src/runtime/AssemblyManager.cs b/src/runtime/AssemblyManager.cs index a8bbd1f6c..82658bf50 100644 --- a/src/runtime/AssemblyManager.cs +++ b/src/runtime/AssemblyManager.cs @@ -200,6 +200,13 @@ static IEnumerable FindAssemblyCandidates(string name) } else { + int invalidCharIndex = head.IndexOfAny(Path.GetInvalidPathChars()); + if (invalidCharIndex >= 0) + { + using var importWarning = Runtime.PyObject_GetAttrString(Exceptions.exceptions_module, "ImportWarning"); + Exceptions.warn($"Path entry '{head}' has invalid char at position {invalidCharIndex}", importWarning.BorrowOrThrow()); + continue; + } path = Path.Combine(head, name); } diff --git a/src/runtime/ClassManager.cs b/src/runtime/ClassManager.cs index 79ab20e82..b884bfa92 100644 --- a/src/runtime/ClassManager.cs +++ b/src/runtime/ClassManager.cs @@ -213,6 +213,7 @@ internal static void InitClassBase(Type type, ClassBase impl, ReflectedClrType p ClassInfo info = GetClassInfo(type, impl); impl.indexer = info.indexer; + impl.del = info.del; impl.richcompare.Clear(); @@ -290,11 +291,13 @@ internal static void InitClassBase(Type type, ClassBase impl, ReflectedClrType p internal static bool ShouldBindMethod(MethodBase mb) { + if (mb is null) throw new ArgumentNullException(nameof(mb)); return (mb.IsPublic || mb.IsFamily || mb.IsFamilyOrAssembly); } internal static bool ShouldBindField(FieldInfo fi) { + if (fi is null) throw new ArgumentNullException(nameof(fi)); return (fi.IsPublic || fi.IsFamily || fi.IsFamilyOrAssembly); } @@ -326,7 +329,7 @@ internal static bool ShouldBindProperty(PropertyInfo pi) internal static bool ShouldBindEvent(EventInfo ei) { - return ShouldBindMethod(ei.GetAddMethod(true)); + return ei.GetAddMethod(true) is { } add && ShouldBindMethod(add); } private static ClassInfo GetClassInfo(Type type, ClassBase impl) @@ -536,6 +539,21 @@ private static ClassInfo GetClassInfo(Type type, ClassBase impl) ob = new MethodObject(type, name, mlist); ci.members[name] = ob.AllocObject(); + if (name == nameof(IDictionary.Remove) + && mlist.Any(m => m.DeclaringType?.GetInterfaces() + .Any(i => i.TryGetGenericDefinition() == typeof(IDictionary<,>)) is true)) + { + ci.del = new(); + ci.del.AddRange(mlist.Where(m => !m.IsStatic)); + } + else if (name == nameof(IList.RemoveAt) + && mlist.Any(m => m.DeclaringType?.GetInterfaces() + .Any(i => i.TryGetGenericDefinition() == typeof(IList<>)) is true)) + { + ci.del = new(); + ci.del.AddRange(mlist.Where(m => !m.IsStatic)); + } + if (mlist.Any(OperatorMethod.IsOperatorMethod)) { string pyName = OperatorMethod.GetPyMethodName(name); @@ -546,7 +564,7 @@ private static ClassInfo GetClassInfo(Type type, ClassBase impl) ci.members[pyName] = new MethodObject(type, name, forwardMethods).AllocObject(); // Only methods where only the right operand is the declaring type. if (reverseMethods.Length > 0) - ci.members[pyNameReverse] = new MethodObject(type, name, reverseMethods).AllocObject(); + ci.members[pyNameReverse] = new MethodObject(type, name, reverseMethods, argsReversed: true).AllocObject(); } } @@ -579,6 +597,7 @@ private static ClassInfo GetClassInfo(Type type, ClassBase impl) private class ClassInfo { public Indexer? indexer; + public MethodBinder? del; public readonly Dictionary members = new(); internal ClassInfo() diff --git a/src/runtime/CollectionWrappers/IterableWrapper.cs b/src/runtime/CollectionWrappers/IterableWrapper.cs index d30e584d3..849e3997b 100644 --- a/src/runtime/CollectionWrappers/IterableWrapper.cs +++ b/src/runtime/CollectionWrappers/IterableWrapper.cs @@ -24,18 +24,22 @@ public IEnumerator GetEnumerator() { iterObject = PyIter.GetIter(pyObject); } - - using var _ = iterObject; - while (true) + try { - using var GIL = Py.GIL(); - - if (!iterObject.MoveNext()) + while (true) { - iterObject.Dispose(); - break; + using var _ = Py.GIL(); + if (!iterObject.MoveNext()) + { + break; + } + yield return iterObject.Current.As()!; } - yield return iterObject.Current.As()!; + } + finally + { + using var _ = Py.GIL(); + iterObject.Dispose(); } } } diff --git a/src/runtime/Converter.cs b/src/runtime/Converter.cs index 73bbd4a3a..50b33e60e 100644 --- a/src/runtime/Converter.cs +++ b/src/runtime/Converter.cs @@ -133,7 +133,8 @@ internal static NewReference ToPython(object? value, Type type) if (EncodableByUser(type, value)) { var encoded = PyObjectConversions.TryEncode(value, type); - if (encoded != null) { + if (encoded != null) + { return new NewReference(encoded); } } @@ -334,7 +335,7 @@ internal static bool ToManagedValue(BorrowedReference value, Type obType, if (obType.IsGenericType && obType.GetGenericTypeDefinition() == typeof(Nullable<>)) { - if( value == Runtime.PyNone ) + if (value == Runtime.PyNone) { result = null; return true; @@ -686,10 +687,8 @@ internal static bool ToPrimitive(BorrowedReference value, Type obType, out objec { if (Runtime.PyUnicode_GetLength(value) == 1) { - IntPtr unicodePtr = Runtime.PyUnicode_AsUnicode(value); - Char[] buff = new Char[1]; - Marshal.Copy(unicodePtr, buff, 0, 1); - result = buff[0]; + int chr = Runtime.PyUnicode_ReadChar(value, 0); + result = (Char)chr; return true; } goto type_error; @@ -982,5 +981,11 @@ public static PyObject ToPython(this object? o) if (o is null) return Runtime.None; return Converter.ToPython(o, o.GetType()).MoveToPyObject(); } + + public static PyObject ToPythonAs(this T? o) + { + if (o is null) return Runtime.None; + return Converter.ToPython(o, typeof(T)).MoveToPyObject(); + } } } diff --git a/src/runtime/Exceptions.cs b/src/runtime/Exceptions.cs index da095e030..85e56eace 100644 --- a/src/runtime/Exceptions.cs +++ b/src/runtime/Exceptions.cs @@ -270,7 +270,7 @@ public static void warn(string message, BorrowedReference exception, int stackle } using var warn = Runtime.PyObject_GetAttrString(warnings_module.obj, "warn"); - Exceptions.ErrorCheck(warn.Borrow()); + warn.BorrowOrThrow(); using var argsTemp = Runtime.PyTuple_New(3); BorrowedReference args = argsTemp.BorrowOrThrow(); @@ -283,7 +283,7 @@ public static void warn(string message, BorrowedReference exception, int stackle Runtime.PyTuple_SetItem(args, 2, level.StealOrThrow()); using var result = Runtime.PyObject_CallObject(warn.Borrow(), args); - Exceptions.ErrorCheck(result.Borrow()); + result.BorrowOrThrow(); } public static void warn(string message, BorrowedReference exception) diff --git a/src/runtime/Finalizer.cs b/src/runtime/Finalizer.cs index 713564f08..5b5ecfcfc 100644 --- a/src/runtime/Finalizer.cs +++ b/src/runtime/Finalizer.cs @@ -191,7 +191,7 @@ internal static void Shutdown() Instance.started = false; } - internal nint DisposeAll() + internal nint DisposeAll(bool disposeObj = true, bool disposeDerived = true, bool disposeBuffer = true) { if (_objQueue.IsEmpty && _derivedQueue.IsEmpty && _bufferQueue.IsEmpty) return 0; @@ -216,7 +216,7 @@ internal nint DisposeAll() try { - while (!_objQueue.IsEmpty) + if (disposeObj) while (!_objQueue.IsEmpty) { if (!_objQueue.TryDequeue(out var obj)) continue; @@ -240,7 +240,7 @@ internal nint DisposeAll() } } - while (!_derivedQueue.IsEmpty) + if (disposeDerived) while (!_derivedQueue.IsEmpty) { if (!_derivedQueue.TryDequeue(out var derived)) continue; @@ -258,7 +258,7 @@ internal nint DisposeAll() collected++; } - while (!_bufferQueue.IsEmpty) + if (disposeBuffer) while (!_bufferQueue.IsEmpty) { if (!_bufferQueue.TryDequeue(out var buffer)) continue; diff --git a/src/runtime/InternalPythonnetException.cs b/src/runtime/InternalPythonnetException.cs new file mode 100644 index 000000000..d0ea1bece --- /dev/null +++ b/src/runtime/InternalPythonnetException.cs @@ -0,0 +1,9 @@ +using System; + +namespace Python.Runtime; + +public class InternalPythonnetException : Exception +{ + public InternalPythonnetException(string message, Exception innerException) + : base(message, innerException) { } +} diff --git a/src/runtime/Loader.cs b/src/runtime/Loader.cs index 516b9ab9c..c0e964abc 100644 --- a/src/runtime/Loader.cs +++ b/src/runtime/Loader.cs @@ -12,7 +12,7 @@ public unsafe static int Initialize(IntPtr data, int size) { try { - var dllPath = Encoding.UTF8.GetString((byte*)data.ToPointer(), size); + var dllPath = Encodings.UTF8.GetString((byte*)data.ToPointer(), size); if (!string.IsNullOrEmpty(dllPath)) { @@ -33,7 +33,7 @@ public unsafe static int Initialize(IntPtr data, int size) ); return 1; } - + return 0; } @@ -41,7 +41,7 @@ public unsafe static int Shutdown(IntPtr data, int size) { try { - var command = Encoding.UTF8.GetString((byte*)data.ToPointer(), size); + var command = Encodings.UTF8.GetString((byte*)data.ToPointer(), size); if (command == "full_shutdown") { diff --git a/src/runtime/MethodBinder.cs b/src/runtime/MethodBinder.cs index 07ed4fe22..af75a34b4 100644 --- a/src/runtime/MethodBinder.cs +++ b/src/runtime/MethodBinder.cs @@ -28,9 +28,12 @@ internal class MethodBinder [NonSerialized] public bool init = false; + public const bool DefaultAllowThreads = true; public bool allow_threads = DefaultAllowThreads; + public bool argsReversed = false; + internal MethodBinder() { list = new List(); @@ -51,6 +54,11 @@ internal void AddMethod(MethodBase m) list.Add(m); } + internal void AddRange(IEnumerable methods) + { + list.AddRange(methods.Select(m => new MaybeMethodBase(m))); + } + /// /// Given a sequence of MethodInfo and a sequence of types, return the /// MethodInfo that matches the signature represented by those types. @@ -363,10 +371,10 @@ public MismatchedMethod(Exception exception, MethodBase mb) _methods = GetMethods(); } - return Bind(inst, args, kwargDict, _methods, matchGenerics: true); + return Bind(inst, args, kwargDict, _methods, matchGenerics: true, argsReversed); } - static Binding? Bind(BorrowedReference inst, BorrowedReference args, Dictionary kwargDict, MethodBase[] methods, bool matchGenerics) + private static Binding? Bind(BorrowedReference inst, BorrowedReference args, Dictionary kwargDict, MethodBase[] methods, bool matchGenerics, bool argsReversed = false) { var pynargs = (int)Runtime.PyTuple_Size(args); var isGeneric = false; @@ -386,7 +394,7 @@ public MismatchedMethod(Exception exception, MethodBase mb) // Binary operator methods will have 2 CLR args but only one Python arg // (unary operators will have 1 less each), since Python operator methods are bound. isOperator = isOperator && pynargs == pi.Length - 1; - bool isReverse = isOperator && OperatorMethod.IsReverse((MethodInfo)mi); // Only cast if isOperator. + bool isReverse = isOperator && argsReversed; // Only cast if isOperator. if (isReverse && OperatorMethod.IsComparisonOp((MethodInfo)mi)) continue; // Comparison operators in Python have no reverse mode. if (!MatchesArgumentCount(pynargs, pi, kwargDict, out bool paramsArray, out ArrayList? defaultArgList, out int kwargsMatched, out int defaultsNeeded) && !isOperator) @@ -394,12 +402,14 @@ public MismatchedMethod(Exception exception, MethodBase mb) continue; } // Preprocessing pi to remove either the first or second argument. - if (isOperator && !isReverse) { + if (isOperator && !isReverse) + { // The first Python arg is the right operand, while the bound instance is the left. // We need to skip the first (left operand) CLR argument. pi = pi.Skip(1).ToArray(); } - else if (isOperator && isReverse) { + else if (isOperator && isReverse) + { // The first Python arg is the left operand. // We need to take the first CLR argument. pi = pi.Take(1).ToArray(); diff --git a/src/runtime/Native/CustomMarshaler.cs b/src/runtime/Native/CustomMarshaler.cs index 62c027150..299af3a33 100644 --- a/src/runtime/Native/CustomMarshaler.cs +++ b/src/runtime/Native/CustomMarshaler.cs @@ -42,7 +42,7 @@ public int GetNativeDataSize() internal class UcsMarshaler : MarshalerBase { internal static readonly int _UCS = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? 2 : 4; - internal static readonly Encoding PyEncoding = _UCS == 2 ? Encoding.Unicode : Encoding.UTF32; + internal static readonly Encoding PyEncoding = _UCS == 2 ? Encodings.UTF16 : Encodings.UTF32; private static readonly MarshalerBase Instance = new UcsMarshaler(); public override IntPtr MarshalManagedToNative(object managedObj) diff --git a/src/runtime/Native/NativeTypeSpec.cs b/src/runtime/Native/NativeTypeSpec.cs index 8b84df536..90e07afd7 100644 --- a/src/runtime/Native/NativeTypeSpec.cs +++ b/src/runtime/Native/NativeTypeSpec.cs @@ -17,7 +17,7 @@ public NativeTypeSpec(TypeSpec spec) { if (spec is null) throw new ArgumentNullException(nameof(spec)); - this.Name = new StrPtr(spec.Name, Encoding.UTF8); + this.Name = new StrPtr(spec.Name); this.BasicSize = spec.BasicSize; this.ItemSize = spec.ItemSize; this.Flags = (int)spec.Flags; diff --git a/src/runtime/Native/StrPtr.cs b/src/runtime/Native/StrPtr.cs index 4f73be9b5..c9f4db660 100644 --- a/src/runtime/Native/StrPtr.cs +++ b/src/runtime/Native/StrPtr.cs @@ -10,6 +10,8 @@ struct StrPtr : IDisposable public IntPtr RawPointer { get; set; } unsafe byte* Bytes => (byte*)this.RawPointer; + public unsafe StrPtr(string value) : this(value, Encodings.UTF8) {} + public unsafe StrPtr(string value, Encoding encoding) { if (value is null) throw new ArgumentNullException(nameof(value)); diff --git a/src/runtime/Native/TypeOffset312.cs b/src/runtime/Native/TypeOffset312.cs new file mode 100644 index 000000000..8ba30e816 --- /dev/null +++ b/src/runtime/Native/TypeOffset312.cs @@ -0,0 +1,144 @@ + +// Auto-generated by geninterop.py. +// DO NOT MODIFY BY HAND. + +// Python 3.12: ABI flags: '' + +// ReSharper disable InconsistentNaming +// ReSharper disable IdentifierTypo + +using System; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.InteropServices; + +using Python.Runtime.Native; + +namespace Python.Runtime +{ + [SuppressMessage("Style", "IDE1006:Naming Styles", + Justification = "Following CPython", + Scope = "type")] + + [StructLayout(LayoutKind.Sequential)] + internal class TypeOffset312 : GeneratedTypeOffsets, ITypeOffsets + { + public TypeOffset312() { } + // Auto-generated from PyHeapTypeObject in Python.h + public int ob_refcnt { get; private set; } + public int ob_type { get; private set; } + public int ob_size { get; private set; } + public int tp_name { get; private set; } + public int tp_basicsize { get; private set; } + public int tp_itemsize { get; private set; } + public int tp_dealloc { get; private set; } + public int tp_vectorcall_offset { get; private set; } + public int tp_getattr { get; private set; } + public int tp_setattr { get; private set; } + public int tp_as_async { get; private set; } + public int tp_repr { get; private set; } + public int tp_as_number { get; private set; } + public int tp_as_sequence { get; private set; } + public int tp_as_mapping { get; private set; } + public int tp_hash { get; private set; } + public int tp_call { get; private set; } + public int tp_str { get; private set; } + public int tp_getattro { get; private set; } + public int tp_setattro { get; private set; } + public int tp_as_buffer { get; private set; } + public int tp_flags { get; private set; } + public int tp_doc { get; private set; } + public int tp_traverse { get; private set; } + public int tp_clear { get; private set; } + public int tp_richcompare { get; private set; } + public int tp_weaklistoffset { get; private set; } + public int tp_iter { get; private set; } + public int tp_iternext { get; private set; } + public int tp_methods { get; private set; } + public int tp_members { get; private set; } + public int tp_getset { get; private set; } + public int tp_base { get; private set; } + public int tp_dict { get; private set; } + public int tp_descr_get { get; private set; } + public int tp_descr_set { get; private set; } + public int tp_dictoffset { get; private set; } + public int tp_init { get; private set; } + public int tp_alloc { get; private set; } + public int tp_new { get; private set; } + public int tp_free { get; private set; } + public int tp_is_gc { get; private set; } + public int tp_bases { get; private set; } + public int tp_mro { get; private set; } + public int tp_cache { get; private set; } + public int tp_subclasses { get; private set; } + public int tp_weaklist { get; private set; } + public int tp_del { get; private set; } + public int tp_version_tag { get; private set; } + public int tp_finalize { get; private set; } + public int tp_vectorcall { get; private set; } + public int tp_watched { get; private set; } + public int am_await { get; private set; } + public int am_aiter { get; private set; } + public int am_anext { get; private set; } + public int am_send { get; private set; } + public int nb_add { get; private set; } + public int nb_subtract { get; private set; } + public int nb_multiply { get; private set; } + public int nb_remainder { get; private set; } + public int nb_divmod { get; private set; } + public int nb_power { get; private set; } + public int nb_negative { get; private set; } + public int nb_positive { get; private set; } + public int nb_absolute { get; private set; } + public int nb_bool { get; private set; } + public int nb_invert { get; private set; } + public int nb_lshift { get; private set; } + public int nb_rshift { get; private set; } + public int nb_and { get; private set; } + public int nb_xor { get; private set; } + public int nb_or { get; private set; } + public int nb_int { get; private set; } + public int nb_reserved { get; private set; } + public int nb_float { get; private set; } + public int nb_inplace_add { get; private set; } + public int nb_inplace_subtract { get; private set; } + public int nb_inplace_multiply { get; private set; } + public int nb_inplace_remainder { get; private set; } + public int nb_inplace_power { get; private set; } + public int nb_inplace_lshift { get; private set; } + public int nb_inplace_rshift { get; private set; } + public int nb_inplace_and { get; private set; } + public int nb_inplace_xor { get; private set; } + public int nb_inplace_or { get; private set; } + public int nb_floor_divide { get; private set; } + public int nb_true_divide { get; private set; } + public int nb_inplace_floor_divide { get; private set; } + public int nb_inplace_true_divide { get; private set; } + public int nb_index { get; private set; } + public int nb_matrix_multiply { get; private set; } + public int nb_inplace_matrix_multiply { get; private set; } + public int mp_length { get; private set; } + public int mp_subscript { get; private set; } + public int mp_ass_subscript { get; private set; } + public int sq_length { get; private set; } + public int sq_concat { get; private set; } + public int sq_repeat { get; private set; } + public int sq_item { get; private set; } + public int was_sq_slice { get; private set; } + public int sq_ass_item { get; private set; } + public int was_sq_ass_slice { get; private set; } + public int sq_contains { get; private set; } + public int sq_inplace_concat { get; private set; } + public int sq_inplace_repeat { get; private set; } + public int bf_getbuffer { get; private set; } + public int bf_releasebuffer { get; private set; } + public int name { get; private set; } + public int ht_slots { get; private set; } + public int qualname { get; private set; } + public int ht_cached_keys { get; private set; } + public int ht_module { get; private set; } + public int _ht_tpname { get; private set; } + public int spec_cache_getitem { get; private set; } + public int getitem_version { get; private set; } + } +} + diff --git a/src/runtime/Native/TypeOffset313.cs b/src/runtime/Native/TypeOffset313.cs new file mode 100644 index 000000000..4c2e71295 --- /dev/null +++ b/src/runtime/Native/TypeOffset313.cs @@ -0,0 +1,152 @@ + +// Auto-generated by geninterop.py. +// DO NOT MODIFY BY HAND. + +// Python 3.13: ABI flags: '' + +// ReSharper disable InconsistentNaming +// ReSharper disable IdentifierTypo + +using System; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.InteropServices; + +using Python.Runtime.Native; + +namespace Python.Runtime +{ + [SuppressMessage("Style", "IDE1006:Naming Styles", + Justification = "Following CPython", + Scope = "type")] + + [StructLayout(LayoutKind.Sequential)] + internal class TypeOffset313 : GeneratedTypeOffsets, ITypeOffsets + { + public TypeOffset313() { } + // Auto-generated from PyHeapTypeObject in Python.h + public int ob_refcnt { get; private set; } + public int ob_type { get; private set; } + public int ob_size { get; private set; } + public int tp_name { get; private set; } + public int tp_basicsize { get; private set; } + public int tp_itemsize { get; private set; } + public int tp_dealloc { get; private set; } + public int tp_vectorcall_offset { get; private set; } + public int tp_getattr { get; private set; } + public int tp_setattr { get; private set; } + public int tp_as_async { get; private set; } + public int tp_repr { get; private set; } + public int tp_as_number { get; private set; } + public int tp_as_sequence { get; private set; } + public int tp_as_mapping { get; private set; } + public int tp_hash { get; private set; } + public int tp_call { get; private set; } + public int tp_str { get; private set; } + public int tp_getattro { get; private set; } + public int tp_setattro { get; private set; } + public int tp_as_buffer { get; private set; } + public int tp_flags { get; private set; } + public int tp_doc { get; private set; } + public int tp_traverse { get; private set; } + public int tp_clear { get; private set; } + public int tp_richcompare { get; private set; } + public int tp_weaklistoffset { get; private set; } + public int tp_iter { get; private set; } + public int tp_iternext { get; private set; } + public int tp_methods { get; private set; } + public int tp_members { get; private set; } + public int tp_getset { get; private set; } + public int tp_base { get; private set; } + public int tp_dict { get; private set; } + public int tp_descr_get { get; private set; } + public int tp_descr_set { get; private set; } + public int tp_dictoffset { get; private set; } + public int tp_init { get; private set; } + public int tp_alloc { get; private set; } + public int tp_new { get; private set; } + public int tp_free { get; private set; } + public int tp_is_gc { get; private set; } + public int tp_bases { get; private set; } + public int tp_mro { get; private set; } + public int tp_cache { get; private set; } + public int tp_subclasses { get; private set; } + public int tp_weaklist { get; private set; } + public int tp_del { get; private set; } + public int tp_version_tag { get; private set; } + public int tp_finalize { get; private set; } + public int tp_vectorcall { get; private set; } + // This is an error in our generator: + // + // The fields below are actually not pointers (like we incorrectly + // assume for all other fields) but instead a char (1 byte) and a short + // (2 bytes). By dropping one of the fields, we still get the correct + // overall size of the struct. + public int tp_watched { get; private set; } + // public int tp_versions_used { get; private set; } + public int am_await { get; private set; } + public int am_aiter { get; private set; } + public int am_anext { get; private set; } + public int am_send { get; private set; } + public int nb_add { get; private set; } + public int nb_subtract { get; private set; } + public int nb_multiply { get; private set; } + public int nb_remainder { get; private set; } + public int nb_divmod { get; private set; } + public int nb_power { get; private set; } + public int nb_negative { get; private set; } + public int nb_positive { get; private set; } + public int nb_absolute { get; private set; } + public int nb_bool { get; private set; } + public int nb_invert { get; private set; } + public int nb_lshift { get; private set; } + public int nb_rshift { get; private set; } + public int nb_and { get; private set; } + public int nb_xor { get; private set; } + public int nb_or { get; private set; } + public int nb_int { get; private set; } + public int nb_reserved { get; private set; } + public int nb_float { get; private set; } + public int nb_inplace_add { get; private set; } + public int nb_inplace_subtract { get; private set; } + public int nb_inplace_multiply { get; private set; } + public int nb_inplace_remainder { get; private set; } + public int nb_inplace_power { get; private set; } + public int nb_inplace_lshift { get; private set; } + public int nb_inplace_rshift { get; private set; } + public int nb_inplace_and { get; private set; } + public int nb_inplace_xor { get; private set; } + public int nb_inplace_or { get; private set; } + public int nb_floor_divide { get; private set; } + public int nb_true_divide { get; private set; } + public int nb_inplace_floor_divide { get; private set; } + public int nb_inplace_true_divide { get; private set; } + public int nb_index { get; private set; } + public int nb_matrix_multiply { get; private set; } + public int nb_inplace_matrix_multiply { get; private set; } + public int mp_length { get; private set; } + public int mp_subscript { get; private set; } + public int mp_ass_subscript { get; private set; } + public int sq_length { get; private set; } + public int sq_concat { get; private set; } + public int sq_repeat { get; private set; } + public int sq_item { get; private set; } + public int was_sq_slice { get; private set; } + public int sq_ass_item { get; private set; } + public int was_sq_ass_slice { get; private set; } + public int sq_contains { get; private set; } + public int sq_inplace_concat { get; private set; } + public int sq_inplace_repeat { get; private set; } + public int bf_getbuffer { get; private set; } + public int bf_releasebuffer { get; private set; } + public int name { get; private set; } + public int ht_slots { get; private set; } + public int qualname { get; private set; } + public int ht_cached_keys { get; private set; } + public int ht_module { get; private set; } + public int _ht_tpname { get; private set; } + public int spec_cache_getitem { get; private set; } + public int getitem_version { get; private set; } + public int init { get; private set; } + } +} + diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index 5072f23cd..0a7f58a3c 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -12,8 +12,6 @@ https://github.com/pythonnet/pythonnet git python interop dynamic dlr Mono pinvoke - python-clear.png - https://raw.githubusercontent.com/pythonnet/pythonnet/master/src/console/python-clear.ico https://pythonnet.github.io/ README.md true @@ -48,7 +46,6 @@ - diff --git a/src/runtime/PythonEngine.cs b/src/runtime/PythonEngine.cs index 4ed45b9e9..0b28c3a35 100644 --- a/src/runtime/PythonEngine.cs +++ b/src/runtime/PythonEngine.cs @@ -135,7 +135,7 @@ public static string PythonPath } public static Version MinSupportedVersion => new(3, 7); - public static Version MaxSupportedVersion => new(3, 11, int.MaxValue, int.MaxValue); + public static Version MaxSupportedVersion => new(3, 13, int.MaxValue, int.MaxValue); public static bool IsSupportedVersion(Version version) => version >= MinSupportedVersion && version <= MaxSupportedVersion; public static string Version diff --git a/src/runtime/PythonTypes/PyBuffer.cs b/src/runtime/PythonTypes/PyBuffer.cs index de0e7122f..120582494 100644 --- a/src/runtime/PythonTypes/PyBuffer.cs +++ b/src/runtime/PythonTypes/PyBuffer.cs @@ -11,7 +11,7 @@ public sealed class PyBuffer : IDisposable private PyObject _exporter; private Py_buffer _view; - unsafe internal PyBuffer(PyObject exporter, PyBUF flags) + internal PyBuffer(PyObject exporter, PyBUF flags) { _view = new Py_buffer(); @@ -25,17 +25,17 @@ unsafe internal PyBuffer(PyObject exporter, PyBUF flags) var intPtrBuf = new IntPtr[_view.ndim]; if (_view.shape != IntPtr.Zero) { - Marshal.Copy(_view.shape, intPtrBuf, 0, (int)_view.len * sizeof(IntPtr)); + Marshal.Copy(_view.shape, intPtrBuf, 0, _view.ndim); Shape = intPtrBuf.Select(x => (long)x).ToArray(); } if (_view.strides != IntPtr.Zero) { - Marshal.Copy(_view.strides, intPtrBuf, 0, (int)_view.len * sizeof(IntPtr)); + Marshal.Copy(_view.strides, intPtrBuf, 0, _view.ndim); Strides = intPtrBuf.Select(x => (long)x).ToArray(); } if (_view.suboffsets != IntPtr.Zero) { - Marshal.Copy(_view.suboffsets, intPtrBuf, 0, (int)_view.len * sizeof(IntPtr)); + Marshal.Copy(_view.suboffsets, intPtrBuf, 0, _view.ndim); SubOffsets = intPtrBuf.Select(x => (long)x).ToArray(); } } diff --git a/src/runtime/PythonTypes/PyFloat.IComparable.cs b/src/runtime/PythonTypes/PyFloat.IComparable.cs new file mode 100644 index 000000000..c12fc283a --- /dev/null +++ b/src/runtime/PythonTypes/PyFloat.IComparable.cs @@ -0,0 +1,34 @@ +using System; + +namespace Python.Runtime; + +partial class PyFloat : IComparable, IComparable + , IEquatable, IEquatable + , IComparable, IEquatable +{ + public override bool Equals(object o) + { + using var _ = Py.GIL(); + return o switch + { + double f64 => this.Equals(f64), + float f32 => this.Equals(f32), + _ => base.Equals(o), + }; + } + + public int CompareTo(double other) => this.ToDouble().CompareTo(other); + + public int CompareTo(float other) => this.ToDouble().CompareTo(other); + + public bool Equals(double other) => this.ToDouble().Equals(other); + + public bool Equals(float other) => this.ToDouble().Equals(other); + + public int CompareTo(PyFloat? other) + { + return other is null ? 1 : this.CompareTo(other.BorrowNullable()); + } + + public bool Equals(PyFloat? other) => base.Equals(other); +} diff --git a/src/runtime/PythonTypes/PyFloat.cs b/src/runtime/PythonTypes/PyFloat.cs index c09ec93ba..50621d5c2 100644 --- a/src/runtime/PythonTypes/PyFloat.cs +++ b/src/runtime/PythonTypes/PyFloat.cs @@ -8,7 +8,7 @@ namespace Python.Runtime /// PY3: https://docs.python.org/3/c-api/float.html /// for details. /// - public class PyFloat : PyNumber + public partial class PyFloat : PyNumber { internal PyFloat(in StolenReference ptr) : base(ptr) { @@ -100,6 +100,8 @@ public static PyFloat AsFloat(PyObject value) return new PyFloat(op.Steal()); } + public double ToDouble() => Runtime.PyFloat_AsDouble(obj); + public override TypeCode GetTypeCode() => TypeCode.Double; } } diff --git a/src/runtime/PythonTypes/PyInt.IComparable.cs b/src/runtime/PythonTypes/PyInt.IComparable.cs new file mode 100644 index 000000000..a96f02e10 --- /dev/null +++ b/src/runtime/PythonTypes/PyInt.IComparable.cs @@ -0,0 +1,136 @@ +using System; + +namespace Python.Runtime; + +partial class PyInt : IComparable, IComparable, IComparable, IComparable + , IComparable, IComparable, IComparable, IComparable + , IEquatable, IEquatable, IEquatable, IEquatable + , IEquatable, IEquatable, IEquatable, IEquatable + , IComparable, IEquatable +{ + public override bool Equals(object o) + { + using var _ = Py.GIL(); + return o switch + { + long i64 => this.Equals(i64), + int i32 => this.Equals(i32), + short i16 => this.Equals(i16), + sbyte i8 => this.Equals(i8), + + ulong u64 => this.Equals(u64), + uint u32 => this.Equals(u32), + ushort u16 => this.Equals(u16), + byte u8 => this.Equals(u8), + + _ => base.Equals(o), + }; + } + + #region Signed + public int CompareTo(long other) + { + using var pyOther = Runtime.PyInt_FromInt64(other); + return this.CompareTo(pyOther.BorrowOrThrow()); + } + + public int CompareTo(int other) + { + using var pyOther = Runtime.PyInt_FromInt32(other); + return this.CompareTo(pyOther.BorrowOrThrow()); + } + + public int CompareTo(short other) + { + using var pyOther = Runtime.PyInt_FromInt32(other); + return this.CompareTo(pyOther.BorrowOrThrow()); + } + + public int CompareTo(sbyte other) + { + using var pyOther = Runtime.PyInt_FromInt32(other); + return this.CompareTo(pyOther.BorrowOrThrow()); + } + + public bool Equals(long other) + { + using var pyOther = Runtime.PyInt_FromInt64(other); + return this.Equals(pyOther.BorrowOrThrow()); + } + + public bool Equals(int other) + { + using var pyOther = Runtime.PyInt_FromInt32(other); + return this.Equals(pyOther.BorrowOrThrow()); + } + + public bool Equals(short other) + { + using var pyOther = Runtime.PyInt_FromInt32(other); + return this.Equals(pyOther.BorrowOrThrow()); + } + + public bool Equals(sbyte other) + { + using var pyOther = Runtime.PyInt_FromInt32(other); + return this.Equals(pyOther.BorrowOrThrow()); + } + #endregion Signed + + #region Unsigned + public int CompareTo(ulong other) + { + using var pyOther = Runtime.PyLong_FromUnsignedLongLong(other); + return this.CompareTo(pyOther.BorrowOrThrow()); + } + + public int CompareTo(uint other) + { + using var pyOther = Runtime.PyLong_FromUnsignedLongLong(other); + return this.CompareTo(pyOther.BorrowOrThrow()); + } + + public int CompareTo(ushort other) + { + using var pyOther = Runtime.PyLong_FromUnsignedLongLong(other); + return this.CompareTo(pyOther.BorrowOrThrow()); + } + + public int CompareTo(byte other) + { + using var pyOther = Runtime.PyLong_FromUnsignedLongLong(other); + return this.CompareTo(pyOther.BorrowOrThrow()); + } + + public bool Equals(ulong other) + { + using var pyOther = Runtime.PyLong_FromUnsignedLongLong(other); + return this.Equals(pyOther.BorrowOrThrow()); + } + + public bool Equals(uint other) + { + using var pyOther = Runtime.PyLong_FromUnsignedLongLong(other); + return this.Equals(pyOther.BorrowOrThrow()); + } + + public bool Equals(ushort other) + { + using var pyOther = Runtime.PyLong_FromUnsignedLongLong(other); + return this.Equals(pyOther.BorrowOrThrow()); + } + + public bool Equals(byte other) + { + using var pyOther = Runtime.PyLong_FromUnsignedLongLong(other); + return this.Equals(pyOther.BorrowOrThrow()); + } + #endregion Unsigned + + public int CompareTo(PyInt? other) + { + return other is null ? 1 : this.CompareTo(other.BorrowNullable()); + } + + public bool Equals(PyInt? other) => base.Equals(other); +} diff --git a/src/runtime/PythonTypes/PyInt.cs b/src/runtime/PythonTypes/PyInt.cs index e71462b74..0d00f5a13 100644 --- a/src/runtime/PythonTypes/PyInt.cs +++ b/src/runtime/PythonTypes/PyInt.cs @@ -9,7 +9,7 @@ namespace Python.Runtime /// Represents a Python integer object. /// See the documentation at https://docs.python.org/3/c-api/long.html /// - public class PyInt : PyNumber, IFormattable + public partial class PyInt : PyNumber, IFormattable { internal PyInt(in StolenReference ptr) : base(ptr) { diff --git a/src/runtime/PythonTypes/PyModule.cs b/src/runtime/PythonTypes/PyModule.cs index 4549678ed..243f77ecc 100644 --- a/src/runtime/PythonTypes/PyModule.cs +++ b/src/runtime/PythonTypes/PyModule.cs @@ -82,7 +82,18 @@ public PyModule Reload() public static PyModule FromString(string name, string code) { - using NewReference c = Runtime.Py_CompileString(code, "none", (int)RunFlagType.File); + return FromString(name, code, ""); + } + + public static PyModule FromString(string name, string code, string file) + { + // Force valid value + if (string.IsNullOrWhiteSpace(file)) + { + file = "none"; + } + + using NewReference c = Runtime.Py_CompileString(code, file, (int)RunFlagType.File); NewReference m = Runtime.PyImport_ExecCodeModule(name, c.BorrowOrThrow()); return new PyModule(m.StealOrThrow()); } diff --git a/src/runtime/PythonTypes/PyObject.cs b/src/runtime/PythonTypes/PyObject.cs index bda2d9c02..cf0c2a03f 100644 --- a/src/runtime/PythonTypes/PyObject.cs +++ b/src/runtime/PythonTypes/PyObject.cs @@ -1136,6 +1136,23 @@ public long Refcount } } + internal int CompareTo(BorrowedReference other) + { + int greater = Runtime.PyObject_RichCompareBool(this.Reference, other, Runtime.Py_GT); + Debug.Assert(greater != -1); + if (greater > 0) + return 1; + int less = Runtime.PyObject_RichCompareBool(this.Reference, other, Runtime.Py_LT); + Debug.Assert(less != -1); + return less > 0 ? -1 : 0; + } + + internal bool Equals(BorrowedReference other) + { + int equal = Runtime.PyObject_RichCompareBool(this.Reference, other, Runtime.Py_EQ); + Debug.Assert(equal != -1); + return equal > 0; + } public override bool TryGetMember(GetMemberBinder binder, out object? result) { @@ -1325,7 +1342,7 @@ private bool TryCompare(PyObject arg, int op, out object @out) } return true; } - + public override bool TryBinaryOperation(BinaryOperationBinder binder, object arg, out object? result) { using var _ = Py.GIL(); diff --git a/src/runtime/PythonTypes/PyString.cs b/src/runtime/PythonTypes/PyString.cs index d54397fcf..6fed25c3e 100644 --- a/src/runtime/PythonTypes/PyString.cs +++ b/src/runtime/PythonTypes/PyString.cs @@ -1,4 +1,5 @@ using System; +using System.Diagnostics; using System.Runtime.Serialization; namespace Python.Runtime @@ -13,7 +14,7 @@ namespace Python.Runtime /// 2011-01-29: ...Then why does the string constructor call PyUnicode_FromUnicode()??? /// [Serializable] - public class PyString : PySequence + public class PyString : PySequence, IComparable, IEquatable { internal PyString(in StolenReference reference) : base(reference) { } internal PyString(BorrowedReference reference) : base(reference) { } @@ -61,5 +62,23 @@ public static bool IsStringType(PyObject value) } public override TypeCode GetTypeCode() => TypeCode.String; + + internal string ToStringUnderGIL() + { + string? result = Runtime.GetManagedString(this.Reference); + Debug.Assert(result is not null); + return result!; + } + + public bool Equals(string? other) + => this.ToStringUnderGIL().Equals(other, StringComparison.CurrentCulture); + public int CompareTo(string? other) + => string.Compare(this.ToStringUnderGIL(), other, StringComparison.CurrentCulture); + + public override string ToString() + { + using var _ = Py.GIL(); + return this.ToStringUnderGIL(); + } } } diff --git a/src/runtime/PythonTypes/PyType.cs b/src/runtime/PythonTypes/PyType.cs index af796a5c5..28bda5d3e 100644 --- a/src/runtime/PythonTypes/PyType.cs +++ b/src/runtime/PythonTypes/PyType.cs @@ -53,7 +53,7 @@ public string Name { RawPointer = Util.ReadIntPtr(this, TypeOffset.tp_name), }; - return namePtr.ToString(System.Text.Encoding.UTF8)!; + return namePtr.ToString(Encodings.UTF8)!; } } diff --git a/src/runtime/Runtime.Delegates.cs b/src/runtime/Runtime.Delegates.cs index 0b6b75872..262dc1e19 100644 --- a/src/runtime/Runtime.Delegates.cs +++ b/src/runtime/Runtime.Delegates.cs @@ -23,7 +23,17 @@ static Delegates() Py_EndInterpreter = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_EndInterpreter), GetUnmanagedDll(_PythonDll)); PyThreadState_New = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyThreadState_New), GetUnmanagedDll(_PythonDll)); PyThreadState_Get = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyThreadState_Get), GetUnmanagedDll(_PythonDll)); - _PyThreadState_UncheckedGet = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(_PyThreadState_UncheckedGet), GetUnmanagedDll(_PythonDll)); + try + { + // Up until Python 3.13, this function was private and named + // slightly differently. + PyThreadState_GetUnchecked = (delegate* unmanaged[Cdecl])GetFunctionByName("_PyThreadState_UncheckedGet", GetUnmanagedDll(_PythonDll)); + } + catch (MissingMethodException) + { + + PyThreadState_GetUnchecked = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyThreadState_GetUnchecked), GetUnmanagedDll(_PythonDll)); + } try { PyGILState_Check = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyGILState_Check), GetUnmanagedDll(_PythonDll)); @@ -35,7 +45,6 @@ static Delegates() PyGILState_Ensure = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyGILState_Ensure), GetUnmanagedDll(_PythonDll)); PyGILState_Release = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyGILState_Release), GetUnmanagedDll(_PythonDll)); PyGILState_GetThisThreadState = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyGILState_GetThisThreadState), GetUnmanagedDll(_PythonDll)); - Py_Main = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_Main), GetUnmanagedDll(_PythonDll)); PyEval_InitThreads = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyEval_InitThreads), GetUnmanagedDll(_PythonDll)); PyEval_ThreadsInitialized = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyEval_ThreadsInitialized), GetUnmanagedDll(_PythonDll)); PyEval_AcquireLock = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyEval_AcquireLock), GetUnmanagedDll(_PythonDll)); @@ -164,8 +173,8 @@ static Delegates() PyUnicode_AsUTF8 = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_AsUTF8), GetUnmanagedDll(_PythonDll)); PyUnicode_DecodeUTF16 = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_DecodeUTF16), GetUnmanagedDll(_PythonDll)); PyUnicode_GetLength = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_GetLength), GetUnmanagedDll(_PythonDll)); - PyUnicode_AsUnicode = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_AsUnicode), GetUnmanagedDll(_PythonDll)); PyUnicode_AsUTF16String = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_AsUTF16String), GetUnmanagedDll(_PythonDll)); + PyUnicode_ReadChar = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_ReadChar), GetUnmanagedDll(_PythonDll)); PyUnicode_FromOrdinal = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_FromOrdinal), GetUnmanagedDll(_PythonDll)); PyUnicode_InternFromString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_InternFromString), GetUnmanagedDll(_PythonDll)); PyUnicode_Compare = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_Compare), GetUnmanagedDll(_PythonDll)); @@ -314,12 +323,11 @@ static Delegates() internal static delegate* unmanaged[Cdecl] Py_EndInterpreter { get; } internal static delegate* unmanaged[Cdecl] PyThreadState_New { get; } internal static delegate* unmanaged[Cdecl] PyThreadState_Get { get; } - internal static delegate* unmanaged[Cdecl] _PyThreadState_UncheckedGet { get; } + internal static delegate* unmanaged[Cdecl] PyThreadState_GetUnchecked { get; } internal static delegate* unmanaged[Cdecl] PyGILState_Check { get; } internal static delegate* unmanaged[Cdecl] PyGILState_Ensure { get; } internal static delegate* unmanaged[Cdecl] PyGILState_Release { get; } internal static delegate* unmanaged[Cdecl] PyGILState_GetThisThreadState { get; } - internal static delegate* unmanaged[Cdecl] Py_Main { get; } internal static delegate* unmanaged[Cdecl] PyEval_InitThreads { get; } internal static delegate* unmanaged[Cdecl] PyEval_ThreadsInitialized { get; } internal static delegate* unmanaged[Cdecl] PyEval_AcquireLock { get; } @@ -441,7 +449,7 @@ static Delegates() internal static delegate* unmanaged[Cdecl] PyUnicode_AsUTF8 { get; } internal static delegate* unmanaged[Cdecl] PyUnicode_DecodeUTF16 { get; } internal static delegate* unmanaged[Cdecl] PyUnicode_GetLength { get; } - internal static delegate* unmanaged[Cdecl] PyUnicode_AsUnicode { get; } + internal static delegate* unmanaged[Cdecl] PyUnicode_ReadChar { get; } internal static delegate* unmanaged[Cdecl] PyUnicode_AsUTF16String { get; } internal static delegate* unmanaged[Cdecl] PyUnicode_FromOrdinal { get; } internal static delegate* unmanaged[Cdecl] PyUnicode_InternFromString { get; } diff --git a/src/runtime/Runtime.cs b/src/runtime/Runtime.cs index beb577e45..c8f022860 100644 --- a/src/runtime/Runtime.cs +++ b/src/runtime/Runtime.cs @@ -158,6 +158,7 @@ internal static void Initialize(bool initSigs = false) ClassManager.Reset(); ClassDerivedObject.Reset(); TypeManager.Initialize(); + CLRObject.creationBlocked = false; _typesInitialized = true; // Initialize modules that depend on the runtime class. @@ -278,6 +279,10 @@ internal static void Shutdown() ClearClrModules(); RemoveClrRootModule(); + TryCollectingGarbage(MaxCollectRetriesOnShutdown, forceBreakLoops: true, + obj: true, derived: false, buffer: false); + CLRObject.creationBlocked = true; + NullGCHandles(ExtensionType.loadedExtensions); ClassManager.RemoveClasses(); TypeManager.RemoveTypes(); @@ -295,8 +300,7 @@ internal static void Shutdown() PyObjectConversions.Reset(); PyGC_Collect(); - bool everythingSeemsCollected = TryCollectingGarbage(MaxCollectRetriesOnShutdown, - forceBreakLoops: true); + bool everythingSeemsCollected = TryCollectingGarbage(MaxCollectRetriesOnShutdown); Debug.Assert(everythingSeemsCollected); Finalizer.Shutdown(); @@ -312,7 +316,7 @@ internal static void Shutdown() // Then release the GIL for good, if there is somehting to release // Use the unchecked version as the checked version calls `abort()` // if the current state is NULL. - if (_PyThreadState_UncheckedGet() != (PyThreadState*)0) + if (PyThreadState_GetUnchecked() != (PyThreadState*)0) { PyEval_SaveThread(); } @@ -328,7 +332,8 @@ internal static void Shutdown() const int MaxCollectRetriesOnShutdown = 20; internal static int _collected; - static bool TryCollectingGarbage(int runs, bool forceBreakLoops) + static bool TryCollectingGarbage(int runs, bool forceBreakLoops, + bool obj = true, bool derived = true, bool buffer = true) { if (runs <= 0) throw new ArgumentOutOfRangeException(nameof(runs)); @@ -341,7 +346,9 @@ static bool TryCollectingGarbage(int runs, bool forceBreakLoops) GC.Collect(); GC.WaitForPendingFinalizers(); pyCollected += PyGC_Collect(); - pyCollected += Finalizer.Instance.DisposeAll(); + pyCollected += Finalizer.Instance.DisposeAll(disposeObj: obj, + disposeDerived: derived, + disposeBuffer: buffer); } if (Volatile.Read(ref _collected) == 0 && pyCollected == 0) { @@ -598,23 +605,8 @@ internal static void CheckExceptionOccurred() [Obsolete("Use NewReference or PyObject constructor instead")] internal static unsafe void XIncref(BorrowedReference op) { -#if !CUSTOM_INCDEC_REF Py_IncRef(op); return; -#else - var p = (void*)op; - if ((void*)0 != p) - { - if (Is32Bit) - { - (*(int*)p)++; - } - else - { - (*(long*)p)++; - } - } -#endif } internal static unsafe void XDecref(StolenReference op) @@ -623,40 +615,9 @@ internal static unsafe void XDecref(StolenReference op) Debug.Assert(op == null || Refcount(new BorrowedReference(op.Pointer)) > 0); Debug.Assert(_isInitialized || Py_IsInitialized() != 0 || _Py_IsFinalizing() != false); #endif -#if !CUSTOM_INCDEC_REF if (op == null) return; Py_DecRef(op.AnalyzerWorkaround()); return; -#else - var p = (void*)op; - if ((void*)0 != p) - { - if (Is32Bit) - { - --(*(int*)p); - } - else - { - --(*(long*)p); - } - if ((*(int*)p) == 0) - { - // PyObject_HEAD: struct _typeobject *ob_type - void* t = Is32Bit - ? (void*)(*((uint*)p + 1)) - : (void*)(*((ulong*)p + 1)); - // PyTypeObject: destructor tp_dealloc - void* f = Is32Bit - ? (void*)(*((uint*)t + 6)) - : (void*)(*((ulong*)t + 6)); - if ((void*)0 == f) - { - return; - } - NativeCall.Void_Call_1(new IntPtr(f), op); - } - } -#endif } [Pure] @@ -744,7 +705,7 @@ internal static T TryUsingDll(Func op) internal static PyThreadState* PyThreadState_Get() => Delegates.PyThreadState_Get(); - internal static PyThreadState* _PyThreadState_UncheckedGet() => Delegates._PyThreadState_UncheckedGet(); + internal static PyThreadState* PyThreadState_GetUnchecked() => Delegates.PyThreadState_GetUnchecked(); internal static int PyGILState_Check() => Delegates.PyGILState_Check(); @@ -758,20 +719,6 @@ internal static T TryUsingDll(Func op) internal static PyThreadState* PyGILState_GetThisThreadState() => Delegates.PyGILState_GetThisThreadState(); - public static int Py_Main(int argc, string[] argv) - { - var marshaler = StrArrayMarshaler.GetInstance(null); - var argvPtr = marshaler.MarshalManagedToNative(argv); - try - { - return Delegates.Py_Main(argc, argvPtr); - } - finally - { - marshaler.CleanUpNativeData(argvPtr); - } - } - internal static void PyEval_InitThreads() => Delegates.PyEval_InitThreads(); @@ -841,13 +788,13 @@ public static int Py_Main(int argc, string[] argv) internal static int PyRun_SimpleString(string code) { - using var codePtr = new StrPtr(code, Encoding.UTF8); + using var codePtr = new StrPtr(code); return Delegates.PyRun_SimpleStringFlags(codePtr, Utf8String); } internal static NewReference PyRun_String(string code, RunFlagType st, BorrowedReference globals, BorrowedReference locals) { - using var codePtr = new StrPtr(code, Encoding.UTF8); + using var codePtr = new StrPtr(code); return Delegates.PyRun_StringFlags(codePtr, st, globals, locals, Utf8String); } @@ -859,14 +806,15 @@ internal static NewReference PyRun_String(string code, RunFlagType st, BorrowedR /// internal static NewReference Py_CompileString(string str, string file, int start) { - using var strPtr = new StrPtr(str, Encoding.UTF8); + using var strPtr = new StrPtr(str); + using var fileObj = new PyString(file); return Delegates.Py_CompileStringObject(strPtr, fileObj, start, Utf8String, -1); } internal static NewReference PyImport_ExecCodeModule(string name, BorrowedReference code) { - using var namePtr = new StrPtr(name, Encoding.UTF8); + using var namePtr = new StrPtr(name); return Delegates.PyImport_ExecCodeModule(namePtr, code); } @@ -913,13 +861,13 @@ internal static bool PyObject_IsIterable(BorrowedReference ob) internal static int PyObject_HasAttrString(BorrowedReference pointer, string name) { - using var namePtr = new StrPtr(name, Encoding.UTF8); + using var namePtr = new StrPtr(name); return Delegates.PyObject_HasAttrString(pointer, namePtr); } internal static NewReference PyObject_GetAttrString(BorrowedReference pointer, string name) { - using var namePtr = new StrPtr(name, Encoding.UTF8); + using var namePtr = new StrPtr(name); return Delegates.PyObject_GetAttrString(pointer, namePtr); } @@ -930,12 +878,12 @@ internal static NewReference PyObject_GetAttrString(BorrowedReference pointer, S internal static int PyObject_DelAttr(BorrowedReference @object, BorrowedReference name) => Delegates.PyObject_SetAttr(@object, name, null); internal static int PyObject_DelAttrString(BorrowedReference @object, string name) { - using var namePtr = new StrPtr(name, Encoding.UTF8); + using var namePtr = new StrPtr(name); return Delegates.PyObject_SetAttrString(@object, namePtr, null); } internal static int PyObject_SetAttrString(BorrowedReference @object, string name, BorrowedReference value) { - using var namePtr = new StrPtr(name, Encoding.UTF8); + using var namePtr = new StrPtr(name); return Delegates.PyObject_SetAttrString(@object, namePtr, value); } @@ -985,7 +933,7 @@ internal static BorrowedReference PyObject_GetWeakRefList(BorrowedReference ob) Debug.Assert(ob != null); var type = PyObject_TYPE(ob); int offset = Util.ReadInt32(type, TypeOffset.tp_weaklistoffset); - if (offset == 0) return BorrowedReference.Null; + if (offset <= 0) return BorrowedReference.Null; Debug.Assert(offset > 0); return Util.ReadRef(ob, offset); } @@ -1117,7 +1065,7 @@ internal static bool PyBool_CheckExact(BorrowedReference ob) internal static NewReference PyLong_FromString(string value, int radix) { - using var valPtr = new StrPtr(value, Encoding.UTF8); + using var valPtr = new StrPtr(value); return Delegates.PyLong_FromString(valPtr, IntPtr.Zero, radix); } @@ -1298,12 +1246,14 @@ internal static bool PyString_CheckExact(BorrowedReference ob) internal static NewReference PyString_FromString(string value) { + int byteorder = BitConverter.IsLittleEndian ? -1 : 1; + int* byteorderPtr = &byteorder; fixed(char* ptr = value) return Delegates.PyUnicode_DecodeUTF16( (IntPtr)ptr, value.Length * sizeof(Char), IntPtr.Zero, - IntPtr.Zero + (IntPtr)byteorderPtr ); } @@ -1318,7 +1268,7 @@ internal static NewReference EmptyPyBytes() internal static NewReference PyByteArray_FromStringAndSize(IntPtr strPtr, nint len) => Delegates.PyByteArray_FromStringAndSize(strPtr, len); internal static NewReference PyByteArray_FromStringAndSize(string s) { - using var ptr = new StrPtr(s, Encoding.UTF8); + using var ptr = new StrPtr(s); return PyByteArray_FromStringAndSize(ptr.RawPointer, checked((nint)ptr.ByteCount)); } @@ -1336,16 +1286,17 @@ internal static IntPtr PyBytes_AsString(BorrowedReference ob) internal static nint PyUnicode_GetLength(BorrowedReference ob) => Delegates.PyUnicode_GetLength(ob); - internal static IntPtr PyUnicode_AsUnicode(BorrowedReference ob) => Delegates.PyUnicode_AsUnicode(ob); internal static NewReference PyUnicode_AsUTF16String(BorrowedReference ob) => Delegates.PyUnicode_AsUTF16String(ob); + internal static int PyUnicode_ReadChar(BorrowedReference ob, nint index) => Delegates.PyUnicode_ReadChar(ob, index); + internal static NewReference PyUnicode_FromOrdinal(int c) => Delegates.PyUnicode_FromOrdinal(c); internal static NewReference PyUnicode_InternFromString(string s) { - using var ptr = new StrPtr(s, Encoding.UTF8); + using var ptr = new StrPtr(s); return Delegates.PyUnicode_InternFromString(ptr); } @@ -1420,7 +1371,7 @@ internal static bool PyDict_Check(BorrowedReference ob) internal static BorrowedReference PyDict_GetItemString(BorrowedReference pointer, string key) { - using var keyStr = new StrPtr(key, Encoding.UTF8); + using var keyStr = new StrPtr(key); return Delegates.PyDict_GetItemString(pointer, keyStr); } @@ -1436,7 +1387,7 @@ internal static BorrowedReference PyDict_GetItemString(BorrowedReference pointer /// internal static int PyDict_SetItemString(BorrowedReference dict, string key, BorrowedReference value) { - using var keyPtr = new StrPtr(key, Encoding.UTF8); + using var keyPtr = new StrPtr(key); return Delegates.PyDict_SetItemString(dict, keyPtr, value); } @@ -1445,7 +1396,7 @@ internal static int PyDict_SetItemString(BorrowedReference dict, string key, Bor internal static int PyDict_DelItemString(BorrowedReference pointer, string key) { - using var keyPtr = new StrPtr(key, Encoding.UTF8); + using var keyPtr = new StrPtr(key); return Delegates.PyDict_DelItemString(pointer, keyPtr); } @@ -1560,7 +1511,7 @@ internal static bool PyIter_Check(BorrowedReference ob) internal static NewReference PyModule_New(string name) { - using var namePtr = new StrPtr(name, Encoding.UTF8); + using var namePtr = new StrPtr(name); return Delegates.PyModule_New(namePtr); } @@ -1574,7 +1525,7 @@ internal static NewReference PyModule_New(string name) /// Return -1 on error, 0 on success. internal static int PyModule_AddObject(BorrowedReference module, string name, StolenReference value) { - using var namePtr = new StrPtr(name, Encoding.UTF8); + using var namePtr = new StrPtr(name); IntPtr valueAddr = value.DangerousGetAddressOrNull(); int res = Delegates.PyModule_AddObject(module, namePtr, valueAddr); // We can't just exit here because the reference is stolen only on success. @@ -1592,7 +1543,7 @@ internal static int PyModule_AddObject(BorrowedReference module, string name, St internal static NewReference PyImport_ImportModule(string name) { - using var namePtr = new StrPtr(name, Encoding.UTF8); + using var namePtr = new StrPtr(name); return Delegates.PyImport_ImportModule(namePtr); } @@ -1601,7 +1552,7 @@ internal static NewReference PyImport_ImportModule(string name) internal static BorrowedReference PyImport_AddModule(string name) { - using var namePtr = new StrPtr(name, Encoding.UTF8); + using var namePtr = new StrPtr(name); return Delegates.PyImport_AddModule(namePtr); } @@ -1629,13 +1580,13 @@ internal static void PySys_SetArgvEx(int argc, string[] argv, int updatepath) internal static BorrowedReference PySys_GetObject(string name) { - using var namePtr = new StrPtr(name, Encoding.UTF8); + using var namePtr = new StrPtr(name); return Delegates.PySys_GetObject(namePtr); } internal static int PySys_SetObject(string name, BorrowedReference ob) { - using var namePtr = new StrPtr(name, Encoding.UTF8); + using var namePtr = new StrPtr(name); return Delegates.PySys_SetObject(namePtr, ob); } @@ -1734,7 +1685,7 @@ internal static IntPtr PyMem_Malloc(long size) internal static void PyErr_SetString(BorrowedReference ob, string message) { - using var msgPtr = new StrPtr(message, Encoding.UTF8); + using var msgPtr = new StrPtr(message); Delegates.PyErr_SetString(ob, msgPtr); } diff --git a/src/runtime/StateSerialization/MaybeType.cs b/src/runtime/StateSerialization/MaybeType.cs index f3c96e369..884b7edb0 100644 --- a/src/runtime/StateSerialization/MaybeType.cs +++ b/src/runtime/StateSerialization/MaybeType.cs @@ -15,7 +15,7 @@ internal struct MaybeType : ISerializable const string SerializationName = "n"; readonly string name; readonly Type type; - + public string DeletedMessage { get @@ -61,4 +61,4 @@ public void GetObjectData(SerializationInfo serializationInfo, StreamingContext serializationInfo.AddValue(SerializationName, name); } } -} \ No newline at end of file +} diff --git a/src/runtime/StateSerialization/NoopFormatter.cs b/src/runtime/StateSerialization/NoopFormatter.cs new file mode 100644 index 000000000..f05b7ebb2 --- /dev/null +++ b/src/runtime/StateSerialization/NoopFormatter.cs @@ -0,0 +1,14 @@ +using System; +using System.IO; +using System.Runtime.Serialization; + +namespace Python.Runtime; + +public class NoopFormatter : IFormatter { + public object Deserialize(Stream s) => throw new NotImplementedException(); + public void Serialize(Stream s, object o) {} + + public SerializationBinder? Binder { get; set; } + public StreamingContext Context { get; set; } + public ISurrogateSelector? SurrogateSelector { get; set; } +} diff --git a/src/runtime/StateSerialization/RuntimeData.cs b/src/runtime/StateSerialization/RuntimeData.cs index 204e15b5b..8eda9ce0b 100644 --- a/src/runtime/StateSerialization/RuntimeData.cs +++ b/src/runtime/StateSerialization/RuntimeData.cs @@ -1,7 +1,5 @@ using System; -using System.Collections; using System.Collections.Generic; -using System.Collections.ObjectModel; using System.Diagnostics; using System.IO; using System.Linq; @@ -17,7 +15,34 @@ namespace Python.Runtime { public static class RuntimeData { - private static Type? _formatterType; + + public readonly static Func DefaultFormatterFactory = () => + { + try + { + return new BinaryFormatter(); + } + catch + { + return new NoopFormatter(); + } + }; + + private static Func _formatterFactory { get; set; } = DefaultFormatterFactory; + + public static Func FormatterFactory + { + get => _formatterFactory; + set + { + if (value == null) + throw new ArgumentNullException(nameof(value)); + + _formatterFactory = value; + } + } + + private static Type? _formatterType = null; public static Type? FormatterType { get => _formatterType; @@ -31,6 +56,14 @@ public static Type? FormatterType } } + /// + /// Callback called as a last step in the serialization process + /// + public static Action? PostStashHook { get; set; } = null; + /// + /// Callback called as the first step in the deserialization process + /// + public static Action? PreRestoreHook { get; set; } = null; public static ICLRObjectStorer? WrappersStorer { get; set; } /// @@ -74,6 +107,7 @@ internal static void Stash() using NewReference capsule = PyCapsule_New(mem, IntPtr.Zero, IntPtr.Zero); int res = PySys_SetObject("clr_data", capsule.BorrowOrThrow()); PythonException.ThrowIfIsNotZero(res); + PostStashHook?.Invoke(); } internal static void RestoreRuntimeData() @@ -90,6 +124,7 @@ internal static void RestoreRuntimeData() private static void RestoreRuntimeDataImpl() { + PreRestoreHook?.Invoke(); BorrowedReference capsule = PySys_GetObject("clr_data"); if (capsule.IsNull) { @@ -250,11 +285,102 @@ private static void RestoreRuntimeDataObjects(SharedObjectsState storage) } } + static readonly string serialization_key_namepsace = "pythonnet_serialization_"; + /// + /// Removes the serialization capsule from the `sys` module object. + /// + /// + /// The serialization data must have been set with StashSerializationData + /// + /// The name given to the capsule on the `sys` module object + public static void FreeSerializationData(string key) + { + key = serialization_key_namepsace + key; + BorrowedReference oldCapsule = PySys_GetObject(key); + if (!oldCapsule.IsNull) + { + IntPtr oldData = PyCapsule_GetPointer(oldCapsule, IntPtr.Zero); + Marshal.FreeHGlobal(oldData); + PyCapsule_SetPointer(oldCapsule, IntPtr.Zero); + PySys_SetObject(key, null); + } + } + + /// + /// Stores the data in the argument in a Python capsule and stores + /// the capsule on the `sys` module object with the name . + /// + /// + /// No checks on pre-existing names on the `sys` module object are made. + /// + /// The name given to the capsule on the `sys` module object + /// A MemoryStream that contains the data to be placed in the capsule + public static void StashSerializationData(string key, MemoryStream stream) + { + if (stream.TryGetBuffer(out var data)) + { + IntPtr mem = Marshal.AllocHGlobal(IntPtr.Size + data.Count); + + // store the length of the buffer first + Marshal.WriteIntPtr(mem, (IntPtr)data.Count); + Marshal.Copy(data.Array, data.Offset, mem + IntPtr.Size, data.Count); + + try + { + using NewReference capsule = PyCapsule_New(mem, IntPtr.Zero, IntPtr.Zero); + int res = PySys_SetObject(key, capsule.BorrowOrThrow()); + PythonException.ThrowIfIsNotZero(res); + } + catch + { + Marshal.FreeHGlobal(mem); + } + } + else + { + throw new NotImplementedException($"{nameof(stream)} must be exposable"); + } + + } + + static byte[] emptyBuffer = new byte[0]; + /// + /// Retreives the previously stored data on a Python capsule. + /// Throws if the object corresponding to the parameter + /// on the `sys` module object is not a capsule. + /// + /// The name given to the capsule on the `sys` module object + /// A MemoryStream containing the previously saved serialization data. + /// The stream is empty if no name matches the key. + public static MemoryStream GetSerializationData(string key) + { + BorrowedReference capsule = PySys_GetObject(key); + if (capsule.IsNull) + { + // nothing to do. + return new MemoryStream(emptyBuffer, writable:false); + } + var ptr = PyCapsule_GetPointer(capsule, IntPtr.Zero); + if (ptr == IntPtr.Zero) + { + // The PyCapsule API returns NULL on error; NULL cannot be stored + // as a capsule's value + PythonException.ThrowIfIsNull(null); + } + var len = (int)Marshal.ReadIntPtr(ptr); + byte[] buffer = new byte[len]; + Marshal.Copy(ptr+IntPtr.Size, buffer, 0, len); + return new MemoryStream(buffer, writable:false); + } + internal static IFormatter CreateFormatter() { - return FormatterType != null ? - (IFormatter)Activator.CreateInstance(FormatterType) - : new BinaryFormatter(); + + if (FormatterType != null) + { + return (IFormatter)Activator.CreateInstance(FormatterType); + } + return FormatterFactory(); } } } diff --git a/src/runtime/TypeManager.cs b/src/runtime/TypeManager.cs index e0a78ba49..559d5148e 100644 --- a/src/runtime/TypeManager.cs +++ b/src/runtime/TypeManager.cs @@ -374,7 +374,7 @@ static PyTuple GetBaseTypeTuple(Type clrType) return new PyTuple(bases); } - internal static NewReference CreateSubType(BorrowedReference py_name, BorrowedReference py_base_type, BorrowedReference dictRef) + internal static NewReference CreateSubType(BorrowedReference py_name, ClassBase py_base_type, IList interfaces, BorrowedReference dictRef) { // Utility to create a subtype of a managed type with the ability for the // a python subtype able to override the managed implementation @@ -415,17 +415,10 @@ internal static NewReference CreateSubType(BorrowedReference py_name, BorrowedRe } // create the new managed type subclassing the base managed type - if (ManagedType.GetManagedObject(py_base_type) is ClassBase baseClass) - { - return ReflectedClrType.CreateSubclass(baseClass, name, - ns: (string?)namespaceStr, - assembly: (string?)assembly, - dict: dictRef); - } - else - { - return Exceptions.RaiseTypeError("invalid base class, expected CLR class type"); - } + return ReflectedClrType.CreateSubclass(py_base_type, interfaces, name, + ns: (string?)namespaceStr, + assembly: (string?)assembly, + dict: dictRef); } internal static IntPtr WriteMethodDef(IntPtr mdef, IntPtr name, IntPtr func, PyMethodFlags flags, IntPtr doc) diff --git a/src/runtime/Types/ArrayObject.cs b/src/runtime/Types/ArrayObject.cs index b95934baf..f220d53fb 100644 --- a/src/runtime/Types/ArrayObject.cs +++ b/src/runtime/Types/ArrayObject.cs @@ -247,6 +247,12 @@ public static NewReference mp_subscript(BorrowedReference ob, BorrowedReference /// public static int mp_ass_subscript(BorrowedReference ob, BorrowedReference idx, BorrowedReference v) { + if (v.IsNull) + { + Exceptions.RaiseTypeError("'System.Array' object does not support item deletion"); + return -1; + } + var obj = (CLRObject)GetManagedObject(ob)!; var items = (Array)obj.inst; Type itemType = obj.inst.GetType().GetElementType(); diff --git a/src/runtime/Types/ClassBase.cs b/src/runtime/Types/ClassBase.cs index 7296a1900..2d6ce8a47 100644 --- a/src/runtime/Types/ClassBase.cs +++ b/src/runtime/Types/ClassBase.cs @@ -25,6 +25,7 @@ internal class ClassBase : ManagedType, IDeserializationCallback [NonSerialized] internal List dotNetMembers = new(); internal Indexer? indexer; + internal MethodBinder? del; internal readonly Dictionary richcompare = new(); internal MaybeType type; @@ -465,6 +466,11 @@ static int mp_ass_subscript_impl(BorrowedReference ob, BorrowedReference idx, Bo // with the index arg (method binders expect arg tuples). NewReference argsTuple = default; + if (v.IsNull) + { + return DelImpl(ob, idx, cls); + } + if (!Runtime.PyTuple_Check(idx)) { argsTuple = Runtime.PyTuple_New(1); @@ -497,13 +503,45 @@ static int mp_ass_subscript_impl(BorrowedReference ob, BorrowedReference idx, Bo // Add value to argument list Runtime.PyTuple_SetItem(real.Borrow(), i, v); - cls.indexer.SetItem(ob, real.Borrow()); + using var result = cls.indexer.SetItem(ob, real.Borrow()); + return result.IsNull() ? -1 : 0; + } + + /// Implements __delitem__ (del x[...]) for IList<T> and IDictionary<TKey, TValue>. + private static int DelImpl(BorrowedReference ob, BorrowedReference idx, ClassBase cls) + { + if (cls.del is null) + { + Exceptions.SetError(Exceptions.TypeError, "object does not support item deletion"); + return -1; + } + + if (Runtime.PyTuple_Check(idx)) + { + Exceptions.SetError(Exceptions.TypeError, "multi-index deletion not supported"); + return -1; + } + + using var argsTuple = Runtime.PyTuple_New(1); + Runtime.PyTuple_SetItem(argsTuple.Borrow(), 0, idx); + using var result = cls.del.Invoke(ob, argsTuple.Borrow(), kw: null); + if (result.IsNull()) + return -1; - if (Exceptions.ErrorOccurred()) + if (Runtime.PyBool_CheckExact(result.Borrow())) { + if (Runtime.PyObject_IsTrue(result.Borrow()) != 0) + return 0; + + Exceptions.SetError(Exceptions.KeyError, "key not found"); return -1; } + if (!result.IsNone()) + { + Exceptions.warn("unsupported return type for __delitem__", Exceptions.TypeError); + } + return 0; } diff --git a/src/runtime/Types/ClassDerived.cs b/src/runtime/Types/ClassDerived.cs index cf6d9b16b..592eefd55 100644 --- a/src/runtime/Types/ClassDerived.cs +++ b/src/runtime/Types/ClassDerived.cs @@ -144,6 +144,7 @@ internal static NewReference ToPython(IPythonDerivedType obj) /// internal static Type CreateDerivedType(string name, Type baseType, + IList typeInterfaces, BorrowedReference py_dict, string? namespaceStr, string? assemblyName, @@ -163,7 +164,9 @@ internal static Type CreateDerivedType(string name, ModuleBuilder moduleBuilder = GetModuleBuilder(assemblyName, moduleName); Type baseClass = baseType; - var interfaces = new List { typeof(IPythonDerivedType) }; + var interfaces = new HashSet { typeof(IPythonDerivedType) }; + foreach(var interfaceType in typeInterfaces) + interfaces.Add(interfaceType); // if the base type is an interface then use System.Object as the base class // and add the base type to the list of interfaces this new class will implement. @@ -214,13 +217,17 @@ internal static Type CreateDerivedType(string name, } } - // override any virtual methods not already overridden by the properties above - MethodInfo[] methods = baseType.GetMethods(); + // override any virtual not already overridden by the properties above + // also override any interface method. + var methods = baseType.GetMethods().Concat(interfaces.SelectMany(x => x.GetMethods())); var virtualMethods = new HashSet(); foreach (MethodInfo method in methods) { if (!method.Attributes.HasFlag(MethodAttributes.Virtual) | - method.Attributes.HasFlag(MethodAttributes.Final)) + method.Attributes.HasFlag(MethodAttributes.Final) + // overriding generic virtual methods is not supported + // so a call to that should be deferred to the base class method. + || method.IsGenericMethod) { continue; } diff --git a/src/runtime/Types/ClassObject.cs b/src/runtime/Types/ClassObject.cs index cc42039e8..f0585ffa6 100644 --- a/src/runtime/Types/ClassObject.cs +++ b/src/runtime/Types/ClassObject.cs @@ -1,7 +1,9 @@ using System; using System.Diagnostics; +using System.Globalization; using System.Linq; using System.Reflection; +using System.Runtime.InteropServices; using System.Runtime.Serialization; namespace Python.Runtime @@ -47,6 +49,55 @@ internal NewReference GetDocString() return Runtime.PyString_FromString(str); } + private static string ConvertFlags(Enum value) + { + Type primitiveType = value.GetType().GetEnumUnderlyingType(); + string format = "X" + (Marshal.SizeOf(primitiveType) * 2).ToString(CultureInfo.InvariantCulture); + var primitive = (IFormattable)Convert.ChangeType(value, primitiveType); + return "0x" + primitive.ToString(format, null); + + } + + private static string ConvertValue(Enum value) + { + Type primitiveType = value.GetType().GetEnumUnderlyingType(); + return Convert.ChangeType(value, primitiveType).ToString()!; + } + + /// + /// given an enum, write a __repr__ string formatted in the same + /// way as a python repr string. Something like: + /// '<Color.GREEN: 2>'; + /// with a binary value for [Flags] enums + /// + /// Instace of the enum object + /// + private static string GetEnumReprString(Enum inst) + { + var obType = inst.GetType(); + + string strValue2 = obType.IsFlagsEnum() ? ConvertFlags(inst) : ConvertValue(inst); + + var repr = $"<{obType.Name}.{inst}: {strValue2}>"; + return repr; + } + + /// + /// ClassObject __repr__ implementation. + /// + public new static NewReference tp_repr(BorrowedReference ob) + { + if (GetManagedObject(ob) is not CLRObject co) + { + return Exceptions.RaiseTypeError("invalid object"); + } + if (co.inst.GetType().IsEnum) + { + return Runtime.PyString_FromString(GetEnumReprString((Enum)co.inst)); + } + + return ClassBase.tp_repr(ob); + } /// /// Implements __new__ for reflected classes and value types. diff --git a/src/runtime/Types/ClrObject.cs b/src/runtime/Types/ClrObject.cs index 4cf9062cb..afa136414 100644 --- a/src/runtime/Types/ClrObject.cs +++ b/src/runtime/Types/ClrObject.cs @@ -11,10 +11,15 @@ internal sealed class CLRObject : ManagedType { internal readonly object inst; + internal static bool creationBlocked = false; + // "borrowed" references internal static readonly HashSet reflectedObjects = new(); static NewReference Create(object ob, BorrowedReference tp) { + if (creationBlocked) + throw new InvalidOperationException("Reflected objects should not be created anymore."); + Debug.Assert(tp != null); var py = Runtime.PyType_GenericAlloc(tp, 0); @@ -61,6 +66,9 @@ internal static void Restore(object ob, BorrowedReference pyHandle, Dictionary? context) { + if (creationBlocked) + throw new InvalidOperationException("Reflected objects should not be loaded anymore."); + base.OnLoad(ob, context); GCHandle gc = GCHandle.Alloc(this); SetGCHandle(ob, gc); diff --git a/src/runtime/Types/Indexer.cs b/src/runtime/Types/Indexer.cs index 4903b6f76..fe6dab4c9 100644 --- a/src/runtime/Types/Indexer.cs +++ b/src/runtime/Types/Indexer.cs @@ -50,9 +50,9 @@ internal NewReference GetItem(BorrowedReference inst, BorrowedReference args) } - internal void SetItem(BorrowedReference inst, BorrowedReference args) + internal NewReference SetItem(BorrowedReference inst, BorrowedReference args) { - SetterBinder.Invoke(inst, args, null); + return SetterBinder.Invoke(inst, args, null); } internal bool NeedsDefaultArgs(BorrowedReference args) diff --git a/src/runtime/Types/MetaType.cs b/src/runtime/Types/MetaType.cs index 5b59f5139..57fcaa232 100644 --- a/src/runtime/Types/MetaType.cs +++ b/src/runtime/Types/MetaType.cs @@ -1,5 +1,8 @@ using System; +using System.Collections.Generic; using System.Diagnostics; +using System.Linq; +using System.Reflection; using System.Runtime.InteropServices; using System.Runtime.Serialization; @@ -79,41 +82,80 @@ public static NewReference tp_new(BorrowedReference tp, BorrowedReference args, BorrowedReference bases = Runtime.PyTuple_GetItem(args, 1); BorrowedReference dict = Runtime.PyTuple_GetItem(args, 2); - // We do not support multiple inheritance, so the bases argument - // should be a 1-item tuple containing the type we are subtyping. - // That type must itself have a managed implementation. We check - // that by making sure its metatype is the CLR metatype. + // Extract interface types and base class types. + var interfaces = new List(); - if (Runtime.PyTuple_Size(bases) != 1) - { - return Exceptions.RaiseTypeError("cannot use multiple inheritance with managed classes"); - } + // More than one base type case be declared, but an exception will be thrown + // if more than one is a class/not an interface. + var baseTypes = new List(); - BorrowedReference base_type = Runtime.PyTuple_GetItem(bases, 0); - BorrowedReference mt = Runtime.PyObject_TYPE(base_type); - - if (!(mt == PyCLRMetaType || mt == Runtime.PyTypeType)) + var baseClassCount = Runtime.PyTuple_Size(bases); + if (baseClassCount == 0) { - return Exceptions.RaiseTypeError("invalid metatype"); + return Exceptions.RaiseTypeError("zero base classes "); } - // Ensure that the reflected type is appropriate for subclassing, - // disallowing subclassing of delegates, enums and array types. - - if (GetManagedObject(base_type) is ClassBase cb) + for (nint i = 0; i < baseClassCount; i++) { - try + var baseTypeIt = Runtime.PyTuple_GetItem(bases, (int)i); + + if (GetManagedObject(baseTypeIt) is ClassBase classBaseIt) { - if (!cb.CanSubclass()) + if (!classBaseIt.type.Valid) + { + return Exceptions.RaiseTypeError("Invalid type used as a super type."); + } + if (classBaseIt.type.Value.IsInterface) { - return Exceptions.RaiseTypeError("delegates, enums and array types cannot be subclassed"); + interfaces.Add(classBaseIt.type.Value); } + else + { + baseTypes.Add(classBaseIt); + } + } + else + { + return Exceptions.RaiseTypeError("Non .NET type used as super class for meta type. This is not supported."); } - catch (SerializationException) + } + // if the base type count is 0, there might still be interfaces to implement. + if (baseTypes.Count == 0) + { + baseTypes.Add(new ClassBase(typeof(object))); + } + + // Multiple inheritance is not supported, unless the other types are interfaces + if (baseTypes.Count > 1) + { + var types = string.Join(", ", baseTypes.Select(baseType => baseType.type.Value)); + return Exceptions.RaiseTypeError($"Multiple inheritance with managed classes cannot be used. Types: {types} "); + } + + // check if the list of interfaces contains no duplicates. + if (interfaces.Distinct().Count() != interfaces.Count) + { + // generate a string containing the problematic types. + var duplicateTypes = interfaces.GroupBy(type => type) + .Where(typeGroup => typeGroup.Count() > 1) + .Select(typeGroup => typeGroup.Key); + var duplicateTypesString = string.Join(", ", duplicateTypes); + + return Exceptions.RaiseTypeError($"An interface can only be implemented once. Duplicate types: {duplicateTypesString}"); + } + + var cb = baseTypes[0]; + try + { + if (!cb.CanSubclass()) { - return Exceptions.RaiseTypeError($"Underlying C# Base class {cb.type} has been deleted"); + return Exceptions.RaiseTypeError("delegates, enums and array types cannot be subclassed"); } } + catch (SerializationException) + { + return Exceptions.RaiseTypeError($"Underlying C# Base class {cb.type} has been deleted"); + } BorrowedReference slots = Runtime.PyDict_GetItem(dict, PyIdentifier.__slots__); if (slots != null) @@ -130,10 +172,12 @@ public static NewReference tp_new(BorrowedReference tp, BorrowedReference args, using var clsDict = new PyDict(dict); if (clsDict.HasKey("__assembly__") || clsDict.HasKey("__namespace__")) { - return TypeManager.CreateSubType(name, base_type, clsDict); + return TypeManager.CreateSubType(name, baseTypes[0], interfaces, clsDict); } } + var base_type = Runtime.PyTuple_GetItem(bases, 0); + // otherwise just create a basic type without reflecting back into the managed side. IntPtr func = Util.ReadIntPtr(Runtime.PyTypeType, TypeOffset.tp_new); NewReference type = NativeCall.Call_3(func, tp, args, kw); diff --git a/src/runtime/Types/MethodBinding.cs b/src/runtime/Types/MethodBinding.cs index 334d705a6..bfe22b0f3 100644 --- a/src/runtime/Types/MethodBinding.cs +++ b/src/runtime/Types/MethodBinding.cs @@ -237,6 +237,7 @@ public static NewReference tp_call(BorrowedReference ob, BorrowedReference args, } } } + return self.m.Invoke(target is null ? BorrowedReference.Null : target, args, kw, self.info.UnsafeValue); } finally diff --git a/src/runtime/Types/MethodObject.cs b/src/runtime/Types/MethodObject.cs index 05198da76..12484d301 100644 --- a/src/runtime/Types/MethodObject.cs +++ b/src/runtime/Types/MethodObject.cs @@ -19,20 +19,20 @@ internal class MethodObject : ExtensionType { [NonSerialized] private MethodBase[]? _info = null; + private readonly List infoList; internal string name; internal readonly MethodBinder binder; internal bool is_static = false; - internal PyString? doc; internal MaybeType type; - public MethodObject(MaybeType type, string name, MethodBase[] info, bool allow_threads) + public MethodObject(MaybeType type, string name, MethodBase[] info, bool allow_threads, bool argsReversed = false) { this.type = type; this.name = name; this.infoList = new List(); - binder = new MethodBinder(); + binder = new MethodBinder() { argsReversed = argsReversed }; foreach (MethodBase item in info) { this.infoList.Add(item); @@ -45,8 +45,8 @@ public MethodObject(MaybeType type, string name, MethodBase[] info, bool allow_t binder.allow_threads = allow_threads; } - public MethodObject(MaybeType type, string name, MethodBase[] info) - : this(type, name, info, allow_threads: AllowThreads(info)) + public MethodObject(MaybeType type, string name, MethodBase[] info, bool argsReversed = false) + : this(type, name, info, allow_threads: AllowThreads(info), argsReversed) { } diff --git a/src/runtime/Types/OperatorMethod.cs b/src/runtime/Types/OperatorMethod.cs index e3cc23370..a2ca73982 100644 --- a/src/runtime/Types/OperatorMethod.cs +++ b/src/runtime/Types/OperatorMethod.cs @@ -177,17 +177,14 @@ public static string ReversePyMethodName(string pyName) } /// - /// Check if the method is performing a reverse operation. + /// Check if the method should have a reversed operation. /// /// The operator method. /// - public static bool IsReverse(MethodBase method) + public static bool HaveReverse(MethodBase method) { - Type primaryType = method.IsOpsHelper() - ? method.DeclaringType.GetGenericArguments()[0] - : method.DeclaringType; - Type leftOperandType = method.GetParameters()[0].ParameterType; - return leftOperandType != primaryType; + var pi = method.GetParameters(); + return OpMethodMap.ContainsKey(method.Name) && pi.Length == 2; } public static void FilterMethods(MethodBase[] methods, out MethodBase[] forwardMethods, out MethodBase[] reverseMethods) @@ -196,14 +193,11 @@ public static void FilterMethods(MethodBase[] methods, out MethodBase[] forwardM var reverseMethodsList = new List(); foreach (var method in methods) { - if (IsReverse(method)) + forwardMethodsList.Add(method); + if (HaveReverse(method)) { reverseMethodsList.Add(method); - } else - { - forwardMethodsList.Add(method); } - } forwardMethods = forwardMethodsList.ToArray(); reverseMethods = reverseMethodsList.ToArray(); diff --git a/src/runtime/Types/ReflectedClrType.cs b/src/runtime/Types/ReflectedClrType.cs index d3d89bdb8..df9b26c29 100644 --- a/src/runtime/Types/ReflectedClrType.cs +++ b/src/runtime/Types/ReflectedClrType.cs @@ -30,22 +30,29 @@ public static ReflectedClrType GetOrCreate(Type type) return pyType; } - // Ensure, that matching Python type exists first. - // It is required for self-referential classes - // (e.g. with members, that refer to the same class) - pyType = AllocateClass(type); - ClassManager.cache.Add(type, pyType); + try + { + // Ensure, that matching Python type exists first. + // It is required for self-referential classes + // (e.g. with members, that refer to the same class) + pyType = AllocateClass(type); + ClassManager.cache.Add(type, pyType); - var impl = ClassManager.CreateClass(type); + var impl = ClassManager.CreateClass(type); - TypeManager.InitializeClassCore(type, pyType, impl); + TypeManager.InitializeClassCore(type, pyType, impl); - ClassManager.InitClassBase(type, impl, pyType); + ClassManager.InitClassBase(type, impl, pyType); - // Now we force initialize the Python type object to reflect the given - // managed type, filling the Python type slots with thunks that - // point to the managed methods providing the implementation. - TypeManager.InitializeClass(pyType, impl, type); + // Now we force initialize the Python type object to reflect the given + // managed type, filling the Python type slots with thunks that + // point to the managed methods providing the implementation. + TypeManager.InitializeClass(pyType, impl, type); + } + catch (Exception e) + { + throw new InternalPythonnetException($"Failed to create Python type for {type.FullName}", e); + } return pyType; } @@ -68,7 +75,7 @@ internal void Restore(ClassBase cb) TypeManager.InitializeClass(this, cb, cb.type.Value); } - internal static NewReference CreateSubclass(ClassBase baseClass, + internal static NewReference CreateSubclass(ClassBase baseClass, IList interfaces, string name, string? assembly, string? ns, BorrowedReference dict) { @@ -76,6 +83,7 @@ internal static NewReference CreateSubclass(ClassBase baseClass, { Type subType = ClassDerivedObject.CreateDerivedType(name, baseClass.type.Value, + interfaces, dict, ns, assembly); diff --git a/src/runtime/Util/Encodings.cs b/src/runtime/Util/Encodings.cs new file mode 100644 index 000000000..d5a0c6ff8 --- /dev/null +++ b/src/runtime/Util/Encodings.cs @@ -0,0 +1,10 @@ +using System; +using System.Text; + +namespace Python.Runtime; + +static class Encodings { + public static System.Text.Encoding UTF8 = new UTF8Encoding(false, true); + public static System.Text.Encoding UTF16 = new UnicodeEncoding(!BitConverter.IsLittleEndian, false, true); + public static System.Text.Encoding UTF32 = new UTF32Encoding(!BitConverter.IsLittleEndian, false, true); +} diff --git a/src/runtime/Util/ReflectionUtil.cs b/src/runtime/Util/ReflectionUtil.cs index 58d0a506e..0fad2d4b2 100644 --- a/src/runtime/Util/ReflectionUtil.cs +++ b/src/runtime/Util/ReflectionUtil.cs @@ -53,4 +53,11 @@ public static BindingFlags GetBindingFlags(this PropertyInfo property) flags |= accessor.IsPublic ? BindingFlags.Public : BindingFlags.NonPublic; return flags; } + + public static Type? TryGetGenericDefinition(this Type type) + { + if (type is null) throw new ArgumentNullException(nameof(type)); + + return type.IsConstructedGenericType ? type.GetGenericTypeDefinition() : null; + } } diff --git a/src/testing/classtest.cs b/src/testing/classtest.cs index 68c0d8c55..993afdfc9 100644 --- a/src/testing/classtest.cs +++ b/src/testing/classtest.cs @@ -1,3 +1,4 @@ +using System; using System.Collections; namespace Python.Test @@ -59,4 +60,15 @@ public ClassCtorTest2(string v) internal class InternalClass { } + + public class SimpleClass + { + public static void TestObject(object obj) + { + if ((!(obj is ISayHello1 && obj is SimpleClass))) + { + throw new Exception("Expected ISayHello and SimpleClass instance"); + } + } + } } diff --git a/src/testing/generictest.cs b/src/testing/generictest.cs index 238435811..b333910c2 100644 --- a/src/testing/generictest.cs +++ b/src/testing/generictest.cs @@ -136,4 +136,12 @@ public static T[] EchoRange(T[] items) return items; } } + + public abstract class GenericVirtualMethodTest + { + public virtual Q VirtMethod(Q arg1) + { + return arg1; + } + } } diff --git a/tests/domain_tests/TestRunner.cs b/tests/domain_tests/TestRunner.cs index 4f6a3ea28..bbee81b3d 100644 --- a/tests/domain_tests/TestRunner.cs +++ b/tests/domain_tests/TestRunner.cs @@ -1132,6 +1132,66 @@ import System ", }, + new TestCase + { + Name = "test_serialize_unserializable_object", + DotNetBefore = @" + namespace TestNamespace + { + public class NotSerializableTextWriter : System.IO.TextWriter + { + override public System.Text.Encoding Encoding { get { return System.Text.Encoding.ASCII;} } + } + [System.Serializable] + public static class SerializableWriter + { + private static System.IO.TextWriter _writer = null; + public static System.IO.TextWriter Writer {get { return _writer; }} + public static void CreateInternalWriter() + { + _writer = System.IO.TextWriter.Synchronized(new NotSerializableTextWriter()); + } + } + } +", + DotNetAfter = @" + namespace TestNamespace + { + public class NotSerializableTextWriter : System.IO.TextWriter + { + override public System.Text.Encoding Encoding { get { return System.Text.Encoding.ASCII;} } + } + [System.Serializable] + public static class SerializableWriter + { + private static System.IO.TextWriter _writer = null; + public static System.IO.TextWriter Writer {get { return _writer; }} + public static void CreateInternalWriter() + { + _writer = System.IO.TextWriter.Synchronized(new NotSerializableTextWriter()); + } + } + } + ", + PythonCode = @" +import sys + +def before_reload(): + import clr + import System + clr.AddReference('DomainTests') + import TestNamespace + TestNamespace.SerializableWriter.CreateInternalWriter(); + sys.__obj = TestNamespace.SerializableWriter.Writer + sys.__obj.WriteLine('test') + +def after_reload(): + import clr + import System + sys.__obj.WriteLine('test') + + ", + } }; /// @@ -1142,7 +1202,59 @@ import System const string CaseRunnerTemplate = @" using System; using System.IO; +using System.Runtime.Serialization; +using System.Runtime.Serialization.Formatters.Binary; using Python.Runtime; + +namespace Serialization +{{ + // Classes in this namespace is mostly useful for test_serialize_unserializable_object + class NotSerializableSerializer : ISerializationSurrogate + {{ + public NotSerializableSerializer() + {{ + }} + public void GetObjectData(object obj, SerializationInfo info, StreamingContext context) + {{ + info.AddValue(""notSerialized_tp"", obj.GetType()); + }} + public object SetObjectData(object obj, SerializationInfo info, StreamingContext context, ISurrogateSelector selector) + {{ + if (info == null) + {{ + return null; + }} + Type typeObj = info.GetValue(""notSerialized_tp"", typeof(Type)) as Type; + if (typeObj == null) + {{ + return null; + }} + + obj = Activator.CreateInstance(typeObj); + return obj; + }} + }} + class NonSerializableSelector : SurrogateSelector + {{ + public override ISerializationSurrogate GetSurrogate(Type type, StreamingContext context, out ISurrogateSelector selector) + {{ + if (type == null) + {{ + throw new ArgumentNullException(); + }} + selector = (ISurrogateSelector)this; + if (type.IsSerializable) + {{ + return null; // use whichever default + }} + else + {{ + return (ISerializationSurrogate)(new NotSerializableSerializer()); + }} + }} + }} +}} + namespace CaseRunner {{ class CaseRunner @@ -1151,6 +1263,11 @@ public static int Main() {{ try {{ + RuntimeData.FormatterFactory = () => + {{ + return new BinaryFormatter(){{SurrogateSelector = new Serialization.NonSerializableSelector()}}; + }}; + PythonEngine.Initialize(); using (Py.GIL()) {{ diff --git a/tests/domain_tests/test_domain_reload.py b/tests/domain_tests/test_domain_reload.py index 8999e481b..1e5e8e81b 100644 --- a/tests/domain_tests/test_domain_reload.py +++ b/tests/domain_tests/test_domain_reload.py @@ -88,3 +88,6 @@ def test_nested_type(): def test_import_after_reload(): _run_test("import_after_reload") + +def test_import_after_reload(): + _run_test("test_serialize_unserializable_object") \ No newline at end of file diff --git a/tests/test_conversion.py b/tests/test_conversion.py index bb686dd52..dd70f900a 100644 --- a/tests/test_conversion.py +++ b/tests/test_conversion.py @@ -510,6 +510,9 @@ def test_string_conversion(): ob.StringField = System.String(u'\uffff\uffff') assert ob.StringField == u'\uffff\uffff' + ob.StringField = System.String("\ufeffbom") + assert ob.StringField == "\ufeffbom" + ob.StringField = None assert ob.StringField is None diff --git a/tests/test_enum.py b/tests/test_enum.py index f24f95b36..3d3edba10 100644 --- a/tests/test_enum.py +++ b/tests/test_enum.py @@ -143,6 +143,16 @@ def test_enum_undefined_value(): Test.FieldTest().EnumField = Test.ShortEnum(20, True) +def test_enum_repr(): + """Test enumeration repr.""" + from System import DayOfWeek + + assert repr(DayOfWeek.Monday) == "" + + assert repr(Test.FlagsEnum(7)) == "" + assert repr(Test.FlagsEnum(8)) == "" + + def test_enum_conversion(): """Test enumeration conversion.""" ob = Test.FieldTest() diff --git a/tests/test_indexer.py b/tests/test_indexer.py index 8cf3150ba..108573f0d 100644 --- a/tests/test_indexer.py +++ b/tests/test_indexer.py @@ -668,3 +668,39 @@ def test_public_inherited_overloaded_indexer(): with pytest.raises(TypeError): ob[[]] + +def test_del_indexer_dict(): + """Test deleting indexers (__delitem__).""" + from System.Collections.Generic import Dictionary, KeyNotFoundException + d = Dictionary[str, str]() + d["delme"] = "1" + with pytest.raises(KeyError): + del d["nonexistent"] + del d["delme"] + with pytest.raises(KeyError): + del d["delme"] + +def test_del_indexer_list(): + """Test deleting indexers (__delitem__).""" + from System import ArgumentOutOfRangeException + from System.Collections.Generic import List + l = List[str]() + l.Add("1") + with pytest.raises(ArgumentOutOfRangeException): + del l[3] + del l[0] + assert len(l) == 0 + +def test_del_indexer_array(): + """Test deleting indexers (__delitem__).""" + from System import Array + l = Array[str](0) + with pytest.raises(TypeError): + del l[0] + +def test_del_indexer_absent(): + """Test deleting indexers (__delitem__).""" + from System import Uri + l = Uri("http://www.example.com") + with pytest.raises(TypeError): + del l[0] diff --git a/tests/test_method.py b/tests/test_method.py index b86bbd6b4..c70200c7e 100644 --- a/tests/test_method.py +++ b/tests/test_method.py @@ -1023,6 +1023,7 @@ def test_getting_method_overloads_binding_does_not_leak_ref_count(): refCount = sys.getrefcount(PlainOldClass().OverloadedMethod.Overloads) assert refCount == 1 +@pytest.mark.xfail(reason="Fails locally, need to investigate later", strict=False) def test_getting_method_overloads_binding_does_not_leak_memory(): """Test that managed object is freed after calling overloaded method. Issue #691""" diff --git a/tests/test_module.py b/tests/test_module.py index ddfa7bb36..0c20dcfc0 100644 --- a/tests/test_module.py +++ b/tests/test_module.py @@ -344,6 +344,20 @@ def test_clr_add_reference(): with pytest.raises(FileNotFoundException): AddReference("somethingtotallysilly") + +def test_clr_add_reference_bad_path(): + import sys + from clr import AddReference + from System.IO import FileNotFoundException + bad_path = "hello\0world" + sys.path.append(bad_path) + try: + with pytest.raises(FileNotFoundException): + AddReference("test_clr_add_reference_bad_path") + finally: + sys.path.remove(bad_path) + + def test_clr_get_clr_type(): """Test clr.GetClrType().""" from clr import GetClrType diff --git a/tests/test_subclass.py b/tests/test_subclass.py index a51e89da3..c6ab7650f 100644 --- a/tests/test_subclass.py +++ b/tests/test_subclass.py @@ -9,7 +9,7 @@ import System import pytest from Python.Test import (IInterfaceTest, SubClassTest, EventArgsTest, - FunctionsTest, IGenericInterface) + FunctionsTest, IGenericInterface, GenericVirtualMethodTest, SimpleClass, ISayHello1) from System.Collections.Generic import List @@ -327,3 +327,20 @@ def test_generic_interface(): obj = GenericInterfaceImpl() SpecificInterfaceUser(obj, Int32(0)) GenericInterfaceUser[Int32](obj, Int32(0)) + +def test_virtual_generic_method(): + class OverloadingSubclass(GenericVirtualMethodTest): + __namespace__ = "test_virtual_generic_method_cls" + class OverloadingSubclass2(OverloadingSubclass): + __namespace__ = "test_virtual_generic_method_cls" + obj = OverloadingSubclass() + assert obj.VirtMethod[int](5) == 5 + obj = OverloadingSubclass2() + assert obj.VirtMethod[int](5) == 5 + +def test_implement_interface_and_class(): + class DualSubClass0(ISayHello1, SimpleClass): + __namespace__ = "Test" + def SayHello(self): + return "hello" + obj = DualSubClass0() diff --git a/tools/geninterop/geninterop.py b/tools/geninterop/geninterop.py index 78e4d45c2..6d80bcfa6 100755 --- a/tools/geninterop/geninterop.py +++ b/tools/geninterop/geninterop.py @@ -76,6 +76,8 @@ def visit(self, node): self.visit_ptrdecl(node) elif isinstance(node, c_ast.IdentifierType): self.visit_identifier(node) + elif isinstance(node, c_ast.Union): + self.visit_union(node) def visit_ast(self, ast): for _name, node in ast.children(): @@ -119,6 +121,16 @@ def visit_identifier(self, identifier): type_name = " ".join(identifier.names) self._add_struct_member(type_name) + def visit_union(self, union): + # Treat the field as if it was just the first declaration for now. This + # is not really correct, but handles the one case that is relevant for + # us right now (ob_refcnt being "split" in Python 3.12) + if self._struct_members_stack and union.decls: + decl = union.decls[0] + self._struct_members_stack.pop(0) + self._struct_members_stack.insert(0, decl.name) + self.visit(decl) + def _add_struct_member(self, type_name): if not (self._struct_stack and self._struct_members_stack): return @@ -245,7 +257,7 @@ def gen_interop_head(writer, version, abi_flags): // Auto-generated by {filename}. // DO NOT MODIFY BY HAND. -// Python {".".join(version[:2])}: ABI flags: '{abi_flags}' +// Python {".".join(map(str, version[:2]))}: ABI flags: '{abi_flags}' // ReSharper disable InconsistentNaming // ReSharper disable IdentifierTypo diff --git a/version.txt b/version.txt index cb2b00e4f..0f9d6b15d 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -3.0.1 +3.1.0-dev pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy