From daf3e1f537cf891bb7eeaf08af18a1bc3fade05e Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Thu, 27 Feb 2025 09:15:36 +0100 Subject: [PATCH 1/3] Update HPy inlined files: b0fbdf73 --- .../include/hpy/version.h | 4 - .../src/hpytest/conftest.py | 2 +- .../src/hpytest/debug/__init__.py | 0 .../src/hpytest/debug/test_charptr.py | 4 +- .../src/hpytest/debug/test_handles_invalid.py | 13 +- .../src/hpytest/debug/test_misc.py | 2 +- .../src/hpytest/hpy_devel/__init__.py | 0 .../src/hpytest/hpy_devel/test_distutils.py | 2 +- .../src/hpytest/support.py | 28 +-- .../src/hpytest/test_00_basic.py | 22 +- .../src/hpytest/test_argparse.py | 18 +- .../src/hpytest/test_capsule.py | 4 + .../src/hpytest/test_eval.py | 2 +- .../src/hpytest/test_hpyfield.py | 3 +- .../src/hpytest/test_hpyiter.py | 90 +++++++ .../src/hpytest/test_hpylist.py | 35 +++ .../src/hpytest/test_hpymodule.py | 10 +- .../src/hpytest/test_hpyslice.py | 28 +++ .../src/hpytest/test_hpytype.py | 10 +- .../src/hpytest/test_hpytype_legacy.py | 52 ++++ .../src/hpytest/test_hpyunicode.py | 10 +- .../src/hpytest/test_importing.py | 2 +- .../src/hpytest/test_legacy_forbidden.py | 2 - .../src/hpytest/test_object.py | 223 ++++++++++++++++++ .../src/hpytest/test_slots.py | 26 +- .../include/hpy.h | 0 .../include/hpy/autogen_hpyfunc_declare.h | 0 .../include/hpy/autogen_hpyslot.h | 2 + .../include/hpy/cpy_types.h | 0 .../include/hpy/cpython/autogen_api_impl.h | 39 ++- .../include/hpy/cpython/autogen_ctx.h | 1 + .../hpy/cpython/autogen_hpyfunc_trampolines.h | 0 .../include/hpy/cpython/hpyfunc_trampolines.h | 0 .../include/hpy/cpython/misc.h | 15 +- .../include/hpy/forbid_python_h/Python.h | 0 .../include/hpy/hpydef.h | 2 +- .../include/hpy/hpyexports.h | 0 .../include/hpy/hpyfunc.h | 0 .../include/hpy/hpymodule.h | 10 +- .../include/hpy/hpytype.h | 18 +- .../include/hpy/inline_helpers.h | 0 .../include/hpy/macros.h | 0 .../include/hpy/runtime/argparse.h | 0 .../include/hpy/runtime/buildvalue.h | 0 .../include/hpy/runtime/ctx_funcs.h | 4 +- .../include/hpy/runtime/ctx_module.h | 0 .../include/hpy/runtime/ctx_type.h | 0 .../include/hpy/runtime/format.h | 0 .../include/hpy/runtime/helpers.h | 0 .../include/hpy/runtime/structseq.h | 0 .../include/hpy/universal/autogen_ctx.h | 12 +- .../universal/autogen_hpyfunc_trampolines.h | 0 .../hpy/universal/autogen_trampolines.h | 38 ++- .../hpy/universal/hpyfunc_trampolines.h | 0 .../include/hpy/universal/misc_trampolines.h | 0 .../include/hpy/version.h | 4 + .../src/debug/_debugmod.c | 2 +- .../src/debug/autogen_debug_ctx_init.h | 21 +- .../src/debug/autogen_debug_wrappers.c | 100 ++++++++ .../src/debug/debug_ctx.c | 5 +- .../src/trace/autogen_trace_ctx_init.h | 25 +- .../src/trace/autogen_trace_func_table.c | 14 +- .../src/trace/autogen_trace_wrappers.c | 119 +++++++++- .../modules/hpy/devel/__init__.py | 2 +- .../hpy/devel/src/runtime/ctx_object.c | 14 +- .../modules/hpy/devel/src/runtime/ctx_tuple.c | 2 +- .../modules/hpy/devel/src/runtime/ctx_type.c | 9 + .../modules/hpy/devel/version.py | 4 +- .../modules/hpy/trace/leakdetector.py | 43 ++++ .../modules/hpy/trace/pytest.py | 31 +++ 70 files changed, 1047 insertions(+), 81 deletions(-) delete mode 100644 graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/version.h create mode 100644 graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/__init__.py create mode 100644 graalpython/com.oracle.graal.python.hpy.test/src/hpytest/hpy_devel/__init__.py create mode 100644 graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyiter.py rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy.h (100%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/autogen_hpyfunc_declare.h (100%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/autogen_hpyslot.h (98%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/cpy_types.h (100%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/cpython/autogen_api_impl.h (92%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/cpython/autogen_ctx.h (99%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/cpython/autogen_hpyfunc_trampolines.h (100%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/cpython/hpyfunc_trampolines.h (100%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/cpython/misc.h (97%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/forbid_python_h/Python.h (100%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/hpydef.h (99%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/hpyexports.h (100%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/hpyfunc.h (100%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/hpymodule.h (92%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/hpytype.h (94%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/inline_helpers.h (100%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/macros.h (100%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/runtime/argparse.h (100%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/runtime/buildvalue.h (100%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/runtime/ctx_funcs.h (96%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/runtime/ctx_module.h (100%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/runtime/ctx_type.h (100%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/runtime/format.h (100%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/runtime/helpers.h (100%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/runtime/structseq.h (100%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/universal/autogen_ctx.h (95%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/universal/autogen_hpyfunc_trampolines.h (100%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/universal/autogen_trampolines.h (94%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/universal/hpyfunc_trampolines.h (100%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/universal/misc_trampolines.h (100%) create mode 100644 graalpython/com.oracle.graal.python.hpy/include/hpy/version.h create mode 100644 graalpython/lib-graalpython/modules/hpy/trace/leakdetector.py create mode 100644 graalpython/lib-graalpython/modules/hpy/trace/pytest.py diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/version.h b/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/version.h deleted file mode 100644 index decdc2922f..0000000000 --- a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/version.h +++ /dev/null @@ -1,4 +0,0 @@ - -// automatically generated by setup.py:get_scm_config() -#define HPY_VERSION "0.9.0" -#define HPY_GIT_REVISION "f6114734" diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/conftest.py b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/conftest.py index 8c5c78aa08..372843459f 100644 --- a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/conftest.py +++ b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/conftest.py @@ -1,7 +1,6 @@ import pytest from .support import ExtensionCompiler, DefaultExtensionTemplate,\ PythonSubprocessRunner, HPyDebugCapture, make_hpy_abi_fixture -from hpy.debug.leakdetector import LeakDetector from pathlib import Path IS_VALGRIND_RUN = False @@ -48,6 +47,7 @@ def leakdetector(hpy_abi): """ Automatically detect leaks when the hpy_abi == 'debug' """ + from hpy.debug.leakdetector import LeakDetector if 'debug' in hpy_abi: with LeakDetector() as ld: yield ld diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/__init__.py b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_charptr.py b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_charptr.py index cf59cd5e9f..7df8d8f2c3 100644 --- a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_charptr.py +++ b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_charptr.py @@ -1,6 +1,6 @@ import os import pytest -from test.support import SUPPORTS_SYS_EXECUTABLE, SUPPORTS_MEM_PROTECTION +from ..support import SUPPORTS_SYS_EXECUTABLE, SUPPORTS_MEM_PROTECTION # Tests detection of usage of char pointers associated with invalid already # closed handles. For now, the debug mode does not provide any hook for this @@ -293,4 +293,4 @@ def clear_raw_data_in_closed_handles(): assert h.raw_data_size == -1 finally: _debug.set_protected_raw_data_max_size(old_raw_data_max_size) - _debug.set_closed_handles_queue_max_size(old_closed_handles_max_size) \ No newline at end of file + _debug.set_closed_handles_queue_max_size(old_closed_handles_max_size) diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_handles_invalid.py b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_handles_invalid.py index 32aec8e415..36622eece9 100644 --- a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_handles_invalid.py +++ b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_handles_invalid.py @@ -1,7 +1,8 @@ import pytest +import sys from hpy.debug.leakdetector import LeakDetector -from test.support import SUPPORTS_SYS_EXECUTABLE, IS_PYTHON_DEBUG_BUILD -from test.conftest import IS_VALGRIND_RUN +from ..support import SUPPORTS_SYS_EXECUTABLE, IS_PYTHON_DEBUG_BUILD +from ..conftest import IS_VALGRIND_RUN @pytest.fixture def hpy_abi(): @@ -9,6 +10,8 @@ def hpy_abi(): yield "debug" +@pytest.mark.skipif(sys.implementation.name == 'pypy', + reason="Cannot recover from use-after-close on pypy") def test_no_invalid_handle(compiler, hpy_debug_capture): # Basic sanity check that valid code does not trigger any error reports mod = compiler.make_module(""" @@ -33,6 +36,8 @@ def test_no_invalid_handle(compiler, hpy_debug_capture): assert hpy_debug_capture.invalid_handles_count == 0 +@pytest.mark.skipif(sys.implementation.name == 'pypy', + reason="Cannot recover from use-after-close on pypy") def test_cant_use_closed_handle(compiler, hpy_debug_capture): mod = compiler.make_module(""" HPyDef_METH(f, "f", HPyFunc_O, .doc="double close") @@ -106,6 +111,8 @@ def test_cant_use_closed_handle(compiler, hpy_debug_capture): assert hpy_debug_capture.invalid_handles_count == 6 +@pytest.mark.skipif(sys.implementation.name == 'pypy', + reason="Cannot recover from use-after-close on pypy") def test_keeping_and_reusing_argument_handle(compiler, hpy_debug_capture): mod = compiler.make_module(""" HPy keep; @@ -197,4 +204,4 @@ def test_invalid_handle_crashes_python_if_no_hook(compiler, python_subprocess, f """) result = python_subprocess.run(mod, "mod.f(42);") assert result.returncode == fatal_exit_code - assert b"Invalid usage of already closed handle" in result.stderr \ No newline at end of file + assert b"Invalid usage of already closed handle" in result.stderr diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_misc.py b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_misc.py index 0c6e0a4ca2..2994da4b3e 100644 --- a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_misc.py +++ b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_misc.py @@ -1,5 +1,5 @@ import pytest -from test.support import SUPPORTS_SYS_EXECUTABLE, SUPPORTS_MEM_PROTECTION +from ..support import SUPPORTS_SYS_EXECUTABLE, SUPPORTS_MEM_PROTECTION @pytest.fixture def hpy_abi(): diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/hpy_devel/__init__.py b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/hpy_devel/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/hpy_devel/test_distutils.py b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/hpy_devel/test_distutils.py index 00bbb3a52a..f6b2daae39 100644 --- a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/hpy_devel/test_distutils.py +++ b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/hpy_devel/test_distutils.py @@ -16,7 +16,7 @@ import py import pytest -from test.support import atomic_run, HPY_ROOT +from ..support import atomic_run, HPY_ROOT # ====== IMPORTANT DEVELOPMENT TIP ===== # You can use py.test --reuse-venv to speed up local testing. diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/support.py b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/support.py index 4278c3da13..250640548f 100644 --- a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/support.py +++ b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/support.py @@ -49,16 +49,19 @@ def make_hpy_abi_fixture(ABIs, class_fixture=False): class TestFoo(HPyTest): hpy_abi = make_hpy_abi_fixture('with hybrid', class_fixture=True) """ - if ABIs == 'default': - ABIs = ['cpython', 'universal', 'debug'] - elif ABIs == 'with hybrid': - ABIs = ['cpython', 'hybrid', 'hybrid+debug'] - elif isinstance(ABIs, list): + is_cpython = not hasattr(sys, "implementation") or sys.implementation.name == 'cpython' + if isinstance(ABIs, list): pass else: - raise ValueError("ABIs must be 'default', 'with hybrid' " - "or a list of strings. Got: %s" % ABIs) - + if ABIs == 'default': + ABIs = ['universal', 'debug'] + elif ABIs == 'with hybrid': + ABIs = ['hybrid', 'hybrid+debug'] + else: + raise ValueError("ABIs must be 'default', 'with hybrid' " + "or a list of strings. Got: %s" % ABIs) + if is_cpython: + ABIs.append('cpython') if class_fixture: @pytest.fixture(params=ABIs) def hpy_abi(self, request): @@ -495,15 +498,6 @@ def supports_ordinary_make_module_imports(self): """ return True - def supports_refcounts(self): - """ Returns True if the underlying Python implementation supports - the vectorcall protocol. - - By default, this returns True for Python version 3.8+ on all - implementations. - """ - return sys.version_info >= (3, 8) - class HPyDebugCapture: """ diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_00_basic.py b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_00_basic.py index a17f10e3a6..7fc483407e 100644 --- a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_00_basic.py +++ b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_00_basic.py @@ -8,7 +8,6 @@ """ from .support import HPyTest from hpy.devel.abitag import HPY_ABI_VERSION, HPY_ABI_VERSION_MINOR -import shutil class TestBasic(HPyTest): @@ -48,6 +47,7 @@ def test_abi_version_check(self): assert False, "Expected exception" def test_abi_tag_check(self): + import shutil if self.compiler.hpy_abi != 'universal': return @@ -262,6 +262,7 @@ def test_builtin_handles(self): case 20: h = ctx->h_MemoryViewType; break; case 21: h = ctx->h_SliceType; break; case 22: h = ctx->h_Builtins; break; + case 23: h = ctx->h_DictType; break; case 2048: h = ctx->h_CapsuleType; break; default: HPyErr_SetString(ctx, ctx->h_ValueError, "invalid choice"); @@ -278,7 +279,7 @@ def test_builtin_handles(self): '', None, False, True, ValueError, TypeError, IndexError, SystemError, object, type, bool, int, float, str, tuple, list, NotImplemented, Ellipsis, complex, bytes, memoryview, slice, - builtins.__dict__ + builtins.__dict__, dict ) for i, obj in enumerate(builtin_objs): if i == 0: @@ -560,3 +561,20 @@ def test_leave_python(self): @INIT """) assert mod.f("abraka") == 3 + + def test_dup_null(self): + mod = self.make_module(""" + HPyDef_METH(f, "f", HPyFunc_NOARGS) + static HPy f_impl(HPyContext *ctx, HPy self) + { + HPy h = HPy_Dup(ctx, HPy_NULL); + if (HPy_IsNull(h)) { + return HPyLong_FromSize_t(ctx, 0); + } + HPy_Close(ctx, h); + return HPyLong_FromSize_t(ctx, -1); + } + @EXPORT(f) + @INIT + """) + assert mod.f() == 0 diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_argparse.py b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_argparse.py index 51aa35c71e..55bdd31c6d 100644 --- a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_argparse.py +++ b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_argparse.py @@ -75,7 +75,7 @@ def test_s(self): "function a str is required" ) - def test_B(self): + def test_upper_b(self): mod = self.make_parse_item("B", "char", "char_to_hpybytes") assert mod.f(0) == b"\x00" assert mod.f(1) == b"\x01" @@ -102,7 +102,7 @@ def test_h(self): "function signed short integer is less than minimum" ) - def test_H_short(self): + def test_upper_h_short(self): mod = self.make_parse_item("H", "short", "HPyLong_FromLong") assert mod.f(0) == 0 assert mod.f(1) == 1 @@ -114,7 +114,7 @@ def test_H_short(self): assert mod.f(2**16) == 0 assert mod.f(-2**16) == 0 - def test_H_unsigned_short(self): + def test_upper_h_unsigned_short(self): mod = self.make_parse_item( "H", "unsigned short", "HPyLong_FromUnsignedLong" ) @@ -147,7 +147,7 @@ def test_i(self): "Python int too large to convert to C long", # where sizeof(long) == 4 ) - def test_I_signed(self): + def test_upper_i_signed(self): mod = self.make_parse_item("I", "int", "HPyLong_FromLong") assert mod.f(0) == 0 assert mod.f(1) == 1 @@ -159,7 +159,7 @@ def test_I_signed(self): assert mod.f(2**32) == 0 assert mod.f(-2**32) == 0 - def test_I_unsigned(self): + def test_upper_i_unsigned(self): mod = self.make_parse_item( "I", "unsigned int", "HPyLong_FromUnsignedLong" ) @@ -211,7 +211,7 @@ def test_k_unsigned(self): assert mod.f(2**ULONG_BITS) == 0 assert mod.f(-2**ULONG_BITS) == 0 - def test_L(self): + def test_upper_l(self): import pytest mod = self.make_parse_item("L", "long long", "HPyLong_FromLongLong") assert mod.f(0) == 0 @@ -224,7 +224,7 @@ def test_L(self): with pytest.raises(OverflowError): mod.f(-2**63 - 1) - def test_K_signed(self): + def test_upper_k_signed(self): mod = self.make_parse_item("K", "long long", "HPyLong_FromLongLong") assert mod.f(0) == 0 assert mod.f(1) == 1 @@ -236,7 +236,7 @@ def test_K_signed(self): assert mod.f(2**64) == 0 assert mod.f(-2**64) == 0 - def test_K_unsigned(self): + def test_upper_k_unsigned(self): mod = self.make_parse_item( "K", "unsigned long long", "HPyLong_FromUnsignedLongLong" ) @@ -277,7 +277,7 @@ def test_d(self): with pytest.raises(TypeError): mod.f("x") - def test_O(self): + def test_upper_o(self): mod = self.make_parse_item("O", "HPy", "HPy_Dup") assert mod.f("a") == "a" assert mod.f(5) == 5 diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_capsule.py b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_capsule.py index 74c7134b34..546b264f01 100644 --- a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_capsule.py +++ b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_capsule.py @@ -156,6 +156,7 @@ class TestHPyCapsule(HPyTest): ExtensionTemplate = CapsuleTemplate def test_capsule_new(self): + import pytest mod = self.make_module(""" @DEFINE_SomeObject @DEFINE_Capsule_New @@ -178,6 +179,7 @@ def test_capsule_new(self): mod.capsule_new() def test_capsule_getter_and_setter(self): + import pytest mod = self.make_module(""" #include @@ -385,6 +387,7 @@ def test_capsule_getter_and_setter(self): mod.capsule_freename(p) def test_capsule_isvalid(self): + import pytest mod = self.make_module(""" @DEFINE_SomeObject @DEFINE_Capsule_New @@ -455,6 +458,7 @@ def test_capsule_new_with_destructor(self): assert mod.pointer_freed() def test_capsule_new_with_invalid_destructor(self): + import pytest mod = self.make_module(""" static HPyCapsule_Destructor mydtor = { NULL, NULL }; diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_eval.py b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_eval.py index ffb89df8e4..441c0ab838 100644 --- a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_eval.py +++ b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_eval.py @@ -1,9 +1,9 @@ -from textwrap import dedent from .support import HPyTest class TestEval(HPyTest): def test_compile(self): import pytest + from textwrap import dedent mod = self.make_module(""" HPyDef_METH(f, "f", HPyFunc_VARARGS) static HPy f_impl(HPyContext *ctx, HPy self, const HPy *args, size_t nargs) diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyfield.py b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyfield.py index 55e68e5cdc..29ef99dba7 100644 --- a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyfield.py +++ b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyfield.py @@ -326,6 +326,7 @@ def test_tp_finalize(self): // nicely with pytest, but if the finalizer gets called after the // test has finished, we have no choice but to abort to signal // the error + static void on_unexpected_finalize_call(void); static void on_unexpected_finalize_call() { if (test_finished) abort(); @@ -420,4 +421,4 @@ def test_tp_finalize(self): owner.set_a(42) from gc import collect collect() - assert mod.check_finalize_calls() \ No newline at end of file + assert mod.check_finalize_calls() diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyiter.py b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyiter.py new file mode 100644 index 0000000000..58387253f9 --- /dev/null +++ b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyiter.py @@ -0,0 +1,90 @@ +from .support import HPyTest + +class TestIter(HPyTest): + + def test_Check(self): + mod = self.make_module(""" + HPyDef_METH(f, "f", HPyFunc_O) + static HPy f_impl(HPyContext *ctx, HPy self, HPy arg) + { + if (HPyIter_Check(ctx, arg)) + return HPy_Dup(ctx, ctx->h_True); + return HPy_Dup(ctx, ctx->h_False); + } + @EXPORT(f) + @INIT + """) + + class CustomIterable: + def __init__(self): + self._iter = iter([1, 2, 3]) + + def __iter__(self): + return self._iter + + class CustomIterator: + def __init__(self): + self._iter = iter([1, 2, 3]) + + def __iter__(self): + return self._iter + + def __next__(self): + return next(self._iter) + + assert mod.f(object()) is False + assert mod.f(10) is False + + assert mod.f((1, 2)) is False + assert mod.f(iter((1, 2))) is True + + assert mod.f([]) is False + assert mod.f(iter([])) is True + + assert mod.f('hello') is False + assert mod.f(iter('hello')) is True + + assert mod.f(map(int, ("1", "2"))) is True + assert mod.f(range(1, 10)) is False + + assert mod.f(CustomIterable()) is False + assert mod.f(iter(CustomIterable())) is True + assert mod.f(CustomIterator()) is True + + def test_Next(self): + mod = self.make_module(""" + HPyDef_METH(f, "f", HPyFunc_O) + static HPy f_impl(HPyContext *ctx, HPy self, HPy arg) + { + HPy result = HPyIter_Next(ctx, arg); + int is_null = HPy_IsNull(result); + + if (is_null && HPyErr_Occurred(ctx)) + return HPy_NULL; + if (is_null) + return HPyErr_SetObject(ctx, ctx->h_StopIteration, ctx->h_None); + return result; + } + @EXPORT(f) + @INIT + """) + + class CustomIterator: + def __init__(self): + self._iter = iter(["a", "b", "c"]) + + def __iter__(self): + return self._iter + + def __next__(self): + return next(self._iter) + + assert mod.f(iter([3, 2, 1])) == 3 + assert mod.f((i for i in range(1, 10))) == 1 + assert mod.f(CustomIterator()) == "a" + + import pytest + with pytest.raises(StopIteration): + assert mod.f(iter([])) + + diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpylist.py b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpylist.py index 9a048ccb5d..42270cdc6d 100644 --- a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpylist.py +++ b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpylist.py @@ -75,3 +75,38 @@ def test_ListBuilder(self): @INIT """) assert mod.f("xy") == ["xy", True, -42] + + def test_Insert(self): + import pytest + mod = self.make_module(""" + HPyDef_METH(f, "f", HPyFunc_VARARGS) + static HPy f_impl(HPyContext *ctx, HPy self, const HPy *args, size_t nargs) + { + HPy_ssize_t index; + if (nargs != 3) { + HPyErr_SetString(ctx, ctx->h_ValueError, "expected exactly three arguments"); + return HPy_NULL; + } + index = HPyLong_AsSsize_t(ctx, args[1]); + if (index == -1 && HPyErr_Occurred(ctx)) { + return HPy_NULL; + } + if (HPyList_Insert(ctx, args[0], index, args[2]) == -1) + return HPy_NULL; + return HPy_Dup(ctx, args[0]); + } + @EXPORT(f) + @INIT + """) + l = [] + assert mod.f(l, 0, 0) == [0] + l = [] + assert mod.f(l, -1, 0) == [0] + l = [1, 2, 4] + assert mod.f(l, 0, 0) == [0, 1, 2, 4] + assert mod.f(l, -1, 3) == [0, 1, 2, 3, 4] + assert mod.f(l, -3, 1.5) == [0, 1, 1.5, 2, 3, 4] + assert mod.f(l, 1000, 5) == [0, 1, 1.5, 2, 3, 4, 5] + assert mod.f(l, -1000, -1) == [-1, 0, 1, 1.5, 2, 3, 4, 5] + with pytest.raises(SystemError): + mod.f(None, 0, 0) diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpymodule.py b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpymodule.py index f154485ffe..5b675c98df 100644 --- a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpymodule.py +++ b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpymodule.py @@ -1,5 +1,3 @@ -import types -import pytest from .support import HPyTest class TestModule(HPyTest): @@ -93,6 +91,7 @@ def test_HPyModule_custom_create_returns_non_module(self): runtime and the extension can only populate that module object in the init slots. """ + import types mod = self.make_module(""" HPyDef_SLOT(create, HPy_mod_create) static HPy create_impl(HPyContext *ctx, HPy spec) @@ -104,7 +103,7 @@ def test_HPyModule_custom_create_returns_non_module(self): return HPy_NULL; ns_type = HPy_GetAttr_s(ctx, types, "SimpleNamespace"); - if (HPy_IsNull(types)) + if (HPy_IsNull(ns_type)) goto cleanup; dict = HPyDict_New(ctx); HPy_SetItem_s(ctx, dict, "spec", spec); @@ -144,6 +143,7 @@ def test_HPyModule_error_when_create_returns_module(self): there are any actual use-cases, the purpose of the 'create' slot is to create non-builtin-module objects. """ + import pytest expected_message = "HPy_mod_create slot returned a builtin module " \ "object. This is currently not supported." with pytest.raises(SystemError, match=expected_message): @@ -171,6 +171,7 @@ def test_HPyModule_error_when_create_returns_module(self): """) def test_HPyModule_create_raises(self): + import pytest with pytest.raises(RuntimeError, match="Test error"): self.make_module(""" HPyDef_SLOT(create, HPy_mod_create) @@ -197,6 +198,7 @@ def test_HPyModule_create_raises(self): """) def test_HPyModule_create_and_nondefault_values(self): + import pytest expected_message = r'^HPyModuleDef defines a HPy_mod_create slot.*' with pytest.raises(SystemError, match=expected_message): self.make_module(""" @@ -224,6 +226,7 @@ def test_HPyModule_create_and_nondefault_values(self): """) def test_HPyModule_create_and_exec_slots(self): + import pytest expected_message = r'^HPyModuleDef defines a HPy_mod_create slot.*' with pytest.raises(SystemError, match=expected_message): self.make_module(""" @@ -261,6 +264,7 @@ def test_HPyModule_negative_size(self): """ The simplest fully declarative module creation. """ + import pytest expected_message = "HPy does not permit HPyModuleDef.size < 0" with pytest.raises(SystemError, match=expected_message): self.make_module(""" diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyslice.py b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyslice.py index 8f8126828f..97f45b77b7 100644 --- a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyslice.py +++ b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyslice.py @@ -108,3 +108,31 @@ def test_adjust_indices(self): assert mod.f(10, 9, 0, -3) == (3, 9, 0, -3) assert mod.f(10, -1, -10, -3) == (3, 9, 0, -3) assert mod.f(10, 5, 5, -3) == (0, 5, 5, -3) + + def test_new(self): + mod = self.make_module(""" + HPyDef_METH(f, "f", HPyFunc_VARARGS) + static HPy f_impl(HPyContext *ctx, HPy self, const HPy *args, size_t nargs) + { + HPy start, stop, step; + + if (nargs != 3) { + HPyErr_SetString(ctx, ctx->h_TypeError, + "expected exactly 3 arguments"); + return HPy_NULL; + } + + if (HPyArg_Parse(ctx, NULL, args, nargs, "OOO", + &start, &stop, &step) < 0) { + return HPy_NULL; + } + return HPySlice_New(ctx, start, stop, step); + } + @EXPORT(f) + @INIT + """) + + assert mod.f(0, 10, 1) == slice(0, 10, 1) + assert mod.f(None, 10, 1) == slice(None, 10, 1) + assert mod.f(1, None, 1) == slice(1, None, 1) + assert mod.f(0, 10, None) == slice(0, 10, None) \ No newline at end of file diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpytype.py b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpytype.py index aa14006b47..3b8469fd16 100644 --- a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpytype.py +++ b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpytype.py @@ -1102,10 +1102,12 @@ def test_call_invalid_specs(self): @EXPORT(create_vcall) @INIT """) - with pytest.raises(TypeError): + with pytest.raises(TypeError) as err: mod.create_var_type() - with pytest.raises(TypeError): + assert "Cannot use HPy call protocol with var" in str(err.value) + with pytest.raises(TypeError) as err: mod.create_call_and_vectorcalloffset_type() + # assert "Cannot have HPy_tp_call and explicit member" in str(err.value) def test_call_explicit_offset(self): mod = self.make_module(""" @@ -1483,7 +1485,7 @@ def __new__(self): mod.create_type("mytest.DummyIntMeta", int) def test_get_name(self): - import array + import struct mod = self.make_module(""" static HPyType_Spec Dummy_spec = { .name = "mytest.Dummy", @@ -1508,7 +1510,7 @@ def test_get_name(self): assert mod.Dummy.__name__ == "Dummy" assert mod.get_name(mod.Dummy) == "Dummy" assert mod.get_name(str) == "str" - assert mod.get_name(array.array) == "array" + assert mod.get_name(struct.Struct) == "Struct" def test_issubtype(self): mod = self.make_module(""" diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpytype_legacy.py b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpytype_legacy.py index 776c1a2c03..fa9710c58f 100644 --- a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpytype_legacy.py +++ b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpytype_legacy.py @@ -1,6 +1,7 @@ """ HPyType tests on legacy types. """ import pytest +import sys from .support import HPyTest, make_hpy_abi_fixture from .test_hpytype import PointTemplate, TestType as _TestType @@ -257,6 +258,57 @@ def test_call_zero_basicsize(self): @INIT """) + @pytest.mark.skipif(sys.version_info < (3, 9), reason="not for python<3.9") + def test_legacy_class_method(self): + mod = self.make_module(""" + @DEFINE_PointObject + @DEFINE_Point_xy + static PyObject * point_class_getitem(PyObject *cls, PyObject *args) + { + Py_ssize_t args_len = PyTuple_Check(args) ? PyTuple_Size(args) : 1; + if (args_len != 1) { + return PyErr_Format(PyExc_TypeError, + "Too %s arguments for %s", + args_len > 1 ? "many" : "few", + ((PyTypeObject *)cls)->tp_name); + } + return Py_GenericAlias(cls, args); + } + + static PyMethodDef point_methods[] = { + /* for typing; requires python >= 3.9 */ + {"__class_getitem__", + (PyCFunction)point_class_getitem, + METH_CLASS | METH_O, NULL}, + {NULL, NULL, 0, NULL} /* sentinel */ + }; + + + static PyType_Slot Point_slots[] = { + {Py_tp_methods, point_methods}, + {Py_tp_new, (void*)PyType_GenericNew}, + {0, NULL}, + }; + static HPyDef *Point_defines[] = {&Point_x, &Point_y, NULL}; + static HPyType_Spec Point_spec = { + .name = "mytest.Point", + .basicsize = sizeof(PointObject), + .builtin_shape = SHAPE(PointObject), + .legacy_slots = Point_slots, + .defines = Point_defines, + }; + + @EXPORT_TYPE("Point", Point_spec) + @INIT + """) + # Calls __class_getitem__ + t = mod.Point[int] + assert str(t) == "mytest.Point[int]" + pt = mod.Point() + assert pt.x == 0 + assert pt.y == 0 + + class TestCustomLegacyFeatures(HPyTest): def test_legacy_methods(self): diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyunicode.py b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyunicode.py index 481d3029ca..6f6ce29d16 100644 --- a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyunicode.py +++ b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyunicode.py @@ -1,10 +1,6 @@ # -*- encoding: utf-8 -*- -import itertools -import re import sys -import pytest - from .support import HPyTest class TestUnicode(HPyTest): @@ -226,6 +222,9 @@ def test_FromFormat(self, hpy_abi): # Later we generate an HPy function for each case described below: # Most of the test cases are taken from CPython:Modules/_testcapi/unicode.c # Future work can improve this to add tests from Lib/test/test_capi/test_unicode.py + import itertools + import re + import pytest cases = [ # Unrecognized ( "%y%d", (SystemError, "invalid format string"), "'w', 42"), @@ -692,6 +691,7 @@ def check_cpython_raises_any(name): def test_FromFormat_Ptr(self): # '%p' is platform dependent to some extent, so we need to use regex + import re mod = self.make_module(""" HPyDef_METH(p, "p", HPyFunc_NOARGS) static HPy p_impl(HPyContext *ctx, HPy self) @@ -750,6 +750,7 @@ def __repr__(self): assert mod.A(MyObj()) == 'prefix-MyObj.__repr__\\xfc-suffix' def test_FromFormat_NoAsciiEncodedFmt(self): + import pytest mod = self.make_module(""" HPyDef_METH(no_ascii_fmt, "no_ascii_fmt", HPyFunc_O) static HPy no_ascii_fmt_impl(HPyContext *ctx, HPy self, HPy arg) @@ -809,6 +810,7 @@ def test_FromFormat_LongFormat(self): def test_FromFormat_Limits(self): import sys + import pytest mod = self.make_module(""" #include diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_importing.py b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_importing.py index c903d2faec..a16fa74c74 100644 --- a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_importing.py +++ b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_importing.py @@ -1,6 +1,5 @@ import pytest from .support import HPyTest -from hpy.devel.abitag import get_hpy_ext_suffix @pytest.fixture(params=['cpython', 'universal', 'hybrid', 'debug']) def hpy_abi(request): @@ -36,6 +35,7 @@ def test_importing_attributes(self, hpy_abi, tmpdir): import pytest if not self.supports_ordinary_make_module_imports(): pytest.skip() + from hpy.devel.abitag import get_hpy_ext_suffix mod = self.make_module(""" @INIT """, name='mytest') diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_legacy_forbidden.py b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_legacy_forbidden.py index ceeb69413c..6dfe29be51 100644 --- a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_legacy_forbidden.py +++ b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_legacy_forbidden.py @@ -3,8 +3,6 @@ get the expected compile time errors """ -import sys -import pytest from .support import HPyTest, make_hpy_abi_fixture, ONLY_LINUX # this is not strictly correct, we should check whether the actual compiler diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_object.py b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_object.py index 46e82e9e83..8e50450457 100644 --- a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_object.py +++ b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_object.py @@ -530,6 +530,54 @@ def test_getitem_s(self): with pytest.raises(TypeError): mod.f([]) + def test_getslice(self): + import pytest + mod = self.make_module(""" + HPyDef_METH(f, "f", HPyFunc_VARARGS) + static HPy f_impl(HPyContext *ctx, HPy self, const HPy *args, size_t nargs) + { + HPy seq; + HPy_ssize_t i1, i2; + if (!HPyArg_Parse(ctx, NULL, args, nargs, "Onn", &seq, &i1, &i2)) { + return HPy_NULL; + } + if (HPy_Is(ctx, seq, ctx->h_None)) { + seq = HPy_NULL; + } + return HPy_GetSlice(ctx, seq, i1, i2); + } + @EXPORT(f) + @INIT + """) + l = [1,2,3,4,5] + assert mod.f(l, 0, 5) == l + assert mod.f(l, 2, 3) == [3] + assert mod.f(l, 1, 3) == [2, 3] + + s = "hello" + assert mod.f(s, 0, 5) == "hello" + assert mod.f(s, 1, 3) == "el" + + class Sliceable: + def __getitem__(self, key): + # assume 'key' is a slice + if key.start < 0: + raise ValueError + return key.start + key.stop + + o = Sliceable() + assert mod.f(o, 5, 13) == 18 + + # test that errors are propagated + with pytest.raises(ValueError): + mod.f(o, -1, 1) + + # 'None' will be mapped to 'HPy_NULL' by the C function + with pytest.raises(SystemError): + mod.f(None, 0, 1) + with pytest.raises(TypeError): + mod.f(123, 0, 1) + def test_setitem(self): import pytest mod = self.make_module(""" @@ -603,6 +651,67 @@ def test_setitem_s(self): with pytest.raises(TypeError): mod.f([]) + def test_setslice(self): + import pytest + mod = self.make_module(""" + HPyDef_METH(f, "f", HPyFunc_VARARGS) + static HPy f_impl(HPyContext *ctx, HPy self, const HPy *args, size_t nargs) + { + int res; + HPy seq, value; + HPy_ssize_t i1, i2; + if (!HPyArg_Parse(ctx, NULL, args, nargs, "OnnO", &seq, &i1, &i2, &value)) { + return HPy_NULL; + } + if (HPy_Is(ctx, seq, ctx->h_None)) { + seq = HPy_NULL; + } + res = HPy_SetSlice(ctx, seq, i1, i2, value); + if (res == 0) { + return HPy_Dup(ctx, seq); + } else if (res == -1) { + return HPy_NULL; + } + HPyErr_SetString(ctx, ctx->h_SystemError, + "HPy_SetSlice returned an invalid result code"); + return HPy_NULL; + } + @EXPORT(f) + @INIT + """) + l = [1,2,3,4,5] + val = [11,22,33,44,55] + x = mod.f(l, 0, 5, val) + assert x is l + assert x is not val + assert l == val + + l = [1,2,3,4,5] + assert mod.f(l, 0, 5, []) == [] + + l = [1,2,3,4,5] + assert mod.f(l, 1, 3, [10, 11, 12]) == [1, 10, 11, 12, 4, 5] + + class Sliceable: + def __setitem__(self, key, value): + # assume 'key' is a slice + if key.start < 0: + raise ValueError + self.value = (key.start, key.stop, value) + + o = Sliceable() + assert mod.f(o, 5, 13, [7, 8, 9]).value == (5, 13, [7, 8, 9]) + + # test that errors are propagated + with pytest.raises(ValueError): + mod.f(o, -1, 1, []) + + # 'None' will be mapped to 'HPy_NULL' by the C function + with pytest.raises(SystemError): + mod.f(None, 0, 1, []) + with pytest.raises(TypeError): + mod.f(123, 0, 1, []) + def test_delitem(self): import pytest mod = self.make_module(""" @@ -675,6 +784,62 @@ def test_delitem(self): with pytest.raises(TypeError): mod.delitem_s3([1, 2, 3, 4]) + def test_delslice(self): + import pytest + mod = self.make_module(""" + HPyDef_METH(f, "f", HPyFunc_VARARGS) + static HPy f_impl(HPyContext *ctx, HPy self, const HPy *args, size_t nargs) + { + int res; + HPy seq; + HPy_ssize_t i1, i2; + if (!HPyArg_Parse(ctx, NULL, args, nargs, "Onn", &seq, &i1, &i2)) { + return HPy_NULL; + } + if (HPy_Is(ctx, seq, ctx->h_None)) { + seq = HPy_NULL; + } + res = HPy_DelSlice(ctx, seq, i1, i2); + if (res == 0) { + return HPy_Dup(ctx, seq); + } else if (res == -1) { + return HPy_NULL; + } + HPyErr_SetString(ctx, ctx->h_SystemError, + "HPy_DelSlice returned an invalid result code"); + return HPy_NULL; + } + @EXPORT(f) + @INIT + """) + l = [1,2,3,4,5] + x = mod.f(l, 0, 5) + assert x is l + assert l == [] + + l = [1,2,3,4,5] + assert mod.f(l, 1, 3) == [1, 4, 5] + + class Sliceable: + def __delitem__(self, key): + # assume 'key' is a slice + if key.start < 0: + raise ValueError + self.value = key.start + key.stop + + o = Sliceable() + assert mod.f(o, 5, 13).value == 18 + + # test that errors are propagated + with pytest.raises(ValueError): + mod.f(o, -1, 1) + + # 'None' will be mapped to 'HPy_NULL' by the C function + with pytest.raises(SystemError): + mod.f(None, 0, 1) + with pytest.raises(TypeError): + mod.f(123, 0, 1) + def test_length(self): mod = self.make_module(""" HPyDef_METH(f, "f", HPyFunc_O) @@ -740,6 +905,64 @@ class Dummy: import pytest with pytest.raises(TypeError): mod.f(Dummy(), 42) + + def test_getiter(self): + mod = self.make_module(""" + HPyDef_METH(f, "f", HPyFunc_O) + static HPy f_impl(HPyContext *ctx, HPy self, HPy arg) + { + HPy iterator; + iterator = HPy_GetIter(ctx, arg); + if HPy_IsNull(iterator) + return HPy_NULL; + return iterator; + } + @EXPORT(f) + @INIT + """) + + def test_for_loop(iterator): + results = [] + for obj in iterator: + results.append(obj) + return results + + class WithIter: + def __iter__(self): + return (1, 2, 3).__iter__() + + class WithoutIter: + pass + + case = [1, 2, 3] + result = mod.f(case) + assert result + assert test_for_loop(result) == [1, 2, 3] + + case = iter([1, 2, 3]) + result = mod.f(case) + assert result + assert test_for_loop(result) == [1, 2, 3] + + case = zip((1, 2, 3), [4, 5, 6]) + result = mod.f(case) + assert result + assert test_for_loop(result) == [(1, 4), (2, 5), (3, 6)] + + case = range(10) + result = mod.f(case) + assert result + assert test_for_loop(result) == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + + case = WithIter() + result = mod.f(case) + assert result + assert test_for_loop(result) == [1, 2, 3] + + import pytest + with pytest.raises(TypeError): + assert mod.f(WithoutIter()) + def test_dump(self): # _HPy_Dump is supposed to be used e.g. inside a gdb session: it diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_slots.py b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_slots.py index 356fb73b0e..ac9cbe2fc7 100644 --- a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_slots.py +++ b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_slots.py @@ -462,6 +462,7 @@ def test_tp_repr_and_tp_str(self): assert repr(p) == 'repr(Point(1, 2))' def test_tp_hash(self): + import pytest mod = self.make_module(""" @DEFINE_PointObject @DEFINE_Point_new @@ -771,7 +772,6 @@ def test_sq_contains(self): 'hello' in p def test_tp_richcompare(self): - import pytest mod = self.make_module(""" @DEFINE_PointObject @DEFINE_Point_new @@ -807,3 +807,27 @@ def test_tp_richcompare(self): # assert not p1 >= p2 assert p1 >= p1 + + def test_tp_descr_get(self): + mod = self.make_module(""" + @DEFINE_PointObject + @DEFINE_Point_new + + HPyDef_SLOT(Point_get, HPy_tp_descr_get); + static HPy + Point_get_impl(HPyContext *ctx, HPy self, HPy obj, HPy type) + { + if (HPy_IsNull(obj) || HPy_Is(ctx, self, ctx->h_None)) { + return HPy_Dup(ctx, self); + } + return HPyLong_FromLong(ctx, 123); + } + + @EXPORT_POINT_TYPE(&Point_new, &Point_get) + @INIT + """) + p = mod.Point(10, 10) + class Dummy: + point_func = p + assert Dummy.point_func is p + assert Dummy().point_func == 123 diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy.h b/graalpython/com.oracle.graal.python.hpy/include/hpy.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy.h diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/autogen_hpyfunc_declare.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/autogen_hpyfunc_declare.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/autogen_hpyfunc_declare.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/autogen_hpyfunc_declare.h diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/autogen_hpyslot.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/autogen_hpyslot.h similarity index 98% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/autogen_hpyslot.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/autogen_hpyslot.h index 0fcf6608a2..117e6ccae0 100644 --- a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/autogen_hpyslot.h +++ b/graalpython/com.oracle.graal.python.hpy/include/hpy/autogen_hpyslot.h @@ -58,6 +58,7 @@ typedef enum { HPy_sq_length = 45, HPy_sq_repeat = 46, HPy_tp_call = 50, + HPy_tp_descr_get = 54, HPy_tp_hash = 59, HPy_tp_init = 60, HPy_tp_new = 65, @@ -120,6 +121,7 @@ typedef enum { #define _HPySlot_SIG__HPy_sq_length HPyFunc_LENFUNC #define _HPySlot_SIG__HPy_sq_repeat HPyFunc_SSIZEARGFUNC #define _HPySlot_SIG__HPy_tp_call HPyFunc_KEYWORDS +#define _HPySlot_SIG__HPy_tp_descr_get HPyFunc_TERNARYFUNC #define _HPySlot_SIG__HPy_tp_hash HPyFunc_HASHFUNC #define _HPySlot_SIG__HPy_tp_init HPyFunc_INITPROC #define _HPySlot_SIG__HPy_tp_new HPyFunc_NEWFUNC diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/cpy_types.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/cpy_types.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/cpy_types.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/cpy_types.h diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/cpython/autogen_api_impl.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/cpython/autogen_api_impl.h similarity index 92% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/cpython/autogen_api_impl.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/cpython/autogen_api_impl.h index bde41f9e62..4a55fc6dc6 100644 --- a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/cpython/autogen_api_impl.h +++ b/graalpython/com.oracle.graal.python.hpy/include/hpy/cpython/autogen_api_impl.h @@ -240,6 +240,21 @@ HPyAPI_FUNC int HPyCallable_Check(HPyContext *ctx, HPy h) return PyCallable_Check(_h2py(h)); } +HPyAPI_FUNC HPy HPy_GetIter(HPyContext *ctx, HPy obj) +{ + return _py2h(PyObject_GetIter(_h2py(obj))); +} + +HPyAPI_FUNC HPy HPyIter_Next(HPyContext *ctx, HPy obj) +{ + return _py2h(PyIter_Next(_h2py(obj))); +} + +HPyAPI_FUNC int HPyIter_Check(HPyContext *ctx, HPy obj) +{ + return PyIter_Check(_h2py(obj)); +} + HPyAPI_FUNC HPy HPyErr_SetString(HPyContext *ctx, HPy h_type, const char *utf8_message) { PyErr_SetString(_h2py(h_type), utf8_message); @@ -339,6 +354,11 @@ HPyAPI_FUNC HPy HPy_GetItem(HPyContext *ctx, HPy obj, HPy key) return _py2h(PyObject_GetItem(_h2py(obj), _h2py(key))); } +HPyAPI_FUNC HPy HPy_GetSlice(HPyContext *ctx, HPy obj, HPy_ssize_t start, HPy_ssize_t end) +{ + return _py2h(PySequence_GetSlice(_h2py(obj), start, end)); +} + HPyAPI_FUNC int HPy_Contains(HPyContext *ctx, HPy container, HPy key) { return PySequence_Contains(_h2py(container), _h2py(key)); @@ -349,14 +369,19 @@ HPyAPI_FUNC int HPy_SetItem(HPyContext *ctx, HPy obj, HPy key, HPy value) return PyObject_SetItem(_h2py(obj), _h2py(key), _h2py(value)); } +HPyAPI_FUNC int HPy_SetSlice(HPyContext *ctx, HPy obj, HPy_ssize_t start, HPy_ssize_t end, HPy value) +{ + return PySequence_SetSlice(_h2py(obj), start, end, _h2py(value)); +} + HPyAPI_FUNC int HPy_DelItem(HPyContext *ctx, HPy obj, HPy key) { return PyObject_DelItem(_h2py(obj), _h2py(key)); } -HPyAPI_FUNC HPy HPy_Type(HPyContext *ctx, HPy obj) +HPyAPI_FUNC int HPy_DelSlice(HPyContext *ctx, HPy obj, HPy_ssize_t start, HPy_ssize_t end) { - return _py2h(PyObject_Type(_h2py(obj))); + return PySequence_DelSlice(_h2py(obj), start, end); } HPyAPI_FUNC HPy HPy_Repr(HPyContext *ctx, HPy obj) @@ -514,6 +539,11 @@ HPyAPI_FUNC int HPyList_Append(HPyContext *ctx, HPy h_list, HPy h_item) return PyList_Append(_h2py(h_list), _h2py(h_item)); } +HPyAPI_FUNC int HPyList_Insert(HPyContext *ctx, HPy h_list, HPy_ssize_t index, HPy h_item) +{ + return PyList_Insert(_h2py(h_list), index, _h2py(h_item)); +} + HPyAPI_FUNC int HPyDict_Check(HPyContext *ctx, HPy h) { return PyDict_Check(_h2py(h)); @@ -539,6 +569,11 @@ HPyAPI_FUNC int HPyTuple_Check(HPyContext *ctx, HPy h) return PyTuple_Check(_h2py(h)); } +HPyAPI_FUNC HPy HPySlice_New(HPyContext *ctx, HPy start, HPy stop, HPy step) +{ + return _py2h(PySlice_New(_h2py(start), _h2py(stop), _h2py(step))); +} + HPyAPI_FUNC int HPySlice_Unpack(HPyContext *ctx, HPy slice, HPy_ssize_t *start, HPy_ssize_t *stop, HPy_ssize_t *step) { return PySlice_Unpack(_h2py(slice), start, stop, step); diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/cpython/autogen_ctx.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/cpython/autogen_ctx.h similarity index 99% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/cpython/autogen_ctx.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/cpython/autogen_ctx.h index 87813ec23b..6f9fb0d7dc 100644 --- a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/cpython/autogen_ctx.h +++ b/graalpython/com.oracle.graal.python.hpy/include/hpy/cpython/autogen_ctx.h @@ -96,4 +96,5 @@ struct _HPyContext_s { HPy h_CapsuleType; HPy h_SliceType; HPy h_Builtins; + HPy h_DictType; }; diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/cpython/autogen_hpyfunc_trampolines.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/cpython/autogen_hpyfunc_trampolines.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/cpython/autogen_hpyfunc_trampolines.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/cpython/autogen_hpyfunc_trampolines.h diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/cpython/hpyfunc_trampolines.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/cpython/hpyfunc_trampolines.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/cpython/hpyfunc_trampolines.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/cpython/hpyfunc_trampolines.h diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/cpython/misc.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/cpython/misc.h similarity index 97% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/cpython/misc.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/cpython/misc.h index 5665460de7..6df576e7ff 100644 --- a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/cpython/misc.h +++ b/graalpython/com.oracle.graal.python.hpy/include/hpy/cpython/misc.h @@ -144,6 +144,7 @@ HPyAPI_FUNC HPyContext * _HPyGetContext(void) { ctx->h_MemoryViewType = _py2h((PyObject *)&PyMemoryView_Type); ctx->h_CapsuleType = _py2h((PyObject *)&PyCapsule_Type); ctx->h_SliceType = _py2h((PyObject *)&PySlice_Type); + ctx->h_DictType = _py2h((PyObject *)&PyDict_Type); /* Reflection */ ctx->h_Builtins = _py2h(PyEval_GetBuiltins()); } @@ -271,6 +272,11 @@ HPyAPI_FUNC void* _HPy_AsStruct_List(HPyContext *ctx, HPy h) return ctx_AsStruct_List(ctx, h); } +HPyAPI_FUNC void* _HPy_AsStruct_Dict(HPyContext *ctx, HPy h) +{ + return ctx_AsStruct_Dict(ctx, h); +} + HPyAPI_FUNC HPy HPy_CallTupleDict(HPyContext *ctx, HPy callable, HPy args, HPy kw) { return ctx_CallTupleDict(ctx, callable, args, kw); @@ -305,6 +311,13 @@ HPyAPI_FUNC void _HPy_Dump(HPyContext *ctx, HPy h) ctx_Dump(ctx, h); } +HPyAPI_FUNC HPy HPy_Type(HPyContext *ctx, HPy h_obj) +{ + PyTypeObject *tp = Py_TYPE(_h2py(h_obj)); + Py_INCREF(tp); + return _py2h((PyObject *)tp); +} + HPyAPI_FUNC int HPy_TypeCheck(HPyContext *ctx, HPy h_obj, HPy h_type) { return ctx_TypeCheck(ctx, h_obj, h_type); @@ -312,7 +325,7 @@ HPyAPI_FUNC int HPy_TypeCheck(HPyContext *ctx, HPy h_obj, HPy h_type) HPyAPI_FUNC int HPy_Is(HPyContext *ctx, HPy h_obj, HPy h_other) { - return ctx_Is(ctx, h_obj, h_other); + return _h2py(h_obj) == _h2py(h_other); } HPyAPI_FUNC HPyListBuilder HPyListBuilder_New(HPyContext *ctx, HPy_ssize_t initial_size) diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/forbid_python_h/Python.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/forbid_python_h/Python.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/forbid_python_h/Python.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/forbid_python_h/Python.h diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/hpydef.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/hpydef.h similarity index 99% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/hpydef.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/hpydef.h index a0478727f1..db45de2e80 100644 --- a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/hpydef.h +++ b/graalpython/com.oracle.graal.python.hpy/include/hpy/hpydef.h @@ -10,7 +10,7 @@ extern "C" { #include "hpy/autogen_hpyslot.h" #include "hpy/cpy_types.h" -typedef void* (*HPyCFunction)(); +typedef void* (*HPyCFunction)(void); typedef void (*HPyFunc_Capsule_Destructor)(const char *name, void *pointer, void *context); /** diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/hpyexports.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/hpyexports.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/hpyexports.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/hpyexports.h diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/hpyfunc.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/hpyfunc.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/hpyfunc.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/hpyfunc.h diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/hpymodule.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/hpymodule.h similarity index 92% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/hpymodule.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/hpymodule.h index 1ad5802aa2..fc6a72f608 100644 --- a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/hpymodule.h +++ b/graalpython/com.oracle.graal.python.hpy/include/hpy/hpymodule.h @@ -129,17 +129,23 @@ typedef struct { */ #define HPy_MODINIT(ext_name, mod_def) \ HPy_EXPORTED_FUNC uint32_t \ - get_required_hpy_major_version_##ext_name() \ + get_required_hpy_major_version_##ext_name(void); \ + uint32_t \ + get_required_hpy_major_version_##ext_name(void) \ { \ return HPY_ABI_VERSION; \ } \ HPy_EXPORTED_FUNC uint32_t \ - get_required_hpy_minor_version_##ext_name() \ + get_required_hpy_minor_version_##ext_name(void); \ + uint32_t \ + get_required_hpy_minor_version_##ext_name(void) \ { \ return HPY_ABI_VERSION_MINOR; \ } \ _HPy_CTX_MODIFIER HPyContext *_ctx_for_trampolines; \ HPy_EXPORTED_FUNC void \ + HPyInitGlobalContext_##ext_name(HPyContext *ctx); \ + void \ HPyInitGlobalContext_##ext_name(HPyContext *ctx) \ { \ _ctx_for_trampolines = ctx; \ diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/hpytype.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/hpytype.h similarity index 94% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/hpytype.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/hpytype.h index 7ba2a5640b..167bd6def5 100644 --- a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/hpytype.h +++ b/graalpython/com.oracle.graal.python.hpy/include/hpy/hpytype.h @@ -73,6 +73,12 @@ typedef enum { * need to specify base class ``ctx->h_ListType``. */ HPyType_BuiltinShape_List = 6, + + /** + * The type inherits from built-in type ``dict``. If using this shape, you + * need to specify base class ``ctx->h_DictType``. + */ + HPyType_BuiltinShape_Dict = 7, } HPyType_BuiltinShape; typedef struct { @@ -202,7 +208,16 @@ typedef struct { #define HPy_TPFLAGS_HAVE_GC (1UL << 14) /** Convenience macro which is equivalent to: - ``HPyType_HELPERS(TYPE, HPyType_BuiltinShape_Legacy)`` */ + ``HPyType_HELPERS(TYPE, HPyType_BuiltinShape_Legacy)`` + For instance, HPyType_LEGACY_HELPERS(DummyMeta) will produce:: + + enum { DummyMeta_SHAPE = (int)HPyType_BuiltinShape_Legacy }; + __attribute__((unused)) static inline + DummyMeta * + DummyMeta_AsStruct(HPyContext *ctx, HPy h) { + return (DummyMeta *) _HPy_AsStruct_Legacy(ctx, h); + } +*/ #define HPyType_LEGACY_HELPERS(TYPE) \ HPyType_HELPERS(TYPE, HPyType_BuiltinShape_Legacy) @@ -283,5 +298,6 @@ _HPyType_HELPER_X(_HPyType_HELPER_FNAME(__VA_ARGS__))(HPyContext *ctx, HPy h) \ #define HPyType_BuiltinShape_Unicode_AsStruct _HPy_AsStruct_Unicode #define HPyType_BuiltinShape_Tuple_AsStruct _HPy_AsStruct_Tuple #define HPyType_BuiltinShape_List_AsStruct _HPy_AsStruct_List +#define HPyType_BuiltinShape_Dict_AsStruct _HPy_AsStruct_Dict #endif /* HPY_UNIVERSAL_HPYTYPE_H */ diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/inline_helpers.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/inline_helpers.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/inline_helpers.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/inline_helpers.h diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/macros.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/macros.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/macros.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/macros.h diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/runtime/argparse.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/argparse.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/runtime/argparse.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/argparse.h diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/runtime/buildvalue.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/buildvalue.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/runtime/buildvalue.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/buildvalue.h diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/runtime/ctx_funcs.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/ctx_funcs.h similarity index 96% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/runtime/ctx_funcs.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/ctx_funcs.h index 4fc1f319fc..d32aa75592 100644 --- a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/runtime/ctx_funcs.h +++ b/graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/ctx_funcs.h @@ -31,6 +31,7 @@ _HPy_HIDDEN HPy ctx_Module_Create(HPyContext *ctx, HPyModuleDef *hpydef); // ctx_object.c _HPy_HIDDEN void ctx_Dump(HPyContext *ctx, HPy h); +_HPy_HIDDEN HPy ctx_Type(HPyContext *ctx, HPy h_obj); _HPy_HIDDEN int ctx_TypeCheck(HPyContext *ctx, HPy h_obj, HPy h_type); _HPy_HIDDEN int ctx_Is(HPyContext *ctx, HPy h_obj, HPy h_other); _HPy_HIDDEN HPy ctx_GetItem_i(HPyContext *ctx, HPy obj, HPy_ssize_t idx); @@ -56,7 +57,7 @@ _HPy_HIDDEN void ctx_TupleBuilder_Cancel(HPyContext *ctx, HPyTupleBuilder builder); // ctx_tuple.c -_HPy_HIDDEN HPy ctx_Tuple_FromArray(HPyContext *ctx, HPy items[], HPy_ssize_t n); +_HPy_HIDDEN HPy ctx_Tuple_FromArray(HPyContext *ctx, const HPy items[], HPy_ssize_t n); // ctx_capsule.c _HPy_HIDDEN HPy ctx_Capsule_New(HPyContext *ctx, @@ -87,6 +88,7 @@ _HPy_HIDDEN void* ctx_AsStruct_Float(HPyContext *ctx, HPy h); _HPy_HIDDEN void* ctx_AsStruct_Unicode(HPyContext *ctx, HPy h); _HPy_HIDDEN void* ctx_AsStruct_Tuple(HPyContext *ctx, HPy h); _HPy_HIDDEN void* ctx_AsStruct_List(HPyContext *ctx, HPy h); +_HPy_HIDDEN void* ctx_AsStruct_Dict(HPyContext *ctx, HPy h); _HPy_HIDDEN void* ctx_AsStruct_Slow(HPyContext *ctx, HPy h); _HPy_HIDDEN HPy ctx_Type_FromSpec(HPyContext *ctx, HPyType_Spec *hpyspec, HPyType_SpecParam *params); diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/runtime/ctx_module.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/ctx_module.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/runtime/ctx_module.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/ctx_module.h diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/runtime/ctx_type.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/ctx_type.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/runtime/ctx_type.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/ctx_type.h diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/runtime/format.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/format.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/runtime/format.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/format.h diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/runtime/helpers.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/helpers.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/runtime/helpers.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/helpers.h diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/runtime/structseq.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/structseq.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/runtime/structseq.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/structseq.h diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/universal/autogen_ctx.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/universal/autogen_ctx.h similarity index 95% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/universal/autogen_ctx.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/universal/autogen_ctx.h index 6133fc633f..1abb8e5d1c 100644 --- a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/universal/autogen_ctx.h +++ b/graalpython/com.oracle.graal.python.hpy/include/hpy/universal/autogen_ctx.h @@ -218,7 +218,7 @@ struct _HPyContext_s { int (*ctx_Dict_Check)(HPyContext *ctx, HPy h); HPy (*ctx_Dict_New)(HPyContext *ctx); int (*ctx_Tuple_Check)(HPyContext *ctx, HPy h); - HPy (*ctx_Tuple_FromArray)(HPyContext *ctx, HPy items[], HPy_ssize_t n); + HPy (*ctx_Tuple_FromArray)(HPyContext *ctx, const HPy items[], HPy_ssize_t n); HPy (*ctx_Import_ImportModule)(HPyContext *ctx, const char *utf8_name); HPy (*ctx_FromPyObject)(HPyContext *ctx, cpy_PyObject *obj); cpy_PyObject *(*ctx_AsPyObject)(HPyContext *ctx, HPy h); @@ -277,4 +277,14 @@ struct _HPyContext_s { int (*ctx_SetCallFunction)(HPyContext *ctx, HPy h, HPyCallFunction *func); HPy (*ctx_Call)(HPyContext *ctx, HPy callable, const HPy *args, size_t nargs, HPy kwnames); HPy (*ctx_CallMethod)(HPyContext *ctx, HPy name, const HPy *args, size_t nargs, HPy kwnames); + HPy h_DictType; + void *(*ctx_AsStruct_Dict)(HPyContext *ctx, HPy h); + int (*ctx_List_Insert)(HPyContext *ctx, HPy h_list, HPy_ssize_t index, HPy h_item); + HPy (*ctx_GetSlice)(HPyContext *ctx, HPy obj, HPy_ssize_t start, HPy_ssize_t end); + int (*ctx_SetSlice)(HPyContext *ctx, HPy obj, HPy_ssize_t start, HPy_ssize_t end, HPy value); + int (*ctx_DelSlice)(HPyContext *ctx, HPy obj, HPy_ssize_t start, HPy_ssize_t end); + HPy (*ctx_GetIter)(HPyContext *ctx, HPy obj); + HPy (*ctx_Iter_Next)(HPyContext *ctx, HPy obj); + int (*ctx_Iter_Check)(HPyContext *ctx, HPy obj); + HPy (*ctx_Slice_New)(HPyContext *ctx, HPy start, HPy stop, HPy step); }; diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/universal/autogen_hpyfunc_trampolines.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/universal/autogen_hpyfunc_trampolines.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/universal/autogen_hpyfunc_trampolines.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/universal/autogen_hpyfunc_trampolines.h diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/universal/autogen_trampolines.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/universal/autogen_trampolines.h similarity index 94% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/universal/autogen_trampolines.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/universal/autogen_trampolines.h index 0e57d3f1be..4579b5591b 100644 --- a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/universal/autogen_trampolines.h +++ b/graalpython/com.oracle.graal.python.hpy/include/hpy/universal/autogen_trampolines.h @@ -254,6 +254,18 @@ HPyAPI_FUNC HPy HPy_CallMethod(HPyContext *ctx, HPy name, const HPy *args, size_ return ctx->ctx_CallMethod ( ctx, name, args, nargs, kwnames ); } +HPyAPI_FUNC HPy HPy_GetIter(HPyContext *ctx, HPy obj) { + return ctx->ctx_GetIter ( ctx, obj ); +} + +HPyAPI_FUNC HPy HPyIter_Next(HPyContext *ctx, HPy obj) { + return ctx->ctx_Iter_Next ( ctx, obj ); +} + +HPyAPI_FUNC int HPyIter_Check(HPyContext *ctx, HPy obj) { + return ctx->ctx_Iter_Check ( ctx, obj ); +} + HPyAPI_FUNC HPy HPyErr_SetString(HPyContext *ctx, HPy h_type, const char *utf8_message) { ctx->ctx_Err_SetString ( ctx, h_type, utf8_message ); return HPy_NULL; } @@ -350,6 +362,10 @@ HPyAPI_FUNC HPy HPy_GetItem_s(HPyContext *ctx, HPy obj, const char *utf8_key) { return ctx->ctx_GetItem_s ( ctx, obj, utf8_key ); } +HPyAPI_FUNC HPy HPy_GetSlice(HPyContext *ctx, HPy obj, HPy_ssize_t start, HPy_ssize_t end) { + return ctx->ctx_GetSlice ( ctx, obj, start, end ); +} + HPyAPI_FUNC int HPy_Contains(HPyContext *ctx, HPy container, HPy key) { return ctx->ctx_Contains ( ctx, container, key ); } @@ -366,6 +382,10 @@ HPyAPI_FUNC int HPy_SetItem_s(HPyContext *ctx, HPy obj, const char *utf8_key, HP return ctx->ctx_SetItem_s ( ctx, obj, utf8_key, value ); } +HPyAPI_FUNC int HPy_SetSlice(HPyContext *ctx, HPy obj, HPy_ssize_t start, HPy_ssize_t end, HPy value) { + return ctx->ctx_SetSlice ( ctx, obj, start, end, value ); +} + HPyAPI_FUNC int HPy_DelItem(HPyContext *ctx, HPy obj, HPy key) { return ctx->ctx_DelItem ( ctx, obj, key ); } @@ -378,6 +398,10 @@ HPyAPI_FUNC int HPy_DelItem_s(HPyContext *ctx, HPy obj, const char *utf8_key) { return ctx->ctx_DelItem_s ( ctx, obj, utf8_key ); } +HPyAPI_FUNC int HPy_DelSlice(HPyContext *ctx, HPy obj, HPy_ssize_t start, HPy_ssize_t end) { + return ctx->ctx_DelSlice ( ctx, obj, start, end ); +} + HPyAPI_FUNC HPy HPy_Type(HPyContext *ctx, HPy obj) { return ctx->ctx_Type ( ctx, obj ); } @@ -430,6 +454,10 @@ HPyAPI_FUNC void *_HPy_AsStruct_List(HPyContext *ctx, HPy h) { return ctx->ctx_AsStruct_List ( ctx, h ); } +HPyAPI_FUNC void *_HPy_AsStruct_Dict(HPyContext *ctx, HPy h) { + return ctx->ctx_AsStruct_Dict ( ctx, h ); +} + HPyAPI_FUNC HPyType_BuiltinShape _HPyType_GetBuiltinShape(HPyContext *ctx, HPy h_type) { return ctx->ctx_Type_GetBuiltinShape ( ctx, h_type ); } @@ -562,6 +590,10 @@ HPyAPI_FUNC int HPyList_Append(HPyContext *ctx, HPy h_list, HPy h_item) { return ctx->ctx_List_Append ( ctx, h_list, h_item ); } +HPyAPI_FUNC int HPyList_Insert(HPyContext *ctx, HPy h_list, HPy_ssize_t index, HPy h_item) { + return ctx->ctx_List_Insert ( ctx, h_list, index, h_item ); +} + HPyAPI_FUNC int HPyDict_Check(HPyContext *ctx, HPy h) { return ctx->ctx_Dict_Check ( ctx, h ); } @@ -582,10 +614,14 @@ HPyAPI_FUNC int HPyTuple_Check(HPyContext *ctx, HPy h) { return ctx->ctx_Tuple_Check ( ctx, h ); } -HPyAPI_FUNC HPy HPyTuple_FromArray(HPyContext *ctx, HPy items[], HPy_ssize_t n) { +HPyAPI_FUNC HPy HPyTuple_FromArray(HPyContext *ctx, const HPy items[], HPy_ssize_t n) { return ctx->ctx_Tuple_FromArray ( ctx, items, n ); } +HPyAPI_FUNC HPy HPySlice_New(HPyContext *ctx, HPy start, HPy stop, HPy step) { + return ctx->ctx_Slice_New ( ctx, start, stop, step ); +} + HPyAPI_FUNC int HPySlice_Unpack(HPyContext *ctx, HPy slice, HPy_ssize_t *start, HPy_ssize_t *stop, HPy_ssize_t *step) { return ctx->ctx_Slice_Unpack ( ctx, slice, start, stop, step ); } diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/universal/hpyfunc_trampolines.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/universal/hpyfunc_trampolines.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/universal/hpyfunc_trampolines.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/universal/hpyfunc_trampolines.h diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/universal/misc_trampolines.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/universal/misc_trampolines.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/universal/misc_trampolines.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/universal/misc_trampolines.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/version.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/version.h new file mode 100644 index 0000000000..81d60ed7d5 --- /dev/null +++ b/graalpython/com.oracle.graal.python.hpy/include/hpy/version.h @@ -0,0 +1,4 @@ + +// automatically generated by setup.py:get_scm_config() +#define HPY_VERSION "0.9.1.dev79+gb0fbdf73" +#define HPY_GIT_REVISION "b0fbdf73" diff --git a/graalpython/com.oracle.graal.python.jni/src/debug/_debugmod.c b/graalpython/com.oracle.graal.python.jni/src/debug/_debugmod.c index ce3a57e886..2bd71f3262 100644 --- a/graalpython/com.oracle.graal.python.jni/src/debug/_debugmod.c +++ b/graalpython/com.oracle.graal.python.jni/src/debug/_debugmod.c @@ -10,7 +10,7 @@ static UHPy new_DebugHandleObj(HPyContext *uctx, UHPy u_DebugHandleType, DebugHandle *handle); -HPY_MOD_EMBEDDABLE(_trace) +HPY_MOD_EMBEDDABLE(_debug) HPyDef_METH(new_generation, "new_generation", HPyFunc_NOARGS) static UHPy new_generation_impl(HPyContext *uctx, UHPy self) diff --git a/graalpython/com.oracle.graal.python.jni/src/debug/autogen_debug_ctx_init.h b/graalpython/com.oracle.graal.python.jni/src/debug/autogen_debug_ctx_init.h index 6c242cfe48..c394c362ce 100644 --- a/graalpython/com.oracle.graal.python.jni/src/debug/autogen_debug_ctx_init.h +++ b/graalpython/com.oracle.graal.python.jni/src/debug/autogen_debug_ctx_init.h @@ -71,6 +71,9 @@ int debug_ctx_Callable_Check(HPyContext *dctx, DHPy h); DHPy debug_ctx_CallTupleDict(HPyContext *dctx, DHPy callable, DHPy args, DHPy kw); DHPy debug_ctx_Call(HPyContext *dctx, DHPy callable, const DHPy *args, size_t nargs, DHPy kwnames); DHPy debug_ctx_CallMethod(HPyContext *dctx, DHPy name, const DHPy *args, size_t nargs, DHPy kwnames); +DHPy debug_ctx_GetIter(HPyContext *dctx, DHPy obj); +DHPy debug_ctx_Iter_Next(HPyContext *dctx, DHPy obj); +int debug_ctx_Iter_Check(HPyContext *dctx, DHPy obj); void debug_ctx_FatalError(HPyContext *dctx, const char *message); void debug_ctx_Err_SetString(HPyContext *dctx, DHPy h_type, const char *utf8_message); void debug_ctx_Err_SetObject(HPyContext *dctx, DHPy h_type, DHPy h_value); @@ -96,13 +99,16 @@ int debug_ctx_SetAttr_s(HPyContext *dctx, DHPy obj, const char *utf8_name, DHPy DHPy debug_ctx_GetItem(HPyContext *dctx, DHPy obj, DHPy key); DHPy debug_ctx_GetItem_i(HPyContext *dctx, DHPy obj, HPy_ssize_t idx); DHPy debug_ctx_GetItem_s(HPyContext *dctx, DHPy obj, const char *utf8_key); +DHPy debug_ctx_GetSlice(HPyContext *dctx, DHPy obj, HPy_ssize_t start, HPy_ssize_t end); int debug_ctx_Contains(HPyContext *dctx, DHPy container, DHPy key); int debug_ctx_SetItem(HPyContext *dctx, DHPy obj, DHPy key, DHPy value); int debug_ctx_SetItem_i(HPyContext *dctx, DHPy obj, HPy_ssize_t idx, DHPy value); int debug_ctx_SetItem_s(HPyContext *dctx, DHPy obj, const char *utf8_key, DHPy value); +int debug_ctx_SetSlice(HPyContext *dctx, DHPy obj, HPy_ssize_t start, HPy_ssize_t end, DHPy value); int debug_ctx_DelItem(HPyContext *dctx, DHPy obj, DHPy key); int debug_ctx_DelItem_i(HPyContext *dctx, DHPy obj, HPy_ssize_t idx); int debug_ctx_DelItem_s(HPyContext *dctx, DHPy obj, const char *utf8_key); +int debug_ctx_DelSlice(HPyContext *dctx, DHPy obj, HPy_ssize_t start, HPy_ssize_t end); DHPy debug_ctx_Type(HPyContext *dctx, DHPy obj); int debug_ctx_TypeCheck(HPyContext *dctx, DHPy obj, DHPy type); const char *debug_ctx_Type_GetName(HPyContext *dctx, DHPy type); @@ -116,6 +122,7 @@ void *debug_ctx_AsStruct_Float(HPyContext *dctx, DHPy h); void *debug_ctx_AsStruct_Unicode(HPyContext *dctx, DHPy h); void *debug_ctx_AsStruct_Tuple(HPyContext *dctx, DHPy h); void *debug_ctx_AsStruct_List(HPyContext *dctx, DHPy h); +void *debug_ctx_AsStruct_Dict(HPyContext *dctx, DHPy h); HPyType_BuiltinShape debug_ctx_Type_GetBuiltinShape(HPyContext *dctx, DHPy h_type); DHPy debug_ctx_New(HPyContext *dctx, DHPy h_type, void **data); DHPy debug_ctx_Repr(HPyContext *dctx, DHPy obj); @@ -150,12 +157,14 @@ DHPy debug_ctx_Unicode_Substring(HPyContext *dctx, DHPy str, HPy_ssize_t start, int debug_ctx_List_Check(HPyContext *dctx, DHPy h); DHPy debug_ctx_List_New(HPyContext *dctx, HPy_ssize_t len); int debug_ctx_List_Append(HPyContext *dctx, DHPy h_list, DHPy h_item); +int debug_ctx_List_Insert(HPyContext *dctx, DHPy h_list, HPy_ssize_t index, DHPy h_item); int debug_ctx_Dict_Check(HPyContext *dctx, DHPy h); DHPy debug_ctx_Dict_New(HPyContext *dctx); DHPy debug_ctx_Dict_Keys(HPyContext *dctx, DHPy h); DHPy debug_ctx_Dict_Copy(HPyContext *dctx, DHPy h); int debug_ctx_Tuple_Check(HPyContext *dctx, DHPy h); -DHPy debug_ctx_Tuple_FromArray(HPyContext *dctx, DHPy items[], HPy_ssize_t n); +DHPy debug_ctx_Tuple_FromArray(HPyContext *dctx, const DHPy items[], HPy_ssize_t n); +DHPy debug_ctx_Slice_New(HPyContext *dctx, DHPy start, DHPy stop, DHPy step); int debug_ctx_Slice_Unpack(HPyContext *dctx, DHPy slice, HPy_ssize_t *start, HPy_ssize_t *stop, HPy_ssize_t *step); DHPy debug_ctx_Import_ImportModule(HPyContext *dctx, const char *utf8_name); DHPy debug_ctx_Capsule_New(HPyContext *dctx, void *pointer, const char *utf8_name, HPyCapsule_Destructor *destructor); @@ -275,6 +284,7 @@ static inline void debug_ctx_init_fields(HPyContext *dctx, HPyContext *uctx) dctx->h_MemoryViewType = DHPy_open_immortal(dctx, uctx->h_MemoryViewType); dctx->h_CapsuleType = DHPy_open_immortal(dctx, uctx->h_CapsuleType); dctx->h_SliceType = DHPy_open_immortal(dctx, uctx->h_SliceType); + dctx->h_DictType = DHPy_open_immortal(dctx, uctx->h_DictType); dctx->h_Builtins = DHPy_open_immortal(dctx, uctx->h_Builtins); dctx->ctx_Dup = &debug_ctx_Dup; dctx->ctx_Close = &debug_ctx_Close; @@ -337,6 +347,9 @@ static inline void debug_ctx_init_fields(HPyContext *dctx, HPyContext *uctx) dctx->ctx_CallTupleDict = &debug_ctx_CallTupleDict; dctx->ctx_Call = &debug_ctx_Call; dctx->ctx_CallMethod = &debug_ctx_CallMethod; + dctx->ctx_GetIter = &debug_ctx_GetIter; + dctx->ctx_Iter_Next = &debug_ctx_Iter_Next; + dctx->ctx_Iter_Check = &debug_ctx_Iter_Check; dctx->ctx_FatalError = &debug_ctx_FatalError; dctx->ctx_Err_SetString = &debug_ctx_Err_SetString; dctx->ctx_Err_SetObject = &debug_ctx_Err_SetObject; @@ -362,13 +375,16 @@ static inline void debug_ctx_init_fields(HPyContext *dctx, HPyContext *uctx) dctx->ctx_GetItem = &debug_ctx_GetItem; dctx->ctx_GetItem_i = &debug_ctx_GetItem_i; dctx->ctx_GetItem_s = &debug_ctx_GetItem_s; + dctx->ctx_GetSlice = &debug_ctx_GetSlice; dctx->ctx_Contains = &debug_ctx_Contains; dctx->ctx_SetItem = &debug_ctx_SetItem; dctx->ctx_SetItem_i = &debug_ctx_SetItem_i; dctx->ctx_SetItem_s = &debug_ctx_SetItem_s; + dctx->ctx_SetSlice = &debug_ctx_SetSlice; dctx->ctx_DelItem = &debug_ctx_DelItem; dctx->ctx_DelItem_i = &debug_ctx_DelItem_i; dctx->ctx_DelItem_s = &debug_ctx_DelItem_s; + dctx->ctx_DelSlice = &debug_ctx_DelSlice; dctx->ctx_Type = &debug_ctx_Type; dctx->ctx_TypeCheck = &debug_ctx_TypeCheck; dctx->ctx_Type_GetName = &debug_ctx_Type_GetName; @@ -382,6 +398,7 @@ static inline void debug_ctx_init_fields(HPyContext *dctx, HPyContext *uctx) dctx->ctx_AsStruct_Unicode = &debug_ctx_AsStruct_Unicode; dctx->ctx_AsStruct_Tuple = &debug_ctx_AsStruct_Tuple; dctx->ctx_AsStruct_List = &debug_ctx_AsStruct_List; + dctx->ctx_AsStruct_Dict = &debug_ctx_AsStruct_Dict; dctx->ctx_Type_GetBuiltinShape = &debug_ctx_Type_GetBuiltinShape; dctx->ctx_New = &debug_ctx_New; dctx->ctx_Repr = &debug_ctx_Repr; @@ -416,12 +433,14 @@ static inline void debug_ctx_init_fields(HPyContext *dctx, HPyContext *uctx) dctx->ctx_List_Check = &debug_ctx_List_Check; dctx->ctx_List_New = &debug_ctx_List_New; dctx->ctx_List_Append = &debug_ctx_List_Append; + dctx->ctx_List_Insert = &debug_ctx_List_Insert; dctx->ctx_Dict_Check = &debug_ctx_Dict_Check; dctx->ctx_Dict_New = &debug_ctx_Dict_New; dctx->ctx_Dict_Keys = &debug_ctx_Dict_Keys; dctx->ctx_Dict_Copy = &debug_ctx_Dict_Copy; dctx->ctx_Tuple_Check = &debug_ctx_Tuple_Check; dctx->ctx_Tuple_FromArray = &debug_ctx_Tuple_FromArray; + dctx->ctx_Slice_New = &debug_ctx_Slice_New; dctx->ctx_Slice_Unpack = &debug_ctx_Slice_Unpack; dctx->ctx_Import_ImportModule = &debug_ctx_Import_ImportModule; dctx->ctx_Capsule_New = &debug_ctx_Capsule_New; diff --git a/graalpython/com.oracle.graal.python.jni/src/debug/autogen_debug_wrappers.c b/graalpython/com.oracle.graal.python.jni/src/debug/autogen_debug_wrappers.c index 59663daecc..e9eb6a15c3 100644 --- a/graalpython/com.oracle.graal.python.jni/src/debug/autogen_debug_wrappers.c +++ b/graalpython/com.oracle.graal.python.jni/src/debug/autogen_debug_wrappers.c @@ -731,6 +731,42 @@ DHPy debug_ctx_CallTupleDict(HPyContext *dctx, DHPy callable, DHPy args, DHPy kw return DHPy_open(dctx, universal_result); } +DHPy debug_ctx_GetIter(HPyContext *dctx, DHPy obj) +{ + if (!get_ctx_info(dctx)->is_valid) { + report_invalid_debug_context(); + } + HPy dh_obj = DHPy_unwrap(dctx, obj); + get_ctx_info(dctx)->is_valid = false; + HPy universal_result = HPy_GetIter(get_info(dctx)->uctx, dh_obj); + get_ctx_info(dctx)->is_valid = true; + return DHPy_open(dctx, universal_result); +} + +DHPy debug_ctx_Iter_Next(HPyContext *dctx, DHPy obj) +{ + if (!get_ctx_info(dctx)->is_valid) { + report_invalid_debug_context(); + } + HPy dh_obj = DHPy_unwrap(dctx, obj); + get_ctx_info(dctx)->is_valid = false; + HPy universal_result = HPyIter_Next(get_info(dctx)->uctx, dh_obj); + get_ctx_info(dctx)->is_valid = true; + return DHPy_open(dctx, universal_result); +} + +int debug_ctx_Iter_Check(HPyContext *dctx, DHPy obj) +{ + if (!get_ctx_info(dctx)->is_valid) { + report_invalid_debug_context(); + } + HPy dh_obj = DHPy_unwrap(dctx, obj); + get_ctx_info(dctx)->is_valid = false; + int universal_result = HPyIter_Check(get_info(dctx)->uctx, dh_obj); + get_ctx_info(dctx)->is_valid = true; + return universal_result; +} + void debug_ctx_FatalError(HPyContext *dctx, const char *message) { if (!get_ctx_info(dctx)->is_valid) { @@ -1007,6 +1043,18 @@ DHPy debug_ctx_GetItem_s(HPyContext *dctx, DHPy obj, const char *utf8_key) return DHPy_open(dctx, universal_result); } +DHPy debug_ctx_GetSlice(HPyContext *dctx, DHPy obj, HPy_ssize_t start, HPy_ssize_t end) +{ + if (!get_ctx_info(dctx)->is_valid) { + report_invalid_debug_context(); + } + HPy dh_obj = DHPy_unwrap(dctx, obj); + get_ctx_info(dctx)->is_valid = false; + HPy universal_result = HPy_GetSlice(get_info(dctx)->uctx, dh_obj, start, end); + get_ctx_info(dctx)->is_valid = true; + return DHPy_open(dctx, universal_result); +} + int debug_ctx_Contains(HPyContext *dctx, DHPy container, DHPy key) { if (!get_ctx_info(dctx)->is_valid) { @@ -1060,6 +1108,19 @@ int debug_ctx_SetItem_s(HPyContext *dctx, DHPy obj, const char *utf8_key, DHPy v return universal_result; } +int debug_ctx_SetSlice(HPyContext *dctx, DHPy obj, HPy_ssize_t start, HPy_ssize_t end, DHPy value) +{ + if (!get_ctx_info(dctx)->is_valid) { + report_invalid_debug_context(); + } + HPy dh_obj = DHPy_unwrap(dctx, obj); + HPy dh_value = DHPy_unwrap(dctx, value); + get_ctx_info(dctx)->is_valid = false; + int universal_result = HPy_SetSlice(get_info(dctx)->uctx, dh_obj, start, end, dh_value); + get_ctx_info(dctx)->is_valid = true; + return universal_result; +} + int debug_ctx_DelItem(HPyContext *dctx, DHPy obj, DHPy key) { if (!get_ctx_info(dctx)->is_valid) { @@ -1097,6 +1158,18 @@ int debug_ctx_DelItem_s(HPyContext *dctx, DHPy obj, const char *utf8_key) return universal_result; } +int debug_ctx_DelSlice(HPyContext *dctx, DHPy obj, HPy_ssize_t start, HPy_ssize_t end) +{ + if (!get_ctx_info(dctx)->is_valid) { + report_invalid_debug_context(); + } + HPy dh_obj = DHPy_unwrap(dctx, obj); + get_ctx_info(dctx)->is_valid = false; + int universal_result = HPy_DelSlice(get_info(dctx)->uctx, dh_obj, start, end); + get_ctx_info(dctx)->is_valid = true; + return universal_result; +} + DHPy debug_ctx_Type(HPyContext *dctx, DHPy obj) { if (!get_ctx_info(dctx)->is_valid) { @@ -1476,6 +1549,19 @@ int debug_ctx_List_Append(HPyContext *dctx, DHPy h_list, DHPy h_item) return universal_result; } +int debug_ctx_List_Insert(HPyContext *dctx, DHPy h_list, HPy_ssize_t index, DHPy h_item) +{ + if (!get_ctx_info(dctx)->is_valid) { + report_invalid_debug_context(); + } + HPy dh_h_list = DHPy_unwrap(dctx, h_list); + HPy dh_h_item = DHPy_unwrap(dctx, h_item); + get_ctx_info(dctx)->is_valid = false; + int universal_result = HPyList_Insert(get_info(dctx)->uctx, dh_h_list, index, dh_h_item); + get_ctx_info(dctx)->is_valid = true; + return universal_result; +} + int debug_ctx_Dict_Check(HPyContext *dctx, DHPy h) { if (!get_ctx_info(dctx)->is_valid) { @@ -1535,6 +1621,20 @@ int debug_ctx_Tuple_Check(HPyContext *dctx, DHPy h) return universal_result; } +DHPy debug_ctx_Slice_New(HPyContext *dctx, DHPy start, DHPy stop, DHPy step) +{ + if (!get_ctx_info(dctx)->is_valid) { + report_invalid_debug_context(); + } + HPy dh_start = DHPy_unwrap(dctx, start); + HPy dh_stop = DHPy_unwrap(dctx, stop); + HPy dh_step = DHPy_unwrap(dctx, step); + get_ctx_info(dctx)->is_valid = false; + HPy universal_result = HPySlice_New(get_info(dctx)->uctx, dh_start, dh_stop, dh_step); + get_ctx_info(dctx)->is_valid = true; + return DHPy_open(dctx, universal_result); +} + int debug_ctx_Slice_Unpack(HPyContext *dctx, DHPy slice, HPy_ssize_t *start, HPy_ssize_t *stop, HPy_ssize_t *step) { if (!get_ctx_info(dctx)->is_valid) { diff --git a/graalpython/com.oracle.graal.python.jni/src/debug/debug_ctx.c b/graalpython/com.oracle.graal.python.jni/src/debug/debug_ctx.c index 1f021daa5c..de661ea2c9 100644 --- a/graalpython/com.oracle.graal.python.jni/src/debug/debug_ctx.c +++ b/graalpython/com.oracle.graal.python.jni/src/debug/debug_ctx.c @@ -256,7 +256,7 @@ const char *debug_ctx_Bytes_AS_STRING(HPyContext *dctx, DHPy h) return (const char *)protect_and_associate_data_ptr(h, (void *)ptr, data_size); } -DHPy debug_ctx_Tuple_FromArray(HPyContext *dctx, DHPy dh_items[], HPy_ssize_t n) +DHPy debug_ctx_Tuple_FromArray(HPyContext *dctx, const DHPy dh_items[], HPy_ssize_t n) { if (!get_ctx_info(dctx)->is_valid) { report_invalid_debug_context(); @@ -328,6 +328,7 @@ static const char *get_builtin_shape_name(HPyType_BuiltinShape shape) SHAPE_NAME(HPyType_BuiltinShape_Unicode) SHAPE_NAME(HPyType_BuiltinShape_Tuple) SHAPE_NAME(HPyType_BuiltinShape_List) + SHAPE_NAME(HPyType_BuiltinShape_Dict) } return ""; #undef SHAPE_NAME @@ -372,6 +373,8 @@ MAKE_debug_ctx_AsStruct(Tuple) MAKE_debug_ctx_AsStruct(List) +MAKE_debug_ctx_AsStruct(Dict) + /* ~~~ debug mode implementation of HPyTracker ~~~ This is a bit special and it's worth explaining what is going on. diff --git a/graalpython/com.oracle.graal.python.jni/src/trace/autogen_trace_ctx_init.h b/graalpython/com.oracle.graal.python.jni/src/trace/autogen_trace_ctx_init.h index 4e452b7e28..fef5028491 100644 --- a/graalpython/com.oracle.graal.python.jni/src/trace/autogen_trace_ctx_init.h +++ b/graalpython/com.oracle.graal.python.jni/src/trace/autogen_trace_ctx_init.h @@ -71,6 +71,9 @@ int trace_ctx_Callable_Check(HPyContext *tctx, HPy h); HPy trace_ctx_CallTupleDict(HPyContext *tctx, HPy callable, HPy args, HPy kw); HPy trace_ctx_Call(HPyContext *tctx, HPy callable, const HPy *args, size_t nargs, HPy kwnames); HPy trace_ctx_CallMethod(HPyContext *tctx, HPy name, const HPy *args, size_t nargs, HPy kwnames); +HPy trace_ctx_GetIter(HPyContext *tctx, HPy obj); +HPy trace_ctx_Iter_Next(HPyContext *tctx, HPy obj); +int trace_ctx_Iter_Check(HPyContext *tctx, HPy obj); void trace_ctx_Err_SetString(HPyContext *tctx, HPy h_type, const char *utf8_message); void trace_ctx_Err_SetObject(HPyContext *tctx, HPy h_type, HPy h_value); HPy trace_ctx_Err_SetFromErrnoWithFilename(HPyContext *tctx, HPy h_type, const char *filename_fsencoded); @@ -95,13 +98,16 @@ int trace_ctx_SetAttr_s(HPyContext *tctx, HPy obj, const char *utf8_name, HPy va HPy trace_ctx_GetItem(HPyContext *tctx, HPy obj, HPy key); HPy trace_ctx_GetItem_i(HPyContext *tctx, HPy obj, HPy_ssize_t idx); HPy trace_ctx_GetItem_s(HPyContext *tctx, HPy obj, const char *utf8_key); +HPy trace_ctx_GetSlice(HPyContext *tctx, HPy obj, HPy_ssize_t start, HPy_ssize_t end); int trace_ctx_Contains(HPyContext *tctx, HPy container, HPy key); int trace_ctx_SetItem(HPyContext *tctx, HPy obj, HPy key, HPy value); int trace_ctx_SetItem_i(HPyContext *tctx, HPy obj, HPy_ssize_t idx, HPy value); int trace_ctx_SetItem_s(HPyContext *tctx, HPy obj, const char *utf8_key, HPy value); +int trace_ctx_SetSlice(HPyContext *tctx, HPy obj, HPy_ssize_t start, HPy_ssize_t end, HPy value); int trace_ctx_DelItem(HPyContext *tctx, HPy obj, HPy key); int trace_ctx_DelItem_i(HPyContext *tctx, HPy obj, HPy_ssize_t idx); int trace_ctx_DelItem_s(HPyContext *tctx, HPy obj, const char *utf8_key); +int trace_ctx_DelSlice(HPyContext *tctx, HPy obj, HPy_ssize_t start, HPy_ssize_t end); HPy trace_ctx_Type(HPyContext *tctx, HPy obj); int trace_ctx_TypeCheck(HPyContext *tctx, HPy obj, HPy type); const char *trace_ctx_Type_GetName(HPyContext *tctx, HPy type); @@ -115,6 +121,7 @@ void *trace_ctx_AsStruct_Float(HPyContext *tctx, HPy h); void *trace_ctx_AsStruct_Unicode(HPyContext *tctx, HPy h); void *trace_ctx_AsStruct_Tuple(HPyContext *tctx, HPy h); void *trace_ctx_AsStruct_List(HPyContext *tctx, HPy h); +void *trace_ctx_AsStruct_Dict(HPyContext *tctx, HPy h); HPyType_BuiltinShape trace_ctx_Type_GetBuiltinShape(HPyContext *tctx, HPy h_type); HPy trace_ctx_New(HPyContext *tctx, HPy h_type, void **data); HPy trace_ctx_Repr(HPyContext *tctx, HPy obj); @@ -149,12 +156,14 @@ HPy trace_ctx_Unicode_Substring(HPyContext *tctx, HPy str, HPy_ssize_t start, HP int trace_ctx_List_Check(HPyContext *tctx, HPy h); HPy trace_ctx_List_New(HPyContext *tctx, HPy_ssize_t len); int trace_ctx_List_Append(HPyContext *tctx, HPy h_list, HPy h_item); +int trace_ctx_List_Insert(HPyContext *tctx, HPy h_list, HPy_ssize_t index, HPy h_item); int trace_ctx_Dict_Check(HPyContext *tctx, HPy h); HPy trace_ctx_Dict_New(HPyContext *tctx); HPy trace_ctx_Dict_Keys(HPyContext *tctx, HPy h); HPy trace_ctx_Dict_Copy(HPyContext *tctx, HPy h); int trace_ctx_Tuple_Check(HPyContext *tctx, HPy h); -HPy trace_ctx_Tuple_FromArray(HPyContext *tctx, HPy items[], HPy_ssize_t n); +HPy trace_ctx_Tuple_FromArray(HPyContext *tctx, const HPy items[], HPy_ssize_t n); +HPy trace_ctx_Slice_New(HPyContext *tctx, HPy start, HPy stop, HPy step); int trace_ctx_Slice_Unpack(HPyContext *tctx, HPy slice, HPy_ssize_t *start, HPy_ssize_t *stop, HPy_ssize_t *step); HPy trace_ctx_Import_ImportModule(HPyContext *tctx, const char *utf8_name); HPy trace_ctx_Capsule_New(HPyContext *tctx, void *pointer, const char *utf8_name, HPyCapsule_Destructor *destructor); @@ -193,8 +202,8 @@ static inline void trace_ctx_init_info(HPyTraceInfo *info, HPyContext *uctx) { info->magic_number = HPY_TRACE_MAGIC; info->uctx = uctx; - info->call_counts = (uint64_t *)calloc(263, sizeof(uint64_t)); - info->durations = (_HPyTime_t *)calloc(263, sizeof(_HPyTime_t)); + info->call_counts = (uint64_t *)calloc(273, sizeof(uint64_t)); + info->durations = (_HPyTime_t *)calloc(273, sizeof(_HPyTime_t)); info->on_enter_func = HPy_NULL; info->on_exit_func = HPy_NULL; } @@ -292,6 +301,7 @@ static inline void trace_ctx_init_fields(HPyContext *tctx, HPyContext *uctx) tctx->h_MemoryViewType = uctx->h_MemoryViewType; tctx->h_CapsuleType = uctx->h_CapsuleType; tctx->h_SliceType = uctx->h_SliceType; + tctx->h_DictType = uctx->h_DictType; tctx->h_Builtins = uctx->h_Builtins; tctx->ctx_Dup = &trace_ctx_Dup; tctx->ctx_Close = &trace_ctx_Close; @@ -354,6 +364,9 @@ static inline void trace_ctx_init_fields(HPyContext *tctx, HPyContext *uctx) tctx->ctx_CallTupleDict = &trace_ctx_CallTupleDict; tctx->ctx_Call = &trace_ctx_Call; tctx->ctx_CallMethod = &trace_ctx_CallMethod; + tctx->ctx_GetIter = &trace_ctx_GetIter; + tctx->ctx_Iter_Next = &trace_ctx_Iter_Next; + tctx->ctx_Iter_Check = &trace_ctx_Iter_Check; tctx->ctx_FatalError = uctx->ctx_FatalError; tctx->ctx_Err_SetString = &trace_ctx_Err_SetString; tctx->ctx_Err_SetObject = &trace_ctx_Err_SetObject; @@ -379,13 +392,16 @@ static inline void trace_ctx_init_fields(HPyContext *tctx, HPyContext *uctx) tctx->ctx_GetItem = &trace_ctx_GetItem; tctx->ctx_GetItem_i = &trace_ctx_GetItem_i; tctx->ctx_GetItem_s = &trace_ctx_GetItem_s; + tctx->ctx_GetSlice = &trace_ctx_GetSlice; tctx->ctx_Contains = &trace_ctx_Contains; tctx->ctx_SetItem = &trace_ctx_SetItem; tctx->ctx_SetItem_i = &trace_ctx_SetItem_i; tctx->ctx_SetItem_s = &trace_ctx_SetItem_s; + tctx->ctx_SetSlice = &trace_ctx_SetSlice; tctx->ctx_DelItem = &trace_ctx_DelItem; tctx->ctx_DelItem_i = &trace_ctx_DelItem_i; tctx->ctx_DelItem_s = &trace_ctx_DelItem_s; + tctx->ctx_DelSlice = &trace_ctx_DelSlice; tctx->ctx_Type = &trace_ctx_Type; tctx->ctx_TypeCheck = &trace_ctx_TypeCheck; tctx->ctx_Type_GetName = &trace_ctx_Type_GetName; @@ -399,6 +415,7 @@ static inline void trace_ctx_init_fields(HPyContext *tctx, HPyContext *uctx) tctx->ctx_AsStruct_Unicode = &trace_ctx_AsStruct_Unicode; tctx->ctx_AsStruct_Tuple = &trace_ctx_AsStruct_Tuple; tctx->ctx_AsStruct_List = &trace_ctx_AsStruct_List; + tctx->ctx_AsStruct_Dict = &trace_ctx_AsStruct_Dict; tctx->ctx_Type_GetBuiltinShape = &trace_ctx_Type_GetBuiltinShape; tctx->ctx_New = &trace_ctx_New; tctx->ctx_Repr = &trace_ctx_Repr; @@ -433,12 +450,14 @@ static inline void trace_ctx_init_fields(HPyContext *tctx, HPyContext *uctx) tctx->ctx_List_Check = &trace_ctx_List_Check; tctx->ctx_List_New = &trace_ctx_List_New; tctx->ctx_List_Append = &trace_ctx_List_Append; + tctx->ctx_List_Insert = &trace_ctx_List_Insert; tctx->ctx_Dict_Check = &trace_ctx_Dict_Check; tctx->ctx_Dict_New = &trace_ctx_Dict_New; tctx->ctx_Dict_Keys = &trace_ctx_Dict_Keys; tctx->ctx_Dict_Copy = &trace_ctx_Dict_Copy; tctx->ctx_Tuple_Check = &trace_ctx_Tuple_Check; tctx->ctx_Tuple_FromArray = &trace_ctx_Tuple_FromArray; + tctx->ctx_Slice_New = &trace_ctx_Slice_New; tctx->ctx_Slice_Unpack = &trace_ctx_Slice_Unpack; tctx->ctx_Import_ImportModule = &trace_ctx_Import_ImportModule; tctx->ctx_Capsule_New = &trace_ctx_Capsule_New; diff --git a/graalpython/com.oracle.graal.python.jni/src/trace/autogen_trace_func_table.c b/graalpython/com.oracle.graal.python.jni/src/trace/autogen_trace_func_table.c index 7a1a16332f..97d8308d53 100644 --- a/graalpython/com.oracle.graal.python.jni/src/trace/autogen_trace_func_table.c +++ b/graalpython/com.oracle.graal.python.jni/src/trace/autogen_trace_func_table.c @@ -12,7 +12,7 @@ #include "trace_internal.h" -#define TRACE_NFUNC 180 +#define TRACE_NFUNC 189 #define NO_FUNC "" static const char *trace_func_table[] = { @@ -279,6 +279,16 @@ static const char *trace_func_table[] = { "ctx_SetCallFunction", "ctx_Call", "ctx_CallMethod", + NO_FUNC, + "ctx_AsStruct_Dict", + "ctx_List_Insert", + "ctx_GetSlice", + "ctx_SetSlice", + "ctx_DelSlice", + "ctx_GetIter", + "ctx_Iter_Next", + "ctx_Iter_Check", + "ctx_Slice_New", NULL /* sentinel */ }; @@ -289,7 +299,7 @@ int hpy_trace_get_nfunc(void) const char * hpy_trace_get_func_name(int idx) { - if (idx >= 0 && idx < 263) + if (idx >= 0 && idx < 273) return trace_func_table[idx]; return NULL; } diff --git a/graalpython/com.oracle.graal.python.jni/src/trace/autogen_trace_wrappers.c b/graalpython/com.oracle.graal.python.jni/src/trace/autogen_trace_wrappers.c index cd8a4b1308..4940a12a9d 100644 --- a/graalpython/com.oracle.graal.python.jni/src/trace/autogen_trace_wrappers.c +++ b/graalpython/com.oracle.graal.python.jni/src/trace/autogen_trace_wrappers.c @@ -804,6 +804,45 @@ HPy trace_ctx_CallMethod(HPyContext *tctx, HPy name, const HPy *args, size_t nar return res; } +HPy trace_ctx_GetIter(HPyContext *tctx, HPy obj) +{ + HPyTraceInfo *info = hpy_trace_on_enter(tctx, 269); + HPyContext *uctx = info->uctx; + _HPyTime_t _ts_start, _ts_end; + _HPyClockStatus_t r0, r1; + r0 = get_monotonic_clock(&_ts_start); + HPy res = HPy_GetIter(uctx, obj); + r1 = get_monotonic_clock(&_ts_end); + hpy_trace_on_exit(info, 269, r0, r1, &_ts_start, &_ts_end); + return res; +} + +HPy trace_ctx_Iter_Next(HPyContext *tctx, HPy obj) +{ + HPyTraceInfo *info = hpy_trace_on_enter(tctx, 270); + HPyContext *uctx = info->uctx; + _HPyTime_t _ts_start, _ts_end; + _HPyClockStatus_t r0, r1; + r0 = get_monotonic_clock(&_ts_start); + HPy res = HPyIter_Next(uctx, obj); + r1 = get_monotonic_clock(&_ts_end); + hpy_trace_on_exit(info, 270, r0, r1, &_ts_start, &_ts_end); + return res; +} + +int trace_ctx_Iter_Check(HPyContext *tctx, HPy obj) +{ + HPyTraceInfo *info = hpy_trace_on_enter(tctx, 271); + HPyContext *uctx = info->uctx; + _HPyTime_t _ts_start, _ts_end; + _HPyClockStatus_t r0, r1; + r0 = get_monotonic_clock(&_ts_start); + int res = HPyIter_Check(uctx, obj); + r1 = get_monotonic_clock(&_ts_end); + hpy_trace_on_exit(info, 271, r0, r1, &_ts_start, &_ts_end); + return res; +} + void trace_ctx_Err_SetString(HPyContext *tctx, HPy h_type, const char *utf8_message) { HPyTraceInfo *info = hpy_trace_on_enter(tctx, 137); @@ -1110,6 +1149,19 @@ HPy trace_ctx_GetItem_s(HPyContext *tctx, HPy obj, const char *utf8_key) return res; } +HPy trace_ctx_GetSlice(HPyContext *tctx, HPy obj, HPy_ssize_t start, HPy_ssize_t end) +{ + HPyTraceInfo *info = hpy_trace_on_enter(tctx, 266); + HPyContext *uctx = info->uctx; + _HPyTime_t _ts_start, _ts_end; + _HPyClockStatus_t r0, r1; + r0 = get_monotonic_clock(&_ts_start); + HPy res = HPy_GetSlice(uctx, obj, start, end); + r1 = get_monotonic_clock(&_ts_end); + hpy_trace_on_exit(info, 266, r0, r1, &_ts_start, &_ts_end); + return res; +} + int trace_ctx_Contains(HPyContext *tctx, HPy container, HPy key) { HPyTraceInfo *info = hpy_trace_on_enter(tctx, 161); @@ -1162,6 +1214,19 @@ int trace_ctx_SetItem_s(HPyContext *tctx, HPy obj, const char *utf8_key, HPy val return res; } +int trace_ctx_SetSlice(HPyContext *tctx, HPy obj, HPy_ssize_t start, HPy_ssize_t end, HPy value) +{ + HPyTraceInfo *info = hpy_trace_on_enter(tctx, 267); + HPyContext *uctx = info->uctx; + _HPyTime_t _ts_start, _ts_end; + _HPyClockStatus_t r0, r1; + r0 = get_monotonic_clock(&_ts_start); + int res = HPy_SetSlice(uctx, obj, start, end, value); + r1 = get_monotonic_clock(&_ts_end); + hpy_trace_on_exit(info, 267, r0, r1, &_ts_start, &_ts_end); + return res; +} + int trace_ctx_DelItem(HPyContext *tctx, HPy obj, HPy key) { HPyTraceInfo *info = hpy_trace_on_enter(tctx, 235); @@ -1201,6 +1266,19 @@ int trace_ctx_DelItem_s(HPyContext *tctx, HPy obj, const char *utf8_key) return res; } +int trace_ctx_DelSlice(HPyContext *tctx, HPy obj, HPy_ssize_t start, HPy_ssize_t end) +{ + HPyTraceInfo *info = hpy_trace_on_enter(tctx, 268); + HPyContext *uctx = info->uctx; + _HPyTime_t _ts_start, _ts_end; + _HPyClockStatus_t r0, r1; + r0 = get_monotonic_clock(&_ts_start); + int res = HPy_DelSlice(uctx, obj, start, end); + r1 = get_monotonic_clock(&_ts_end); + hpy_trace_on_exit(info, 268, r0, r1, &_ts_start, &_ts_end); + return res; +} + HPy trace_ctx_Type(HPyContext *tctx, HPy obj) { HPyTraceInfo *info = hpy_trace_on_enter(tctx, 165); @@ -1370,6 +1448,19 @@ void *trace_ctx_AsStruct_List(HPyContext *tctx, HPy h) return res; } +void *trace_ctx_AsStruct_Dict(HPyContext *tctx, HPy h) +{ + HPyTraceInfo *info = hpy_trace_on_enter(tctx, 264); + HPyContext *uctx = info->uctx; + _HPyTime_t _ts_start, _ts_end; + _HPyClockStatus_t r0, r1; + r0 = get_monotonic_clock(&_ts_start); + void * res = _HPy_AsStruct_Dict(uctx, h); + r1 = get_monotonic_clock(&_ts_end); + hpy_trace_on_exit(info, 264, r0, r1, &_ts_start, &_ts_end); + return res; +} + HPyType_BuiltinShape trace_ctx_Type_GetBuiltinShape(HPyContext *tctx, HPy h_type) { HPyTraceInfo *info = hpy_trace_on_enter(tctx, 234); @@ -1812,6 +1903,19 @@ int trace_ctx_List_Append(HPyContext *tctx, HPy h_list, HPy h_item) return res; } +int trace_ctx_List_Insert(HPyContext *tctx, HPy h_list, HPy_ssize_t index, HPy h_item) +{ + HPyTraceInfo *info = hpy_trace_on_enter(tctx, 265); + HPyContext *uctx = info->uctx; + _HPyTime_t _ts_start, _ts_end; + _HPyClockStatus_t r0, r1; + r0 = get_monotonic_clock(&_ts_start); + int res = HPyList_Insert(uctx, h_list, index, h_item); + r1 = get_monotonic_clock(&_ts_end); + hpy_trace_on_exit(info, 265, r0, r1, &_ts_start, &_ts_end); + return res; +} + int trace_ctx_Dict_Check(HPyContext *tctx, HPy h) { HPyTraceInfo *info = hpy_trace_on_enter(tctx, 201); @@ -1877,7 +1981,7 @@ int trace_ctx_Tuple_Check(HPyContext *tctx, HPy h) return res; } -HPy trace_ctx_Tuple_FromArray(HPyContext *tctx, HPy items[], HPy_ssize_t n) +HPy trace_ctx_Tuple_FromArray(HPyContext *tctx, const HPy items[], HPy_ssize_t n) { HPyTraceInfo *info = hpy_trace_on_enter(tctx, 204); HPyContext *uctx = info->uctx; @@ -1890,6 +1994,19 @@ HPy trace_ctx_Tuple_FromArray(HPyContext *tctx, HPy items[], HPy_ssize_t n) return res; } +HPy trace_ctx_Slice_New(HPyContext *tctx, HPy start, HPy stop, HPy step) +{ + HPyTraceInfo *info = hpy_trace_on_enter(tctx, 272); + HPyContext *uctx = info->uctx; + _HPyTime_t _ts_start, _ts_end; + _HPyClockStatus_t r0, r1; + r0 = get_monotonic_clock(&_ts_start); + HPy res = HPySlice_New(uctx, start, stop, step); + r1 = get_monotonic_clock(&_ts_end); + hpy_trace_on_exit(info, 272, r0, r1, &_ts_start, &_ts_end); + return res; +} + int trace_ctx_Slice_Unpack(HPyContext *tctx, HPy slice, HPy_ssize_t *start, HPy_ssize_t *stop, HPy_ssize_t *step) { HPyTraceInfo *info = hpy_trace_on_enter(tctx, 259); diff --git a/graalpython/lib-graalpython/modules/hpy/devel/__init__.py b/graalpython/lib-graalpython/modules/hpy/devel/__init__.py index e59dd027ac..3b7f6e6d18 100644 --- a/graalpython/lib-graalpython/modules/hpy/devel/__init__.py +++ b/graalpython/lib-graalpython/modules/hpy/devel/__init__.py @@ -428,7 +428,7 @@ def write_stub(self, output_dir, ext, compile=False): ext_file = os.path.basename(ext._file_name) module_name = ext_file.split(".")[0] if not self.dry_run: - with open(stub_file, 'w') as f: + with open(stub_file, 'w', encoding='utf-8') as f: f.write(_HPY_UNIVERSAL_MODULE_STUB_TEMPLATE.format( ext_file=ext_file, module_name=module_name) ) diff --git a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_object.c b/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_object.c index dc6ee2b2bf..9fa72a205c 100644 --- a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_object.c +++ b/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_object.c @@ -15,6 +15,14 @@ ctx_Dump(HPyContext *ctx, HPy h) _PyObject_Dump(_h2py(h)); } +_HPy_HIDDEN HPy +ctx_Type(HPyContext *ctx, HPy obj) +{ + PyTypeObject *tp = Py_TYPE(_h2py(obj)); + Py_INCREF(tp); + return _py2h((PyObject *)tp); +} + /* NOTE: In contrast to CPython, HPy has to check that 'h_type' is a type. This is not necessary on CPython because it requires C type 'PyTypeObject *' but here we can only receive an HPy handle. Appropriate checking of the argument @@ -34,10 +42,14 @@ ctx_Is(HPyContext *ctx, HPy h_obj, HPy h_other) _HPy_HIDDEN HPy ctx_GetItem_i(HPyContext *ctx, HPy obj, HPy_ssize_t idx) { + PyObject *py_obj = _h2py(obj); + if (PySequence_Check(py_obj)) { + return _py2h(PySequence_GetItem(py_obj, idx)); + } PyObject* key = PyLong_FromSsize_t(idx); if (key == NULL) return HPy_NULL; - HPy result = _py2h(PyObject_GetItem(_h2py(obj), key)); + HPy result = _py2h(PyObject_GetItem(py_obj, key)); Py_DECREF(key); return result; } diff --git a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_tuple.c b/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_tuple.c index eb16a87b57..42661f8fea 100644 --- a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_tuple.c +++ b/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_tuple.c @@ -8,7 +8,7 @@ _HPy_HIDDEN HPy -ctx_Tuple_FromArray(HPyContext *ctx, HPy items[], HPy_ssize_t n) +ctx_Tuple_FromArray(HPyContext *ctx, const HPy items[], HPy_ssize_t n) { PyObject *res = PyTuple_New(n); if (!res) diff --git a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_type.c b/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_type.c index 2310aa4fbe..ef89e2590d 100644 --- a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_type.c +++ b/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_type.c @@ -76,6 +76,7 @@ FULLY_ALIGNED_SPACE(PyFloatObject) FULLY_ALIGNED_SPACE(PyUnicodeObject) FULLY_ALIGNED_SPACE(PyTupleObject) FULLY_ALIGNED_SPACE(PyListObject) +FULLY_ALIGNED_SPACE(PyDictObject) #define _HPy_HEAD_SIZE(HEAD) (offsetof(_HPy_FullyAlignedSpaceFor##HEAD, payload)) @@ -101,6 +102,8 @@ _HPy_GetHeaderSize(HPyType_BuiltinShape shape) return _HPy_HEAD_SIZE(PyTupleObject); case HPyType_BuiltinShape_List: return _HPy_HEAD_SIZE(PyListObject); + case HPyType_BuiltinShape_Dict: + return _HPy_HEAD_SIZE(PyDictObject); } return -1; } @@ -1504,6 +1507,12 @@ ctx_AsStruct_List(HPyContext *ctx, HPy h) return _HPy_Payload(_h2py(h), HPyType_BuiltinShape_List); } +_HPy_HIDDEN void* +ctx_AsStruct_Dict(HPyContext *ctx, HPy h) +{ + return _HPy_Payload(_h2py(h), HPyType_BuiltinShape_Dict); +} + _HPy_HIDDEN void* ctx_AsStruct_Slow(HPyContext *ctx, HPy h) { diff --git a/graalpython/lib-graalpython/modules/hpy/devel/version.py b/graalpython/lib-graalpython/modules/hpy/devel/version.py index 356bae1ebb..0cd1dbcc22 100644 --- a/graalpython/lib-graalpython/modules/hpy/devel/version.py +++ b/graalpython/lib-graalpython/modules/hpy/devel/version.py @@ -1,4 +1,4 @@ # automatically generated by setup.py:get_scm_config() -__version__ = "0.9.0" -__git_revision__ = "f6114734" +__version__ = "0.9.1.dev79+gb0fbdf73" +__git_revision__ = "b0fbdf73" diff --git a/graalpython/lib-graalpython/modules/hpy/trace/leakdetector.py b/graalpython/lib-graalpython/modules/hpy/trace/leakdetector.py new file mode 100644 index 0000000000..56a68d344c --- /dev/null +++ b/graalpython/lib-graalpython/modules/hpy/trace/leakdetector.py @@ -0,0 +1,43 @@ +from hpy.universal import _debug + +class HPyDebugError(Exception): + pass + +class HPyLeakError(HPyDebugError): + def __init__(self, leaks): + super().__init__() + self.leaks = leaks + + def __str__(self): + lines = [] + n = len(self.leaks) + s = 's' if n != 1 else '' + lines.append(f'{n} unclosed handle{s}:') + for dh in self.leaks: + lines.append(' %r' % dh) + return '\n'.join(lines) + + +class LeakDetector: + + def __init__(self): + self.generation = None + + def start(self): + if self.generation is not None: + raise ValueError('LeakDetector already started') + self.generation = _debug.new_generation() + + def stop(self): + if self.generation is None: + raise ValueError('LeakDetector not started yet') + leaks = _debug.get_open_handles(self.generation) + if leaks: + raise HPyLeakError(leaks) + + def __enter__(self): + self.start() + return self + + def __exit__(self, etype, evalue, tb): + self.stop() diff --git a/graalpython/lib-graalpython/modules/hpy/trace/pytest.py b/graalpython/lib-graalpython/modules/hpy/trace/pytest.py new file mode 100644 index 0000000000..9a33c51343 --- /dev/null +++ b/graalpython/lib-graalpython/modules/hpy/trace/pytest.py @@ -0,0 +1,31 @@ +# hpy.debug / pytest integration + +import pytest +from .leakdetector import LeakDetector + +# For now "hpy_debug" just does leak detection, but in the future it might +# grows extra features: that's why it's called generically "hpy_debug" instead +# of "detect_leaks". + +# NOTE: the fixture itself is currently untested :(. It turns out that testing +# that the fixture raises during the teardown is complicated and probably +# requires to write a full-fledged plugin. We might want to turn this into a +# real plugin in the future, but for now I think this is enough. + +# pypy still uses a very ancient version of pytest, 2.9.2: pytest<3.x needs to +# use @yield_fixture, which is deprecated in newer version of pytest (where +# you can just use @fixture) +if pytest.__version__ < '3': + fixture = pytest.yield_fixture +else: + fixture = pytest.fixture + +@fixture +def hpy_debug(request): + """ + pytest fixture which makes it possible to control hpy.debug from within a test. + + In particular, it automatically check that the test doesn't leak. + """ + with LeakDetector() as ld: + yield ld From e6b441c18c563e2bb328fb9b71d5067c2a0f4b34 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Tue, 8 Apr 2025 15:34:01 +0200 Subject: [PATCH 2/3] include hpy as submodule, commit b0fbdf73bb74bc96310835781bb3fe3386138253 --- .../hpy/debug_internal.h | 233 --- .../hpy/hpy_debug.h | 59 - .../modules/_hpy_debug.c | 403 ---- .../include/hpy/version.h | 4 - .../src/debug/include/hpy_debug.h | 57 - .../src/trace/include/hpy_trace.h | 42 - graalpython/hpy/.gitattributes | 1 + graalpython/hpy/.github/FUNDING.yml | 1 + graalpython/hpy/.github/workflows/ci.yml | 427 ++++ .../hpy/.github/workflows/release-pypi.yml | 136 ++ .../hpy/.github/workflows/valgrind-tests.yml | 38 + graalpython/hpy/.gitignore | 92 + graalpython/hpy/.readthedocs.yml | 10 + graalpython/hpy/AUTHORS | 38 + graalpython/hpy/CONTRIBUTING.md | 55 + graalpython/hpy/LICENSE | 21 + graalpython/hpy/MANIFEST.in | 1 + graalpython/hpy/Makefile | 86 + graalpython/hpy/README-gdb.md | 207 ++ graalpython/hpy/README.md | 81 + graalpython/hpy/c_test/Makefile | 16 + graalpython/hpy/c_test/acutest.h | 1794 +++++++++++++++++ graalpython/hpy/c_test/test_debug_handles.c | 99 + graalpython/hpy/c_test/test_stacktrace.c | 22 + graalpython/hpy/docs/Makefile | 28 + graalpython/hpy/docs/_static/README.txt | 1 + graalpython/hpy/docs/_templates/README.txt | 1 + .../docs/api-reference/argument-parsing.rst | 5 + .../hpy/docs/api-reference/build-value.rst | 5 + .../hpy/docs/api-reference/formatting.rst | 5 + .../hpy/docs/api-reference/function-index.rst | 187 ++ .../hpy/docs/api-reference/helpers.rst | 5 + .../hpy/docs/api-reference/hpy-call.rst | 5 + .../hpy/docs/api-reference/hpy-ctx.rst | 9 + .../hpy/docs/api-reference/hpy-dict.rst | 5 + .../hpy/docs/api-reference/hpy-err.rst | 5 + .../hpy/docs/api-reference/hpy-eval.rst | 8 + .../hpy/docs/api-reference/hpy-field.rst | 5 + .../hpy/docs/api-reference/hpy-gil.rst | 5 + .../hpy/docs/api-reference/hpy-global.rst | 5 + .../hpy/docs/api-reference/hpy-object.rst | 5 + .../hpy/docs/api-reference/hpy-sequence.rst | 23 + .../hpy/docs/api-reference/hpy-type.rst | 50 + graalpython/hpy/docs/api-reference/index.rst | 69 + .../hpy/docs/api-reference/inline-helpers.rst | 14 + .../hpy/docs/api-reference/public-api.rst | 9 + .../hpy/docs/api-reference/structseq.rst | 15 + graalpython/hpy/docs/api.rst | 581 ++++++ graalpython/hpy/docs/changelog.rst | 235 +++ graalpython/hpy/docs/conf.py | 134 ++ graalpython/hpy/docs/contributing/index.rst | 33 + graalpython/hpy/docs/debug-mode.rst | 169 ++ .../hpy/docs/examples/debug-example.py | 10 + .../examples/hpytype-example/builtin_type.c | 106 + .../docs/examples/hpytype-example/setup.py | 11 + .../examples/hpytype-example/simple_type.c | 102 + .../examples/hpytype-example/simple_type.rst | 8 + .../hpy/docs/examples/mixed-example/mixed.c | 49 + .../hpy/docs/examples/mixed-example/setup.py | 10 + .../hpy/docs/examples/quickstart/quickstart.c | 39 + .../hpy/docs/examples/quickstart/setup.py | 13 + .../hpy/docs/examples/simple-example/setup.py | 10 + .../hpy/docs/examples/simple-example/simple.c | 43 + .../hpy/docs/examples/snippets/hpycall.c | 196 ++ .../hpy/docs/examples/snippets/hpyinit.c | 22 + .../hpy/docs/examples/snippets/hpyvarargs.c | 45 + .../hpy/docs/examples/snippets/legacyinit.c | 20 + .../hpy/docs/examples/snippets/setup.py | 16 + .../hpy/docs/examples/snippets/snippets.c | 81 + graalpython/hpy/docs/examples/tests.py | 124 ++ .../hpy/docs/examples/trace-example.py | 9 + graalpython/hpy/docs/index.rst | 80 + .../hpy/docs/leysin-2020-design-decisions.md | 295 +++ graalpython/hpy/docs/misc/embedding.rst | 46 + graalpython/hpy/docs/misc/index.rst | 7 + graalpython/hpy/docs/module-state.txt | 44 + graalpython/hpy/docs/overview.rst | 444 ++++ .../hpy/docs/porting-example/index.rst | 356 ++++ .../hpy/docs/porting-example/steps/.gitignore | 1 + .../docs/porting-example/steps/conftest.py | 35 + .../hpy/docs/porting-example/steps/setup00.py | 12 + .../hpy/docs/porting-example/steps/setup01.py | 13 + .../hpy/docs/porting-example/steps/setup02.py | 13 + .../hpy/docs/porting-example/steps/setup03.py | 14 + .../porting-example/steps/step_00_c_api.c | 165 ++ .../porting-example/steps/step_00_c_api.rst | 8 + .../steps/step_01_hpy_legacy.c | 189 ++ .../steps/step_01_hpy_legacy.rst | 8 + .../steps/step_02_hpy_legacy.c | 169 ++ .../steps/step_02_hpy_legacy.rst | 8 + .../porting-example/steps/step_03_hpy_final.c | 153 ++ .../steps/step_03_hpy_final.rst | 8 + .../steps/test_porting_example.py | 60 + graalpython/hpy/docs/porting-guide.rst | 569 ++++++ graalpython/hpy/docs/quickstart.rst | 55 + graalpython/hpy/docs/requirements.txt | 8 + graalpython/hpy/docs/trace-mode.rst | 64 + graalpython/hpy/docs/xxx-unsorted-notes.txt | 91 + graalpython/hpy/gdb-py.test | 3 + .../modules => hpy}/hpy/debug/__init__.py | 0 .../modules => hpy}/hpy/debug/leakdetector.py | 0 .../modules => hpy}/hpy/debug/pytest.py | 0 .../debug => hpy/hpy/debug/src}/_debugmod.c | 0 .../hpy/debug/src/autogen_debug_ctx_call.i | 468 +++++ .../hpy/debug/src}/autogen_debug_ctx_init.h | 0 .../hpy/debug/src}/autogen_debug_wrappers.c | 0 .../debug => hpy/hpy/debug/src}/debug_ctx.c | 0 .../hpy/hpy/debug/src/debug_ctx_cpython.c | 299 +++ .../hpy/debug/src}/debug_ctx_not_cpython.c | 0 .../hpy/debug/src}/debug_handles.c | 0 .../hpy/debug/src}/debug_internal.h | 0 .../src/debug => hpy/hpy/debug/src}/dhqueue.c | 0 .../hpy/debug/src/include}/hpy_debug.h | 0 .../debug => hpy/hpy/debug/src}/memprotect.c | 0 .../debug => hpy/hpy/debug/src}/stacktrace.c | 0 .../modules => hpy}/hpy/devel/__init__.py | 0 .../modules => hpy}/hpy/devel/abitag.py | 0 .../hpy/devel}/include/hpy.h | 0 .../include/hpy/autogen_hpyfunc_declare.h | 0 .../hpy/devel}/include/hpy/autogen_hpyslot.h | 0 .../hpy/devel}/include/hpy/cpy_types.h | 0 .../include/hpy/cpython/autogen_api_impl.h | 0 .../devel}/include/hpy/cpython/autogen_ctx.h | 0 .../hpy/cpython/autogen_hpyfunc_trampolines.h | 0 .../include/hpy/cpython/hpyfunc_trampolines.h | 0 .../hpy/devel}/include/hpy/cpython/misc.h | 0 .../include/hpy/forbid_python_h/Python.h | 0 .../hpy/devel}/include/hpy/hpydef.h | 0 .../hpy/devel}/include/hpy/hpyexports.h | 0 .../hpy/devel}/include/hpy/hpyfunc.h | 0 .../hpy/devel}/include/hpy/hpymodule.h | 0 .../hpy/devel}/include/hpy/hpytype.h | 0 .../hpy/devel}/include/hpy/inline_helpers.h | 0 .../hpy/devel}/include/hpy/macros.h | 0 .../hpy/devel}/include/hpy/runtime/argparse.h | 0 .../devel}/include/hpy/runtime/buildvalue.h | 0 .../devel}/include/hpy/runtime/ctx_funcs.h | 0 .../devel}/include/hpy/runtime/ctx_module.h | 0 .../hpy/devel}/include/hpy/runtime/ctx_type.h | 0 .../hpy/devel}/include/hpy/runtime/format.h | 0 .../hpy/devel}/include/hpy/runtime/helpers.h | 0 .../devel}/include/hpy/runtime/structseq.h | 0 .../include/hpy/universal/autogen_ctx.h | 0 .../universal/autogen_hpyfunc_trampolines.h | 0 .../hpy/universal/autogen_trampolines.h | 0 .../hpy/universal/hpyfunc_trampolines.h | 0 .../include/hpy/universal/misc_trampolines.h | 0 .../hpy/devel/src/runtime/argparse.c | 0 .../hpy/devel/src/runtime/buildvalue.c | 0 .../hpy/devel/src/runtime/ctx_bytes.c | 0 .../hpy/devel/src/runtime/ctx_call.c | 0 .../hpy/devel/src/runtime/ctx_capsule.c | 0 .../hpy/devel/src/runtime/ctx_contextvar.c | 0 .../hpy/devel/src/runtime/ctx_err.c | 0 .../hpy/devel/src/runtime/ctx_eval.c | 0 .../hpy/devel/src/runtime/ctx_listbuilder.c | 0 .../hpy/devel/src/runtime/ctx_long.c | 0 .../hpy/devel/src/runtime/ctx_module.c | 0 .../hpy/devel/src/runtime/ctx_object.c | 0 .../hpy/devel/src/runtime}/ctx_tracker.c | 0 .../hpy/devel/src/runtime/ctx_tuple.c | 0 .../hpy/devel/src/runtime/ctx_tuplebuilder.c | 0 .../hpy/devel/src/runtime/ctx_type.c | 0 .../hpy/devel/src/runtime/format.c | 0 .../hpy/devel/src/runtime/helpers.c | 0 .../hpy/devel/src/runtime/structseq.c | 0 graalpython/hpy/hpy/tools/autogen/__init__.py | 21 + graalpython/hpy/hpy/tools/autogen/__main__.py | 66 + graalpython/hpy/hpy/tools/autogen/autogen.h | 40 + .../hpy/hpy/tools/autogen/autogenfile.py | 35 + graalpython/hpy/hpy/tools/autogen/conf.py | 211 ++ graalpython/hpy/hpy/tools/autogen/ctx.py | 106 + graalpython/hpy/hpy/tools/autogen/debug.py | 228 +++ graalpython/hpy/hpy/tools/autogen/doc.py | 173 ++ graalpython/hpy/hpy/tools/autogen/hpyfunc.py | 204 ++ graalpython/hpy/hpy/tools/autogen/hpyslot.py | 18 + graalpython/hpy/hpy/tools/autogen/parse.py | 278 +++ .../hpy/hpy/tools/autogen/public_api.h | 1557 ++++++++++++++ graalpython/hpy/hpy/tools/autogen/pypy.py | 40 + .../hpy/tools/autogen/testing}/__init__.py | 0 .../hpy/tools/autogen/testing/test_autogen.py | 224 ++ .../hpy/tools/autogen/testing/test_hpyfunc.py | 145 ++ graalpython/hpy/hpy/tools/autogen/trace.py | 182 ++ .../hpy/hpy/tools/autogen/trampolines.py | 140 ++ graalpython/hpy/hpy/tools/include_path.py | 4 + graalpython/hpy/hpy/tools/valgrind/hpy.supp | 44 + .../hpy/hpy/tools/valgrind/python.supp | 389 ++++ graalpython/hpy/hpy/trace/__init__.py | 6 + .../trace => hpy/hpy/trace/src}/_tracemod.c | 0 .../hpy/trace/src}/autogen_trace_ctx_init.h | 0 .../hpy/trace/src}/autogen_trace_func_table.c | 0 .../hpy/trace/src}/autogen_trace_wrappers.c | 0 .../hpy/trace/src/include}/hpy_trace.h | 0 .../trace => hpy/hpy/trace/src}/trace_ctx.c | 0 .../hpy/trace/src}/trace_internal.h | 0 graalpython/hpy/hpy/universal/src/api.h | 18 + .../hpy/hpy/universal/src/autogen_ctx_call.i | 180 ++ .../hpy/hpy/universal/src/autogen_ctx_def.h | 207 ++ .../hpy/hpy/universal/src/autogen_ctx_impl.h | 612 ++++++ graalpython/hpy/hpy/universal/src/ctx.c | 10 + graalpython/hpy/hpy/universal/src/ctx_meth.c | 136 ++ graalpython/hpy/hpy/universal/src/ctx_meth.h | 6 + graalpython/hpy/hpy/universal/src/ctx_misc.c | 83 + graalpython/hpy/hpy/universal/src/ctx_misc.h | 20 + graalpython/hpy/hpy/universal/src/handles.h | 58 + graalpython/hpy/hpy/universal/src/hpymodule.c | 691 +++++++ .../hpy/hpy/universal/src/misc_win32.h | 55 + graalpython/hpy/microbench/README.md | 29 + graalpython/hpy/microbench/_valgrind_build.py | 29 + graalpython/hpy/microbench/conftest.py | 124 ++ graalpython/hpy/microbench/pytest_valgrind.sh | 3 + graalpython/hpy/microbench/setup.py | 17 + graalpython/hpy/microbench/src/cpy_simple.c | 174 ++ graalpython/hpy/microbench/src/hpy_simple.c | 135 ++ graalpython/hpy/microbench/test_microbench.py | 226 +++ graalpython/hpy/proof-of-concept/pof.c | 109 + graalpython/hpy/proof-of-concept/pofcpp.cpp | 117 ++ .../hpy/proof-of-concept/pofpackage/bar.cpp | 45 + .../hpy/proof-of-concept/pofpackage/foo.c | 20 + .../hpy/proof-of-concept/requirements.txt | 1 + graalpython/hpy/proof-of-concept/setup.py | 36 + graalpython/hpy/proof-of-concept/test_pof.py | 44 + graalpython/hpy/proof-of-concept/test_pof.sh | 177 ++ graalpython/hpy/pyproject.toml | 3 + graalpython/hpy/requirements-autogen.txt | 4 + graalpython/hpy/setup.py | 269 +++ .../hpytest/debug => hpy/test}/__init__.py | 0 .../hpytest => hpy/test}/check_py27_compat.py | 0 .../src/hpytest => hpy/test}/conftest.py | 0 .../hpy_devel => hpy/test/debug}/__init__.py | 0 .../test}/debug/test_builder_invalid.py | 0 .../test}/debug/test_charptr.py | 0 .../test}/debug/test_context_reuse.py | 0 .../test}/debug/test_handles_invalid.py | 0 .../test}/debug/test_handles_leak.py | 0 .../hpytest => hpy/test}/debug/test_misc.py | 0 graalpython/hpy/test/hpy_devel/__init__.py | 0 .../test}/hpy_devel/test_abitag.py | 0 .../test}/hpy_devel/test_distutils.py | 0 .../src/hpytest => hpy/test}/support.py | 0 .../src/hpytest => hpy/test}/test_00_basic.py | 0 .../src/hpytest => hpy/test}/test_argparse.py | 0 .../src/hpytest => hpy/test}/test_call.py | 0 .../src/hpytest => hpy/test}/test_capsule.py | 0 .../test}/test_capsule_legacy.py | 0 .../hpytest => hpy/test}/test_contextvar.py | 0 .../hpytest => hpy/test}/test_cpy_compat.py | 0 .../src/hpytest => hpy/test}/test_eval.py | 0 .../src/hpytest => hpy/test}/test_helpers.py | 0 .../test}/test_hpybuildvalue.py | 0 .../src/hpytest => hpy/test}/test_hpybytes.py | 0 .../src/hpytest => hpy/test}/test_hpydict.py | 0 .../src/hpytest => hpy/test}/test_hpyerr.py | 0 .../src/hpytest => hpy/test}/test_hpyfield.py | 0 .../hpytest => hpy/test}/test_hpyglobal.py | 0 .../hpytest => hpy/test}/test_hpyimport.py | 0 .../src/hpytest => hpy/test}/test_hpyiter.py | 0 .../src/hpytest => hpy/test}/test_hpylist.py | 0 .../src/hpytest => hpy/test}/test_hpylong.py | 0 .../hpytest => hpy/test}/test_hpymodule.py | 0 .../src/hpytest => hpy/test}/test_hpyslice.py | 0 .../hpytest => hpy/test}/test_hpystructseq.py | 0 .../src/hpytest => hpy/test}/test_hpytuple.py | 0 .../src/hpytest => hpy/test}/test_hpytype.py | 0 .../test}/test_hpytype_legacy.py | 0 .../hpytest => hpy/test}/test_hpyunicode.py | 0 .../hpytest => hpy/test}/test_importing.py | 0 .../test}/test_legacy_forbidden.py | 0 .../src/hpytest => hpy/test}/test_number.py | 0 .../src/hpytest => hpy/test}/test_object.py | 0 .../src/hpytest => hpy/test}/test_slots.py | 0 .../hpytest => hpy/test}/test_slots_legacy.py | 0 .../src/hpytest => hpy/test}/test_support.py | 0 .../src/hpytest => hpy/test}/test_tracker.py | 0 .../hpytest => hpy/test}/trace/test_trace.py | 0 .../hpy/devel/src/runtime/ctx_tracker.c | 155 -- .../modules/hpy/devel/version.py | 4 - .../modules/hpy/trace/__init__.py | 11 - .../modules/hpy/trace/leakdetector.py | 43 - .../modules/hpy/trace/pytest.py | 31 - 280 files changed, 17094 insertions(+), 1042 deletions(-) delete mode 100644 graalpython/com.oracle.graal.python.cext/hpy/debug_internal.h delete mode 100644 graalpython/com.oracle.graal.python.cext/hpy/hpy_debug.h delete mode 100644 graalpython/com.oracle.graal.python.cext/modules/_hpy_debug.c delete mode 100644 graalpython/com.oracle.graal.python.hpy/include/hpy/version.h delete mode 100644 graalpython/com.oracle.graal.python.jni/src/debug/include/hpy_debug.h delete mode 100644 graalpython/com.oracle.graal.python.jni/src/trace/include/hpy_trace.h create mode 100644 graalpython/hpy/.gitattributes create mode 100644 graalpython/hpy/.github/FUNDING.yml create mode 100644 graalpython/hpy/.github/workflows/ci.yml create mode 100644 graalpython/hpy/.github/workflows/release-pypi.yml create mode 100644 graalpython/hpy/.github/workflows/valgrind-tests.yml create mode 100644 graalpython/hpy/.gitignore create mode 100644 graalpython/hpy/.readthedocs.yml create mode 100644 graalpython/hpy/AUTHORS create mode 100644 graalpython/hpy/CONTRIBUTING.md create mode 100644 graalpython/hpy/LICENSE create mode 100644 graalpython/hpy/MANIFEST.in create mode 100644 graalpython/hpy/Makefile create mode 100644 graalpython/hpy/README-gdb.md create mode 100644 graalpython/hpy/README.md create mode 100644 graalpython/hpy/c_test/Makefile create mode 100644 graalpython/hpy/c_test/acutest.h create mode 100644 graalpython/hpy/c_test/test_debug_handles.c create mode 100644 graalpython/hpy/c_test/test_stacktrace.c create mode 100644 graalpython/hpy/docs/Makefile create mode 100644 graalpython/hpy/docs/_static/README.txt create mode 100644 graalpython/hpy/docs/_templates/README.txt create mode 100644 graalpython/hpy/docs/api-reference/argument-parsing.rst create mode 100644 graalpython/hpy/docs/api-reference/build-value.rst create mode 100644 graalpython/hpy/docs/api-reference/formatting.rst create mode 100644 graalpython/hpy/docs/api-reference/function-index.rst create mode 100644 graalpython/hpy/docs/api-reference/helpers.rst create mode 100644 graalpython/hpy/docs/api-reference/hpy-call.rst create mode 100644 graalpython/hpy/docs/api-reference/hpy-ctx.rst create mode 100644 graalpython/hpy/docs/api-reference/hpy-dict.rst create mode 100644 graalpython/hpy/docs/api-reference/hpy-err.rst create mode 100644 graalpython/hpy/docs/api-reference/hpy-eval.rst create mode 100644 graalpython/hpy/docs/api-reference/hpy-field.rst create mode 100644 graalpython/hpy/docs/api-reference/hpy-gil.rst create mode 100644 graalpython/hpy/docs/api-reference/hpy-global.rst create mode 100644 graalpython/hpy/docs/api-reference/hpy-object.rst create mode 100644 graalpython/hpy/docs/api-reference/hpy-sequence.rst create mode 100644 graalpython/hpy/docs/api-reference/hpy-type.rst create mode 100644 graalpython/hpy/docs/api-reference/index.rst create mode 100644 graalpython/hpy/docs/api-reference/inline-helpers.rst create mode 100644 graalpython/hpy/docs/api-reference/public-api.rst create mode 100644 graalpython/hpy/docs/api-reference/structseq.rst create mode 100644 graalpython/hpy/docs/api.rst create mode 100644 graalpython/hpy/docs/changelog.rst create mode 100644 graalpython/hpy/docs/conf.py create mode 100644 graalpython/hpy/docs/contributing/index.rst create mode 100644 graalpython/hpy/docs/debug-mode.rst create mode 100644 graalpython/hpy/docs/examples/debug-example.py create mode 100644 graalpython/hpy/docs/examples/hpytype-example/builtin_type.c create mode 100644 graalpython/hpy/docs/examples/hpytype-example/setup.py create mode 100644 graalpython/hpy/docs/examples/hpytype-example/simple_type.c create mode 100644 graalpython/hpy/docs/examples/hpytype-example/simple_type.rst create mode 100644 graalpython/hpy/docs/examples/mixed-example/mixed.c create mode 100644 graalpython/hpy/docs/examples/mixed-example/setup.py create mode 100644 graalpython/hpy/docs/examples/quickstart/quickstart.c create mode 100644 graalpython/hpy/docs/examples/quickstart/setup.py create mode 100644 graalpython/hpy/docs/examples/simple-example/setup.py create mode 100644 graalpython/hpy/docs/examples/simple-example/simple.c create mode 100644 graalpython/hpy/docs/examples/snippets/hpycall.c create mode 100644 graalpython/hpy/docs/examples/snippets/hpyinit.c create mode 100644 graalpython/hpy/docs/examples/snippets/hpyvarargs.c create mode 100644 graalpython/hpy/docs/examples/snippets/legacyinit.c create mode 100644 graalpython/hpy/docs/examples/snippets/setup.py create mode 100644 graalpython/hpy/docs/examples/snippets/snippets.c create mode 100644 graalpython/hpy/docs/examples/tests.py create mode 100644 graalpython/hpy/docs/examples/trace-example.py create mode 100644 graalpython/hpy/docs/index.rst create mode 100644 graalpython/hpy/docs/leysin-2020-design-decisions.md create mode 100644 graalpython/hpy/docs/misc/embedding.rst create mode 100644 graalpython/hpy/docs/misc/index.rst create mode 100644 graalpython/hpy/docs/module-state.txt create mode 100644 graalpython/hpy/docs/overview.rst create mode 100644 graalpython/hpy/docs/porting-example/index.rst create mode 100644 graalpython/hpy/docs/porting-example/steps/.gitignore create mode 100644 graalpython/hpy/docs/porting-example/steps/conftest.py create mode 100644 graalpython/hpy/docs/porting-example/steps/setup00.py create mode 100644 graalpython/hpy/docs/porting-example/steps/setup01.py create mode 100644 graalpython/hpy/docs/porting-example/steps/setup02.py create mode 100644 graalpython/hpy/docs/porting-example/steps/setup03.py create mode 100644 graalpython/hpy/docs/porting-example/steps/step_00_c_api.c create mode 100644 graalpython/hpy/docs/porting-example/steps/step_00_c_api.rst create mode 100644 graalpython/hpy/docs/porting-example/steps/step_01_hpy_legacy.c create mode 100644 graalpython/hpy/docs/porting-example/steps/step_01_hpy_legacy.rst create mode 100644 graalpython/hpy/docs/porting-example/steps/step_02_hpy_legacy.c create mode 100644 graalpython/hpy/docs/porting-example/steps/step_02_hpy_legacy.rst create mode 100644 graalpython/hpy/docs/porting-example/steps/step_03_hpy_final.c create mode 100644 graalpython/hpy/docs/porting-example/steps/step_03_hpy_final.rst create mode 100644 graalpython/hpy/docs/porting-example/steps/test_porting_example.py create mode 100644 graalpython/hpy/docs/porting-guide.rst create mode 100644 graalpython/hpy/docs/quickstart.rst create mode 100644 graalpython/hpy/docs/requirements.txt create mode 100644 graalpython/hpy/docs/trace-mode.rst create mode 100644 graalpython/hpy/docs/xxx-unsorted-notes.txt create mode 100755 graalpython/hpy/gdb-py.test rename graalpython/{lib-graalpython/modules => hpy}/hpy/debug/__init__.py (100%) rename graalpython/{lib-graalpython/modules => hpy}/hpy/debug/leakdetector.py (100%) rename graalpython/{lib-graalpython/modules => hpy}/hpy/debug/pytest.py (100%) rename graalpython/{com.oracle.graal.python.jni/src/debug => hpy/hpy/debug/src}/_debugmod.c (100%) create mode 100644 graalpython/hpy/hpy/debug/src/autogen_debug_ctx_call.i rename graalpython/{com.oracle.graal.python.jni/src/debug => hpy/hpy/debug/src}/autogen_debug_ctx_init.h (100%) rename graalpython/{com.oracle.graal.python.jni/src/debug => hpy/hpy/debug/src}/autogen_debug_wrappers.c (100%) rename graalpython/{com.oracle.graal.python.jni/src/debug => hpy/hpy/debug/src}/debug_ctx.c (100%) create mode 100644 graalpython/hpy/hpy/debug/src/debug_ctx_cpython.c rename graalpython/{com.oracle.graal.python.jni/src/debug => hpy/hpy/debug/src}/debug_ctx_not_cpython.c (100%) rename graalpython/{com.oracle.graal.python.jni/src/debug => hpy/hpy/debug/src}/debug_handles.c (100%) rename graalpython/{com.oracle.graal.python.jni/src/debug => hpy/hpy/debug/src}/debug_internal.h (100%) rename graalpython/{com.oracle.graal.python.jni/src/debug => hpy/hpy/debug/src}/dhqueue.c (100%) rename graalpython/{com.oracle.graal.python.jni/src/debug => hpy/hpy/debug/src/include}/hpy_debug.h (100%) rename graalpython/{com.oracle.graal.python.jni/src/debug => hpy/hpy/debug/src}/memprotect.c (100%) rename graalpython/{com.oracle.graal.python.jni/src/debug => hpy/hpy/debug/src}/stacktrace.c (100%) rename graalpython/{lib-graalpython/modules => hpy}/hpy/devel/__init__.py (100%) rename graalpython/{lib-graalpython/modules => hpy}/hpy/devel/abitag.py (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/autogen_hpyfunc_declare.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/autogen_hpyslot.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/cpy_types.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/cpython/autogen_api_impl.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/cpython/autogen_ctx.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/cpython/autogen_hpyfunc_trampolines.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/cpython/hpyfunc_trampolines.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/cpython/misc.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/forbid_python_h/Python.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/hpydef.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/hpyexports.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/hpyfunc.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/hpymodule.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/hpytype.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/inline_helpers.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/macros.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/runtime/argparse.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/runtime/buildvalue.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/runtime/ctx_funcs.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/runtime/ctx_module.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/runtime/ctx_type.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/runtime/format.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/runtime/helpers.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/runtime/structseq.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/universal/autogen_ctx.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/universal/autogen_hpyfunc_trampolines.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/universal/autogen_trampolines.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/universal/hpyfunc_trampolines.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/universal/misc_trampolines.h (100%) rename graalpython/{lib-graalpython/modules => hpy}/hpy/devel/src/runtime/argparse.c (100%) rename graalpython/{lib-graalpython/modules => hpy}/hpy/devel/src/runtime/buildvalue.c (100%) rename graalpython/{lib-graalpython/modules => hpy}/hpy/devel/src/runtime/ctx_bytes.c (100%) rename graalpython/{lib-graalpython/modules => hpy}/hpy/devel/src/runtime/ctx_call.c (100%) rename graalpython/{lib-graalpython/modules => hpy}/hpy/devel/src/runtime/ctx_capsule.c (100%) rename graalpython/{lib-graalpython/modules => hpy}/hpy/devel/src/runtime/ctx_contextvar.c (100%) rename graalpython/{lib-graalpython/modules => hpy}/hpy/devel/src/runtime/ctx_err.c (100%) rename graalpython/{lib-graalpython/modules => hpy}/hpy/devel/src/runtime/ctx_eval.c (100%) rename graalpython/{lib-graalpython/modules => hpy}/hpy/devel/src/runtime/ctx_listbuilder.c (100%) rename graalpython/{lib-graalpython/modules => hpy}/hpy/devel/src/runtime/ctx_long.c (100%) rename graalpython/{lib-graalpython/modules => hpy}/hpy/devel/src/runtime/ctx_module.c (100%) rename graalpython/{lib-graalpython/modules => hpy}/hpy/devel/src/runtime/ctx_object.c (100%) rename graalpython/{com.oracle.graal.python.jni/src => hpy/hpy/devel/src/runtime}/ctx_tracker.c (100%) rename graalpython/{lib-graalpython/modules => hpy}/hpy/devel/src/runtime/ctx_tuple.c (100%) rename graalpython/{lib-graalpython/modules => hpy}/hpy/devel/src/runtime/ctx_tuplebuilder.c (100%) rename graalpython/{lib-graalpython/modules => hpy}/hpy/devel/src/runtime/ctx_type.c (100%) rename graalpython/{lib-graalpython/modules => hpy}/hpy/devel/src/runtime/format.c (100%) rename graalpython/{lib-graalpython/modules => hpy}/hpy/devel/src/runtime/helpers.c (100%) rename graalpython/{lib-graalpython/modules => hpy}/hpy/devel/src/runtime/structseq.c (100%) create mode 100644 graalpython/hpy/hpy/tools/autogen/__init__.py create mode 100644 graalpython/hpy/hpy/tools/autogen/__main__.py create mode 100644 graalpython/hpy/hpy/tools/autogen/autogen.h create mode 100644 graalpython/hpy/hpy/tools/autogen/autogenfile.py create mode 100644 graalpython/hpy/hpy/tools/autogen/conf.py create mode 100644 graalpython/hpy/hpy/tools/autogen/ctx.py create mode 100644 graalpython/hpy/hpy/tools/autogen/debug.py create mode 100644 graalpython/hpy/hpy/tools/autogen/doc.py create mode 100644 graalpython/hpy/hpy/tools/autogen/hpyfunc.py create mode 100644 graalpython/hpy/hpy/tools/autogen/hpyslot.py create mode 100644 graalpython/hpy/hpy/tools/autogen/parse.py create mode 100644 graalpython/hpy/hpy/tools/autogen/public_api.h create mode 100644 graalpython/hpy/hpy/tools/autogen/pypy.py rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/hpy/tools/autogen/testing}/__init__.py (100%) create mode 100644 graalpython/hpy/hpy/tools/autogen/testing/test_autogen.py create mode 100644 graalpython/hpy/hpy/tools/autogen/testing/test_hpyfunc.py create mode 100644 graalpython/hpy/hpy/tools/autogen/trace.py create mode 100644 graalpython/hpy/hpy/tools/autogen/trampolines.py create mode 100644 graalpython/hpy/hpy/tools/include_path.py create mode 100644 graalpython/hpy/hpy/tools/valgrind/hpy.supp create mode 100644 graalpython/hpy/hpy/tools/valgrind/python.supp create mode 100644 graalpython/hpy/hpy/trace/__init__.py rename graalpython/{com.oracle.graal.python.jni/src/trace => hpy/hpy/trace/src}/_tracemod.c (100%) rename graalpython/{com.oracle.graal.python.jni/src/trace => hpy/hpy/trace/src}/autogen_trace_ctx_init.h (100%) rename graalpython/{com.oracle.graal.python.jni/src/trace => hpy/hpy/trace/src}/autogen_trace_func_table.c (100%) rename graalpython/{com.oracle.graal.python.jni/src/trace => hpy/hpy/trace/src}/autogen_trace_wrappers.c (100%) rename graalpython/{com.oracle.graal.python.jni/src/trace => hpy/hpy/trace/src/include}/hpy_trace.h (100%) rename graalpython/{com.oracle.graal.python.jni/src/trace => hpy/hpy/trace/src}/trace_ctx.c (100%) rename graalpython/{com.oracle.graal.python.jni/src/trace => hpy/hpy/trace/src}/trace_internal.h (100%) create mode 100644 graalpython/hpy/hpy/universal/src/api.h create mode 100644 graalpython/hpy/hpy/universal/src/autogen_ctx_call.i create mode 100644 graalpython/hpy/hpy/universal/src/autogen_ctx_def.h create mode 100644 graalpython/hpy/hpy/universal/src/autogen_ctx_impl.h create mode 100644 graalpython/hpy/hpy/universal/src/ctx.c create mode 100644 graalpython/hpy/hpy/universal/src/ctx_meth.c create mode 100644 graalpython/hpy/hpy/universal/src/ctx_meth.h create mode 100644 graalpython/hpy/hpy/universal/src/ctx_misc.c create mode 100644 graalpython/hpy/hpy/universal/src/ctx_misc.h create mode 100644 graalpython/hpy/hpy/universal/src/handles.h create mode 100644 graalpython/hpy/hpy/universal/src/hpymodule.c create mode 100644 graalpython/hpy/hpy/universal/src/misc_win32.h create mode 100644 graalpython/hpy/microbench/README.md create mode 100644 graalpython/hpy/microbench/_valgrind_build.py create mode 100644 graalpython/hpy/microbench/conftest.py create mode 100755 graalpython/hpy/microbench/pytest_valgrind.sh create mode 100644 graalpython/hpy/microbench/setup.py create mode 100644 graalpython/hpy/microbench/src/cpy_simple.c create mode 100644 graalpython/hpy/microbench/src/hpy_simple.c create mode 100644 graalpython/hpy/microbench/test_microbench.py create mode 100644 graalpython/hpy/proof-of-concept/pof.c create mode 100644 graalpython/hpy/proof-of-concept/pofcpp.cpp create mode 100644 graalpython/hpy/proof-of-concept/pofpackage/bar.cpp create mode 100644 graalpython/hpy/proof-of-concept/pofpackage/foo.c create mode 100644 graalpython/hpy/proof-of-concept/requirements.txt create mode 100644 graalpython/hpy/proof-of-concept/setup.py create mode 100644 graalpython/hpy/proof-of-concept/test_pof.py create mode 100755 graalpython/hpy/proof-of-concept/test_pof.sh create mode 100644 graalpython/hpy/pyproject.toml create mode 100644 graalpython/hpy/requirements-autogen.txt create mode 100644 graalpython/hpy/setup.py rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest/debug => hpy/test}/__init__.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/check_py27_compat.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/conftest.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest/hpy_devel => hpy/test/debug}/__init__.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/debug/test_builder_invalid.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/debug/test_charptr.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/debug/test_context_reuse.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/debug/test_handles_invalid.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/debug/test_handles_leak.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/debug/test_misc.py (100%) create mode 100644 graalpython/hpy/test/hpy_devel/__init__.py rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/hpy_devel/test_abitag.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/hpy_devel/test_distutils.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/support.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_00_basic.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_argparse.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_call.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_capsule.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_capsule_legacy.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_contextvar.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_cpy_compat.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_eval.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_helpers.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_hpybuildvalue.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_hpybytes.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_hpydict.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_hpyerr.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_hpyfield.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_hpyglobal.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_hpyimport.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_hpyiter.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_hpylist.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_hpylong.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_hpymodule.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_hpyslice.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_hpystructseq.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_hpytuple.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_hpytype.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_hpytype_legacy.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_hpyunicode.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_importing.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_legacy_forbidden.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_number.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_object.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_slots.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_slots_legacy.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_support.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_tracker.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/trace/test_trace.py (100%) delete mode 100644 graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_tracker.c delete mode 100644 graalpython/lib-graalpython/modules/hpy/devel/version.py delete mode 100644 graalpython/lib-graalpython/modules/hpy/trace/__init__.py delete mode 100644 graalpython/lib-graalpython/modules/hpy/trace/leakdetector.py delete mode 100644 graalpython/lib-graalpython/modules/hpy/trace/pytest.py diff --git a/graalpython/com.oracle.graal.python.cext/hpy/debug_internal.h b/graalpython/com.oracle.graal.python.cext/hpy/debug_internal.h deleted file mode 100644 index e5bda5199b..0000000000 --- a/graalpython/com.oracle.graal.python.cext/hpy/debug_internal.h +++ /dev/null @@ -1,233 +0,0 @@ -/* Internal header for all the files in hpy/debug/src. The public API is in - include/hpy_debug.h -*/ -#ifndef HPY_DEBUG_INTERNAL_H -#define HPY_DEBUG_INTERNAL_H - -#include -#include "hpy.h" -#include "hpy_debug.h" - -#define HPY_DEBUG_MAGIC 0xDEB00FF - -/* The Debug context is a wrapper around an underlying context, which we will - call Universal. Inside the debug mode we manipulate handles which belongs - to both contexts, so to make things easier we create two typedefs to make - it clear what kind of handle we expect: UHPy and DHPy: - - * UHPy are opaque from our point of view. - - * DHPy are actually DebugHandle* in disguise. DebugHandles are wrappers - around a UHPy, with a bunch of extra info. - - To cast between DHPy and DebugHandle*, use as_DebugHandle and as_DHPy: - these are just no-op casts. - - Each DHPy wraps a corresponding UHPy: DHPys are created by calling - DHPy_open, and they must be eventually closed by DHPy_close. Note that if - you call DHPy_open twice on the same UHPy, you get two different DHPy. - - To unwrap a DHPy and get the underlying UHPy, call DHPy_unwrap. If you call - DHPy_unwrap multiple times on the same DHPy, you always get the same UHPy. - - WARNING: both UHPy and DHPy are alias of HPy, so we need to take care of - not mixing them, because the compiler cannot help. - - Each DebugHandle has a "generation", which is just an int to be able to get - only the handles which were created after a certain point. - - DHPys/DebugHandles are memory-managed by using a free list: - - - info->open_handles is a list of all DHPys which are currently open - - - DHPy_close() moves a DHPy from info->open_handles to info->closed_handles - - - if closed_handles is too big, the oldest DHPy is freed by DHPy_free() - - - to allocate memory for a new DHPy, DHPy_open() does the following: - - * if closed_handles is full, it reuses the memory of the oldest DHPy - in the queue - - * else, it malloc()s memory for a new DHPy - - - Each DebugHandle can have some "raw" data associated with it. It is a - generic pointer to any data. The validity, or life-time, of such pointer - is supposed to be the same as the that of the handle and the debug mode - enforces it. Additionally, the data can be also marked as write protected. - - Example is the `const char*` handed out by `HPyUnicode_AsUTF8AndSize`. It - must not be written by the user (users may discard the const modifier), and - the pointer is considered invalid once the handle is closed, so it must not - be accessed even for reading. Most Python implementations, will choose to - hand out pointer to the actual internal data, which happen to stay valid and - accessible and this may lead the users to a wrong conclusion that they can - use the pointer after the handle is closed. - - The memory protection mechanism is abstracted by several functions that - may have different implementations depending on the compile-time - configuration. Those are: - - * `raw_data_copy`: makes a copy of some data, optionally the copy can be - made read-only. - * `raw_data_protect`: protects the result of `raw_data_copy` from reading - * `raw_data_free`: if `raw_data_protect` retained any actual memory or other - resources, this indicates that those can be freed - - Any HPy context function that wishes to attach raw data to a handle should - make a copy of the actual data by using `raw_data_copy`. This copy should be - then set as the value of the associated_data field. Once the handle is - closed, the raw data pointer is passed to raw_data_protect and once the handle - is reused the raw data pointer is passed to raw_data_free. - - This means that if the implementation of `raw_data_protect` retains some - resources, we are leaking them. To mitigate this a bit, we have a limit on the - overall size of data that can be leaked and once it is reached, we use - raw_data_free immediately once the associated handle is closed. - - Note that, for example, the mmap based implementation of `raw_data_copy` - never allocates less than a page, so it actually takes more memory than - what is the size of the raw data. This is, however, mostly covered by the - limit on closed handles. For the default configuration we have: - - DEFAULT_CLOSED_HANDLES_QUEUE_MAX_SIZE = 1024 - DEFAULT_PROTECTED_RAW_DATA_MAX_SIZE = 1024 * 1024 * 10 - - the total leaked raw data size limit of 10MB is larger than if we created - and leaked 1024 handles with only a small raw data attached to them (4MB - for 1024 pages of 4KB). This ratio may be different for larger pages or for - different configuration of the limits. For the sake of keeping the - implementation reasonably simple and portable, we choose to ignore this - for the time being. -*/ - -typedef HPy UHPy; -typedef HPy DHPy; - -/* Under CPython: - - UHPy always end with 1 (see hpy.universal's _py2h and _h2py) - - DHPy are pointers, so they always end with 0 - - DHPy_sanity_check is a minimal check to ensure that we are not treating a - UHPy as a DHPy. Note that DHPy_sanity_check works fine also on HPy_NULL. - - NOTE: UHPy_sanity_check works ONLY with CPython's hpy.universal, because - UHPys are computed in such a way that the last bit it's always 1. On other - implementations this assumption might not hold. By default, - UHPy_sanity_check does nothing, unless you #define - HPY_DEBUG_ENABLE_UHPY_SANITY_CHECK, which for CPython is done by setup.py -*/ -static inline void DHPy_sanity_check(DHPy dh) { - assert( (dh._i & 1) == 0 ); -} - -static inline void UHPy_sanity_check(UHPy uh) { -#ifdef HPY_DEBUG_ENABLE_UHPY_SANITY_CHECK - if (!HPy_IsNull(uh)) - assert( (uh._i & 1) == 1 ); -#endif -} - -// NOTE: having a "generation" field is the easiest way to know when a handle -// was created, but we waste 8 bytes per handle. Since all handles of the same -// generation are stored sequentially in the open_handles list, a possible -// alternative implementation is to put special placeholders inside the list -// to mark the creation of a new generation -typedef struct DebugHandle { - UHPy uh; - long generation; - bool is_closed; - // pointer to and size of any raw data associated with - // the lifetime of the handle: - void *associated_data; - // allocation_stacktrace information if available - char *allocation_stacktrace; - HPy_ssize_t associated_data_size; - struct DebugHandle *prev; - struct DebugHandle *next; -} DebugHandle; - -static inline DebugHandle * as_DebugHandle(DHPy dh) { - DHPy_sanity_check(dh); - return (DebugHandle *)dh._i; -} - -static inline DHPy as_DHPy(DebugHandle *handle) { - return (DHPy){(HPy_ssize_t)handle}; -} - -DHPy DHPy_open(HPyContext *dctx, UHPy uh); -void DHPy_close(HPyContext *dctx, DHPy dh); -void DHPy_close_and_check(HPyContext *dctx, DHPy dh); -void DHPy_free(HPyContext *dctx, DHPy dh); -void DHPy_invalid_handle(HPyContext *dctx, DHPy dh); - -static inline UHPy DHPy_unwrap(HPyContext *dctx, DHPy dh) -{ - if (HPy_IsNull(dh)) - return HPy_NULL; - DebugHandle *handle = as_DebugHandle(dh); - if (handle->is_closed) - DHPy_invalid_handle(dctx, dh); - return handle->uh; -} - -/* === DHQueue === */ - -typedef struct { - DebugHandle *head; - DebugHandle *tail; - HPy_ssize_t size; -} DHQueue; - -void DHQueue_init(DHQueue *q); -void DHQueue_append(DHQueue *q, DebugHandle *h); -DebugHandle *DHQueue_popfront(DHQueue *q); -void DHQueue_remove(DHQueue *q, DebugHandle *h); -void DHQueue_sanity_check(DHQueue *q); - -/* === HPyDebugInfo === */ - -static const HPy_ssize_t DEFAULT_CLOSED_HANDLES_QUEUE_MAX_SIZE = 1024; -static const HPy_ssize_t DEFAULT_PROTECTED_RAW_DATA_MAX_SIZE = 1024 * 1024 * 10; - -typedef struct { - long magic_number; // used just for sanity checks - HPyContext *uctx; - long current_generation; - - // the following should be an HPyField, but it's complicate: - // HPyFields should be used only on memory which is known by the GC, which - // happens automatically if you use e.g. HPy_New, but currently - // HPy_DebugInfo is malloced(). We need either: - // 1. a generic HPy_GcMalloc() OR - // 2. HPy_{Un}TrackMemory(), so that we can add manually allocated - // memory as a GC root - UHPy uh_on_invalid_handle; - HPy_ssize_t closed_handles_queue_max_size; // configurable by the user - HPy_ssize_t protected_raw_data_max_size; - HPy_ssize_t protected_raw_data_size; - // Limit for the stack traces captured for allocated handles - // Value 0 implies that stack traces should not be captured - HPy_ssize_t handle_alloc_stacktrace_limit; - DHQueue open_handles; - DHQueue closed_handles; -} HPyDebugInfo; - -static inline HPyDebugInfo *get_info(HPyContext *dctx) -{ - HPyDebugInfo *info = (HPyDebugInfo*)dctx->_private; - assert(info->magic_number == HPY_DEBUG_MAGIC); // sanity check - return info; -} - - -void *raw_data_copy(const void* data, HPy_ssize_t size, bool write_protect); -void raw_data_protect(void* data, HPy_ssize_t size); -/* Return value: 0 indicates success, any different value indicates an error */ -int raw_data_free(void *data, HPy_ssize_t size); - -void create_stacktrace(char **target, HPy_ssize_t max_frames_count); - -#endif /* HPY_DEBUG_INTERNAL_H */ diff --git a/graalpython/com.oracle.graal.python.cext/hpy/hpy_debug.h b/graalpython/com.oracle.graal.python.cext/hpy/hpy_debug.h deleted file mode 100644 index c68abbe677..0000000000 --- a/graalpython/com.oracle.graal.python.cext/hpy/hpy_debug.h +++ /dev/null @@ -1,59 +0,0 @@ -#ifndef HPY_DEBUG_H -#define HPY_DEBUG_H - -#include "hpy.h" - -/* - This is the main public API for the debug mode, and it's meant to be used - by hpy.universal implementations (including but not limited to the - CPython's version of hpy.universal which is included in this repo). - - The idea is that for every uctx there is a corresponding unique dctx which - wraps it. - - If you call hpy_debug_get_ctx twice on the same uctx, you get the same - result. - - IMPLEMENTATION NOTE: at the moment of writing, the only known user of the - debug mode is CPython's hpy.universal: in that module, the uctx is a - statically allocated singleton, so for simplicity of implementation - currently we do the same inside debug_ctx.c, with a sanity check to ensure - that we don't call hpy_debug_get_ctx with different uctxs. But this is a - limitation of the current implementation and users should not rely on it. It - is likely that we will need to change it in the future, e.g. if we want to - have per-subinterpreter uctxs. -*/ - -HPyContext * hpy_debug_get_ctx(HPyContext *uctx); -int hpy_debug_ctx_init(HPyContext *dctx, HPyContext *uctx); -void hpy_debug_set_ctx(HPyContext *dctx); - -// convert between debug and universal handles. These are basically -// the same as DHPy_open and DHPy_unwrap but with a different name -// because this is the public-facing API and DHPy/UHPy are only internal -// implementation details. -HPy hpy_debug_open_handle(HPyContext *dctx, HPy uh); -HPy hpy_debug_unwrap_handle(HPyContext *dctx, HPy dh); -void hpy_debug_close_handle(HPyContext *dctx, HPy dh); - -// this is the HPy init function created by HPy_MODINIT. In CPython's version -// of hpy.universal the code is embedded inside the extension, so we can call -// this function directly instead of dlopen it. This is similar to what -// CPython does for its own built-in modules. But we must use the same -// signature as HPy_MODINIT - -// Copied from Python's exports.h, pyport.h -#ifndef Py_EXPORTED_SYMBOL - #if defined(_WIN32) || defined(__CYGWIN__) - #define Py_EXPORTED_SYMBOL __declspec(dllexport) - #else - #define Py_EXPORTED_SYMBOL __attribute__ ((visibility ("default"))) - #endif -#endif -#ifdef ___cplusplus -extern "C" -#endif -Py_EXPORTED_SYMBOL -HPy HPyInit__debug(HPyContext *uctx); - -#endif /* HPY_DEBUG_H */ diff --git a/graalpython/com.oracle.graal.python.cext/modules/_hpy_debug.c b/graalpython/com.oracle.graal.python.cext/modules/_hpy_debug.c deleted file mode 100644 index 9d0573f36c..0000000000 --- a/graalpython/com.oracle.graal.python.cext/modules/_hpy_debug.c +++ /dev/null @@ -1,403 +0,0 @@ -// Python-level interface for the _debug module. Written in HPy itself, the -// idea is that it should be reusable by other implementations - -// NOTE: hpy.debug._debug is loaded using the UNIVERSAL ctx. To make it -// clearer, we will use "uctx" and "dctx" to distinguish them. - -#include "hpy.h" -#include "debug_internal.h" - -static UHPy new_DebugHandleObj(HPyContext *uctx, UHPy u_DebugHandleType, - DebugHandle *handle); - - -HPyDef_METH(new_generation, "new_generation", new_generation_impl, HPyFunc_NOARGS) -static UHPy new_generation_impl(HPyContext *uctx, UHPy self) -{ - HPyContext *dctx = hpy_debug_get_ctx(uctx); - HPyDebugInfo *info = get_info(dctx); - info->current_generation++; - return HPyLong_FromLong(uctx, info->current_generation); -} - -static UHPy build_list_of_handles(HPyContext *uctx, UHPy u_self, DHQueue *q, - long gen) -{ - UHPy u_DebugHandleType = HPy_NULL; - UHPy u_result = HPy_NULL; - UHPy u_item = HPy_NULL; - - u_DebugHandleType = HPy_GetAttr_s(uctx, u_self, "DebugHandle"); - if (HPy_IsNull(u_DebugHandleType)) - goto error; - - u_result = HPyList_New(uctx, 0); - if (HPy_IsNull(u_result)) - goto error; - - DebugHandle *dh = q->head; - while(dh != NULL) { - if (dh->generation >= gen) { - UHPy u_item = new_DebugHandleObj(uctx, u_DebugHandleType, dh); - if (HPy_IsNull(u_item)) - goto error; - if (HPyList_Append(uctx, u_result, u_item) == -1) - goto error; - HPy_Close(uctx, u_item); - } - dh = dh->next; - } - - HPy_Close(uctx, u_DebugHandleType); - return u_result; - - error: - HPy_Close(uctx, u_DebugHandleType); - HPy_Close(uctx, u_result); - HPy_Close(uctx, u_item); - return HPy_NULL; -} - - -HPyDef_METH(get_open_handles, "get_open_handles", get_open_handles_impl, HPyFunc_O, .doc= - "Return a list containing all the open handles whose generation is >= " - "of the given arg") -static UHPy get_open_handles_impl(HPyContext *uctx, UHPy u_self, UHPy u_gen) -{ - HPyContext *dctx = hpy_debug_get_ctx(uctx); - HPyDebugInfo *info = get_info(dctx); - - long gen = HPyLong_AsLong(uctx, u_gen); - if (HPyErr_Occurred(uctx)) - return HPy_NULL; - - return build_list_of_handles(uctx, u_self, &info->open_handles, gen); -} - -HPyDef_METH(get_closed_handles, "get_closed_handles", get_closed_handles_impl, - HPyFunc_VARARGS, .doc= - "Return a list of all the closed handle in the cache") -static UHPy get_closed_handles_impl(HPyContext *uctx, UHPy u_self, HPy *args, HPy_ssize_t nargs) -{ - HPyContext *dctx = hpy_debug_get_ctx(uctx); - HPyDebugInfo *info = get_info(dctx); - long gen = 0; - if (nargs > 0) { - if (nargs != 1) { - HPyErr_SetString(uctx, uctx->h_TypeError, - "get_closed_handles expects no arguments or exactly one argument"); - return HPy_NULL; - } - gen = HPyLong_AsLong(uctx, args[0]); - if (HPyErr_Occurred(uctx)) - return HPy_NULL; - } - return build_list_of_handles(uctx, u_self, &info->closed_handles, gen); -} - -HPyDef_METH(get_closed_handles_queue_max_size, "get_closed_handles_queue_max_size", - get_closed_handles_queue_max_size_impl, HPyFunc_NOARGS, .doc= - "Return the maximum size of the closed handles queue") -static UHPy get_closed_handles_queue_max_size_impl(HPyContext *uctx, UHPy u_self) -{ - HPyContext *dctx = hpy_debug_get_ctx(uctx); - HPyDebugInfo *info = get_info(dctx); - return HPyLong_FromSsize_t(uctx, info->closed_handles_queue_max_size); -} - -HPyDef_METH(set_closed_handles_queue_max_size, "set_closed_handles_queue_max_size", - set_closed_handles_queue_max_size_impl, HPyFunc_O, .doc= - "Set the maximum size of the closed handles queue") -static UHPy set_closed_handles_queue_max_size_impl(HPyContext *uctx, UHPy u_self, UHPy u_size) -{ - HPyContext *dctx = hpy_debug_get_ctx(uctx); - HPyDebugInfo *info = get_info(dctx); - HPy_ssize_t size = HPyLong_AsSize_t(uctx, u_size); - if (HPyErr_Occurred(uctx)) - return HPy_NULL; - info->closed_handles_queue_max_size = size; - return HPy_Dup(uctx, uctx->h_None); -} - -HPyDef_METH(get_protected_raw_data_max_size, "get_protected_raw_data_max_size", -get_protected_raw_data_max_size_impl, HPyFunc_NOARGS, .doc= -"Return the maximum size of the retained raw memory associated with closed handles") -static UHPy get_protected_raw_data_max_size_impl(HPyContext *uctx, UHPy u_self) -{ - HPyContext *dctx = hpy_debug_get_ctx(uctx); - HPyDebugInfo *info = get_info(dctx); - return HPyLong_FromSsize_t(uctx, info->protected_raw_data_max_size); -} - -HPyDef_METH(set_protected_raw_data_max_size, "set_protected_raw_data_max_size", -set_protected_raw_data_max_size_impl, HPyFunc_O, .doc= -"Set the maximum size of the retained raw memory associated with closed handles") -static UHPy set_protected_raw_data_max_size_impl(HPyContext *uctx, UHPy u_self, UHPy u_size) -{ - HPyContext *dctx = hpy_debug_get_ctx(uctx); - HPyDebugInfo *info = get_info(dctx); - HPy_ssize_t size = HPyLong_AsSize_t(uctx, u_size); - if (HPyErr_Occurred(uctx)) - return HPy_NULL; - info->protected_raw_data_max_size = size; - return HPy_Dup(uctx, uctx->h_None); -} - -HPyDef_METH(set_on_invalid_handle, "set_on_invalid_handle", set_on_invalid_handle_impl, - HPyFunc_O, .doc= - "Set the function to call when we detect the usage of an invalid handle") -static UHPy set_on_invalid_handle_impl(HPyContext *uctx, UHPy u_self, UHPy u_arg) -{ - HPyContext *dctx = hpy_debug_get_ctx(uctx); - HPyDebugInfo *info = get_info(dctx); - if (HPy_Is(uctx, u_arg, uctx->h_None)) { - info->uh_on_invalid_handle = HPy_NULL; - } else if (!HPyCallable_Check(uctx, u_arg)) { - HPyErr_SetString(uctx, uctx->h_TypeError, "Expected a callable object"); - return HPy_NULL; - } else { - info->uh_on_invalid_handle = HPy_Dup(uctx, u_arg); - } - return HPy_Dup(uctx, uctx->h_None); -} - -HPyDef_METH(set_handle_stack_trace_limit, "set_handle_stack_trace_limit", - set_handle_stack_trace_limit_impl, HPyFunc_O, .doc= - "Set the limit to captured HPy handles allocations stack traces. " - "None means do not capture the stack traces.") -static UHPy set_handle_stack_trace_limit_impl(HPyContext *uctx, UHPy u_self, UHPy u_arg) -{ - HPyContext *dctx = hpy_debug_get_ctx(uctx); - HPyDebugInfo *info = get_info(dctx); - if (HPy_Is(uctx, u_arg, uctx->h_None)) { - info->handle_alloc_stacktrace_limit = 0; - } else { - assert(!HPyErr_Occurred(uctx)); - HPy_ssize_t newlimit = HPyLong_AsSsize_t(uctx, u_arg); - if (newlimit == -1 && HPyErr_Occurred(uctx)) { - return HPy_NULL; - } - info->handle_alloc_stacktrace_limit = newlimit; - } - return HPy_Dup(uctx, uctx->h_None); -} - - -/* ~~~~~~ DebugHandleType and DebugHandleObject ~~~~~~~~ - - This is the applevel view of a DebugHandle/DHPy. - - Note that there are two different ways to expose DebugHandle to applevel: - - 1. make DebugHandle itself a Python object: this is simple but means that - you have to pay the PyObject_HEAD overhead (16 bytes) for all of them - - 2. make DebugHandle a plain C struct, and expose them through a - Python-level wrapper. - - We choose to implement solution 2 because we expect to have many - DebugHandle around, but to expose only few of them to applevel, when you - call get_open_handles. This way, we save 16 bytes per DebugHandle. - - This means that you can have different DebugHandleObjects wrapping the same - DebugHandle. To make it easier to compare them, they expose the .id - attribute, which is the address of the wrapped DebugHandle. Also, - DebugHandleObjects compare equal if their .id is equal. -*/ - -typedef struct { - DebugHandle *handle; -} DebugHandleObject; - -HPyType_HELPERS(DebugHandleObject) - -HPyDef_GET(DebugHandle_obj, "obj", DebugHandle_obj_get, - .doc="The object which the handle points to") -static UHPy DebugHandle_obj_get(HPyContext *uctx, UHPy self, void *closure) -{ - DebugHandleObject *dh = DebugHandleObject_AsStruct(uctx, self); - return HPy_Dup(uctx, dh->handle->uh); -} - -HPyDef_GET(DebugHandle_id, "id", DebugHandle_id_get, - .doc="A numeric identifier representing the underlying universal handle") -static UHPy DebugHandle_id_get(HPyContext *uctx, UHPy self, void *closure) -{ - DebugHandleObject *dh = DebugHandleObject_AsStruct(uctx, self); - return HPyLong_FromSsize_t(uctx, (HPy_ssize_t)dh->handle); -} - -HPyDef_GET(DebugHandle_is_closed, "is_closed", DebugHandle_is_closed_get, - .doc="Self-explanatory") -static UHPy DebugHandle_is_closed_get(HPyContext *uctx, UHPy self, void *closure) -{ - DebugHandleObject *dh = DebugHandleObject_AsStruct(uctx, self); - return HPyBool_FromLong(uctx, dh->handle->is_closed); -} - -HPyDef_GET(DebugHandle_raw_data_size, "raw_data_size", DebugHandle_raw_data_size_get, -.doc="Size of retained raw memory. FOR TESTS ONLY.") -static UHPy DebugHandle_raw_data_size_get(HPyContext *uctx, UHPy self, void *closure) -{ - DebugHandleObject *dh = DebugHandleObject_AsStruct(uctx, self); - if (dh->handle->associated_data) { - return HPyLong_FromSsize_t(uctx, dh->handle->associated_data_size); - } else { - return HPyLong_FromLong(uctx, -1); - } -} - -HPyDef_SLOT(DebugHandle_cmp, DebugHandle_cmp_impl, HPy_tp_richcompare) -static UHPy DebugHandle_cmp_impl(HPyContext *uctx, UHPy self, UHPy o, HPy_RichCmpOp op) -{ - UHPy T = HPy_Type(uctx, self); - if (!HPy_TypeCheck(uctx, o, T)) - return HPy_Dup(uctx, uctx->h_NotImplemented); - DebugHandleObject *dh_self = DebugHandleObject_AsStruct(uctx, self); - DebugHandleObject *dh_o = DebugHandleObject_AsStruct(uctx, o); - - switch(op) { - case HPy_EQ: - return HPyBool_FromLong(uctx, dh_self->handle == dh_o->handle); - case HPy_NE: - return HPyBool_FromLong(uctx, dh_self->handle != dh_o->handle); - default: - return HPy_Dup(uctx, uctx->h_NotImplemented); - } -} - -HPyDef_SLOT(DebugHandle_repr, DebugHandle_repr_impl, HPy_tp_repr) -static UHPy DebugHandle_repr_impl(HPyContext *uctx, UHPy self) -{ - DebugHandleObject *dh = DebugHandleObject_AsStruct(uctx, self); - UHPy uh_fmt = HPy_NULL; - UHPy uh_id = HPy_NULL; - UHPy uh_args = HPy_NULL; - UHPy uh_result = HPy_NULL; - UHPy h_trace_header = HPy_NULL; - UHPy h_trace = HPy_NULL; - - const char *fmt = NULL; - if (dh->handle->is_closed) - fmt = "\n%s%s"; - else - fmt = "\n%s%s"; - - // XXX: switch to HPyUnicode_FromFormat when we have it - uh_fmt = HPyUnicode_FromString(uctx, fmt); - if (HPy_IsNull(uh_fmt)) - goto exit; - - uh_id = HPyLong_FromSsize_t(uctx, (HPy_ssize_t)dh->handle); - if (HPy_IsNull(uh_id)) - goto exit; - - const char *trace_header; - const char *trace; - if (dh->handle->allocation_stacktrace) { - trace_header = "Allocation stacktrace:\n"; - trace = dh->handle->allocation_stacktrace; - } else { - trace_header = "To get the stack trace of where it was allocated use:\nhpy.debug."; - trace = set_handle_stack_trace_limit.meth.name; - } - h_trace_header = HPyUnicode_FromString(uctx, trace_header); - h_trace = HPyUnicode_FromString(uctx, trace); - - if (dh->handle->is_closed) - uh_args = HPyTuple_FromArray(uctx, (UHPy[]){uh_id, - h_trace_header, h_trace}, 3); - else - uh_args = HPyTuple_FromArray(uctx, (UHPy[]){uh_id, dh->handle->uh, - h_trace_header, h_trace}, 4); - if (HPy_IsNull(uh_args)) - goto exit; - - uh_result = HPy_Remainder(uctx, uh_fmt, uh_args); - - exit: - HPy_Close(uctx, uh_fmt); - HPy_Close(uctx, uh_id); - HPy_Close(uctx, uh_args); - HPy_Close(uctx, h_trace); - HPy_Close(uctx, h_trace_header); - return uh_result; -} - - -HPyDef_METH(DebugHandle__force_close, "_force_close", DebugHandle__force_close_impl, - HPyFunc_NOARGS, .doc="Close the underyling handle. FOR TESTS ONLY.") -static UHPy DebugHandle__force_close_impl(HPyContext *uctx, UHPy self) -{ - DebugHandleObject *dh = DebugHandleObject_AsStruct(uctx, self); - HPyContext *dctx = hpy_debug_get_ctx(uctx); - HPy_Close(dctx, as_DHPy(dh->handle)); - return HPy_Dup(uctx, uctx->h_None); -} - -static HPyDef *DebugHandleType_defs[] = { - &DebugHandle_obj, - &DebugHandle_id, - &DebugHandle_is_closed, - &DebugHandle_raw_data_size, - &DebugHandle_cmp, - &DebugHandle_repr, - &DebugHandle__force_close, - NULL -}; - -static HPyType_Spec DebugHandleType_spec = { - .name = "hpy.debug._debug.DebugHandle", - .basicsize = sizeof(DebugHandleObject), - .flags = HPy_TPFLAGS_DEFAULT, - .defines = DebugHandleType_defs, -}; - - -static UHPy new_DebugHandleObj(HPyContext *uctx, UHPy u_DebugHandleType, - DebugHandle *handle) -{ - DebugHandleObject *dhobj; - UHPy u_result = HPy_New(uctx, u_DebugHandleType, &dhobj); - dhobj->handle = handle; - return u_result; -} - - -/* ~~~~~~ definition of the module hpy.debug._debug ~~~~~~~ */ - -static HPyDef *module_defines[] = { - &new_generation, - &get_open_handles, - &get_closed_handles, - &get_closed_handles_queue_max_size, - &set_closed_handles_queue_max_size, - &get_protected_raw_data_max_size, - &set_protected_raw_data_max_size, - &set_on_invalid_handle, - &set_handle_stack_trace_limit, - NULL -}; - -static HPyModuleDef moduledef = { - .name = "hpy.debug._debug", - .doc = "HPy debug mode", - .size = -1, - .defines = module_defines -}; - - -HPy_MODINIT(_debug) -static UHPy init__debug_impl(HPyContext *uctx) -{ - UHPy m = HPyModule_Create(uctx, &moduledef); - if (HPy_IsNull(m)) - return HPy_NULL; - - UHPy h_DebugHandleType = HPyType_FromSpec(uctx, &DebugHandleType_spec, NULL); - if (HPy_IsNull(h_DebugHandleType)) - return HPy_NULL; - HPy_SetAttr_s(uctx, m, "DebugHandle", h_DebugHandleType); - HPy_Close(uctx, h_DebugHandleType); - return m; -} diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/version.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/version.h deleted file mode 100644 index 81d60ed7d5..0000000000 --- a/graalpython/com.oracle.graal.python.hpy/include/hpy/version.h +++ /dev/null @@ -1,4 +0,0 @@ - -// automatically generated by setup.py:get_scm_config() -#define HPY_VERSION "0.9.1.dev79+gb0fbdf73" -#define HPY_GIT_REVISION "b0fbdf73" diff --git a/graalpython/com.oracle.graal.python.jni/src/debug/include/hpy_debug.h b/graalpython/com.oracle.graal.python.jni/src/debug/include/hpy_debug.h deleted file mode 100644 index f2fb9471b3..0000000000 --- a/graalpython/com.oracle.graal.python.jni/src/debug/include/hpy_debug.h +++ /dev/null @@ -1,57 +0,0 @@ -#ifndef HPY_DEBUG_H -#define HPY_DEBUG_H - -#include "hpy.h" - -/* - This is the main public API for the debug mode, and it's meant to be used - by hpy.universal implementations (including but not limited to the - CPython's version of hpy.universal which is included in this repo). - - The idea is that for every uctx there is a corresponding unique dctx which - wraps it. - - If you call hpy_debug_get_ctx twice on the same uctx, you get the same - result. - - IMPLEMENTATION NOTE: at the moment of writing, the only known user of the - debug mode is CPython's hpy.universal: in that module, the uctx is a - statically allocated singleton, so for simplicity of implementation - currently we do the same inside debug_ctx.c, with a sanity check to ensure - that we don't call hpy_debug_get_ctx with different uctxs. But this is a - limitation of the current implementation and users should not rely on it. It - is likely that we will need to change it in the future, e.g. if we want to - have per-subinterpreter uctxs. -*/ - -HPyContext * hpy_debug_get_ctx(HPyContext *uctx); -int hpy_debug_ctx_init(HPyContext *dctx, HPyContext *uctx); -void hpy_debug_set_ctx(HPyContext *dctx); - -// convert between debug and universal handles. These are basically -// the same as DHPy_open and DHPy_unwrap but with a different name -// because this is the public-facing API and DHPy/UHPy are only internal -// implementation details. -HPy hpy_debug_open_handle(HPyContext *dctx, HPy uh); -HPy hpy_debug_unwrap_handle(HPyContext *dctx, HPy dh); -void hpy_debug_close_handle(HPyContext *dctx, HPy dh); - -// this is the HPy init function created by HPy_MODINIT. In CPython's version -// of hpy.universal the code is embedded inside the extension, so we can call -// this function directly instead of dlopen it. This is similar to what -// CPython does for its own built-in modules. But we must use the same -// signature as HPy_MODINIT - -#ifdef ___cplusplus -extern "C" -#endif -HPy_EXPORTED_SYMBOL -HPyModuleDef* HPyInit__debug(); - -#ifdef ___cplusplus -extern "C" -#endif -HPy_EXPORTED_SYMBOL -void HPyInitGlobalContext__debug(HPyContext *ctx); - -#endif /* HPY_DEBUG_H */ diff --git a/graalpython/com.oracle.graal.python.jni/src/trace/include/hpy_trace.h b/graalpython/com.oracle.graal.python.jni/src/trace/include/hpy_trace.h deleted file mode 100644 index e75b76186e..0000000000 --- a/graalpython/com.oracle.graal.python.jni/src/trace/include/hpy_trace.h +++ /dev/null @@ -1,42 +0,0 @@ -#ifndef HPY_TRACE_H -#define HPY_TRACE_H - -#include "hpy.h" - -/* - This is the main public API for the trace mode, and it's meant to be used - by hpy.universal implementations (including but not limited to the - CPython's version of hpy.universal which is included in this repo). - - The idea is that for every uctx there is a corresponding unique tctx which - wraps it. - - If you call hpy_trace_get_ctx twice on the same uctx, you get the same - result. -*/ - -HPyContext * hpy_trace_get_ctx(HPyContext *uctx); -int hpy_trace_ctx_init(HPyContext *tctx, HPyContext *uctx); -int hpy_trace_ctx_free(HPyContext *tctx); -int hpy_trace_get_nfunc(void); -const char * hpy_trace_get_func_name(int idx); - -// this is the HPy init function created by HPy_MODINIT. In CPython's version -// of hpy.universal the code is embedded inside the extension, so we can call -// this function directly instead of dlopen it. This is similar to what -// CPython does for its own built-in modules. But we must use the same -// signature as HPy_MODINIT - -#ifdef ___cplusplus -extern "C" -#endif -HPy_EXPORTED_SYMBOL -HPyModuleDef* HPyInit__trace(); - -#ifdef ___cplusplus -extern "C" -#endif -HPy_EXPORTED_SYMBOL -void HPyInitGlobalContext__trace(HPyContext *ctx); - -#endif /* HPY_TRACE_H */ diff --git a/graalpython/hpy/.gitattributes b/graalpython/hpy/.gitattributes new file mode 100644 index 0000000000..1c5f632434 --- /dev/null +++ b/graalpython/hpy/.gitattributes @@ -0,0 +1 @@ +**/autogen_* linguist-generated=true diff --git a/graalpython/hpy/.github/FUNDING.yml b/graalpython/hpy/.github/FUNDING.yml new file mode 100644 index 0000000000..950b3021c6 --- /dev/null +++ b/graalpython/hpy/.github/FUNDING.yml @@ -0,0 +1 @@ +open_collective: hpy diff --git a/graalpython/hpy/.github/workflows/ci.yml b/graalpython/hpy/.github/workflows/ci.yml new file mode 100644 index 0000000000..0ec548c068 --- /dev/null +++ b/graalpython/hpy/.github/workflows/ci.yml @@ -0,0 +1,427 @@ +--- +name: tests + +on: [pull_request] + +jobs: + main_tests: + name: Main tests ${{ matrix.os }} ${{ matrix.python-version }} ${{ matrix.compiler }} + runs-on: ${{ matrix.os }} + continue-on-error: true + strategy: + # Duplicate changes to this matrix to 'poc_tests' + matrix: + os: [ubuntu-latest, macos-13, windows-latest] + python-version: ['3.9', '3.10'] + compiler: [""] + include: + - os: ubuntu-latest + python-version: '3.8' + - os: ubuntu-latest + python-version: '3.10' + compiler: 'g++' + - os: ubuntu-latest + python-version: '3.11' + - os: ubuntu-latest + python-version: '3.12' + + steps: + - uses: actions/checkout@v4 + + # - template: azure-templates/ccache.yml + # parameters: + # pythonVersion: $(python.version) + # - template: azure-templates/python.yml + # parameters: + # pythonVersion: $(python.version) + + - name: Set up Python + uses: actions/setup-python@v5.4 + with: + python-version: ${{ matrix.python-version }} + allow-prereleases: true + + - name: Install/Upgrade Python dependencies + run: python -m pip install --upgrade pip wheel 'setuptools>=60.2' + + - name: Build + run: | + make + python -m pip install . + + - if: ${{ matrix.compiler }} + # Only set the compiler for the tests, not for the build + run: echo "CC=${{ matrix.compiler }}" >> $GITHUB_ENV + + - name: Run tests + run: | + python -m pip install pytest pytest-xdist filelock + python -m pytest --basetemp=.tmpdir --durations=16 -n auto test/ + + main_tests_debug: + name: Main tests on CPython debug builds + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + # Cannot install 3.12 on ubuntu-latest + python-version: ['3.11', '3.10', '3.9', '3.8'] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python from deadsnakes + uses: deadsnakes/action@v3.2.0 + with: + python-version: ${{ matrix.python-version }} + debug: true + + - name: Check Python debug build + run: python -c "import sys; print(hasattr(sys, 'gettotalrefcount'))" + + - name: Install/Upgrade Python dependencies + run: python -m pip install --upgrade pip wheel + + - name: Build + run: | + make + python -m pip install . + + - name: Run tests + run: | + python -m pip install pytest pytest-xdist filelock + python -m pytest --durations=16 -n auto test/ + + poc_tests: + name: Proof of concept tests + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-13, windows-latest] + python-version: ['3.10'] + include: + - os: ubuntu-latest + python-version: '3.8' + - os: ubuntu-latest + python-version: '3.9' + - os: ubuntu-latest + python-version: '3.11' + - os: ubuntu-latest + python-version: '3.12' + + steps: + - uses: actions/checkout@v4 + + # - template: azure-templates/ccache.yml + # parameters: + # pythonVersion: $(python.version) + # - template: azure-templates/python.yml + # parameters: + # pythonVersion: $(python.version) + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + allow-prereleases: true + + - name: Install/Upgrade Python dependencies + run: python -m pip install --upgrade pip wheel + shell: bash + + - name: 'Test setup.py --hpy-abi=cpython bdist_wheel' + run: proof-of-concept/test_pof.sh wheel cpython + shell: bash + + - name: 'Test setup.py --hpy-abi=universal bdist_wheel' + run: proof-of-concept/test_pof.sh wheel universal + shell: bash + + - name: 'Test setup.py --hpy-abi=cpython install' + run: proof-of-concept/test_pof.sh setup_py_install cpython + shell: bash + + - name: 'Test setup.py --hpy-abi=universal install' + run: proof-of-concept/test_pof.sh setup_py_install universal + shell: bash + + - name: 'Test setup.py --hpy-abi=cpython build_ext --inplace' + run: proof-of-concept/test_pof.sh setup_py_build_ext_inplace cpython + shell: bash + + - name: 'Test setup.py --hpy-abi=universal build_ext --inplace' + run: proof-of-concept/test_pof.sh setup_py_build_ext_inplace universal + shell: bash + + + porting_example_tests: + name: Porting example tests + runs-on: ${{ matrix.os }} + continue-on-error: true + strategy: + matrix: + os: [ubuntu-latest, macos-13, windows-latest] + python-version: ['3.9'] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install/Upgrade Python dependencies + run: python -m pip install --upgrade pip wheel + shell: bash + + - name: Install HPy + run: python -m pip install . + + - name: Install pytest + run: | + python -m pip install pytest + + - name: Run tests + run: make porting-example-tests + shell: bash + + - name: Run tests of completed port in debug mode + env: + HPY_DEBUG: "1" + TEST_ARGS: "-s -k hpy_final" + run: make porting-example-tests + shell: bash + + + valgrind_tests_1: + name: 'Valgrind tests (1/3)' + uses: ./.github/workflows/valgrind-tests.yml + with: + portion: '1/3' + + + valgrind_tests_2: + name: 'Valgrind tests (2/3)' + uses: ./.github/workflows/valgrind-tests.yml + with: + portion: '2/3' + + + valgrind_tests_3: + name: 'Valgrind tests (3/3)' + uses: ./.github/workflows/valgrind-tests.yml + with: + portion: '3/3' + + + docs_examples_tests: + name: Documentation examples tests + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-13, windows-latest] + python-version: ['3.10'] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install/Upgrade Python dependencies + run: python -m pip install --upgrade pip wheel + shell: bash + + - name: Install HPy + run: python -m pip install . + + - name: Install pytest + run: | + python -m pip install pytest pytest-xdist filelock + - name: Run tests + run: make docs-examples-tests + shell: bash + + build_docs: + name: Build documentation + runs-on: 'ubuntu-latest' + steps: + - uses: actions/checkout@v4 + + # - template: azure-templates/python.yml + + - name: Install / Upgrade system requirements + run: sudo apt update && sudo apt install -y libclang-17-dev + + - name: Install / Upgrade Python requirements + run: | + python -m pip install --upgrade pip + python -m pip install -r docs/requirements.txt + + - name: Build docs + run: | + cd docs; + python -m sphinx -T -W -E -b html -d _build/doctrees -D language=en . _build/html + + - name: Upload built HTML files + uses: actions/upload-artifact@v4 + with: + name: hpy_html_docs + path: docs/_build/html/* + if-no-files-found: error + retention-days: 5 + + c_tests: + name: C tests + runs-on: 'ubuntu-latest' + steps: + - uses: actions/checkout@v4 + - run: make -C c_test + + + check_autogen: + name: Check autogen + runs-on: 'ubuntu-latest' + steps: + - uses: actions/checkout@v4 + + # - template: azure-templates/python.yml + + - name: Set up Python + uses: actions/setup-python@v5 + with: + # autogen needs distutils + python-version: '3.11' + + - name: Install/Upgrade Python dependencies + run: python -m pip install --upgrade pip wheel + + - name: Install autogen dependencies + run: pip install -r requirements-autogen.txt + + - name: make autogen + run: | + make autogen + if [ -z "$(git status --porcelain)" ]; then + # clean working copy + echo "Working copy is clean, everything ok" + else + # Uncommitted changes + echo "ERROR: uncommitted changes after running make autogen" + echo "git status" + git status + echo + echo "git diff" + git diff + exit 1 + fi + + + check_py27_compat: + name: Check Python 2.7 compatibility + runs-on: 'ubuntu-20.04' + steps: + - uses: actions/checkout@v4 + + # - template: azure-templates/python.yml + # parameters: + # pythonVersion: "2.7" + + - name: Set up Python2 + # Copied from cython's ci.yml + run: | + sudo ln -fs python2 /usr/bin/python + sudo apt-get update + sudo apt-get install python-setuptools python2.7 python2.7-dev + curl https://bootstrap.pypa.io/pip/2.7/get-pip.py --output get-pip.py + sudo python2 get-pip.py + ls -l /usr/bin/pip* /usr/local/bin/pip* + which pip + + - name: Install/Upgrade Python dependencies + run: python -m pip install --upgrade pip wheel + + - name: check_py27_compat.py + run: | + python -m pip install pytest pytest-xdist filelock pathlib + python test/check_py27_compat.py + + + cpp_check: + name: Cppcheck static analysis + runs-on: 'ubuntu-22.04' + continue-on-error: true + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Install Cppcheck + run: sudo apt-get -qq -y install cppcheck=2.7-1 + + - name: Run Cppcheck + run: make cppcheck + + + infer: + name: Infer static analysis + runs-on: 'ubuntu-latest' + steps: + - uses: actions/checkout@v4 + + # - template: azure-templates/python.yml + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Install/Upgrade Python dependencies + run: python -m pip install --upgrade pip wheel + + - name: Install Infer + run: | + python -m pip install compiledb wheel; + VERSION=1.1.0; \ + curl -sSL "https://github.com/facebook/infer/releases/download/v$VERSION/infer-linux64-v$VERSION.tar.xz" \ + | sudo tar -C /opt -xJ && \ + echo "/opt/infer-linux64-v$VERSION/bin" >> $GITHUB_PATH + + - name: Run Infer + run: make infer + + check_microbench: + name: Check micro benchmarks + runs-on: 'ubuntu-latest' + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install / Upgrade system dependencies + run: sudo apt update && sudo apt install -y valgrind + + - name: Install / Upgrade Python requirements + run: | + python -m pip install --upgrade pip wheel 'setuptools>=60.2' + python -m pip install pytest cffi + + - name: Build and install HPy + run: | + make + python -m pip install . + + - name: Run microbenchmarks + run: | + cd microbench + python setup.py build_ext -i + python -m pytest -v diff --git a/graalpython/hpy/.github/workflows/release-pypi.yml b/graalpython/hpy/.github/workflows/release-pypi.yml new file mode 100644 index 0000000000..959a779170 --- /dev/null +++ b/graalpython/hpy/.github/workflows/release-pypi.yml @@ -0,0 +1,136 @@ +--- +name: Publish to PyPI + +# manually trigger this workflow +on: + workflow_dispatch: + inputs: + tag: + description: 'The Git tag to create or test a release for. Official releases should always be made from an existing tag.' + required: true + type: string + official: + description: 'If true, publish to official PyPI and create GitHub release. Otherwise publish only to test PyPI.' + default: false + type: boolean + +jobs: + build_sdist: + name: Build source package + runs-on: 'ubuntu-latest' + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ inputs.tag }} + + - name: Set up Python + uses: actions/setup-python@v4 + with: + # HPy requires at least Python 3.8; we mostly work with >=3.10 + python-version: '>=3.10' + + - name: Install/Upgrade Python dependencies + run: python -m pip install --upgrade pip wheel 'setuptools>=60.2' + + - name: Build and install Python source package + run: | + python setup.py sdist + python -m pip install dist/*.tar.gz + + - name: Run tests + run: | + make + pip install pytest pytest-xdist filelock + pytest -n auto test/ + + - uses: actions/upload-artifact@v3 + with: + path: dist/*.tar.gz + retention-days: 5 + + + build_bdist: + name: Build binary wheels + runs-on: ${{ matrix.os }} + strategy: + matrix: + # Windows tests fail when built as a binary wheel for some reason + # 'macos-12' doesn't pass the tests + os: [ubuntu-latest, macos-11] # windows-latest + + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ inputs.tag }} + + # setup Python for cibuildwheel + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.x' + + # for other architectures, see: https://cibuildwheel.readthedocs.io/en/stable/faq/#emulation + - name: Build wheels for CPython + uses: pypa/cibuildwheel@v2.12.3 + env: + # cibuildwheel automatically reads 'python_requires' from 'setup.py' + CIBW_BUILD: 'cp*' # no PyPy builds + CIBW_ARCHS_LINUX: "x86_64" # only Intel 64-bit + CIBW_ARCHS_MACOS: "x86_64" # only Intel 64-bit + CIBW_TEST_REQUIRES: pytest pytest-xdist filelock setuptools>=60.2 + # only copy test dir to current working dir + CIBW_TEST_COMMAND: cp -R {project}/test ./ && pytest --basetemp=.tmpdir --ignore=test/hpy_devel -n auto ./test/ + + - uses: actions/upload-artifact@v3 + with: + path: ./wheelhouse/*.whl + retention-days: 5 + + upload_pypi: + name: Publish packages to (Test) PyPI + needs: [build_sdist, build_bdist] + runs-on: ubuntu-latest + # don't do this action on forks by default + if: github.repository == 'hpyproject/hpy' + steps: + - uses: actions/download-artifact@v3 + with: + name: artifact + path: dist + + - name: Upload to Test PyPI + uses: pypa/gh-action-pypi-publish@v1.8.5 + if: ${{ !inputs.official }} + with: + verbose: true + user: __token__ + password: ${{ secrets.TEST_PYPI_API_TOKEN }} + repository-url: https://test.pypi.org/legacy/ + + - name: Upload to PyPI + uses: pypa/gh-action-pypi-publish@v1.8.5 + if: ${{ inputs.official }} + with: + verbose: true + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} + + create_gh_release: + needs: [upload_pypi] + runs-on: ubuntu-latest + # don't do this action on forks by default + if: github.repository == 'hpyproject/hpy' + steps: + - uses: actions/download-artifact@v3 + with: + name: artifact + path: dist + + - name: Release + uses: softprops/action-gh-release@v1 + with: + files: dist/* + # consider tags in form '*.*.*rc*' to indicate a pre-release + prerelease: ${{ contains(inputs.tag, 'rc') }} + draft: ${{ !inputs.official }} + tag_name: ${{ inputs.tag }} diff --git a/graalpython/hpy/.github/workflows/valgrind-tests.yml b/graalpython/hpy/.github/workflows/valgrind-tests.yml new file mode 100644 index 0000000000..badbc9da6e --- /dev/null +++ b/graalpython/hpy/.github/workflows/valgrind-tests.yml @@ -0,0 +1,38 @@ +on: + workflow_call: + inputs: + portion: + description: 'Select portion of tests to run under Valgrind (default runs all)' + default: '' + type: string + required: false + +jobs: + valgrind_tests: + # name: Valgrind tests + runs-on: 'ubuntu-latest' + steps: + - uses: actions/checkout@v4 + + - name: Install / Upgrade system dependencies + run: sudo apt update && sudo apt install -y valgrind + + - name: Set up Python + uses: actions/setup-python@v5.4 + with: + python-version: 3.9 + + - name: Install / Upgrade Python dependencies + run: python -m pip install --upgrade pip wheel + + - name: Build + run: | + make + python -m pip install . + + - name: Run tests + env: + HPY_TEST_PORTION: ${{ inputs.portion }} + run: | + python -m pip install pytest pytest-valgrind pytest-portion filelock + make valgrind diff --git a/graalpython/hpy/.gitignore b/graalpython/hpy/.gitignore new file mode 100644 index 0000000000..2877066982 --- /dev/null +++ b/graalpython/hpy/.gitignore @@ -0,0 +1,92 @@ +# HPy autogen +hpy/tools/autogen/autogen_pypy.txt + +# generated by setup.py:get_scm_config() +hpy/devel/include/hpy/version.h +hpy/devel/version.py + +# stubs created by setup.py when doing build_ext --inplace +proof-of-concept/pof.py +proof-of-concept/pofpackage/foo.py + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions and static libs +*.so +*.o +*.a +*.lib +vc140.pdb +c_test/test_debug_handles + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +test-output.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +.hpy.lock + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# cppcheck +.cppcheck + +# Sphinx +docs/_build/ + +# vscode +.vscode diff --git a/graalpython/hpy/.readthedocs.yml b/graalpython/hpy/.readthedocs.yml new file mode 100644 index 0000000000..7ee9eedb2e --- /dev/null +++ b/graalpython/hpy/.readthedocs.yml @@ -0,0 +1,10 @@ +# readthedocs.org configuration + +version: 2 +sphinx: + configuration: docs/conf.py +formats: all +python: + version: 3.7 + install: + - requirements: docs/requirements.txt diff --git a/graalpython/hpy/AUTHORS b/graalpython/hpy/AUTHORS new file mode 100644 index 0000000000..707ed38f94 --- /dev/null +++ b/graalpython/hpy/AUTHORS @@ -0,0 +1,38 @@ +# This is the list of HPy's significant contributors. +# +# This does not necessarily list everyone who has contributed code, +# especially since many employees of one corporation may be contributing. +# To see the full list of contributors, see the revision history in +# source control. +Alexander Schremmer +Ammar Askar +Ankith (ankith26) +Antonio Cuni +Armin Rigo +Charlie Hayden +Christian Klein +Daniel Alley +Dominic Davis-Foster +Jaime Resano Aísa +Joachim Trouverie +Julian Berman +Krish Patel +Kyle Lawlor-Bagcal +Mathieu Dupuy +Matti Picus +Max Horn +Maxwell Bernstein +Mohamed Koubaa +Nico Rittinghaus +Omer Katz +Oracle and/or its affiliates +Paul Prescod +Pierre Augier +Robert O'Shea +Ronan Lamy +Simon Cross +Stefan Behnel +Stefano Rivera +Victor Stinner +Vytautas Liuolia +Łukasz Langa diff --git a/graalpython/hpy/CONTRIBUTING.md b/graalpython/hpy/CONTRIBUTING.md new file mode 100644 index 0000000000..7178db511c --- /dev/null +++ b/graalpython/hpy/CONTRIBUTING.md @@ -0,0 +1,55 @@ +Contributing +============ + +Contributions are welcome, and they are greatly appreciated! Every +little bit helps. + +You can contribute in many ways: + +Types of Contributions +---------------------- + +### Report Bugs + +Report bugs at . + +When reporting a bug, please include: + +- Your operating system name and version. +- Any details about your local setup that might be helpful in + troubleshooting. +- Detailed steps to reproduce the bug. + +### Donate to HPy + +Become a financial contributor and help us sustain the HPy community: [Contribute to HPy](https://opencollective.com/hpy/contribute). + +### Fix Bugs or Implement Features + +Look through the GitHub issues for bugs or proposals being discussed +or ready for implementation. + +### Write Documentation + +HPy could always use more documentation, whether as part of the +official HPy docs, in docstrings, or even on the web in blog +posts, articles, and such. + +Get Started! +------------ + +You can test your environment by running + +```bash +proof-of-concept/test_pof.sh wheel universal +``` + +This should build HPy and a proof of concept (demo) extension and +then run a pytest that validates both. + +Until we get around to more comprehensive documentation, you +can reverse engineer how the system works by looking at these +files: + +- `proof-of-concept/test_pof.sh` +- `proof-of-concept/setup.py` diff --git a/graalpython/hpy/LICENSE b/graalpython/hpy/LICENSE new file mode 100644 index 0000000000..a5b403aa3f --- /dev/null +++ b/graalpython/hpy/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 The HPy Authors. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/graalpython/hpy/MANIFEST.in b/graalpython/hpy/MANIFEST.in new file mode 100644 index 0000000000..7a4b248c98 --- /dev/null +++ b/graalpython/hpy/MANIFEST.in @@ -0,0 +1 @@ +recursive-include hpy/devel/include *.h diff --git a/graalpython/hpy/Makefile b/graalpython/hpy/Makefile new file mode 100644 index 0000000000..6d0f1c0db8 --- /dev/null +++ b/graalpython/hpy/Makefile @@ -0,0 +1,86 @@ +.PHONY: all +all: hpy.universal + +.PHONY: hpy.universal +hpy.universal: + python3 setup.py build_clib -f build_ext -if + +.PHONY: dist-info +dist-info: + python3 setup.py dist_info + +debug: + HPY_DEBUG_BUILD=1 make all + +autogen: + python3 -m hpy.tools.autogen . + +cppcheck-build-dir: + mkdir -p $(or ${CPPCHECK_BUILD_DIR}, .cppcheck) + +.PHONY: cppcheck +cppcheck: cppcheck-build-dir + # azure pipelines doesn't show stderr, so we write the errors to a file and cat it later :( + $(eval PYTHON_INC = $(shell python3 -q -c "from sysconfig import get_paths as gp; print(gp()['include'])")) + $(eval PYTHON_PLATINC = $(shell python3 -q -c "from sysconfig import get_paths as gp; print(gp()['platinclude'])")) + cppcheck --version + cppcheck \ + -v \ + --error-exitcode=1 \ + --cppcheck-build-dir=$(or ${CPPCHECK_BUILD_DIR}, .cppcheck) \ + --enable=warning,performance,portability,information,missingInclude \ + --inline-suppr \ + --suppress=syntaxError \ + -I /usr/local/include/ \ + -I /usr/include/ \ + -I ${PYTHON_INC} \ + -I ${PYTHON_PLATINC} \ + -I . \ + -I hpy/devel/include/ \ + -I hpy/devel/include/hpy/ \ + -I hpy/devel/include/hpy/cpython/ \ + -I hpy/devel/include/hpy/universal/ \ + -I hpy/devel/include/hpy/runtime/ \ + -I hpy/universal/src/ \ + -I hpy/debug/src/ \ + -I hpy/debug/src/include \ + -I hpy/trace/src/ \ + -I hpy/trace/src/include \ + --force \ + -D NULL=0 \ + -D HPY_ABI_CPYTHON \ + -D __linux__=1 \ + -D __x86_64__=1 \ + -D __LP64__=1 \ + . + +infer: + python3 setup.py build_ext -if -U NDEBUG | compiledb + # see commit cd8cd6e for why we need to ignore debug_ctx.c + @infer --fail-on-issue --compilation-database compile_commands.json --report-blacklist-path-regex "hpy/debug/src/debug_ctx.c" + +valgrind_args = --suppressions=hpy/tools/valgrind/python.supp --suppressions=hpy/tools/valgrind/hpy.supp --leak-check=full --show-leak-kinds=definite,indirect --log-file=/tmp/valgrind-output +python_args = -m pytest --valgrind --valgrind-log=/tmp/valgrind-output + +.PHONY: valgrind +valgrind: +ifeq ($(HPY_TEST_PORTION),) + PYTHONMALLOC=malloc valgrind $(valgrind_args) python3 $(python_args) test/ +else + PYTHONMALLOC=malloc valgrind $(valgrind_args) python3 $(python_args) --portion $(HPY_TEST_PORTION) test/ +endif + +porting-example-tests: + cd docs/porting-example/steps && python3 setup00.py build_ext -i + cd docs/porting-example/steps && python3 setup01.py build_ext -i + cd docs/porting-example/steps && python3 setup02.py build_ext -i + cd docs/porting-example/steps && python3 setup03.py --hpy-abi=universal build_ext -i + python3 -m pytest docs/porting-example/steps/ ${TEST_ARGS} + +docs-examples-tests: + cd docs/examples/simple-example && python3 setup.py --hpy-abi=universal install + cd docs/examples/mixed-example && python3 setup.py install + cd docs/examples/snippets && python3 setup.py --hpy-abi=universal install + cd docs/examples/quickstart && python3 setup.py --hpy-abi=universal install + cd docs/examples/hpytype-example && python3 setup.py --hpy-abi=universal install + python3 -m pytest docs/examples/tests.py ${TEST_ARGS} diff --git a/graalpython/hpy/README-gdb.md b/graalpython/hpy/README-gdb.md new file mode 100644 index 0000000000..c1bacca196 --- /dev/null +++ b/graalpython/hpy/README-gdb.md @@ -0,0 +1,207 @@ +How to debug HPy on CPython +============================ + +This document describes how to make debugging easier when running HPy on +CPython. At the moment it is structured as a collection of notes, PRs to make +it more structured are welcome. + +**NOTE**: this document is **not** about the HPy debug mode, but it's about +debugging HPy itself. + + +Build a debug version of CPython +--------------------------------- + +This is highly recommended. It takes only few minutes and helps a lot during +debugging for two reasons: + +1. You can easily view CPython's source code inside GDB + +2. CPython is compiled with `-Og`, which means it will be easier to follow the + code step-by-step and to inspect variables + +3. The `py-*` GDB commands work out of the box. + +``` +$ cd /path/to/cpython +$ git checkout v3.8.2 +$ ./configure --with-pydebug --prefix=/opt/python-debug/ +$ make install +``` + +Enable `python-gdb.py` +---------------------- + +`python-gdb.py` is a GDB script to make it easier to inspect the state of a +Python process from whitin GDB. It is documented +[in the CPython's dev guide](https://devguide.python.org/gdb/). + +The script add a series of GBD commands such as: + + - `py-bt`: prints the Python-level traceback of the current function + + - `py-up`, `py-down`: navigate up and down the Python function stack by + going to the previous/next occurrence of `_PyEval_EvalFrameDefault`. They + are more or less equivalent to `up` and `down` inside `pdb`. + + - `py-print`: print the value of a Python-level variable + +**WARNING**: the CPython's dev guide suggests to add `add-auto-load-safe-path` +to your `~/.gdbinit`, but it doesn't work for me. What works for me is the +following: + +``` +# add this to your ~/.gdbinit +source /path/to/cpython/python-gdb.py +``` + +To check that the `py-*` commands work as expected, you can do the following: + +``` +$ gdb --args /opt/python-debug/bin/python3 -c 'for i in range(10000000): pass' +GNU gdb (Ubuntu 9.2-0ubuntu1~20.04) 9.2 +... +(gdb) run # start the python process +... + +^C +... +(gdb) py-bt +Traceback (most recent call first): + File "", line 1, in +(gdb) py-print i +global 'i' = 9657683 +``` + +Inspect `PyObject *` and `HPy` inside GDB +------------------------------------------ + +**WARNING**: `py-dump` and `hpy-dump` prints to stderr, and this interacts +badly with pytest's capturing. Make sure to run `py.test -s`, else you might +not see the output. The included script `gdb-py.test` automatically pass `-s`. + +`python-gdb.py` installs a GDB pretty-printer for `PyObject *` variables, +which sometimes can be confusing: often, it prints the Python repr of the +variable: + +``` +(gdb) set $x = PyLong_FromLong(42) +(gdb) p $x +$5 = 42 +(gdb) p (void*)$x +$7 = (void *) 0x555555903ca0 +(gdb) p *$x +$6 = {ob_refcnt = 10, ob_type = 0x5555558d5560 } +``` + +For some reason which is unknown to me, sometimes it prints a rather obscure +string: here, `type` is the Python type of the object, and `0x5555558ce88` is +its address: + +``` +(gdb) p PyExc_ValueError +$12 = +(gdb) p (void*)PyExc_ValueError +$13 = (void *) 0x5555558ce880 <_PyExc_ValueError> +``` + +Another useful trick is to define these two custom GDB commands in your +`~/.gdbinit`: + +``` +# put this in your ~/.gdbinit +define py-dump +call _PyObject_Dump($arg0) +end + +# NOTE: +# 1. this assumes that you have a variable called "ctx" available +# 2. this assumes that it's a debug version of HPy, which was compiled with +# -fkeep-inline-functions +# 3. if you don't have _HPy_Dump available, you can call manually +# ctx->ctx_Dump (but only in universal mode) +define hpy-dump +call _HPy_Dump(ctx, $arg0) +end +``` + +Example of usage: + +``` +(gdb) set $x = PyLong_FromLong(42) +(gdb) py-dump $x +object address : 0x555555903ca0 +object refcount : 11 +object type : 0x5555558d5560 +object type name: int +object repr : 42 + +(gdb) py-dump PyExc_ValueError +object address : 0x5555558ce880 +object refcount : 14 +object type : 0x5555558dd8e0 +object type name: type +object repr : + +(gdb) set $h = ctx->ctx_Long_FromLong(ctx, 1234) +(gdb) p $h +$2 = {_i = 77} +(gdb) hpy-dump $h +object address : 0x7ffff64b0580 +object refcount : 1 +object type : 0x5555558d5560 +object type name: int +object repr : 1234 +``` + + + +Create a venv for python-debug and install hpy +----------------------------------------------- + +The following commands create a `venv` based on the newly built +`python-debug`, and installs `hpy` in "editable mode". + +``` +$ cd /path/to/hpy +$ /opt/python-debug/bin/python3 -m venv venv/hpy-debug +$ . venv/hpy-debug/bin/activate +$ python setup.py develop +``` + +Once hpy is installed this way, you can edit the C files and rebuild it easily +by using `make`: + +``` +$ make # build normally +$ make debug # build HPY with -O0 -g +``` + +Run a specific HPy test under GDB +--------------------------------- + +To run `py.test` under GDB, use the script `./gdb-py.test`. + +**Tip:** most HPy tests export a Python function called `f`. You can easily +put a breakpoint into it by using `b f_impl`: + +``` +$ ./gdb-py.test -s test/test_00_basic.py -k 'test_float_asdouble and universal' +... +(gdb) b f_impl +Breakpoint 1 at 0x7ffff63a0284: file /tmp/pytest-of-antocuni/pytest-219/test_float_asdouble_universal_0/mytest.c, line 5. +(gdb) run +... +Breakpoint 1, f_impl (ctx=0x7ffff64191d0, self=..., arg=...) at /tmp/pytest-of-antocuni/pytest-220/test_float_asdouble_universal_0/mytest.c:5 +5 { +(gdb) next +6 double a = HPyFloat_AsDouble(ctx, arg); +(gdb) p arg +$1 = {_i = 75} +(gdb) hpy-dump arg +object address : 0x7ffff7017700 +object refcount : 3 +object type : 0x5555558d3400 +object type name: float +object repr : 1.0 +``` diff --git a/graalpython/hpy/README.md b/graalpython/hpy/README.md new file mode 100644 index 0000000000..311199e6bc --- /dev/null +++ b/graalpython/hpy/README.md @@ -0,0 +1,81 @@ +# HPy: a better API for Python + +[![Build](https://github.com/hpyproject/hpy/actions/workflows/ci.yml/badge.svg)](https://github.com/hpyproject/hpy/actions/workflows/ci.yml) +[![Documentation](https://readthedocs.org/projects/hpy/badge/)](https://hpy.readthedocs.io/) +[![Join the discord server at https://discord.gg/xSzxUbPkTQ](https://img.shields.io/discord/1077164940906995813.svg?color=7389D8&labelColor=6A7EC2&logo=discord&logoColor=ffffff&style=flat-square)](https://discord.gg/xSzxUbPkTQ) + +**Website**: [hpyproject.org](https://hpyproject.org/) \ +**Community**: [HPy Discord server](https://discord.gg/xSzxUbPkTQ) \ +**Mailing list**: [hpy-dev@python.org](https://mail.python.org/mailman3/lists/hpy-dev.python.org/) + +## Summary + +HPy is a better API for extending Python +in C. The old C API is specific to the current implementation of CPython. +It exposes a lot of internal details which makes it hard to: + + - implement it for other Python implementations (e.g. PyPy, GraalPy, + Jython, IronPython, etc.). + - experiment with new things inside CPython itself: e.g. using a GC + instead of refcounting, or to remove the GIL + - guarantee binary stability + +HPy is a specification of a new API and ABI for extending Python that is +Python implementation agnostic and designed to hide and abstract internal +details such that it: + + - can stay binary compatible even if the underlying Python internals change significantly + - does not hinder internal progress of CPython and other Pythons + +Please read the [documentation](https://docs.hpyproject.org/en/latest/overview.html) +for more details on HPy motivation, goals, and features, for example: + + - debug mode for better developer experience + - support for incremental porting from CPython API to HPy + - CPython ABI for raw performance on CPython + - and others + +Do you want to see how HPy API looks in code? Check out +our [quickstart example](https://docs.hpyproject.org/en/latest/quickstart.html). + +You may also be interested in HPy's +[API reference](https://docs.hpyproject.org/en/latest/api-reference/index.html). + +This repository contains the API and ABI specification and implementation +for the CPython interpreter. Other interpreters that support HPy natively: GraalPy +and PyPy, provide their own builtin HPy implementations. + + +## Why should I care about this stuff? + + - the existing C API is becoming a problem for CPython and for the + evolution of the language itself: this project makes it possible to make + experiments which might be "officially" adopted in the future + + - for PyPy, it will give obvious speed benefits: for example, data + scientists will be able to get the benefit of fast C libraries *and* fast + Python code at the same time, something which is hard to achieve now + + - the current implementation is too tied to CPython and proved to be a + problem for almost all the other alternative implementations. Having an + API which is designed to work well on two different implementations will + make the job much easier for future ones: going from 2 to N is much easier + than going from 1 to 2 + + - arguably, it will be easier to learn and understand than the massive + CPython C API + +See also [Python Performance: Past, Present, +Future](https://github.com/vstinner/talks/raw/main/2019-EuroPython/python_performance.pdf) +by Victor Stinner. + + +## What does `HPy` mean? + +The "H" in `HPy` stands for "handle": one of the key idea of the new API is to +use fully opaque handles to represent and pass around Python objects. + + +## Donate to HPy + +Become a financial contributor and help us sustain the HPy community: [Contribute to HPy](https://opencollective.com/hpy/contribute). diff --git a/graalpython/hpy/c_test/Makefile b/graalpython/hpy/c_test/Makefile new file mode 100644 index 0000000000..2e93170d4c --- /dev/null +++ b/graalpython/hpy/c_test/Makefile @@ -0,0 +1,16 @@ +CC ?= gcc +INCLUDE=-I.. -I../hpy/devel/include -I../hpy/debug/src/include +CFLAGS = -O0 -UNDEBUG -g -Wall -Werror -Wfatal-errors $(INCLUDE) -DHPY_ABI_UNIVERSAL + +test: test_debug_handles test_stacktrace + ./test_debug_handles + ./test_stacktrace + +test_debug_handles: test_debug_handles.o ../hpy/debug/src/dhqueue.o + $(CC) -o $@ $^ + +test_stacktrace: test_stacktrace.o ../hpy/debug/src/stacktrace.o + $(CC) -o $@ $^ + +%.o : %.c + $(CC) -c $(CFLAGS) $< -o $@ diff --git a/graalpython/hpy/c_test/acutest.h b/graalpython/hpy/c_test/acutest.h new file mode 100644 index 0000000000..a05eb41cd0 --- /dev/null +++ b/graalpython/hpy/c_test/acutest.h @@ -0,0 +1,1794 @@ +/* + * Acutest -- Another C/C++ Unit Test facility + * + * + * Copyright 2013-2020 Martin Mitas + * Copyright 2019 Garrett D'Amore + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#ifndef ACUTEST_H +#define ACUTEST_H + + +/************************ + *** Public interface *** + ************************/ + +/* By default, "acutest.h" provides the main program entry point (function + * main()). However, if the test suite is composed of multiple source files + * which include "acutest.h", then this causes a problem of multiple main() + * definitions. To avoid this problem, #define macro TEST_NO_MAIN in all + * compilation units but one. + */ + +/* Macro to specify list of unit tests in the suite. + * The unit test implementation MUST provide list of unit tests it implements + * with this macro: + * + * TEST_LIST = { + * { "test1_name", test1_func_ptr }, + * { "test2_name", test2_func_ptr }, + * ... + * { NULL, NULL } // zeroed record marking the end of the list + * }; + * + * The list specifies names of each test (must be unique) and pointer to + * a function implementing it. The function does not take any arguments + * and has no return values, i.e. every test function has to be compatible + * with this prototype: + * + * void test_func(void); + * + * Note the list has to be ended with a zeroed record. + */ +#define TEST_LIST const struct acutest_test_ acutest_list_[] + + +/* Macros for testing whether an unit test succeeds or fails. These macros + * can be used arbitrarily in functions implementing the unit tests. + * + * If any condition fails throughout execution of a test, the test fails. + * + * TEST_CHECK takes only one argument (the condition), TEST_CHECK_ allows + * also to specify an error message to print out if the condition fails. + * (It expects printf-like format string and its parameters). The macros + * return non-zero (condition passes) or 0 (condition fails). + * + * That can be useful when more conditions should be checked only if some + * preceding condition passes, as illustrated in this code snippet: + * + * SomeStruct* ptr = allocate_some_struct(); + * if(TEST_CHECK(ptr != NULL)) { + * TEST_CHECK(ptr->member1 < 100); + * TEST_CHECK(ptr->member2 > 200); + * } + */ +#define TEST_CHECK_(cond,...) acutest_check_((cond), __FILE__, __LINE__, __VA_ARGS__) +#define TEST_CHECK(cond) acutest_check_((cond), __FILE__, __LINE__, "%s", #cond) + + +/* These macros are the same as TEST_CHECK_ and TEST_CHECK except that if the + * condition fails, the currently executed unit test is immediately aborted. + * + * That is done either by calling abort() if the unit test is executed as a + * child process; or via longjmp() if the unit test is executed within the + * main Acutest process. + * + * As a side effect of such abortion, your unit tests may cause memory leaks, + * unflushed file descriptors, and other phenomena caused by the abortion. + * + * Therefore you should not use these as a general replacement for TEST_CHECK. + * Use it with some caution, especially if your test causes some other side + * effects to the outside world (e.g. communicating with some server, inserting + * into a database etc.). + */ +#define TEST_ASSERT_(cond,...) \ + do { \ + if(!acutest_check_((cond), __FILE__, __LINE__, __VA_ARGS__)) \ + acutest_abort_(); \ + } while(0) +#define TEST_ASSERT(cond) \ + do { \ + if(!acutest_check_((cond), __FILE__, __LINE__, "%s", #cond)) \ + acutest_abort_(); \ + } while(0) + + +#ifdef __cplusplus +/* Macros to verify that the code (the 1st argument) throws exception of given + * type (the 2nd argument). (Note these macros are only available in C++.) + * + * TEST_EXCEPTION_ is like TEST_EXCEPTION but accepts custom printf-like + * message. + * + * For example: + * + * TEST_EXCEPTION(function_that_throw(), ExpectedExceptionType); + * + * If the function_that_throw() throws ExpectedExceptionType, the check passes. + * If the function throws anything incompatible with ExpectedExceptionType + * (or if it does not thrown an exception at all), the check fails. + */ +#define TEST_EXCEPTION(code, exctype) \ + do { \ + bool exc_ok_ = false; \ + const char *msg_ = NULL; \ + try { \ + code; \ + msg_ = "No exception thrown."; \ + } catch(exctype const&) { \ + exc_ok_= true; \ + } catch(...) { \ + msg_ = "Unexpected exception thrown."; \ + } \ + acutest_check_(exc_ok_, __FILE__, __LINE__, #code " throws " #exctype);\ + if(msg_ != NULL) \ + acutest_message_("%s", msg_); \ + } while(0) +#define TEST_EXCEPTION_(code, exctype, ...) \ + do { \ + bool exc_ok_ = false; \ + const char *msg_ = NULL; \ + try { \ + code; \ + msg_ = "No exception thrown."; \ + } catch(exctype const&) { \ + exc_ok_= true; \ + } catch(...) { \ + msg_ = "Unexpected exception thrown."; \ + } \ + acutest_check_(exc_ok_, __FILE__, __LINE__, __VA_ARGS__); \ + if(msg_ != NULL) \ + acutest_message_("%s", msg_); \ + } while(0) +#endif /* #ifdef __cplusplus */ + + +/* Sometimes it is useful to split execution of more complex unit tests to some + * smaller parts and associate those parts with some names. + * + * This is especially handy if the given unit test is implemented as a loop + * over some vector of multiple testing inputs. Using these macros allow to use + * sort of subtitle for each iteration of the loop (e.g. outputting the input + * itself or a name associated to it), so that if any TEST_CHECK condition + * fails in the loop, it can be easily seen which iteration triggers the + * failure, without the need to manually output the iteration-specific data in + * every single TEST_CHECK inside the loop body. + * + * TEST_CASE allows to specify only single string as the name of the case, + * TEST_CASE_ provides all the power of printf-like string formatting. + * + * Note that the test cases cannot be nested. Starting a new test case ends + * implicitly the previous one. To end the test case explicitly (e.g. to end + * the last test case after exiting the loop), you may use TEST_CASE(NULL). + */ +#define TEST_CASE_(...) acutest_case_(__VA_ARGS__) +#define TEST_CASE(name) acutest_case_("%s", name) + + +/* Maximal output per TEST_CASE call. Longer messages are cut. + * You may define another limit prior including "acutest.h" + */ +#ifndef TEST_CASE_MAXSIZE + #define TEST_CASE_MAXSIZE 64 +#endif + + +/* printf-like macro for outputting an extra information about a failure. + * + * Intended use is to output some computed output versus the expected value, + * e.g. like this: + * + * if(!TEST_CHECK(produced == expected)) { + * TEST_MSG("Expected: %d", expected); + * TEST_MSG("Produced: %d", produced); + * } + * + * Note the message is only written down if the most recent use of any checking + * macro (like e.g. TEST_CHECK or TEST_EXCEPTION) in the current test failed. + * This means the above is equivalent to just this: + * + * TEST_CHECK(produced == expected); + * TEST_MSG("Expected: %d", expected); + * TEST_MSG("Produced: %d", produced); + * + * The macro can deal with multi-line output fairly well. It also automatically + * adds a final new-line if there is none present. + */ +#define TEST_MSG(...) acutest_message_(__VA_ARGS__) + + +/* Maximal output per TEST_MSG call. Longer messages are cut. + * You may define another limit prior including "acutest.h" + */ +#ifndef TEST_MSG_MAXSIZE + #define TEST_MSG_MAXSIZE 1024 +#endif + + +/* Macro for dumping a block of memory. + * + * Its intended use is very similar to what TEST_MSG is for, but instead of + * generating any printf-like message, this is for dumping raw block of a + * memory in a hexadecimal form: + * + * TEST_CHECK(size_produced == size_expected && + * memcmp(addr_produced, addr_expected, size_produced) == 0); + * TEST_DUMP("Expected:", addr_expected, size_expected); + * TEST_DUMP("Produced:", addr_produced, size_produced); + */ +#define TEST_DUMP(title, addr, size) acutest_dump_(title, addr, size) + +/* Maximal output per TEST_DUMP call (in bytes to dump). Longer blocks are cut. + * You may define another limit prior including "acutest.h" + */ +#ifndef TEST_DUMP_MAXSIZE + #define TEST_DUMP_MAXSIZE 1024 +#endif + + +/* Common test initialization/clean-up + * + * In some test suites, it may be needed to perform some sort of the same + * initialization and/or clean-up in all the tests. + * + * Such test suites may use macros TEST_INIT and/or TEST_FINI prior including + * this header. The expansion of the macro is then used as a body of helper + * function called just before executing every single (TEST_INIT) or just after + * it ends (TEST_FINI). + * + * Examples of various ways how to use the macro TEST_INIT: + * + * #define TEST_INIT my_init_func(); + * #define TEST_INIT my_init_func() // Works even without the semicolon + * #define TEST_INIT setlocale(LC_ALL, NULL); + * #define TEST_INIT { setlocale(LC_ALL, NULL); my_init_func(); } + * + * TEST_FINI is to be used in the same way. + */ + + +/********************** + *** Implementation *** + **********************/ + +/* The unit test files should not rely on anything below. */ + +#include +#include +#include +#include +#include +#include + +#if defined(unix) || defined(__unix__) || defined(__unix) || defined(__APPLE__) + #define ACUTEST_UNIX_ 1 + #include + #include + #include + #include + #include + #include + #include + + #if defined CLOCK_PROCESS_CPUTIME_ID && defined CLOCK_MONOTONIC + #define ACUTEST_HAS_POSIX_TIMER_ 1 + #endif +#endif + +#if defined(_gnu_linux_) || defined(__linux__) + #define ACUTEST_LINUX_ 1 + #include + #include +#endif + +#if defined(_WIN32) || defined(__WIN32__) || defined(__WINDOWS__) + #define ACUTEST_WIN_ 1 + #include + #include +#endif + +#ifdef __cplusplus + #include +#endif + +#ifdef __has_include + #if __has_include() + #include + #endif +#endif + +/* Enable the use of the non-standard keyword __attribute__ to silence warnings under some compilers */ +#if defined(__GNUC__) || defined(__clang__) + #define ACUTEST_ATTRIBUTE_(attr) __attribute__((attr)) +#else + #define ACUTEST_ATTRIBUTE_(attr) +#endif + +/* Note our global private identifiers end with '_' to mitigate risk of clash + * with the unit tests implementation. */ + +#ifdef __cplusplus + extern "C" { +#endif + +#ifdef _MSC_VER + /* In the multi-platform code like ours, we cannot use the non-standard + * "safe" functions from Microsoft C lib like e.g. sprintf_s() instead of + * standard sprintf(). Hence, lets disable the warning C4996. */ + #pragma warning(push) + #pragma warning(disable: 4996) +#endif + + +struct acutest_test_ { + const char* name; + void (*func)(void); +}; + +struct acutest_test_data_ { + unsigned char flags; + double duration; +}; + +enum { + ACUTEST_FLAG_RUN_ = 1 << 0, + ACUTEST_FLAG_SUCCESS_ = 1 << 1, + ACUTEST_FLAG_FAILURE_ = 1 << 2, +}; + +extern const struct acutest_test_ acutest_list_[]; + +int acutest_check_(int cond, const char* file, int line, const char* fmt, ...); +void acutest_case_(const char* fmt, ...); +void acutest_message_(const char* fmt, ...); +void acutest_dump_(const char* title, const void* addr, size_t size); +void acutest_abort_(void) ACUTEST_ATTRIBUTE_(noreturn); + + +#ifndef TEST_NO_MAIN + +static char* acutest_argv0_ = NULL; +static size_t acutest_list_size_ = 0; +static struct acutest_test_data_* acutest_test_data_ = NULL; +static size_t acutest_count_ = 0; +static int acutest_no_exec_ = -1; +static int acutest_no_summary_ = 0; +static int acutest_tap_ = 0; +static int acutest_skip_mode_ = 0; +static int acutest_worker_ = 0; +static int acutest_worker_index_ = 0; +static int acutest_cond_failed_ = 0; +static int acutest_was_aborted_ = 0; +static FILE *acutest_xml_output_ = NULL; + +static int acutest_stat_failed_units_ = 0; +static int acutest_stat_run_units_ = 0; + +static const struct acutest_test_* acutest_current_test_ = NULL; +static int acutest_current_index_ = 0; +static char acutest_case_name_[TEST_CASE_MAXSIZE] = ""; +static int acutest_test_already_logged_ = 0; +static int acutest_case_already_logged_ = 0; +static int acutest_verbose_level_ = 2; +static int acutest_test_failures_ = 0; +static int acutest_colorize_ = 0; +static int acutest_timer_ = 0; + +static int acutest_abort_has_jmp_buf_ = 0; +static jmp_buf acutest_abort_jmp_buf_; + + +static void +acutest_cleanup_(void) +{ + free((void*) acutest_test_data_); +} + +static void ACUTEST_ATTRIBUTE_(noreturn) +acutest_exit_(int exit_code) +{ + acutest_cleanup_(); + exit(exit_code); +} + +#if defined ACUTEST_WIN_ + typedef LARGE_INTEGER acutest_timer_type_; + static LARGE_INTEGER acutest_timer_freq_; + static acutest_timer_type_ acutest_timer_start_; + static acutest_timer_type_ acutest_timer_end_; + + static void + acutest_timer_init_(void) + { + QueryPerformanceFrequency(´st_timer_freq_); + } + + static void + acutest_timer_get_time_(LARGE_INTEGER* ts) + { + QueryPerformanceCounter(ts); + } + + static double + acutest_timer_diff_(LARGE_INTEGER start, LARGE_INTEGER end) + { + double duration = (double)(end.QuadPart - start.QuadPart); + duration /= (double)acutest_timer_freq_.QuadPart; + return duration; + } + + static void + acutest_timer_print_diff_(void) + { + printf("%.6lf secs", acutest_timer_diff_(acutest_timer_start_, acutest_timer_end_)); + } +#elif defined ACUTEST_HAS_POSIX_TIMER_ + static clockid_t acutest_timer_id_; + typedef struct timespec acutest_timer_type_; + static acutest_timer_type_ acutest_timer_start_; + static acutest_timer_type_ acutest_timer_end_; + + static void + acutest_timer_init_(void) + { + if(acutest_timer_ == 1) + acutest_timer_id_ = CLOCK_MONOTONIC; + else if(acutest_timer_ == 2) + acutest_timer_id_ = CLOCK_PROCESS_CPUTIME_ID; + } + + static void + acutest_timer_get_time_(struct timespec* ts) + { + clock_gettime(acutest_timer_id_, ts); + } + + static double + acutest_timer_diff_(struct timespec start, struct timespec end) + { + double endns; + double startns; + + endns = end.tv_sec; + endns *= 1e9; + endns += end.tv_nsec; + + startns = start.tv_sec; + startns *= 1e9; + startns += start.tv_nsec; + + return ((endns - startns)/ 1e9); + } + + static void + acutest_timer_print_diff_(void) + { + printf("%.6lf secs", + acutest_timer_diff_(acutest_timer_start_, acutest_timer_end_)); + } +#else + typedef int acutest_timer_type_; + static acutest_timer_type_ acutest_timer_start_; + static acutest_timer_type_ acutest_timer_end_; + + void + acutest_timer_init_(void) + {} + + static void + acutest_timer_get_time_(int* ts) + { + (void) ts; + } + + static double + acutest_timer_diff_(int start, int end) + { + (void) start; + (void) end; + return 0.0; + } + + static void + acutest_timer_print_diff_(void) + {} +#endif + +#define ACUTEST_COLOR_DEFAULT_ 0 +#define ACUTEST_COLOR_GREEN_ 1 +#define ACUTEST_COLOR_RED_ 2 +#define ACUTEST_COLOR_DEFAULT_INTENSIVE_ 3 +#define ACUTEST_COLOR_GREEN_INTENSIVE_ 4 +#define ACUTEST_COLOR_RED_INTENSIVE_ 5 + +static int ACUTEST_ATTRIBUTE_(format (printf, 2, 3)) +acutest_colored_printf_(int color, const char* fmt, ...) +{ + va_list args; + char buffer[256]; + int n; + + va_start(args, fmt); + vsnprintf(buffer, sizeof(buffer), fmt, args); + va_end(args); + buffer[sizeof(buffer)-1] = '\0'; + + if(!acutest_colorize_) { + return printf("%s", buffer); + } + +#if defined ACUTEST_UNIX_ + { + const char* col_str; + switch(color) { + case ACUTEST_COLOR_GREEN_: col_str = "\033[0;32m"; break; + case ACUTEST_COLOR_RED_: col_str = "\033[0;31m"; break; + case ACUTEST_COLOR_GREEN_INTENSIVE_: col_str = "\033[1;32m"; break; + case ACUTEST_COLOR_RED_INTENSIVE_: col_str = "\033[1;31m"; break; + case ACUTEST_COLOR_DEFAULT_INTENSIVE_: col_str = "\033[1m"; break; + default: col_str = "\033[0m"; break; + } + printf("%s", col_str); + n = printf("%s", buffer); + printf("\033[0m"); + return n; + } +#elif defined ACUTEST_WIN_ + { + HANDLE h; + CONSOLE_SCREEN_BUFFER_INFO info; + WORD attr; + + h = GetStdHandle(STD_OUTPUT_HANDLE); + GetConsoleScreenBufferInfo(h, &info); + + switch(color) { + case ACUTEST_COLOR_GREEN_: attr = FOREGROUND_GREEN; break; + case ACUTEST_COLOR_RED_: attr = FOREGROUND_RED; break; + case ACUTEST_COLOR_GREEN_INTENSIVE_: attr = FOREGROUND_GREEN | FOREGROUND_INTENSITY; break; + case ACUTEST_COLOR_RED_INTENSIVE_: attr = FOREGROUND_RED | FOREGROUND_INTENSITY; break; + case ACUTEST_COLOR_DEFAULT_INTENSIVE_: attr = FOREGROUND_BLUE | FOREGROUND_GREEN | FOREGROUND_RED | FOREGROUND_INTENSITY; break; + default: attr = 0; break; + } + if(attr != 0) + SetConsoleTextAttribute(h, attr); + n = printf("%s", buffer); + SetConsoleTextAttribute(h, info.wAttributes); + return n; + } +#else + n = printf("%s", buffer); + return n; +#endif +} + +static void +acutest_begin_test_line_(const struct acutest_test_* test) +{ + if(!acutest_tap_) { + if(acutest_verbose_level_ >= 3) { + acutest_colored_printf_(ACUTEST_COLOR_DEFAULT_INTENSIVE_, "Test %s:\n", test->name); + acutest_test_already_logged_++; + } else if(acutest_verbose_level_ >= 1) { + int n; + char spaces[48]; + + n = acutest_colored_printf_(ACUTEST_COLOR_DEFAULT_INTENSIVE_, "Test %s... ", test->name); + memset(spaces, ' ', sizeof(spaces)); + if(n < (int) sizeof(spaces)) + printf("%.*s", (int) sizeof(spaces) - n, spaces); + } else { + acutest_test_already_logged_ = 1; + } + } +} + +static void +acutest_finish_test_line_(int result) +{ + if(acutest_tap_) { + const char* str = (result == 0) ? "ok" : "not ok"; + + printf("%s %d - %s\n", str, acutest_current_index_ + 1, acutest_current_test_->name); + + if(result == 0 && acutest_timer_) { + printf("# Duration: "); + acutest_timer_print_diff_(); + printf("\n"); + } + } else { + int color = (result == 0) ? ACUTEST_COLOR_GREEN_INTENSIVE_ : ACUTEST_COLOR_RED_INTENSIVE_; + const char* str = (result == 0) ? "OK" : "FAILED"; + printf("[ "); + acutest_colored_printf_(color, "%s", str); + printf(" ]"); + + if(result == 0 && acutest_timer_) { + printf(" "); + acutest_timer_print_diff_(); + } + + printf("\n"); + } +} + +static void +acutest_line_indent_(int level) +{ + static const char spaces[] = " "; + int n = level * 2; + + if(acutest_tap_ && n > 0) { + n--; + printf("#"); + } + + while(n > 16) { + printf("%s", spaces); + n -= 16; + } + printf("%.*s", n, spaces); +} + +int ACUTEST_ATTRIBUTE_(format (printf, 4, 5)) +acutest_check_(int cond, const char* file, int line, const char* fmt, ...) +{ + const char *result_str; + int result_color; + int verbose_level; + + if(cond) { + result_str = "ok"; + result_color = ACUTEST_COLOR_GREEN_; + verbose_level = 3; + } else { + if(!acutest_test_already_logged_ && acutest_current_test_ != NULL) + acutest_finish_test_line_(-1); + + result_str = "failed"; + result_color = ACUTEST_COLOR_RED_; + verbose_level = 2; + acutest_test_failures_++; + acutest_test_already_logged_++; + } + + if(acutest_verbose_level_ >= verbose_level) { + va_list args; + + if(!acutest_case_already_logged_ && acutest_case_name_[0]) { + acutest_line_indent_(1); + acutest_colored_printf_(ACUTEST_COLOR_DEFAULT_INTENSIVE_, "Case %s:\n", acutest_case_name_); + acutest_test_already_logged_++; + acutest_case_already_logged_++; + } + + acutest_line_indent_(acutest_case_name_[0] ? 2 : 1); + if(file != NULL) { +#ifdef ACUTEST_WIN_ + const char* lastsep1 = strrchr(file, '\\'); + const char* lastsep2 = strrchr(file, '/'); + if(lastsep1 == NULL) + lastsep1 = file-1; + if(lastsep2 == NULL) + lastsep2 = file-1; + file = (lastsep1 > lastsep2 ? lastsep1 : lastsep2) + 1; +#else + const char* lastsep = strrchr(file, '/'); + if(lastsep != NULL) + file = lastsep+1; +#endif + printf("%s:%d: Check ", file, line); + } + + va_start(args, fmt); + vprintf(fmt, args); + va_end(args); + + printf("... "); + acutest_colored_printf_(result_color, "%s", result_str); + printf("\n"); + acutest_test_already_logged_++; + } + + acutest_cond_failed_ = (cond == 0); + return !acutest_cond_failed_; +} + +void ACUTEST_ATTRIBUTE_(format (printf, 1, 2)) +acutest_case_(const char* fmt, ...) +{ + va_list args; + + if(acutest_verbose_level_ < 2) + return; + + if(acutest_case_name_[0]) { + acutest_case_already_logged_ = 0; + acutest_case_name_[0] = '\0'; + } + + if(fmt == NULL) + return; + + va_start(args, fmt); + vsnprintf(acutest_case_name_, sizeof(acutest_case_name_) - 1, fmt, args); + va_end(args); + acutest_case_name_[sizeof(acutest_case_name_) - 1] = '\0'; + + if(acutest_verbose_level_ >= 3) { + acutest_line_indent_(1); + acutest_colored_printf_(ACUTEST_COLOR_DEFAULT_INTENSIVE_, "Case %s:\n", acutest_case_name_); + acutest_test_already_logged_++; + acutest_case_already_logged_++; + } +} + +void ACUTEST_ATTRIBUTE_(format (printf, 1, 2)) +acutest_message_(const char* fmt, ...) +{ + char buffer[TEST_MSG_MAXSIZE]; + char* line_beg; + char* line_end; + va_list args; + + if(acutest_verbose_level_ < 2) + return; + + /* We allow extra message only when something is already wrong in the + * current test. */ + if(acutest_current_test_ == NULL || !acutest_cond_failed_) + return; + + va_start(args, fmt); + vsnprintf(buffer, TEST_MSG_MAXSIZE, fmt, args); + va_end(args); + buffer[TEST_MSG_MAXSIZE-1] = '\0'; + + line_beg = buffer; + while(1) { + line_end = strchr(line_beg, '\n'); + if(line_end == NULL) + break; + acutest_line_indent_(acutest_case_name_[0] ? 3 : 2); + printf("%.*s\n", (int)(line_end - line_beg), line_beg); + line_beg = line_end + 1; + } + if(line_beg[0] != '\0') { + acutest_line_indent_(acutest_case_name_[0] ? 3 : 2); + printf("%s\n", line_beg); + } +} + +void +acutest_dump_(const char* title, const void* addr, size_t size) +{ + static const size_t BYTES_PER_LINE = 16; + size_t line_beg; + size_t truncate = 0; + + if(acutest_verbose_level_ < 2) + return; + + /* We allow extra message only when something is already wrong in the + * current test. */ + if(acutest_current_test_ == NULL || !acutest_cond_failed_) + return; + + if(size > TEST_DUMP_MAXSIZE) { + truncate = size - TEST_DUMP_MAXSIZE; + size = TEST_DUMP_MAXSIZE; + } + + acutest_line_indent_(acutest_case_name_[0] ? 3 : 2); + printf((title[strlen(title)-1] == ':') ? "%s\n" : "%s:\n", title); + + for(line_beg = 0; line_beg < size; line_beg += BYTES_PER_LINE) { + size_t line_end = line_beg + BYTES_PER_LINE; + size_t off; + + acutest_line_indent_(acutest_case_name_[0] ? 4 : 3); + printf("%08lx: ", (unsigned long)line_beg); + for(off = line_beg; off < line_end; off++) { + if(off < size) + printf(" %02x", ((const unsigned char*)addr)[off]); + else + printf(" "); + } + + printf(" "); + for(off = line_beg; off < line_end; off++) { + unsigned char byte = ((const unsigned char*)addr)[off]; + if(off < size) + printf("%c", (iscntrl(byte) ? '.' : byte)); + else + break; + } + + printf("\n"); + } + + if(truncate > 0) { + acutest_line_indent_(acutest_case_name_[0] ? 4 : 3); + printf(" ... (and more %u bytes)\n", (unsigned) truncate); + } +} + +/* This is called just before each test */ +static void +acutest_init_(const char *test_name) +{ +#ifdef TEST_INIT + TEST_INIT + ; /* Allow for a single unterminated function call */ +#endif + + /* Suppress any warnings about unused variable. */ + (void) test_name; +} + +/* This is called after each test */ +static void +acutest_fini_(const char *test_name) +{ +#ifdef TEST_FINI + TEST_FINI + ; /* Allow for a single unterminated function call */ +#endif + + /* Suppress any warnings about unused variable. */ + (void) test_name; +} + +void +acutest_abort_(void) +{ + if(acutest_abort_has_jmp_buf_) { + longjmp(acutest_abort_jmp_buf_, 1); + } else { + if(acutest_current_test_ != NULL) + acutest_fini_(acutest_current_test_->name); + abort(); + } +} + +static void +acutest_list_names_(void) +{ + const struct acutest_test_* test; + + printf("Unit tests:\n"); + for(test = ´st_list_[0]; test->func != NULL; test++) + printf(" %s\n", test->name); +} + +static void +acutest_remember_(int i) +{ + if(acutest_test_data_[i].flags & ACUTEST_FLAG_RUN_) + return; + + acutest_test_data_[i].flags |= ACUTEST_FLAG_RUN_; + acutest_count_++; +} + +static void +acutest_set_success_(int i, int success) +{ + acutest_test_data_[i].flags |= success ? ACUTEST_FLAG_SUCCESS_ : ACUTEST_FLAG_FAILURE_; +} + +static void +acutest_set_duration_(int i, double duration) +{ + acutest_test_data_[i].duration = duration; +} + +static int +acutest_name_contains_word_(const char* name, const char* pattern) +{ + static const char word_delim[] = " \t-_/.,:;"; + const char* substr; + size_t pattern_len; + + pattern_len = strlen(pattern); + + substr = strstr(name, pattern); + while(substr != NULL) { + int starts_on_word_boundary = (substr == name || strchr(word_delim, substr[-1]) != NULL); + int ends_on_word_boundary = (substr[pattern_len] == '\0' || strchr(word_delim, substr[pattern_len]) != NULL); + + if(starts_on_word_boundary && ends_on_word_boundary) + return 1; + + substr = strstr(substr+1, pattern); + } + + return 0; +} + +static int +acutest_lookup_(const char* pattern) +{ + int i; + int n = 0; + + /* Try exact match. */ + for(i = 0; i < (int) acutest_list_size_; i++) { + if(strcmp(acutest_list_[i].name, pattern) == 0) { + acutest_remember_(i); + n++; + break; + } + } + if(n > 0) + return n; + + /* Try word match. */ + for(i = 0; i < (int) acutest_list_size_; i++) { + if(acutest_name_contains_word_(acutest_list_[i].name, pattern)) { + acutest_remember_(i); + n++; + } + } + if(n > 0) + return n; + + /* Try relaxed match. */ + for(i = 0; i < (int) acutest_list_size_; i++) { + if(strstr(acutest_list_[i].name, pattern) != NULL) { + acutest_remember_(i); + n++; + } + } + + return n; +} + + +/* Called if anything goes bad in Acutest, or if the unit test ends in other + * way then by normal returning from its function (e.g. exception or some + * abnormal child process termination). */ +static void ACUTEST_ATTRIBUTE_(format (printf, 1, 2)) +acutest_error_(const char* fmt, ...) +{ + if(acutest_verbose_level_ == 0) + return; + + if(acutest_verbose_level_ >= 2) { + va_list args; + + acutest_line_indent_(1); + if(acutest_verbose_level_ >= 3) + acutest_colored_printf_(ACUTEST_COLOR_RED_INTENSIVE_, "ERROR: "); + va_start(args, fmt); + vprintf(fmt, args); + va_end(args); + printf("\n"); + } + + if(acutest_verbose_level_ >= 3) { + printf("\n"); + } +} + +/* Call directly the given test unit function. */ +static int +acutest_do_run_(const struct acutest_test_* test, int index) +{ + int status = -1; + + acutest_was_aborted_ = 0; + acutest_current_test_ = test; + acutest_current_index_ = index; + acutest_test_failures_ = 0; + acutest_test_already_logged_ = 0; + acutest_cond_failed_ = 0; + +#ifdef __cplusplus + try { +#endif + acutest_init_(test->name); + acutest_begin_test_line_(test); + + /* This is good to do in case the test unit crashes. */ + fflush(stdout); + fflush(stderr); + + if(!acutest_worker_) { + acutest_abort_has_jmp_buf_ = 1; + if(setjmp(acutest_abort_jmp_buf_) != 0) { + acutest_was_aborted_ = 1; + goto aborted; + } + } + + acutest_timer_get_time_(´st_timer_start_); + test->func(); +aborted: + acutest_abort_has_jmp_buf_ = 0; + acutest_timer_get_time_(´st_timer_end_); + + if(acutest_verbose_level_ >= 3) { + acutest_line_indent_(1); + if(acutest_test_failures_ == 0) { + acutest_colored_printf_(ACUTEST_COLOR_GREEN_INTENSIVE_, "SUCCESS: "); + printf("All conditions have passed.\n"); + + if(acutest_timer_) { + acutest_line_indent_(1); + printf("Duration: "); + acutest_timer_print_diff_(); + printf("\n"); + } + } else { + acutest_colored_printf_(ACUTEST_COLOR_RED_INTENSIVE_, "FAILED: "); + if(!acutest_was_aborted_) { + printf("%d condition%s %s failed.\n", + acutest_test_failures_, + (acutest_test_failures_ == 1) ? "" : "s", + (acutest_test_failures_ == 1) ? "has" : "have"); + } else { + printf("Aborted.\n"); + } + } + printf("\n"); + } else if(acutest_verbose_level_ >= 1 && acutest_test_failures_ == 0) { + acutest_finish_test_line_(0); + } + + status = (acutest_test_failures_ == 0) ? 0 : -1; + +#ifdef __cplusplus + } catch(std::exception& e) { + const char* what = e.what(); + acutest_check_(0, NULL, 0, "Threw std::exception"); + if(what != NULL) + acutest_message_("std::exception::what(): %s", what); + + if(acutest_verbose_level_ >= 3) { + acutest_line_indent_(1); + acutest_colored_printf_(ACUTEST_COLOR_RED_INTENSIVE_, "FAILED: "); + printf("C++ exception.\n\n"); + } + } catch(...) { + acutest_check_(0, NULL, 0, "Threw an exception"); + + if(acutest_verbose_level_ >= 3) { + acutest_line_indent_(1); + acutest_colored_printf_(ACUTEST_COLOR_RED_INTENSIVE_, "FAILED: "); + printf("C++ exception.\n\n"); + } + } +#endif + + acutest_fini_(test->name); + acutest_case_(NULL); + acutest_current_test_ = NULL; + + return status; +} + +/* Trigger the unit test. If possible (and not suppressed) it starts a child + * process who calls acutest_do_run_(), otherwise it calls acutest_do_run_() + * directly. */ +static void +acutest_run_(const struct acutest_test_* test, int index, int master_index) +{ + int failed = 1; + acutest_timer_type_ start, end; + + acutest_current_test_ = test; + acutest_test_already_logged_ = 0; + acutest_timer_get_time_(&start); + + if(!acutest_no_exec_) { + +#if defined(ACUTEST_UNIX_) + + pid_t pid; + int exit_code; + + /* Make sure the child starts with empty I/O buffers. */ + fflush(stdout); + fflush(stderr); + + pid = fork(); + if(pid == (pid_t)-1) { + acutest_error_("Cannot fork. %s [%d]", strerror(errno), errno); + failed = 1; + } else if(pid == 0) { + /* Child: Do the test. */ + acutest_worker_ = 1; + failed = (acutest_do_run_(test, index) != 0); + acutest_exit_(failed ? 1 : 0); + } else { + /* Parent: Wait until child terminates and analyze its exit code. */ + waitpid(pid, &exit_code, 0); + if(WIFEXITED(exit_code)) { + switch(WEXITSTATUS(exit_code)) { + case 0: failed = 0; break; /* test has passed. */ + case 1: /* noop */ break; /* "normal" failure. */ + default: acutest_error_("Unexpected exit code [%d]", WEXITSTATUS(exit_code)); + } + } else if(WIFSIGNALED(exit_code)) { + char tmp[32]; + const char* signame; + switch(WTERMSIG(exit_code)) { + case SIGINT: signame = "SIGINT"; break; + case SIGHUP: signame = "SIGHUP"; break; + case SIGQUIT: signame = "SIGQUIT"; break; + case SIGABRT: signame = "SIGABRT"; break; + case SIGKILL: signame = "SIGKILL"; break; + case SIGSEGV: signame = "SIGSEGV"; break; + case SIGILL: signame = "SIGILL"; break; + case SIGTERM: signame = "SIGTERM"; break; + default: sprintf(tmp, "signal %d", WTERMSIG(exit_code)); signame = tmp; break; + } + acutest_error_("Test interrupted by %s.", signame); + } else { + acutest_error_("Test ended in an unexpected way [%d].", exit_code); + } + } + +#elif defined(ACUTEST_WIN_) + + char buffer[512] = {0}; + STARTUPINFOA startupInfo; + PROCESS_INFORMATION processInfo; + DWORD exitCode; + + /* Windows has no fork(). So we propagate all info into the child + * through a command line arguments. */ + _snprintf(buffer, sizeof(buffer)-1, + "%s --worker=%d %s --no-exec --no-summary %s --verbose=%d --color=%s -- \"%s\"", + acutest_argv0_, index, acutest_timer_ ? "--time" : "", + acutest_tap_ ? "--tap" : "", acutest_verbose_level_, + acutest_colorize_ ? "always" : "never", + test->name); + memset(&startupInfo, 0, sizeof(startupInfo)); + startupInfo.cb = sizeof(STARTUPINFO); + if(CreateProcessA(NULL, buffer, NULL, NULL, FALSE, 0, NULL, NULL, &startupInfo, &processInfo)) { + WaitForSingleObject(processInfo.hProcess, INFINITE); + GetExitCodeProcess(processInfo.hProcess, &exitCode); + CloseHandle(processInfo.hThread); + CloseHandle(processInfo.hProcess); + failed = (exitCode != 0); + if(exitCode > 1) { + switch(exitCode) { + case 3: acutest_error_("Aborted."); break; + case 0xC0000005: acutest_error_("Access violation."); break; + default: acutest_error_("Test ended in an unexpected way [%lu].", exitCode); break; + } + } + } else { + acutest_error_("Cannot create unit test subprocess [%ld].", GetLastError()); + failed = 1; + } + +#else + + /* A platform where we don't know how to run child process. */ + failed = (acutest_do_run_(test, index) != 0); + +#endif + + } else { + /* Child processes suppressed through --no-exec. */ + failed = (acutest_do_run_(test, index) != 0); + } + acutest_timer_get_time_(&end); + + acutest_current_test_ = NULL; + + acutest_stat_run_units_++; + if(failed) + acutest_stat_failed_units_++; + + acutest_set_success_(master_index, !failed); + acutest_set_duration_(master_index, acutest_timer_diff_(start, end)); +} + +#if defined(ACUTEST_WIN_) +/* Callback for SEH events. */ +static LONG CALLBACK +acutest_seh_exception_filter_(EXCEPTION_POINTERS *ptrs) +{ + acutest_check_(0, NULL, 0, "Unhandled SEH exception"); + acutest_message_("Exception code: 0x%08lx", ptrs->ExceptionRecord->ExceptionCode); + acutest_message_("Exception address: 0x%p", ptrs->ExceptionRecord->ExceptionAddress); + + fflush(stdout); + fflush(stderr); + + return EXCEPTION_EXECUTE_HANDLER; +} +#endif + + +#define ACUTEST_CMDLINE_OPTFLAG_OPTIONALARG_ 0x0001 +#define ACUTEST_CMDLINE_OPTFLAG_REQUIREDARG_ 0x0002 + +#define ACUTEST_CMDLINE_OPTID_NONE_ 0 +#define ACUTEST_CMDLINE_OPTID_UNKNOWN_ (-0x7fffffff + 0) +#define ACUTEST_CMDLINE_OPTID_MISSINGARG_ (-0x7fffffff + 1) +#define ACUTEST_CMDLINE_OPTID_BOGUSARG_ (-0x7fffffff + 2) + +typedef struct acutest_test_CMDLINE_OPTION_ { + char shortname; + const char* longname; + int id; + unsigned flags; +} ACUTEST_CMDLINE_OPTION_; + +static int +acutest_cmdline_handle_short_opt_group_(const ACUTEST_CMDLINE_OPTION_* options, + const char* arggroup, + int (*callback)(int /*optval*/, const char* /*arg*/)) +{ + const ACUTEST_CMDLINE_OPTION_* opt; + int i; + int ret = 0; + + for(i = 0; arggroup[i] != '\0'; i++) { + for(opt = options; opt->id != 0; opt++) { + if(arggroup[i] == opt->shortname) + break; + } + + if(opt->id != 0 && !(opt->flags & ACUTEST_CMDLINE_OPTFLAG_REQUIREDARG_)) { + ret = callback(opt->id, NULL); + } else { + /* Unknown option. */ + char badoptname[3]; + badoptname[0] = '-'; + badoptname[1] = arggroup[i]; + badoptname[2] = '\0'; + ret = callback((opt->id != 0 ? ACUTEST_CMDLINE_OPTID_MISSINGARG_ : ACUTEST_CMDLINE_OPTID_UNKNOWN_), + badoptname); + } + + if(ret != 0) + break; + } + + return ret; +} + +#define ACUTEST_CMDLINE_AUXBUF_SIZE_ 32 + +static int +acutest_cmdline_read_(const ACUTEST_CMDLINE_OPTION_* options, int argc, char** argv, + int (*callback)(int /*optval*/, const char* /*arg*/)) +{ + + const ACUTEST_CMDLINE_OPTION_* opt; + char auxbuf[ACUTEST_CMDLINE_AUXBUF_SIZE_+1]; + int after_doubledash = 0; + int i = 1; + int ret = 0; + + auxbuf[ACUTEST_CMDLINE_AUXBUF_SIZE_] = '\0'; + + while(i < argc) { + if(after_doubledash || strcmp(argv[i], "-") == 0) { + /* Non-option argument. */ + ret = callback(ACUTEST_CMDLINE_OPTID_NONE_, argv[i]); + } else if(strcmp(argv[i], "--") == 0) { + /* End of options. All the remaining members are non-option arguments. */ + after_doubledash = 1; + } else if(argv[i][0] != '-') { + /* Non-option argument. */ + ret = callback(ACUTEST_CMDLINE_OPTID_NONE_, argv[i]); + } else { + for(opt = options; opt->id != 0; opt++) { + if(opt->longname != NULL && strncmp(argv[i], "--", 2) == 0) { + size_t len = strlen(opt->longname); + if(strncmp(argv[i]+2, opt->longname, len) == 0) { + /* Regular long option. */ + if(argv[i][2+len] == '\0') { + /* with no argument provided. */ + if(!(opt->flags & ACUTEST_CMDLINE_OPTFLAG_REQUIREDARG_)) + ret = callback(opt->id, NULL); + else + ret = callback(ACUTEST_CMDLINE_OPTID_MISSINGARG_, argv[i]); + break; + } else if(argv[i][2+len] == '=') { + /* with an argument provided. */ + if(opt->flags & (ACUTEST_CMDLINE_OPTFLAG_OPTIONALARG_ | ACUTEST_CMDLINE_OPTFLAG_REQUIREDARG_)) { + ret = callback(opt->id, argv[i]+2+len+1); + } else { + sprintf(auxbuf, "--%s", opt->longname); + ret = callback(ACUTEST_CMDLINE_OPTID_BOGUSARG_, auxbuf); + } + break; + } else { + continue; + } + } + } else if(opt->shortname != '\0' && argv[i][0] == '-') { + if(argv[i][1] == opt->shortname) { + /* Regular short option. */ + if(opt->flags & ACUTEST_CMDLINE_OPTFLAG_REQUIREDARG_) { + if(argv[i][2] != '\0') + ret = callback(opt->id, argv[i]+2); + else if(i+1 < argc) + ret = callback(opt->id, argv[++i]); + else + ret = callback(ACUTEST_CMDLINE_OPTID_MISSINGARG_, argv[i]); + break; + } else { + ret = callback(opt->id, NULL); + + /* There might be more (argument-less) short options + * grouped together. */ + if(ret == 0 && argv[i][2] != '\0') + ret = acutest_cmdline_handle_short_opt_group_(options, argv[i]+2, callback); + break; + } + } + } + } + + if(opt->id == 0) { /* still not handled? */ + if(argv[i][0] != '-') { + /* Non-option argument. */ + ret = callback(ACUTEST_CMDLINE_OPTID_NONE_, argv[i]); + } else { + /* Unknown option. */ + char* badoptname = argv[i]; + + if(strncmp(badoptname, "--", 2) == 0) { + /* Strip any argument from the long option. */ + char* assignment = strchr(badoptname, '='); + if(assignment != NULL) { + size_t len = assignment - badoptname; + if(len > ACUTEST_CMDLINE_AUXBUF_SIZE_) + len = ACUTEST_CMDLINE_AUXBUF_SIZE_; + strncpy(auxbuf, badoptname, len); + auxbuf[len] = '\0'; + badoptname = auxbuf; + } + } + + ret = callback(ACUTEST_CMDLINE_OPTID_UNKNOWN_, badoptname); + } + } + } + + if(ret != 0) + return ret; + i++; + } + + return ret; +} + +static void +acutest_help_(void) +{ + printf("Usage: %s [options] [test...]\n", acutest_argv0_); + printf("\n"); + printf("Run the specified unit tests; or if the option '--skip' is used, run all\n"); + printf("tests in the suite but those listed. By default, if no tests are specified\n"); + printf("on the command line, all unit tests in the suite are run.\n"); + printf("\n"); + printf("Options:\n"); + printf(" -s, --skip Execute all unit tests but the listed ones\n"); + printf(" --exec[=WHEN] If supported, execute unit tests as child processes\n"); + printf(" (WHEN is one of 'auto', 'always', 'never')\n"); + printf(" -E, --no-exec Same as --exec=never\n"); +#if defined ACUTEST_WIN_ + printf(" -t, --time Measure test duration\n"); +#elif defined ACUTEST_HAS_POSIX_TIMER_ + printf(" -t, --time Measure test duration (real time)\n"); + printf(" --time=TIMER Measure test duration, using given timer\n"); + printf(" (TIMER is one of 'real', 'cpu')\n"); +#endif + printf(" --no-summary Suppress printing of test results summary\n"); + printf(" --tap Produce TAP-compliant output\n"); + printf(" (See https://testanything.org/)\n"); + printf(" -x, --xml-output=FILE Enable XUnit output to the given file\n"); + printf(" -l, --list List unit tests in the suite and exit\n"); + printf(" -v, --verbose Make output more verbose\n"); + printf(" --verbose=LEVEL Set verbose level to LEVEL:\n"); + printf(" 0 ... Be silent\n"); + printf(" 1 ... Output one line per test (and summary)\n"); + printf(" 2 ... As 1 and failed conditions (this is default)\n"); + printf(" 3 ... As 1 and all conditions (and extended summary)\n"); + printf(" -q, --quiet Same as --verbose=0\n"); + printf(" --color[=WHEN] Enable colorized output\n"); + printf(" (WHEN is one of 'auto', 'always', 'never')\n"); + printf(" --no-color Same as --color=never\n"); + printf(" -h, --help Display this help and exit\n"); + + if(acutest_list_size_ < 16) { + printf("\n"); + acutest_list_names_(); + } +} + +static const ACUTEST_CMDLINE_OPTION_ acutest_cmdline_options_[] = { + { 's', "skip", 's', 0 }, + { 0, "exec", 'e', ACUTEST_CMDLINE_OPTFLAG_OPTIONALARG_ }, + { 'E', "no-exec", 'E', 0 }, +#if defined ACUTEST_WIN_ + { 't', "time", 't', 0 }, + { 0, "timer", 't', 0 }, /* kept for compatibility */ +#elif defined ACUTEST_HAS_POSIX_TIMER_ + { 't', "time", 't', ACUTEST_CMDLINE_OPTFLAG_OPTIONALARG_ }, + { 0, "timer", 't', ACUTEST_CMDLINE_OPTFLAG_OPTIONALARG_ }, /* kept for compatibility */ +#endif + { 0, "no-summary", 'S', 0 }, + { 0, "tap", 'T', 0 }, + { 'l', "list", 'l', 0 }, + { 'v', "verbose", 'v', ACUTEST_CMDLINE_OPTFLAG_OPTIONALARG_ }, + { 'q', "quiet", 'q', 0 }, + { 0, "color", 'c', ACUTEST_CMDLINE_OPTFLAG_OPTIONALARG_ }, + { 0, "no-color", 'C', 0 }, + { 'h', "help", 'h', 0 }, + { 0, "worker", 'w', ACUTEST_CMDLINE_OPTFLAG_REQUIREDARG_ }, /* internal */ + { 'x', "xml-output", 'x', ACUTEST_CMDLINE_OPTFLAG_REQUIREDARG_ }, + { 0, NULL, 0, 0 } +}; + +static int +acutest_cmdline_callback_(int id, const char* arg) +{ + switch(id) { + case 's': + acutest_skip_mode_ = 1; + break; + + case 'e': + if(arg == NULL || strcmp(arg, "always") == 0) { + acutest_no_exec_ = 0; + } else if(strcmp(arg, "never") == 0) { + acutest_no_exec_ = 1; + } else if(strcmp(arg, "auto") == 0) { + /*noop*/ + } else { + fprintf(stderr, "%s: Unrecognized argument '%s' for option --exec.\n", acutest_argv0_, arg); + fprintf(stderr, "Try '%s --help' for more information.\n", acutest_argv0_); + acutest_exit_(2); + } + break; + + case 'E': + acutest_no_exec_ = 1; + break; + + case 't': +#if defined ACUTEST_WIN_ || defined ACUTEST_HAS_POSIX_TIMER_ + if(arg == NULL || strcmp(arg, "real") == 0) { + acutest_timer_ = 1; + #ifndef ACUTEST_WIN_ + } else if(strcmp(arg, "cpu") == 0) { + acutest_timer_ = 2; + #endif + } else { + fprintf(stderr, "%s: Unrecognized argument '%s' for option --time.\n", acutest_argv0_, arg); + fprintf(stderr, "Try '%s --help' for more information.\n", acutest_argv0_); + acutest_exit_(2); + } +#endif + break; + + case 'S': + acutest_no_summary_ = 1; + break; + + case 'T': + acutest_tap_ = 1; + break; + + case 'l': + acutest_list_names_(); + acutest_exit_(0); + break; + + case 'v': + acutest_verbose_level_ = (arg != NULL ? atoi(arg) : acutest_verbose_level_+1); + break; + + case 'q': + acutest_verbose_level_ = 0; + break; + + case 'c': + if(arg == NULL || strcmp(arg, "always") == 0) { + acutest_colorize_ = 1; + } else if(strcmp(arg, "never") == 0) { + acutest_colorize_ = 0; + } else if(strcmp(arg, "auto") == 0) { + /*noop*/ + } else { + fprintf(stderr, "%s: Unrecognized argument '%s' for option --color.\n", acutest_argv0_, arg); + fprintf(stderr, "Try '%s --help' for more information.\n", acutest_argv0_); + acutest_exit_(2); + } + break; + + case 'C': + acutest_colorize_ = 0; + break; + + case 'h': + acutest_help_(); + acutest_exit_(0); + break; + + case 'w': + acutest_worker_ = 1; + acutest_worker_index_ = atoi(arg); + break; + case 'x': + acutest_xml_output_ = fopen(arg, "w"); + if (!acutest_xml_output_) { + fprintf(stderr, "Unable to open '%s': %s\n", arg, strerror(errno)); + acutest_exit_(2); + } + break; + + case 0: + if(acutest_lookup_(arg) == 0) { + fprintf(stderr, "%s: Unrecognized unit test '%s'\n", acutest_argv0_, arg); + fprintf(stderr, "Try '%s --list' for list of unit tests.\n", acutest_argv0_); + acutest_exit_(2); + } + break; + + case ACUTEST_CMDLINE_OPTID_UNKNOWN_: + fprintf(stderr, "Unrecognized command line option '%s'.\n", arg); + fprintf(stderr, "Try '%s --help' for more information.\n", acutest_argv0_); + acutest_exit_(2); + break; + + case ACUTEST_CMDLINE_OPTID_MISSINGARG_: + fprintf(stderr, "The command line option '%s' requires an argument.\n", arg); + fprintf(stderr, "Try '%s --help' for more information.\n", acutest_argv0_); + acutest_exit_(2); + break; + + case ACUTEST_CMDLINE_OPTID_BOGUSARG_: + fprintf(stderr, "The command line option '%s' does not expect an argument.\n", arg); + fprintf(stderr, "Try '%s --help' for more information.\n", acutest_argv0_); + acutest_exit_(2); + break; + } + + return 0; +} + + +#ifdef ACUTEST_LINUX_ +static int +acutest_is_tracer_present_(void) +{ + /* Must be large enough so the line 'TracerPid: ${PID}' can fit in. */ + static const int OVERLAP = 32; + + char buf[256+OVERLAP+1]; + int tracer_present = 0; + int fd; + size_t n_read = 0; + + fd = open("/proc/self/status", O_RDONLY); + if(fd == -1) + return 0; + + while(1) { + static const char pattern[] = "TracerPid:"; + const char* field; + + while(n_read < sizeof(buf) - 1) { + ssize_t n; + + n = read(fd, buf + n_read, sizeof(buf) - 1 - n_read); + if(n <= 0) + break; + n_read += n; + } + buf[n_read] = '\0'; + + field = strstr(buf, pattern); + if(field != NULL && field < buf + sizeof(buf) - OVERLAP) { + pid_t tracer_pid = (pid_t) atoi(field + sizeof(pattern) - 1); + tracer_present = (tracer_pid != 0); + break; + } + + if(n_read == sizeof(buf)-1) { + memmove(buf, buf + sizeof(buf)-1 - OVERLAP, OVERLAP); + n_read = OVERLAP; + } else { + break; + } + } + + close(fd); + return tracer_present; +} +#endif + +int +main(int argc, char** argv) +{ + int i; + + acutest_argv0_ = argv[0]; + +#if defined ACUTEST_UNIX_ + acutest_colorize_ = isatty(STDOUT_FILENO); +#elif defined ACUTEST_WIN_ + #if defined _BORLANDC_ + acutest_colorize_ = isatty(_fileno(stdout)); + #else + acutest_colorize_ = _isatty(_fileno(stdout)); + #endif +#else + acutest_colorize_ = 0; +#endif + + /* Count all test units */ + acutest_list_size_ = 0; + for(i = 0; acutest_list_[i].func != NULL; i++) + acutest_list_size_++; + + acutest_test_data_ = (struct acutest_test_data_*)calloc(acutest_list_size_, sizeof(struct acutest_test_data_)); + if(acutest_test_data_ == NULL) { + fprintf(stderr, "Out of memory.\n"); + acutest_exit_(2); + } + + /* Parse options */ + acutest_cmdline_read_(acutest_cmdline_options_, argc, argv, acutest_cmdline_callback_); + + /* Initialize the proper timer. */ + acutest_timer_init_(); + +#if defined(ACUTEST_WIN_) + SetUnhandledExceptionFilter(acutest_seh_exception_filter_); +#ifdef _MSC_VER + _set_abort_behavior(0, _WRITE_ABORT_MSG); +#endif +#endif + + /* By default, we want to run all tests. */ + if(acutest_count_ == 0) { + for(i = 0; acutest_list_[i].func != NULL; i++) + acutest_remember_(i); + } + + /* Guess whether we want to run unit tests as child processes. */ + if(acutest_no_exec_ < 0) { + acutest_no_exec_ = 0; + + if(acutest_count_ <= 1) { + acutest_no_exec_ = 1; + } else { +#ifdef ACUTEST_WIN_ + if(IsDebuggerPresent()) + acutest_no_exec_ = 1; +#endif +#ifdef ACUTEST_LINUX_ + if(acutest_is_tracer_present_()) + acutest_no_exec_ = 1; +#endif +#ifdef RUNNING_ON_VALGRIND + /* RUNNING_ON_VALGRIND is provided by optionally included */ + if(RUNNING_ON_VALGRIND) + acutest_no_exec_ = 1; +#endif + } + } + + if(acutest_tap_) { + /* TAP requires we know test result ("ok", "not ok") before we output + * anything about the test, and this gets problematic for larger verbose + * levels. */ + if(acutest_verbose_level_ > 2) + acutest_verbose_level_ = 2; + + /* TAP harness should provide some summary. */ + acutest_no_summary_ = 1; + + if(!acutest_worker_) + printf("1..%d\n", (int) acutest_count_); + } + + int index = acutest_worker_index_; + for(i = 0; acutest_list_[i].func != NULL; i++) { + int run = (acutest_test_data_[i].flags & ACUTEST_FLAG_RUN_); + if (acutest_skip_mode_) /* Run all tests except those listed. */ + run = !run; + if(run) + acutest_run_(´st_list_[i], index++, i); + } + + /* Write a summary */ + if(!acutest_no_summary_ && acutest_verbose_level_ >= 1) { + if(acutest_verbose_level_ >= 3) { + acutest_colored_printf_(ACUTEST_COLOR_DEFAULT_INTENSIVE_, "Summary:\n"); + + printf(" Count of all unit tests: %4d\n", (int) acutest_list_size_); + printf(" Count of run unit tests: %4d\n", acutest_stat_run_units_); + printf(" Count of failed unit tests: %4d\n", acutest_stat_failed_units_); + printf(" Count of skipped unit tests: %4d\n", (int) acutest_list_size_ - acutest_stat_run_units_); + } + + if(acutest_stat_failed_units_ == 0) { + acutest_colored_printf_(ACUTEST_COLOR_GREEN_INTENSIVE_, "SUCCESS:"); + printf(" All unit tests have passed.\n"); + } else { + acutest_colored_printf_(ACUTEST_COLOR_RED_INTENSIVE_, "FAILED:"); + printf(" %d of %d unit tests %s failed.\n", + acutest_stat_failed_units_, acutest_stat_run_units_, + (acutest_stat_failed_units_ == 1) ? "has" : "have"); + } + + if(acutest_verbose_level_ >= 3) + printf("\n"); + } + + if (acutest_xml_output_) { +#if defined ACUTEST_UNIX_ + char *suite_name = basename(argv[0]); +#elif defined ACUTEST_WIN_ + char suite_name[_MAX_FNAME]; + _splitpath(argv[0], NULL, NULL, suite_name, NULL); +#else + const char *suite_name = argv[0]; +#endif + fprintf(acutest_xml_output_, "\n"); + fprintf(acutest_xml_output_, "\n", + suite_name, (int)acutest_list_size_, acutest_stat_failed_units_, acutest_stat_failed_units_, + (int)acutest_list_size_ - acutest_stat_run_units_); + for(i = 0; acutest_list_[i].func != NULL; i++) { + struct acutest_test_data_ *details = ´st_test_data_[i]; + fprintf(acutest_xml_output_, " \n", acutest_list_[i].name, details->duration); + if (details->flags & ACUTEST_FLAG_FAILURE_) + fprintf(acutest_xml_output_, " \n"); + if (!(details->flags & ACUTEST_FLAG_FAILURE_) && !(details->flags & ACUTEST_FLAG_SUCCESS_)) + fprintf(acutest_xml_output_, " \n"); + fprintf(acutest_xml_output_, " \n"); + } + fprintf(acutest_xml_output_, "\n"); + fclose(acutest_xml_output_); + } + + acutest_cleanup_(); + + return (acutest_stat_failed_units_ == 0) ? 0 : 1; +} + + +#endif /* #ifndef TEST_NO_MAIN */ + +#ifdef _MSC_VER + #pragma warning(pop) +#endif + +#ifdef __cplusplus + } /* extern "C" */ +#endif + +#endif /* #ifndef ACUTEST_H */ diff --git a/graalpython/hpy/c_test/test_debug_handles.c b/graalpython/hpy/c_test/test_debug_handles.c new file mode 100644 index 0000000000..a5584fdb39 --- /dev/null +++ b/graalpython/hpy/c_test/test_debug_handles.c @@ -0,0 +1,99 @@ +#include +#include "acutest.h" // https://github.com/mity/acutest +#include "hpy/debug/src/debug_internal.h" + +static void check_DHQueue(DHQueue *q, HPy_ssize_t size, ...) +{ + va_list argp; + va_start(argp, size); + DHQueue_sanity_check(q); + TEST_CHECK(q->size == size); + DHQueueNode *h = q->head; + while(h != NULL) { + DHQueueNode *expected = va_arg(argp, DHQueueNode*); + TEST_CHECK(h == expected); + h = h->next; + } + va_end(argp); +} + +void test_DHQueue_init(void) +{ + DHQueue q; + DHQueue_init(&q); + TEST_CHECK(q.head == NULL); + TEST_CHECK(q.tail == NULL); + TEST_CHECK(q.size == 0); + DHQueue_sanity_check(&q); +} + +void test_DHQueue_append(void) +{ + DHQueue q; + DHQueueNode h1; + DHQueueNode h2; + DHQueueNode h3; + DHQueue_init(&q); + DHQueue_append(&q, &h1); + DHQueue_append(&q, &h2); + DHQueue_append(&q, &h3); + check_DHQueue(&q, 3, &h1, &h2, &h3); +} + +void test_DHQueue_popfront(void) +{ + DHQueue q; + DHQueueNode h1; + DHQueueNode h2; + DHQueueNode h3; + DHQueue_init(&q); + DHQueue_append(&q, &h1); + DHQueue_append(&q, &h2); + DHQueue_append(&q, &h3); + + TEST_CHECK(DHQueue_popfront(&q) == &h1); + check_DHQueue(&q, 2, &h2, &h3); + + TEST_CHECK(DHQueue_popfront(&q) == &h2); + check_DHQueue(&q, 1, &h3); + + TEST_CHECK(DHQueue_popfront(&q) == &h3); + check_DHQueue(&q, 0); +} + + +void test_DHQueue_remove(void) +{ + DHQueue q; + DHQueueNode h1; + DHQueueNode h2; + DHQueueNode h3; + DHQueueNode h4; + DHQueue_init(&q); + DHQueue_append(&q, &h1); + DHQueue_append(&q, &h2); + DHQueue_append(&q, &h3); + DHQueue_append(&q, &h4); + + DHQueue_remove(&q, &h3); // try to remove something in the middle + check_DHQueue(&q, 3, &h1, &h2, &h4); + + DHQueue_remove(&q, &h1); // try to remove the head + check_DHQueue(&q, 2, &h2, &h4); + + DHQueue_remove(&q, &h4); // try to remove the tail + check_DHQueue(&q, 1, &h2); + + DHQueue_remove(&q, &h2); // try to remove the only element + check_DHQueue(&q, 0); +} + +#define MYTEST(X) { #X, X } + +TEST_LIST = { + MYTEST(test_DHQueue_init), + MYTEST(test_DHQueue_append), + MYTEST(test_DHQueue_popfront), + MYTEST(test_DHQueue_remove), + { NULL, NULL } +}; diff --git a/graalpython/hpy/c_test/test_stacktrace.c b/graalpython/hpy/c_test/test_stacktrace.c new file mode 100644 index 0000000000..5d02eced84 --- /dev/null +++ b/graalpython/hpy/c_test/test_stacktrace.c @@ -0,0 +1,22 @@ +// Smoke test for the create_stacktrace function + +#include +#include "acutest.h" // https://github.com/mity/acutest +#include "hpy/debug/src/debug_internal.h" + +void test_create_stacktrace(void) +{ + char *trace; + create_stacktrace(&trace, 16); + if (trace != NULL) { + printf("\n\nTRACE:\n%s\n\n", trace); + free(trace); + } +} + +#define MYTEST(X) { #X, X } + +TEST_LIST = { + MYTEST(test_create_stacktrace), + { NULL, NULL } +}; \ No newline at end of file diff --git a/graalpython/hpy/docs/Makefile b/graalpython/hpy/docs/Makefile new file mode 100644 index 0000000000..7164c8d0e5 --- /dev/null +++ b/graalpython/hpy/docs/Makefile @@ -0,0 +1,28 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +livehtml: + @sphinx-autobuild \ + "$(SOURCEDIR)" \ + -b html \ + --ignore "*~" \ + --ignore ".#*" \ + $(SPHINXOPTS) $(BUILDDIR)/html diff --git a/graalpython/hpy/docs/_static/README.txt b/graalpython/hpy/docs/_static/README.txt new file mode 100644 index 0000000000..4d871f5a5a --- /dev/null +++ b/graalpython/hpy/docs/_static/README.txt @@ -0,0 +1 @@ +Static files for the Sphinx documentation. diff --git a/graalpython/hpy/docs/_templates/README.txt b/graalpython/hpy/docs/_templates/README.txt new file mode 100644 index 0000000000..2dc51448da --- /dev/null +++ b/graalpython/hpy/docs/_templates/README.txt @@ -0,0 +1 @@ +Template files for the Sphinx documentation. diff --git a/graalpython/hpy/docs/api-reference/argument-parsing.rst b/graalpython/hpy/docs/api-reference/argument-parsing.rst new file mode 100644 index 0000000000..ff22665254 --- /dev/null +++ b/graalpython/hpy/docs/api-reference/argument-parsing.rst @@ -0,0 +1,5 @@ +Argument Parsing +================ + +.. autocmodule:: runtime/argparse.c + :members: diff --git a/graalpython/hpy/docs/api-reference/build-value.rst b/graalpython/hpy/docs/api-reference/build-value.rst new file mode 100644 index 0000000000..a8cafb6bfe --- /dev/null +++ b/graalpython/hpy/docs/api-reference/build-value.rst @@ -0,0 +1,5 @@ +Building Complex Python Objects +=============================== + +.. autocmodule:: runtime/buildvalue.c + :members: diff --git a/graalpython/hpy/docs/api-reference/formatting.rst b/graalpython/hpy/docs/api-reference/formatting.rst new file mode 100644 index 0000000000..f4ea6f26f4 --- /dev/null +++ b/graalpython/hpy/docs/api-reference/formatting.rst @@ -0,0 +1,5 @@ +String Formatting Helpers +========================= + +.. autocmodule:: runtime/format.c + :no-members: diff --git a/graalpython/hpy/docs/api-reference/function-index.rst b/graalpython/hpy/docs/api-reference/function-index.rst new file mode 100644 index 0000000000..5e2cbcb59b --- /dev/null +++ b/graalpython/hpy/docs/api-reference/function-index.rst @@ -0,0 +1,187 @@ + +.. note: DO NOT EDIT THIS FILE! + This file is automatically generated by hpy.tools.autogen.doc.autogen_function_index + See also hpy.tools.autogen and hpy/tools/public_api.h + + Run this to regenerate: + make autogen + +HPy Core API Function Index +########################### + +* :c:func:`HPyBool_FromBool` +* :c:func:`HPyBytes_AS_STRING` +* :c:func:`HPyBytes_AsString` +* :c:func:`HPyBytes_Check` +* :c:func:`HPyBytes_FromString` +* :c:func:`HPyBytes_FromStringAndSize` +* :c:func:`HPyBytes_GET_SIZE` +* :c:func:`HPyBytes_Size` +* :c:func:`HPyCallable_Check` +* :c:func:`HPyCapsule_Get` +* :c:func:`HPyCapsule_IsValid` +* :c:func:`HPyCapsule_New` +* :c:func:`HPyCapsule_Set` +* :c:func:`HPyContextVar_Get` +* :c:func:`HPyContextVar_New` +* :c:func:`HPyContextVar_Set` +* :c:func:`HPyDict_Check` +* :c:func:`HPyDict_Copy` +* :c:func:`HPyDict_Keys` +* :c:func:`HPyDict_New` +* :c:func:`HPyErr_Clear` +* :c:func:`HPyErr_ExceptionMatches` +* :c:func:`HPyErr_NewException` +* :c:func:`HPyErr_NewExceptionWithDoc` +* :c:func:`HPyErr_NoMemory` +* :c:func:`HPyErr_Occurred` +* :c:func:`HPyErr_SetFromErrnoWithFilename` +* :c:func:`HPyErr_SetFromErrnoWithFilenameObjects` +* :c:func:`HPyErr_SetObject` +* :c:func:`HPyErr_SetString` +* :c:func:`HPyErr_WarnEx` +* :c:func:`HPyErr_WriteUnraisable` +* :c:func:`HPyField_Load` +* :c:func:`HPyField_Store` +* :c:func:`HPyFloat_AsDouble` +* :c:func:`HPyFloat_FromDouble` +* :c:func:`HPyGlobal_Load` +* :c:func:`HPyGlobal_Store` +* :c:func:`HPyImport_ImportModule` +* :c:func:`HPyIter_Check` +* :c:func:`HPyIter_Next` +* :c:func:`HPyListBuilder_Build` +* :c:func:`HPyListBuilder_Cancel` +* :c:func:`HPyListBuilder_New` +* :c:func:`HPyListBuilder_Set` +* :c:func:`HPyList_Append` +* :c:func:`HPyList_Check` +* :c:func:`HPyList_Insert` +* :c:func:`HPyList_New` +* :c:func:`HPyLong_AsDouble` +* :c:func:`HPyLong_AsInt32_t` +* :c:func:`HPyLong_AsInt64_t` +* :c:func:`HPyLong_AsSize_t` +* :c:func:`HPyLong_AsSsize_t` +* :c:func:`HPyLong_AsUInt32_t` +* :c:func:`HPyLong_AsUInt32_tMask` +* :c:func:`HPyLong_AsUInt64_t` +* :c:func:`HPyLong_AsUInt64_tMask` +* :c:func:`HPyLong_AsVoidPtr` +* :c:func:`HPyLong_FromInt32_t` +* :c:func:`HPyLong_FromInt64_t` +* :c:func:`HPyLong_FromSize_t` +* :c:func:`HPyLong_FromSsize_t` +* :c:func:`HPyLong_FromUInt32_t` +* :c:func:`HPyLong_FromUInt64_t` +* :c:func:`HPyNumber_Check` +* :c:func:`HPySlice_New` +* :c:func:`HPySlice_Unpack` +* :c:func:`HPyTracker_Add` +* :c:func:`HPyTracker_Close` +* :c:func:`HPyTracker_ForgetAll` +* :c:func:`HPyTracker_New` +* :c:func:`HPyTupleBuilder_Build` +* :c:func:`HPyTupleBuilder_Cancel` +* :c:func:`HPyTupleBuilder_New` +* :c:func:`HPyTupleBuilder_Set` +* :c:func:`HPyTuple_Check` +* :c:func:`HPyTuple_FromArray` +* :c:func:`HPyType_FromSpec` +* :c:func:`HPyType_GenericNew` +* :c:func:`HPyType_GetName` +* :c:func:`HPyType_IsSubtype` +* :c:func:`HPyUnicode_AsASCIIString` +* :c:func:`HPyUnicode_AsLatin1String` +* :c:func:`HPyUnicode_AsUTF8AndSize` +* :c:func:`HPyUnicode_AsUTF8String` +* :c:func:`HPyUnicode_Check` +* :c:func:`HPyUnicode_DecodeASCII` +* :c:func:`HPyUnicode_DecodeFSDefault` +* :c:func:`HPyUnicode_DecodeFSDefaultAndSize` +* :c:func:`HPyUnicode_DecodeLatin1` +* :c:func:`HPyUnicode_EncodeFSDefault` +* :c:func:`HPyUnicode_FromEncodedObject` +* :c:func:`HPyUnicode_FromString` +* :c:func:`HPyUnicode_FromWideChar` +* :c:func:`HPyUnicode_ReadChar` +* :c:func:`HPyUnicode_Substring` +* :c:func:`HPy_ASCII` +* :c:func:`HPy_Absolute` +* :c:func:`HPy_Add` +* :c:func:`HPy_And` +* :c:func:`HPy_AsPyObject` +* :c:func:`HPy_Bytes` +* :c:func:`HPy_Call` +* :c:func:`HPy_CallMethod` +* :c:func:`HPy_CallTupleDict` +* :c:func:`HPy_Close` +* :c:func:`HPy_Compile_s` +* :c:func:`HPy_Contains` +* :c:func:`HPy_DelItem` +* :c:func:`HPy_DelItem_i` +* :c:func:`HPy_DelItem_s` +* :c:func:`HPy_DelSlice` +* :c:func:`HPy_Divmod` +* :c:func:`HPy_Dup` +* :c:func:`HPy_EvalCode` +* :c:func:`HPy_FatalError` +* :c:func:`HPy_Float` +* :c:func:`HPy_FloorDivide` +* :c:func:`HPy_FromPyObject` +* :c:func:`HPy_GetAttr` +* :c:func:`HPy_GetAttr_s` +* :c:func:`HPy_GetItem` +* :c:func:`HPy_GetItem_i` +* :c:func:`HPy_GetItem_s` +* :c:func:`HPy_GetIter` +* :c:func:`HPy_GetSlice` +* :c:func:`HPy_HasAttr` +* :c:func:`HPy_HasAttr_s` +* :c:func:`HPy_Hash` +* :c:func:`HPy_InPlaceAdd` +* :c:func:`HPy_InPlaceAnd` +* :c:func:`HPy_InPlaceFloorDivide` +* :c:func:`HPy_InPlaceLshift` +* :c:func:`HPy_InPlaceMatrixMultiply` +* :c:func:`HPy_InPlaceMultiply` +* :c:func:`HPy_InPlaceOr` +* :c:func:`HPy_InPlacePower` +* :c:func:`HPy_InPlaceRemainder` +* :c:func:`HPy_InPlaceRshift` +* :c:func:`HPy_InPlaceSubtract` +* :c:func:`HPy_InPlaceTrueDivide` +* :c:func:`HPy_InPlaceXor` +* :c:func:`HPy_Index` +* :c:func:`HPy_Invert` +* :c:func:`HPy_Is` +* :c:func:`HPy_IsTrue` +* :c:func:`HPy_LeavePythonExecution` +* :c:func:`HPy_Length` +* :c:func:`HPy_Long` +* :c:func:`HPy_Lshift` +* :c:func:`HPy_MatrixMultiply` +* :c:func:`HPy_Multiply` +* :c:func:`HPy_Negative` +* :c:func:`HPy_Or` +* :c:func:`HPy_Positive` +* :c:func:`HPy_Power` +* :c:func:`HPy_ReenterPythonExecution` +* :c:func:`HPy_Remainder` +* :c:func:`HPy_Repr` +* :c:func:`HPy_RichCompare` +* :c:func:`HPy_RichCompareBool` +* :c:func:`HPy_Rshift` +* :c:func:`HPy_SetAttr` +* :c:func:`HPy_SetAttr_s` +* :c:func:`HPy_SetCallFunction` +* :c:func:`HPy_SetItem` +* :c:func:`HPy_SetItem_i` +* :c:func:`HPy_SetItem_s` +* :c:func:`HPy_SetSlice` +* :c:func:`HPy_Str` +* :c:func:`HPy_Subtract` +* :c:func:`HPy_TrueDivide` +* :c:func:`HPy_Type` +* :c:func:`HPy_TypeCheck` +* :c:func:`HPy_Xor` diff --git a/graalpython/hpy/docs/api-reference/helpers.rst b/graalpython/hpy/docs/api-reference/helpers.rst new file mode 100644 index 0000000000..df7725f8d4 --- /dev/null +++ b/graalpython/hpy/docs/api-reference/helpers.rst @@ -0,0 +1,5 @@ +Misc Helpers +============ + +.. autocmodule:: runtime/helpers.c + :members: diff --git a/graalpython/hpy/docs/api-reference/hpy-call.rst b/graalpython/hpy/docs/api-reference/hpy-call.rst new file mode 100644 index 0000000000..d685898f1e --- /dev/null +++ b/graalpython/hpy/docs/api-reference/hpy-call.rst @@ -0,0 +1,5 @@ +HPy Call API +============ + +.. autocmodule:: autogen/public_api.h + :members: HPy_Call,HPy_CallMethod,HPy_CallTupleDict diff --git a/graalpython/hpy/docs/api-reference/hpy-ctx.rst b/graalpython/hpy/docs/api-reference/hpy-ctx.rst new file mode 100644 index 0000000000..040903f005 --- /dev/null +++ b/graalpython/hpy/docs/api-reference/hpy-ctx.rst @@ -0,0 +1,9 @@ +HPy Context +=========== + +The ``HPyContext`` structure is also part of the API since it provides handles +for built-in objects. For a high-level description of the context, please also +read :ref:`api:hpycontext`. + +.. autocstruct:: hpy/cpython/autogen_ctx.h::_HPyContext_s + :members: diff --git a/graalpython/hpy/docs/api-reference/hpy-dict.rst b/graalpython/hpy/docs/api-reference/hpy-dict.rst new file mode 100644 index 0000000000..fb9400f4c1 --- /dev/null +++ b/graalpython/hpy/docs/api-reference/hpy-dict.rst @@ -0,0 +1,5 @@ +HPy Dict +======== + +.. autocmodule:: autogen/public_api.h + :members: HPyDict_Check, HPyDict_New, HPyDict_Keys, HPyDict_Copy diff --git a/graalpython/hpy/docs/api-reference/hpy-err.rst b/graalpython/hpy/docs/api-reference/hpy-err.rst new file mode 100644 index 0000000000..aeba618295 --- /dev/null +++ b/graalpython/hpy/docs/api-reference/hpy-err.rst @@ -0,0 +1,5 @@ +Exception Handling +================== + +.. autocmodule:: autogen/public_api.h + :members: HPyErr_SetFromErrnoWithFilename, HPyErr_SetFromErrnoWithFilenameObjects, HPy_FatalError, HPyErr_SetString, HPyErr_SetObject, HPyErr_Occurred, HPyErr_ExceptionMatches, HPyErr_NoMemory, HPyErr_Clear, HPyErr_NewException, HPyErr_NewExceptionWithDoc, HPyErr_WarnEx, HPyErr_WriteUnraisable diff --git a/graalpython/hpy/docs/api-reference/hpy-eval.rst b/graalpython/hpy/docs/api-reference/hpy-eval.rst new file mode 100644 index 0000000000..4e73e6d6fe --- /dev/null +++ b/graalpython/hpy/docs/api-reference/hpy-eval.rst @@ -0,0 +1,8 @@ +HPy Eval +======== + +.. autocmodule:: hpy.h + :members: HPy_SourceKind + +.. autocmodule:: autogen/public_api.h + :members: HPy_Compile_s,HPy_EvalCode diff --git a/graalpython/hpy/docs/api-reference/hpy-field.rst b/graalpython/hpy/docs/api-reference/hpy-field.rst new file mode 100644 index 0000000000..2973af4c5e --- /dev/null +++ b/graalpython/hpy/docs/api-reference/hpy-field.rst @@ -0,0 +1,5 @@ +HPyField +======== + +.. autocmodule:: autogen/public_api.h + :members: HPyField_Load,HPyField_Store diff --git a/graalpython/hpy/docs/api-reference/hpy-gil.rst b/graalpython/hpy/docs/api-reference/hpy-gil.rst new file mode 100644 index 0000000000..b64f96042d --- /dev/null +++ b/graalpython/hpy/docs/api-reference/hpy-gil.rst @@ -0,0 +1,5 @@ +Leave/enter Python execution (GIL) +================================== + +.. autocmodule:: autogen/public_api.h + :members: HPy_LeavePythonExecution,HPy_ReenterPythonExecution diff --git a/graalpython/hpy/docs/api-reference/hpy-global.rst b/graalpython/hpy/docs/api-reference/hpy-global.rst new file mode 100644 index 0000000000..9cbaa539e4 --- /dev/null +++ b/graalpython/hpy/docs/api-reference/hpy-global.rst @@ -0,0 +1,5 @@ +HPyGlobal +========= + +.. autocmodule:: autogen/public_api.h + :members: HPyGlobal_Store,HPyGlobal_Load diff --git a/graalpython/hpy/docs/api-reference/hpy-object.rst b/graalpython/hpy/docs/api-reference/hpy-object.rst new file mode 100644 index 0000000000..eeb5a6544b --- /dev/null +++ b/graalpython/hpy/docs/api-reference/hpy-object.rst @@ -0,0 +1,5 @@ +HPy Object +========== + +.. autocmodule:: autogen/public_api.h + :members: HPy_IsTrue,HPy_GetAttr,HPy_GetAttr_s,HPy_HasAttr,HPy_HasAttr_s,HPy_SetAttr,HPy_SetAttr_s,HPy_GetItem,HPy_GetItem_s,HPy_GetItem_i,HPy_GetSlice,HPy_SetItem,HPy_SetItem_s,HPy_SetItem_i,HPy_SetSlice,HPy_DelItem,HPy_DelItem_s,HPy_DelItem_i,HPy_DelSlice,HPy_Type,HPy_TypeCheck,HPy_Is,HPy_Repr,HPy_Str,HPy_ASCII,HPy_Bytes,HPy_RichCompare,HPy_RichCompareBool,HPy_Hash,HPy_SetCallFunction diff --git a/graalpython/hpy/docs/api-reference/hpy-sequence.rst b/graalpython/hpy/docs/api-reference/hpy-sequence.rst new file mode 100644 index 0000000000..21b7756e03 --- /dev/null +++ b/graalpython/hpy/docs/api-reference/hpy-sequence.rst @@ -0,0 +1,23 @@ +HPy Lists and Tuples +==================== + +Building Tuples and Lists +------------------------- + +.. autocmodule:: autogen/public_api.h + :members: HPyTupleBuilder_New,HPyTupleBuilder_Set,HPyTupleBuilder_Build,HPyTupleBuilder_Cancel + +.. autocmodule:: autogen/public_api.h + :members: HPyListBuilder_New,HPyListBuilder_Set,HPyListBuilder_Build,HPyListBuilder_Cancel + +Tuples +------ + +.. autocmodule:: autogen/public_api.h + :members: HPyTuple_Check,HPyTuple_FromArray + +Lists +----- + +.. autocmodule:: autogen/public_api.h + :members: HPyList_Check,HPyList_New,HPyList_Append,HPyList_Insert diff --git a/graalpython/hpy/docs/api-reference/hpy-type.rst b/graalpython/hpy/docs/api-reference/hpy-type.rst new file mode 100644 index 0000000000..9fccdfa37a --- /dev/null +++ b/graalpython/hpy/docs/api-reference/hpy-type.rst @@ -0,0 +1,50 @@ +HPy Types and Modules +===================== + +Types, modules and their attributes (i.e. methods, members, slots, get-set +descriptors) are defined in a similar way. Section `HPy Type`_ documents the +type-specific and `HPy Module`_ documents the module-specific part. Section +`HPy Definition`_ documents how to define attributes for both, types and +modules. + + +HPy Type +-------- + +Definition +~~~~~~~~~~ + +.. autocmodule:: hpy/hpytype.h + :members: HPyType_Spec,HPyType_BuiltinShape,HPyType_SpecParam,HPyType_SpecParam_Kind,HPyType_HELPERS,HPyType_LEGACY_HELPERS,HPy_TPFLAGS_DEFAULT,HPy_TPFLAGS_BASETYPE,HPy_TPFLAGS_HAVE_GC + +Construction and More +~~~~~~~~~~~~~~~~~~~~~ + +.. autocmodule:: autogen/public_api.h + :members: HPyType_FromSpec, HPyType_GetName, HPyType_IsSubtype + +HPy Module +---------- + +.. c:macro:: HPY_EMBEDDED_MODULES + + If ``HPY_EMBEDDED_MODULES`` is defined, this means that there will be + several embedded HPy modules (and so, several ``HPy_MODINIT`` usages) in the + same binary. In this case, some restrictions apply: + + 1. all of the module's methods/member/slots/... must be defined in the same + file + 2. the embedder **MUST** declare the module to be *embeddable* by using macro + :c:macro:`HPY_MOD_EMBEDDABLE`. + +.. autocmodule:: hpy/hpymodule.h + :members: HPY_MOD_EMBEDDABLE,HPyModuleDef,HPy_MODINIT + +HPy Definition +-------------- + +Defining slots, methods, members, and get-set descriptors for types and modules +is done with HPy definition (represented by C struct :c:struct:`HPyDef`). + +.. autocmodule:: hpy/hpydef.h + :members: HPyDef,HPyDef_Kind,HPySlot,HPyMeth,HPyMember_FieldType,HPyMember,HPyGetSet,HPyDef_SLOT,HPyDef_METH,HPyDef_MEMBER,HPyDef_GET,HPyDef_SET,HPyDef_GETSET,HPyDef_CALL_FUNCTION diff --git a/graalpython/hpy/docs/api-reference/index.rst b/graalpython/hpy/docs/api-reference/index.rst new file mode 100644 index 0000000000..89380d6377 --- /dev/null +++ b/graalpython/hpy/docs/api-reference/index.rst @@ -0,0 +1,69 @@ +API Reference +============= + +HPy's public API consists of three parts: + +1. The **Core API** as defined in the :doc:`public-api` +2. **HPy Helper** functions +3. **Inline Helper** functions + +Core API +-------- + +The **Core API** consists of inline functions that call into the Python +interpreter. Those functions will be implemented by each Python interpreter. In +:term:`CPython ABI` mode, many of these inline functions will just delegate to +a C API functions. In :term:`HPy Universal ABI` mode, they will call a function +pointer from the HPy context. This is the source of the performance change +between the modes. + +.. toctree:: + :maxdepth: 2 + + function-index + hpy-ctx + hpy-object + hpy-type + hpy-call + hpy-field + hpy-global + hpy-dict + hpy-sequence + hpy-gil + hpy-err + hpy-eval + public-api + + +HPy Helper Functions +-------------------- + +**HPy Helper** functions are functions (written in C) that will be compiled +together with the HPy extension's sources. The appropriate source files are +automatically added to the extension sources. The helper functions will, of +course, use the core API to interact with the interpreter. The main reason for +having the helper functions in the HPy extension is to avoid compatibility +problems due to different compilers. + +.. toctree:: + :maxdepth: 2 + + argument-parsing + build-value + formatting + structseq + helpers + + +Inline Helper Functions +----------------------- + +**Inline Helper** functions are ``static inline`` functions (written in C). +Those functions are usually small convenience functions that everyone could +write but in order to avoid duplicated effort, they are defined by HPy. + +.. toctree:: + :maxdepth: 2 + + inline-helpers + diff --git a/graalpython/hpy/docs/api-reference/inline-helpers.rst b/graalpython/hpy/docs/api-reference/inline-helpers.rst new file mode 100644 index 0000000000..2e719caf77 --- /dev/null +++ b/graalpython/hpy/docs/api-reference/inline-helpers.rst @@ -0,0 +1,14 @@ +Inline Helpers +============== + +**Inline Helper** functions are ``static inline`` functions (written in C). +Those functions are usually small convenience functions that everyone could +write but in order to avoid duplicated effort, they are defined by HPy. + +One category of inline helpers are functions that convert the commonly used +but not fixed width C types, such as ``int``, or ``long long``, to HPy API. +The HPy API always uses well-defined fixed width types like ``int32`` or +``unsigned int8``. + +.. autocmodule:: hpy/inline_helpers.h + :members: diff --git a/graalpython/hpy/docs/api-reference/public-api.rst b/graalpython/hpy/docs/api-reference/public-api.rst new file mode 100644 index 0000000000..992df114f8 --- /dev/null +++ b/graalpython/hpy/docs/api-reference/public-api.rst @@ -0,0 +1,9 @@ +Public API Header +================= + +The core API is defined in `public_api.h +`_: + +.. literalinclude:: ../../hpy/tools/autogen/public_api.h + :language: c + :linenos: diff --git a/graalpython/hpy/docs/api-reference/structseq.rst b/graalpython/hpy/docs/api-reference/structseq.rst new file mode 100644 index 0000000000..2772b3edfe --- /dev/null +++ b/graalpython/hpy/docs/api-reference/structseq.rst @@ -0,0 +1,15 @@ +Struct Sequences +================ + +Struct sequences are subclasses of tuple. Similar to the API for creating +tuples, HPy provides an API to create struct sequences. This is a builder API +such that the struct sequence is guaranteed not to be written after it is +created. + +.. note:: + + There is no specific getter function for struct sequences. Just use one of + :c:func:`HPy_GetItem`, :c:func:`HPy_GetItem_i`, or :c:func:`HPy_GetItem_s`. + +.. autocmodule:: hpy/runtime/structseq.h + :members: HPyStructSequence_Field,HPyStructSequence_Desc,HPyStructSequence_UnnamedField,HPyStructSequence_NewType,HPyStructSequence_New diff --git a/graalpython/hpy/docs/api.rst b/graalpython/hpy/docs/api.rst new file mode 100644 index 0000000000..7146e024e2 --- /dev/null +++ b/graalpython/hpy/docs/api.rst @@ -0,0 +1,581 @@ +HPy API Introduction +==================== + +Handles +------- + +The "H" in HPy stands for **handle**, which is a central concept: handles are +used to hold a C reference to Python objects, and they are represented by the +C ``HPy`` type. They play the same role as ``PyObject *`` in the ``Python.h`` +API, albeit with some important differences which are detailed below. + +When they are no longer needed, handles must be closed by calling +``HPy_Close``, which plays more or less the same role as ``Py_DECREF``. +Similarly, if you need a new handle for an existing object, you can duplicate +it by calling ``HPy_Dup``, which plays more or less the same role as +``Py_INCREF``. + +The HPy API strictly follows these rules: + +- ``HPy`` handles returned by a function are **never borrowed**, i.e., + the caller must either close or return it. +- ``HPy`` handles passed as function arguments are **never stolen**; + if you receive a ``HPy`` handle argument from your caller, you should never close it. + +These rules makes the code simpler to reason about. Moreover, no reference +borrowing enables the Python implementations to use whatever internal +representation they wish. For example, the object returned by ``HPy_GetItem_i`` +may be created on demand from some compact internal representation, which does +not need to convert itself to full blown representation in order to hold onto +the borrowed object. + +We strongly encourage the users of HPy to also internally follow these rules +for their own internal APIs and helper functions. For the sake of simplicity +and easier local reasoning and also because in the future, code adhering +to those rules may be suitable target for some scalable and precise static +analysis tool. + +The concept of handles is certainly not unique to HPy. Other examples include +Unix file descriptors, where you have ``dup()`` and ``close()``, and Windows' +``HANDLE``, where you have ``DuplicateHandle()`` and ``CloseHandle()``. + + +Handles vs ``PyObject *`` +~~~~~~~~~~~~~~~~~~~~~~~~~ + +In order to fully understand the way HPy handles work, it is useful to discuss +the ``Pyobject *`` pointer in ``Python.h``. These pointers always +point to the same object, and a python object's identity is completely given +by its address in memory, and two pointers with the same address can +be passed to ``Python.h`` API functions interchangeably. As a result, ``Py_INCREF`` +and ``Py_DECREF`` can be called with any reference to an object as long as the +total number of calls of ``incref`` is equal to the number of calls of ``decref`` +at the end of the object lifetime. + +Whereas using HPy API, each handle must be closed independently. + +Thus, the following perfectly valid piece of code using ``Python.h``:: + + void foo(void) + { + PyObject *x = PyLong_FromLong(42); // implicit INCREF on x + PyObject *y = x; + Py_INCREF(y); // INCREF on y + /* ... */ + Py_DECREF(x); + Py_DECREF(x); // two DECREF on x + } + +Becomes using HPy API: + +.. literalinclude:: examples/snippets/snippets.c + :start-after: // BEGIN: foo + :end-before: // END: foo + +Calling any HPy function on a closed handle is an error. Calling +``HPy_Close()`` on the same handle twice is an error. Forgetting to call +``HPy_Close()`` on a handle results in a memory leak. When running in +:ref:`debug-mode:debug mode`, HPy actively checks that you don't +close a handle twice and that you don't forget to close any. + + +.. note:: + Debug mode is a good example of how powerful it is to decouple the + identity and therefore the lifetime of handles and those of objects. + If you find a memory leak on CPython, you know that you are missing a + ``Py_DECREF`` somewhere but the only way to find the corresponding + ``Py_INCREF`` is to manually and carefully study the source code. + On the other hand, if you forget to call ``HPy_Close()``, debug mode + is able to identify the precise code location which created the unclosed + handle. Similarly, if you try to operate on a closed handle, it will + identify the precise code locations which created and closed it. This is + possible because handles are associated with a single call to a C/API + function. As a result, given a handle that is leaked or used after freeing, + it is possible to identify exactly the C/API function that produced it. + + +Remember that ``Python.h`` guarantees that multiple references to the same +object results in the very same ``PyObject *`` pointer. Thus, it is +possible to compare the pointer addresses to check whether they refer +to the same object:: + + int is_same_object(PyObject *x, PyObject *y) + { + return x == y; + } + +On the other hand, in HPy, each handle is independent and it is common to have +two different handles which point to the same underlying object, so comparing +two handles directly is ill-defined. To prevent this kind of common error +(especially when porting existing code to HPy), the ``HPy`` C type is opaque +and the C compiler actively forbids comparisons between them. To check for +identity, you can use ``HPy_Is()``: + +.. literalinclude:: examples/snippets/snippets.c + :start-after: // BEGIN: is_same_object + :end-before: // END: is_same_object + +.. note:: + The main benefit of opaque handle semantics is that implementations are + allowed to use very different models of memory management. On CPython, + implementing handles is trivial because ``HPy`` is basically ``PyObject *`` + in disguise, and ``HPy_Dup()`` and ``HPy_Close()`` are just aliases for + ``Py_INCREF`` and ``Py_DECREF``. + + Unlike CPython, PyPy does not use reference counting to manage memory: + instead, it uses a *moving GC*, which means that the address of an object + might change during its lifetime, and this makes it hard to implement + semantics like ``PyObject *``'s where the address *identifies* the object, + and this is directly exposed to the user. HPy solves this problem: on + PyPy, handles are integers which represent indices into a list, which + is itself managed by the GC. When an address changes, the GC edits the + list, without having to touch all the handles which have been passed to C. + + +HPyContext +----------- + +All HPy function calls take an ``HPyContext`` as a first argument, which +represents the Python interpreter all the handles belong to. Strictly +speaking, it would be possible to design the HPy API without using +``HPyContext``: after all, all HPy function calls are ultimately mapped to +``Python.h`` function call, where there is no notion of context. + +One of the reasons to include ``HPyContext`` from the day one is to be +future-proof: it is conceivable to use it to hold the interpreter or the +thread state in the future, in particular when there will be support for +sub-interpreters. Another possible usage could be to embed different versions +or implementations of Python inside the same process. In addition, the +``HPyContext`` may also be extended by adding new functions to the end without +breaking any extensions built against the current ``HPyContext``. + +Moreover, ``HPyContext`` is used by the :term:`HPy Universal ABI` to contain a +sort of virtual function table which is used by the C extensions to call back +into the Python interpreter. + +.. _simple example: + +A simple example +----------------- + +In this section, we will see how to write a simple C extension using HPy. It +is assumed that you are already familiar with the existing ``Python.h`` API, so we +will underline the similarities and the differences with it. + +We want to create a function named ``myabs`` and ``double`` which takes a +single argument and computes its absolute value: + +.. literalinclude:: examples/simple-example/simple.c + :start-after: // BEGIN: myabs + :end-before: // END: myabs + +There are a couple of points which are worth noting: + + * We use the macro ``HPyDef_METH`` to declare we are going to define a HPy + function called ``myabs``. + + * The function will be available under the name ``"myabs"`` in our Python + module. + + * The actual C function which implements ``myabs`` is called ``myabs_impl`` + and is inferred by the macro. The macro takes the name and adds ``_impl`` + to the end of it. + + * It uses the ``HPyFunc_O`` calling convention. Like ``METH_O`` in ``Python.h``, + ``HPyFunc_O`` means that the function receives a single argument on top of + ``self``. + + * ``myabs_impl`` takes two arguments of type ``HPy``: handles for ``self`` + and the argument, which are guaranteed to be valid. They are automatically + closed by the caller, so there is no need to call ``HPy_Close`` on them. + + * ``myabs_impl`` returns a handle, which has to be closed by the caller. + + * ``HPy_Absolute`` is the equivalent of ``PyNumber_Absolute`` and + computes the absolute value of the given argument. + + * We also do not call ``HPy_Close`` on the result returned to the caller. + We must return a valid handle. + +.. note:: + Among other things, + the ``HPyDef_METH`` macro is needed to maintain compatibility with CPython. + In CPython, C functions and methods have a C signature that is different to + the one used by HPy: they don't receive an ``HPyContext`` and their arguments + have the type ``PyObject *`` instead of ``HPy``. The macro automatically + generates a trampoline function whose signature is appropriate for CPython and + which calls the ``myabs_impl``. This trampoline is then used from both the + CPython ABI and the CPython implementation of the universal ABI, but other + implementations of the universal ABI will usually call directly the HPy + function itself. + +The second function definition is a bit different: + +.. literalinclude:: examples/simple-example/simple.c + :start-after: // BEGIN: double + :end-before: // END: double + +This shows off the other way of creating functions. + + * This example is much the same but the difference is that we use + ``HPyDef_METH_IMPL`` to define a function named ``double``. + + * The difference between ``HPyDef_METH_IMPL`` and ``HPyDef_METH`` is that + the former needs to be given a name for a the functions as the third + argument. + +Now, we can define our module: + +.. literalinclude:: examples/simple-example/simple.c + :start-after: // BEGIN: methodsdef + :end-before: // END: methodsdef + +This part is very similar to the one you would write with ``Python.h``. Note that +we specify ``myabs`` (and **not** ``myabs_impl``) in the method table. There +is also the ``.legacy_methods`` field, which allows to add methods that use the +``Python.h`` API, i.e., the value should be an array of ``PyMethodDef``. This +feature enables support for hybrid extensions in which some of the methods +are still written using the ``Python.h`` API. + +Note that the HPy module does not specify its name. HPy does not support the legacy +single phase module initialization and the only module initialization approach is +the multi-phase initialization (`PEP 489 `_). +With multi-phase module initialization, +the name of the module is always taken from the ``ModuleSpec`` (`PEP 451 `_) +, i.e., most likely from the name used in the ``import {{name}}`` statement that +imported your module. + +This is the only difference stemming from multi-phase module initialization in this +simple example. +As long as there is no need for any further initialization, we can just "register" +our module using the ``HPy_MODINIT`` convenience macro. The first argument is the +name of the extension file and is needed for HPy, among other things, to be able +to generate the entry point for CPython called ``PyInit_{{name}}``. The second argument +is the ``HPyModuleDef`` we just defined. + +.. literalinclude:: examples/simple-example/simple.c + :start-after: // BEGIN: moduledef + :end-before: // END: moduledef + +Building the module +~~~~~~~~~~~~~~~~~~~~ + +Let's write a ``setup.py`` to build our extension: + +.. literalinclude:: examples/simple-example/setup.py + :language: python + +We can now build the extension by running ``python setup.py build_ext -i``. On +CPython, it will target the :term:`CPython ABI` by default, so you will end up with +a file named e.g. ``simple.cpython-37m-x86_64-linux-gnu.so`` which can be +imported directly on CPython with no dependency on HPy. + +To target the :term:`HPy Universal ABI` instead, it is possible to pass the +option ``--hpy-abi=universal`` to ``setup.py``. The following command will +produce a file called ``simple.hpy.so`` (note that you need to specify +``--hpy-abi`` **before** ``build_ext``, since it is a global option):: + + python setup.py --hpy-abi=universal build_ext -i + +.. note:: + This command will also produce a Python file named ``simple.py``, which + loads the HPy module using the ``universal.load`` function from + the ``hpy`` Python package. + +VARARGS calling convention +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If we want to receive more than a single arguments, we need the +``HPy_METH_VARARGS`` calling convention. Let's add a function ``add_ints`` +which adds two integers: + +.. literalinclude:: examples/snippets/hpyvarargs.c + :start-after: // BEGIN: add_ints + :end-before: // END: add_ints + +There are a few things to note: + + * The C signature is different than the corresponding ``Python.h`` + ``METH_VARARGS``: in particular, instead of taking a tuple ``PyObject *args``, + we take an array of ``HPy`` and its size. This allows the call to happen + more efficiently, because you don't need to create a tuple just to pass the + arguments. + + * We call ``HPyArg_Parse`` to parse the arguments. Contrarily to almost all + the other HPy functions, this is **not** a thin wrapper around + ``PyArg_ParseTuple`` because as stated above we don't have a tuple to pass + to it, although the idea is to mimic its behavior as closely as + possible. The parsing logic is implemented from scratch inside HPy, and as + such there might be missing functionality during the early stages of HPy + development. + + * If an error occurs, we return ``HPy_NULL``: we cannot simply ``return NULL`` + because ``HPy`` is not a pointer type. + +Once we have written our function, we can add it to the ``SimpleMethods[]`` +table, which now becomes: + +.. literalinclude:: examples/snippets/hpyvarargs.c + :start-after: // BEGIN: methodsdef + :end-before: // END: methodsdef + +Creating types in HPy +--------------------- + +Creating Python types in an HPy extension is again very similar to the C API +with the difference that HPy only supports creating types from a specification. +This is necessary because there is no such C-level type as ``PyTypeObject`` +since that would expose the internal implementation. + + +Creating a simple type in HPy +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This section assumes that the user wants to define a type that stores some data +in a C-level structure. As an example, we will create a simple C structure +``PointObject`` that represents a two-dimensional point. + +.. literalinclude:: examples/hpytype-example/simple_type.c + :start-after: // BEGIN: PointObject + :end-before: // END: PointObject + +The macro call ``HPyType_HELPERS(PointObject)`` generates useful helper +facilities for working with the type. It generates a C enum +``PointObject_SHAPE`` and a helper function ``PointObject_AsStruct``. The enum +is used in the type specification. The helper function is used to efficiently +retrieving the pointer ``PointObject *`` from an HPy handle to be able to access +the C structure. We will use this helper function to implement the methods, +get-set descriptors, and slots. + +It makes sense to expose fields ``PointObject.x`` and ``PointObject.y`` as +Python-level members. To do so, we need to define members by specifying their +name, type, and location using HPy's convenience macro ``HPyDef_MEMBER``: + +.. literalinclude:: examples/hpytype-example/simple_type.c + :start-after: // BEGIN: members + :end-before: // END: members + +The first argument of the macro is the name for the C glabal variable that will +store the necessary information. We will need that later for registration of +the type. The second, third, and fourth arguments are the Python-level name, the +C type of the member, and the offset in the C structure, respectively. + +Similarly, methods and get-set descriptors can be defined. For example, method +``foo`` is an instance method that takes no arguments (the self argument is, of +course, implicit), does some computation with fields ``x`` and ``y`` and +returns a Python ``int``: + +.. literalinclude:: examples/hpytype-example/simple_type.c + :start-after: // BEGIN: methods + :end-before: // END: methods + +Get-set descriptors are also defined in a very similar way as methods. The +following example defines a get-set descriptor for attribute ``z`` which is +calculated from the ``x`` and ``y`` fields of the struct. + +.. literalinclude:: examples/hpytype-example/simple_type.c + :start-after: // BEGIN: getset + :end-before: // END: getset + +It is also possible to define a get-descriptor or a set-descriptor by using +HPy's macros ``HPyDef_GET`` and ``HPyDef_SET`` in the same way. + +HPy also supports type slots. In this example, we will define slot +``HPy_tp_new`` (which corresponds to magic method ``__new__``) to initialize +fields ``x`` and ``y`` when constructing the object: + +.. literalinclude:: examples/hpytype-example/simple_type.c + :start-after: // BEGIN: slots + :end-before: // END: slots + +After everything was defined, we need to create a list of all defines such that +we are able to eventually register them to the type: + +.. literalinclude:: examples/hpytype-example/simple_type.c + :start-after: // BEGIN: defines + :end-before: // END: defines + +Please note that it is required to terminate the list with ``NULL``. +We can now create the actual type specification by appropriately filling an +``HPyType_Spec`` structure: + +.. literalinclude:: examples/hpytype-example/simple_type.c + :start-after: // BEGIN: spec + :end-before: // END: spec + +First, we need to define the name of the type by setting a C string to member +``name``. Since this type has a C structure, we need to define the ``basicsize`` +and best practice is to set it to ``sizeof(PointObject)``. Also best practice is +to set ``builtin_shape`` to ``PointObject_SHAPE`` where ``PointObject_SHAPE`` is +generated by the previous usage of macro ``HPyType_HELPERS(PointObject)``. Last +but not least, we need to register the defines by setting field ``defines`` to +the previously defined array ``Point_defines``. + +The type specification for the simple type ``simple_type.Point`` represented in +C by structure ``PointObject`` is now complete. All that remains is to create +the type object and add it to the module. + +We will define a module execute slot, which is executed by the runtime right +after the module is created. The purpose of the execute slot is to initialize +the newly created module object. We can then add the type by using +:c:func:`HPyHelpers_AddType`: + +.. literalinclude:: examples/hpytype-example/simple_type.c + :start-after: // BEGIN: add_type + :end-before: // END: add_type + +Also look at the full example at: :doc:`examples/hpytype-example/simple_type`. + + +Legacy types +~~~~~~~~~~~~ + +A type whose struct starts with ``PyObject_HEAD`` (either directly by +embedding it in the type struct or indirectly by embedding another struct like +``PyLongObject``) is a *legacy type*. A legacy type must set +``.builtin_shape = HPyType_BuiltinShape_Legacy`` +in its ``HPyType_Spec``. The counterpart (i.e. a non-legacy type) is called HPy +pure type. + +Legacy types are available to allow gradual porting of existing CPython +extensions. It is possible to reuse existing ``PyType_Slot`` entities (i.e. +slots, methods, members, and get/set descriptors). The idea is that you can then +migrate one after each other while still running the tests. + +The major restriction when using legacy types is that you cannot build a +universal binary of your HPy extension (i.e. you cannot use :term:`HPy Universal +ABI`). The resulting binary will be specific to the Python interpreter used for +building. Therefore, the goal should always be to fully migrate to HPy pure +types. + +A type with ``.legacy_slots != NULL`` is required to have +``HPyType_BuiltinShape_Legacy`` and to include ``PyObject_HEAD`` at the start of +its struct. It would be easy to relax this requirement on CPython (where the +``PyObject_HEAD`` fields are always present) but a large burden on other +implementations (e.g. PyPy, GraalPy) where a struct starting with +``PyObject_HEAD`` might not exist. + +Types created via the old Python C API are automatically legacy types. + +This section does not provide a dedicated example for how to create and use +legacy types because the :doc:`porting-example/index` already shows how that +is useful during incremental migration to HPy. + +Inherit from a built-in type +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +HPy also supports inheriting from following built-in types: + + * ``type`` + + * ``int`` + + * ``float`` + + * ``unicode`` + + * ``tuple`` + + * ``list`` + +Inheriting from built-in types is straight forward if you don't have a C +structure that represents your type. In other words, you can simply inherit +from, e.g., ``str`` if the ``basicsize`` in your type specification is ``0``. +For example: + +.. literalinclude:: examples/hpytype-example/builtin_type.c + :start-after: // BEGIN: spec_Dummy + :end-before: // END: spec_Dummy + +.. literalinclude:: examples/hpytype-example/builtin_type.c + :start-after: // BEGIN: add_Dummy + :end-before: // END: add_Dummy + +This case is simple because there is no ``Dummy_AsStruct`` since there is no +associated C-level structure. + +It is, however, more involved if your type also defines its own C structure +(i.e. ``basicsize > 0`` in the type specification). In this case, it is strictly +necessary to use the right *built-in shape*. + +**What is the right built-in shape?** + +This question is easy to answer: Each built-in shape (except of +:c:enumerator:`HPyType_BuiltinShape_Legacy`) represents a built-in type. You +need to use the built-in shape that fits to the specified base class. The +mapping is described in :c:enum:`HPyType_BuiltinShape`. + +Let's do an example. Assume we want to define a type that stores the natural +language of a unicode string to the unicode object but the object should still +just behave like a Python unicode object. So, we define struct +``LanguageObject``: + +.. literalinclude:: examples/hpytype-example/builtin_type.c + :start-after: // BEGIN: LanguageObject + :end-before: // END: LanguageObject + +As you can see, we already specify the built-in shape here using +``HPyType_HELPERS(LanguageObject, HPyType_BuiltinShape_Unicode)``. Then, in the +type specification, we do: + +.. literalinclude:: examples/hpytype-example/builtin_type.c + :start-after: // BEGIN: spec_Language + :end-before: // END: spec_Language + +In the last step, when actually creating the type from the specification, we +need to define that its base class is ``str`` (aka. ``UnicodeType``): + +.. literalinclude:: examples/hpytype-example/builtin_type.c + :start-after: // BEGIN: add_Language + :end-before: // END: add_Language + +Function ``LanguageObject_AsStruct`` (which is generated by ``HPyType_HELPERS``) +will then return a pointer to ``LanguageObject``. + +To summarize this: Specifying a type that inherits from a built-in type needs to +be considered in three places: + +1. Pass the appropriate built-in shape to :c:macro:`HPyType_HELPERS`. +2. Assign ``SHAPE(TYPE)`` to :c:member:`HPyType_Spec.builtin_shape`. +3. Specify the desired base class in the type specification parameters. + +For more information about the built-in shape and for a technical explanation +for why it is required, see :c:member:`HPyType_Spec.builtin_shape` and +:c:enum:`HPyType_BuiltinShape`. + +More Examples +------------- + +The :doc:`porting-example/index` shows another complete example +of HPy extension ported from Python/C API. + +The `HPy project space `_ on GitHub +contains forks of some popular Python extensions ported to HPy as +a proof of concept/feasibility studies, such as the +`Kiwi solver `_. +Note that those forks may not be up to date with their upstream projects +or with the upstream HPy changes. + +HPy unit tests +~~~~~~~~~~~~~~ + +HPy usually has tests for each API function. This means that there is lots of +examples available by looking at the tests. However, the test source uses +many macros and is hard to read. To overcome this we supply a utility to +export clean C sources for the tests. Since the HPy tests are not shipped by +default, you need to clone the HPy repository from GitHub: + +.. code-block:: console + + > git clone https://github.com/hpyproject/hpy.git + +After that, install all test requirements and dump the sources: + +.. code-block:: console + + > cd hpy + > python3 -m pip install pytest filelock + > python3 -m pytest --dump-dir=test_sources test/ + +This will dump the generated test sources into folder ``test_sources``. Note, +that the tests won't be executed but skipped with an appropriate message. diff --git a/graalpython/hpy/docs/changelog.rst b/graalpython/hpy/docs/changelog.rst new file mode 100644 index 0000000000..e2915b98d3 --- /dev/null +++ b/graalpython/hpy/docs/changelog.rst @@ -0,0 +1,235 @@ +Changelog +========= + +Version 0.9 (April 25th, 2023) +------------------------------ + +This release adds numerous major features and indicates the end of HPy's *alhpa* +phase. We've migrated several key packages to HPy (for a list, see our website +https://hpyproject.org) and we are now confident that HPy is mature enough for +being used as serious extension API. We also plan that the next major release +will be ``1.0``. + +Major new features +~~~~~~~~~~~~~~~~~~ + +Support subclasses of built-in types + It is now possible to create pure HPy types that inherit from built-in types + like ``type`` or ``float``. This was already possible before but in a very + limited way, i.e., by setting :c:member:`HPyType_Spec.basicsize` to ``0``. In + this case, the type implicitly inherited the basic size of the supertype but + that also means that you cannot have a custom C struct. It is now possible + inherit from a built-in type **AND** have a custom C struct. For further + reference, see :c:member:`HPyType_Spec.builtin_shape` and + :c:enum:`HPyType_BuiltinShape`. + +Support for metaclasses + HPy now supports creating types with metaclasses. This can be done by passing + type specification parameter with kind + :c:enumerator:`HPyType_SpecParam_Metaclass` when calling + :c:func:`HPyType_FromSpec`. + +:term:`HPy Hybrid ABI` + In addition to :term:`CPython ABI` and :term:`HPy Universal ABI`, we now + introduced the Hybrid ABI. The major difference is that whenever you use a + legacy API like :c:func:`HPy_AsPyObject` or :c:func:`HPy_FromPyObject`, the + prdouced binary will then be specific to one interpreter. This was necessary + to ensure that universal binaries are really portable and can be used on any + HPy-capable interpreter. + +:doc:`trace-mode` + Similar to the :doc:`debug-mode`, HPy now provides the Trace Mode that can be + enabled at runtime and helps analyzing API usage and identifying performance + issues. + +:ref:`porting-guide:multi-phase module initialization` + HPy now support multi-phase module initialization which is an important + feature in particular needed for two important use cases: (1) module state + support (which is planned to be introduced in the next major release), and (2) + subinterpreters. We decided to drop support for single-phase module + initialization since this makes the API cleaner and easier to use. + +HPy :ref:`porting-guide:calling protocol` + This was a big missing piece and is now eventually available. It enables slot + ``HPy_tp_call``, which can now be used in the HPy type specification. We + decided to use a calling convention similar to CPython's vectorcall calling + convention. This is: the arguments are passed in a C array and the keyword + argument names are provided as a Python tuple. Before this release, the only + way to create a callable type was to set the special method ``__call__``. + However, this has several disadvantages. In particlar, poor performance on + CPython (and maybe other implementations) and it was not possible to have + specialized call function implementations per object (see + :c:func:`HPy_SetCallFunction`) + +Added APIs +~~~~~~~~~~ + +Deleting attributes and items + :c:func:`HPy_DelAttr`, :c:func:`HPy_DelAttr_s`, :c:func:`HPy_DelItem`, :c:func:`HPy_DelItem_i`, :c:func:`HPy_DelItem_s` + +Capsule API + :c:func:`HPyCapsule_New`, :c:func:`HPyCapsule_IsValid`, :c:func:`HPyCapsule_Get`, :c:func:`HPyCapsule_Set` + +Eval API + :c:func:`HPy_Compile_s` and :c:func:`HPy_EvalCode` + +Formatting helpers + :c:func:`HPyUnicode_FromFormat` and :c:func:`HPyErr_Format` + +Contextvar API + :c:func:`HPyContextVar_New`, :c:func:`HPyContextVar_Get`, :c:func:`HPyContextVar_Set` + +Unicode API + :c:func:`HPyUnicode_FromEncodedObject` and :c:func:`HPyUnicode_Substring` + +Dict API + :c:func:`HPyDict_Keys` and :c:func:`HPyDict_Copy` + +Type API + :c:func:`HPyType_GetName` and :c:func:`HPyType_IsSubtype` + +Slice API + :c:func:`HPySlice_Unpack` and :c:func:`HPySlice_AdjustIndices` + +Structseq API + :c:func:`HPyStructSequence_NewType`, :c:func:`HPyStructSequence_New` + +Call API + :c:func:`HPy_Call`, :c:func:`HPy_CallMethod`, :c:func:`HPy_CallMethodTupleDict`, :c:func:`HPy_CallMethodTupleDict_s` + +HPy call protocol + :c:func:`HPy_SetCallFunction` + +Debug mode +~~~~~~~~~~ + +* Detect closing and returning (without dup) of context handles +* Detect invalid usage of stored ``HPyContext *`` pointer +* Detect invalid usage of tuple and list builders +* Added Windows support for checking invalid use of raw data pointers (e.g + ``HPyUnicode_AsUTF8AndSize``) after handle was closed. +* Added support for backtrace on MacOS + +Documentation +~~~~~~~~~~~~~ + +* Added incremental :doc:`porting-example/index` +* Added :doc:`quickstart` guide +* Extended :doc:`api-reference/index` +* Added :doc:`api-reference/function-index` +* Added possiblity to generate examples from tests with argument ``--dump-dir`` + (see :ref:`api:hpy unit tests`) +* Added initial :doc:`contributing/index` docs + +Incompatible changes to version 0.0.4 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* Simplified ``HPyDef_*`` macros +* Changed macro :c:macro:`HPy_MODINIT` because of multi-phase module init + support. +* Replace environment variable ``HPY_DEBUG`` by ``HPY`` (see :doc:`debug-mode` + or :doc:`trace-mode`). +* Changed signature of ``HPyFunc_VARARGS`` and ``HPyFunc_ KEYWORDS`` to align + with HPy's call protocol calling convention. + +Supported Python versions +~~~~~~~~~~~~~~~~~~~~~~~~~ + +* Added Python 3.11 support +* Preliminary Python 3.12 support +* Dropped Python 3.6 support (since EOL) +* Dropped Python 3.7 support (since EOL by June 2023) + +Misc +~~~~ + +* Ensure deterministic auto-generation +* Ensure ABI backwards compatibility + + * Explicitly define slot within HPyContext of function pointers and handles + * Compile HPy ABI version into binary and verify at load time +* Added proper support for object members ``HPyMember_OBJECT`` +* Changed :c:func:`HPyBytes_AsString` and :c:func:`HPyBytes_AS_STRING` to return ``const char *`` +* Use fixed-width integers in context functions + + + +Version 0.0.4 (May 25th, 2022) +------------------------------ + +New Features/API: + + - HPy headers are C++ compliant + - Python 3.10 support + - `HPyField `_: + References to Python objects that can be stored in raw native memory owned by Python objects. + + - New API functions: ``HPyField_Load``, ``HPyField_Store`` + - `HPyGlobal `_: + References to Python objects that can be stored into a C global variable. + + - New API functions: ``HPyGlobal_Load``, ``HPyGlobal_Store`` + - Note: ``HPyGlobal`` does not allow to share Python objects between (sub)interpreters + + - `GIL support `_ + - New API functions: ``HPy_ReenterPythonExecution``, ``HPy_LeavePythonExecution`` + + - `Value building support `_ (``HPy_BuildValue``) + + - New type slots + + - ``HPy_mp_ass_subscript``, ``HPy_mp_length``, ``HPy_mp_subscript`` + - ``HPy_tp_finalize`` + + - Other new API functions + + - ``HPyErr_SetFromErrnoWithFilename``, ``HPyErr_SetFromErrnoWithFilenameObjects`` + - ``HPyErr_ExceptionMatches`` + - ``HPyErr_WarnEx`` + - ``HPyErr_WriteUnraisable`` + - ``HPy_Contains`` + - ``HPyLong_AsVoidPtr`` + - ``HPyLong_AsDouble`` + - ``HPyUnicode_AsASCIIString``, ``HPyUnicode_DecodeASCII`` + - ``HPyUnicode_AsLatin1String``, ``HPyUnicode_DecodeLatin1`` + - ``HPyUnicode_DecodeFSDefault``, ``HPyUnicode_DecodeFSDefaultAndSize`` + - ``HPyUnicode_ReadChar`` + +Debug mode: + + - Support activation of debug mode via environment variable ``HPY_DEBUG`` + - Support capturing stack traces of handle allocations + - Check for invalid use of raw data pointers (e.g ``HPyUnicode_AsUTF8AndSize``) after handle was closed. + - Detect invalid handles returned from extension functions + - Detect incorrect closing of handles passed as arguments + +Misc Changes: + + - Removed unnecessary prefix ``"m_"`` from fields of ``HPyModuleDef`` (incompatible change) + - For HPy implementors: new pytest mark for HPy tests assuming synchronous GC + +Version 0.0.3 (September 22nd, 2021) +------------------------------------ + +This release adds various new API functions (see below) and extends the debug +mode with the ability to track closed handles. +The default ABI mode now is 'universal' for non-CPython implementations. +Also, the type definition of ``HPyContext`` was changed and it's no longer a +pointer type. +The name of the HPy dev package was changed to 'hpy' (formerly: 'hpy.devel'). +Macro HPy_CAST was replaced by HPy_AsStruct. + +New features: + + - Added helper HPyHelpers_AddType for creating new types + - Support format specifier 's' in HPyArg_Parse + - Added API functions: HPy_Is, HPy_AsStructLegacy (for legacy types), + HPyBytes_FromStringAndSize, HPyErr_NewException, HPyErr_NewExceptionWithDoc, + HPyUnicode_AsUTF8AndSize, HPyUnicode_DecodeFSDefault, HPyImport_ImportModule + - Debug mode: Implemented tracking of closed handles + - Debug mode: Add hook for invalid handle access + +Bug fixes: + + - Distinguish between pure and legacy types + - Fix Sphinx doc errors diff --git a/graalpython/hpy/docs/conf.py b/graalpython/hpy/docs/conf.py new file mode 100644 index 0000000000..45a9077616 --- /dev/null +++ b/graalpython/hpy/docs/conf.py @@ -0,0 +1,134 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + +import sys +import os +import re +import datetime + +# -- Project information ----------------------------------------------------- + +project = "HPy" +copyright = "2019-{}, HPy Collective".format(datetime.date.today().year) +author = "HPy Collective" + +# The full version, including alpha/beta/rc tags +release = "0.9" + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + "sphinx.ext.autosectionlabel", + "sphinx_c_autodoc", + "sphinx_c_autodoc.viewcode", + "sphinx_rtd_theme", +] + +autosectionlabel_prefix_document = True + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + +# -- autodoc ----------------------------------------------------------------- + +autodoc_member_order = "bysource" + +# -- sphinx_c_autodoc -------------------------------------------------------- + +c_autodoc_roots = [ + "../hpy/devel/include", + "../hpy/devel/src", + "../hpy/tools", +] + + +def pre_process(app, filename, contents, *args): + # FIXME: the missing typedef for 'HPy' causes the file formatting to fail + if filename.endswith('public_api.h'): + contents[0] = '#include "../../devel/include/hpy.h"\n' + contents[0] + if filename.endswith('autogen_ctx.h'): + contents[0] = 'typedef int HPy;' + contents[0] + + # remove HPyAPI_HELPER so that the sphinx-c-autodoc and clang + # find and render the API functions + contents[:] = [ + re.sub(r"^(HPyAPI_HELPER|HPyAPI_INLINE_HELPER|HPy_ID\(\d+\))", r"", part, flags=re.MULTILINE) + for part in contents + ] + + +def setup(app): + app.connect("c-autodoc-pre-process", pre_process) + + +def setup_clang(): + """ + Make sure clang is set up correctly for the sphinx_c_autodoc extension. + + The Python clang package requires a matching libclang*.so. Our + ``doc/requirements.txt`` file specifies ``clang==10.0.1``, so we need + ``libclang-10``. + + On Ubuntu 20.04 (and possibly later), this can be installed with + ``apt install libclang-10-dev`` and the Python clang package finds the + appropriate .so automatically. + + However, ReadTheDocs has an older Ubuntu that only packages libclang-6.0. + The Python ``clang==10.0.1`` packages supports this older .so, but + needs to be explicitly told where to find it. + + If you encounter issues with a local build, please start by checking that + the ``libclang-10-dev`` system package or equivalent is installed. + """ + from clang import cindex + if 'READTHEDOCS' in os.environ: + # TODO: Hopefully we can remove this setting of the libclang path once + # readthedocs updates its docker image to Ubuntu 20.04 which + # supports clang-10 and clang-11. + cindex.Config.set_library_file( + "/usr/lib/x86_64-linux-gnu/libclang-6.0.so.1" + ) + elif sys.platform == "darwin": + cindex.Config.set_library_file( + "/Library/Developer/CommandLineTools/usr/lib/libclang.dylib" + ) + + +setup_clang() + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = "sphinx_rtd_theme" + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["_static"] + +# most the the code examples will be in C, let's make it the default +highlight_language = 'C' diff --git a/graalpython/hpy/docs/contributing/index.rst b/graalpython/hpy/docs/contributing/index.rst new file mode 100644 index 0000000000..1dba877a90 --- /dev/null +++ b/graalpython/hpy/docs/contributing/index.rst @@ -0,0 +1,33 @@ +Contributing +============ + +Getting Started +--------------- + +TBD + + +Adding New API +-------------- + +1. Add the function to ``hpy/tools/autogen/public_api.h``. If the CPython + equivalent function name is not the same (after removing the leading ``H``, + add an appropriate CPython function mapping in ``hpy/tools/autogen/conf.py``. + If the signature is complicated or there is no clear equivalent function, + the mapping should be ``None``, and follow the directions in the next step. + Otherwise all the needed functions will be autogenerated. + +2. If the function cannot be autogenerated (i.e. the mapping does not exist), + you must write the wrapper by hand. Add the function to ``NO_WRAPPER`` in + ``hpy/tools/autogen/debug.py``, and add a ``ctx_fname`` function to + ``hyp/devel/src/runtime/*.c`` (possibly adding the new file to ``setup.py``), + add a debug wrapper to ``hpy/debug/src/debug_ctx.c``, add a implementation + that uses the ``ctx`` variant to ``hpy/devel/include/hpy/cpython/misc.h`` and + add the declaration to ``hpy/devel/include/hpy/runtime/ctx_funcs.h``. + +3. Run ``make autogen`` which will turn the mapping into autogenerated functions + +4. Add a test for the functionality + +5. Build with ``python setup.py build_ext``. After that works, build with + ``python -m pip install -e .``, then run the test with ``python -m pytest ...``. diff --git a/graalpython/hpy/docs/debug-mode.rst b/graalpython/hpy/docs/debug-mode.rst new file mode 100644 index 0000000000..0f59bc9486 --- /dev/null +++ b/graalpython/hpy/docs/debug-mode.rst @@ -0,0 +1,169 @@ +Debug Mode +========== + +HPy includes a debug mode which does useful run-time checks to ensure that C +extensions use the API correctly. Its features include: + + 1. No special compilation flags are required: it is enough to compile the + extension with the Universal ABI. + + 2. Debug mode can be activated at *import time*, and it can be activated + per-extension. + + 3. You pay the overhead of debug mode only if you use it. Extensions loaded + without the debug mode run at full speed. + +This is possible because the whole of the HPy API is provided as part of the HPy +context, so debug mode can pass in a special debugging context without affecting +the performance of the regular context at all. + +.. note:: The debug mode is only available if the module (you want to use it + for) was built for :term:`HPy Universal ABI`. + +The debugging context can already check for: + +* Leaked handles. +* Handles used after they are closed. +* Tuple and list builder used after they were *closed* (i.e. cancelled or the + tuple/list was built). +* Reading from a memory which is no longer guaranteed to be still valid, + for example, the buffer returned by :c:func:`HPyUnicode_AsUTF8AndSize`, + :c:func:`HPyBytes_AsString`, and :c:func:`HPyBytes_AS_STRING`, after the + corresponding ``HPy`` handle was closed. +* Writing to memory which should be read-only, for example the buffer + returned by :c:func:`HPyUnicode_AsUTF8AndSize`, :c:func:`HPyBytes_AsString`, + and :c:func:`HPyBytes_AS_STRING` + + +Activating Debug Mode +--------------------- + +Debug mode works *only* for extensions built with HPy universal ABI. + +To enable debug mode, use environment variable ``HPY``. If ``HPY=debug``, then +all HPy modules are loaded with the trace context. Alternatively, it is also +possible to specify the mode per module like this: +``HPY=modA:debug,modB:debug``. + +In order to verify that your extension is being loaded in debug mode, use +environment variable ``HPY_LOG``. If this variable is set, then all HPy +extensions built in universal ABI mode print a message when loaded, such as: + +.. code-block:: console + + > import snippets + Loading 'snippets' in HPy universal mode with a debug context + +.. Note: the output above is tested in test_leak_detector_with_traces_output + +If the extension is built in CPython ABI mode, then the ``HPY_LOG`` environment +variable has no effect. + +An HPy extension module may be also explicitly loaded in debug mode using: + +.. code-block:: python + + from hpy.universal import load, MODE_DEBUG + mod = load(module_name, so_filename, mode=MODE_DEBUG) + +When loading HPy extensions explicitly, environment variables ``HPY_LOG`` +and ``HPY`` have no effect for that extension. + + +Using Debug Mode +---------------- + +By default, when debug mode detects an error it will either abort the process +(using :c:func:`HPy_FatalError`) or raise a fatal exception. This may sound very +strict but in general, it is not safe to continue the execution. + +When testing, aborting the process is unwanted. Module ``hpy.debug`` exposes the +``LeakDetector`` class to detect leaked ``HPy`` handles. For example: + +.. literalinclude:: examples/tests.py + :language: python + :start-at: def test_leak_detector + :end-before: # END: test_leak_detector + +Additionally, the debug module also provides a pytest fixture, ``hpy_debug``, +that for the time being, enables the ``LeakDetector``. In the future, it +may also enable other useful debugging facilities. + +.. literalinclude:: examples/tests.py + :language: python + :start-at: from hpy.debug.pytest import hpy_debug + :end-at: # Run some HPy extension code + +.. warning:: The usage of ``LeakDetector`` or ``hpy_debug`` by itself does not + enable HPy debug mode! If debug mode is not enabled for any extension, then + those features have no effect. + +When dealing with handle leaks, it is useful to get a stack trace of the +allocation of the leaked handle. This feature has large memory requirements +and is therefore opt-in. It can be activated by: + +.. literalinclude:: examples/tests.py + :language: python + :start-at: hpy.debug.set_handle_stack_trace_limit + :end-at: hpy.debug.set_handle_stack_trace_limit + +and disabled by: + +.. literalinclude:: examples/tests.py + :language: python + :start-at: hpy.debug.disable_handle_stack_traces + :end-at: hpy.debug.disable_handle_stack_traces + + +Example +------- + +.. note: The following output is tested in test_leak_detector_with_traces_output + +Following HPy function leaks a handle: + +.. literalinclude:: examples/snippets/snippets.c + :start-after: // BEGIN: test_leak_stacktrace + :end-before: // END: test_leak_stacktrace + +When this script is executed in debug mode: + +.. literalinclude:: examples/debug-example.py + :language: python + :end-before: print("SUCCESS") + +The output is:: + + Traceback (most recent call last): + File "/path/to/hpy/docs/examples/debug-example.py", line 7, in + snippets.test_leak_stacktrace() + File "/path/to/hpy/debug/leakdetector.py", line 43, in __exit__ + self.stop() + File "/path/to/hpy/debug/leakdetector.py", line 36, in stop + raise HPyLeakError(leaks) + hpy.debug.leakdetector.HPyLeakError: 1 unclosed handle: + + Allocation stacktrace: + /path/to/site-packages/hpy-0.0.4.dev227+gd7eeec6.d20220510-py3.8-linux-x86_64.egg/hpy/universal.cpython-38d-x86_64-linux-gnu.so(debug_ctx_Long_FromLong+0x45) [0x7f1d928c48c4] + /path/to/site-packages/hpy_snippets-0.0.0-py3.8-linux-x86_64.egg/snippets.hpy.so(+0x122c) [0x7f1d921a622c] + /path/to/site-packages/hpy_snippets-0.0.0-py3.8-linux-x86_64.egg/snippets.hpy.so(+0x14b1) [0x7f1d921a64b1] + /path/to/site-packages/hpy-0.0.4.dev227+gd7eeec6.d20220510-py3.8-linux-x86_64.egg/hpy/universal.cpython-38d-x86_64-linux-gnu.so(debug_ctx_CallRealFunctionFromTrampoline+0xca) [0x7f1d928bde1e] + /path/to/site-packages/hpy_snippets-0.0.0-py3.8-linux-x86_64.egg/snippets.hpy.so(+0x129b) [0x7f1d921a629b] + /path/to/site-packages/hpy_snippets-0.0.0-py3.8-linux-x86_64.egg/snippets.hpy.so(+0x1472) [0x7f1d921a6472] + /path/to/libpython3.8d.so.1.0(+0x10a022) [0x7f1d93807022] + /path/to/libpython3.8d.so.1.0(+0x1e986b) [0x7f1d938e686b] + /path/to/libpython3.8d.so.1.0(+0x2015e9) [0x7f1d938fe5e9] + /path/to/libpython3.8d.so.1.0(_PyEval_EvalFrameDefault+0x1008c) [0x7f1d938f875a] + /path/to/libpython3.8d.so.1.0(PyEval_EvalFrameEx+0x64) [0x7f1d938e86b8] + /path/to/libpython3.8d.so.1.0(_PyEval_EvalCodeWithName+0xfaa) [0x7f1d938fc8af] + /path/to/libpython3.8d.so.1.0(PyEval_EvalCodeEx+0x86) [0x7f1d938fca25] + /path/to/libpython3.8d.so.1.0(PyEval_EvalCode+0x4b) [0x7f1d938e862b] + +For the time being, HPy uses the glibc ``backtrace`` and ``backtrace_symbols`` +`functions `_. +Therefore all their caveats and limitations apply. Usual recommendations to get +more symbols in the traces and not only addresses, such as ``snippets.hpy.so(+0x122c)``, are: + +* link your native code with ``-rdynamic`` flag (``LDFLAGS="-rdynamic"``) +* build your code without optimizations and with debug symbols (``CFLAGS="-O0 -g"``) +* use ``addr2line`` command line utility, e.g.: ``addr2line -e /path/to/snippets.hpy.so -C -f +0x122c`` diff --git a/graalpython/hpy/docs/examples/debug-example.py b/graalpython/hpy/docs/examples/debug-example.py new file mode 100644 index 0000000000..0cd16cae36 --- /dev/null +++ b/graalpython/hpy/docs/examples/debug-example.py @@ -0,0 +1,10 @@ +# Run with HPY=debug +import hpy.debug +import snippets + +hpy.debug.set_handle_stack_trace_limit(16) +from hpy.debug.pytest import LeakDetector +with LeakDetector() as ld: + snippets.test_leak_stacktrace() + +print("SUCCESS") # Should not be actually printed diff --git a/graalpython/hpy/docs/examples/hpytype-example/builtin_type.c b/graalpython/hpy/docs/examples/hpytype-example/builtin_type.c new file mode 100644 index 0000000000..5ec749b690 --- /dev/null +++ b/graalpython/hpy/docs/examples/hpytype-example/builtin_type.c @@ -0,0 +1,106 @@ +#include +#include + +// BEGIN: spec_Dummy +static HPyType_Spec Dummy_spec = { + .name = "builtin_type.Dummy", + .basicsize = 0 +}; + +// END: spec_Dummy +static void make_Dummy(HPyContext *ctx, HPy module) +{ +// BEGIN: add_Dummy + HPyType_SpecParam param[] = { + { HPyType_SpecParam_Base, ctx->h_UnicodeType }, + { (HPyType_SpecParam_Kind)0 } + }; + if (!HPyHelpers_AddType(ctx, module, "Dummy", &Dummy_spec, param)) + return; +// END: add_Dummy +} + +// BEGIN: LanguageObject +typedef struct { + char *language; +} LanguageObject; +HPyType_HELPERS(LanguageObject, HPyType_BuiltinShape_Unicode) +// END: LanguageObject + +HPyDef_GETSET(Language_lang, "lang") +static HPy Language_lang_get(HPyContext *ctx, HPy self, void *closure) +{ + LanguageObject *data = LanguageObject_AsStruct(ctx, self); + return HPyUnicode_FromString(ctx, data->language); +} +static int Language_lang_set(HPyContext *ctx, HPy self, HPy value, void *closure) +{ + LanguageObject *data = LanguageObject_AsStruct(ctx, self); + HPy_ssize_t size; + const char *s = HPyUnicode_AsUTF8AndSize(ctx, value, &size); + if (s == NULL) + return -1; + data->language = (char *)calloc(size+1, sizeof(char)); + strncpy(data->language, s, size); + return 0; +} + +HPyDef_SLOT(Language_destroy, HPy_tp_destroy) +static void Language_destroy_impl(void *data) +{ + LanguageObject *ldata = (LanguageObject *)data; + if (ldata->language) + free(ldata->language); +} + +HPyDef *Language_defines[] = { + &Language_lang, + &Language_destroy, + NULL +}; + + +// BEGIN: spec_Language +static HPyType_Spec Language_spec = { + .name = "builtin_type.Language", + .basicsize = sizeof(LanguageObject), + .builtin_shape = SHAPE(LanguageObject), + .defines = Language_defines +}; +// END: spec_Language + +static void make_Language(HPyContext *ctx, HPy module) +{ +// BEGIN: add_Language + HPyType_SpecParam param[] = { + { HPyType_SpecParam_Base, ctx->h_UnicodeType }, + { (HPyType_SpecParam_Kind)0 } + }; + if (!HPyHelpers_AddType(ctx, module, "Language", &Language_spec, param)) + return; +// END: add_Language +} + +HPyDef_SLOT(simple_exec, HPy_mod_exec) +static int simple_exec_impl(HPyContext *ctx, HPy m) { + make_Dummy(ctx, m); + if (HPyErr_Occurred(ctx)) + return -1; + + make_Language(ctx, m); + if (HPyErr_Occurred(ctx)) + return -1; + + return 0; // success +} + +static HPyDef *mod_defines[] = { + &simple_exec, // 'simple_exec' is generated by the HPyDef_SLOT macro + NULL, +}; + +static HPyModuleDef moduledef = { + .defines = mod_defines +}; + +HPy_MODINIT(builtin_type, moduledef) diff --git a/graalpython/hpy/docs/examples/hpytype-example/setup.py b/graalpython/hpy/docs/examples/hpytype-example/setup.py new file mode 100644 index 0000000000..f740a424e1 --- /dev/null +++ b/graalpython/hpy/docs/examples/hpytype-example/setup.py @@ -0,0 +1,11 @@ +from setuptools import setup, Extension +from os import path + +setup( + name="hpy-type-example", + hpy_ext_modules=[ + Extension('simple_type', sources=['simple_type.c']), + Extension('builtin_type', sources=['builtin_type.c']), + ], + setup_requires=['hpy'], +) diff --git a/graalpython/hpy/docs/examples/hpytype-example/simple_type.c b/graalpython/hpy/docs/examples/hpytype-example/simple_type.c new file mode 100644 index 0000000000..40b1fba715 --- /dev/null +++ b/graalpython/hpy/docs/examples/hpytype-example/simple_type.c @@ -0,0 +1,102 @@ +#include + +// BEGIN: PointObject +typedef struct { + long x; + long y; +} PointObject; +HPyType_HELPERS(PointObject) +// END: PointObject + +// BEGIN: members +HPyDef_MEMBER(Point_x, "x", HPyMember_LONG, offsetof(PointObject, x)) +HPyDef_MEMBER(Point_y, "y", HPyMember_LONG, offsetof(PointObject, y)) +// END: members + +// BEGIN: methods +HPyDef_METH(Point_foo, "foo", HPyFunc_NOARGS) +static HPy Point_foo_impl(HPyContext *ctx, HPy self) +{ + PointObject *point = PointObject_AsStruct(ctx, self); + return HPyLong_FromLong(ctx, point->x * 10 + point->y); +} +// END: methods + +// BEGIN: getset +HPyDef_GETSET(Point_z, "z", .closure=(void *)1000) +static HPy Point_z_get(HPyContext *ctx, HPy self, void *closure) +{ + PointObject *point = PointObject_AsStruct(ctx, self); + return HPyLong_FromLong(ctx, point->x*10 + point->y + (long)(HPy_ssize_t)closure); +} + +static int Point_z_set(HPyContext *ctx, HPy self, HPy value, void *closure) +{ + PointObject *point = PointObject_AsStruct(ctx, self); + long current = point->x*10 + point->y + (long)(HPy_ssize_t)closure; + long target = HPyLong_AsLong(ctx, value); // assume no exception + point->y += target - current; + return 0; +} +// END: getset + +// BEGIN: slots +HPyDef_SLOT(Point_new, HPy_tp_new) +static HPy Point_new_impl(HPyContext *ctx, HPy cls, const HPy *args, + HPy_ssize_t nargs, HPy kw) +{ + long x, y; + if (!HPyArg_Parse(ctx, NULL, args, nargs, "ll", &x, &y)) + return HPy_NULL; + PointObject *point; + HPy h_point = HPy_New(ctx, cls, &point); + if (HPy_IsNull(h_point)) + return HPy_NULL; + point->x = x; + point->y = y; + return h_point; +} +// END: slots + +// BEGIN: defines +static HPyDef *Point_defines[] = { + &Point_x, + &Point_y, + &Point_z, + &Point_new, + &Point_foo, + NULL +}; +// END: defines + +// BEGIN: spec +static HPyType_Spec Point_spec = { + .name = "simple_type.Point", + .basicsize = sizeof(PointObject), + .builtin_shape = PointObject_SHAPE, + .defines = Point_defines +}; +// END: spec + +// BEGIN: add_type +HPyDef_SLOT(simple_exec, HPy_mod_exec) +static int simple_exec_impl(HPyContext *ctx, HPy m) { + if (!HPyHelpers_AddType(ctx, m, "Point", &Point_spec, NULL)) { + return -1; + } + return 0; // success +} + +static HPyDef *mod_defines[] = { + &simple_exec, // 'simple_exec' is generated by the HPyDef_SLOT macro + NULL, +}; + +static HPyModuleDef moduledef = { + .defines = mod_defines, + // ... +// END: add_type + .doc = "A simple HPy type", +}; + +HPy_MODINIT(simple_type, moduledef) \ No newline at end of file diff --git a/graalpython/hpy/docs/examples/hpytype-example/simple_type.rst b/graalpython/hpy/docs/examples/hpytype-example/simple_type.rst new file mode 100644 index 0000000000..d611e5113c --- /dev/null +++ b/graalpython/hpy/docs/examples/hpytype-example/simple_type.rst @@ -0,0 +1,8 @@ +:orphan: + +simple_type.c +============= + +.. literalinclude:: ./simple_type.c + :language: c + :linenos: diff --git a/graalpython/hpy/docs/examples/mixed-example/mixed.c b/graalpython/hpy/docs/examples/mixed-example/mixed.c new file mode 100644 index 0000000000..d041f3f41c --- /dev/null +++ b/graalpython/hpy/docs/examples/mixed-example/mixed.c @@ -0,0 +1,49 @@ +/* Simple C module that shows how to mix CPython API and HPY. + * At the moment, this code is not referenced from the documentation, but it is + * tested nonetheless. + */ + +#include "hpy.h" + +/* a HPy style function */ +HPyDef_METH(add_ints, "add_ints", HPyFunc_VARARGS) +static HPy add_ints_impl(HPyContext *ctx, HPy self, const HPy *args, size_t nargs) +{ + long a, b; + if (!HPyArg_Parse(ctx, NULL, args, nargs, "ll", &a, &b)) + return HPy_NULL; + return HPyLong_FromLong(ctx, a+b); +} + + +/* Add an old-style function */ +static PyObject * +add_ints2(PyObject *self, PyObject *args) +{ + + long a, b, ret; + if (!PyArg_ParseTuple(args, "ll", &a, &b)) + return NULL; + ret = a + b; + return PyLong_FromLong(ret); +} + +static HPyDef *hpy_defines[] = { + &add_ints, + NULL +}; + +static PyMethodDef py_defines[] = { + {"add_ints_legacy", add_ints2, METH_VARARGS, "add two ints"}, + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +static HPyModuleDef moduledef = { + .doc = "HPy Example of mixing CPython API and HPy API", + .size = 0, + .defines = hpy_defines, + .legacy_methods = py_defines +}; + + +HPy_MODINIT(mixed, moduledef) diff --git a/graalpython/hpy/docs/examples/mixed-example/setup.py b/graalpython/hpy/docs/examples/mixed-example/setup.py new file mode 100644 index 0000000000..75af734583 --- /dev/null +++ b/graalpython/hpy/docs/examples/mixed-example/setup.py @@ -0,0 +1,10 @@ +from setuptools import setup, Extension +from os import path + +setup( + name="hpy-mixed-example", + hpy_ext_modules=[ + Extension('mixed', sources=[path.join(path.dirname(__file__), 'mixed.c')]), + ], + setup_requires=['hpy'], +) diff --git a/graalpython/hpy/docs/examples/quickstart/quickstart.c b/graalpython/hpy/docs/examples/quickstart/quickstart.c new file mode 100644 index 0000000000..ce9542046a --- /dev/null +++ b/graalpython/hpy/docs/examples/quickstart/quickstart.c @@ -0,0 +1,39 @@ +// quickstart.c + +// This header file is the entrypoint to the HPy API: +#include "hpy.h" + +// HPy method: the HPyDef_METH macro generates some boilerplate code, +// the same code can be also written manually if desired +HPyDef_METH(say_hello, "say_hello", HPyFunc_NOARGS) +static HPy say_hello_impl(HPyContext *ctx, HPy self) +{ + // Methods take HPyContext, which must be passed as the first argument to + // all HPy API functions. Other than that HPyUnicode_FromString does the + // same thing as PyUnicode_FromString. + // + // HPy type represents a "handle" to a Python object, but may not be + // a pointer to the object itself. It should be fully "opaque" to the + // users. Try uncommenting the following two lines to see the difference + // from PyObject*: + // + // if (self == self) + // HPyUnicode_FromString(ctx, "Surprise? Try HPy_Is(ctx, self, self)"); + + return HPyUnicode_FromString(ctx, "Hello world"); +} + +static HPyDef *QuickstartMethods[] = { + &say_hello, // 'say_hello' generated for us by the HPyDef_METH macro + NULL, +}; + +static HPyModuleDef quickstart_def = { + .doc = "HPy Quickstart Example", + .defines = QuickstartMethods, +}; + +// The Python interpreter will create the module for us from the +// HPyModuleDef specification. Additional initialization can be +// done in the HPy_mod_exec slot +HPy_MODINIT(quickstart, quickstart_def) diff --git a/graalpython/hpy/docs/examples/quickstart/setup.py b/graalpython/hpy/docs/examples/quickstart/setup.py new file mode 100644 index 0000000000..0d56aefec4 --- /dev/null +++ b/graalpython/hpy/docs/examples/quickstart/setup.py @@ -0,0 +1,13 @@ +# setup.py + +from setuptools import setup, Extension +from os import path + +DIR = path.dirname(__file__) +setup( + name="hpy-quickstart", + hpy_ext_modules=[ + Extension('quickstart', sources=[path.join(DIR, 'quickstart.c')]), + ], + setup_requires=['hpy'], +) diff --git a/graalpython/hpy/docs/examples/simple-example/setup.py b/graalpython/hpy/docs/examples/simple-example/setup.py new file mode 100644 index 0000000000..af81861942 --- /dev/null +++ b/graalpython/hpy/docs/examples/simple-example/setup.py @@ -0,0 +1,10 @@ +from setuptools import setup, Extension +from os import path + +setup( + name="hpy-simple-example", + hpy_ext_modules=[ + Extension('simple', sources=[path.join(path.dirname(__file__), 'simple.c')]), + ], + setup_requires=['hpy'], +) diff --git a/graalpython/hpy/docs/examples/simple-example/simple.c b/graalpython/hpy/docs/examples/simple-example/simple.c new file mode 100644 index 0000000000..49883a950a --- /dev/null +++ b/graalpython/hpy/docs/examples/simple-example/simple.c @@ -0,0 +1,43 @@ +/* Simple C module that defines single simple function "myabs". + * We need to have a separate standalone package for those snippets, because we + * want to show the source code in its entirety, including the HPyDef array + * initialization, the module definition, and the setup.py script, so there is + * no room left for mixing these code snippets with other code snippets. + */ + +// BEGIN: myabs +#include "hpy.h" + +HPyDef_METH(myabs, "myabs", HPyFunc_O) +static HPy myabs_impl(HPyContext *ctx, HPy self, HPy arg) +{ + return HPy_Absolute(ctx, arg); +} +// END: myabs + +// BEGIN: double +HPyDef_METH_IMPL(double_num, "double", double_impl, HPyFunc_O) +static HPy double_impl(HPyContext *ctx, HPy self, HPy arg) +{ + return HPy_Add(ctx, arg, arg); +} +// END: double + +// BEGIN: methodsdef +static HPyDef *SimpleMethods[] = { + &myabs, + &double_num, + NULL, +}; + +static HPyModuleDef simple = { + .doc = "HPy Example", + .size = 0, + .defines = SimpleMethods, + .legacy_methods = NULL +}; +// END: methodsdef + +// BEGIN: moduledef +HPy_MODINIT(simple, simple) +// END: moduledef \ No newline at end of file diff --git a/graalpython/hpy/docs/examples/snippets/hpycall.c b/graalpython/hpy/docs/examples/snippets/hpycall.c new file mode 100644 index 0000000000..9fc51e1f3e --- /dev/null +++ b/graalpython/hpy/docs/examples/snippets/hpycall.c @@ -0,0 +1,196 @@ +#include + +// BEGIN EuclideanVectorObject +typedef struct { + long x; + long y; +} EuclideanVectorObject; +HPyType_HELPERS(EuclideanVectorObject) +// END EuclideanVectorObject + +// BEGIN HPy_tp_call +HPyDef_SLOT(call, HPy_tp_call) +static HPy +call_impl(HPyContext *ctx, HPy self, const HPy *args, size_t nargs, + HPy kwnames) +{ + static const char *keywords[] = { "x1", "y1", NULL }; + long x1, y1; + HPyTracker ht; + if (!HPyArg_ParseKeywords(ctx, &ht, args, nargs, kwnames, "ll", keywords, + &x1, &y1)) { + return HPy_NULL; + } + EuclideanVectorObject *data = EuclideanVectorObject_AsStruct(ctx, self); + return HPyLong_FromLong(ctx, data->x * x1 + data->y * y1); +} +// END HPy_tp_call + +// BEGIN HPy_SetCallFunction +HPyDef_CALL_FUNCTION(special_call) +static HPy +special_call_impl(HPyContext *ctx, HPy self, const HPy *args, size_t nargs, + HPy kwnames) +{ + HPy tmp = call_impl(ctx, self, args, nargs, kwnames); + HPy res = HPy_Negative(ctx, tmp); + HPy_Close(ctx, tmp); + return res; +} + +HPyDef_SLOT(new, HPy_tp_new) +static HPy +new_impl(HPyContext *ctx, HPy cls, const HPy *args, HPy_ssize_t nargs, HPy kw) +{ + static const char *keywords[] = { "x", "y", "use_special_call", NULL }; + HPyTracker ht; + long x, y; + HPy use_special_call = ctx->h_False; + if (!HPyArg_ParseKeywordsDict(ctx, &ht, args, nargs, kw, "ll|O", keywords, + &x, &y, &use_special_call)) { + return HPy_NULL; + } + EuclideanVectorObject *vector; + HPy h_point = HPy_New(ctx, cls, &vector); + if (HPy_IsNull(h_point)) { + HPyTracker_Close(ctx, ht); + return HPy_NULL; + } + if (HPy_IsTrue(ctx, use_special_call) && + HPy_SetCallFunction(ctx, h_point, &special_call) < 0) { + HPyTracker_Close(ctx, ht); + HPy_Close(ctx, h_point); + return HPy_NULL; + } + HPyTracker_Close(ctx, ht); + vector->x = x; + vector->y = y; + return h_point; +} +// END HPy_SetCallFunction + +// BEGIN FooObject +typedef struct { + void *a; + HPyCallFunction call_func; + void *b; +} FooObject; +HPyType_HELPERS(FooObject) +// END FooObject + +// BEGIN vectorcalloffset +HPyDef_MEMBER(Foo_call_func_offset, "__vectorcalloffset__", HPyMember_HPYSSIZET, + offsetof(FooObject, call_func), .readonly=1) + +HPyDef_CALL_FUNCTION(Foo_call_func) +static HPy +Foo_call_func_impl(HPyContext *ctx, HPy self, const HPy *args, size_t nargs, + HPy kwnames) +{ + return HPyUnicode_FromString(ctx, + "hello manually initialized call function"); +} + +HPyDef_SLOT(Foo_new, HPy_tp_new) +static HPy Foo_new_impl(HPyContext *ctx, HPy cls, const HPy *args, + HPy_ssize_t nargs, HPy kw) +{ + FooObject *data; + HPy h_obj = HPy_New(ctx, cls, &data); + if (HPy_IsNull(h_obj)) + return HPy_NULL; + data->call_func = Foo_call_func; + return h_obj; +} +// END vectorcalloffset + +// BEGIN pack_args +// function using legacy 'tp_call' calling convention +static HPy +Pack_call_legacy(HPyContext *ctx, HPy self, HPy args, HPy kwd) +{ + // use 'args' and 'kwd' + return HPy_Dup(ctx, ctx->h_None); +} + +// function using HPy calling convention +HPyDef_SLOT(Pack_call, HPy_tp_call) +static HPy +Pack_call_impl(HPyContext *ctx, HPy self, const HPy *args, size_t nargs, + HPy kwnames) +{ + HPy args_tuple, kwd; + HPy result; + if (!HPyHelpers_PackArgsAndKeywords(ctx, args, nargs, kwnames, + &args_tuple, &kwd)) { + return HPy_NULL; + } + result = Pack_call_legacy(ctx, self, args_tuple, kwd); + HPy_Close(ctx, args_tuple); + HPy_Close(ctx, kwd); + return result; +} +// END pack_args + +static HPyDef *Point_defines[] = { + &call, + &new, + NULL +}; +static HPyType_Spec EuclideanVector_spec = { + .name = "hpycall.EuclideanVector", + .basicsize = sizeof(EuclideanVectorObject), + .builtin_shape = SHAPE(EuclideanVectorObject), + .defines = Point_defines +}; + +static HPyDef *Foo_defines[] = { + &Foo_call_func_offset, + &Foo_new, + NULL +}; +static HPyType_Spec Foo_spec = { + .name = "hpycall.Foo", + .basicsize = sizeof(FooObject), + .builtin_shape = SHAPE(FooObject), + .defines = Foo_defines +}; + +static HPyDef *Pack_defines[] = { + &Pack_call, + NULL +}; +static HPyType_Spec Pack_spec = { + .name = "hpycall.Pack", + .defines = Pack_defines +}; + +HPyDef_SLOT(init, HPy_mod_exec) +static int init_impl(HPyContext *ctx, HPy m) +{ + if (!HPyHelpers_AddType(ctx, m, "EuclideanVector", &EuclideanVector_spec, NULL)) { + return -1; + } + if (!HPyHelpers_AddType(ctx, m, "Foo", &Foo_spec, NULL)) { + return -1; + } + if (!HPyHelpers_AddType(ctx, m, "Pack", &Pack_spec, NULL)) { + return -1; + } + return 0; +} + +static HPyDef *moduledefs[] = { + &init, + NULL +}; + +static HPyModuleDef moduledef = { + .doc = "HPy call protocol usage example", + .size = 0, + .legacy_methods = NULL, + .defines = moduledefs, + +}; + +HPy_MODINIT(hpycall, moduledef) diff --git a/graalpython/hpy/docs/examples/snippets/hpyinit.c b/graalpython/hpy/docs/examples/snippets/hpyinit.c new file mode 100644 index 0000000000..f23bba29eb --- /dev/null +++ b/graalpython/hpy/docs/examples/snippets/hpyinit.c @@ -0,0 +1,22 @@ +#include "hpy.h" + +// BEGIN +HPyDef_SLOT(my_exec, HPy_mod_exec) +int my_exec_impl(HPyContext *ctx, HPy mod) { + + // Some initialization: add types, constants, ... + + return 0; // success +} + +static HPyDef *Methods[] = { + &my_exec, // HPyDef_SLOT macro generated `my_exec` for us + NULL, +}; + +static HPyModuleDef mod_def = { + .defines = Methods +}; + +HPy_MODINIT(hpyinit, mod_def) +// END \ No newline at end of file diff --git a/graalpython/hpy/docs/examples/snippets/hpyvarargs.c b/graalpython/hpy/docs/examples/snippets/hpyvarargs.c new file mode 100644 index 0000000000..a95617d542 --- /dev/null +++ b/graalpython/hpy/docs/examples/snippets/hpyvarargs.c @@ -0,0 +1,45 @@ +/* Simple C module that defines simple functions "myabs" and "add_ints". + * + * This module represents an incremental change over the "simple" package + * and shows how to add a method with VARARGS calling convention. + * + * We need to have a separate standalone C module for those snippets, because we + * want to show the source code including the HPyDef array initialization, so + * there is no room left for adding other entry points for other code snippets. + */ + +#include "hpy.h" + +// This is here to make the module look like an incremental change to simple-example +HPyDef_METH(myabs, "myabs", HPyFunc_O) +static HPy myabs_impl(HPyContext *ctx, HPy self, HPy arg) +{ + return HPy_Absolute(ctx, arg); +} + +// BEGIN: add_ints +HPyDef_METH(add_ints, "add_ints", HPyFunc_VARARGS) +static HPy add_ints_impl(HPyContext *ctx, HPy self, const HPy *args, size_t nargs) +{ + long a, b; + if (!HPyArg_Parse(ctx, NULL, args, nargs, "ll", &a, &b)) + return HPy_NULL; + return HPyLong_FromLong(ctx, a+b); +} +// END: add_ints + +// BEGIN: methodsdef +static HPyDef *SimpleMethods[] = { + &myabs, + &add_ints, + NULL, +}; +// END: methodsdef + +static HPyModuleDef def = { + .doc = "HPy Example of varargs calling convention", + .size = 0, + .defines = SimpleMethods +}; + +HPy_MODINIT(hpyvarargs, def) diff --git a/graalpython/hpy/docs/examples/snippets/legacyinit.c b/graalpython/hpy/docs/examples/snippets/legacyinit.c new file mode 100644 index 0000000000..f10381b617 --- /dev/null +++ b/graalpython/hpy/docs/examples/snippets/legacyinit.c @@ -0,0 +1,20 @@ +#include + +// BEGIN +static struct PyModuleDef mod_def = { + PyModuleDef_HEAD_INIT, + .m_name = "legacyinit", + .m_size = -1 +}; + +PyMODINIT_FUNC +PyInit_legacyinit(void) +{ + PyObject *mod = PyModule_Create(&mod_def); + if (mod == NULL) return NULL; + + // Some initialization: add types, constants, ... + + return mod; +} +// END \ No newline at end of file diff --git a/graalpython/hpy/docs/examples/snippets/setup.py b/graalpython/hpy/docs/examples/snippets/setup.py new file mode 100644 index 0000000000..6c097b0385 --- /dev/null +++ b/graalpython/hpy/docs/examples/snippets/setup.py @@ -0,0 +1,16 @@ +from setuptools import setup, Extension +from os import path + +setup( + name="hpy-snippets", + hpy_ext_modules=[ + Extension('hpyvarargs', sources=[path.join(path.dirname(__file__), 'hpyvarargs.c')]), + Extension('snippets', sources=[path.join(path.dirname(__file__), 'snippets.c')]), + Extension('hpyinit', sources=[path.join(path.dirname(__file__), 'hpyinit.c')]), + Extension('hpycall', sources=[path.join(path.dirname(__file__), 'hpycall.c')]), + ], + ext_modules=[ + Extension('legacyinit', sources=[path.join(path.dirname(__file__), 'legacyinit.c')]), + ], + setup_requires=['hpy'], +) diff --git a/graalpython/hpy/docs/examples/snippets/snippets.c b/graalpython/hpy/docs/examples/snippets/snippets.c new file mode 100644 index 0000000000..7e60a52cab --- /dev/null +++ b/graalpython/hpy/docs/examples/snippets/snippets.c @@ -0,0 +1,81 @@ +/* Module with various code snippets used in the docs. + * All code snippets should be put into this file if possible. Notable + * exception are code snippets showing definition of the module or the + * HPyDef array initialization. Remember to also add tests to ../tests.py + */ +#include "hpy.h" + +// ------------------------------------ +// Snippets used in api.rst + +// BEGIN: foo +void foo(HPyContext *ctx) +{ + HPy x = HPyLong_FromLong(ctx, 42); + HPy y = HPy_Dup(ctx, x); + /* ... */ + // we need to close x and y independently + HPy_Close(ctx, x); + HPy_Close(ctx, y); +} +// END: foo + +// BEGIN: is_same_object +int is_same_object(HPyContext *ctx, HPy x, HPy y) +{ + // return x == y; // compilation error! + return HPy_Is(ctx, x, y); +} +// END: is_same_object + +// dummy entry point so that we can test the snippets: +HPyDef_METH(test_foo_and_is_same_object, "test_foo_and_is_same_object", HPyFunc_VARARGS) +static HPy test_foo_and_is_same_object_impl(HPyContext *ctx, HPy self, + const HPy *args, size_t nargs) +{ + foo(ctx); // not much we can test here + return HPyLong_FromLong(ctx, is_same_object(ctx, args[0], args[1])); +} + +// BEGIN: test_leak_stacktrace +HPyDef_METH(test_leak_stacktrace, "test_leak_stacktrace", HPyFunc_NOARGS) +static HPy test_leak_stacktrace_impl(HPyContext *ctx, HPy self) +{ + HPy num = HPyLong_FromLong(ctx, 42); + if (HPy_IsNull(num)) { + return HPy_NULL; + } + // No HPy_Close(ctx, num); + return HPy_Dup(ctx, ctx->h_None); +} +// END: test_leak_stacktrace + +// BEGIN: add +HPyDef_METH(add, "add", HPyFunc_VARARGS) +static HPy add_impl(HPyContext *ctx, HPy self, const HPy *args, size_t nargs) +{ + if (nargs != 2) { + HPyErr_SetString(ctx, ctx->h_TypeError, "expected exactly two args"); + return HPy_NULL; + } + return HPy_Add(ctx, args[0], args[1]); +} +// END: add + +// ------------------------------------ +// Dummy module definition, so that we can test the snippets + +static HPyDef *Methods[] = { + &test_foo_and_is_same_object, + &test_leak_stacktrace, + &add, + NULL, +}; + +static HPyModuleDef snippets = { + .doc = "Various HPy code snippets for the docs", + .size = 0, + .defines = Methods +}; + +HPy_MODINIT(snippets, snippets) diff --git a/graalpython/hpy/docs/examples/tests.py b/graalpython/hpy/docs/examples/tests.py new file mode 100644 index 0000000000..8a8ee5f5df --- /dev/null +++ b/graalpython/hpy/docs/examples/tests.py @@ -0,0 +1,124 @@ +import os +import os.path +import re +import subprocess +import sys + +import simple +import mixed +import hpyvarargs +import snippets +import simple_type +import builtin_type +import hpycall + + +def test_simple_abs(): + assert simple.myabs(-42) == 42 + assert simple.myabs(42) == 42 + + +def test_hpyvarargs(): + assert hpyvarargs.add_ints(40, 2) == 42 + + +def test_mixed_add_ints(): + assert mixed.add_ints_legacy(40, 2) == 42 + assert mixed.add_ints(40, 2) == 42 + + +def test_snippets(): + x = 2 + assert snippets.test_foo_and_is_same_object(x, x) == 1 + assert snippets.test_foo_and_is_same_object(x, 42) == 0 + + +def test_simple_type(): + p = simple_type.Point(4, 5) + assert p.x == 4 + assert p.y == 5 + assert p.foo() == 45 + assert p.z == 1045 + p.z = 2000 + assert p.y == 960 + assert p.z == 2000 + + +def test_quickstart(): + import quickstart + assert quickstart.say_hello() == "Hello world" +# END: test_quickstart + + +def test_builtin_type(): + obj = builtin_type.Dummy("hello") + assert obj == "hello" + + obj = builtin_type.Language("hello") + obj.lang = "en" + assert obj == "hello" + assert obj.lang == "en" + + +def test_leak_detector(): + from hpy.debug.pytest import LeakDetector + with LeakDetector() as ld: + # add_ints is an HPy C function. If it forgets to close a handle, + # LeakDetector will complain + assert mixed.add_ints(40, 2) == 42 +# END: test_leak_detector + +from hpy.debug.pytest import hpy_debug +def test_that_uses_leak_detector_fixture(hpy_debug): + # Run some HPy extension code + assert mixed.add_ints(40, 2) == 42 + + +def test_leak_detector_with_traces(): + import hpy.debug + hpy.debug.set_handle_stack_trace_limit(16) + assert mixed.add_ints(40, 2) == 42 + hpy.debug.disable_handle_stack_traces() + + +def test_leak_detector_with_traces_output(): + # Update the debug documentation if anything here changes! + env = os.environ.copy() + env['HPY'] = 'debug' + env['HPY_LOG'] = '1' + script = os.path.join(os.path.dirname(__file__), 'debug-example.py') + result = subprocess.run([sys.executable, script], env=env, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # Rudimentary check that the output contains what we have in the documentation + out = result.stdout.decode('latin-1') + assert out == "Loading 'snippets' in HPy universal mode with a debug context" + os.linesep + err = result.stderr.decode('latin-1') + assert 'hpy.debug.leakdetector.HPyLeakError: 1 unclosed handle:' in err + assert re.search('', err) + assert 'Allocation stacktrace:' in err + if sys.platform.startswith(("linux", "darwin")): + assert 'snippets.hpy0.so' in err # Should be somewhere in the stack trace + else: + assert 'At the moment this is only supported on Linux with glibc' in err + +def test_trace_mode_output(): + # Update the trace mode documentation if anything here changes! + env = os.environ.copy() + env['HPY'] = 'trace' + env['HPY_LOG'] = '1' + script = os.path.join(os.path.dirname(__file__), 'trace-example.py') + result = subprocess.run([sys.executable, script], env=env, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # Rudimentary check that the output contains what we have in the documentation + out = result.stdout.decode('latin-1') + assert 'get_call_counts()["ctx_Add"] == 1' in out + +def test_call_dot_product(): + vec = hpycall.EuclideanVector(4, 5) + assert vec(6, 7) == 4 * 6 + 5 * 7 + vec = hpycall.EuclideanVector(4, 5, use_special_call=True) + assert vec(6, 7) == -(4 * 6 + 5 * 7) + foo = hpycall.Foo() + assert foo() == 'hello manually initialized call function' + pack = hpycall.Pack() + assert pack() is None diff --git a/graalpython/hpy/docs/examples/trace-example.py b/graalpython/hpy/docs/examples/trace-example.py new file mode 100644 index 0000000000..2a4d92838e --- /dev/null +++ b/graalpython/hpy/docs/examples/trace-example.py @@ -0,0 +1,9 @@ +# Run with HPY=trace +from hpy.trace import get_call_counts +import snippets + +add_count_0 = get_call_counts()["ctx_Add"] +snippets.add(1, 2) == 3 +add_count_1 = get_call_counts()["ctx_Add"] + +print('get_call_counts()["ctx_Add"] == %d' % (add_count_1 - add_count_0)) diff --git a/graalpython/hpy/docs/index.rst b/graalpython/hpy/docs/index.rst new file mode 100644 index 0000000000..9ca101b753 --- /dev/null +++ b/graalpython/hpy/docs/index.rst @@ -0,0 +1,80 @@ +.. HPy documentation master file, created by + sphinx-quickstart on Thu Apr 2 23:01:08 2020. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +HPy: a better API for Python +=============================== + +HPy provides a new API for extending Python in C. + +There are several advantages to writing C extensions in HPy: + + - **Speed**: it runs much faster on PyPy, GraalPy, and at native speed on CPython + + - **Deployment**: it is possible to compile a single binary which runs unmodified on all + supported Python implementations and versions -- think "stable ABI" on steroids + + - **Simplicity**: it is simpler and more manageable than the ``Python.h`` API, both for + the users and the Pythons implementing it + + - **Debugging**: it provides an improved debugging experience. Debug mode can be turned + on at runtime without the need to recompile the extension or the Python running it. + HPy design is more suitable for automated checks. + +The official `Python/C API `_, +also informally known as ``#include ``, is +specific to the current implementation of CPython: it exposes a lot of +internal details which makes it hard to: + + - implement it for other Python implementations (e.g. PyPy, GraalPy, + Jython, ...) + + - experiment with new approaches inside CPython itself, for example: + + - use a tracing garbage collection instead of reference counting + - remove the global interpreter lock (GIL) to take full advantage of multicore architectures + - use tagged pointers to reduce memory footprint + +Where to go next: +----------------- + + - Show me the code: + + - :doc:`Quickstart` + - :ref:`Simple documented HPy extension example` + - :doc:`Tutorial: porting Python/C API extension to HPy` + + - Details: + + - :doc:`HPy overview: motivation, goals, current status` + - :doc:`HPy API concepts introduction` + - :doc:`Python/C API to HPy Porting guide` + - :doc:`HPy API reference` + + +Full table of contents: +----------------------- + +.. toctree:: + :maxdepth: 2 + + quickstart + overview + api + porting-guide + porting-example/index + debug-mode + trace-mode + api-reference/index + contributing/index + misc/index + changelog + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/graalpython/hpy/docs/leysin-2020-design-decisions.md b/graalpython/hpy/docs/leysin-2020-design-decisions.md new file mode 100644 index 0000000000..b50bb081c5 --- /dev/null +++ b/graalpython/hpy/docs/leysin-2020-design-decisions.md @@ -0,0 +1,295 @@ +Leysin Sprint 2020 Design Decisions +=================================== + +Closing and duping HPy_NULL +--------------------------- + +Question: Should `HPy_Close` silently ignore attempts to close `HPy_NULL`? + +Decisions: + +* `HPy_Close` should function like `Py_XDECREF` and silently not close `HPy_NULL`. +* `HPy_Dup` should silently not dup `HPy_NULL` for symmetry with `HPy_Close`, + i.e. `HPy_Dup(ctx, HPy_NULL)` should return `HPy_NULL`. +* Both the API and ABI versions should have this behaviour. +* We should add tests for this behaviour. + + +Attribute and item access +------------------------- + +Question: Should we have separate HPySequence_GetItem and HPyMapping_GetItem +or just one HPy_GetItem? Should we explore the idea of protocols for such +access. + +Decisions: + +* We should start with `HPy_GetItem` and `HPy_SetItem` and `HPy_GetItem_i` and + `HPy_SetItem_i` for item access and `HPy_GetAttr`, `HPy_SetAttr`, + `HPy_GetAttr_s` and `HPy_SetAttr_s` for attribute access.. +* We could add an `hpy/compat.h` that supports accessing the old Python mapping + and sequence get item slots. For example, `HPyDict_GetItem_NotBorrowed`. + `PyDict_GetItem` returns a borrower reference. `HPyDict_GetItem_NotBorrowed` + should return a new handle that has to be closed. +* HPy API functions MUST NOT return a borrowed reference and we should add this + to our official documentation. +* We should add an `HPyBuffer` protocol later (with the design still to be + decided). + +Proposed function signatures: + +```C +HPy HPy_GetItem(HPyContext ctx, HPy obj, HPy idx); +HPy HPy_GetItem_i(HPyContext ctx, HPy obj, HPy_ssize_t idx); +HPy HPy_GetItem_s(HPyContext ctx, HPy obj, const char * idx); // UTF8 bytes + +int HPy_SetItem(HPyContext ctx, HPy obj, HPy idx, HPy value); +int HPy_SetItem_i(HPyContext ctx, HPy obj, HPy_ssize_t idx, HPy value); +int HPy_SetItem_s(HPyContext ctx, HPy obj, const char * idx, HPy value); // UTF8 bytes + +HPy HPy_GetAttr(HPyContext ctx, HPy obj, HPy idx); +HPy HPy_GetAttr_s(HPyContext ctx, HPy obj, const char * idx); // UTF8 bytes + +int HPy_SetAttr(HPyContext ctx, HPy obj, HPy idx, HPy value); +int HPy_SetAttr_s(HPyContext ctx, HPy obj, const char * idx, HPy value); // UTF8 bytes +``` + +Macro for returning None +------------------------ + +Question: Should we have an HPy_RETURN_NONE macro? + +Decisions: + +* Yes we should, but it should be `HPy_RETURN_NONE(ctx)`. +* We should also change `HPy_IsNull(x)` to `HPy_IsNull(ctx, x)`. + +Proposed macros: + +```C +#define HPy_RETURN_NONE(ctx) return HPy_Dup(ctx, ctx->h_None); +#define HPy_IsNull(ctx, x) ...; +``` + +Passing handles as void pointers +-------------------------------- + +Question: Should we add `HPy_AsVoidP` and `HPy_FromVoidP` to the API? Should they +be on the ctx or not? + +Decisions: + +* Yes, they should be part of the API. +* They should take the ctx as an argument in case a future implementation needs + it (much like `HPy_IsNull`). + +Proposed functions: + +```C +// universal +static inline HPy HPy_FromVoidP(HPyContext ctx, void *p) { return (HPy){(HPy_ssize_t)p}; } +static inline void* HPy_AsVoidP(HPyContext ctx, HPy h) { return (void*)h._i; } + +// cpython (the -/+4 is to avoid people casting it to PyObject) +static inline HPy HPy_FromVoidP(HPyContext ctx, void *p) { return (HPy){(HPy_ssize_t)p - 4}; } +static inline void* HPy_AsVoidP(HPyContext ctx, HPy h) { return (void*)h._o + 4; } +``` + +/* + * Should we implement HPy_Dump? + * + * Question: What should it print and where should it print it? + * + * Decision: It's useful for debugging if all macros are also available as functions + * definitions. + * + * Decision: It should dump to stderr just like PyObject_Dump. + */ + +/* + * How do we silence warnings from using HPy_METH_KEYWORDS? + * + * Decision: Write a cast to HPyMeth. + */ + +{"add_ints_kw", (HPyMeth) add_ints_kw, HPy_METH_KEYWORDS, ""} + +/* + * How should HPyErr_Format be implemented? Should we avoid va_args? + * + * Decision: + */ + +HPyErr_Format(ctx, const char *fmt, ...) { + const char *msg = HPyStr_Format(ctx, fmt, ...); + HPyErr_SetString(msg); +} + +ctx->ctx_HPyErr_Format(???) + +/* + * Should make specifying values for optional arguments & dup / closing them + * less messy? + * + * Decision: Right now, No. In the future, someone should invent + * ARGUMENT_CLINIC for HPy. + */ + +/* + * Should we rename "struct _object*" to "PyObject*" and "_HPy_PyCFunction" + * to "PyCFunction" in universal/hpy.h? + * + * Decision: No. This would generate a warning if one imports Python.h. + * + * Decision: Comment in the code that this _object* is PyObject* and why we + * cannot call it that. + * + * Decision: In user documentation, just call ing PyObject *. + */ + +/* + * How should the API pass around exceptions? + * + * Decision: Follow CPython for each API call. + * + */ + +// E.g. API call returns HPy: + +HPy h = HPyLong_FromLong(ctx, 5); +if (HPy_IsNull(ctx, h)) { + // handle error + // python error has been set by the API call +} + +// or + +HPy h = HPyLong_FromLong(ctx, 5); +if (HPyErr_Occurred(ctx)) { + // handle error + // python error has been set by the API call +} + +// E.g. API call returns a value that is not an HPy: + +long l = HPyLong_AsLong(ctx, h); +if (l == -1 && HPyErr_Occurred(ctx)) { + // handle error + // python error has been set by the API call +} + +// E.g. API call returns a success or error flag: + +// int error = HPyArg_Parse(...); +if (!HPyArg_Parse(...)) { + // handle error + // python error has been set by the API call +} + +/* + * How should support for creating custom Python types look? + * + * Decisions: + * + * + */ + +// When using C-API: + +typedef struct { + PyObject_HEAD + /* Type-specific fields go here. */ + PyObject *x; + PyObject *y; + double *data; + int size; +} PointObject; + +// When using HPy: + +typedef struct { + HPyObject_HEAD + double x; + double y; +} HPy_Point; + +typedef struct { + HPyObject_HEAD + HPyField a; + HPyField b; +} HPy_Rectangle; + +/* Possible HPy code */ + +typedef struct { + ??? ob_type; +} _HPy_ObjectHeader; + +#define HPyObject_HEAD _HPy_ObjectHeader head; + +#define HPy_STORE(ctx, obj, field, value) ((ctx)->ctx_HPy_StoreInto((_HPy_ObjectHeader *) obj, &((obj)->field), value)) +#define HPy_STORE_INTO(ctx, obj, pointer, value) ((ctx)->ctx_HPy_StoreInto((_HPy_ObjectHeader *) obj, pointer, value)) + +// Using the debug mode ctx, this should check that obj->ob_type->tp_traverse is not NULL. +void HPy_StoreInto(HPyContext ctx, _HPy_ObjectHeader *obj, HPyField *pointer, HPy value) + +// HPy_New: +// +// * Should return a handle. +// * It should be possible to go from the handle to struct, but not +// from struct to handle. E.g. HPy_CAST(ctx, HPyRectangle, h) -> HPyRectangle, +// but no inverse. +// * We should not have an HPy_Init at the moment -- HPy_New both allocates +// the object and initializes it. We will add separate allocation and init +// later if we encounter a need for it. +// * HPyTypeSpec should follow the CPython type spec. + +#define HPy_CAST(ctx, return_type, h) (return_type *) ctx->ctx_HPy_Cast(h) + +void* HPy_Cast(ctx, HPy h); + +HPy HPy_TypeFromSpec(ctx, HPyTypeSpec type_spec); +HPy HPy_New(ctx, HPy h_type); + +/* end of possible HPy code */ + +HPy new_rect(HPy p1, HPy p2) { + // HPy_New always initialize the whole object to 0. We can also have + // HPy_NewUninitialized if we don't want to pay the penalty + HPy_Rectangle *rect; + HPy rect_handle = HPy_New(ctx, HPy_Rectangle, h_rectangle_type, &rect); + + // HPy_Store does a write barrier on PyPy, and DECREF the old rect->a on + // CPython if needed + HPy_Store(ctx, rect, a, p1); + HPy_Store(ctx, rect, b, p2); + return rect_handle; +} + +double calc_diagonal(HPy rect_handle) { + // rect is valid until rect_handle is closed. on PyPy we pin the object, and + // we unpin it when we close the handle + HPy_Rectangle *rect = HPy_Cast(HPy_Rectangle, rect_handle); + + // HPy_Load reads a field and turn it into a handle + HPy p1_handle = HPy_Load(rect->a); // p1 is a handle which must be closed + HPy_Point *p1 = HPy_Cast(HPy_Point, p1_handle); + + // for C99 compilers, we can also provide a macro which declares p2 and + // p2_handle automatically and does the equivalent of the two lines above + HPY_LOAD(HPy_Point, p2, rect->b); + + double diag = sqrt(p1->x - p2->x /* etc. etc. */); + + // close all the handles + HPy_Close(p1_handle); + HPy_Close(p2_handle); + + return diag; +} + +/* + * Should HPy support Python without the GIL? + * + * Problem left as an exercise for the reader. + */ diff --git a/graalpython/hpy/docs/misc/embedding.rst b/graalpython/hpy/docs/misc/embedding.rst new file mode 100644 index 0000000000..cb85d2bc10 --- /dev/null +++ b/graalpython/hpy/docs/misc/embedding.rst @@ -0,0 +1,46 @@ +Embedding HPy modules +===================== + +There might be cases where it is beneficial or even necessary to embed multiple +HPy modules into one library. HPy itself already makes use of that. The debug +and the trace module do not have individual libraries but are embedded into the +universal module. + +To achieve that, the embedder will use the macro :c:macro:`HPy_MODINIT` several times. +Unfortunately, this macro defines global state and cannot repeatedly be used by +default. In order to correctly embed several HPy modules into one library, the +embedder needs to consider following: + +* The modules must be compiled with preprocessor macro + :c:macro:`HPY_EMBEDDED_MODULES` defined to enable this feature. + +* There is one major restriction: All HPy-specific module pieces must be + in the same compilation unit. *HPy-specific pieces* are things like the + module's init function (``HPy_MODINIT``) and all slots, members, methods of + the module or any type of it (``HPyDef_*``). The implementation functions + (usually the ``*_impl`` functions) of the slots, members, methods, etc. and + any helper functions may still be in different compilation units. The reason + for this is that the global state induced by ``HPy_MODINIT`` is, of course, + made local (e.g. using C modifier ``static``). + +* It is also necessary to use macro :c:macro:`HPY_MOD_EMBEDDABLE` before the + first usage of any ``HPyDef_*`` macro. + +Also refer to the API reference :ref:`api-reference/hpy-type:hpy module`. + + +**Example** + +.. code-block:: c + + // compile with -DHPY_EMBEDDED_MODULES + + HPY_MOD_EMBEDDABLE(hpymodA) + + HPyDef_METH(foo, /* ... */) + static HPy foo_impl(/* ... */) + { + // ... + } + + HPy_MODINIT(extension_name, hpymodA) diff --git a/graalpython/hpy/docs/misc/index.rst b/graalpython/hpy/docs/misc/index.rst new file mode 100644 index 0000000000..f305c8ed01 --- /dev/null +++ b/graalpython/hpy/docs/misc/index.rst @@ -0,0 +1,7 @@ +Misc Notes +========== + +.. toctree:: + :maxdepth: 1 + + embedding diff --git a/graalpython/hpy/docs/module-state.txt b/graalpython/hpy/docs/module-state.txt new file mode 100644 index 0000000000..3682a8820e --- /dev/null +++ b/graalpython/hpy/docs/module-state.txt @@ -0,0 +1,44 @@ +How to replace global variables +------------------------------- + +In a given .c source, write: + + +typedef struct { + long x; + HPy y; +} my_globals_t; + +static void my_globals_traverse(traversefunc traverse, my_globals_t *g) +{ + traverse(g->y); +} + +HPyGlobalSpec my_globals = { + .m_size = sizeof(my_globals_t), + .m_traverse = my_globals_traverse +}; + + +There can be several HPyGlobalSpec structures around; in CPython it's done as +part of the PyModuleDef type, but there is no real reason for why it should +be tightly tied to a module. + + +To use: + + my_globals_t *g = HPy_GetState(ctx, &my_globals); + g->x++; + HPy_DoRandomStuffWithHandle(ctx, g->y); + + +Implementation: the type HPyGlobalSpec contains extra internal fields +which should give us a very fast cache: _last_ctx and _last_result, +and HPy_GetState() can be: + + if (ctx == globspec->_last_ctx) + return globspec->_last_result; + else + look up globspec in a dictionary attached to ctx, or vice-versa, + or maybe initialize globspec->_index with a unique incrementing + index and use that to index an array attached to ctx diff --git a/graalpython/hpy/docs/overview.rst b/graalpython/hpy/docs/overview.rst new file mode 100644 index 0000000000..e4c9e30b86 --- /dev/null +++ b/graalpython/hpy/docs/overview.rst @@ -0,0 +1,444 @@ +HPy Overview +============ + +Motivation and goals +--------------------- + +The superpower of the Python ecosystem is its libraries, which are developed by +users. Over time, these libraries have grown in number, quality, and +applicability. While it is possible to write python libraries entirely in +python, many of them, especially in the scientific community, are written in C +and exposed to Python using the `Python.h API +`_. The existence of these C +extensions using the ``Python.h`` API leads to some issues: + + 1. Usually, alternative implementation of the Python programming language + want to support C extensions. To do so, they must implement the same + ``Python.h`` API or provide a compatibility layer. + + 2. CPython developers cannot experiment with new designs or refactoring + without breaking compatibility with existing extensions. + +Over the years, it has become evident that emulating ``Python.h`` in an +efficient way is `challenging, if not impossible +`_. +To summarize, it is mainly due to leaking of implementation details of CPython +into the C/API - which makes it difficult to make different design choices than +those made by CPython. As such - the main goal of HPy is to provide a **C API +which makes as few assumptions as possible about the design decisions of any +implementation of Python, allowing diverse implementations to support it +efficiently and without compromise**. In particular, **reference counting is not +part of the API**: we want a more generic way of managing resources that is +possible to implement with different strategies, including the existing +reference counting and/or with a moving *Garbage Collector* (like the ones used +by PyPy, GraalPy or Java, for example). Moreover, each implementation can +experiment with new memory layout of objects, add optimizations, etc. The +following is a list of sub-goals. + + +Performance on CPython + HPy is usable on CPython from day 1 with no performance impact compared to + the existing ``Python.h`` API. + + +Incremental adoption + It is possible to port existing C extensions piece by piece and to use + the old and the new API side-by-side during the transition. + + +Easy migration + It should be easy to migrate existing C extensions to HPy. Thanks to an + appropriate and regular naming convention it should be obvious what the + HPy equivalent of any existing ``Python.h`` API is. When a perfect replacement + does not exist, the documentation explains what the alternative options are. + + +Better debugging + In debug mode, you get early and precise errors and warnings when you make + some specific kind of mistakes and/or violate the API rules and + assumptions. For example, you get an error if you try to use a handle + (see :ref:`api:handles`) which has already been closed. It is possible to + turn on the debug mode at startup time, *without needing to recompile*. + +Simplicity + The HPy API aims to be smaller and easier to study/use/manage than the + existing ``Python.h`` API. Sometimes there is a trade-off between this goal and + the others above, in particular *Performance on CPython* and *Easy migration*. + The general approach is to have an API which is "as simple as possible" while + not violating the other goals. + + +Universal binaries + It is possible to compile extensions to a single binary which is + ABI-compatible across multiple Python versions and/or multiple + implementation. See :ref:`hpy-target-abis`. + + +Opt-in low level data structures + Internal details might still be available, but in a opt-in way: for example, + if Cython wants to iterate over a list of integers, it can ask if the + implementation provides a direct low-level access to the content (e.g. in + the form of a ``int64_t[]`` array) and use that. But at the same time, be + ready to handle the generic fallback case. + + +API vs ABI +----------- + +HPy defines *both* an API and an ABI. Before digging further into details, +let's distinguish them: + + - The **API** works at the level of source code: it is the set of functions, + macros, types and structs which developers can use to write their own + extension modules. For C programs, the API is generally made available + through one or more header files (``*.h``). + + - The **ABI** works at the level of compiled code: it is the interface between + the host interpreter and the compiled DLL. Given a target CPU and + operating system it defines things like the set of exported symbols, the + precise memory layout of objects, the size of types, etc. + +In general it is possible to compile the same source into multiple compiled +libraries, each one targeting a different ABI. :pep:`3149` states that the +filename of the compiled extension should contain the *ABI tag* to specify +what the target ABI is. For example, if you compile an extension called +``simple.c`` on CPython 3.8, you get a DLL called +``simple.cpython-38-x86_64-linux-gnu.so``: + + - ``cpython-38`` is the ABI tag, in this case CPython 3.8 + + - ``x86_64`` is the CPU architecture + + - ``linux-gnu`` is the operating system + +The same source code compiled on PyPy3.6 7.2.0 results in a file called +``simple.pypy38-pp73-x86_64-linux-gnu.so``: + + - ``pypy38-pp73`` is the ABI tag, in this case "PyPy3.8", version "7.3.x" + +The HPy C API is exposed to the user by including ``hpy.h`` and it is +explained in its own section of the documentation. + + +Legacy and compatibility features +--------------------------------- + +To allow an incremental transition to HPy, it is possible to use both +``hpy.h`` and ``Python.h`` API calls in the same extension. Using *HPy legacy +features* you can: + + - mix ``Python.h`` and HPy method defs in the same HPy module + + - mix ``Python.h`` and HPy method defs and slots in the same HPy type + + - convert ``HPy`` handles to and from ``PyObject *`` using + ``HPy_AsPyObject()`` and ``HPy_FromPyObject()`` + + +Thanks to this, you can port your code to HPy one method and one type at a +time, while keeping the extension fully functional during the transition +period. See the :ref:`porting-guide:Porting guide` for a concrete example. + +Legacy features are available only if you target the CPython or HPy Hybrid +ABIs, as explained in the next section. + + +.. _hpy-target-abis: + +Target ABIs +----------- + +Depending on the compilation options, an HPy extension can target three +different ABIs: + +.. glossary:: + + CPython ABI + In this mode, HPy is implemented as a set of C macros and ``static inline`` + functions which translate the HPy API into the CPython API at compile + time. The result is a compiled extension which is indistinguishable from a + "normal" one and can be distributed using all the standard tools and will + run at the very same speed. + + *Legacy features* are available. + + The output filename is e.g. ``simple.cpython-38-x86_64-linux-gnu.so``. + + + HPy Universal ABI + As the name suggests, the HPy Universal ABI is designed to be loaded and + executed by a variety of different Python implementations. Compiled + extensions can be loaded unmodified on all the interpreters which support + it. PyPy and GraalPy support it natively. CPython supports it by using the + ``hpy.universal`` package, and there is a small speed penalty [#f1]_ compared to + the CPython ABI. + + *Legacy features* are **not** available and it is forbidden to ``#include ``. + + The resulting filename is e.g. ``simple.hpy0.so``. + + HPy Hybrid ABI + + The HPy Hybrid ABI is essentially the same as the Universal ABI, with + the big difference that it allows to ``#include ``, to use the + legacy features and thus to allow incremental porting. + + At the ABI level the resulting binary depends on *both* HPy and the + specific Python implementation which was used to compile the extension. + As the name suggests, this means that the binary is not "universal", + thus negating some of the benefits of HPy. The main benefit of using + the HPy Hybrid ABI instead of the CPython ABI is being able to use the + :ref:`debug-mode:Debug mode` on the HPy parts, and faster speed on + alternative implementations. + + *Legacy features* are available. + + The resulting filename is e.g. ``simple.hpy0-cp38.so``. + + +Moreover, each alternative Python implementation could decide to implement its +own non-universal ABI if it makes sense for them. For example, a hypothetical +project *DummyPython* could decide to ship its own ``hpy.h`` which implements +the HPy API but generates a DLL which targets the DummyPython ABI. + +This means that to compile an extension for CPython, you can choose whether to +target the CPython ABI or the Universal ABI. The advantage of the former is +that it runs at native speed, while the advantage of the latter is that you +can distribute a single binary, although with a small speed penalty on +CPython. Obviously, nothing stops you compiling and distributing both +versions: this is very similar to what most projects are already doing, since +they automatically compile and distribute extensions for many different +CPython versions. + +From the user point of view, extensions compiled for the CPython ABI can be +distributed and installed as usual, while those compiled for the HPy Universal +or HPy Hybrid ABIs require installing the ``hpy.universal`` package on +CPython and have no further requirements on Pythons that support HPy natively. + + +Benefits for the Python ecosystem +--------------------------------- + +The HPy project offers some benefits to the python ecosystem, both to Python +users and to library developers. + + - C extensions can achieve much better speed on alternative implementions, + including PyPy and GraalPy: according to early :ref:`benchmarks`, an + extension written in HPy can be ~3x faster than the equivalent extension + written using ``Python.h``. + - Improved debugging: when you load extensions in :ref:`debug-mode:debug mode`, + many common mistakes are checked and reported automatically. + - Universal binaries: libraries can choose to distribute only Universal ABI + binaries. By doing so, they can support all Python implementations and + version of CPython (like PyPy, GraalPy, CPython 3.10, CPython 3.11, etc) + for which an HPy loader exists, including those that do not yet exist! This + currently comes with a small speed penalty on CPython, but for + non-performance critical libraries it might still be a good tradeoff. + - Python environments: With general availability of universal ABI binaries for + popular packages, users can create equivalent python environments that + target different Python implementations. Thus, Python users can try their + workload against different implementations and pick the one best suited for + their usage. + - In a situation where most or all popular Python extensions target the + universal ABI, it will be more feasible for CPython to make breaking changes + to its C/API for performance or maintainability reasons. + + +Cython extensions +----------------- + +If you use Cython, you can't use HPy directly. There is a +`work in progress `_ to +add Cython backend which emits HPy code instead of using ``Python.h`` code: once this is +done, you will get the benefits of HPy automatically. + + +Extensions in other languages +----------------------------- + +On the API side, HPy is designed with C in mind, so it is not directly useful +if you want to write an extension in a language other than C. + +However, Python bindings for other languages could decide to target the +:term:`HPy Universal ABI` instead of the :term:`CPython ABI`, and generate +extensions which can be loaded seamlessly on all Python implementations which +supports it. This is the route taken, for example, by `Rust +`_. + + +Benefits for alternative Python implementations +----------------------------------------------- + +If you are writing an alternative Python implementation, there is a good +chance that you already know how painful it is to support the ``Python.h`` API. +HPy is designed to be both faster and easier to implement! + +You have two choices: + + - support the Universal ABI: in this case, you just need to export the + needed functions and to add a hook to ``dlopen()`` the desired libraries + + - use a custom ABI: in this case, you have to write your own replacement for + ``hpy.h`` and recompile the C extensions with it. + + +Current status and roadmap +-------------------------- + +HPy left the early stages of development and already provides a noticeable set +of features. As on April 2023, the following milestones have been reached: + + - some prominent real-world Python packages have been ported to HPy API. There + is a list of HPy-compatible packages we know about on the HPy website + `hpyproject.org `_. + + - one can write extensions which expose module-level functions, with all + the various kinds of calling conventions. + + - there is support for argument parsing (i.e., the equivalents of + ``PyArg_ParseTuple`` and ``PyArg_ParseTupleAndKeywords``), and a + convenient complex value building (i.e., the equivalent ``Py_BuildValue``). + + - one can implement custom types, whose struct may contain references to other + Python objects using ``HPyField``. + + - there is a support for globally accessible Python object handles: ``HPyGlobal``, + which can still provide isolation for subinterpreters if needed. + + - there is support for raising and catching exceptions. + + - debug mode has been implemented and can be activated at run-time without + recompiling. It can detect leaked handles or handles used after + being closed. + + - trace mode has been implemented and can be activated just like the debug + mode. It helps analyzing the API usage (in particular wrt. performance). + + - wheels can be built for HPy extensions with ``python setup.py bdist_wheel`` + and can be installed with ``pip install``. + + - it is possible to choose between the :term:`CPython ABI` and the + :term:`HPy Universal ABI` when compiling an extension module. + + - extensions compiled with the CPython ABI work out of the box on + CPython. + + - it is possible to load HPy Universal extensions on CPython, thanks to the + ``hpy.universal`` package. + + - it is possible to load HPy Universal extensions on + PyPy (using the PyPy `hpy branch `_). + + - it is possible to load HPy Universal extensions on `GraalPy + `_. + + - there is support for multi-phase module initialization. + + - support for metaclasses has been added. + + +However, there is still a long road before HPy is usable for the general +public. In particular, the following features are on our roadmap but have not +been implemented yet: + + - many of the original ``Python.h`` functions have not been ported to + HPy yet. Porting most of them is straightforward, so for now the priority + is to test HPy with real-world Python packages and primarily resolve the + "hard" features to prove that the HPy approach works. + + - add C-level module state to complement the ``HPyGlobal`` approach. While ``HPyGlobal`` + is easier to use, it will make the migration simpler for existing extensions that + use CPython module state. + + - the integration with Cython is work in progress + + - it is not clear yet how to approach pybind11 and similar C++ bindings. They serve two use-cases: + + - As C++ wrappers for CPython API. HPy is fundamentally different in some ways, so fully compatible + pybind11 port of this API to HPy does not make sense. There can be a similar or even partially pybind11 + compatible C++ wrapper for HPy adhering to the HPy semantics and conventions (e.g., passing the + HPyContext pointer argument around, no reference stealing, etc.). + + - Way to expose (or "bind") mostly pure C++ functions as Python functions where the C++ templating + machinery takes care of the conversion between the Python world, i.e., ``PyObject*``, and the C++ + types. Porting this abstraction to HPy is possible and desired in the future. To determine the priority + or such effort, we need to get more knowledge about existing pybind11 use-cases. + + +.. _benchmarks: + +Early benchmarks +----------------- + +To validate our approach, we ported a simple yet performance critical module +to HPy. We chose `ultrajson `_ +because it is simple enough to require porting only a handful of API +functions, but at the same time it is performance critical and performs many +API calls during the parsing of a JSON file. + +This `blog post `_ +explains the results in more detail, but they can be summarized as follows: + + - ``ujson-hpy`` compiled with the CPython ABI is as fast as the original + ``ujson``. + + - A bit surprisingly, ``ujson-hpy`` compiled with the HPy Universal ABI is + only 10% slower on CPython. We need more evidence than a single benchmark + of course, but if the overhead of the HPy Universal ABI is only 10% on + CPython, many projects may find it small enough that the benefits + of distributing extensions using only the HPy Universal ABI out weight + the performance costs. + + - On PyPy, ``ujson-hpy`` runs 3x faster than the original ``ujson``. Note + the HPy implementation on PyPy is not fully optimized yet, so we expect + even bigger speedups eventually. + + +Projects involved +----------------- + +HPy was born during EuroPython 2019, were a small group of people started to +discuss the problems of the ``Python.h`` API and how it would be nice to +have a way to fix them. Since then, it has gathered the attention and interest +of people who are involved in many projects within the Python ecosystem. The +following is a (probably incomplete) list of projects whose core developers +are involved in HPy, in one way or the other. The mere presence in this list +does not mean that the project as a whole endorse or recognize HPy in any way, +just that some of the people involved contributed to the +code/design/discussions of HPy: + + - PyPy + + - CPython + + - Cython + + - GraalPy + + - RustPython + + - rust-hpy (fork of the `cpython crate `_) + + +Related work +------------- + +A partial list of alternative implementations which offer a ``Python.h`` +compatibility layer include: + + - `PyPy `_ + + - `Jython `_ + + - `IronPython `_ + + - `GraalPy `_ + +.. rubric:: Footnotes + +.. [#f1] The reason for this minor performance penalty is a layer of pointer + indirection. For instance, ``ctx->HPyLong_FromLong`` is called from the + CPython extension, which in universal mode simply forwards the call to + ``PyLong_FromLong``. It is technically possible to implement a CPython + universal module loader which edits the program's executable code at runtime + to replace that call. Note that this is not at all trivial. diff --git a/graalpython/hpy/docs/porting-example/index.rst b/graalpython/hpy/docs/porting-example/index.rst new file mode 100644 index 0000000000..e5a6dcd7b8 --- /dev/null +++ b/graalpython/hpy/docs/porting-example/index.rst @@ -0,0 +1,356 @@ +Porting Example +=============== + +HPy supports *incrementally* porting an existing C extension from the +original Python C API to the HPy API and to have the extension compile and +run at each step along the way. + +Here we walk through porting a small C extension that implements a Point type +with some simple methods (a norm and a dot product). The Point type is minimal, +but does contain additional C attributes (the x and y values of the point) +and an attribute (obj) that contains a Python object (that we will need to +convert from a ``PyObject *`` to an ``HPyField``). + +There is a separate C file illustrating each step of the incremental port: + +* :doc:`steps/step_00_c_api`: The original C API version that we are going to + port. + +* :doc:`steps/step_01_hpy_legacy`: A possible first step where all methods still + receive ``PyObject *`` arguments and may still cast them to ``PyPointObject *`` + if they are instances of Point. + +* :doc:`steps/step_02_hpy_legacy`: Shows how to transition some methods to HPy + methods that receive ``HPy`` handles as arguments while still supporting legacy + methods that receive ``PyObject *`` arguments. + +* :doc:`steps/step_03_hpy_final`: The completed port to HPy where all methods + receive ``HPy`` handles and ``PyObject_HEAD`` has been removed. + +Take a moment to read through :doc:`steps/step_00_c_api`. Then, once you're +ready, keep reading. + +Each section below corresponds to one of the three porting steps above: + +.. contents:: + :local: + :depth: 2 + +.. note:: + The steps used here are one approach to porting a module. The specific + steps are not required. They're just an example approach. + + +Step 01: Converting the module to a (legacy) HPy module +------------------------------------------------------- + +First for the easy bit -- let's include ``hpy.h``: + +.. literalinclude:: steps/step_01_hpy_legacy.c + :lineno-match: + :start-at: #include + :end-at: #include + +We'd like to differentiate between references to ``PyPointObject`` that have +been ported to HPy and those that haven't, so let's rename it to ``PointObject`` +and alias ``PyPointObject`` to ``PointObject``. We'll keep ``PyPointObject`` for +the instances that haven't been ported yet (the legacy ones) and use +``PointObject`` where we have ported the references: + +.. literalinclude:: steps/step_01_hpy_legacy.c + :lineno-match: + :start-at: typedef struct { + :end-at: } PointObject; + +.. literalinclude:: steps/step_01_hpy_legacy.c + :lineno-match: + :start-at: typedef PointObject PyPointObject; + :end-at: typedef PointObject PyPointObject; + +For this step, all references will be to ``PyPointObject`` -- we'll only start +porting references in the next step. + +Let's also call ``HPyType_LEGACY_HELPERS`` to define some helper functions +for use with the ``PointObject`` struct: + +.. literalinclude:: steps/step_01_hpy_legacy.c + :lineno-match: + :start-at: HPyType_LEGACY_HELPERS(PointObject) + :end-at: HPyType_LEGACY_HELPERS(PointObject) + +Again, we won't use these helpers in this step -- we're just setting things +up for later. + +Now for the big steps. + +We need to replace ``PyType_Spec`` for the ``Point`` type with the equivalent +``HPyType_Spec``: + +.. literalinclude:: steps/step_01_hpy_legacy.c + :lineno-match: + :start-at: // HPy type methods and slots (no methods or slots have been ported yet) + :end-before: // Legacy module methods (the "dot" method is still a PyCFunction) + +Initially the list of ported methods in ``point_defines`` is empty and all of +the methods are still in ``Point_slots`` which we have renamed to +``Point_legacy_slots`` for clarity. + +``SHAPE(PointObject)`` is a macro that retrieves the shape of ``PointObject`` as it +was defined by the ``HPyType_LEGACY_HELPERS`` macro and will be set to +``HPyType_BuiltinShape_Legacy`` until we replace the legacy macro with the +``HPyType_HELPERS`` one. Any type with ``legacy_slots`` or that still includes +``PyObject_HEAD`` in its struct should have ``.builtin_shape`` set to +``HPyType_BuiltinShape_Legacy``. + +Similarly we replace ``PyModuleDef`` with ``HPyModuleDef``: + +.. literalinclude:: steps/step_01_hpy_legacy.c + :lineno-match: + :start-at: // Legacy module methods (the "dot" method is still a PyCFunction) + :end-before: // END-OF: HPyModuleDef + +Like the type, the list of ported methods in ``module_defines`` is initially +almost empty: all the regular methods are still in ``PointModuleMethods`` which has +been renamed to ``PointModuleLegacyMethods``. However, because HPy supports only +multiphase module initialization, we must convert our module initialization code +to an "exec" slot on the module and add that slot to ``module_defines``. + +Now all that is left is to replace the module initialization function with +one that uses ``HPy_MODINIT``. The first argument is the name of the extension, +i.e., what was ``XXX`` in ``PyInit_XXX``, and the second argument +is the ``HPyModuleDef``. + +.. literalinclude:: steps/step_01_hpy_legacy.c + :lineno-match: + :start-at: HPy_MODINIT(step_01_hpy_legacy, moduledef) + +And we're done! + +Instead of the ``PyInit_XXX``, we now have an "exec" slot on the module. +We implement it with a C function that that takes an ``HPyContext *ctx`` and ``HPy mod`` +as arguments. The ``ctx`` must be forwarded as the first argument to calls to +HPy API methods. The ``mod`` argument is a handle for the module object. The runtime +creates the module for us from the provided ``HPyModuleDef``. There is no need to +call API like ``PyModule_Create`` explicitly. + +Next step is to replace ``PyType_FromSpec`` by ``HPyType_FromSpec``. + +``HPy_SetAttr_s`` is used to add the ``Point`` class to the module. HPy requires no +special ``PyModule_AddObject`` method. + +.. literalinclude:: steps/step_01_hpy_legacy.c + :lineno-match: + :start-at: HPyDef_SLOT(module_exec, HPy_mod_exec) + :end-at: } + + +Step 02: Transition some methods to HPy +--------------------------------------- + +In the previous step we put in place the type and module definitions required +to create an HPy extension module. In this step we will port some individual +methods. + +Let us start by migrating ``Point_traverse``. First we need to change +``PyObject *obj`` in the ``PointObject`` struct to ``HPyField obj``: + +.. literalinclude:: steps/step_02_hpy_legacy.c + :lineno-match: + :start-at: typedef struct { + :end-at: } PointObject; + +``HPy`` handles can only be short-lived -- i.e. local variables, arguments to +functions or return values. ``HPyField`` is the way to store long-lived +references to Python objects. For more information, please refer to the +documentation of :ref:`api-reference/hpy-field:HPyField`. + +Now we can update ``Point_traverse``: + +.. literalinclude:: steps/step_02_hpy_legacy.c + :lineno-match: + :start-at: HPyDef_SLOT(Point_traverse, HPy_tp_traverse) + :end-before: // this is a method for creating a Point + +In the first line we used the ``HPyDef_SLOT`` macro to define a small structure +that describes the slot being implemented. The first argument, ``Point_traverse``, +is the name to assign the structure to. By convention, the ``HPyDef_SLOT`` macro +expects a function called ``Point_traverse_impl`` implementing the slot. The +second argument, ``HPy_tp_traverse``, specifies the kind of slot. + +This is a change from how slots are defined in the old C API. In the old API, +the kind of slot is only specified much lower down in ``Point_legacy_slots``. In +HPy the implementation and kind are defined in one place using a syntax +reminiscent of Python decorators. + +The implementation of traverse is now a bit simpler than in the old C API. +We no longer need to visit ``Py_TYPE(self)`` and need only ``HPy_VISIT`` +``self->obj``. HPy ensures that interpreter knows that the type of the instance +is still referenced. + +Only struct members of type ``HPyField`` can be visited with ``HPy_VISIT``, which +is why we needed to convert ``obj`` to an ``HPyField`` before we implemented the +HPy traverse. + +Next we must update ``Point_init`` to store the value of ``obj`` as an ``HPyField``: + +.. literalinclude:: steps/step_02_hpy_legacy.c + :lineno-match: + :start-at: HPyDef_SLOT(Point_init, HPy_tp_init) + :end-before: // this is the getter for the associated object + +There are a few new HPy constructs used here: + +- The kind of the slot passed to ``HPyDef_SLOT`` is ``HPy_tp_init``. + +- ``PointObject_AsStruct`` is defined by ``HPyType_LEGACY_HELPERS`` and returns + an instance of the ``PointObject`` struct. Because we still include + ``PyObject_HEAD`` at the start of the struct this is still a valid ``PyObject *`` + but once we finish the port the struct will no longer contain ``PyObject_HEAD`` + and this will just be an ordinary C struct with no memory overhead! + +- We use ``HPyTracker`` when parsing the arguments with ``HPyArg_ParseKeywords``. + The ``HPyTracker`` keeps track of open handles so that they can be closed + easily at the end with ``HPyTracker_Close``. + +- ``HPyArg_ParseKeywords`` is the equivalent of ``PyArg_ParseTupleAndKeywords``. + Note that the HPy version does not steal a reference like the Python + version. + +- ``HPyField_Store`` is used to store a reference to ``obj`` in the struct. The + arguments are the context (``ctx``), a handle to the object that owns the + reference (``self``), the address of the ``HPyField`` (``&p->obj``), and the + handle to the object (``obj``). + +.. note:: + + An ``HPyTracker`` is not strictly needed for ``HPyArg_ParseKeywords`` + in ``Point_init``. The arguments ``x`` and ``y`` are C floats (so there are no + handles to close) and the handle stored in ``obj`` was passed in to the + ``Point_init`` as an argument and so should not be closed. + + We showed the tracker here to demonstrate its use. You can read more + about argument parsing in the + :doc:`API docs `. + + If a tracker is needed and one is not provided, ``HPyArg_ParseKeywords`` + will return an error. + + +The last update we need to make for the change to ``HPyField`` is to migrate +``Point_obj_get`` which retrieves ``obj`` from the stored ``HPyField``: + +.. literalinclude:: steps/step_02_hpy_legacy.c + :lineno-match: + :start-at: HPyDef_GET(Point_obj, "obj", .doc="Associated object.") + :end-before: // an HPy method of Point + +Above we have used ``PointObject_AsStruct`` again, and then ``HPyField_Load`` to +retrieve the value of ``obj`` from the ``HPyField``. + +We've now finished all of the changes needed by introducing ``HPyField``. We +could stop here, but let's migrate one ordinary method, ``Point_norm``, to end +off this stage of the port: + +.. literalinclude:: steps/step_02_hpy_legacy.c + :lineno-match: + :start-at: HPyDef_METH(Point_norm, "norm", HPyFunc_NOARGS, .doc="Distance from origin.") + :end-before: // this is an LEGACY function which casts a PyObject* into a PyPointObject* + +To define a method we use ``HPyDef_METH`` instead of ``HPyDef_SLOT``. ``HPyDef_METH`` +creates a small structure defining the method. The first argument is the name +to assign to the structure (``Point_norm``). The second is the Python name of +the method (``norm``). The third specifies the method signature (``HPyFunc_NOARGS`` +-- i.e. no additional arguments in this case). The last provides the docstring. +The macro then expects a function named ``Point_norm_impl`` implementing the +method. + +The rest of the implementation remains similar, except that we use +``HPyFloat_FromDouble`` to create a handle to a Python float containing the +result (i.e. the distance of the point from the origin). + +Now we are done and just have to remove the old implementations from +``Point_legacy_slots`` and add them to ``point_defines``: + +.. literalinclude:: steps/step_02_hpy_legacy.c + :lineno-match: + :start-at: static HPyDef *point_defines[] = { + :end-before: static HPyType_Spec Point_Type_spec = { + + +Step 03: Complete the port to HPy +--------------------------------- + +In this step we'll complete the port. We'll no longer include Python, remove +``PyObject_HEAD`` from the ``PointObject`` struct, and port the remaining methods. + +First, let's remove the import of ``Python.h``: + +.. literalinclude:: steps/step_03_hpy_final.c + :lineno-match: + :start-at: // #include // disallow use of the old C API + :end-at: // #include // disallow use of the old C API + +And ``PyObject_HEAD`` from the struct: + +.. literalinclude:: steps/step_03_hpy_final.c + :lineno-match: + :start-at: typedef struct { + :end-at: } PointObject; + +And the typedef of ``PointObject`` to ``PyPointObject``: + +.. literalinclude:: steps/step_03_hpy_final.c + :lineno-match: + :start-at: // typedef PointObject PyPointObject; + :end-at: // typedef PointObject PyPointObject; + +Now any code that has not been ported should result in a compilation error. + +We must also change the type helpers from ``HPyType_LEGACY_HELPERS`` to +``HPyType_HELPERS`` so that ``PointObject_AsStruct`` knows that ``PyObject_HEAD`` +has been removed: + +.. literalinclude:: steps/step_03_hpy_final.c + :lineno-match: + :start-at: HPyType_HELPERS(PointObject) + :end-at: HPyType_HELPERS(PointObject) + +There is one more method to port, the ``dot`` method which is a module method +that implements the dot product between two points: + +.. literalinclude:: steps/step_03_hpy_final.c + :lineno-match: + :start-at: HPyDef_METH(dot, "dot", HPyFunc_VARARGS, .doc="Dot product.") + :end-before: // Method, type and module definitions. In this porting step all + +The changes are similar to those used in porting the ``norm`` method, except: + +- We use ``HPyArg_Parse`` instead of ``HPyArg_ParseKeywordsDict``. + +- We opted not to use an ``HPyTracker`` by passing ``NULL`` as the pointer to the + tracker when calling ``HPyArg_Parse``. There is no reason not to use a + tracker here, but the handles to the two points are passed in as arguments + to ``dot_impl`` and thus there is no need to close them (and they should not + be closed). + +We use ``PointObject_AsStruct`` and ``HPyFloat_FromDouble`` as before. + +Now that we have ported everything we can remove ``PointMethods``, +``Point_legacy_slots`` and ``PointModuleLegacyMethods``. The resulting +type definition is much cleaner: + +.. literalinclude:: steps/step_03_hpy_final.c + :lineno-match: + :start-at: static HPyDef *point_defines[] = { + :end-before: // HPy module methods + +and the module definition is simpler too: + +.. literalinclude:: steps/step_03_hpy_final.c + :lineno-match: + :start-at: static HPyDef *module_defines[] = { + :end-before: HPy_MODINIT(step_03_hpy_final, moduledef) + +Now that the port is complete, when we compile our extension in HPy +universal mode, we obtain a built extension that depends only on the HPy ABI +and not on the CPython ABI at all! diff --git a/graalpython/hpy/docs/porting-example/steps/.gitignore b/graalpython/hpy/docs/porting-example/steps/.gitignore new file mode 100644 index 0000000000..36733d4937 --- /dev/null +++ b/graalpython/hpy/docs/porting-example/steps/.gitignore @@ -0,0 +1 @@ +step_03_hpy_final.py diff --git a/graalpython/hpy/docs/porting-example/steps/conftest.py b/graalpython/hpy/docs/porting-example/steps/conftest.py new file mode 100644 index 0000000000..06b7c319ee --- /dev/null +++ b/graalpython/hpy/docs/porting-example/steps/conftest.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- + +""" Pytest configuration for the porting example tests. """ + +import glob +import os +import sys + +import pytest + + +sys.path.insert(0, os.path.join(os.path.dirname(__file__))) + + +class PortingStep: + def __init__(self, src): + self.name = os.path.splitext(os.path.basename(src))[0] + self.src = src + + def import_step(self): + return __import__(self.name) + + +PORTING_STEPS = [ + PortingStep(src) for src in sorted( + glob.glob(os.path.join(os.path.dirname(__file__), "step_*.c"))) +] + + +@pytest.fixture( + params=PORTING_STEPS, + ids=[step.name for step in PORTING_STEPS], +) +def step(request): + return request.param diff --git a/graalpython/hpy/docs/porting-example/steps/setup00.py b/graalpython/hpy/docs/porting-example/steps/setup00.py new file mode 100644 index 0000000000..b3ea47b7aa --- /dev/null +++ b/graalpython/hpy/docs/porting-example/steps/setup00.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- + +from setuptools import setup, Extension + + +setup( + name="hpy-porting-example", + ext_modules=[ + Extension("step_00_c_api", sources=["step_00_c_api.c"]) + ], + py_modules=["step_00_c_api"], +) diff --git a/graalpython/hpy/docs/porting-example/steps/setup01.py b/graalpython/hpy/docs/porting-example/steps/setup01.py new file mode 100644 index 0000000000..33803453a7 --- /dev/null +++ b/graalpython/hpy/docs/porting-example/steps/setup01.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- + +from setuptools import setup, Extension + + +setup( + name="hpy-porting-example", + hpy_ext_modules=[ + Extension("step_01_hpy_legacy", sources=["step_01_hpy_legacy.c"]) + ], + py_modules=["step_01_hpy_legacy"], + setup_requires=['hpy'], +) diff --git a/graalpython/hpy/docs/porting-example/steps/setup02.py b/graalpython/hpy/docs/porting-example/steps/setup02.py new file mode 100644 index 0000000000..6ab9b95e73 --- /dev/null +++ b/graalpython/hpy/docs/porting-example/steps/setup02.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- + +from setuptools import setup, Extension + + +setup( + name="hpy-porting-example", + hpy_ext_modules=[ + Extension("step_02_hpy_legacy", sources=["step_02_hpy_legacy.c"]) + ], + py_modules=["step_02_hpy_legacy"], + setup_requires=['hpy'], +) diff --git a/graalpython/hpy/docs/porting-example/steps/setup03.py b/graalpython/hpy/docs/porting-example/steps/setup03.py new file mode 100644 index 0000000000..c12ec40020 --- /dev/null +++ b/graalpython/hpy/docs/porting-example/steps/setup03.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- + +from setuptools import setup, Extension + +# now we can add --hpy-abi=universal to the invocation of setup.py to build a +# universal binary +setup( + name="hpy-porting-example", + hpy_ext_modules=[ + Extension("step_03_hpy_final", sources=["step_03_hpy_final.c"]) + ], + py_modules=["step_03_hpy_final"], + setup_requires=['hpy'], +) diff --git a/graalpython/hpy/docs/porting-example/steps/step_00_c_api.c b/graalpython/hpy/docs/porting-example/steps/step_00_c_api.c new file mode 100644 index 0000000000..0b2742fd4d --- /dev/null +++ b/graalpython/hpy/docs/porting-example/steps/step_00_c_api.c @@ -0,0 +1,165 @@ +#include +#include + +// Porting to HPy, Step 0: Original Python C API version +// +// An example of porting a C extension that implements a Point type +// with a couple of simple methods (a norm and a dot product). It +// illustrates the steps needed to port types that contain additional +// C attributes (in this case, x and y). +// +// This file contains the original C API version that needs to be ported. +// +// HPy supports porting C extensions piece by piece. +// +// point_hpy_legacy_1.c illustrates a possible first step where all +// methods still receive PyObject arguments and may still cast them to +// PyPointObject if they are instances of Point. +// +// point_hpy_legacy_2.c shows how to transition some methods to HPy methods +// that receive HPy handles as arguments while still supporting legacy +// methods that receive PyObject arguments. +// +// point_hpy_final.c shows the completed port to HPy where all methods receive +// HPy handles and PyObject_HEAD has been removed. + +typedef struct { + PyObject_HEAD + double x; + double y; + PyObject *obj; +} PyPointObject; + +int Point_traverse(PyObject *self, visitproc visit, void *arg) +{ + Py_VISIT(((PyPointObject*)self)->obj); + Py_VISIT(Py_TYPE(self)); + return 0; +} + +void Point_dealloc(PyObject *self) +{ + Py_CLEAR(((PyPointObject*)self)->obj); + PyTypeObject *tp = Py_TYPE(self); + tp->tp_free(self); + Py_DECREF(tp); +} + +// this is a method for creating a Point +int Point_init(PyObject *self, PyObject *args, PyObject *kw) +{ + static char *kwlist[] = {"x", "y", "obj", NULL}; + PyPointObject *p = (PyPointObject *)self; + p->x = 0.0; + p->y = 0.0; + p->obj = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kw, "|ddO", kwlist, + &p->x, &p->y, &p->obj)) + return -1; + if (p->obj == NULL) + p->obj = Py_None; + Py_INCREF(p->obj); + return 0; +} + +// this is a method of Point +PyObject* Point_norm(PyObject *self) +{ + PyPointObject *p = (PyPointObject *)self; + double norm; + PyObject *result; + norm = sqrt(p->x * p->x + p->y * p->y); + result = PyFloat_FromDouble(norm); + return result; +} + +// this is the getter for the associated object +PyObject* Point_obj_get(PyObject *self, void *context) +{ + PyPointObject *p = (PyPointObject *)self; + Py_INCREF(p->obj); + return p->obj; +} + +// this is an unrelated function which happens to cast a PyObject* into a +// PyPointObject* +PyObject* dot(PyObject *self, PyObject *args) +{ + PyObject *point1, *point2; + if (!PyArg_ParseTuple(args, "OO", &point1, &point2)) + return NULL; + + PyPointObject *p1 = (PyPointObject *)point1; + PyPointObject *p2 = (PyPointObject *)point2; + + double dp; + PyObject *result; + dp = p1->x * p2->x + p1->y * p2->y; + result = PyFloat_FromDouble(dp); + return result; +} + + +// Method, type and module definitions. These will be updated to add HPy +// module support in point_hpy_legacy_1.c. + +static PyMethodDef PointMethods[] = { + {"norm", (PyCFunction)Point_norm, METH_NOARGS, "Distance from origin."}, + {NULL, NULL, 0, NULL} +}; + +static PyGetSetDef PointGetSets[] = { + {"obj", (getter)Point_obj_get, NULL, "Associated object.", NULL}, + {NULL, NULL, 0, NULL} +}; + +static PyType_Slot Point_slots[] = { + {Py_tp_doc, "Point (Step 0; C API implementation)"}, + {Py_tp_init, Point_init}, + {Py_tp_methods, PointMethods}, + {Py_tp_getset, PointGetSets}, + {Py_tp_traverse, Point_traverse}, + {Py_tp_dealloc, Point_dealloc}, + {0, 0} +}; + +static PyType_Spec Point_Type_spec = { + .name = "point_capi.Point", + .basicsize = sizeof(PyPointObject), + .itemsize = 0, + .flags = Py_TPFLAGS_DEFAULT, + .slots = Point_slots +}; + +static PyMethodDef PointModuleMethods[] = { + {"dot", (PyCFunction)dot, METH_VARARGS, "Dot product."}, + {NULL, NULL, 0, NULL} +}; + +static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + "step_00_c_api", + "Point module (Step 0; C API implementation)", + -1, + PointModuleMethods, + NULL, + NULL, + NULL, + NULL, +}; + +PyMODINIT_FUNC +PyInit_step_00_c_api(void) +{ + PyObject* m; + m = PyModule_Create(&moduledef); + if (m == NULL) + return NULL; + + PyObject *point_type = PyType_FromSpec(&Point_Type_spec); + if (point_type == NULL) + return NULL; + PyModule_AddObject(m, "Point", point_type); + + return m; +} diff --git a/graalpython/hpy/docs/porting-example/steps/step_00_c_api.rst b/graalpython/hpy/docs/porting-example/steps/step_00_c_api.rst new file mode 100644 index 0000000000..c9e9b76c18 --- /dev/null +++ b/graalpython/hpy/docs/porting-example/steps/step_00_c_api.rst @@ -0,0 +1,8 @@ +:orphan: + +step_00_c_api.c +=============== + +.. literalinclude:: ./step_00_c_api.c + :language: c + :linenos: diff --git a/graalpython/hpy/docs/porting-example/steps/step_01_hpy_legacy.c b/graalpython/hpy/docs/porting-example/steps/step_01_hpy_legacy.c new file mode 100644 index 0000000000..48137cc589 --- /dev/null +++ b/graalpython/hpy/docs/porting-example/steps/step_01_hpy_legacy.c @@ -0,0 +1,189 @@ +#include +#include +#include + +// Porting to HPy, Step 1: All legacy methods +// +// An example of porting a C extension that implements a Point type +// with a couple of simple methods (a norm and a dot product). It +// illustrates the steps needed to port types that contain additional +// C attributes (in this case, x and y). +// +// This file contains an example first step of the port in which all methods +// still receive PyObject arguments and may still cast them to +// PyPointObject if they are instances of Point. + +typedef struct { + // PyObject_HEAD is required while legacy_slots are still used + // but can (and should) be removed once the port to HPy is completed. + PyObject_HEAD + double x; + double y; + PyObject *obj; +} PointObject; + +// This defines PyPointObject as an alias of PointObject so that existing +// code that still uses PyPointObject and expects PyObject_HEAD continues to +// compile and run. Once PyObject_HEAD has been removed, this alias should be +// removed so that code that still expects PyObject_HEAD will fail to compile. +typedef PointObject PyPointObject; + +// The legacy type helper macro defines an PointObject_AsStruct function allows +// non-legacy methods to convert HPy handles to PointObject structs. It is not +// used in this file, but is provided so that methods can start to be ported +// (see point_hpy_legacy_2.c). The legacy type helper macro is used because +// PyObject_HEAD is still present in PointObject. Once PyObject_HEAD has been +// removed (see point_hpy_final.c) we will use HPy_TYPE_HELPERS instead. +HPyType_LEGACY_HELPERS(PointObject) + +int Point_traverse(PyObject *self, visitproc visit, void *arg) +{ + Py_VISIT(((PyPointObject*)self)->obj); + Py_VISIT(Py_TYPE(self)); + return 0; +} + +void Point_dealloc(PyObject *self) +{ + Py_CLEAR(((PyPointObject*)self)->obj); + PyTypeObject *tp = Py_TYPE(self); + tp->tp_free(self); + Py_DECREF(tp); +} + +// this is a method for creating a Point +int Point_init(PyObject *self, PyObject *args, PyObject *kw) +{ + static char *kwlist[] = {"x", "y", "obj", NULL}; + PyPointObject *p = (PyPointObject *)self; + p->x = 0.0; + p->y = 0.0; + p->obj = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kw, "|ddO", kwlist, + &p->x, &p->y, &p->obj)) + return -1; + if (p->obj == NULL) + p->obj = Py_None; + Py_INCREF(p->obj); + return 0; +} + +// this is a LEGACY method of Point +PyObject* Point_norm(PyObject *self) +{ + PyPointObject *p = (PyPointObject *)self; + double norm; + norm = sqrt(p->x * p->x + p->y * p->y); + return PyFloat_FromDouble(norm); +} + +// this is the getter for the associated object +PyObject* Point_obj_get(PyObject *self, void *context) +{ + PyPointObject *p = (PyPointObject *)self; + Py_INCREF(p->obj); + return p->obj; +} + +// this is an LEGACY function which casts a PyObject* into a PyPointObject* +PyObject* dot(PyObject *self, PyObject *args) +{ + PyObject *point1, *point2; + if (!PyArg_ParseTuple(args, "OO", &point1, &point2)) + return NULL; + + PyPointObject *p1 = (PyPointObject *)point1; + PyPointObject *p2 = (PyPointObject *)point2; + + double dp; + dp = p1->x * p2->x + p1->y * p2->y; + return PyFloat_FromDouble(dp); +} + + +// Method, type and module definitions. In this porting step, the module and +// type definitions have been ported to HPy, but the methods themselves +// remaining legacy methods. + +// Legacy methods (all methods are still legacy methods) +static PyMethodDef PointMethods[] = { + {"norm", (PyCFunction)Point_norm, METH_NOARGS, "Distance from origin."}, + {NULL, NULL, 0, NULL} +}; + +// Legacy getsets +static PyGetSetDef PointGetSets[] = { + {"obj", (getter)Point_obj_get, NULL, "Associated object.", NULL}, + {NULL, NULL, 0, NULL} +}; + +// Legacy slots (all slots are still legacy slots) +static PyType_Slot Point_legacy_slots[] = { + {Py_tp_doc, "Point (Step 1; All legacy methods)"}, + {Py_tp_init, Point_init}, + {Py_tp_methods, PointMethods}, + {Py_tp_getset, PointGetSets}, + {Py_tp_traverse, Point_traverse}, + {Py_tp_dealloc, Point_dealloc}, + {0, 0} +}; + +// HPy type methods and slots (no methods or slots have been ported yet) +static HPyDef *point_defines[] = { + NULL +}; + +static HPyType_Spec Point_Type_spec = { + .name = "point_hpy_legacy_1.Point", + .basicsize = sizeof(PointObject), + .itemsize = 0, + .flags = HPy_TPFLAGS_DEFAULT, + .builtin_shape = SHAPE(PointObject), + .legacy_slots = Point_legacy_slots, + .defines = point_defines, +}; + +// HPy supports only multiphase module initialization, so we must migrate the +// single phase initialization by extracting the code that populates the module +// object with attributes into a separate 'exec' slot. The module is not +// created manually by calling API like PyModule_Create, but the runtime creates +// the module for us from the specification in HPyModuleDef, and we can provide +// additional slots to populate the module before its initialization is finalized +HPyDef_SLOT(module_exec, HPy_mod_exec) +static int module_exec_impl(HPyContext *ctx, HPy mod) +{ + HPy point_type = HPyType_FromSpec(ctx, &Point_Type_spec, NULL); + if (HPy_IsNull(point_type)) + return -1; + HPy_SetAttr_s(ctx, mod, "Point", point_type); + return 0; +} + +// Legacy module methods (the "dot" method is still a PyCFunction) +static PyMethodDef PointModuleLegacyMethods[] = { + {"dot", (PyCFunction)dot, METH_VARARGS, "Dot product."}, + {NULL, NULL, 0, NULL} +}; + +// HPy module methods: no regular methods have been ported yet, +// but we add the module execute slot +static HPyDef *module_defines[] = { + &module_exec, + NULL +}; + +static HPyModuleDef moduledef = { + // .name = "step_01_hpy_legacy", + // ^-- .name is not needed for multiphase module initialization, + // it is always taken from the ModuleSpec + .doc = "Point module (Step 1; All legacy methods)", + .size = 0, + .legacy_methods = PointModuleLegacyMethods, + .defines = module_defines, +}; +// END-OF: HPyModuleDef + +// HPy_MODINIT takes the extension name, i.e., what would be XXX in PyInit_XXX, +// and the module definition. The module will be created by the runtime and +// passed to the HPy_mod_exec slots if any are defined +HPy_MODINIT(step_01_hpy_legacy, moduledef) diff --git a/graalpython/hpy/docs/porting-example/steps/step_01_hpy_legacy.rst b/graalpython/hpy/docs/porting-example/steps/step_01_hpy_legacy.rst new file mode 100644 index 0000000000..7ba453ac10 --- /dev/null +++ b/graalpython/hpy/docs/porting-example/steps/step_01_hpy_legacy.rst @@ -0,0 +1,8 @@ +:orphan: + +step_01_hpy_legacy.c +==================== + +.. literalinclude:: ./step_01_hpy_legacy.c + :language: c + :linenos: diff --git a/graalpython/hpy/docs/porting-example/steps/step_02_hpy_legacy.c b/graalpython/hpy/docs/porting-example/steps/step_02_hpy_legacy.c new file mode 100644 index 0000000000..0d4ff7b04b --- /dev/null +++ b/graalpython/hpy/docs/porting-example/steps/step_02_hpy_legacy.c @@ -0,0 +1,169 @@ +#include +#include +#include + +// Porting to HPy, Step 2: Porting some methods +// +// An example of porting a C extension that implements a Point type +// with a couple of simple methods (a norm and a dot product). It +// illustrates the steps needed to port types that contain additional +// C attributes (in this case, x and y). +// +// This file contains an example second step of the port in which some methods +// have been converted to HPy methods that receive handles as arguments, but +// other methods are still legacy methods that receive PyObject arguments. + +typedef struct { + // PyObject_HEAD is required while legacy methods still access + // PointObject and should be removed once the port to HPy is completed. + PyObject_HEAD + double x; + double y; + // HPy handles are shortlived to support all GC strategies + // For that reason, PyObject* in C structs are replaced by HPyField + HPyField obj; +} PointObject; + +// This defines PyPointObject as an alias of PointObject so that existing +// code that still uses PyPointObject and expects PyObject_HEAD continues to +// compile and run. Once PyObject_HEAD has been removed, this alias should be +// removed so that code that still expects PyObject_HEAD will fail to compile. +typedef PointObject PyPointObject; + +// The legacy type helper macro defines an PointObject_AsStruct function allows +// non-legacy methods to convert HPy handles to PointObject structs. The legacy +// type helper macro is used because PyObject_HEAD is still present in +// PointObject. Once PyObject_HEAD has been removed (see point_hpy_final.c) we +// will use HPy_TYPE_HELPERS instead. +HPyType_LEGACY_HELPERS(PointObject) + +HPyDef_SLOT(Point_traverse, HPy_tp_traverse) +int Point_traverse_impl(void *self, HPyFunc_visitproc visit, void *arg) +{ + HPy_VISIT(&((PointObject*)self)->obj); + return 0; +} + +// this is a method for creating a Point +HPyDef_SLOT(Point_init, HPy_tp_init) +int Point_init_impl(HPyContext *ctx, HPy self, const HPy *args, + HPy_ssize_t nargs, HPy kw) +{ + static const char *kwlist[] = {"x", "y", "obj", NULL}; + PointObject *p = PointObject_AsStruct(ctx, self); + p->x = 0.0; + p->y = 0.0; + HPy obj = HPy_NULL; + HPyTracker ht; + if (!HPyArg_ParseKeywordsDict(ctx, &ht, args, nargs, kw, "|ddO", kwlist, + &p->x, &p->y, &obj)) + return -1; + if (HPy_IsNull(obj)) + obj = ctx->h_None; + /* INCREF not needed because HPyArg_ParseKeywordsDict does not steal a + reference */ + HPyField_Store(ctx, self, &p->obj, obj); + HPyTracker_Close(ctx, ht); + return 0; +} + +// this is the getter for the associated object +HPyDef_GET(Point_obj, "obj", .doc="Associated object.") +HPy Point_obj_get(HPyContext *ctx, HPy self, void* closure) +{ + PointObject *p = PointObject_AsStruct(ctx, self); + return HPyField_Load(ctx, self, p->obj); +} + +// an HPy method of Point +HPyDef_METH(Point_norm, "norm", HPyFunc_NOARGS, .doc="Distance from origin.") +HPy Point_norm_impl(HPyContext *ctx, HPy self) +{ + PointObject *p = PointObject_AsStruct(ctx, self); + double norm; + norm = sqrt(p->x * p->x + p->y * p->y); + return HPyFloat_FromDouble(ctx, norm); +} + +// this is an LEGACY function which casts a PyObject* into a PyPointObject* +PyObject* dot(PyObject *self, PyObject *args) +{ + PyObject *point1, *point2; + if (!PyArg_ParseTuple(args, "OO", &point1, &point2)) + return NULL; + + PyPointObject *p1 = (PyPointObject *)point1; + PyPointObject *p2 = (PyPointObject *)point2; + + double dp; + dp = p1->x * p2->x + p1->y * p2->y; + return PyFloat_FromDouble(dp); +} + +// Method, type and module definitions. In this porting step .norm() +// is ported to HPy, but dot(...) remains a legacy methods. +// Point.__init__ and Point.__doc__ are ported from legacy slots to +// HPy type defines. + +// Legacy methods (there are no legacy methods left now) +static PyMethodDef PointMethods[] = { + {NULL, NULL, 0, NULL} +}; + +// Legacy slots (all slots are still legacy slots) +static PyType_Slot Point_legacy_slots[] = { + {Py_tp_doc, "Point (Step 2; Porting some methods)"}, + {Py_tp_methods, PointMethods}, + {0, 0} +}; + +// HPy type methods and slots +static HPyDef *point_defines[] = { + &Point_init, + &Point_norm, + &Point_obj, + &Point_traverse, + NULL +}; + +static HPyType_Spec Point_Type_spec = { + .name = "point_hpy_legacy_2.Point", + .basicsize = sizeof(PointObject), + .itemsize = 0, + .flags = HPy_TPFLAGS_DEFAULT, + .builtin_shape = SHAPE(PointObject), + .legacy_slots = Point_legacy_slots, + .defines = point_defines +}; + +// Legacy module methods (the "dot" method is still a PyCFunction) +static PyMethodDef PointModuleLegacyMethods[] = { + {"dot", (PyCFunction)dot, METH_VARARGS, "Dot product."}, + {NULL, NULL, 0, NULL} +}; + +HPyDef_SLOT(module_exec, HPy_mod_exec) +static int module_exec_impl(HPyContext *ctx, HPy mod) +{ + HPy point_type = HPyType_FromSpec(ctx, &Point_Type_spec, NULL); + if (HPy_IsNull(point_type)) + return -1; + HPy_SetAttr_s(ctx, mod, "Point", point_type); + return 0; +} + +// HPy module methods: no regular methods have been ported yet, +// but we add the module execute slot +static HPyDef *module_defines[] = { + &module_exec, + NULL +}; + +static HPyModuleDef moduledef = { + .doc = "Point module (Step 2; Porting some methods)", + .size = 0, + .legacy_methods = PointModuleLegacyMethods, + .defines = module_defines, +}; + +HPy_MODINIT(step_02_hpy_legacy, moduledef) diff --git a/graalpython/hpy/docs/porting-example/steps/step_02_hpy_legacy.rst b/graalpython/hpy/docs/porting-example/steps/step_02_hpy_legacy.rst new file mode 100644 index 0000000000..5523681465 --- /dev/null +++ b/graalpython/hpy/docs/porting-example/steps/step_02_hpy_legacy.rst @@ -0,0 +1,8 @@ +:orphan: + +step_02_hpy_legacy.c +==================== + +.. literalinclude:: ./step_02_hpy_legacy.c + :language: c + :linenos: diff --git a/graalpython/hpy/docs/porting-example/steps/step_03_hpy_final.c b/graalpython/hpy/docs/porting-example/steps/step_03_hpy_final.c new file mode 100644 index 0000000000..8f49ecf857 --- /dev/null +++ b/graalpython/hpy/docs/porting-example/steps/step_03_hpy_final.c @@ -0,0 +1,153 @@ +#include +// #include // disallow use of the old C API +#include + +// Porting to HPy, Step 3: All methods ported +// +// An example of porting a C extension that implements a Point type +// with a couple of simple methods (a norm and a dot product). It +// illustrates the steps needed to port types that contain additional +// C attributes (in this case, x and y). +// +// This file contains an example final step of the port in which all methods +// have been converted to HPy methods and PyObject_HEAD has been removed. + +typedef struct { + // PyObject_HEAD is no longer available in PointObject. In CPython, + // of course, it still exists but is inaccessible from HPy_AsStruct. In + // other Python implementations (e.g. PyPy) it might no longer exist at + // all. + double x; + double y; + HPyField obj; +} PointObject; + +// Code using PyPointObject relied on PyObject_HEAD and is no longer valid +// (PyObject_HEAD has been removed from the PointObject struct above). The +// typedef below has been deleted to ensure that such code is now generates +// an error during compilation. +// typedef PointObject PyPointObject; + +// The type helper macro defines an PointObject_AsStruct function allows +// converting HPy handles to PointObject structs. We no longer need to use +// the legacy type helper macro because PyObject_HEAD has been removed from +// PointObject. +HPyType_HELPERS(PointObject) + +HPyDef_SLOT(Point_traverse, HPy_tp_traverse) +int Point_traverse_impl(void *self, HPyFunc_visitproc visit, void *arg) +{ + HPy_VISIT(&((PointObject*)self)->obj); + return 0; +} + +// this is a method for creating a Point +HPyDef_SLOT(Point_init, HPy_tp_init) +int Point_init_impl(HPyContext *ctx, HPy self, const HPy *args, + HPy_ssize_t nargs, HPy kw) +{ + static const char *kwlist[] = {"x", "y", "obj", NULL}; + PointObject *p = PointObject_AsStruct(ctx, self); + p->x = 0.0; + p->y = 0.0; + HPy obj = HPy_NULL; + HPyTracker ht; + if (!HPyArg_ParseKeywordsDict(ctx, &ht, args, nargs, kw, "|ddO", kwlist, + &p->x, &p->y, &obj)) + return -1; + if (HPy_IsNull(obj)) + obj = ctx->h_None; + /* INCREF not needed because HPyArg_ParseKeywordsDict does not steal a + reference */ + HPyField_Store(ctx, self, &p->obj, obj); + HPyTracker_Close(ctx, ht); + return 0; +} + +// this is the getter for the associated object +HPyDef_GET(Point_obj, "obj", .doc="Associated object.") +HPy Point_obj_get(HPyContext *ctx, HPy self, void* closure) +{ + PointObject *p = PointObject_AsStruct(ctx, self); + return HPyField_Load(ctx, self, p->obj); +} + +// an HPy method of Point +HPyDef_METH(Point_norm, "norm", HPyFunc_NOARGS, .doc="Distance from origin.") +HPy Point_norm_impl(HPyContext *ctx, HPy self) +{ + PointObject *p = PointObject_AsStruct(ctx, self); + double norm; + norm = sqrt(p->x * p->x + p->y * p->y); + return HPyFloat_FromDouble(ctx, norm); +} + +// this is an HPy function that uses Point +HPyDef_METH(dot, "dot", HPyFunc_VARARGS, .doc="Dot product.") +HPy dot_impl(HPyContext *ctx, HPy self, const HPy *args, size_t nargs) +{ + HPy point1, point2; + if (!HPyArg_Parse(ctx, NULL, args, nargs, "OO", &point1, &point2)) + return HPy_NULL; + PointObject *p1 = PointObject_AsStruct(ctx, point1); + PointObject *p2 = PointObject_AsStruct(ctx, point2); + double dp; + dp = p1->x * p2->x + p1->y * p2->y; + return HPyFloat_FromDouble(ctx, dp); +} + +// Method, type and module definitions. In this porting step all +// methods and slots have been ported to HPy and all legacy support +// has been removed. + +// Support for legacy methods and slots has been removed. It used to be: +/// +// static PyMethodDef PointMethods[] = { ... } +// static PyType_Slot Point_legacy_slots[] = { ... } +// static PyMethodDef PointModuleLegacyMethods[] = { ... } +// +// and .legacy_slots and .legacy_defines have been removed from HPyType_Spec +// HPyModuleDef respectively. + +// HPy type methods and slots +static HPyDef *point_defines[] = { + &Point_init, + &Point_norm, + &Point_obj, + &Point_traverse, + NULL +}; + +static HPyType_Spec Point_Type_spec = { + .name = "point_hpy_final.Point", + .doc = "Point (Step 03)", + .basicsize = sizeof(PointObject), + .itemsize = 0, + .flags = HPy_TPFLAGS_DEFAULT, + .defines = point_defines +}; + +HPyDef_SLOT(module_exec, HPy_mod_exec) +static int module_exec_impl(HPyContext *ctx, HPy mod) +{ + HPy point_type = HPyType_FromSpec(ctx, &Point_Type_spec, NULL); + if (HPy_IsNull(point_type)) + return -1; + HPy_SetAttr_s(ctx, mod, "Point", point_type); + return 0; +} + +// HPy module methods +static HPyDef *module_defines[] = { + &module_exec, + &dot, + NULL +}; + +static HPyModuleDef moduledef = { + .doc = "Point module (Step 3; Porting complete)", + .size = 0, + .defines = module_defines, +}; + +HPy_MODINIT(step_03_hpy_final, moduledef) diff --git a/graalpython/hpy/docs/porting-example/steps/step_03_hpy_final.rst b/graalpython/hpy/docs/porting-example/steps/step_03_hpy_final.rst new file mode 100644 index 0000000000..3b8f561fc7 --- /dev/null +++ b/graalpython/hpy/docs/porting-example/steps/step_03_hpy_final.rst @@ -0,0 +1,8 @@ +:orphan: + +step_03_hpy_final.c +=================== + +.. literalinclude:: ./step_03_hpy_final.c + :language: c + :linenos: diff --git a/graalpython/hpy/docs/porting-example/steps/test_porting_example.py b/graalpython/hpy/docs/porting-example/steps/test_porting_example.py new file mode 100644 index 0000000000..013b010c03 --- /dev/null +++ b/graalpython/hpy/docs/porting-example/steps/test_porting_example.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- + +""" Porting example tests. """ + +import pytest +import math +import types + + +class TestPorting: + def test_load_module(self, step): + mod = step.import_step() + assert isinstance(mod, types.ModuleType) + assert mod.__name__ == step.name + assert mod.__doc__.startswith("Point module (Step ") + assert type(mod.Point) == type + assert mod.Point.__doc__.startswith("Point (Step ") + assert isinstance(mod.dot, types.BuiltinFunctionType) + assert mod.dot.__doc__ == "Dot product." + + def test_create_point(self, step): + mod = step.import_step() + p = mod.Point(1, 2) + assert type(p) == mod.Point + + def test_norm(self, step): + mod = step.import_step() + assert mod.Point(1, 2).norm() == math.sqrt(5.0) + assert mod.Point(1.5).norm() == 1.5 + + def test_dot(self, step): + mod = step.import_step() + p1 = mod.Point(1, 2) + p2 = mod.Point(3, 2) + assert mod.dot(p1, p2) == 7.0 + + def test_object(self, step): + mod = step.import_step() + p1 = mod.Point(23, 42, ...) + assert p1.obj is ... + p2 = mod.Point(23, 42) + assert p2.obj is None + + def test_leak_checker(self, step): + if "hpy_final" not in step.name: + pytest.skip("Can only check for leaks in universal mode") + mod = step.import_step() + import hpy.debug + hpy.debug.set_handle_stack_trace_limit(10) + with hpy.debug.LeakDetector(): + p1 = mod.Point(1, 2, ...) + p2 = mod.Point(3, 2) + + assert p1.obj is ... + assert p2.obj is None + assert p1.norm() == math.sqrt(5.0) + assert mod.dot(p1, p2) == 7.0 + + del p1 + del p2 diff --git a/graalpython/hpy/docs/porting-guide.rst b/graalpython/hpy/docs/porting-guide.rst new file mode 100644 index 0000000000..c21961366d --- /dev/null +++ b/graalpython/hpy/docs/porting-guide.rst @@ -0,0 +1,569 @@ +Porting Guide +============= + +Porting ``PyObject *`` to HPy API constructs +-------------------------------------------- + +While in CPython one always uses ``PyObject *`` to reference to Python objects, +in HPy there are several types of handles that should be used depending on the +life-time of the handle: ``HPy``, ``HPyField``, and ``HPyGlobal``. + +- ``HPy`` represents short lived handles that live no longer than the duration of + one call from Python to HPy extension function. Rule of thumb: use for local + variables, arguments, and return values. + +- ``HPyField`` represents handles that are Python object struct fields, i.e., + live in native memory attached to some Python object. + +- ``HPyGlobal`` represents handles stored in C global variables. ``HPyGlobal`` + can provide isolation between subinterpreters. + +.. warning:: Never use a local variable of type ``HPyField``, for any reason! If + the GC kicks in, it might become invalid and become a dangling pointer. + +.. warning:: Never store `HPy` handles to a long-lived memory, for example: C + global variables or Python object structs. + +The ``HPy``/``HPyField`` dichotomy might seem arbitrary at first, but it is +needed to allow Python implementations to use a moving GC, such as PyPy. It is +easier to explain and understand the rules by thinking about how a moving GC +interacts with the C code inside an HPy extension. + +It is worth remembering that during the collection phase, a moving GC might +move an existing object to another memory location, and in that case it needs +to update all the places which store a pointer to it. In order to do so, it +needs to *know* where the pointers are. If there is a local C variable which is +unknown to the GC but contains a pointer to a GC-managed object, the variable +will point to invalid memory as soon as the object is moved. + +Back to ``HPy`` vs ``HPyField`` vs ``HPyGlobal``: + + * ``HPy`` handles must be used for all C local variables, function arguments + and function return values. They are supposed to be short-lived and closed + as soon as they are no longer needed. The debug mode will report a + long-lived ``HPy`` as a potential memory leak. + + * In PyPy and GraalPy, ``HPy`` handles are implemented using an + indirection: they are indexes inside a big list of GC-managed objects: this + big list is tracked by the GC, so when an object moves its pointer is + correctly updated. + + * ``HPyField`` is for long-lived references, and the GC must be aware of + their location in memory. In PyPy, an ``HPyField`` is implemented as a + direct pointer to the object, and thus we need a way to inform the GC + where it is in memory, so that it can update its value upon moving: this + job is done by ``tp_traverse``, as explained in the next section. + + * ``HPyGlobal`` is for long-lived references that are supposed to be closed + implicitly when the module is unloaded (once module unloading is actually + implemented). ``HPyGlobal`` provides indirection to isolate subinterpreters. + Implementation wise, ``HPyGlobal`` will usually contain an index to a table + with Python objects stored in the interpreter state. + + * On CPython without subinterpreters support, ``HPy``, ``HPyGlobal``, + and ``HPyField`` are implemented as ``PyObject *``. + + * On CPython with subinterpreters support, ``HPyGlobal`` will be implemented + by an indirection through the interpreter state. Note that thanks to the HPy + design, switching between this and the more efficient implementation without + subinterpreter support will not require rebuilding of the extension (in HPy + universal mode), nor rebuilding of CPython. + +.. note:: If you write a custom type using ``HPyField``, you **MUST** also write + a ``tp_traverse`` slot. Note that this is different than the old ``Python.h`` + API, where you need ``tp_traverse`` only under certain conditions. See the + next section for more details. + +.. note:: The contract of ``tp_traverse`` is that it must visit all members of + type ``HPyField`` contained within given struct, or more precisely *owned* by + given Python object (in the sense of the *owner* argument to + ``HPyField_Store``), and nothing more, nothing less. Some Python + implementations may choose to not call the provided ``tp_traverse`` if they + know how to visit all members of type ``HPyField`` by other means (for + example, when they track them internally already). The debug mode will check + this contract. + +``tp_traverse``, ``tp_clear``, ``Py_TPFLAGS_HAVE_GC`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Let's quote the ``Python.h`` documentation about `GC support +`_ + + Python's support for detecting and collecting garbage which involves + circular references requires support from object types which are + “containers” for other objects which may also be containers. Types which do + not store references to other objects, or which only store references to + atomic types (such as numbers or strings), do not need to provide any + explicit support for garbage collection. + +A good rule of thumb is that if your type contains ``PyObject *`` fields, you +need to: + + 1. provide a ``tp_traverse`` slot; + + 2. provide a ``tp_clear`` slot; + + 3. add the ``Py_TPFLAGS_GC`` to the ``tp_flags``. + + +However, if you know that your ``PyObject *`` fields will contain only +"atomic" types, you can avoid these steps. + +In HPy the rules are slightly different: + + 1. if you have a field of type ``HPyField``, you always **MUST** provide a + ``tp_traverse``. This is needed so that a moving GC can track the + relevant areas of memory. However, you **MUST NOT** rely on + ``tp_traverse`` to be called; + + 2. ``tp_clear`` does not exist. On CPython, ``HPy`` automatically generates + one for you, by using ``tp_traverse`` to know which are the fields to + clear. Other implementations are free to ignore it, if it's not needed; + + 3. ``HPy_TPFLAGS_GC`` is still needed, especially on CPython. If you don't + specify it, your type will not be tracked by CPython's GC and thus it + might cause memory leaks if it's part of a reference cycle. However, + other implementations are free to ignore the flag and track the objects + anyway, if their GC implementation allows it. + +``tp_dealloc`` and ``Py_DECREF`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Generally speaking, if you have one or more ``PyObject *`` fields in the old +``Python.h``, you must provide a ``tp_dealloc`` slot where you ``Py_DECREF`` all +of them. In HPy this is not needed and will be handled automatically by the +system. + +In particular, when running on top of CPython, HPy will automatically provide +a ``tp_dealloc`` which decrefs all the fields listed by ``tp_traverse``. + +See also, :ref:`dealloc`. + + +Direct C API to HPy mappings +---------------------------- + +In many cases, migrating to HPy is as easy as just replacing a certain C API +function by the appropriate HPy API function. Table :ref:`table-mapping` gives a +mapping between C API and HPy API functions. This mapping is generated together +with the code for the :term:`CPython ABI` mode, so it is guaranteed to be correct. + + +.. mark: BEGIN API MAPPING +.. _table-mapping: +.. table:: Safe API function mapping + :widths: auto + + ================================================================================================================================== ================================================ + C API function HPY API function + ================================================================================================================================== ================================================ + `PyBool_FromLong `_ :c:func:`HPyBool_FromLong` + `PyBytes_AS_STRING `_ :c:func:`HPyBytes_AS_STRING` + `PyBytes_AsString `_ :c:func:`HPyBytes_AsString` + `PyBytes_Check `_ :c:func:`HPyBytes_Check` + `PyBytes_FromString `_ :c:func:`HPyBytes_FromString` + `PyBytes_GET_SIZE `_ :c:func:`HPyBytes_GET_SIZE` + `PyBytes_Size `_ :c:func:`HPyBytes_Size` + `PyCallable_Check `_ :c:func:`HPyCallable_Check` + `PyCapsule_IsValid `_ :c:func:`HPyCapsule_IsValid` + `PyContextVar_Get `_ :c:func:`HPyContextVar_Get` + `PyContextVar_New `_ :c:func:`HPyContextVar_New` + `PyContextVar_Set `_ :c:func:`HPyContextVar_Set` + `PyDict_Check `_ :c:func:`HPyDict_Check` + `PyDict_Copy `_ :c:func:`HPyDict_Copy` + `PyDict_Keys `_ :c:func:`HPyDict_Keys` + `PyDict_New `_ :c:func:`HPyDict_New` + `PyErr_Clear `_ :c:func:`HPyErr_Clear` + `PyErr_ExceptionMatches `_ :c:func:`HPyErr_ExceptionMatches` + `PyErr_NewException `_ :c:func:`HPyErr_NewException` + `PyErr_NewExceptionWithDoc `_ :c:func:`HPyErr_NewExceptionWithDoc` + `PyErr_NoMemory `_ :c:func:`HPyErr_NoMemory` + `PyErr_SetFromErrnoWithFilename `_ :c:func:`HPyErr_SetFromErrnoWithFilename` + `PyErr_SetFromErrnoWithFilenameObjects `_ :c:func:`HPyErr_SetFromErrnoWithFilenameObjects` + `PyErr_SetObject `_ :c:func:`HPyErr_SetObject` + `PyErr_SetString `_ :c:func:`HPyErr_SetString` + `PyErr_WarnEx `_ :c:func:`HPyErr_WarnEx` + `PyErr_WriteUnraisable `_ :c:func:`HPyErr_WriteUnraisable` + `PyEval_EvalCode `_ :c:func:`HPy_EvalCode` + `PyEval_RestoreThread `_ :c:func:`HPy_ReenterPythonExecution` + `PyEval_SaveThread `_ :c:func:`HPy_LeavePythonExecution` + `PyFloat_AsDouble `_ :c:func:`HPyFloat_AsDouble` + `PyFloat_FromDouble `_ :c:func:`HPyFloat_FromDouble` + `PyImport_ImportModule `_ :c:func:`HPyImport_ImportModule` + `PyIter_Check `_ :c:func:`HPyIter_Check` + `PyIter_Next `_ :c:func:`HPyIter_Next` + `PyList_Append `_ :c:func:`HPyList_Append` + `PyList_Check `_ :c:func:`HPyList_Check` + `PyList_Insert `_ :c:func:`HPyList_Insert` + `PyList_New `_ :c:func:`HPyList_New` + `PyLong_AsDouble `_ :c:func:`HPyLong_AsDouble` + `PyLong_AsLong `_ :c:func:`HPyLong_AsLong` + `PyLong_AsLongLong `_ :c:func:`HPyLong_AsLongLong` + `PyLong_AsSize_t `_ :c:func:`HPyLong_AsSize_t` + `PyLong_AsSsize_t `_ :c:func:`HPyLong_AsSsize_t` + `PyLong_AsUnsignedLong `_ :c:func:`HPyLong_AsUnsignedLong` + `PyLong_AsUnsignedLongLong `_ :c:func:`HPyLong_AsUnsignedLongLong` + `PyLong_AsUnsignedLongLongMask `_ :c:func:`HPyLong_AsUnsignedLongLongMask` + `PyLong_AsUnsignedLongMask `_ :c:func:`HPyLong_AsUnsignedLongMask` + `PyLong_AsVoidPtr `_ :c:func:`HPyLong_AsVoidPtr` + `PyLong_FromLong `_ :c:func:`HPyLong_FromLong` + `PyLong_FromLongLong `_ :c:func:`HPyLong_FromLongLong` + `PyLong_FromSize_t `_ :c:func:`HPyLong_FromSize_t` + `PyLong_FromSsize_t `_ :c:func:`HPyLong_FromSsize_t` + `PyLong_FromUnsignedLong `_ :c:func:`HPyLong_FromUnsignedLong` + `PyLong_FromUnsignedLongLong `_ :c:func:`HPyLong_FromUnsignedLongLong` + `PyNumber_Absolute `_ :c:func:`HPy_Absolute` + `PyNumber_Add `_ :c:func:`HPy_Add` + `PyNumber_And `_ :c:func:`HPy_And` + `PyNumber_Check `_ :c:func:`HPyNumber_Check` + `PyNumber_Divmod `_ :c:func:`HPy_Divmod` + `PyNumber_Float `_ :c:func:`HPy_Float` + `PyNumber_FloorDivide `_ :c:func:`HPy_FloorDivide` + `PyNumber_InPlaceAdd `_ :c:func:`HPy_InPlaceAdd` + `PyNumber_InPlaceAnd `_ :c:func:`HPy_InPlaceAnd` + `PyNumber_InPlaceFloorDivide `_ :c:func:`HPy_InPlaceFloorDivide` + `PyNumber_InPlaceLshift `_ :c:func:`HPy_InPlaceLshift` + `PyNumber_InPlaceMatrixMultiply `_ :c:func:`HPy_InPlaceMatrixMultiply` + `PyNumber_InPlaceMultiply `_ :c:func:`HPy_InPlaceMultiply` + `PyNumber_InPlaceOr `_ :c:func:`HPy_InPlaceOr` + `PyNumber_InPlacePower `_ :c:func:`HPy_InPlacePower` + `PyNumber_InPlaceRemainder `_ :c:func:`HPy_InPlaceRemainder` + `PyNumber_InPlaceRshift `_ :c:func:`HPy_InPlaceRshift` + `PyNumber_InPlaceSubtract `_ :c:func:`HPy_InPlaceSubtract` + `PyNumber_InPlaceTrueDivide `_ :c:func:`HPy_InPlaceTrueDivide` + `PyNumber_InPlaceXor `_ :c:func:`HPy_InPlaceXor` + `PyNumber_Index `_ :c:func:`HPy_Index` + `PyNumber_Invert `_ :c:func:`HPy_Invert` + `PyNumber_Long `_ :c:func:`HPy_Long` + `PyNumber_Lshift `_ :c:func:`HPy_Lshift` + `PyNumber_MatrixMultiply `_ :c:func:`HPy_MatrixMultiply` + `PyNumber_Multiply `_ :c:func:`HPy_Multiply` + `PyNumber_Negative `_ :c:func:`HPy_Negative` + `PyNumber_Or `_ :c:func:`HPy_Or` + `PyNumber_Positive `_ :c:func:`HPy_Positive` + `PyNumber_Power `_ :c:func:`HPy_Power` + `PyNumber_Remainder `_ :c:func:`HPy_Remainder` + `PyNumber_Rshift `_ :c:func:`HPy_Rshift` + `PyNumber_Subtract `_ :c:func:`HPy_Subtract` + `PyNumber_TrueDivide `_ :c:func:`HPy_TrueDivide` + `PyNumber_Xor `_ :c:func:`HPy_Xor` + `PyObject_ASCII `_ :c:func:`HPy_ASCII` + `PyObject_Bytes `_ :c:func:`HPy_Bytes` + `PyObject_Call `_ :c:func:`HPy_CallTupleDict` + `PyObject_DelItem `_ :c:func:`HPy_DelItem` + `PyObject_GetAttr `_ :c:func:`HPy_GetAttr` + `PyObject_GetAttrString `_ :c:func:`HPy_GetAttr_s` + `PyObject_GetItem `_ :c:func:`HPy_GetItem` + `PyObject_GetIter `_ :c:func:`HPy_GetIter` + `PyObject_HasAttr `_ :c:func:`HPy_HasAttr` + `PyObject_HasAttrString `_ :c:func:`HPy_HasAttr_s` + `PyObject_Hash `_ :c:func:`HPy_Hash` + `PyObject_IsTrue `_ :c:func:`HPy_IsTrue` + `PyObject_Length `_ :c:func:`HPy_Length` + `PyObject_Repr `_ :c:func:`HPy_Repr` + `PyObject_RichCompare `_ :c:func:`HPy_RichCompare` + `PyObject_RichCompareBool `_ :c:func:`HPy_RichCompareBool` + `PyObject_SetAttr `_ :c:func:`HPy_SetAttr` + `PyObject_SetAttrString `_ :c:func:`HPy_SetAttr_s` + `PyObject_SetItem `_ :c:func:`HPy_SetItem` + `PyObject_Str `_ :c:func:`HPy_Str` + `PyObject_Type `_ :c:func:`HPy_Type` + `PyObject_TypeCheck `_ :c:func:`HPy_TypeCheck` + `PyObject_Vectorcall `_ :c:func:`HPy_Call` + `PyObject_VectorcallMethod `_ :c:func:`HPy_CallMethod` + `PySequence_Contains `_ :c:func:`HPy_Contains` + `PySequence_DelSlice `_ :c:func:`HPy_DelSlice` + `PySequence_GetSlice `_ :c:func:`HPy_GetSlice` + `PySequence_SetSlice `_ :c:func:`HPy_SetSlice` + `PySlice_AdjustIndices `_ :c:func:`HPySlice_AdjustIndices` + `PySlice_New `_ :c:func:`HPySlice_New` + `PySlice_Unpack `_ :c:func:`HPySlice_Unpack` + `PyTuple_Check `_ :c:func:`HPyTuple_Check` + `PyType_IsSubtype `_ :c:func:`HPyType_IsSubtype` + `PyUnicode_AsASCIIString `_ :c:func:`HPyUnicode_AsASCIIString` + `PyUnicode_AsLatin1String `_ :c:func:`HPyUnicode_AsLatin1String` + `PyUnicode_AsUTF8AndSize `_ :c:func:`HPyUnicode_AsUTF8AndSize` + `PyUnicode_AsUTF8String `_ :c:func:`HPyUnicode_AsUTF8String` + `PyUnicode_Check `_ :c:func:`HPyUnicode_Check` + `PyUnicode_DecodeASCII `_ :c:func:`HPyUnicode_DecodeASCII` + `PyUnicode_DecodeFSDefault `_ :c:func:`HPyUnicode_DecodeFSDefault` + `PyUnicode_DecodeFSDefaultAndSize `_ :c:func:`HPyUnicode_DecodeFSDefaultAndSize` + `PyUnicode_DecodeLatin1 `_ :c:func:`HPyUnicode_DecodeLatin1` + `PyUnicode_EncodeFSDefault `_ :c:func:`HPyUnicode_EncodeFSDefault` + `PyUnicode_FromEncodedObject `_ :c:func:`HPyUnicode_FromEncodedObject` + `PyUnicode_FromString `_ :c:func:`HPyUnicode_FromString` + `PyUnicode_FromWideChar `_ :c:func:`HPyUnicode_FromWideChar` + `PyUnicode_ReadChar `_ :c:func:`HPyUnicode_ReadChar` + `PyUnicode_Substring `_ :c:func:`HPyUnicode_Substring` + `Py_FatalError `_ :c:func:`HPy_FatalError` + ================================================================================================================================== ================================================ +.. mark: END API MAPPING + + +.. note: There are, of course, also cases where it is not possible to map directly and safely from a C API function (or concept) to an HPy API function (or concept). + +Reference Counting ``Py_INCREF`` and ``Py_DECREF`` +-------------------------------------------------- + +The equivalents of ``Py_INCREF`` and ``Py_DECREF`` are essentially +:c:func:`HPy_Dup` and :c:func:`HPy_Close`, respectively. The main difference is +that :c:func:`HPy_Dup` gives you a *new handle* to the same object which means +that the two handles may be different if comparing them with ``memcmp`` but +still reference the same object. As a consequence, you may close a handle only +once, i.e., you cannot call :c:func:`HPy_Close` twice on the same ``HPy`` +handle, even if returned from ``HPy_Dup``. For examples, see also sections +:ref:`api:handles` and :ref:`api:handles vs ``pyobject *``` + +Calling functions ``PyObject_Call`` and ``PyObject_CallObject`` +--------------------------------------------------------------- + +Both ``PyObject_Call`` and ``PyObject_CallObject`` are replaced by +``HPy_CallTupleDict(callable, args, kwargs)`` in which either or both of +``args`` and ``kwargs`` may be null handles. + +``PyObject_Call(callable, args, kwargs)`` becomes:: + + HPy result = HPy_CallTupleDict(ctx, callable, args, kwargs); + +``PyObject_CallObject(callable, args)`` becomes:: + + HPy result = HPy_CallTupleDict(ctx, callable, args, HPy_NULL); + +If ``args`` is not a handle to a tuple or ``kwargs`` is not a handle to a +dictionary, ``HPy_CallTupleDict`` will return ``HPy_NULL`` and raise a +``TypeError``. This is different to ``PyObject_Call`` and +``PyObject_CallObject`` which may segfault instead. + +Calling Protocol +---------------- + +Both the *tp_call* and *vectorcall* calling protocols are replaced by HPy's +calling protocol. This is done by defining slot ``HPy_tp_call``. HPy uses only +one calling convention which is similar to the vectorcall calling convention. +In the following example, we implement a call function for a simple Euclidean +vector type. The function computes the dot product of two vectors. + +.. literalinclude:: examples/snippets/hpycall.c + :start-after: // BEGIN EuclideanVectorObject + :end-before: // END EuclideanVectorObject + +.. literalinclude:: examples/snippets/hpycall.c + :start-after: // BEGIN HPy_tp_call + :end-before: // END HPy_tp_call + +Positional and keyword arguments are passed as C array ``args``. Argument +``nargs`` specifies the number of positional arguments. Argument ``kwnames`` is +a tuple containing the names of the keyword arguments. The keyword argument +values are appended to positional arguments and start at ``args[nargs]`` (if +there are any). + +In the above example, function ``call_impl`` will be used by default to call all +instances of the corresponding type. It is also possible to install (maybe +specialized) call function implementations per instances by using function +:c:func:`HPy_SetCallFunction`. This needs to be done in the constructor of an +object. For example: + +.. literalinclude:: examples/snippets/hpycall.c + :start-after: // BEGIN HPy_SetCallFunction + :end-before: // END HPy_SetCallFunction + +Limitations +~~~~~~~~~~~ + + 1. It is not possible to use slot ``HPy_tp_call`` for a *var object* (i.e. if + :c:member:`HPyType_Spec.itemsize` is greater ``0``). Reason: HPy installs + a hidden field in the object's data to store the call function pointer + which is appended to everything else. In case of ``EuclideanVectorObject``, + a field is implicitly appended after member ``y``. This is not possible for + var objects because the variable part will also start after the fixed + members. + + 2. It is also not possible to use slot ``HPy_tp_call`` with a legacy type that + inherits the basicsize (i.e. if :c:member:`HPyType_Spec.basicsize` is + ``0``) for the same reason as above. + +To overcome these limitations, it is still possible to manually embed a field +for the call function pointer in a type's C struct and tell HPy where this field +is. In this case, it is always necessary to set the call function pointer using +:c:func:`HPy_SetCallFunction` in the object's constructor. This procedure is +less convenient than just using slot ``HPy_tp_cal`` but still not hard to use. +Consider following example. We define a struct ``FooObject`` and declare field +``HPyCallFunction call_func`` which will be used to store the call function's +pointer. We need to register the offset of that field with member +``__vectorcalloffset__`` and in the constructor ``Foo_new``, we assign the call +function ``Foo_call_func``. + +.. literalinclude:: examples/snippets/hpycall.c + :start-after: // BEGIN FooObject + :end-before: // END FooObject + +.. literalinclude:: examples/snippets/hpycall.c + :start-after: // BEGIN vectorcalloffset + :end-before: // END vectorcalloffset + +.. note:: + + In contrast to CPython's vectorcall protocol, ``nargs`` will never have flag + ``PY_VECTORCALL_ARGUMENTS_OFFSET`` set. It will **only** be the positional + argument count. + +.. _call-migration: + +Incremental Migration to HPy's Calling Protocol +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In order to support incremental migration, HPy provides helper function +:c:func:`HPyHelpers_PackArgsAndKeywords` that converts from HPy's calling +convention to CPython's *tp_call* calling convention. Consider following +example: + +.. literalinclude:: examples/snippets/hpycall.c + :start-after: // BEGIN pack_args + :end-before: // END pack_args + +In this example, ``args``, ``nargs``, and ``kwnames`` are used to create a tuple +of positional arguments ``args_tuple`` and a keyword arguments dictionary +``kwd``. + +PyModule_AddObject +------------------ + +``PyModule_AddObject`` is replaced with a regular :c:func:`HPy_SetAttr_s`. There +is no ``HPyModule_AddObject`` function because it has an unusual refcount +behavior (stealing a reference but only when it returns ``0``). + + +.. _dealloc: + +Deallocator slot ``Py_tp_dealloc`` +---------------------------------- + +``Py_tp_dealloc`` essentially becomes ``HPy_tp_destroy``. The name intentionally +differs because there are major differences: while the slot function of +``Py_tp_dealloc`` receives the full object (which makes it possible to resurrect +it) and while there are no restrictions on what you may call in the C API +deallocator, you must not do that in HPy's deallocator. + +The two major restrictions apply to the slot function of ``HPy_tp_destroy``: + +1. The function must be **thread-safe**. +2. The function **must not** call into the interpreter. + +The idea is, that ``HPy_tp_destroy`` just releases native resources (e.g. by +using C lib's ``free`` function). Therefore, it only receives a pointer to the +object's native data (and not a handle to the object) and it does not receive an +``HPyContext`` pointer argument. + +For the time being, HPy will support the ``HPy_tp_finalize`` slot where those +tight restrictions do not apply at the (significant) cost of performance. + +Special slots ``Py_tp_methods``, ``Py_tp_members``, and ``Py_tp_getset`` +------------------------------------------------------------------------ + +There is no direct replacement for C API slots ``Py_tp_methods``, +``Py_tp_members``, and ``Py_tp_getset`` because they are no longer needed. +Methods, members, and get/set descriptors are specified *flatly* together with +the other slots, using the standard mechanisms of :c:macro:`HPyDef_METH`, +:c:macro:`HPyDef_MEMBER`, and :c:macro:`HPyDef_GETSET`. The resulting ``HPyDef`` +structures are then accumulated in :c:member:`HPyType_Spec.defines`. + +Creating lists and tuples +------------------------- + +The C API way of creating lists and tuples is to create an empty list or tuple +object using ``PyList_New(n)`` or ``PyTuple_New(n)``, respectively, and then to +fill the empty object using ``PyList_SetItem / PyList_SET_ITEM`` or +``PyTuple_SetItem / PyTuple_SET_ITEM``, respectively. + +This is in particular problematic for tuples because they are actually +immutable. HPy goes a different way and provides a dedicated *builder* API to +avoid the (temporary) inconsistent state during object initialization. + +Long story short, doing the same in HPy with builders is still very simple and +straight forward. Following an example for creating a list: + +.. code-block:: c + + PyObject *list = PyList_New(5); + if (list == NULL) + return NULL; /* error */ + PyList_SET_ITEM(list, 0, item0); + PyList_SET_ITEM(list, 1, item0); + ... + PyList_SET_ITEM(list, 4, item0); + /* now 'list' is ready to use */ + +becomes + +.. code-block:: c + + HPyListBuilder builder = HPyListBuilder_New(ctx, 5); + HPyListBuilder_Set(ctx, builder, 0, h_item0); + HPyListBuilder_Set(ctx, builder, 1, h_item1); + ... + HPyListBuilder_Set(ctx, builder, 4, h_item4); + HPy h_list = HPyListBuilder_Build(ctx, builder); + if (HPy_IsNull(h_list)) + return HPy_NULL; /* error */ + +.. note:: In contrast to ``PyList_SetItem``, ``PyList_SET_ITEM``, + ``PyTuple_SetItem``, and ``PyTuple_SET_ITEM``, the builder functions + :c:func:`HPyListBuilder_Set` and :c:func:`HPyTupleBuilder_Set` are **NOT** + stealing references. It is necessary to close the passed item handles (e.g. + ``h_item0`` in the above example) if they are no longer needed. + +If an error occurs during building the list or tuple, it is necessary to call +:c:func:`HPyListBuilder_Cancel` or :c:func:`HPyTupleBuilder_Cancel`, +respectively, to avoid memory leaks. + +For details, see the API reference documentation +:doc:`api-reference/hpy-sequence`. + +Buffers +------- + +The buffer API in HPy is implemented using the ``HPy_buffer`` struct, which looks +very similar to ``Py_buffer`` (refer to the `CPython documentation +`_ for the +meaning of the fields):: + + typedef struct { + void *buf; + HPy obj; + HPy_ssize_t len; + HPy_ssize_t itemsize; + int readonly; + int ndim; + char *format; + HPy_ssize_t *shape; + HPy_ssize_t *strides; + HPy_ssize_t *suboffsets; + void *internal; + } HPy_buffer; + +Buffer slots for HPy types are specified using slots ``HPy_bf_getbuffer`` and +``HPy_bf_releasebuffer`` on all supported Python versions, even though the +matching PyType_Spec slots, ``Py_bf_getbuffer`` and ``Py_bf_releasebuffer``, are +only available starting from CPython 3.9. + +Multi-phase Module Initialization +--------------------------------- + +HPy supports only multi-phase module initialization (PEP 451). This means that +the module object is typically created by interpreter from the ``HPyModuleDef`` +specification and there is no "init" function. However, the module can define +one or more ``HPy_mod_exec`` slots, which will be executed just after the module +object is created. Inside the code of those slots, one can usually perform the same +initialization as before. + +Example of legacy single phase module initialization that uses Python/C API: + +.. literalinclude:: examples/snippets/legacyinit.c + :start-after: // BEGIN + :end-before: // END + +The same code structure ported to HPy and multi-phase module initialization: + +.. literalinclude:: examples/snippets/hpyinit.c + :start-after: // BEGIN + :end-before: // END diff --git a/graalpython/hpy/docs/quickstart.rst b/graalpython/hpy/docs/quickstart.rst new file mode 100644 index 0000000000..f57063cfa8 --- /dev/null +++ b/graalpython/hpy/docs/quickstart.rst @@ -0,0 +1,55 @@ +HPy Quickstart +============== + +This section shows how to quickly get started with HPy by creating +a simple HPy extension from scratch. + +Install latest HPy release: + +.. code-block:: console + + python3 -m pip install hpy + +Alternatively, you can also install HPy from the development repository: + +.. code-block:: console + + python3 -m pip install git+https://github.com/hpyproject/hpy.git#egg=hpy + +Create a new directory for the new HPy extension. Location and name of the directory +do not matter. Add the following two files: + +.. literalinclude:: examples/quickstart/quickstart.c + +.. literalinclude:: examples/quickstart/setup.py + :language: python + +Build the extension: + +.. code-block:: console + + python3 setup.py --hpy-abi=universal develop + +Try it out -- start Python console in the same directory and type: + +.. literalinclude:: examples/tests.py + :start-after: test_quickstart + :end-before: # END: test_quickstart + +Notice the shared library that was created by running ``setup.py``: + +.. code-block:: console + + > ls *.so + quickstart.hpy0.so + +It does not have Python version encoded in it. If you happen to have +GraalPy or PyPy installation that supports given HPy version, you can +try running the same extension on it. For example, start +``$GRAALVM_HOME/bin/graalpy`` in the same directory and type the same +Python code: the extension should load and work just fine. + +Where to go next? + + - :ref:`Simple documented HPy extension example` + - :doc:`Tutorial: porting Python/C API extension to HPy` diff --git a/graalpython/hpy/docs/requirements.txt b/graalpython/hpy/docs/requirements.txt new file mode 100644 index 0000000000..bb561e010f --- /dev/null +++ b/graalpython/hpy/docs/requirements.txt @@ -0,0 +1,8 @@ +# Requirements for building the documentation +Jinja2==3.1.3 +sphinx==5.0.2 +sphinx-rtd-theme==2.0.0 +sphinx-autobuild==2021.3.14 +sphinx-c-autodoc==1.3.0 +clang==17.0.6 +docutils==0.16 # docutils >= 0.17 fails to render bullet lists :/ diff --git a/graalpython/hpy/docs/trace-mode.rst b/graalpython/hpy/docs/trace-mode.rst new file mode 100644 index 0000000000..b8763b4220 --- /dev/null +++ b/graalpython/hpy/docs/trace-mode.rst @@ -0,0 +1,64 @@ +Trace Mode +========== + +HPy's trace mode allows you to analyze the usage of the HPy API. The two +fundamental metrics are ``call count`` and ``duration``. As the name already +suggests, ``call count`` tells you how often a certain HPy API function was called +and ``duration`` uses a monotonic clock to measure how much (accumulated) time was +spent in a certain HPy API function. It is further possible to register custom +*on-enter* and *on-exit* Python functions. + +As with the debug mode, the trace mode can be activated at *import time*, so no +recompilation is required. + + +Activating Trace Mode +--------------------- + +Similar to how the +:ref:`debug mode is activated `, use +environment variable ``HPY``. If ``HPY=trace``, then all HPy modules are loaded +with the trace context. Alternatively, it is also possible to specify the mode +per module like this: ``HPY=modA:trace,modB:trace``. +Environment variable ``HPY_LOG`` also works. + + +Using Trace Mode +---------------- + +The trace mode can be accessed via the shipped module ``hpy.trace``. It provides +following functions: + +* ``get_call_counts()`` returns a dict. The HPy API function names are used as + keys and the corresponding call count is the value. +* ``get_durations()`` also returns a dict similar to ``get_call_counts`` but + the value is the accumulated time spent in the corresponding HPy API + function (in nanoseconds). Note, the used clock does not necessarily have a + nanosecond resolution which means that the least significant digits may not be + accurate. +* ``set_trace_functions(on_enter=None, on_exit=None)`` allows the user to + register custom trace functions. The function provided for ``on_enter`` and + ``on_exit`` functions will be executed before and after and HPy API function + is and was executed, respectively. Passing ``None`` to any of the two + arguments or omitting one will clear the corresponding function. +* ``get_frequency()`` returns the resolution of the used clock to measure the + time in Hertz. For example, a value of ``10000000`` corresponds to + ``10 MHz``. In that case, the two least significant digits of the durations + are inaccurate. + + +Example +------- + +Following HPy function uses ``HPy_Add``: + +.. literalinclude:: examples/snippets/snippets.c + :start-after: // BEGIN: add + :end-before: // END: add + +When this script is executed in trace mode: + +.. literalinclude:: examples/trace-example.py + :language: python + +The output is ``get_call_counts()["ctx_Add"] == 1``. diff --git a/graalpython/hpy/docs/xxx-unsorted-notes.txt b/graalpython/hpy/docs/xxx-unsorted-notes.txt new file mode 100644 index 0000000000..159b6e315c --- /dev/null +++ b/graalpython/hpy/docs/xxx-unsorted-notes.txt @@ -0,0 +1,91 @@ +Goal: better C API for CPython and 1st-class citizen on PyPy + +- everything should be opaque by default + + - should we have an API to "query" if an object supports a certain low-level layout? E.g list-of-integer + + - should we have an API to e.g. ask "give me the nth C-level long in the + list? And then rely on the compiler to turn this into an efficient loop: + long PyObject_GetLongItem(PyHandle h, long index) ? + + +- we need PyObject_GetItem(handle) and PyObject_GetItem(int_index) + +- PyHandle PyObject_GetMappingProtocol(PyHandle o): ask a python object if it + has the "mapping" interface; then you can close it when you are done with it + and e.g. PyPy can release when it's no longer needed + +- should we write a tool to convert from the old API to the new API? + +- how we do deploy it? Should we have a single PyHandle.h file which is enough + to include? Or do like numpy.get_include_dirs()? + + - we need to do what cffi does (and ship our version on PyPy) + + - cython might need/want to ship its own vendored version of PyHandle + +- we need a versioning system which is possible to query at runtime? (to check + that it was compiled with the "correct/expected" PyHandle version + +- what to do with existing code which actively check whether the refcount is 1? E.g. PyString_Resize? + +- fast c-to-c calls: should we use argument clinic or something similar? + + + +Protocol sketch +---------------- + +HPySequence_long x = HPy_AsSequence_long(obj); /* it is possible to fail and you should be ready to handle the fallback */ +int len = HPy_Sequence_Len_long(x, obj); +for(int i=0; i sequence index access +* comparisons (<, <=, ==, !=, >=, >) + * rich comparisons + * boolean comparisons (0/1) +* call (call/vectorcall/method/special-method) + * e.g. PyCall_SpecialMethodOneArg(obj, __add__, arg) -> call specific macro + * … +* context manager (enter/exit) + * -> call special method +* async (await/aiter/anext) + * -> call special method diff --git a/graalpython/hpy/gdb-py.test b/graalpython/hpy/gdb-py.test new file mode 100755 index 0000000000..7412a06214 --- /dev/null +++ b/graalpython/hpy/gdb-py.test @@ -0,0 +1,3 @@ +#!/bin/bash + +gdb --eval-command run --args python3 -m pytest -s "$@" diff --git a/graalpython/lib-graalpython/modules/hpy/debug/__init__.py b/graalpython/hpy/hpy/debug/__init__.py similarity index 100% rename from graalpython/lib-graalpython/modules/hpy/debug/__init__.py rename to graalpython/hpy/hpy/debug/__init__.py diff --git a/graalpython/lib-graalpython/modules/hpy/debug/leakdetector.py b/graalpython/hpy/hpy/debug/leakdetector.py similarity index 100% rename from graalpython/lib-graalpython/modules/hpy/debug/leakdetector.py rename to graalpython/hpy/hpy/debug/leakdetector.py diff --git a/graalpython/lib-graalpython/modules/hpy/debug/pytest.py b/graalpython/hpy/hpy/debug/pytest.py similarity index 100% rename from graalpython/lib-graalpython/modules/hpy/debug/pytest.py rename to graalpython/hpy/hpy/debug/pytest.py diff --git a/graalpython/com.oracle.graal.python.jni/src/debug/_debugmod.c b/graalpython/hpy/hpy/debug/src/_debugmod.c similarity index 100% rename from graalpython/com.oracle.graal.python.jni/src/debug/_debugmod.c rename to graalpython/hpy/hpy/debug/src/_debugmod.c diff --git a/graalpython/hpy/hpy/debug/src/autogen_debug_ctx_call.i b/graalpython/hpy/hpy/debug/src/autogen_debug_ctx_call.i new file mode 100644 index 0000000000..0cd23a98aa --- /dev/null +++ b/graalpython/hpy/hpy/debug/src/autogen_debug_ctx_call.i @@ -0,0 +1,468 @@ + +/* + DO NOT EDIT THIS FILE! + + This file is automatically generated by hpy.tools.autogen.debug.autogen_debug_ctx_call_i + See also hpy.tools.autogen and hpy/tools/public_api.h + + Run this to regenerate: + make autogen + +*/ + + case HPyFunc_NOARGS: { + HPyFunc_noargs f = (HPyFunc_noargs)func; + _HPyFunc_args_NOARGS *a = (_HPyFunc_args_NOARGS*)args; + DHPy dh_self = _py2dh(dctx, a->self); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = NULL; + return; + } + DHPy dh_result = f(next_dctx, dh_self); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_self); + a->result = _dh2py(dctx, dh_result); + DHPy_close(dctx, dh_result); + return; + } + case HPyFunc_O: { + HPyFunc_o f = (HPyFunc_o)func; + _HPyFunc_args_O *a = (_HPyFunc_args_O*)args; + DHPy dh_self = _py2dh(dctx, a->self); + DHPy dh_arg = _py2dh(dctx, a->arg); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = NULL; + return; + } + DHPy dh_result = f(next_dctx, dh_self, dh_arg); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_self); + DHPy_close_and_check(dctx, dh_arg); + a->result = _dh2py(dctx, dh_result); + DHPy_close(dctx, dh_result); + return; + } + case HPyFunc_UNARYFUNC: { + HPyFunc_unaryfunc f = (HPyFunc_unaryfunc)func; + _HPyFunc_args_UNARYFUNC *a = (_HPyFunc_args_UNARYFUNC*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = NULL; + return; + } + DHPy dh_result = f(next_dctx, dh_arg0); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + a->result = _dh2py(dctx, dh_result); + DHPy_close(dctx, dh_result); + return; + } + case HPyFunc_BINARYFUNC: { + HPyFunc_binaryfunc f = (HPyFunc_binaryfunc)func; + _HPyFunc_args_BINARYFUNC *a = (_HPyFunc_args_BINARYFUNC*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + DHPy dh_arg1 = _py2dh(dctx, a->arg1); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = NULL; + return; + } + DHPy dh_result = f(next_dctx, dh_arg0, dh_arg1); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + DHPy_close_and_check(dctx, dh_arg1); + a->result = _dh2py(dctx, dh_result); + DHPy_close(dctx, dh_result); + return; + } + case HPyFunc_TERNARYFUNC: { + HPyFunc_ternaryfunc f = (HPyFunc_ternaryfunc)func; + _HPyFunc_args_TERNARYFUNC *a = (_HPyFunc_args_TERNARYFUNC*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + DHPy dh_arg1 = _py2dh(dctx, a->arg1); + DHPy dh_arg2 = _py2dh(dctx, a->arg2); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = NULL; + return; + } + DHPy dh_result = f(next_dctx, dh_arg0, dh_arg1, dh_arg2); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + DHPy_close_and_check(dctx, dh_arg1); + DHPy_close_and_check(dctx, dh_arg2); + a->result = _dh2py(dctx, dh_result); + DHPy_close(dctx, dh_result); + return; + } + case HPyFunc_INQUIRY: { + HPyFunc_inquiry f = (HPyFunc_inquiry)func; + _HPyFunc_args_INQUIRY *a = (_HPyFunc_args_INQUIRY*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = -1; + return; + } + a->result = f(next_dctx, dh_arg0); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + return; + } + case HPyFunc_LENFUNC: { + HPyFunc_lenfunc f = (HPyFunc_lenfunc)func; + _HPyFunc_args_LENFUNC *a = (_HPyFunc_args_LENFUNC*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = -1; + return; + } + a->result = f(next_dctx, dh_arg0); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + return; + } + case HPyFunc_SSIZEARGFUNC: { + HPyFunc_ssizeargfunc f = (HPyFunc_ssizeargfunc)func; + _HPyFunc_args_SSIZEARGFUNC *a = (_HPyFunc_args_SSIZEARGFUNC*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = NULL; + return; + } + DHPy dh_result = f(next_dctx, dh_arg0, a->arg1); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + a->result = _dh2py(dctx, dh_result); + DHPy_close(dctx, dh_result); + return; + } + case HPyFunc_SSIZESSIZEARGFUNC: { + HPyFunc_ssizessizeargfunc f = (HPyFunc_ssizessizeargfunc)func; + _HPyFunc_args_SSIZESSIZEARGFUNC *a = (_HPyFunc_args_SSIZESSIZEARGFUNC*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = NULL; + return; + } + DHPy dh_result = f(next_dctx, dh_arg0, a->arg1, a->arg2); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + a->result = _dh2py(dctx, dh_result); + DHPy_close(dctx, dh_result); + return; + } + case HPyFunc_SSIZEOBJARGPROC: { + HPyFunc_ssizeobjargproc f = (HPyFunc_ssizeobjargproc)func; + _HPyFunc_args_SSIZEOBJARGPROC *a = (_HPyFunc_args_SSIZEOBJARGPROC*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + DHPy dh_arg2 = _py2dh(dctx, a->arg2); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = -1; + return; + } + a->result = f(next_dctx, dh_arg0, a->arg1, dh_arg2); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + DHPy_close_and_check(dctx, dh_arg2); + return; + } + case HPyFunc_SSIZESSIZEOBJARGPROC: { + HPyFunc_ssizessizeobjargproc f = (HPyFunc_ssizessizeobjargproc)func; + _HPyFunc_args_SSIZESSIZEOBJARGPROC *a = (_HPyFunc_args_SSIZESSIZEOBJARGPROC*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + DHPy dh_arg3 = _py2dh(dctx, a->arg3); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = -1; + return; + } + a->result = f(next_dctx, dh_arg0, a->arg1, a->arg2, dh_arg3); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + DHPy_close_and_check(dctx, dh_arg3); + return; + } + case HPyFunc_OBJOBJARGPROC: { + HPyFunc_objobjargproc f = (HPyFunc_objobjargproc)func; + _HPyFunc_args_OBJOBJARGPROC *a = (_HPyFunc_args_OBJOBJARGPROC*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + DHPy dh_arg1 = _py2dh(dctx, a->arg1); + DHPy dh_arg2 = _py2dh(dctx, a->arg2); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = -1; + return; + } + a->result = f(next_dctx, dh_arg0, dh_arg1, dh_arg2); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + DHPy_close_and_check(dctx, dh_arg1); + DHPy_close_and_check(dctx, dh_arg2); + return; + } + case HPyFunc_FREEFUNC: { + HPyFunc_freefunc f = (HPyFunc_freefunc)func; + _HPyFunc_args_FREEFUNC *a = (_HPyFunc_args_FREEFUNC*)args; + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + return; + } + f(next_dctx, a->arg0); + _switch_back_to_original_dctx(dctx, next_dctx); + return; + } + case HPyFunc_GETATTRFUNC: { + HPyFunc_getattrfunc f = (HPyFunc_getattrfunc)func; + _HPyFunc_args_GETATTRFUNC *a = (_HPyFunc_args_GETATTRFUNC*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = NULL; + return; + } + DHPy dh_result = f(next_dctx, dh_arg0, a->arg1); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + a->result = _dh2py(dctx, dh_result); + DHPy_close(dctx, dh_result); + return; + } + case HPyFunc_GETATTROFUNC: { + HPyFunc_getattrofunc f = (HPyFunc_getattrofunc)func; + _HPyFunc_args_GETATTROFUNC *a = (_HPyFunc_args_GETATTROFUNC*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + DHPy dh_arg1 = _py2dh(dctx, a->arg1); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = NULL; + return; + } + DHPy dh_result = f(next_dctx, dh_arg0, dh_arg1); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + DHPy_close_and_check(dctx, dh_arg1); + a->result = _dh2py(dctx, dh_result); + DHPy_close(dctx, dh_result); + return; + } + case HPyFunc_SETATTRFUNC: { + HPyFunc_setattrfunc f = (HPyFunc_setattrfunc)func; + _HPyFunc_args_SETATTRFUNC *a = (_HPyFunc_args_SETATTRFUNC*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + DHPy dh_arg2 = _py2dh(dctx, a->arg2); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = -1; + return; + } + a->result = f(next_dctx, dh_arg0, a->arg1, dh_arg2); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + DHPy_close_and_check(dctx, dh_arg2); + return; + } + case HPyFunc_SETATTROFUNC: { + HPyFunc_setattrofunc f = (HPyFunc_setattrofunc)func; + _HPyFunc_args_SETATTROFUNC *a = (_HPyFunc_args_SETATTROFUNC*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + DHPy dh_arg1 = _py2dh(dctx, a->arg1); + DHPy dh_arg2 = _py2dh(dctx, a->arg2); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = -1; + return; + } + a->result = f(next_dctx, dh_arg0, dh_arg1, dh_arg2); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + DHPy_close_and_check(dctx, dh_arg1); + DHPy_close_and_check(dctx, dh_arg2); + return; + } + case HPyFunc_REPRFUNC: { + HPyFunc_reprfunc f = (HPyFunc_reprfunc)func; + _HPyFunc_args_REPRFUNC *a = (_HPyFunc_args_REPRFUNC*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = NULL; + return; + } + DHPy dh_result = f(next_dctx, dh_arg0); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + a->result = _dh2py(dctx, dh_result); + DHPy_close(dctx, dh_result); + return; + } + case HPyFunc_HASHFUNC: { + HPyFunc_hashfunc f = (HPyFunc_hashfunc)func; + _HPyFunc_args_HASHFUNC *a = (_HPyFunc_args_HASHFUNC*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = -1; + return; + } + a->result = f(next_dctx, dh_arg0); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + return; + } + case HPyFunc_RICHCMPFUNC: { + HPyFunc_richcmpfunc f = (HPyFunc_richcmpfunc)func; + _HPyFunc_args_RICHCMPFUNC *a = (_HPyFunc_args_RICHCMPFUNC*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + DHPy dh_arg1 = _py2dh(dctx, a->arg1); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = NULL; + return; + } + DHPy dh_result = f(next_dctx, dh_arg0, dh_arg1, a->arg2); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + DHPy_close_and_check(dctx, dh_arg1); + a->result = _dh2py(dctx, dh_result); + DHPy_close(dctx, dh_result); + return; + } + case HPyFunc_GETITERFUNC: { + HPyFunc_getiterfunc f = (HPyFunc_getiterfunc)func; + _HPyFunc_args_GETITERFUNC *a = (_HPyFunc_args_GETITERFUNC*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = NULL; + return; + } + DHPy dh_result = f(next_dctx, dh_arg0); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + a->result = _dh2py(dctx, dh_result); + DHPy_close(dctx, dh_result); + return; + } + case HPyFunc_ITERNEXTFUNC: { + HPyFunc_iternextfunc f = (HPyFunc_iternextfunc)func; + _HPyFunc_args_ITERNEXTFUNC *a = (_HPyFunc_args_ITERNEXTFUNC*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = NULL; + return; + } + DHPy dh_result = f(next_dctx, dh_arg0); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + a->result = _dh2py(dctx, dh_result); + DHPy_close(dctx, dh_result); + return; + } + case HPyFunc_DESCRGETFUNC: { + HPyFunc_descrgetfunc f = (HPyFunc_descrgetfunc)func; + _HPyFunc_args_DESCRGETFUNC *a = (_HPyFunc_args_DESCRGETFUNC*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + DHPy dh_arg1 = _py2dh(dctx, a->arg1); + DHPy dh_arg2 = _py2dh(dctx, a->arg2); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = NULL; + return; + } + DHPy dh_result = f(next_dctx, dh_arg0, dh_arg1, dh_arg2); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + DHPy_close_and_check(dctx, dh_arg1); + DHPy_close_and_check(dctx, dh_arg2); + a->result = _dh2py(dctx, dh_result); + DHPy_close(dctx, dh_result); + return; + } + case HPyFunc_DESCRSETFUNC: { + HPyFunc_descrsetfunc f = (HPyFunc_descrsetfunc)func; + _HPyFunc_args_DESCRSETFUNC *a = (_HPyFunc_args_DESCRSETFUNC*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + DHPy dh_arg1 = _py2dh(dctx, a->arg1); + DHPy dh_arg2 = _py2dh(dctx, a->arg2); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = -1; + return; + } + a->result = f(next_dctx, dh_arg0, dh_arg1, dh_arg2); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + DHPy_close_and_check(dctx, dh_arg1); + DHPy_close_and_check(dctx, dh_arg2); + return; + } + case HPyFunc_GETTER: { + HPyFunc_getter f = (HPyFunc_getter)func; + _HPyFunc_args_GETTER *a = (_HPyFunc_args_GETTER*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = NULL; + return; + } + DHPy dh_result = f(next_dctx, dh_arg0, a->arg1); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + a->result = _dh2py(dctx, dh_result); + DHPy_close(dctx, dh_result); + return; + } + case HPyFunc_SETTER: { + HPyFunc_setter f = (HPyFunc_setter)func; + _HPyFunc_args_SETTER *a = (_HPyFunc_args_SETTER*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + DHPy dh_arg1 = _py2dh(dctx, a->arg1); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = -1; + return; + } + a->result = f(next_dctx, dh_arg0, dh_arg1, a->arg2); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + DHPy_close_and_check(dctx, dh_arg1); + return; + } + case HPyFunc_OBJOBJPROC: { + HPyFunc_objobjproc f = (HPyFunc_objobjproc)func; + _HPyFunc_args_OBJOBJPROC *a = (_HPyFunc_args_OBJOBJPROC*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + DHPy dh_arg1 = _py2dh(dctx, a->arg1); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = -1; + return; + } + a->result = f(next_dctx, dh_arg0, dh_arg1); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + DHPy_close_and_check(dctx, dh_arg1); + return; + } + case HPyFunc_DESTRUCTOR: { + HPyFunc_destructor f = (HPyFunc_destructor)func; + _HPyFunc_args_DESTRUCTOR *a = (_HPyFunc_args_DESTRUCTOR*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + return; + } + f(next_dctx, dh_arg0); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + return; + } diff --git a/graalpython/com.oracle.graal.python.jni/src/debug/autogen_debug_ctx_init.h b/graalpython/hpy/hpy/debug/src/autogen_debug_ctx_init.h similarity index 100% rename from graalpython/com.oracle.graal.python.jni/src/debug/autogen_debug_ctx_init.h rename to graalpython/hpy/hpy/debug/src/autogen_debug_ctx_init.h diff --git a/graalpython/com.oracle.graal.python.jni/src/debug/autogen_debug_wrappers.c b/graalpython/hpy/hpy/debug/src/autogen_debug_wrappers.c similarity index 100% rename from graalpython/com.oracle.graal.python.jni/src/debug/autogen_debug_wrappers.c rename to graalpython/hpy/hpy/debug/src/autogen_debug_wrappers.c diff --git a/graalpython/com.oracle.graal.python.jni/src/debug/debug_ctx.c b/graalpython/hpy/hpy/debug/src/debug_ctx.c similarity index 100% rename from graalpython/com.oracle.graal.python.jni/src/debug/debug_ctx.c rename to graalpython/hpy/hpy/debug/src/debug_ctx.c diff --git a/graalpython/hpy/hpy/debug/src/debug_ctx_cpython.c b/graalpython/hpy/hpy/debug/src/debug_ctx_cpython.c new file mode 100644 index 0000000000..01d9c64788 --- /dev/null +++ b/graalpython/hpy/hpy/debug/src/debug_ctx_cpython.c @@ -0,0 +1,299 @@ +/* =========== CPython-ONLY =========== + In the following code, we use _py2h and _h2py and we assumed they are the + ones defined by CPython's version of hpy.universal. + + DO NOT COMPILE THIS FILE UNLESS YOU ARE BUILDING CPython's hpy.universal. + + If you want to compile the debug mode into your own non-CPython version of + hpy.universal, you should include debug_ctx_not_cpython.c. + ==================================== + + In theory, the debug mode is completely generic and can wrap a generic + uctx. However, CPython is special because it does not have native support + for HPy, so uctx contains the logic to call HPy functions from CPython, by + using _HPy_CallRealFunctionFromTrampoline. + + uctx->ctx_CallRealFunctionFromTrampoline converts PyObject* into UHPy. So + for the debug mode we need to: + + 1. convert the PyObject* args into UHPys. + 2. wrap the UHPys into DHPys. + 3. unwrap the resulting DHPy and convert to PyObject*. +*/ + +#include +#include "debug_internal.h" +#include "hpy/runtime/ctx_type.h" // for call_traverseproc_from_trampoline +#include "hpy/runtime/ctx_module.h" +#include "handles.h" // for _py2h and _h2py + +#if defined(_MSC_VER) +# include /* for alloca() */ +#endif + +static inline DHPy _py2dh(HPyContext *dctx, PyObject *obj) +{ + return DHPy_open(dctx, _py2h(obj)); +} + +static inline PyObject *_dh2py(HPyContext *dctx, DHPy dh) +{ + return _h2py(DHPy_unwrap(dctx, dh)); +} + +static void _buffer_h2py(HPyContext *dctx, const HPy_buffer *src, Py_buffer *dest) +{ + dest->buf = src->buf; + dest->obj = HPy_AsPyObject(dctx, src->obj); + dest->len = src->len; + dest->itemsize = src->itemsize; + dest->readonly = src->readonly; + dest->ndim = src->ndim; + dest->format = src->format; + dest->shape = src->shape; + dest->strides = src->strides; + dest->suboffsets = src->suboffsets; + dest->internal = src->internal; +} + +static void _buffer_py2h(HPyContext *dctx, const Py_buffer *src, HPy_buffer *dest) +{ + dest->buf = src->buf; + dest->obj = HPy_FromPyObject(dctx, src->obj); + dest->len = src->len; + dest->itemsize = src->itemsize; + dest->readonly = src->readonly; + dest->ndim = src->ndim; + dest->format = src->format; + dest->shape = src->shape; + dest->strides = src->strides; + dest->suboffsets = src->suboffsets; + dest->internal = src->internal; +} + +static HPyContext* _switch_to_next_dctx_from_cache(HPyContext *current_dctx) { + HPyContext *next_dctx = hpy_debug_get_next_dctx_from_cache(current_dctx); + if (next_dctx == NULL) { + HPyErr_NoMemory(current_dctx); + get_ctx_info(current_dctx)->is_valid = false; + get_ctx_info(next_dctx)->is_valid = true; + return NULL; + } + get_ctx_info(current_dctx)->is_valid = false; + get_ctx_info(next_dctx)->is_valid = true; + return next_dctx; +} + +static void _switch_back_to_original_dctx(HPyContext *original_dctx, HPyContext *next_dctx) { + get_ctx_info(next_dctx)->is_valid = false; + get_ctx_info(original_dctx)->is_valid = true; +} + +void debug_ctx_CallRealFunctionFromTrampoline(HPyContext *dctx, + HPyFunc_Signature sig, + void *func, void *args) +{ + switch (sig) { + case HPyFunc_VARARGS: { + HPyFunc_varargs f = (HPyFunc_varargs)func; + _HPyFunc_args_VARARGS *a = (_HPyFunc_args_VARARGS*)args; + DHPy dh_self = _py2dh(dctx, a->self); + DHPy *dh_args = (DHPy *)alloca(a->nargs * sizeof(DHPy)); + for (HPy_ssize_t i = 0; i < a->nargs; i++) { + dh_args[i] = _py2dh(dctx, a->args[i]); + } + + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = NULL; + return; + } + + DHPy dh_result = f(next_dctx, dh_self, dh_args, a->nargs); + + _switch_back_to_original_dctx(dctx, next_dctx); + + DHPy_close_and_check(dctx, dh_self); + for (HPy_ssize_t i = 0; i < a->nargs; i++) { + DHPy_close_and_check(dctx, dh_args[i]); + } + a->result = _dh2py(dctx, dh_result); + DHPy_close(dctx, dh_result); + return; + } + case HPyFunc_KEYWORDS: { + HPyFunc_keywords f = (HPyFunc_keywords)func; + _HPyFunc_args_KEYWORDS *a = (_HPyFunc_args_KEYWORDS*)args; + DHPy dh_self = _py2dh(dctx, a->self); + size_t n_kwnames = a->kwnames != NULL ? PyTuple_GET_SIZE(a->kwnames) : 0; + size_t nargs = PyVectorcall_NARGS(a->nargsf); + size_t nargs_with_kw = nargs + n_kwnames; + DHPy *dh_args = (DHPy *)alloca(nargs_with_kw * sizeof(DHPy)); + for (size_t i = 0; i < nargs_with_kw; i++) { + dh_args[i] = _py2dh(dctx, a->args[i]); + } + DHPy dh_kwnames = _py2dh(dctx, a->kwnames); + + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = NULL; + return; + } + + DHPy dh_result = f(next_dctx, dh_self, dh_args, nargs, dh_kwnames); + + _switch_back_to_original_dctx(dctx, next_dctx); + + DHPy_close_and_check(dctx, dh_self); + for (size_t i = 0; i < nargs_with_kw; i++) { + DHPy_close_and_check(dctx, dh_args[i]); + } + DHPy_close_and_check(dctx, dh_kwnames); + a->result = _dh2py(dctx, dh_result); + DHPy_close(dctx, dh_result); + return; + } + case HPyFunc_INITPROC: { + HPyFunc_initproc f = (HPyFunc_initproc)func; + _HPyFunc_args_INITPROC *a = (_HPyFunc_args_INITPROC*)args; + DHPy dh_self = _py2dh(dctx, a->self); + Py_ssize_t nargs = PyTuple_GET_SIZE(a->args); + DHPy *dh_args = (DHPy *)alloca(nargs * sizeof(DHPy)); + for (Py_ssize_t i = 0; i < nargs; i++) { + dh_args[i] = _py2dh(dctx, PyTuple_GET_ITEM(a->args, i)); + } + DHPy dh_kw = _py2dh(dctx, a->kw); + + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = -1; + return; + } + + a->result = f(next_dctx, dh_self, dh_args, nargs, dh_kw); + + _switch_back_to_original_dctx(dctx, next_dctx); + + DHPy_close_and_check(dctx, dh_self); + for (Py_ssize_t i = 0; i < nargs; i++) { + DHPy_close_and_check(dctx, dh_args[i]); + } + DHPy_close_and_check(dctx, dh_kw); + return; + } + case HPyFunc_NEWFUNC: { + HPyFunc_newfunc f = (HPyFunc_newfunc)func; + _HPyFunc_args_NEWFUNC *a = (_HPyFunc_args_NEWFUNC*)args; + DHPy dh_self = _py2dh(dctx, a->self); + Py_ssize_t nargs = PyTuple_GET_SIZE(a->args); + DHPy *dh_args = (DHPy *)alloca(nargs * sizeof(DHPy)); + for (Py_ssize_t i = 0; i < nargs; i++) { + dh_args[i] = _py2dh(dctx, PyTuple_GET_ITEM(a->args, i)); + } + DHPy dh_kw = _py2dh(dctx, a->kw); + + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = NULL; + return; + } + + DHPy dh_result = f(next_dctx, dh_self, dh_args, nargs, dh_kw); + + _switch_back_to_original_dctx(dctx, next_dctx); + + DHPy_close_and_check(dctx, dh_self); + for (Py_ssize_t i = 0; i < nargs; i++) { + DHPy_close_and_check(dctx, dh_args[i]); + } + DHPy_close_and_check(dctx, dh_kw); + a->result = _dh2py(dctx, dh_result); + DHPy_close(dctx, dh_result); + return; + } + case HPyFunc_GETBUFFERPROC: { + HPyFunc_getbufferproc f = (HPyFunc_getbufferproc)func; + _HPyFunc_args_GETBUFFERPROC *a = (_HPyFunc_args_GETBUFFERPROC*)args; + HPy_buffer hbuf; + DHPy dh_self = _py2dh(dctx, a->self); + + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = -1; + return; + } + + a->result = f(next_dctx, dh_self, &hbuf, a->flags); + + _switch_back_to_original_dctx(dctx, next_dctx); + + DHPy_close_and_check(dctx, dh_self); + if (a->result < 0) { + a->view->obj = NULL; + return; + } + _buffer_h2py(dctx, &hbuf, a->view); + HPy_Close(dctx, hbuf.obj); + return; + } + case HPyFunc_RELEASEBUFFERPROC: { + HPyFunc_releasebufferproc f = (HPyFunc_releasebufferproc)func; + _HPyFunc_args_RELEASEBUFFERPROC *a = (_HPyFunc_args_RELEASEBUFFERPROC*)args; + HPy_buffer hbuf; + _buffer_py2h(dctx, a->view, &hbuf); + DHPy dh_self = _py2dh(dctx, a->self); + + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + return; + } + + f(next_dctx, dh_self, &hbuf); + + _switch_back_to_original_dctx(dctx, next_dctx); + + DHPy_close_and_check(dctx, dh_self); + // XXX: copy back from hbuf? + HPy_Close(dctx, hbuf.obj); + return; + } + case HPyFunc_TRAVERSEPROC: { + HPyFunc_traverseproc f = (HPyFunc_traverseproc)func; + _HPyFunc_args_TRAVERSEPROC *a = (_HPyFunc_args_TRAVERSEPROC*)args; + + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = -1; + return; + } + + a->result = call_traverseproc_from_trampoline(f, a->self, + a->visit, a->arg); + + _switch_back_to_original_dctx(dctx, next_dctx); + return; + } + case HPyFunc_CAPSULE_DESTRUCTOR: { + HPyFunc_Capsule_Destructor f = (HPyFunc_Capsule_Destructor)func; + PyObject *capsule = (PyObject *)args; + const char *name = PyCapsule_GetName(capsule); + f(name, PyCapsule_GetPointer(capsule, name), + PyCapsule_GetContext(capsule)); + return; + } + case HPyFunc_MOD_CREATE: { + HPyFunc_unaryfunc f = (HPyFunc_unaryfunc)func; + _HPyFunc_args_UNARYFUNC *a = (_HPyFunc_args_UNARYFUNC*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + DHPy dh_result = f(dctx, dh_arg0); + DHPy_close_and_check(dctx, dh_arg0); + a->result = _dh2py(dctx, dh_result); + _HPyModule_CheckCreateSlotResult(&a->result); + DHPy_close(dctx, dh_result); + return; + } +#include "autogen_debug_ctx_call.i" + default: + Py_FatalError("Unsupported HPyFunc_Signature in debug_ctx_cpython.c"); + } +} diff --git a/graalpython/com.oracle.graal.python.jni/src/debug/debug_ctx_not_cpython.c b/graalpython/hpy/hpy/debug/src/debug_ctx_not_cpython.c similarity index 100% rename from graalpython/com.oracle.graal.python.jni/src/debug/debug_ctx_not_cpython.c rename to graalpython/hpy/hpy/debug/src/debug_ctx_not_cpython.c diff --git a/graalpython/com.oracle.graal.python.jni/src/debug/debug_handles.c b/graalpython/hpy/hpy/debug/src/debug_handles.c similarity index 100% rename from graalpython/com.oracle.graal.python.jni/src/debug/debug_handles.c rename to graalpython/hpy/hpy/debug/src/debug_handles.c diff --git a/graalpython/com.oracle.graal.python.jni/src/debug/debug_internal.h b/graalpython/hpy/hpy/debug/src/debug_internal.h similarity index 100% rename from graalpython/com.oracle.graal.python.jni/src/debug/debug_internal.h rename to graalpython/hpy/hpy/debug/src/debug_internal.h diff --git a/graalpython/com.oracle.graal.python.jni/src/debug/dhqueue.c b/graalpython/hpy/hpy/debug/src/dhqueue.c similarity index 100% rename from graalpython/com.oracle.graal.python.jni/src/debug/dhqueue.c rename to graalpython/hpy/hpy/debug/src/dhqueue.c diff --git a/graalpython/com.oracle.graal.python.jni/src/debug/hpy_debug.h b/graalpython/hpy/hpy/debug/src/include/hpy_debug.h similarity index 100% rename from graalpython/com.oracle.graal.python.jni/src/debug/hpy_debug.h rename to graalpython/hpy/hpy/debug/src/include/hpy_debug.h diff --git a/graalpython/com.oracle.graal.python.jni/src/debug/memprotect.c b/graalpython/hpy/hpy/debug/src/memprotect.c similarity index 100% rename from graalpython/com.oracle.graal.python.jni/src/debug/memprotect.c rename to graalpython/hpy/hpy/debug/src/memprotect.c diff --git a/graalpython/com.oracle.graal.python.jni/src/debug/stacktrace.c b/graalpython/hpy/hpy/debug/src/stacktrace.c similarity index 100% rename from graalpython/com.oracle.graal.python.jni/src/debug/stacktrace.c rename to graalpython/hpy/hpy/debug/src/stacktrace.c diff --git a/graalpython/lib-graalpython/modules/hpy/devel/__init__.py b/graalpython/hpy/hpy/devel/__init__.py similarity index 100% rename from graalpython/lib-graalpython/modules/hpy/devel/__init__.py rename to graalpython/hpy/hpy/devel/__init__.py diff --git a/graalpython/lib-graalpython/modules/hpy/devel/abitag.py b/graalpython/hpy/hpy/devel/abitag.py similarity index 100% rename from graalpython/lib-graalpython/modules/hpy/devel/abitag.py rename to graalpython/hpy/hpy/devel/abitag.py diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy.h b/graalpython/hpy/hpy/devel/include/hpy.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy.h rename to graalpython/hpy/hpy/devel/include/hpy.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/autogen_hpyfunc_declare.h b/graalpython/hpy/hpy/devel/include/hpy/autogen_hpyfunc_declare.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/autogen_hpyfunc_declare.h rename to graalpython/hpy/hpy/devel/include/hpy/autogen_hpyfunc_declare.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/autogen_hpyslot.h b/graalpython/hpy/hpy/devel/include/hpy/autogen_hpyslot.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/autogen_hpyslot.h rename to graalpython/hpy/hpy/devel/include/hpy/autogen_hpyslot.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/cpy_types.h b/graalpython/hpy/hpy/devel/include/hpy/cpy_types.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/cpy_types.h rename to graalpython/hpy/hpy/devel/include/hpy/cpy_types.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/cpython/autogen_api_impl.h b/graalpython/hpy/hpy/devel/include/hpy/cpython/autogen_api_impl.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/cpython/autogen_api_impl.h rename to graalpython/hpy/hpy/devel/include/hpy/cpython/autogen_api_impl.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/cpython/autogen_ctx.h b/graalpython/hpy/hpy/devel/include/hpy/cpython/autogen_ctx.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/cpython/autogen_ctx.h rename to graalpython/hpy/hpy/devel/include/hpy/cpython/autogen_ctx.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/cpython/autogen_hpyfunc_trampolines.h b/graalpython/hpy/hpy/devel/include/hpy/cpython/autogen_hpyfunc_trampolines.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/cpython/autogen_hpyfunc_trampolines.h rename to graalpython/hpy/hpy/devel/include/hpy/cpython/autogen_hpyfunc_trampolines.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/cpython/hpyfunc_trampolines.h b/graalpython/hpy/hpy/devel/include/hpy/cpython/hpyfunc_trampolines.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/cpython/hpyfunc_trampolines.h rename to graalpython/hpy/hpy/devel/include/hpy/cpython/hpyfunc_trampolines.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/cpython/misc.h b/graalpython/hpy/hpy/devel/include/hpy/cpython/misc.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/cpython/misc.h rename to graalpython/hpy/hpy/devel/include/hpy/cpython/misc.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/forbid_python_h/Python.h b/graalpython/hpy/hpy/devel/include/hpy/forbid_python_h/Python.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/forbid_python_h/Python.h rename to graalpython/hpy/hpy/devel/include/hpy/forbid_python_h/Python.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/hpydef.h b/graalpython/hpy/hpy/devel/include/hpy/hpydef.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/hpydef.h rename to graalpython/hpy/hpy/devel/include/hpy/hpydef.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/hpyexports.h b/graalpython/hpy/hpy/devel/include/hpy/hpyexports.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/hpyexports.h rename to graalpython/hpy/hpy/devel/include/hpy/hpyexports.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/hpyfunc.h b/graalpython/hpy/hpy/devel/include/hpy/hpyfunc.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/hpyfunc.h rename to graalpython/hpy/hpy/devel/include/hpy/hpyfunc.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/hpymodule.h b/graalpython/hpy/hpy/devel/include/hpy/hpymodule.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/hpymodule.h rename to graalpython/hpy/hpy/devel/include/hpy/hpymodule.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/hpytype.h b/graalpython/hpy/hpy/devel/include/hpy/hpytype.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/hpytype.h rename to graalpython/hpy/hpy/devel/include/hpy/hpytype.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/inline_helpers.h b/graalpython/hpy/hpy/devel/include/hpy/inline_helpers.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/inline_helpers.h rename to graalpython/hpy/hpy/devel/include/hpy/inline_helpers.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/macros.h b/graalpython/hpy/hpy/devel/include/hpy/macros.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/macros.h rename to graalpython/hpy/hpy/devel/include/hpy/macros.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/argparse.h b/graalpython/hpy/hpy/devel/include/hpy/runtime/argparse.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/argparse.h rename to graalpython/hpy/hpy/devel/include/hpy/runtime/argparse.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/buildvalue.h b/graalpython/hpy/hpy/devel/include/hpy/runtime/buildvalue.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/buildvalue.h rename to graalpython/hpy/hpy/devel/include/hpy/runtime/buildvalue.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/ctx_funcs.h b/graalpython/hpy/hpy/devel/include/hpy/runtime/ctx_funcs.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/ctx_funcs.h rename to graalpython/hpy/hpy/devel/include/hpy/runtime/ctx_funcs.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/ctx_module.h b/graalpython/hpy/hpy/devel/include/hpy/runtime/ctx_module.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/ctx_module.h rename to graalpython/hpy/hpy/devel/include/hpy/runtime/ctx_module.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/ctx_type.h b/graalpython/hpy/hpy/devel/include/hpy/runtime/ctx_type.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/ctx_type.h rename to graalpython/hpy/hpy/devel/include/hpy/runtime/ctx_type.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/format.h b/graalpython/hpy/hpy/devel/include/hpy/runtime/format.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/format.h rename to graalpython/hpy/hpy/devel/include/hpy/runtime/format.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/helpers.h b/graalpython/hpy/hpy/devel/include/hpy/runtime/helpers.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/helpers.h rename to graalpython/hpy/hpy/devel/include/hpy/runtime/helpers.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/structseq.h b/graalpython/hpy/hpy/devel/include/hpy/runtime/structseq.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/structseq.h rename to graalpython/hpy/hpy/devel/include/hpy/runtime/structseq.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/universal/autogen_ctx.h b/graalpython/hpy/hpy/devel/include/hpy/universal/autogen_ctx.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/universal/autogen_ctx.h rename to graalpython/hpy/hpy/devel/include/hpy/universal/autogen_ctx.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/universal/autogen_hpyfunc_trampolines.h b/graalpython/hpy/hpy/devel/include/hpy/universal/autogen_hpyfunc_trampolines.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/universal/autogen_hpyfunc_trampolines.h rename to graalpython/hpy/hpy/devel/include/hpy/universal/autogen_hpyfunc_trampolines.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/universal/autogen_trampolines.h b/graalpython/hpy/hpy/devel/include/hpy/universal/autogen_trampolines.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/universal/autogen_trampolines.h rename to graalpython/hpy/hpy/devel/include/hpy/universal/autogen_trampolines.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/universal/hpyfunc_trampolines.h b/graalpython/hpy/hpy/devel/include/hpy/universal/hpyfunc_trampolines.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/universal/hpyfunc_trampolines.h rename to graalpython/hpy/hpy/devel/include/hpy/universal/hpyfunc_trampolines.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/universal/misc_trampolines.h b/graalpython/hpy/hpy/devel/include/hpy/universal/misc_trampolines.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/universal/misc_trampolines.h rename to graalpython/hpy/hpy/devel/include/hpy/universal/misc_trampolines.h diff --git a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/argparse.c b/graalpython/hpy/hpy/devel/src/runtime/argparse.c similarity index 100% rename from graalpython/lib-graalpython/modules/hpy/devel/src/runtime/argparse.c rename to graalpython/hpy/hpy/devel/src/runtime/argparse.c diff --git a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/buildvalue.c b/graalpython/hpy/hpy/devel/src/runtime/buildvalue.c similarity index 100% rename from graalpython/lib-graalpython/modules/hpy/devel/src/runtime/buildvalue.c rename to graalpython/hpy/hpy/devel/src/runtime/buildvalue.c diff --git a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_bytes.c b/graalpython/hpy/hpy/devel/src/runtime/ctx_bytes.c similarity index 100% rename from graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_bytes.c rename to graalpython/hpy/hpy/devel/src/runtime/ctx_bytes.c diff --git a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_call.c b/graalpython/hpy/hpy/devel/src/runtime/ctx_call.c similarity index 100% rename from graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_call.c rename to graalpython/hpy/hpy/devel/src/runtime/ctx_call.c diff --git a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_capsule.c b/graalpython/hpy/hpy/devel/src/runtime/ctx_capsule.c similarity index 100% rename from graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_capsule.c rename to graalpython/hpy/hpy/devel/src/runtime/ctx_capsule.c diff --git a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_contextvar.c b/graalpython/hpy/hpy/devel/src/runtime/ctx_contextvar.c similarity index 100% rename from graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_contextvar.c rename to graalpython/hpy/hpy/devel/src/runtime/ctx_contextvar.c diff --git a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_err.c b/graalpython/hpy/hpy/devel/src/runtime/ctx_err.c similarity index 100% rename from graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_err.c rename to graalpython/hpy/hpy/devel/src/runtime/ctx_err.c diff --git a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_eval.c b/graalpython/hpy/hpy/devel/src/runtime/ctx_eval.c similarity index 100% rename from graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_eval.c rename to graalpython/hpy/hpy/devel/src/runtime/ctx_eval.c diff --git a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_listbuilder.c b/graalpython/hpy/hpy/devel/src/runtime/ctx_listbuilder.c similarity index 100% rename from graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_listbuilder.c rename to graalpython/hpy/hpy/devel/src/runtime/ctx_listbuilder.c diff --git a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_long.c b/graalpython/hpy/hpy/devel/src/runtime/ctx_long.c similarity index 100% rename from graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_long.c rename to graalpython/hpy/hpy/devel/src/runtime/ctx_long.c diff --git a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_module.c b/graalpython/hpy/hpy/devel/src/runtime/ctx_module.c similarity index 100% rename from graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_module.c rename to graalpython/hpy/hpy/devel/src/runtime/ctx_module.c diff --git a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_object.c b/graalpython/hpy/hpy/devel/src/runtime/ctx_object.c similarity index 100% rename from graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_object.c rename to graalpython/hpy/hpy/devel/src/runtime/ctx_object.c diff --git a/graalpython/com.oracle.graal.python.jni/src/ctx_tracker.c b/graalpython/hpy/hpy/devel/src/runtime/ctx_tracker.c similarity index 100% rename from graalpython/com.oracle.graal.python.jni/src/ctx_tracker.c rename to graalpython/hpy/hpy/devel/src/runtime/ctx_tracker.c diff --git a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_tuple.c b/graalpython/hpy/hpy/devel/src/runtime/ctx_tuple.c similarity index 100% rename from graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_tuple.c rename to graalpython/hpy/hpy/devel/src/runtime/ctx_tuple.c diff --git a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_tuplebuilder.c b/graalpython/hpy/hpy/devel/src/runtime/ctx_tuplebuilder.c similarity index 100% rename from graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_tuplebuilder.c rename to graalpython/hpy/hpy/devel/src/runtime/ctx_tuplebuilder.c diff --git a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_type.c b/graalpython/hpy/hpy/devel/src/runtime/ctx_type.c similarity index 100% rename from graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_type.c rename to graalpython/hpy/hpy/devel/src/runtime/ctx_type.c diff --git a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/format.c b/graalpython/hpy/hpy/devel/src/runtime/format.c similarity index 100% rename from graalpython/lib-graalpython/modules/hpy/devel/src/runtime/format.c rename to graalpython/hpy/hpy/devel/src/runtime/format.c diff --git a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/helpers.c b/graalpython/hpy/hpy/devel/src/runtime/helpers.c similarity index 100% rename from graalpython/lib-graalpython/modules/hpy/devel/src/runtime/helpers.c rename to graalpython/hpy/hpy/devel/src/runtime/helpers.c diff --git a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/structseq.c b/graalpython/hpy/hpy/devel/src/runtime/structseq.c similarity index 100% rename from graalpython/lib-graalpython/modules/hpy/devel/src/runtime/structseq.c rename to graalpython/hpy/hpy/devel/src/runtime/structseq.c diff --git a/graalpython/hpy/hpy/tools/autogen/__init__.py b/graalpython/hpy/hpy/tools/autogen/__init__.py new file mode 100644 index 0000000000..81eb61e746 --- /dev/null +++ b/graalpython/hpy/hpy/tools/autogen/__init__.py @@ -0,0 +1,21 @@ +import pycparser +from packaging import version + +if version.parse(pycparser.__version__) < version.parse('2.21'): + raise ImportError('You need pycparser>=2.21 to run autogen') + +from .parse import HPyAPI, AUTOGEN_H + + +def generate(generators, *gen_args): + """ + Takes a sequence of autogen generators that will have access to the parse + tree of 'public_api.c' and can then generate files or whatever. + :param generators: A sequence (e.g. tuple) of classes or callables that + will produce objects with a 'write' method. The 'gen_args' will be passed + to the 'write' method on invocation. + :param gen_args: Arguments for the autogen generator instances. + """ + api = HPyAPI.parse(AUTOGEN_H) + for cls in generators: + cls(api).write(*gen_args) diff --git a/graalpython/hpy/hpy/tools/autogen/__main__.py b/graalpython/hpy/hpy/tools/autogen/__main__.py new file mode 100644 index 0000000000..59cdf04872 --- /dev/null +++ b/graalpython/hpy/hpy/tools/autogen/__main__.py @@ -0,0 +1,66 @@ +""" +Parse public_api.h and generate various stubs around +""" +import sys +import py +import pycparser +from packaging import version + +if version.parse(pycparser.__version__) < version.parse('2.21'): + raise ImportError('You need pycparser>=2.21 to run autogen') + +from . import generate +from .ctx import (autogen_ctx_h, + autogen_ctx_def_h, + cpython_autogen_ctx_h) +from .trampolines import (autogen_trampolines_h, + cpython_autogen_api_impl_h, + universal_autogen_ctx_impl_h) +from .hpyfunc import autogen_hpyfunc_declare_h +from .hpyfunc import autogen_hpyfunc_trampoline_h +from .hpyfunc import autogen_ctx_call_i +from .hpyfunc import autogen_cpython_hpyfunc_trampoline_h +from .hpyslot import autogen_hpyslot_h +from .debug import (autogen_debug_ctx_init_h, + autogen_debug_wrappers, + autogen_debug_ctx_call_i) +from .trace import (autogen_tracer_ctx_init_h, + autogen_tracer_wrappers, + autogen_trace_func_table_c) +from .pypy import autogen_pypy_txt +from .doc import (autogen_function_index, + autogen_doc_api_mapping) + +DEFAULT_GENERATORS = (autogen_ctx_h, + autogen_ctx_def_h, + cpython_autogen_ctx_h, + autogen_trampolines_h, + cpython_autogen_api_impl_h, + universal_autogen_ctx_impl_h, + autogen_hpyfunc_declare_h, + autogen_hpyfunc_trampoline_h, + autogen_ctx_call_i, + autogen_cpython_hpyfunc_trampoline_h, + autogen_hpyslot_h, + autogen_debug_ctx_init_h, + autogen_debug_wrappers, + autogen_debug_ctx_call_i, + autogen_tracer_ctx_init_h, + autogen_tracer_wrappers, + autogen_trace_func_table_c, + autogen_pypy_txt, + autogen_function_index, + autogen_doc_api_mapping) + + +def main(): + if len(sys.argv) != 2: + print('Usage: python -m hpy.tools.autogen OUTDIR') + sys.exit(1) + outdir = py.path.local(sys.argv[1]) + + generate(DEFAULT_GENERATORS, outdir) + + +if __name__ == '__main__': + main() diff --git a/graalpython/hpy/hpy/tools/autogen/autogen.h b/graalpython/hpy/hpy/tools/autogen/autogen.h new file mode 100644 index 0000000000..78bae31617 --- /dev/null +++ b/graalpython/hpy/hpy/tools/autogen/autogen.h @@ -0,0 +1,40 @@ +#define STRINGIFY(X) #X +#define HPy_ID(X) _Pragma(STRINGIFY(id=X)) \ + +#define AUTOGEN 1 + +// These are not real typedefs: they are there only to make pycparser happy +typedef int HPy; +typedef int HPyContext; +typedef int HPyModuleDef; +typedef int HPyType_Spec; +typedef int HPyType_SpecParam; +typedef int HPyCFunction; +typedef int HPy_ssize_t; +typedef int HPy_hash_t; +typedef int wchar_t; +typedef int size_t; +typedef int HPyFunc_Signature; +typedef int cpy_PyObject; +typedef int HPyField; +typedef int HPyGlobal; +typedef int HPyListBuilder; +typedef int HPyTupleBuilder; +typedef int HPyTracker; +typedef int HPy_RichCmpOp; +typedef int HPy_buffer; +typedef int HPyFunc_visitproc; +typedef int HPy_UCS4; +typedef int HPyThreadState; +typedef int HPyType_BuiltinShape; +typedef int _HPyCapsule_key; +typedef int HPyCapsule_Destructor; +typedef int int32_t; +typedef int uint32_t; +typedef int int64_t; +typedef int uint64_t; +typedef int bool; +typedef int HPy_SourceKind; +typedef int HPyCallFunction; + +#include "public_api.h" diff --git a/graalpython/hpy/hpy/tools/autogen/autogenfile.py b/graalpython/hpy/hpy/tools/autogen/autogenfile.py new file mode 100644 index 0000000000..d4fddb74c7 --- /dev/null +++ b/graalpython/hpy/hpy/tools/autogen/autogenfile.py @@ -0,0 +1,35 @@ +C_DISCLAIMER = """ +/* + DO NOT EDIT THIS FILE! + + This file is automatically generated by {clsname} + See also hpy.tools.autogen and hpy/tools/public_api.h + + Run this to regenerate: + make autogen + +*/ +""" + +class AutoGenFile: + LANGUAGE = 'C' + PATH = None + DISCLAIMER = None + + def __init__(self, api): + if self.DISCLAIMER is None and self.LANGUAGE == 'C': + self.DISCLAIMER = C_DISCLAIMER + self.api = api + + def generate(self): + raise NotImplementedError + + def write(self, root): + cls = self.__class__ + clsname = '%s.%s' % (cls.__module__, cls.__name__) + with root.join(self.PATH).open('w', encoding='utf-8') as f: + if self.DISCLAIMER is not None: + f.write(self.DISCLAIMER.format(clsname=clsname)) + f.write('\n') + f.write(self.generate()) + f.write('\n') diff --git a/graalpython/hpy/hpy/tools/autogen/conf.py b/graalpython/hpy/hpy/tools/autogen/conf.py new file mode 100644 index 0000000000..5ebf345783 --- /dev/null +++ b/graalpython/hpy/hpy/tools/autogen/conf.py @@ -0,0 +1,211 @@ +# Defines parameters for the code generation + +# Handwritten trampolines: +NO_TRAMPOLINES = { + '_HPy_New', + 'HPy_FatalError', +} + +# Generated trampoline returns given constant, +# but the context function is void +RETURN_CONSTANT = { + 'HPyErr_SetString': 'HPy_NULL', + 'HPyErr_SetObject': 'HPy_NULL', + 'HPyErr_SetFromErrnoWithFilenameObjects': 'HPy_NULL', + 'HPyErr_NoMemory': 'HPy_NULL' +} + +# If the HPy function delegates to C Python API of a different name or, in the +# case of None, if the HPy function is implemented by hand +SPECIAL_CASES = { + 'HPy_Dup': None, + 'HPy_Close': None, + 'HPyField_Load': None, + 'HPyField_Store': None, + 'HPyModule_Create': None, + 'HPy_GetAttr': 'PyObject_GetAttr', + 'HPy_GetAttr_s': 'PyObject_GetAttrString', + 'HPy_HasAttr': 'PyObject_HasAttr', + 'HPy_HasAttr_s': 'PyObject_HasAttrString', + 'HPy_SetAttr': 'PyObject_SetAttr', + 'HPy_SetAttr_s': 'PyObject_SetAttrString', + 'HPy_GetIter': 'PyObject_GetIter', + 'HPy_GetItem': 'PyObject_GetItem', + 'HPy_GetItem_i': None, + 'HPy_GetItem_s': None, + 'HPy_GetSlice': 'PySequence_GetSlice', + 'HPy_SetItem': 'PyObject_SetItem', + 'HPy_SetItem_i': None, + 'HPy_SetItem_s': None, + 'HPy_SetSlice': 'PySequence_SetSlice', + 'HPy_DelItem': 'PyObject_DelItem', + 'HPy_DelItem_i': None, + 'HPy_DelItem_s': None, + 'HPy_DelSlice': 'PySequence_DelSlice', + 'HPy_Contains': 'PySequence_Contains', + 'HPy_Length': 'PyObject_Length', + 'HPy_CallTupleDict': None, + 'HPy_Call': None, # 'PyObject_Vectorcall', no auto arg conversion + 'HPy_CallMethod': None, # 'PyObject_VectorcallMethod',no auto arg conversion + 'HPy_FromPyObject': None, + 'HPy_AsPyObject': None, + '_HPy_AsStruct_Object': None, + '_HPy_AsStruct_Type': None, + '_HPy_AsStruct_Long': None, + '_HPy_AsStruct_Float': None, + '_HPy_AsStruct_Unicode': None, + '_HPy_AsStruct_Tuple': None, + '_HPy_AsStruct_List': None, + '_HPy_AsStruct_Dict': None, + '_HPy_AsStruct_Legacy': None, + '_HPyType_GetBuiltinShape': None, + '_HPy_CallRealFunctionFromTrampoline': None, + '_HPy_CallDestroyAndThenDealloc': None, + 'HPyErr_Occurred': None, + 'HPy_FatalError': None, + 'HPy_Add': 'PyNumber_Add', + 'HPy_Subtract': 'PyNumber_Subtract', + 'HPy_Multiply': 'PyNumber_Multiply', + 'HPy_MatrixMultiply': 'PyNumber_MatrixMultiply', + 'HPy_FloorDivide': 'PyNumber_FloorDivide', + 'HPy_TrueDivide': 'PyNumber_TrueDivide', + 'HPy_Remainder': 'PyNumber_Remainder', + 'HPy_Divmod': 'PyNumber_Divmod', + 'HPy_Power': 'PyNumber_Power', + 'HPy_Negative': 'PyNumber_Negative', + 'HPy_Positive': 'PyNumber_Positive', + 'HPy_Absolute': 'PyNumber_Absolute', + 'HPy_Invert': 'PyNumber_Invert', + 'HPy_Lshift': 'PyNumber_Lshift', + 'HPy_Rshift': 'PyNumber_Rshift', + 'HPy_And': 'PyNumber_And', + 'HPy_Xor': 'PyNumber_Xor', + 'HPy_Or': 'PyNumber_Or', + 'HPy_Index': 'PyNumber_Index', + 'HPy_Long': 'PyNumber_Long', + 'HPy_Float': 'PyNumber_Float', + 'HPy_InPlaceAdd': 'PyNumber_InPlaceAdd', + 'HPy_InPlaceSubtract': 'PyNumber_InPlaceSubtract', + 'HPy_InPlaceMultiply': 'PyNumber_InPlaceMultiply', + 'HPy_InPlaceMatrixMultiply': 'PyNumber_InPlaceMatrixMultiply', + 'HPy_InPlaceFloorDivide': 'PyNumber_InPlaceFloorDivide', + 'HPy_InPlaceTrueDivide': 'PyNumber_InPlaceTrueDivide', + 'HPy_InPlaceRemainder': 'PyNumber_InPlaceRemainder', + 'HPy_InPlacePower': 'PyNumber_InPlacePower', + 'HPy_InPlaceLshift': 'PyNumber_InPlaceLshift', + 'HPy_InPlaceRshift': 'PyNumber_InPlaceRshift', + 'HPy_InPlaceAnd': 'PyNumber_InPlaceAnd', + 'HPy_InPlaceXor': 'PyNumber_InPlaceXor', + 'HPy_InPlaceOr': 'PyNumber_InPlaceOr', + '_HPy_New': None, + 'HPyType_FromSpec': None, + 'HPyType_GenericNew': None, + 'HPy_Repr': 'PyObject_Repr', + 'HPy_Str': 'PyObject_Str', + 'HPy_ASCII': 'PyObject_ASCII', + 'HPy_Bytes': 'PyObject_Bytes', + 'HPy_IsTrue': 'PyObject_IsTrue', + 'HPy_RichCompare': 'PyObject_RichCompare', + 'HPy_RichCompareBool': 'PyObject_RichCompareBool', + 'HPy_Hash': 'PyObject_Hash', + 'HPyIter_Next': 'PyIter_Next', + 'HPyIter_Check': 'PyIter_Check', + 'HPyListBuilder_New': None, + 'HPyListBuilder_Set': None, + 'HPyListBuilder_Build': None, + 'HPyListBuilder_Cancel': None, + 'HPyTuple_FromArray': None, + 'HPyTupleBuilder_New': None, + 'HPyTupleBuilder_Set': None, + 'HPyTupleBuilder_Build': None, + 'HPyTupleBuilder_Cancel': None, + 'HPyTracker_New': None, + 'HPyTracker_Add': None, + 'HPyTracker_ForgetAll': None, + 'HPyTracker_Close': None, + '_HPy_Dump': None, + 'HPy_Type': None, + 'HPy_TypeCheck': None, + 'HPy_Is': None, + 'HPyBytes_FromStringAndSize': None, + 'HPy_LeavePythonExecution': 'PyEval_SaveThread', + 'HPy_ReenterPythonExecution': 'PyEval_RestoreThread', + 'HPyGlobal_Load': None, + 'HPyGlobal_Store': None, + 'HPyCapsule_New': None, + 'HPyCapsule_Get': None, + 'HPyCapsule_Set': None, + 'HPyLong_FromInt32_t': None, + 'HPyLong_FromUInt32_t': None, + 'HPyLong_FromInt64_t': None, + 'HPyLong_FromUInt64_t': None, + 'HPyLong_AsInt32_t': None, + 'HPyLong_AsUInt32_t': None, + 'HPyLong_AsUInt32_tMask': None, + 'HPyLong_AsInt64_t': None, + 'HPyLong_AsUInt64_t': None, + 'HPyLong_AsUInt64_tMask': None, + 'HPyBool_FromBool': 'PyBool_FromLong', + 'HPy_Compile_s': None, + 'HPy_EvalCode': 'PyEval_EvalCode', + 'HPyContextVar_Get': None, + 'HPyType_GetName': None, + 'HPyType_IsSubtype': None, + 'HPy_SetCallFunction': None, +} + +################################################################################ +# Configuration for auto-generating docs # +################################################################################ + +# A manual mapping of between CPython C API functions and HPy API functions. +# Most of the mapping will be generated automatically from 'public_api.h' if an +# HPy API function is not a special case (see 'conf.py'). However, in some +# cases, it might be that we have inline helper functions or something similar +# that map to a CPython C API function which cannot be determined automatically. +# In those cases, the mapping can be manually specified here. Also, manual +# mapping will always take precedence over automatically derived mappings. +DOC_MANUAL_API_MAPPING = { + # key = C API function name + # value = HPy API function name + 'Py_FatalError': 'HPy_FatalError', + 'PyContextVar_Get': 'HPyContextVar_Get', + 'PyLong_FromLong': 'HPyLong_FromLong', + 'PyLong_FromLongLong': 'HPyLong_FromLongLong', + 'PyLong_FromUnsignedLong': 'HPyLong_FromUnsignedLong', + 'PyLong_FromUnsignedLongLong': 'HPyLong_FromUnsignedLongLong', + 'PyLong_AsLong': 'HPyLong_AsLong', + 'PyLong_AsLongLong': 'HPyLong_AsLongLong', + 'PyLong_AsUnsignedLong': 'HPyLong_AsUnsignedLong', + 'PyLong_AsUnsignedLongMask': 'HPyLong_AsUnsignedLongMask', + 'PyLong_AsUnsignedLongLong': 'HPyLong_AsUnsignedLongLong', + 'PyLong_AsUnsignedLongLongMask': 'HPyLong_AsUnsignedLongLongMask', + 'PyBool_FromLong': 'HPyBool_FromLong', + 'PyObject_TypeCheck': 'HPy_TypeCheck', + 'PySlice_AdjustIndices': 'HPySlice_AdjustIndices', + 'PyType_IsSubtype': 'HPyType_IsSubtype', + 'PyObject_Call': 'HPy_CallTupleDict', + 'PyObject_Type': 'HPy_Type', + 'PyObject_Vectorcall': 'HPy_Call', + 'PyObject_VectorcallMethod': 'HPy_CallMethod', +} + +# Some C API functions are documented in very different pages. +DOC_C_API_PAGES_SPECIAL_CASES = { + 'Py_FatalError': 'sys', + 'PyEval_SaveThread': 'init', + 'PyEval_RestoreThread': 'init', + 'PyEval_EvalCode': 'veryhigh', + 'PyObject_Call': 'call', + 'PyObject_Vectorcall': 'call', + 'PyObject_VectorcallMethod': 'call', +} + +# We assume that, e.g., prefix 'PyLong_Something' belongs to 'longobject.c' and +# its documentation is in '.../3/c-api/long.html'. In some cases, the prefix +# maps to a different page and this can be specified here. E.g. +# 'PyErr_Something' is documented in page '.../3/c-api/exceptions.html' +DOC_PREFIX_TABLE = { + 'err': 'exceptions', + 'contextvar': 'contextvars' +} \ No newline at end of file diff --git a/graalpython/hpy/hpy/tools/autogen/ctx.py b/graalpython/hpy/hpy/tools/autogen/ctx.py new file mode 100644 index 0000000000..4f569f1f35 --- /dev/null +++ b/graalpython/hpy/hpy/tools/autogen/ctx.py @@ -0,0 +1,106 @@ +from copy import deepcopy +from pycparser import c_ast +from .autogenfile import AutoGenFile +from .parse import toC, find_typedecl, maybe_make_void + + +class autogen_ctx_h(AutoGenFile): + PATH = 'hpy/devel/include/hpy/universal/autogen_ctx.h' + + ## struct _HPyContext_s { + ## const char *name; + ## void *_private; + ## int abi_version; + ## HPy h_None; + ## ... + ## HPy (*ctx_Module_Create)(HPyContext *ctx, HPyModuleDef *def); + ## ... + ## } + + def generate(self): + # Put all declarations (variables and functions) into one list in order + # to be able to sort them by their given context index. + # We need to remember the output function per item (either + # 'declare_var' or 'declare_func'). So, we create tuples with structure + # '(decl, declare_*)'. + all_decls = [(x, self.declare_var) for x in self.api.variables] + all_decls += [(x, self.declare_func) for x in self.api.functions] + + # sort the list of all declaration by 'decl.ctx_index' + all_decls.sort(key=lambda x: x[0].ctx_index) + + lines = [] + w = lines.append + w('struct _HPyContext_s {') + w(' const char *name; // used just to make debugging and testing easier') + w(' void *_private; // used by implementations to store custom data') + w(' int abi_version;') + for var, cons in all_decls: + w(' %s;' % cons(var)) + w('};') + return '\n'.join(lines) + + def declare_var(self, var): + return toC(var.node) + + def declare_func(self, func): + # e.g. "HPy (*ctx_Module_Create)(HPyContext *ctx, HPyModuleDef *def)" + # + # turn the function declaration into a function POINTER declaration + newnode = deepcopy(func.node) + newnode.type = c_ast.PtrDecl(type=newnode.type, quals=[]) + maybe_make_void(func, newnode) + + # fix the name of the function pointer + typedecl = find_typedecl(newnode) + typedecl.declname = func.ctx_name() + return toC(newnode) + + +class autogen_ctx_def_h(AutoGenFile): + PATH = 'hpy/universal/src/autogen_ctx_def.h' + + ## struct _HPyContext_s g_universal_ctx = { + ## .name = "...", + ## ._private = NULL, + ## .abi_version = HPY_ABI_VERSION, + ## .h_None = {CONSTANT_H_NONE}, + ## ... + ## .ctx_Module_Create = &ctx_Module_Create, + ## ... + ## } + + def generate(self): + lines = [] + w = lines.append + w('struct _HPyContext_s g_universal_ctx = {') + w(' .name = "HPy Universal ABI (CPython backend)",') + w(' ._private = NULL,') + w(' .abi_version = HPY_ABI_VERSION,') + w(' /* h_None & co. are initialized by init_universal_ctx() */') + for func in self.api.functions: + w(' .%s = &%s,' % (func.ctx_name(), func.ctx_name())) + w('};') + return '\n'.join(lines) + + +class cpython_autogen_ctx_h(AutoGenFile): + PATH = 'hpy/devel/include/hpy/cpython/autogen_ctx.h' + + def generate(self): + # Put all variable declarations into a list in order + # to be able to sort them by their given context index. + var_decls = list(self.api.variables) + + # sort the list of var declaration by 'decl.ctx_index' + var_decls.sort(key=lambda x: x.ctx_index) + + lines = [] + w = lines.append + w('struct _HPyContext_s {') + w(' const char *name;') + w(' int abi_version;') + for var in var_decls: + w(' %s;' % toC(var.node)) + w('};') + return '\n'.join(lines) diff --git a/graalpython/hpy/hpy/tools/autogen/debug.py b/graalpython/hpy/hpy/tools/autogen/debug.py new file mode 100644 index 0000000000..eb8783ba0a --- /dev/null +++ b/graalpython/hpy/hpy/tools/autogen/debug.py @@ -0,0 +1,228 @@ +from copy import deepcopy +from pycparser import c_ast +from .autogenfile import AutoGenFile +from .parse import toC, find_typedecl, get_context_return_type, \ + maybe_make_void, make_void, get_return_constant +from .hpyfunc import NO_CALL + + +class HPy_2_DHPy_Visitor(c_ast.NodeVisitor): + "Visitor which renames all HPy types to DHPy" + + def visit_IdentifierType(self, node): + if node.names == ['HPy']: + node.names = ['DHPy'] + + def visit_TypeDecl(self, node): + if node.declname == 'ctx': + node.declname = 'dctx' + self.generic_visit(node) + +def funcnode_with_new_name(node, name): + newnode = deepcopy(node) + typedecl = find_typedecl(newnode) + typedecl.declname = name + return newnode + +def get_debug_wrapper_node(func): + newnode = funcnode_with_new_name(func.node, 'debug_%s' % func.ctx_name()) + maybe_make_void(func, newnode) + # fix all the types + visitor = HPy_2_DHPy_Visitor() + visitor.visit(newnode) + return newnode + + +class autogen_debug_ctx_init_h(AutoGenFile): + PATH = 'hpy/debug/src/autogen_debug_ctx_init.h' + + def generate(self): + lines = [] + w = lines.append + # emit the declarations for all the debug_ctx_* functions + for func in self.api.functions: + w(toC(get_debug_wrapper_node(func)) + ';') + w('') + w('static inline void debug_ctx_init_fields(HPyContext *dctx, HPyContext *uctx)') + w('{') + for var in self.api.variables: + name = var.name + w(f' dctx->{name} = DHPy_open_immortal(dctx, uctx->{name});') + for func in self.api.functions: + name = func.ctx_name() + w(f' dctx->{name} = &debug_{name};') + + w('}') + return '\n'.join(lines) + + +class autogen_debug_wrappers(AutoGenFile): + PATH = 'hpy/debug/src/autogen_debug_wrappers.c' + + NO_WRAPPER = { + '_HPy_CallRealFunctionFromTrampoline', + 'HPy_Close', + 'HPyUnicode_AsUTF8AndSize', + 'HPyTuple_FromArray', + 'HPyType_GenericNew', + 'HPyType_FromSpec', + '_HPy_AsStruct_Legacy', + '_HPy_AsStruct_Object', + '_HPy_AsStruct_Type', + '_HPy_AsStruct_Long', + '_HPy_AsStruct_Float', + '_HPy_AsStruct_Unicode', + '_HPy_AsStruct_Tuple', + '_HPy_AsStruct_List', + '_HPy_AsStruct_Dict', + 'HPyTracker_New', + 'HPyTracker_Add', + 'HPyTracker_ForgetAll', + 'HPyTracker_Close', + 'HPyBytes_AsString', + 'HPyBytes_AS_STRING', + 'HPyTupleBuilder_New', + 'HPyTupleBuilder_Set', + 'HPyTupleBuilder_Build', + 'HPyTupleBuilder_Cancel', + 'HPyListBuilder_New', + 'HPyListBuilder_Set', + 'HPyListBuilder_Build', + 'HPyListBuilder_Cancel', + 'HPy_TypeCheck', + 'HPyContextVar_Get', + 'HPyType_GetName', + 'HPyType_IsSubtype', + 'HPyUnicode_Substring', + 'HPy_Call', + 'HPy_CallMethod', + } + + def generate(self): + lines = [] + w = lines.append + w('#include "debug_internal.h"') + w('') + for func in self.api.functions: + debug_wrapper = self.gen_debug_wrapper(func) + if debug_wrapper: + w(debug_wrapper) + w('') + return '\n'.join(lines) + + def gen_debug_wrapper(self, func): + if func.name in self.NO_WRAPPER: + return + # + assert not func.is_varargs() + node = get_debug_wrapper_node(func) + const_return = get_return_constant(func) + if const_return: + make_void(node) + signature = toC(node) + rettype = get_context_return_type(node, const_return) + # + def get_params_and_decls(): + lst = [] + decls = [] + for p in node.type.args.params: + if p.name == 'ctx': + lst.append('get_info(dctx)->uctx') + elif toC(p.type) == 'DHPy': + decls.append(' HPy dh_%s = DHPy_unwrap(dctx, %s);' % (p.name, p.name)) + lst.append('dh_%s' % p.name) + elif toC(p.type) in ('DHPy *', 'DHPy []'): + assert False, ('C type %s not supported, please write the wrapper ' + 'for %s by hand' % (toC(p.type), func.name)) + else: + lst.append(p.name) + return (', '.join(lst), '\n'.join(decls)) + (params, param_decls) = get_params_and_decls() + # + lines = [] + w = lines.append + w(signature) + w('{') + w(' if (!get_ctx_info(dctx)->is_valid) {') + w(' report_invalid_debug_context();') + w(' }') + if param_decls: + w(param_decls) + w(' get_ctx_info(dctx)->is_valid = false;') + if rettype == 'void': + w(f' {func.name}({params});') + w(' get_ctx_info(dctx)->is_valid = true;') + elif rettype == 'DHPy': + w(f' HPy universal_result = {func.name}({params});') + w(' get_ctx_info(dctx)->is_valid = true;') + w(' return DHPy_open(dctx, universal_result);') + else: + w(f' {rettype} universal_result = {func.name}({params});') + w(' get_ctx_info(dctx)->is_valid = true;') + w(' return universal_result;') + w('}') + return '\n'.join(lines) + + +class autogen_debug_ctx_call_i(AutoGenFile): + PATH = 'hpy/debug/src/autogen_debug_ctx_call.i' + + def generate(self): + lines = [] + w = lines.append + for hpyfunc in self.api.hpyfunc_typedefs: + name = hpyfunc.base_name() + NAME = name.upper() + if NAME in NO_CALL: + continue + # + c_ret_type = toC(hpyfunc.return_type()) + args = ['next_dctx'] + dhpys = [] + for i, param in enumerate(hpyfunc.params()[1:]): + pname = param.name + if pname is None: + pname = 'arg%d' % i + if toC(param.type) == 'HPy': + dhpys.append(pname) + args.append(f'dh_{pname}') + else: + args.append(f'a->{pname}') + args = ', '.join(args) + # + w(f' case HPyFunc_{NAME}: {{') + w(f' HPyFunc_{name} f = (HPyFunc_{name})func;') + w(f' _HPyFunc_args_{NAME} *a = (_HPyFunc_args_{NAME}*)args;') + for pname in dhpys: + w(f' DHPy dh_{pname} = _py2dh(dctx, a->{pname});') + # + w(' HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx);') + w(' if (next_dctx == NULL) {') + if c_ret_type == 'HPy': + w(' a->result = NULL;') + elif c_ret_type == 'int' or c_ret_type == 'HPy_ssize_t' or c_ret_type == 'HPy_hash_t': + w(' a->result = -1;') + else: + assert c_ret_type == 'void', c_ret_type + " not implemented" + w(' return;') + w(' }') + # + if c_ret_type == 'void': + w(f' f({args});') + elif c_ret_type == 'HPy': + w(f' DHPy dh_result = f({args});') + else: + w(f' a->result = f({args});') + # + w(' _switch_back_to_original_dctx(dctx, next_dctx);') + # + for pname in dhpys: + w(f' DHPy_close_and_check(dctx, dh_{pname});') + # + if c_ret_type == 'HPy': + w(f' a->result = _dh2py(dctx, dh_result);') + w(f' DHPy_close(dctx, dh_result);') + # + w(f' return;') + w(f' }}') + return '\n'.join(lines) diff --git a/graalpython/hpy/hpy/tools/autogen/doc.py b/graalpython/hpy/hpy/tools/autogen/doc.py new file mode 100644 index 0000000000..9824768866 --- /dev/null +++ b/graalpython/hpy/hpy/tools/autogen/doc.py @@ -0,0 +1,173 @@ +import textwrap +import re + +from .autogenfile import AutoGenFile +from .parse import toC +from .ctx import autogen_ctx_h +from .conf import (DOC_C_API_PAGES_SPECIAL_CASES, + DOC_MANUAL_API_MAPPING, + DOC_PREFIX_TABLE) + + +CTX_NAME = '_HPyContext_s' + +RST_DISCLAIMER = """ +.. note: DO NOT EDIT THIS FILE! + This file is automatically generated by {clsname} + See also hpy.tools.autogen and hpy/tools/public_api.h + + Run this to regenerate: + make autogen +""" + +class AutoGenRstFile(AutoGenFile): + LANGUAGE = 'rst' + PATH = None + DISCLAIMER = None + + def __init__(self, api): + if self.DISCLAIMER is None and self.LANGUAGE == 'rst': + self.DISCLAIMER = RST_DISCLAIMER + self.api = api + +class autogen_function_index(AutoGenRstFile): + PATH = 'docs/api-reference/function-index.rst' + LANGUAGE = 'rst' + + def generate(self): + lines = [] + w = lines.append + w('HPy Core API Function Index') + w('###########################') + w('') + functions = list(self.api.functions) + # sort the list of functions by 'func.name' + functions.sort(key=lambda x: x.name) + for func in functions: + if func.name[0] != '_': + w(f'* :c:func:`{func.name}`') + return '\n'.join(lines) + + +class AutoGenFilePart: + PATH = None + BEGIN_MARKER = None + END_MARKER = None + + def __init__(self, api): + self.api = api + + def generate(self, old): + raise NotImplementedError + + def write(self, root): + if not self.BEGIN_MARKER or not self.END_MARKER: + raise RuntimeError("missing BEGIN_MARKER or END_MARKER") + n_begin = len(self.BEGIN_MARKER) + with root.join(self.PATH).open('r', encoding='utf-8') as f: + content = f.read() + start = content.find(self.BEGIN_MARKER) + if start < 0: + raise RuntimeError(f'begin marker "{self.BEGIN_MARKER}" not found' + f'in file {self.PATH}') + end = content.find(self.END_MARKER, start + n_begin) + if end < 0: + raise RuntimeError(f'end marker "{self.END_MARKER}" not found in' + f'file {self.PATH}') + new_content = self.generate(content[(start+n_begin):end]) + with root.join(self.PATH).open('w', encoding='utf-8') as f: + f.write(content[:start + n_begin] + new_content + content[end:]) + + +GROUP_PATTERN = re.compile('Py(\\w+)_.*') + +class autogen_doc_api_mapping(AutoGenFilePart): + PATH = 'docs/porting-guide.rst' + BEGIN_MARKER = '.. mark: BEGIN API MAPPING\n' + END_MARKER = '.. mark: END API MAPPING\n' + + def _get_page(self, cpython_fun_name): + if cpython_fun_name in DOC_C_API_PAGES_SPECIAL_CASES: + return DOC_C_API_PAGES_SPECIAL_CASES[cpython_fun_name] + '.html' + + first_underscore = cpython_fun_name.find('_') + if cpython_fun_name.startswith('Py') and first_underscore != -1: + prefix = cpython_fun_name[2:first_underscore].lower() + return DOC_PREFIX_TABLE.get(prefix, prefix) + '.html' + else: + return 'abstract.html' + + def generate(self, old_content): + table_directive = '.. _table-mapping:\n.. table:: Safe API function mapping\n' + assert old_content.strip().startswith(table_directive) + + lines = [] + w = lines.append + w(':widths: auto') + w('') + mapping = {x.cpython_name: x.name for x in self.api.functions if x.cpython_name} + mapping.update(DOC_MANUAL_API_MAPPING) + max_width0 = 0 + max_width1 = 0 + rows = [] + cpy_functions = list(mapping.keys()) + # sort the list of functions by 'cpython_name' + cpy_functions.sort() + for cpy_func in cpy_functions: + assert cpy_func + page = self._get_page(cpy_func) + col0 = f'`{cpy_func} `_' + col1 = f':c:func:`{mapping[cpy_func]}`' + rows.append((col0, col1)) + max_width0 = max(max_width0, len(col0)) + max_width1 = max(max_width1, len(col1)) + + sep = '=' * max_width0 + ' ' + '=' * max_width1 + w(sep) + w('C API function'.ljust(max_width0) + ' HPY API function') + w(sep) + for row in rows: + w(f'{row[0].ljust(max_width0)} {row[1]}') + w(sep) + w('') + return table_directive + textwrap.indent('\n'.join(lines), ' ' * 4) + + +class autogen_hpy_ctx(AutoGenRstFile): + PATH = 'docs/api-reference/hpy-ctx.rst' + + def generate(self): + lines = [] + w = lines.append + w(textwrap.dedent( + ''' + HPy Context + =========== + + The ``HPyContext`` structure is also part of the API since it provides handles + for built-in objects. For a high-level description of the context, please also + read :ref:`api:hpycontext`. + + .. warning:: It is fine to use handles from the context (e.g. ``ctx->h_None``) + but it is **STRONGLY** discouraged to directly call any context function. + This is because, for example, when compiling for :term:`CPython ABI`, the + context functions won't be used. + ''')) + # Put all variable declarations into a list in order + # to be able to sort them by their given context index. + var_decls = list(self.api.variables) + + # sort the list of var declaration by 'decl.ctx_index' + var_decls.sort(key=lambda x: x.ctx_index) + + w(f'.. c:struct:: {CTX_NAME}') + w(f' :module: {autogen_ctx_h.PATH}') + w('') + w(f' .. c:member:: const char *{CTX_NAME}.name') + w('') + w(f' .. c:member:: int {CTX_NAME}.abi_version') + for var in var_decls: + w('') + w(f' .. c:member:: {toC(var.node)}') + return '\n'.join(lines) + diff --git a/graalpython/hpy/hpy/tools/autogen/hpyfunc.py b/graalpython/hpy/hpy/tools/autogen/hpyfunc.py new file mode 100644 index 0000000000..1e233277f2 --- /dev/null +++ b/graalpython/hpy/hpy/tools/autogen/hpyfunc.py @@ -0,0 +1,204 @@ +from copy import deepcopy +from pycparser import c_ast +from .autogenfile import AutoGenFile +from .parse import toC, find_typedecl + +NO_CALL = ('VARARGS', 'KEYWORDS', 'INITPROC', 'DESTROYFUNC', + 'GETBUFFERPROC', 'RELEASEBUFFERPROC', 'TRAVERSEPROC', 'MOD_CREATE', + 'VECTORCALLFUNC', 'NEWFUNC') +NO_TRAMPOLINE = NO_CALL + ('RICHCMPFUNC',) + +# This is a list of type that can automatically be converted from Python to HPy +# and vice versa. +AUTO_CONVERSION_TYPES = ('HPy', 'HPy_ssize_t', 'void *', 'int', 'char *', + 'HPy_hash_t', 'HPy_RichCmpOp', 'void') + +class autogen_hpyfunc_declare_h(AutoGenFile): + PATH = 'hpy/devel/include/hpy/autogen_hpyfunc_declare.h' + + ## #define _HPyFunc_DECLARE_HPyFunc_NOARGS(SYM) \ + ## static HPy SYM(HPyContext *ctx, HPy self) + + def generate(self): + lines = [] + w = lines.append + for hpyfunc in self.api.hpyfunc_typedefs: + # declare a function named 'SYM' of the appropriate type + funcdecl = hpyfunc.node.type.type + symdecl = deepcopy(funcdecl) + if isinstance(symdecl.type, c_ast.PtrDecl): + symdecl.type = symdecl.type.type + symdecl.type.declname = 'SYM' + symdecl = toC(symdecl) + # + # generate a macro emitting 'symdecl' + name = hpyfunc.base_name().upper() + w(f'#define _HPyFunc_DECLARE_HPyFunc_{name}(SYM) static {symdecl}') + w('') + + for hpyfunc in self.api.hpyfunc_typedefs: + # generate the typedef for HPyFunc_{base_name} + w(f'{toC(hpyfunc.node)};') + + return '\n'.join(lines) + + +def hpy_to_cpy(declnode): + if toC(declnode.type) == 'HPy': + declnode = deepcopy(declnode) + declnode.type.type.names = ['cpy_PyObject'] + declnode.type = c_ast.PtrDecl(type=declnode.type, quals=[]) + return declnode + + +class autogen_hpyfunc_trampoline_h(AutoGenFile): + PATH = 'hpy/devel/include/hpy/universal/autogen_hpyfunc_trampolines.h' + + def generate(self): + lines = [] + w = lines.append + for hpyfunc in self.api.hpyfunc_typedefs: + NAME = hpyfunc.base_name().upper() + if NAME in NO_TRAMPOLINE: + continue + # + tramp_node = deepcopy(hpyfunc.node.type.type) + if isinstance(tramp_node.type, c_ast.PtrDecl): + tramp_node.type = tramp_node.type.type + tramp_node.type.declname = 'SYM' + tramp_node = hpy_to_cpy(tramp_node) + assert toC(tramp_node.args.params[0].type) in ['void', 'HPyContext *'] + tramp_node.args.params = [hpy_to_cpy(p) + for p in tramp_node.args.params[1:]] + for i, param in enumerate(tramp_node.args.params): + typedecl = find_typedecl(param.type) + if typedecl.declname is None: + param.name = 'arg%d' % i + typedecl.declname = 'arg%d' % i + arg_names = [param.name for param in tramp_node.args.params] + arg_names = ', '.join(arg_names) + # + # generate the struct that will contain all parameters + w(f'typedef struct {{') + for param in tramp_node.args.params: + w(f' {toC(param)};') + if toC(tramp_node.type) != 'void': + w(f' {toC(tramp_node.type)} result;') + w(f'}} _HPyFunc_args_{NAME};') + w('') + # + # generate the trampoline itself + w(f'#define _HPyFunc_TRAMPOLINE_HPyFunc_{NAME}(SYM, IMPL) \\') + w(f' static {toC(tramp_node)} \\') + w(f' {{ \\') + w(f' _HPyFunc_args_{NAME} a = {{ {arg_names} }}; \\') + w(f' _HPy_CallRealFunctionFromTrampoline( \\') + w(f' _ctx_for_trampolines, HPyFunc_{NAME}, (HPyCFunction)IMPL, &a); \\') + if toC(tramp_node.type) == 'void': + w(f' return; \\') + else: + w(f' return a.result; \\') + w(f' }}') + w('') + return '\n'.join(lines) + + +def _py2h(type): + if type == 'HPy': + return '_py2h' + elif type in AUTO_CONVERSION_TYPES: + return '' + raise TypeError(f'cannot generate automatic conversion for type \'{type}\'') + + +def _h2py(type): + if type == 'HPy': + return '_h2py' + elif type in AUTO_CONVERSION_TYPES: + return '' + raise TypeError(f'cannot generate automatic conversion for type \'{type}\'') + + +class autogen_ctx_call_i(AutoGenFile): + PATH = 'hpy/universal/src/autogen_ctx_call.i' + + def generate(self): + lines = [] + w = lines.append + for hpyfunc in self.api.hpyfunc_typedefs: + name = hpyfunc.base_name() + NAME = name.upper() + if NAME in NO_CALL: + continue + # + c_ret_type = toC(hpyfunc.return_type()) + args = ['ctx'] + for i, param in enumerate(hpyfunc.params()[1:]): + pname = param.name + if pname is None: + pname = 'arg%d' % i + args.append(f'{_py2h(toC(param.type))}(a->{pname})') + args = ', '.join(args) + # + w(f' case HPyFunc_{NAME}: {{') + w(f' HPyFunc_{name} f = (HPyFunc_{name})func;') + w(f' _HPyFunc_args_{NAME} *a = (_HPyFunc_args_{NAME}*)args;') + if c_ret_type == 'void': + w(f' f({args});') + else: + w(f' a->result = {_h2py(c_ret_type)}(f({args}));') + w(f' return;') + w(f' }}') + return '\n'.join(lines) + + +class autogen_cpython_hpyfunc_trampoline_h(AutoGenFile): + PATH = 'hpy/devel/include/hpy/cpython/autogen_hpyfunc_trampolines.h' + + def generate(self): + lines = [] + w = lines.append + for hpyfunc in self.api.hpyfunc_typedefs: + name = hpyfunc.base_name() + NAME = name.upper() + if NAME in NO_TRAMPOLINE: + continue + # + tramp_node = deepcopy(hpyfunc.node.type.type) + if isinstance(tramp_node.type, c_ast.PtrDecl): + tramp_node.type = tramp_node.type.type + tramp_node.type.declname = 'SYM' + tramp_node = hpy_to_cpy(tramp_node) + tramp_node.args.params = [hpy_to_cpy(p) + for p in tramp_node.args.params[1:]] + for i, param in enumerate(tramp_node.args.params): + typedecl = find_typedecl(param.type) + if typedecl.declname is None: + param.name = 'arg%d' % i + typedecl.declname = 'arg%d' % i + + result = _h2py(toC(hpyfunc.return_type())) + args = ['_HPyGetContext()'] + func_ptr_ret_type = toC(hpyfunc.return_type()) + func_ptr_sig = ['HPyContext *'] + for i, param in enumerate(hpyfunc.params()[1:]): + pname = param.name + if pname is None: + pname = 'arg%d' % i + func_ptr_sig.append(toC(param.type)) + args.append(f'{_py2h(toC(param.type))}({pname})') + args = ', '.join(args) + func_ptr_sig = ', '.join(func_ptr_sig) + # + w(f'typedef {func_ptr_ret_type} (*_HPyCFunction_{NAME})({func_ptr_sig});') + w(f'#define _HPyFunc_TRAMPOLINE_HPyFunc_{NAME}(SYM, IMPL) \\') + w(f' static {toC(tramp_node)} \\') + w(f' {{ \\') + w(f' _HPyCFunction_{NAME} func = (_HPyCFunction_{NAME})IMPL; \\') + if toC(tramp_node.type) == 'void': + w(f' func({args}); \\') + w(f' return; \\') + else: + w(f' return {result}(func({args})); \\') + w(f' }}') + return '\n'.join(lines) diff --git a/graalpython/hpy/hpy/tools/autogen/hpyslot.py b/graalpython/hpy/hpy/tools/autogen/hpyslot.py new file mode 100644 index 0000000000..3eff4d81ca --- /dev/null +++ b/graalpython/hpy/hpy/tools/autogen/hpyslot.py @@ -0,0 +1,18 @@ +from pycparser import c_ast +from .autogenfile import AutoGenFile +from .parse import toC + +class autogen_hpyslot_h(AutoGenFile): + PATH = 'hpy/devel/include/hpy/autogen_hpyslot.h' + + def generate(self): + lines = [] + w = lines.append + w('typedef enum {') + for slot in self.api.hpyslots: + w(f' {slot.name} = {slot.value},') + w('} HPySlot_Slot;') + w('') + for slot in self.api.hpyslots: + w(f'#define _HPySlot_SIG__{slot.name} {slot.hpyfunc}') + return '\n'.join(lines) diff --git a/graalpython/hpy/hpy/tools/autogen/parse.py b/graalpython/hpy/hpy/tools/autogen/parse.py new file mode 100644 index 0000000000..6615eb1d50 --- /dev/null +++ b/graalpython/hpy/hpy/tools/autogen/parse.py @@ -0,0 +1,278 @@ +from copy import deepcopy +import sys +import attr +import re +import py +import pycparser +import shutil +from pycparser import c_ast +from pycparser.c_generator import CGenerator +from sysconfig import get_config_var +from .conf import SPECIAL_CASES, RETURN_CONSTANT + +PUBLIC_API_H = py.path.local(__file__).dirpath('public_api.h') +CURRENT_DIR = py.path.local(__file__).dirpath() +AUTOGEN_H = py.path.local(__file__).dirpath('autogen.h') + + +def toC(node): + return toC.gen.visit(node) +toC.gen = CGenerator() + +def find_typedecl(node): + while not isinstance(node, c_ast.TypeDecl): + node = node.type + return node + +def get_context_return_type(func_node, const_return): + return 'void' if const_return else toC(func_node.type.type) + +def make_void(func_node): + voidid = c_ast.IdentifierType(names=['void']) + func_node.type.type.type = c_ast.TypeDecl(declname='void', quals=[], align=[], type=voidid) + +def get_return_constant(func): + return RETURN_CONSTANT.get(func.node.name) + +def maybe_make_void(func, node): + if RETURN_CONSTANT.get(func.node.name): + make_void(node) + +@attr.s +class Function: + _BASE_NAME = re.compile(r'^_?HPy_?') + + name = attr.ib() + cpython_name = attr.ib() + ctx_index = attr.ib() + node = attr.ib(repr=False) + + def base_name(self): + return self._BASE_NAME.sub('', self.name) + + def ctx_name(self): + # e.g. "ctx_Module_Create" + return self._BASE_NAME.sub(r'ctx_', self.name) + + def is_varargs(self): + return (len(self.node.type.args.params) > 0 and + isinstance(self.node.type.args.params[-1], c_ast.EllipsisParam)) + + +@attr.s +class GlobalVar: + name = attr.ib() + ctx_index = attr.ib() + node = attr.ib(repr=False) + + def ctx_name(self): + return self.name + + +@attr.s +class HPyFunc: + _BASE_NAME = re.compile(r'^HPyFunc_?') + + name = attr.ib() + node = attr.ib(repr=False) + + def base_name(self): + return self._BASE_NAME.sub('', self.name) + + def params(self): + return self.node.type.type.args.params + + def return_type(self): + return self.node.type.type.type + +@attr.s +class HPySlot: + # represent a declaration contained inside enum HPySlot_Slot, such as: + # HPy_nb_add = SLOT(7, HPyFunc_BINARYFUNC) + + name = attr.ib() # "HPy_nb_add" + value = attr.ib() # "7" + hpyfunc = attr.ib() # "HPyFunc_BINARYFUNC" + node = attr.ib(repr=False) + + +class HPyAPIVisitor(pycparser.c_ast.NodeVisitor): + def __init__(self, api, convert_name): + self.api = api + self.convert_name = convert_name + self.cur_index = -1 + self.all_indices = [] + + def _consume_ctx_index(self): + idx = self.cur_index + self.all_indices.append(idx) + self.cur_index = -1 + return idx + + def verify_context_indices(self): + """ + Verifies if context indices are without gaps. This function raises an + assertion error if not. + For example: + [0, 1, 2, 3] is valid + [0, 1, 3] is invalid + """ + self.all_indices.sort() + for i in range(1, len(self.all_indices)): + prev = self.all_indices[i-1] + cur = self.all_indices[i] + assert prev + 1 == cur, \ + "context indices have gaps: %s -> %s" % (prev, cur) + + def _is_function_ptr(self, node): + return (isinstance(node, c_ast.PtrDecl) and + isinstance(node.type, c_ast.FuncDecl)) + + def visit_Decl(self, node): + if isinstance(node.type, c_ast.FuncDecl): + self._visit_function(node) + elif isinstance(node.type, c_ast.TypeDecl): + self._visit_global_var(node) + + def visit_Typedef(self, node): + # find only typedefs to function pointers whose name starts by HPyFunc_ + if node.name.startswith('HPyFunc_') and self._is_function_ptr(node.type): + self._visit_hpyfunc_typedef(node) + elif node.name == 'HPySlot_Slot': + self._visit_hpyslot_slot(node) + + def visit_Pragma(self, node): + parts = node.string.split('=') + if len(parts) != 2: + raise ValueError('invalid pragma: %s' % node) + self.cur_index = int(parts[1]) + + def _visit_function(self, node): + name = node.name + if not name.startswith('HPy') and not name.startswith('_HPy'): + print('WARNING: Ignoring non-hpy declaration: %s' % name) + return + for p in node.type.args.params: + if hasattr(p, 'name') and p.name is None: + raise ValueError("non-named argument in declaration of %s" % + name) + cpy_name = self.convert_name(name) + idx = self._consume_ctx_index() + if idx == -1: + raise ValueError('missing context index for %s' % name) + func = Function(name, cpy_name, idx, node) + self.api.functions.append(func) + + def _visit_global_var(self, node): + name = node.name + if not name.startswith('h_'): + print('WARNING: Ignoring non-hpy variable declaration: %s' % name) + return + assert toC(node.type.type) == "HPy" + idx = self._consume_ctx_index() + if idx == -1: + raise ValueError('missing context index for %s' % name) + var = GlobalVar(name, idx, node) + self.api.variables.append(var) + + def _visit_hpyfunc_typedef(self, node): + hpyfunc = HPyFunc(node.name, node) + self.api.hpyfunc_typedefs.append(hpyfunc) + + def _visit_hpyslot_slot(self, node): + for e in node.type.type.values.enumerators: + call = e.value + assert isinstance(call, c_ast.FuncCall) and call.name.name == 'SLOT' + assert len(call.args.exprs) == 2 + const_value, id_hpyfunc = call.args.exprs + assert isinstance(const_value, c_ast.Constant) and const_value.type == 'int' + assert isinstance(id_hpyfunc, c_ast.ID) + value = const_value.value + hpyfunc = id_hpyfunc.name + self.api.hpyslots.append(HPySlot(e.name, value, hpyfunc, e)) + + +def convert_name(hpy_name): + if hpy_name in SPECIAL_CASES: + return SPECIAL_CASES[hpy_name] + return re.sub(r'^_?HPy_?', 'Py', hpy_name) + + +class HPyAPI: + _r_comment = re.compile(r"/\*.*?\*/|//([^\n\\]|\\.)*?$", + re.DOTALL | re.MULTILINE) + + def __init__(self, filename): + cpp_cmd = get_config_var('CC') + if cpp_cmd: + cpp_cmd = cpp_cmd.split(' ') + elif sys.platform == 'win32': + cpp_cmd = [shutil.which("cl.exe")] + if sys.platform == 'win32': + cpp_cmd += ['/E', '/I%s' % CURRENT_DIR] + else: + cpp_cmd += ['-E', '-I%s' % CURRENT_DIR] + + msvc = "cl.exe" in cpp_cmd[0].casefold() + + csource = pycparser.preprocess_file(filename, + cpp_path=str(cpp_cmd[0]), + cpp_args=cpp_cmd[1:]) + + # MSVC preprocesses _Pragma(foo) to __pragma(foo), + # but cparser needs to see a #pragma, not __pragma. + if msvc: + csource = re.sub(r'__pragma\(([^)]+)\)', r'#pragma \1\n', csource) + + # Remove comments. NOTE: this assumes that comments are never inside + # string literals, but there shouldn't be any here. + def replace_keeping_newlines(m): + return ' ' + m.group().count('\n') * '\n' + csource = self._r_comment.sub(replace_keeping_newlines, csource) + self.ast = pycparser.CParser().parse(csource) + ## print(); self.ast.show() + self.collect_declarations() + + @classmethod + def parse(cls, filename): + return cls(filename) + + def get_func(self, name): + return self._lookup(name, self.functions) + + def get_var(self, name): + return self._lookup(name, self.variables) + + def get_hpyfunc_typedef(self, name): + return self._lookup(name, self.hpyfunc_typedefs) + + def get_slot(self, name): + return self._lookup(name, self.hpyslots) + + def _lookup(self, name, collection): + for x in collection: + if x.name == name: + return x + raise KeyError(name) + + def collect_declarations(self): + self.functions = [] + self.variables = [] + self.hpyfunc_typedefs = [] + self.hpyslots = [] + v = HPyAPIVisitor(self, convert_name) + v.visit(self.ast) + + v.verify_context_indices() + + # Sort lists such that the generated files are deterministic. + # List elements are either 'Function', 'GlobalVar', or 'HPyFunc'. All + # of them have a 'node' attribute and the nodes have a 'coord' attr + # that provides the line and column number. We use that to sort. + def node_key(e): + coord = e.node.coord + return coord.line, coord.column + self.functions.sort(key=node_key) + self.variables.sort(key=node_key) + self.hpyfunc_typedefs.sort(key=node_key) + self.hpyslots.sort(key=node_key) diff --git a/graalpython/hpy/hpy/tools/autogen/public_api.h b/graalpython/hpy/hpy/tools/autogen/public_api.h new file mode 100644 index 0000000000..b081d943a8 --- /dev/null +++ b/graalpython/hpy/hpy/tools/autogen/public_api.h @@ -0,0 +1,1557 @@ +/* HPy public API */ + +/* + * IMPORTANT: In order to ensure backwards compatibility of HPyContext, it is + * necessary to define the order of the context members. To do so, use macro + * 'HPy_ID(idx)' for context handles and functions. When adding members, it + * doesn't matter where they are located in this file. It's just important that + * the maximum context index is incremented by exactly one. + */ + +#ifdef AUTOGEN + +/* Constants */ +HPy_ID(0) HPy h_None; +HPy_ID(1) HPy h_True; +HPy_ID(2) HPy h_False; +HPy_ID(3) HPy h_NotImplemented; +HPy_ID(4) HPy h_Ellipsis; + +/* Exceptions */ +HPy_ID(5) HPy h_BaseException; +HPy_ID(6) HPy h_Exception; +HPy_ID(7) HPy h_StopAsyncIteration; +HPy_ID(8) HPy h_StopIteration; +HPy_ID(9) HPy h_GeneratorExit; +HPy_ID(10) HPy h_ArithmeticError; +HPy_ID(11) HPy h_LookupError; +HPy_ID(12) HPy h_AssertionError; +HPy_ID(13) HPy h_AttributeError; +HPy_ID(14) HPy h_BufferError; +HPy_ID(15) HPy h_EOFError; +HPy_ID(16) HPy h_FloatingPointError; +HPy_ID(17) HPy h_OSError; +HPy_ID(18) HPy h_ImportError; +HPy_ID(19) HPy h_ModuleNotFoundError; +HPy_ID(20) HPy h_IndexError; +HPy_ID(21) HPy h_KeyError; +HPy_ID(22) HPy h_KeyboardInterrupt; +HPy_ID(23) HPy h_MemoryError; +HPy_ID(24) HPy h_NameError; +HPy_ID(25) HPy h_OverflowError; +HPy_ID(26) HPy h_RuntimeError; +HPy_ID(27) HPy h_RecursionError; +HPy_ID(28) HPy h_NotImplementedError; +HPy_ID(29) HPy h_SyntaxError; +HPy_ID(30) HPy h_IndentationError; +HPy_ID(31) HPy h_TabError; +HPy_ID(32) HPy h_ReferenceError; +HPy_ID(33) HPy h_SystemError; +HPy_ID(34) HPy h_SystemExit; +HPy_ID(35) HPy h_TypeError; +HPy_ID(36) HPy h_UnboundLocalError; +HPy_ID(37) HPy h_UnicodeError; +HPy_ID(38) HPy h_UnicodeEncodeError; +HPy_ID(39) HPy h_UnicodeDecodeError; +HPy_ID(40) HPy h_UnicodeTranslateError; +HPy_ID(41) HPy h_ValueError; +HPy_ID(42) HPy h_ZeroDivisionError; +HPy_ID(43) HPy h_BlockingIOError; +HPy_ID(44) HPy h_BrokenPipeError; +HPy_ID(45) HPy h_ChildProcessError; +HPy_ID(46) HPy h_ConnectionError; +HPy_ID(47) HPy h_ConnectionAbortedError; +HPy_ID(48) HPy h_ConnectionRefusedError; +HPy_ID(49) HPy h_ConnectionResetError; +HPy_ID(50) HPy h_FileExistsError; +HPy_ID(51) HPy h_FileNotFoundError; +HPy_ID(52) HPy h_InterruptedError; +HPy_ID(53) HPy h_IsADirectoryError; +HPy_ID(54) HPy h_NotADirectoryError; +HPy_ID(55) HPy h_PermissionError; +HPy_ID(56) HPy h_ProcessLookupError; +HPy_ID(57) HPy h_TimeoutError; +// EnvironmentError, IOError and WindowsError are intentionally omitted (they +// are all aliases of OSError since Python 3.3). + +/* Warnings */ +HPy_ID(58) HPy h_Warning; +HPy_ID(59) HPy h_UserWarning; +HPy_ID(60) HPy h_DeprecationWarning; +HPy_ID(61) HPy h_PendingDeprecationWarning; +HPy_ID(62) HPy h_SyntaxWarning; +HPy_ID(63) HPy h_RuntimeWarning; +HPy_ID(64) HPy h_FutureWarning; +HPy_ID(65) HPy h_ImportWarning; +HPy_ID(66) HPy h_UnicodeWarning; +HPy_ID(67) HPy h_BytesWarning; +HPy_ID(68) HPy h_ResourceWarning; + +/* Types */ +HPy_ID(69) HPy h_BaseObjectType; /* built-in 'object' */ +HPy_ID(70) HPy h_TypeType; /* built-in 'type' */ +HPy_ID(71) HPy h_BoolType; /* built-in 'bool' */ +HPy_ID(72) HPy h_LongType; /* built-in 'int' */ +HPy_ID(73) HPy h_FloatType; /* built-in 'float' */ +HPy_ID(74) HPy h_UnicodeType; /* built-in 'str' */ +HPy_ID(75) HPy h_TupleType; /* built-in 'tuple' */ +HPy_ID(76) HPy h_ListType; /* built-in 'list' */ +HPy_ID(238) HPy h_ComplexType; /* built-in 'complex' */ +HPy_ID(239) HPy h_BytesType; /* built-in 'bytes' */ +HPy_ID(240) HPy h_MemoryViewType; /* built-in 'memoryview' */ +HPy_ID(241) HPy h_CapsuleType; /* built-in 'capsule' */ +HPy_ID(242) HPy h_SliceType; /* built-in 'slice' */ +HPy_ID(263) HPy h_DictType; /* built-in 'dict' */ + +/* Reflection */ +HPy_ID(243) HPy h_Builtins; /* dict of builtins */ + +#endif + +HPy_ID(77) +HPy HPy_Dup(HPyContext *ctx, HPy h); +HPy_ID(78) +void HPy_Close(HPyContext *ctx, HPy h); + +HPy_ID(79) +HPy HPyLong_FromInt32_t(HPyContext *ctx, int32_t value); +HPy_ID(80) +HPy HPyLong_FromUInt32_t(HPyContext *ctx, uint32_t value); +HPy_ID(81) +HPy HPyLong_FromInt64_t(HPyContext *ctx, int64_t v); +HPy_ID(82) +HPy HPyLong_FromUInt64_t(HPyContext *ctx, uint64_t v); +HPy_ID(83) +HPy HPyLong_FromSize_t(HPyContext *ctx, size_t value); +HPy_ID(84) +HPy HPyLong_FromSsize_t(HPyContext *ctx, HPy_ssize_t value); + +HPy_ID(85) +int32_t HPyLong_AsInt32_t(HPyContext *ctx, HPy h); +HPy_ID(86) +uint32_t HPyLong_AsUInt32_t(HPyContext *ctx, HPy h); +HPy_ID(87) +uint32_t HPyLong_AsUInt32_tMask(HPyContext *ctx, HPy h); +HPy_ID(88) +int64_t HPyLong_AsInt64_t(HPyContext *ctx, HPy h); +HPy_ID(89) +uint64_t HPyLong_AsUInt64_t(HPyContext *ctx, HPy h); +HPy_ID(90) +uint64_t HPyLong_AsUInt64_tMask(HPyContext *ctx, HPy h); +HPy_ID(91) +size_t HPyLong_AsSize_t(HPyContext *ctx, HPy h); +HPy_ID(92) +HPy_ssize_t HPyLong_AsSsize_t(HPyContext *ctx, HPy h); +HPy_ID(93) +void* HPyLong_AsVoidPtr(HPyContext *ctx, HPy h); +HPy_ID(94) +double HPyLong_AsDouble(HPyContext *ctx, HPy h); + +HPy_ID(95) +HPy HPyFloat_FromDouble(HPyContext *ctx, double v); +HPy_ID(96) +double HPyFloat_AsDouble(HPyContext *ctx, HPy h); + +HPy_ID(97) +HPy HPyBool_FromBool(HPyContext *ctx, bool v); + + +/* abstract.h */ +HPy_ID(98) +HPy_ssize_t HPy_Length(HPyContext *ctx, HPy h); + +HPy_ID(99) +int HPyNumber_Check(HPyContext *ctx, HPy h); +HPy_ID(100) +HPy HPy_Add(HPyContext *ctx, HPy h1, HPy h2); +HPy_ID(101) +HPy HPy_Subtract(HPyContext *ctx, HPy h1, HPy h2); +HPy_ID(102) +HPy HPy_Multiply(HPyContext *ctx, HPy h1, HPy h2); +HPy_ID(103) +HPy HPy_MatrixMultiply(HPyContext *ctx, HPy h1, HPy h2); +HPy_ID(104) +HPy HPy_FloorDivide(HPyContext *ctx, HPy h1, HPy h2); +HPy_ID(105) +HPy HPy_TrueDivide(HPyContext *ctx, HPy h1, HPy h2); +HPy_ID(106) +HPy HPy_Remainder(HPyContext *ctx, HPy h1, HPy h2); +HPy_ID(107) +HPy HPy_Divmod(HPyContext *ctx, HPy h1, HPy h2); +HPy_ID(108) +HPy HPy_Power(HPyContext *ctx, HPy h1, HPy h2, HPy h3); +HPy_ID(109) +HPy HPy_Negative(HPyContext *ctx, HPy h1); +HPy_ID(110) +HPy HPy_Positive(HPyContext *ctx, HPy h1); +HPy_ID(111) +HPy HPy_Absolute(HPyContext *ctx, HPy h1); +HPy_ID(112) +HPy HPy_Invert(HPyContext *ctx, HPy h1); +HPy_ID(113) +HPy HPy_Lshift(HPyContext *ctx, HPy h1, HPy h2); +HPy_ID(114) +HPy HPy_Rshift(HPyContext *ctx, HPy h1, HPy h2); +HPy_ID(115) +HPy HPy_And(HPyContext *ctx, HPy h1, HPy h2); +HPy_ID(116) +HPy HPy_Xor(HPyContext *ctx, HPy h1, HPy h2); +HPy_ID(117) +HPy HPy_Or(HPyContext *ctx, HPy h1, HPy h2); +HPy_ID(118) +HPy HPy_Index(HPyContext *ctx, HPy h1); +HPy_ID(119) +HPy HPy_Long(HPyContext *ctx, HPy h1); +HPy_ID(120) +HPy HPy_Float(HPyContext *ctx, HPy h1); + +HPy_ID(121) +HPy HPy_InPlaceAdd(HPyContext *ctx, HPy h1, HPy h2); +HPy_ID(122) +HPy HPy_InPlaceSubtract(HPyContext *ctx, HPy h1, HPy h2); +HPy_ID(123) +HPy HPy_InPlaceMultiply(HPyContext *ctx, HPy h1, HPy h2); +HPy_ID(124) +HPy HPy_InPlaceMatrixMultiply(HPyContext *ctx, HPy h1, HPy h2); +HPy_ID(125) +HPy HPy_InPlaceFloorDivide(HPyContext *ctx, HPy h1, HPy h2); +HPy_ID(126) +HPy HPy_InPlaceTrueDivide(HPyContext *ctx, HPy h1, HPy h2); +HPy_ID(127) +HPy HPy_InPlaceRemainder(HPyContext *ctx, HPy h1, HPy h2); +HPy_ID(128) +HPy HPy_InPlacePower(HPyContext *ctx, HPy h1, HPy h2, HPy h3); +HPy_ID(129) +HPy HPy_InPlaceLshift(HPyContext *ctx, HPy h1, HPy h2); +HPy_ID(130) +HPy HPy_InPlaceRshift(HPyContext *ctx, HPy h1, HPy h2); +HPy_ID(131) +HPy HPy_InPlaceAnd(HPyContext *ctx, HPy h1, HPy h2); +HPy_ID(132) +HPy HPy_InPlaceXor(HPyContext *ctx, HPy h1, HPy h2); +HPy_ID(133) +HPy HPy_InPlaceOr(HPyContext *ctx, HPy h1, HPy h2); + +HPy_ID(134) +int HPyCallable_Check(HPyContext *ctx, HPy h); + +/** + * Call a Python object. + * + * :param ctx: + * The execution context. + * :param callable: + * A handle to the Python object to call (must not be ``HPy_NULL``). + * :param args: + * A handle to a tuple containing the positional arguments (must not be + * ``HPy_NULL`` but can, of course, be empty). + * :param kw: + * A handle to a Python dictionary containing the keyword arguments (may be + * ``HPy_NULL``). + * + * :returns: + * The result of the call on success, or ``HPy_NULL`` in case of an error. + */ +HPy_ID(135) +HPy HPy_CallTupleDict(HPyContext *ctx, HPy callable, HPy args, HPy kw); + +/** + * Call a Python object. + * + * :param ctx: + * The execution context. + * :param callable: + * A handle to the Python object to call (must not be ``HPy_NULL``). + * :param args: + * A pointer to an array of positional and keyword arguments. This argument + * must not be ``NULL`` if ``nargs > 0`` or + * ``HPy_Length(ctx, kwnames) > 0``. + * :param nargs: + * The number of positional arguments in ``args``. + * :param kwnames: + * A handle to the tuple of keyword argument names (may be ``HPy_NULL``). + * The values of the keyword arguments are also passed in ``args`` appended + * to the positional arguments. Argument ``nargs`` does not include the + * keyword argument count. + * + * :returns: + * The result of the call on success, or ``HPy_NULL`` in case of an error. + */ +HPy_ID(261) +HPy HPy_Call(HPyContext *ctx, HPy callable, const HPy *args, size_t nargs, HPy kwnames); + +/** + * Call a method of a Python object. + * + * :param ctx: + * The execution context. + * :param name: + * A handle to the name (a Unicode object) of the method. Must not be + * ``HPy_NULL``. + * :param args: + * A pointer to an array of the arguments. The receiver is ``args[0]``, and + * the positional and keyword arguments are starting at ``args[1]``. This + * argument must not be ``NULL`` since a receiver is always required. + * :param nargs: + * The number of positional arguments in ``args`` including the receiver at + * ``args[0]`` (therefore, ``nargs`` must be at least ``1``). + * :param kwnames: + * A handle to the tuple of keyword argument names (may be ``HPy_NULL``). + * The values of the keyword arguments are also passed in ``args`` appended + * to the positional arguments. Argument ``nargs`` does not include the + * keyword argument count. + * + * :returns: + * The result of the call on success, or ``HPy_NULL`` in case of an error. + */ +HPy_ID(262) +HPy HPy_CallMethod(HPyContext *ctx, HPy name, const HPy *args, size_t nargs, HPy kwnames); + +/** + * Return a new iterator for iterable object ``obj``. This is the equivalent + * of the Python expression ``iter(obj)``. + * + * :param ctx: + * The execution context. + * :param obj: + * An iterable Python object (must not be ``HPy_NULL``). If the object is + * not iterable, a ``TypeError`` will be raised. + * + * :returns: + * The new iterator, ``obj`` itself if it is already an iterator, or + * ``HPy_NULL`` on failure. + */ +HPy_ID(269) +HPy HPy_GetIter(HPyContext *ctx, HPy obj); + +/** + * Return the next value from iterator ``obj``. + * + * :param ctx: + * The execution context. + * :param obj: + * An iterator Python object (must not be ``HPy_NULL``). This can be + * verified with ``HPy_IterCheck``. Otherwise, the behavior is undefined. + * + * :returns: + * The new value in iterator ``obj``, or ``HPy_NULL`` on failure. If the + * iterator was exhausted normally, an exception will not be set. In + * case of some other error, one will be set. + */ +HPy_ID(270) +HPy HPyIter_Next(HPyContext *ctx, HPy obj); + +/** + * Tests if an object is an instance of a Python iterator. + * + * :param ctx: + * The execution context. + * :param obj: + * A handle to an arbitrary object (must not be ``HPy_NULL``). + * + * :returns: + * Non-zero if object ``obj`` provides the ``Iterator`` protocol, and ``0`` + * otherwise. + */ +HPy_ID(271) +int HPyIter_Check(HPyContext *ctx, HPy obj); + +/* pyerrors.h */ +HPy_ID(136) +void HPy_FatalError(HPyContext *ctx, const char *message); +HPy_ID(137) +HPy HPyErr_SetString(HPyContext *ctx, HPy h_type, const char *utf8_message); +HPy_ID(138) +HPy HPyErr_SetObject(HPyContext *ctx, HPy h_type, HPy h_value); + +/** + * Similar to :c:func:`HPyErr_SetFromErrnoWithFilenameObjects` but takes one + * filename (a C string) that will be decoded using + * :c:func:`HPyUnicode_DecodeFSDefault`. + * + * :param ctx: + * The execution context. + * :param h_type: + * The exception type to raise. + * :param filename_fsencoded: + * a filename; may be ``NULL`` + * + * :return: + * always returns ``HPy_NULL`` + */ +HPy_ID(139) +HPy HPyErr_SetFromErrnoWithFilename(HPyContext *ctx, HPy h_type, const char *filename_fsencoded); + +/** + * A convenience function to raise an exception when a C library function has + * returned an error and set the C variable ``errno``. It constructs an + * instance of the provided exception type ``h_type`` by calling + * ``h_type(errno, strerror(errno), filename1, 0, filename2)``. The exception + * instance is then raised. + * + * :param ctx: + * The execution context. + * :param h_type: + * The exception type to raise. + * :param filename1: + * A filename; may be ``HPy_NULL``. In the case of ``h_type`` is the + * ``OSError`` exception, this is used to define the filename attribute of + * the exception instance. + * :param filename2: + * another filename argument; may be ``HPy_NULL`` + * + * :return: + * always returns ``HPy_NULL`` + */ +HPy_ID(140) +HPy HPyErr_SetFromErrnoWithFilenameObjects(HPyContext *ctx, HPy h_type, HPy filename1, HPy filename2); +/* note: HPyErr_Occurred() returns a flag 0-or-1, instead of a 'PyObject *' */ +HPy_ID(141) +int HPyErr_Occurred(HPyContext *ctx); +HPy_ID(142) +int HPyErr_ExceptionMatches(HPyContext *ctx, HPy exc); +HPy_ID(143) +HPy HPyErr_NoMemory(HPyContext *ctx); +HPy_ID(144) +void HPyErr_Clear(HPyContext *ctx); +HPy_ID(145) +HPy HPyErr_NewException(HPyContext *ctx, const char *utf8_name, HPy base, HPy dict); +HPy_ID(146) +HPy HPyErr_NewExceptionWithDoc(HPyContext *ctx, const char *utf8_name, const char *utf8_doc, HPy base, HPy dict); +HPy_ID(147) +int HPyErr_WarnEx(HPyContext *ctx, HPy category, const char *utf8_message, HPy_ssize_t stack_level); +HPy_ID(148) +void HPyErr_WriteUnraisable(HPyContext *ctx, HPy obj); + +/* object.h */ +HPy_ID(149) +int HPy_IsTrue(HPyContext *ctx, HPy h); + +/** + * Create a type from a :c:struct:`HPyType_Spec` and an additional list of + * specification parameters. + * + * :param ctx: + * The execution context. + * :param spec: + * The type spec to use to create the type. + * :param params: + * A 0-terminated list of type specification parameters or ``NULL``. + * + * :returns: a handle of the created type on success, ``HPy_NULL`` on failure. + */ +HPy_ID(150) +HPy HPyType_FromSpec(HPyContext *ctx, HPyType_Spec *spec, + HPyType_SpecParam *params); +HPy_ID(151) +HPy HPyType_GenericNew(HPyContext *ctx, HPy type, const HPy *args, HPy_ssize_t nargs, HPy kw); + +HPy_ID(152) +HPy HPy_GetAttr(HPyContext *ctx, HPy obj, HPy name); +HPy_ID(153) +HPy HPy_GetAttr_s(HPyContext *ctx, HPy obj, const char *utf8_name); + +HPy_ID(154) +int HPy_HasAttr(HPyContext *ctx, HPy obj, HPy name); +HPy_ID(155) +int HPy_HasAttr_s(HPyContext *ctx, HPy obj, const char *utf8_name); + +HPy_ID(156) +int HPy_SetAttr(HPyContext *ctx, HPy obj, HPy name, HPy value); +HPy_ID(157) +int HPy_SetAttr_s(HPyContext *ctx, HPy obj, const char *utf8_name, HPy value); + +HPy_ID(158) +HPy HPy_GetItem(HPyContext *ctx, HPy obj, HPy key); +HPy_ID(159) +HPy HPy_GetItem_i(HPyContext *ctx, HPy obj, HPy_ssize_t idx); +HPy_ID(160) +HPy HPy_GetItem_s(HPyContext *ctx, HPy obj, const char *utf8_key); + +/** + * Return the slice of sequence object ``obj`` between ``start`` and ``end``. + * This is the equivalent of the Python expression ``obj[start:end]``. + * + * :param ctx: + * The execution context. + * :param obj: + * A sliceable Python object (must not be ``HPy_NULL`` otherwise a + * ``SystemError`` will be raised). If the object is not sliceable, a + * ``TypeError`` will be raised. + * :param start: + * The start index (inclusive). + * :param end: + * The end index (exclusive). + * + * :returns: + * The requested slice or ``HPy_NULL`` on failure. + */ +HPy_ID(266) +HPy HPy_GetSlice(HPyContext *ctx, HPy obj, HPy_ssize_t start, HPy_ssize_t end); + +HPy_ID(161) +int HPy_Contains(HPyContext *ctx, HPy container, HPy key); + +HPy_ID(162) +int HPy_SetItem(HPyContext *ctx, HPy obj, HPy key, HPy value); +HPy_ID(163) +int HPy_SetItem_i(HPyContext *ctx, HPy obj, HPy_ssize_t idx, HPy value); +HPy_ID(164) +int HPy_SetItem_s(HPyContext *ctx, HPy obj, const char *utf8_key, HPy value); + +/** + * Assign the sequence object ``value`` to the slice in sequence object ``obj`` + * from ``start`` to ``end``. This is the equivalent of the Python statement + * ``obj[start:end] = value``. + * + * :param ctx: + * The execution context. + * :param obj: + * A sliceable Python object (must not be ``HPy_NULL`` otherwise a + * ``SystemError`` will be raised). If the object is not sliceable, a + * ``TypeError`` will be raised. + * :param start: + * The start index (inclusive). + * :param end: + * The end index (exclusive). + * :param value: + * The sequence object to assign (must not be ``HPy_NULL``). + * + * :returns: + * ``0`` on success; ``-1`` on failure + */ +HPy_ID(267) +int HPy_SetSlice(HPyContext *ctx, HPy obj, HPy_ssize_t start, HPy_ssize_t end, HPy value); + +HPy_ID(235) +int HPy_DelItem(HPyContext *ctx, HPy obj, HPy key); +HPy_ID(236) +int HPy_DelItem_i(HPyContext *ctx, HPy obj, HPy_ssize_t idx); +HPy_ID(237) +int HPy_DelItem_s(HPyContext *ctx, HPy obj, const char *utf8_key); + +/** + * Delete the slice of sequence object ``obj`` between ``start`` and ``end``. + * This is the equivalent of the Python statement ``del obj[start:end]``. + * + * :param ctx: + * The execution context. + * :param obj: + * A sliceable Python object (must not be ``HPy_NULL`` otherwise a + * ``SystemError`` will be raised). If the object is not sliceable, a + * ``TypeError`` will be raised. + * :param start: + * The start index (inclusive). + * :param end: + * The end index (exclusive). + * + * :returns: + * ``0`` on success; ``-1`` on failure + */ +HPy_ID(268) +int HPy_DelSlice(HPyContext *ctx, HPy obj, HPy_ssize_t start, HPy_ssize_t end); + +/** + * Returns the type of the given object ``obj``. + * + * On failure, raises ``SystemError`` and returns ``HPy_NULL``. This is + * equivalent to the Python expression``type(obj)``. + * + * :param ctx: + * The execution context. + * :param obj: + * a Python object (must not be ``HPy_NULL``) + * + * :returns: + * The type of ``obj`` or ``HPy_NULL`` in case of errors. + */ +HPy_ID(165) +HPy HPy_Type(HPyContext *ctx, HPy obj); + +/** + * Checks if ``ob`` is an instance of ``type`` or any subtype of ``type``. + * + * :param ctx: + * The execution context. + * :param obj: + * a Python object (must not be ``HPy_NULL``) + * :param type: + * A Python type object. This argument must not be ``HPy_NULL`` and must be + * a type (i.e. it must inherit from Python ``type``). If this is not the + * case, the behavior is undefined (verification of the argument is only + * done in debug mode). + * + * :returns: + * Non-zero if object ``obj`` is an instance of type ``type`` or an instance + * of a subtype of ``type``, and ``0`` otherwise. + */ +HPy_ID(166) +int HPy_TypeCheck(HPyContext *ctx, HPy obj, HPy type); + +/** + * Return the type's name. + * + * Equivalent to getting the type's ``__name__`` attribute. If you want to + * retrieve the type's name as a handle that refers to a ``str``, then just use + * ``HPy_GetAttr_s(ctx, type, "__name__")``. + * + * :param ctx: + * The execution context. + * :param type: + * A Python type object. This argument must not be ``HPy_NULL`` and must be + * a type (i.e. it must inherit from Python ``type``). If this is not the + * case, the behavior is undefined (verification of the argument is only + * done in debug mode). + * + * :returns: + * The name of the type as C string (UTF-8 encoded) or ``NULL`` in case of + * an error. The returned pointer is read-only and guaranteed to be valid as + * long as the handle ``type`` is valid. + */ +HPy_ID(253) +const char *HPyType_GetName(HPyContext *ctx, HPy type); + +/** + * Checks if ``sub`` is a subtype of ``type``. + * + * This function only checks for actual subtypes, which means that + * ``__subclasscheck__()`` is not called on ``type``. + * + * :param ctx: + * The execution context. + * :param sub: + * A Python type object. This argument must not be ``HPy_NULL`` and must be + * a type (i.e. it must inherit from Python ``type``). If this is not the + * case, the behavior is undefined (verification of the argument is only + * done in debug mode). + * :param type: + * A Python type object. This argument must not be ``HPy_NULL`` and must be + * a type (i.e. it must inherit from Python ``type``). If this is not the + * case, the behavior is undefined (verification of the argument is only + * done in debug mode). + * + * :returns: + * Non-zero if ``sub`` is a subtype of ``type``. + */ +HPy_ID(254) +int HPyType_IsSubtype(HPyContext *ctx, HPy sub, HPy type); + +HPy_ID(167) +int HPy_Is(HPyContext *ctx, HPy obj, HPy other); + +HPy_ID(168) +void* _HPy_AsStruct_Object(HPyContext *ctx, HPy h); +HPy_ID(169) +void* _HPy_AsStruct_Legacy(HPyContext *ctx, HPy h); +HPy_ID(228) +void* _HPy_AsStruct_Type(HPyContext *ctx, HPy h); +HPy_ID(229) +void* _HPy_AsStruct_Long(HPyContext *ctx, HPy h); +HPy_ID(230) +void* _HPy_AsStruct_Float(HPyContext *ctx, HPy h); +HPy_ID(231) +void* _HPy_AsStruct_Unicode(HPyContext *ctx, HPy h); +HPy_ID(232) +void* _HPy_AsStruct_Tuple(HPyContext *ctx, HPy h); +HPy_ID(233) +void* _HPy_AsStruct_List(HPyContext *ctx, HPy h); +HPy_ID(264) +void* _HPy_AsStruct_Dict(HPyContext *ctx, HPy h); +HPy_ID(234) +HPyType_BuiltinShape _HPyType_GetBuiltinShape(HPyContext *ctx, HPy h_type); + +HPy_ID(170) +HPy _HPy_New(HPyContext *ctx, HPy h_type, void **data); + +HPy_ID(171) +HPy HPy_Repr(HPyContext *ctx, HPy obj); +HPy_ID(172) +HPy HPy_Str(HPyContext *ctx, HPy obj); +HPy_ID(173) +HPy HPy_ASCII(HPyContext *ctx, HPy obj); +HPy_ID(174) +HPy HPy_Bytes(HPyContext *ctx, HPy obj); + +HPy_ID(175) +HPy HPy_RichCompare(HPyContext *ctx, HPy v, HPy w, int op); +HPy_ID(176) +int HPy_RichCompareBool(HPyContext *ctx, HPy v, HPy w, int op); + +HPy_ID(177) +HPy_hash_t HPy_Hash(HPyContext *ctx, HPy obj); + +/* bytesobject.h */ +HPy_ID(178) +int HPyBytes_Check(HPyContext *ctx, HPy h); +HPy_ID(179) +HPy_ssize_t HPyBytes_Size(HPyContext *ctx, HPy h); +HPy_ID(180) +HPy_ssize_t HPyBytes_GET_SIZE(HPyContext *ctx, HPy h); +HPy_ID(181) +const char* HPyBytes_AsString(HPyContext *ctx, HPy h); +HPy_ID(182) +const char* HPyBytes_AS_STRING(HPyContext *ctx, HPy h); +HPy_ID(183) +HPy HPyBytes_FromString(HPyContext *ctx, const char *bytes); +HPy_ID(184) +HPy HPyBytes_FromStringAndSize(HPyContext *ctx, const char *bytes, HPy_ssize_t len); + +/* unicodeobject.h */ +HPy_ID(185) +HPy HPyUnicode_FromString(HPyContext *ctx, const char *utf8); +HPy_ID(186) +int HPyUnicode_Check(HPyContext *ctx, HPy h); +HPy_ID(187) +HPy HPyUnicode_AsASCIIString(HPyContext *ctx, HPy h); +HPy_ID(188) +HPy HPyUnicode_AsLatin1String(HPyContext *ctx, HPy h); +HPy_ID(189) +HPy HPyUnicode_AsUTF8String(HPyContext *ctx, HPy h); +HPy_ID(190) +const char* HPyUnicode_AsUTF8AndSize(HPyContext *ctx, HPy h, HPy_ssize_t *size); +HPy_ID(191) +HPy HPyUnicode_FromWideChar(HPyContext *ctx, const wchar_t *w, HPy_ssize_t size); +HPy_ID(192) +HPy HPyUnicode_DecodeFSDefault(HPyContext *ctx, const char *v); +HPy_ID(193) +HPy HPyUnicode_DecodeFSDefaultAndSize(HPyContext *ctx, const char *v, HPy_ssize_t size); +HPy_ID(194) +HPy HPyUnicode_EncodeFSDefault(HPyContext *ctx, HPy h); +HPy_ID(195) +HPy_UCS4 HPyUnicode_ReadChar(HPyContext *ctx, HPy h, HPy_ssize_t index); +HPy_ID(196) +HPy HPyUnicode_DecodeASCII(HPyContext *ctx, const char *ascii, HPy_ssize_t size, const char *errors); +HPy_ID(197) +HPy HPyUnicode_DecodeLatin1(HPyContext *ctx, const char *latin1, HPy_ssize_t size, const char *errors); + +/** + * Decode a bytes-like object to a Unicode object. + * + * The bytes of the bytes-like object are decoded according to the given + * encoding and using the error handling defined by ``errors``. + * + * :param ctx: + * The execution context. + * :param obj: + * A bytes-like object. This can be, for example, Python *bytes*, + * *bytearray*, *memoryview*, *array.array* and objects that support the + * Buffer protocol. If this argument is `HPy_NULL``, a ``SystemError`` will + * be raised. If the argument is not a bytes-like object, a ``TypeError`` + * will be raised. + * :param encoding: + * The name (UTF-8 encoded C string) of the encoding to use. If the encoding + * does not exist, a ``LookupError`` will be raised. If this argument is + * ``NULL``, the default encoding ``UTF-8`` will be used. + * :param errors: + * The error handling (UTF-8 encoded C string) to use when decoding. The + * possible values depend on the used encoding. This argument may be + * ``NULL`` in which case it will default to ``"strict"``. + * + * :returns: + * A handle to a ``str`` object created from the decoded bytes or + * ``HPy_NULL`` in case of errors. + */ +HPy_ID(255) +HPy HPyUnicode_FromEncodedObject(HPyContext *ctx, HPy obj, const char *encoding, const char *errors); + +/** + * Return a substring of ``str``, from character index ``start`` (included) to + * character index ``end`` (excluded). + * + * Indices ``start`` and ``end`` must not be negative, otherwise an + * ``IndexError`` will be raised. If ``start >= len(str)`` or if + * ``end < start``, an empty string will be returned. If ``end > len(str)`` then + * ``end == len(str)`` will be assumed. + * + * :param ctx: + * The execution context. + * :param str: + * A Python Unicode object (must not be ``HPy_NULL``). Otherwise, the + * behavior is undefined (verification of the argument is only done in + * debug mode). + * :param start: + * The non-negative start index (inclusive). + * :param end: + * The non-negative end index (exclusive). + * + * :returns: + * The requested substring or ``HPy_NULL`` in case of an error. + */ +HPy_ID(256) +HPy HPyUnicode_Substring(HPyContext *ctx, HPy str, HPy_ssize_t start, HPy_ssize_t end); + +/* listobject.h */ + +/** + * Tests if an object is an instance of a Python list. + * + * :param ctx: + * The execution context. + * :param h: + * A handle to an arbitrary object (must not be ``HPy_NULL``). + * + * :returns: + * Non-zero if object ``h`` is an instance of type ``list`` or an instance + * of a subtype of ``list``, and ``0`` otherwise. + */ +HPy_ID(198) +int HPyList_Check(HPyContext *ctx, HPy h); + +/** + * Creates a new list instance with length ``len``. + * + * :param ctx: + * The execution context. + * :param len: + * A Python list object (must not be ``HPy_NULL``). Otherwise, a + * ``SystemError`` will be raised. + * + * :returns: + * The new list instance on success, or ``HPy_NULL`` on failure. + */ +HPy_ID(199) +HPy HPyList_New(HPyContext *ctx, HPy_ssize_t len); + +/** + * Append item ``h_item`` to list ``h_list``. + * + * :param ctx: + * The execution context. + * :param h_list: + * A Python list object (must not be ``HPy_NULL``). Otherwise, a + * ``SystemError`` will be raised. + * :param h_item: + * The item to append (must not be ``HPy_NULL``). + * + * :returns: + * Return ``0`` if successful; return ``-1`` and set an exception if + * unsuccessful. + */ +HPy_ID(200) +int HPyList_Append(HPyContext *ctx, HPy h_list, HPy h_item); + +/** + * Insert the item ``h_item`` into list ``h_list`` in front of index ``index``. + * + * :param ctx: + * The execution context. + * :param h_list: + * A Python list object (must not be ``HPy_NULL``). Otherwise, a + * ``SystemError`` will be raised. + * :param index: + * The index where the element should be inserted before. A negative index + * is allowed and is then interpreted to be relative to the end of sequence. + * E.g. ``index == -1`` is the last element. + * If ``index < -n`` (where ``n`` is the length of the list), it will be + * replaced by ``0``. If ``index > n``, it will be replaced by ``n``. + * :param h_item: + * The item to insert (must not be ``HPy_NULL``). + * + * :returns: + * Return ``0`` if successful; return ``-1`` and set an exception if + * unsuccessful. + */ +HPy_ID(265) +int HPyList_Insert(HPyContext *ctx, HPy h_list, HPy_ssize_t index, HPy h_item); + +/* dictobject.h */ + +/** + * Tests if an object is an instance of a Python dict. + * + * :param ctx: + * The execution context. + * :param h: + * A handle to an arbitrary object (must not be ``HPy_NULL``). + * + * :returns: + * Non-zero if object ``h`` is an instance of type ``dict`` or an instance + * of a subtype of ``dict``, and ``0`` otherwise. + */ +HPy_ID(201) +int HPyDict_Check(HPyContext *ctx, HPy h); + +/** + * Creates a new empty Python dictionary. + * + * :param ctx: + * The execution context. + * + * :returns: + * A handle to the new and empty Python dictionary or ``HPy_NULL`` in case + * of an error. + */ +HPy_ID(202) +HPy HPyDict_New(HPyContext *ctx); + +/** + * Returns a list of all keys from the dictionary. + * + * Note: This function will directly access the storage of the dict object and + * therefore ignores if method ``keys`` was overwritten. + * + * :param ctx: + * The execution context. + * :param h: + * A Python dict object. If this argument is ``HPy_NULL`` or not an + * instance of a Python dict, a ``SystemError`` will be raised. + * + * :returns: + * A Python list object containing all keys of the given dictionary or + * ``HPy_NULL`` in case of an error. + */ +HPy_ID(257) +HPy HPyDict_Keys(HPyContext *ctx, HPy h); + +/** + * Creates a copy of the provided Python dict object. + * + * :param ctx: + * The execution context. + * :param h: + * A Python dict object. If this argument is ``HPy_NULL`` or not an + * instance of a Python dict, a ``SystemError`` will be raised. + * + * :returns: + * Return a new dictionary that contains the same key-value pairs as ``h`` + * or ``HPy_NULL`` in case of an error. + */ +HPy_ID(258) +HPy HPyDict_Copy(HPyContext *ctx, HPy h); + +/* tupleobject.h */ + +/** + * Tests if an object is an instance of a Python tuple. + * + * :param ctx: + * The execution context. + * :param h: + * A handle to an arbitrary object (must not be ``HPy_NULL``). + * + * :returns: + * Non-zero if object ``h`` is an instance of type ``tuple`` or an instance + * of a subtype of ``tuple``, and ``0`` otherwise. + */ +HPy_ID(203) +int HPyTuple_Check(HPyContext *ctx, HPy h); + +/** + * Create a tuple from an array. + * + * Note: Consider to use the convenience function :c:func:`HPyTuple_Pack` to + * create a tuple. + * + * :param ctx: + * The execution context. + * :param items: + * An array of items to use for initialization of the tuple. + * :param n: + * The number of elements in array ``items``. + * + * :return: + * A new tuple with ``n`` elements or ``HPy_NULL`` in case of an error + * occurred. + */ +HPy_ID(204) +HPy HPyTuple_FromArray(HPyContext *ctx, const HPy items[], HPy_ssize_t n); + +/* sliceobject.h */ + +/** + * Creates a new empty Python slice object. + * + * :param ctx: + * The execution context. + * + * :param start: + * A handle to an object to be used as the slice start value. + * :param end: + * A handle to an object to be used as the slice end value. + * :param step: + * A handle to an object to be used as the slice step value. + * + * :returns: + * A handle to the new and empty Python slice object or ``HPy_NULL`` in case + * of an error. + */ +HPy_ID(272) +HPy HPySlice_New(HPyContext *ctx, HPy start, HPy stop, HPy step); + +/** + * Extract the start, stop and step data members from a slice object as C + * integers. + * + * The slice members may be arbitrary int-like objects. If they are not Python + * int objects, they will be coerced to int objects by calling their + * ``__index__`` method. + * + * If a slice member value is out of bounds, it will be set to the maximum value + * of ``HPy_ssize_t`` if the member was a positive number, or to the minimum + * value of ``HPy_ssize_t`` if it was a negative number. + * + * :param ctx: + * The execution context. + * :param slice: + * A handle to a Python slice object. This argument must be a slice object + * and must not be ``HPy_NULL``. Otherwise, behavior is undefined. + * :param start: + * A pointer to a variable where to write the unpacked slice start. Must not + * be ``NULL``. + * :param end: + * A pointer to a variable where to write the unpacked slice end. Must not + * :param step: + * A pointer to a variable where to write the unpacked slice step. Must not + * be ``NULL``. + * + * :returns: + * ``-1`` on error, ``0`` on success + */ + +HPy_ID(259) +int HPySlice_Unpack(HPyContext *ctx, HPy slice, HPy_ssize_t *start, HPy_ssize_t *stop, HPy_ssize_t *step); + +/* import.h */ +HPy_ID(205) +HPy HPyImport_ImportModule(HPyContext *ctx, const char *utf8_name); + +/* pycapsule.h */ +HPy_ID(244) +HPy HPyCapsule_New(HPyContext *ctx, void *pointer, const char *utf8_name, HPyCapsule_Destructor *destructor); +HPy_ID(245) +void* HPyCapsule_Get(HPyContext *ctx, HPy capsule, _HPyCapsule_key key, const char *utf8_name); +HPy_ID(246) +int HPyCapsule_IsValid(HPyContext *ctx, HPy capsule, const char *utf8_name); +HPy_ID(247) +int HPyCapsule_Set(HPyContext *ctx, HPy capsule, _HPyCapsule_key key, void *value); + +/* integration with the old CPython API */ +HPy_ID(206) +HPy HPy_FromPyObject(HPyContext *ctx, cpy_PyObject *obj); +HPy_ID(207) +cpy_PyObject *HPy_AsPyObject(HPyContext *ctx, HPy h); + +/* internal helpers which need to be exposed to modules for practical reasons :( */ +HPy_ID(208) +void _HPy_CallRealFunctionFromTrampoline(HPyContext *ctx, + HPyFunc_Signature sig, + HPyCFunction func, + void *args); + +/* Builders */ + +/** + * Create a new list builder for ``size`` elements. The builder is then able to + * take at most ``size`` elements. This function does not raise any + * exception (even if running out of memory). + * + * :param ctx: + * The execution context. + * :param size: + * The number of elements to hold. + */ +HPy_ID(209) +HPyListBuilder HPyListBuilder_New(HPyContext *ctx, HPy_ssize_t size); + +/** + * Assign an element to a certain index of the builder. Valid indices are in + * range ``0 <= index < size`` where ``size`` is the value passed to + * :c:func:`HPyListBuilder_New`. This function does not raise any exception. + * + * :param ctx: + * The execution context. + * :param builder: + * A list builder handle. + * :param index: + * The index to assign the object to. + * :param h_item: + * An HPy handle of the object to store or ``HPy_NULL``. Please note that + * HPy **never** steals handles and so, ``h_item`` needs to be closed by + * the caller. + */ +HPy_ID(210) +void HPyListBuilder_Set(HPyContext *ctx, HPyListBuilder builder, + HPy_ssize_t index, HPy h_item); + +/** + * Build a list from a list builder. + * + * :param ctx: + * The execution context. + * :param builder: + * A list builder handle. + * + * :returns: + * An HPy handle to a list containing the values inserted with + * :c:func:`HPyListBuilder_Set` or ``HPy_NULL`` in case an error occurred + * during building or earlier when creating the builder or setting the + * items. + */ +HPy_ID(211) +HPy HPyListBuilder_Build(HPyContext *ctx, HPyListBuilder builder); + +/** + * Cancel building of a tuple and free any acquired resources. + * This function ignores if any error occurred previously when using the tuple + * builder. + * + * :param ctx: + * The execution context. + * :param builder: + * A tuple builder handle. + */ +HPy_ID(212) +void HPyListBuilder_Cancel(HPyContext *ctx, HPyListBuilder builder); + +/** + * Create a new tuple builder for ``size`` elements. The builder is then able + * to take at most ``size`` elements. This function does not raise any + * exception (even if running out of memory). + * + * :param ctx: + * The execution context. + * :param size: + * The number of elements to hold. + */ +HPy_ID(213) +HPyTupleBuilder HPyTupleBuilder_New(HPyContext *ctx, HPy_ssize_t size); + +/** + * Assign an element to a certain index of the builder. Valid indices are in + * range ``0 <= index < size`` where ``size`` is the value passed to + * :c:func:`HPyTupleBuilder_New`. This function does not raise * any exception. + * + * :param ctx: + * The execution context. + * :param builder: + * A tuple builder handle. + * :param index: + * The index to assign the object to. + * :param h_item: + * An HPy handle of the object to store or ``HPy_NULL``. Please note that + * HPy **never** steals handles and so, ``h_item`` needs to be closed by + * the caller. + */ +HPy_ID(214) +void HPyTupleBuilder_Set(HPyContext *ctx, HPyTupleBuilder builder, + HPy_ssize_t index, HPy h_item); + +/** + * Build a tuple from a tuple builder. + * + * :param ctx: + * The execution context. + * :param builder: + * A tuple builder handle. + * + * :returns: + * An HPy handle to a tuple containing the values inserted with + * :c:func:`HPyTupleBuilder_Set` or ``HPy_NULL`` in case an error occurred + * during building or earlier when creating the builder or setting the + * items. + */ +HPy_ID(215) +HPy HPyTupleBuilder_Build(HPyContext *ctx, HPyTupleBuilder builder); + +/** + * Cancel building of a tuple and free any acquired resources. + * This function ignores if any error occurred previously when using the tuple + * builder. + * + * :param ctx: + * The execution context. + * :param builder: + * A tuple builder handle. + */ +HPy_ID(216) +void HPyTupleBuilder_Cancel(HPyContext *ctx, HPyTupleBuilder builder); + +/* Helper for correctly closing handles */ + +HPy_ID(217) +HPyTracker HPyTracker_New(HPyContext *ctx, HPy_ssize_t size); +HPy_ID(218) +int HPyTracker_Add(HPyContext *ctx, HPyTracker ht, HPy h); +HPy_ID(219) +void HPyTracker_ForgetAll(HPyContext *ctx, HPyTracker ht); +HPy_ID(220) +void HPyTracker_Close(HPyContext *ctx, HPyTracker ht); + +/** + * HPyFields should be used ONLY in parts of memory which is known to the GC, + * e.g. memory allocated by HPy_New: + * + * - NEVER declare a local variable of type HPyField + * - NEVER use HPyField on a struct allocated by e.g. malloc() + * + * **CPython's note**: contrary to PyObject*, you don't need to manually + * manage refcounting when using HPyField: if you use HPyField_Store to + * overwrite an existing value, the old object will be automatically decrefed. + * This means that you CANNOT use HPyField_Store to write memory which + * contains uninitialized values, because it would try to decref a dangling + * pointer. + * + * Note that HPy_New automatically zeroes the memory it allocates, so + * everything works well out of the box. In case you are using manually + * allocated memory, you should initialize the HPyField to HPyField_NULL. + * + * Note the difference: + * + * - ``obj->f = HPyField_NULL``: this should be used only to initialize + * uninitialized memory. If you use it to overwrite a valid HPyField, you + * will cause a memory leak (at least on CPython) + * + * - HPyField_Store(ctx, &obj->f, HPy_NULL): this does the right thing and + * decref the old value. However, you CANNOT use it if the memory is not + * initialized. + * + * Note: target_object and source_object are there in case an implementation + * needs to add write and/or read barriers on the objects. They are ignored by + * CPython but e.g. PyPy needs a write barrier. +*/ +HPy_ID(221) +void HPyField_Store(HPyContext *ctx, HPy target_object, HPyField *target_field, HPy h); +HPy_ID(222) +HPy HPyField_Load(HPyContext *ctx, HPy source_object, HPyField source_field); + +/** + * Leaving Python execution: for releasing GIL and other use-cases. + * + * In most situations, users should prefer using convenience macros: + * HPy_BEGIN_LEAVE_PYTHON(context)/HPy_END_LEAVE_PYTHON(context) + * + * HPy extensions may leave Python execution when running Python independent + * code: long-running computations or blocking operations. When an extension + * has left the Python execution it must not call any HPy API other than + * HPy_ReenterPythonExecution. It can access pointers returned by HPy API, + * e.g., HPyUnicode_AsUTF8String, provided that they are valid at the point + * of calling HPy_LeavePythonExecution. + * + * Python execution must be reentered on the same thread as where it was left. + * The leave/enter calls must not be nested. Debug mode will, in the future, + * enforce these constraints. + * + * Python implementations may use this knowledge however they wish. The most + * obvious use case is to release the GIL, in which case the + * HPy_BEGIN_LEAVE_PYTHON/HPy_END_LEAVE_PYTHON becomes equivalent to + * Py_BEGIN_ALLOW_THREADS/Py_END_ALLOW_THREADS. +*/ +HPy_ID(223) +void HPy_ReenterPythonExecution(HPyContext *ctx, HPyThreadState state); +HPy_ID(224) +HPyThreadState HPy_LeavePythonExecution(HPyContext *ctx); + +/** + * HPyGlobal is an alternative to module state. HPyGlobal must be a statically + * allocated C global variable registered in HPyModuleDef.globals array. + * A HPyGlobal can be used only after the HPy module where it is registered was + * created using HPyModule_Create. + * + * HPyGlobal serves as an identifier of a Python object that should be globally + * available per one Python interpreter. Python objects referenced by HPyGlobals + * are destroyed automatically on the interpreter exit (not necessarily the + * process exit). + * + * HPyGlobal instance does not allow anything else but loading and storing + * a HPy handle using a HPyContext. Even if the HPyGlobal C variable may + * be shared between threads or different interpreter instances within one + * process, the API to load and store a handle from HPyGlobal is thread-safe (but + * like any other HPy API must not be called in HPy_LeavePythonExecution blocks). + * + * Given that a handle to object X1 is stored to HPyGlobal using HPyContext of + * Python interpreter I1, then loading a handle from the same HPyGlobal using + * HPyContext of Python interpreter I1 should give a handle to the same object + * X1. Another Python interpreter I2 running within the same process and using + * the same HPyGlobal variable will not be able to load X1 from it, it will have + * its own view on what is stored in the given HPyGlobal. + * + * Python interpreters may use indirection to isolate different interpreter + * instances, but alternative techniques such as copy-on-write or immortal + * objects can be used to avoid that indirection (even selectively on per + * object basis using tagged pointers). + * + * CPython HPy implementation may even provide configuration option that + * switches between a faster version that directly stores PyObject* to + * HPyGlobal but does not support subinterpreters, or a version that supports + * subinterpreters. For now, CPython HPy always stores PyObject* directly + * to HPyGlobal. + * + * While the standard implementation does not fully enforce the documented + * contract, the HPy debug mode will enforce it (not implemented yet). + * + * **Implementation notes:** + * All Python interpreters running in one process must be compatible, because + * they will share all HPyGlobal C level variables. The internal data stored + * in HPyGlobal are specific for each HPy implementation, each implementation + * is also responsible for handling thread-safety when initializing the + * internal data in HPyModule_Create. Note that HPyModule_Create may be called + * concurrently depending on the semantics of the Python implementation (GIL vs + * no GIL) and also depending on the whether there may be multiple instances of + * given Python interpreter running within the same process. In the future, HPy + * ABI may include a contract that internal data of each HPyGlobal must be + * initialized to its address using atomic write and HPy implementations will + * not be free to choose what to store in HPyGlobal, however, this will allow + * multiple different HPy implementations within one process. This contract may + * also be activated only by some runtime option, letting the HPy implementation + * use more optimized HPyGlobal implementation otherwise. +*/ +HPy_ID(225) +void HPyGlobal_Store(HPyContext *ctx, HPyGlobal *global, HPy h); +HPy_ID(226) +HPy HPyGlobal_Load(HPyContext *ctx, HPyGlobal global); + +/* Debugging helpers */ +HPy_ID(227) +void _HPy_Dump(HPyContext *ctx, HPy h); + +/* Evaluating Python statements/expressions */ + +/** + * Parse and compile the Python source code. + * + * :param ctx: + * The execution context. + * :param utf8_source: + * Python source code given as UTF-8 encoded C string (must not be ``NULL``). + * :param utf8_filename: + * The filename (UTF-8 encoded C string) to use for construction of the code + * object. It may appear in tracebacks or in ``SyntaxError`` exception + * messages. + * :param kind: + * The source kind which tells the parser if a single expression, statement, + * or a whole file should be parsed (see enum :c:enum:`HPy_SourceKind`). + * + * :returns: + * A Python code object resulting from the parsed and compiled Python source + * code or ``HPy_NULL`` in case of errors. + */ +HPy_ID(248) +HPy HPy_Compile_s(HPyContext *ctx, const char *utf8_source, const char *utf8_filename, HPy_SourceKind kind); + +/** + * Evaluate a precompiled code object. + * + * Code objects can be compiled from a string using :c:func:`HPy_Compile_s`. + * + * :param ctx: + * The execution context. + * :param code: + * The code object to evaluate. + * :param globals: + * A Python dictionary defining the global variables for the evaluation. + * :param locals: + * A mapping object defining the local variables for the evaluation. + * + * :returns: + * The result produced by the executed code. May be ``HPy_NULL`` in case of + * errors. + */ +HPy_ID(249) +HPy HPy_EvalCode(HPyContext *ctx, HPy code, HPy globals, HPy locals); +HPy_ID(250) +HPy HPyContextVar_New(HPyContext *ctx, const char *name, HPy default_value); +HPy_ID(251) +int32_t HPyContextVar_Get(HPyContext *ctx, HPy context_var, HPy default_value, HPy *result); +HPy_ID(252) +HPy HPyContextVar_Set(HPyContext *ctx, HPy context_var, HPy value); + +/** + * Set the call function for the given object. + * + * By defining slot ``HPy_tp_call`` for some type, instances of this type will + * be callable objects. The specified call function will be used by default for + * every instance. This should account for the most common case (every instance + * of an object uses the same call function) but to still provide the necessary + * flexibility, function ``HPy_SetCallFunction`` allows to set different (maybe + * specialized) call functions for each instance. This must be done in the + * constructor of an object. + * + * A more detailed description on how to use that function can be found in + * section :ref:`porting-guide:calling protocol`. + * + * :param ctx: + * The execution context. + * :param h: + * A handle to an object implementing the call protocol, i.e., the object's + * type must have slot ``HPy_tp_call``. Otherwise, a ``TypeError`` will be + * raised. This argument must not be ``HPy_NULL``. + * :param def: + * A pointer to the call function definition to set (must not be + * ``NULL``). The definition is usually created using + * :c:macro:`HPyDef_CALL_FUNCTION` + * + * :returns: + * ``0`` in case of success and ``-1`` in case of an error. + */ +HPy_ID(260) +int HPy_SetCallFunction(HPyContext *ctx, HPy h, HPyCallFunction *func); + +/* ******* + hpyfunc + ******* + + These typedefs are used to generate the various macros used by + include/common/hpyfunc.h +*/ +typedef HPy (*HPyFunc_noargs)(HPyContext *ctx, HPy self); +typedef HPy (*HPyFunc_o)(HPyContext *ctx, HPy self, HPy arg); +typedef HPy (*HPyFunc_varargs)(HPyContext *ctx, HPy self, const HPy *args, size_t nargs); +typedef HPy (*HPyFunc_keywords)(HPyContext *ctx, HPy self, const HPy *args, + size_t nargs, HPy kwnames); + +typedef HPy (*HPyFunc_unaryfunc)(HPyContext *ctx, HPy); +typedef HPy (*HPyFunc_binaryfunc)(HPyContext *ctx, HPy, HPy); +typedef HPy (*HPyFunc_ternaryfunc)(HPyContext *ctx, HPy, HPy, HPy); +typedef int (*HPyFunc_inquiry)(HPyContext *ctx, HPy); +typedef HPy_ssize_t (*HPyFunc_lenfunc)(HPyContext *ctx, HPy); +typedef HPy (*HPyFunc_ssizeargfunc)(HPyContext *ctx, HPy, HPy_ssize_t); +typedef HPy (*HPyFunc_ssizessizeargfunc)(HPyContext *ctx, HPy, HPy_ssize_t, HPy_ssize_t); +typedef int (*HPyFunc_ssizeobjargproc)(HPyContext *ctx, HPy, HPy_ssize_t, HPy); +typedef int (*HPyFunc_ssizessizeobjargproc)(HPyContext *ctx, HPy, HPy_ssize_t, HPy_ssize_t, HPy); +typedef int (*HPyFunc_objobjargproc)(HPyContext *ctx, HPy, HPy, HPy); +typedef void (*HPyFunc_freefunc)(HPyContext *ctx, void *); +typedef HPy (*HPyFunc_getattrfunc)(HPyContext *ctx, HPy, char *); +typedef HPy (*HPyFunc_getattrofunc)(HPyContext *ctx, HPy, HPy); +typedef int (*HPyFunc_setattrfunc)(HPyContext *ctx, HPy, char *, HPy); +typedef int (*HPyFunc_setattrofunc)(HPyContext *ctx, HPy, HPy, HPy); +typedef HPy (*HPyFunc_reprfunc)(HPyContext *ctx, HPy); +typedef HPy_hash_t (*HPyFunc_hashfunc)(HPyContext *ctx, HPy); +typedef HPy (*HPyFunc_richcmpfunc)(HPyContext *ctx, HPy, HPy, HPy_RichCmpOp); +typedef HPy (*HPyFunc_getiterfunc)(HPyContext *ctx, HPy); +typedef HPy (*HPyFunc_iternextfunc)(HPyContext *ctx, HPy); +typedef HPy (*HPyFunc_descrgetfunc)(HPyContext *ctx, HPy, HPy, HPy); +typedef int (*HPyFunc_descrsetfunc)(HPyContext *ctx, HPy, HPy, HPy); +typedef int (*HPyFunc_initproc)(HPyContext *ctx, HPy self, + const HPy *args, HPy_ssize_t nargs, HPy kw); +typedef HPy (*HPyFunc_newfunc)(HPyContext *ctx, HPy type, const HPy *args, + HPy_ssize_t nargs, HPy kw); +typedef HPy (*HPyFunc_getter)(HPyContext *ctx, HPy, void *); +typedef int (*HPyFunc_setter)(HPyContext *ctx, HPy, HPy, void *); +typedef int (*HPyFunc_objobjproc)(HPyContext *ctx, HPy, HPy); +typedef int (*HPyFunc_getbufferproc)(HPyContext *ctx, HPy, HPy_buffer *, int); +typedef void (*HPyFunc_releasebufferproc)(HPyContext *ctx, HPy, HPy_buffer *); +typedef int (*HPyFunc_traverseproc)(void *object, HPyFunc_visitproc visit, void *arg); +typedef void (*HPyFunc_destructor)(HPyContext *ctx, HPy); + +typedef void (*HPyFunc_destroyfunc)(void *); + +// Note: separate type, because we need a different trampoline +typedef HPy (*HPyFunc_mod_create)(HPyContext *ctx, HPy); + + +/* ~~~ HPySlot_Slot ~~~ + + The following enum is used to generate autogen_hpyslot.h, which contains: + + - The real definition of the enum HPySlot_Slot + + - the macros #define _HPySlot_SIGNATURE_* + +*/ + +// NOTE: if you uncomment/enable a slot below, make sure to write a corresponding +// test in test_slots.py + +/* Note that the magic numbers are the same as CPython */ +typedef enum { + HPy_bf_getbuffer = SLOT(1, HPyFunc_GETBUFFERPROC), + HPy_bf_releasebuffer = SLOT(2, HPyFunc_RELEASEBUFFERPROC), + HPy_mp_ass_subscript = SLOT(3, HPyFunc_OBJOBJARGPROC), + HPy_mp_length = SLOT(4, HPyFunc_LENFUNC), + HPy_mp_subscript = SLOT(5, HPyFunc_BINARYFUNC), + HPy_nb_absolute = SLOT(6, HPyFunc_UNARYFUNC), + HPy_nb_add = SLOT(7, HPyFunc_BINARYFUNC), + HPy_nb_and = SLOT(8, HPyFunc_BINARYFUNC), + HPy_nb_bool = SLOT(9, HPyFunc_INQUIRY), + HPy_nb_divmod = SLOT(10, HPyFunc_BINARYFUNC), + HPy_nb_float = SLOT(11, HPyFunc_UNARYFUNC), + HPy_nb_floor_divide = SLOT(12, HPyFunc_BINARYFUNC), + HPy_nb_index = SLOT(13, HPyFunc_UNARYFUNC), + HPy_nb_inplace_add = SLOT(14, HPyFunc_BINARYFUNC), + HPy_nb_inplace_and = SLOT(15, HPyFunc_BINARYFUNC), + HPy_nb_inplace_floor_divide = SLOT(16, HPyFunc_BINARYFUNC), + HPy_nb_inplace_lshift = SLOT(17, HPyFunc_BINARYFUNC), + HPy_nb_inplace_multiply = SLOT(18, HPyFunc_BINARYFUNC), + HPy_nb_inplace_or = SLOT(19, HPyFunc_BINARYFUNC), + HPy_nb_inplace_power = SLOT(20, HPyFunc_TERNARYFUNC), + HPy_nb_inplace_remainder = SLOT(21, HPyFunc_BINARYFUNC), + HPy_nb_inplace_rshift = SLOT(22, HPyFunc_BINARYFUNC), + HPy_nb_inplace_subtract = SLOT(23, HPyFunc_BINARYFUNC), + HPy_nb_inplace_true_divide = SLOT(24, HPyFunc_BINARYFUNC), + HPy_nb_inplace_xor = SLOT(25, HPyFunc_BINARYFUNC), + HPy_nb_int = SLOT(26, HPyFunc_UNARYFUNC), + HPy_nb_invert = SLOT(27, HPyFunc_UNARYFUNC), + HPy_nb_lshift = SLOT(28, HPyFunc_BINARYFUNC), + HPy_nb_multiply = SLOT(29, HPyFunc_BINARYFUNC), + HPy_nb_negative = SLOT(30, HPyFunc_UNARYFUNC), + HPy_nb_or = SLOT(31, HPyFunc_BINARYFUNC), + HPy_nb_positive = SLOT(32, HPyFunc_UNARYFUNC), + HPy_nb_power = SLOT(33, HPyFunc_TERNARYFUNC), + HPy_nb_remainder = SLOT(34, HPyFunc_BINARYFUNC), + HPy_nb_rshift = SLOT(35, HPyFunc_BINARYFUNC), + HPy_nb_subtract = SLOT(36, HPyFunc_BINARYFUNC), + HPy_nb_true_divide = SLOT(37, HPyFunc_BINARYFUNC), + HPy_nb_xor = SLOT(38, HPyFunc_BINARYFUNC), + HPy_sq_ass_item = SLOT(39, HPyFunc_SSIZEOBJARGPROC), + HPy_sq_concat = SLOT(40, HPyFunc_BINARYFUNC), + HPy_sq_contains = SLOT(41, HPyFunc_OBJOBJPROC), + HPy_sq_inplace_concat = SLOT(42, HPyFunc_BINARYFUNC), + HPy_sq_inplace_repeat = SLOT(43, HPyFunc_SSIZEARGFUNC), + HPy_sq_item = SLOT(44, HPyFunc_SSIZEARGFUNC), + HPy_sq_length = SLOT(45, HPyFunc_LENFUNC), + HPy_sq_repeat = SLOT(46, HPyFunc_SSIZEARGFUNC), + //HPy_tp_alloc = SLOT(47, HPyFunc_X), NOT SUPPORTED + //HPy_tp_base = SLOT(48, HPyFunc_X), + //HPy_tp_bases = SLOT(49, HPyFunc_X), + HPy_tp_call = SLOT(50, HPyFunc_KEYWORDS), + //HPy_tp_clear = SLOT(51, HPyFunc_X), NOT SUPPORTED, use tp_traverse + //HPy_tp_dealloc = SLOT(52, HPyFunc_X), NOT SUPPORTED + //HPy_tp_del = SLOT(53, HPyFunc_X), + HPy_tp_descr_get = SLOT(54, HPyFunc_TERNARYFUNC), + //HPy_tp_descr_set = SLOT(55, HPyFunc_X), + //HPy_tp_doc = SLOT(56, HPyFunc_X), + //HPy_tp_getattr = SLOT(57, HPyFunc_X), + //HPy_tp_getattro = SLOT(58, HPyFunc_X), + HPy_tp_hash = SLOT(59, HPyFunc_HASHFUNC), + HPy_tp_init = SLOT(60, HPyFunc_INITPROC), + //HPy_tp_is_gc = SLOT(61, HPyFunc_X), + //HPy_tp_iter = SLOT(62, HPyFunc_X), + //HPy_tp_iternext = SLOT(63, HPyFunc_X), + //HPy_tp_methods = SLOT(64, HPyFunc_X), NOT SUPPORTED + HPy_tp_new = SLOT(65, HPyFunc_NEWFUNC), + HPy_tp_repr = SLOT(66, HPyFunc_REPRFUNC), + HPy_tp_richcompare = SLOT(67, HPyFunc_RICHCMPFUNC), + //HPy_tp_setattr = SLOT(68, HPyFunc_X), + //HPy_tp_setattro = SLOT(69, HPyFunc_X), + HPy_tp_str = SLOT(70, HPyFunc_REPRFUNC), + HPy_tp_traverse = SLOT(71, HPyFunc_TRAVERSEPROC), + //HPy_tp_members = SLOT(72, HPyFunc_X), NOT SUPPORTED + //HPy_tp_getset = SLOT(73, HPyFunc_X), NOT SUPPORTED + //HPy_tp_free = SLOT(74, HPyFunc_X), NOT SUPPORTED + HPy_nb_matrix_multiply = SLOT(75, HPyFunc_BINARYFUNC), + HPy_nb_inplace_matrix_multiply = SLOT(76, HPyFunc_BINARYFUNC), + //HPy_am_await = SLOT(77, HPyFunc_X), + //HPy_am_aiter = SLOT(78, HPyFunc_X), + //HPy_am_anext = SLOT(79, HPyFunc_X), + HPy_tp_finalize = SLOT(80, HPyFunc_DESTRUCTOR), + + /* extra HPy slots */ + HPy_tp_destroy = SLOT(1000, HPyFunc_DESTROYFUNC), + + /** + * Module create slot: the function receives loader spec and should + * return an HPy handle representing the module. Currently, creating + * real module objects cannot be done by user code, so the only other + * useful thing that this slot can do is to create another object that + * can work as a module, such as SimpleNamespace. + */ + HPy_mod_create = SLOT(2000, HPyFunc_MOD_CREATE), + /** + * Module exec slot: the function receives module object that was created + * by the runtime from HPyModuleDef. This slot can do any initialization + * of the module, such as adding types. There can be multiple exec slots + * and they will be executed in the declaration order. + */ + HPy_mod_exec = SLOT(2001, HPyFunc_INQUIRY), + +} HPySlot_Slot; diff --git a/graalpython/hpy/hpy/tools/autogen/pypy.py b/graalpython/hpy/hpy/tools/autogen/pypy.py new file mode 100644 index 0000000000..702a519e84 --- /dev/null +++ b/graalpython/hpy/hpy/tools/autogen/pypy.py @@ -0,0 +1,40 @@ +from .autogenfile import AutoGenFile +from .parse import toC + +# this class should probably be moved somewhere in the PyPy repo +class autogen_pypy_txt(AutoGenFile): + PATH = 'hpy/tools/autogen/autogen_pypy.txt' + LANGUAGE = 'txt' # to avoid inserting the disclaimer + + def generate(self): + lines = [] + w = lines.append + w("typedef struct _HPyContext_s {") + w(" int abi_version;") + for var in self.api.variables: + w(" struct _HPy_s %s;" % var.ctx_name()) + for func in self.api.functions: + w(" void * %s;" % func.ctx_name()) + w("} _struct_HPyContext_s;") + w("") + w("") + # generate stubs for all the API functions + for func in self.api.functions: + w(self.stub(func)) + return '\n'.join(lines) + + def stub(self, func): + signature = toC(func.node) + if func.is_varargs(): + return '# %s' % signature + # + argnames = [p.name for p in func.node.type.args.params] + lines = [] + w = lines.append + w('@API.func("%s")' % signature) + w('def %s(space, %s):' % (func.name, ', '.join(argnames))) + w(' from rpython.rlib.nonconst import NonConstant # for the annotator') + w(' if NonConstant(False): return 0') + w(' raise NotImplementedError') + w('') + return '\n'.join(lines) diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/__init__.py b/graalpython/hpy/hpy/tools/autogen/testing/__init__.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/__init__.py rename to graalpython/hpy/hpy/tools/autogen/testing/__init__.py diff --git a/graalpython/hpy/hpy/tools/autogen/testing/test_autogen.py b/graalpython/hpy/hpy/tools/autogen/testing/test_autogen.py new file mode 100644 index 0000000000..2e1a971d63 --- /dev/null +++ b/graalpython/hpy/hpy/tools/autogen/testing/test_autogen.py @@ -0,0 +1,224 @@ +import textwrap +import difflib +import py +import pytest +from hpy.tools.autogen.parse import HPyAPI +from hpy.tools.autogen.ctx import autogen_ctx_h, autogen_ctx_def_h +from hpy.tools.autogen.trampolines import (autogen_trampolines_h, + cpython_autogen_api_impl_h) +from hpy.tools.autogen.hpyslot import autogen_hpyslot_h + +def src_equal(exp, got): + # try to compare two C sources, ignoring whitespace + exp = textwrap.dedent(exp).strip() + got = textwrap.dedent(got).strip() + if exp.split() != got.split(): + diff = difflib.unified_diff(exp.splitlines(), got.splitlines(), + fromfile='expected', + tofile='got') + print() + for line in diff: + print(line) + return False + return True + +@pytest.mark.usefixtures('initargs') +class BaseTestAutogen: + + @pytest.fixture + def initargs(self, tmpdir): + self.tmpdir = tmpdir + + def parse(self, src): + fname = self.tmpdir.join('test_api.h') + # automatically add useful typedefs + src = """ + #define STRINGIFY(X) #X + #define HPy_ID(X) _Pragma(STRINGIFY(id=X)) \\ + + typedef int HPy; + typedef int HPyContext; + """ + src + fname.write(src) + return HPyAPI.parse(fname) + + +class TestHPyAPI(BaseTestAutogen): + + def test_ctx_name(self): + api = self.parse(""" + HPy_ID(0) HPy h_None; + HPy_ID(1) HPy HPy_Dup(HPyContext *ctx, HPy h); + HPy_ID(2) void* _HPy_AsStruct(HPyContext *ctx, HPy h); + """) + assert api.get_var('h_None').ctx_name() == 'h_None' + assert api.get_func('HPy_Dup').ctx_name() == 'ctx_Dup' + assert api.get_func('_HPy_AsStruct').ctx_name() == 'ctx_AsStruct' + + def test_cpython_name(self): + api = self.parse(""" + HPy_ID(0) HPy HPy_Dup(HPyContext *ctx, HPy h); + HPy_ID(1) long HPyLong_AsLong(HPyContext *ctx, HPy h); + HPy_ID(2) HPy HPy_Add(HPyContext *ctx, HPy h1, HPy h2); + """) + assert api.get_func('HPy_Dup').cpython_name is None + assert api.get_func('HPyLong_AsLong').cpython_name == 'PyLong_AsLong' + assert api.get_func('HPy_Add').cpython_name == 'PyNumber_Add' + + def test_hpyslot(self): + api = self.parse(""" + typedef enum { + HPy_nb_add = SLOT(7, HPyFunc_BINARYFUNC), + HPy_tp_repr = SLOT(66, HPyFunc_REPRFUNC), + } HPySlot_Slot; + """) + nb_add = api.get_slot('HPy_nb_add') + assert nb_add.value == '7' + assert nb_add.hpyfunc == 'HPyFunc_BINARYFUNC' + # + tp_repr = api.get_slot('HPy_tp_repr') + assert tp_repr.value == '66' + assert tp_repr.hpyfunc == 'HPyFunc_REPRFUNC' + + def test_parse_id(self): + api = self.parse(""" + HPy_ID(0) HPy h_Foo; + HPy_ID(1) + long HPyFoo_Bar(HPyContext *ctx, HPy h); + """) + assert len(api.variables) == 1 + assert len(api.functions) == 1 + assert api.variables[0].ctx_index == 0 + assert api.functions[0].ctx_index == 1 + + # don't allow gaps in the sequence of IDs + with pytest.raises(AssertionError): + self.parse(""" + HPy_ID(0) HPy h_Foo; + HPy_ID(3) long HPyFoo_Bar(HPyContext *ctx, HPy h); + """) + + # don't allow re-using of IDs + with pytest.raises(AssertionError): + self.parse(""" + HPy_ID(0) HPy h_Foo; + HPy_ID(0) HPy h_Foo; + """) + + # all context members must have an ID + with pytest.raises(ValueError): + self.parse("HPy h_Foo;") + + # pragmas must be of form '#pramga key=value' + with pytest.raises(ValueError): + self.parse("#pragma hello\nHPy h_Foo;") + + # pragmas value must be an integer + with pytest.raises(ValueError): + self.parse("#pragma hello=world\nHPy h_Foo;") + +class TestAutoGen(BaseTestAutogen): + + def test_autogen_ctx_h(self): + api = self.parse(""" + HPy_ID(0) HPy h_None; + HPy_ID(1) HPy HPy_Add(HPyContext *ctx, HPy h1, HPy h2); + """) + got = autogen_ctx_h(api).generate() + exp = """ + struct _HPyContext_s { + const char *name; // used just to make debugging and testing easier + void *_private; // used by implementations to store custom data + int abi_version; + HPy h_None; + HPy (*ctx_Add)(HPyContext *ctx, HPy h1, HPy h2); + }; + """ + assert src_equal(exp, got) + + def test_autogen_ctx_def_h(self): + api = self.parse(""" + HPy_ID(0) HPy h_None; + HPy_ID(1) HPy HPy_Add(HPyContext *ctx, HPy h1, HPy h2); + """) + got = autogen_ctx_def_h(api).generate() + exp = """ + struct _HPyContext_s g_universal_ctx = { + .name = "HPy Universal ABI (CPython backend)", + ._private = NULL, + .abi_version = HPY_ABI_VERSION, + /* h_None & co. are initialized by init_universal_ctx() */ + .ctx_Add = &ctx_Add, + }; + """ + assert src_equal(exp, got) + + def test_autogen_trampolines_h(self): + api = self.parse(""" + HPy_ID(0) HPy HPy_Add(HPyContext *ctx, HPy h1, HPy h2); + HPy_ID(1) void HPy_Close(HPyContext *ctx, HPy h); + HPy_ID(2) void* _HPy_AsStruct(HPyContext *ctx, HPy h); + """) + got = autogen_trampolines_h(api).generate() + exp = """ + HPyAPI_FUNC HPy HPy_Add(HPyContext *ctx, HPy h1, HPy h2) { + return ctx->ctx_Add ( ctx, h1, h2 ); + } + + HPyAPI_FUNC void HPy_Close(HPyContext *ctx, HPy h) { + ctx->ctx_Close ( ctx, h ); + } + + HPyAPI_FUNC void *_HPy_AsStruct(HPyContext *ctx, HPy h) { + return ctx->ctx_AsStruct ( ctx, h ); + } + """ + assert src_equal(got, exp) + + def test_cpython_api_impl_h(self): + api = self.parse(""" + HPy_ID(0) HPy HPy_Dup(HPyContext *ctx, HPy h); + HPy_ID(1) HPy HPy_Add(HPyContext *ctx, HPy h1, HPy h2); + HPy_ID(2) HPy HPyLong_FromLong(HPyContext *ctx, long value); + HPy_ID(3) char* HPyBytes_AsString(HPyContext *ctx, HPy h); + """) + got = cpython_autogen_api_impl_h(api).generate() + exp = """ + HPyAPI_FUNC + HPy HPy_Add(HPyContext *ctx, HPy h1, HPy h2) + { + return _py2h(PyNumber_Add(_h2py(h1), _h2py(h2))); + } + + HPyAPI_FUNC + HPy HPyLong_FromLong(HPyContext *ctx, long value) + { + return _py2h(PyLong_FromLong(value)); + } + + HPyAPI_FUNC + char *HPyBytes_AsString(HPyContext *ctx, HPy h) + { + return PyBytes_AsString(_h2py(h)); + } + """ + assert src_equal(got, exp) + + def test_autogen_hpyslot_h(self): + api = self.parse(""" + typedef enum { + HPy_nb_add = SLOT(7, HPyFunc_BINARYFUNC), + HPy_tp_repr = SLOT(66, HPyFunc_REPRFUNC), + } HPySlot_Slot; + """) + got = autogen_hpyslot_h(api).generate() + exp = """ + typedef enum { + HPy_nb_add = 7, + HPy_tp_repr = 66, + } HPySlot_Slot; + + #define _HPySlot_SIG__HPy_nb_add HPyFunc_BINARYFUNC + #define _HPySlot_SIG__HPy_tp_repr HPyFunc_REPRFUNC + """ + assert src_equal(got, exp) diff --git a/graalpython/hpy/hpy/tools/autogen/testing/test_hpyfunc.py b/graalpython/hpy/hpy/tools/autogen/testing/test_hpyfunc.py new file mode 100644 index 0000000000..7e0032979d --- /dev/null +++ b/graalpython/hpy/hpy/tools/autogen/testing/test_hpyfunc.py @@ -0,0 +1,145 @@ +from hpy.tools.autogen.hpyfunc import autogen_hpyfunc_declare_h +from hpy.tools.autogen.hpyfunc import autogen_hpyfunc_trampoline_h +from hpy.tools.autogen.hpyfunc import autogen_ctx_call_i +from hpy.tools.autogen.hpyfunc import autogen_cpython_hpyfunc_trampoline_h +from hpy.tools.autogen.testing.test_autogen import BaseTestAutogen, src_equal + +class TestHPyFunc(BaseTestAutogen): + + def test_parse(self): + api = self.parse(""" + typedef int HPyFunc_Signature; + typedef HPy (*HPyFunc_noargs)(HPyContext *ctx, HPy self); + """) + assert len(api.hpyfunc_typedefs) == 1 + hpyfunc = api.get_hpyfunc_typedef('HPyFunc_noargs') + assert hpyfunc.name == 'HPyFunc_noargs' + assert hpyfunc.base_name() == 'noargs' + + def test_autogen_hpyfunc_declare_h(self): + api = self.parse(""" + typedef HPy (*HPyFunc_foo)(HPyContext *ctx, HPy self); + typedef HPy (*HPyFunc_bar)(HPyContext *ctx, HPy, int); + """) + got = autogen_hpyfunc_declare_h(api).generate() + exp = """ + #define _HPyFunc_DECLARE_HPyFunc_FOO(SYM) static HPy SYM(HPyContext *ctx, HPy self) + #define _HPyFunc_DECLARE_HPyFunc_BAR(SYM) static HPy SYM(HPyContext *ctx, HPy, int) + + typedef HPy (*HPyFunc_foo)(HPyContext *ctx, HPy self); + typedef HPy (*HPyFunc_bar)(HPyContext *ctx, HPy, int); + """ + assert src_equal(got, exp) + + def test_autogen_hpyfunc_trampoline_h(self): + api = self.parse(""" + typedef HPy (*HPyFunc_foo)(HPyContext *ctx, HPy arg, int xy); + typedef HPy (*HPyFunc_bar)(HPyContext *ctx, HPy, int); + typedef void (*HPyFunc_proc)(HPyContext *ctx, int x); + """) + got = autogen_hpyfunc_trampoline_h(api).generate() + exp = r""" + typedef struct { + cpy_PyObject *arg; + int xy; + cpy_PyObject * result; + } _HPyFunc_args_FOO; + + #define _HPyFunc_TRAMPOLINE_HPyFunc_FOO(SYM, IMPL) \ + static cpy_PyObject *SYM(cpy_PyObject *arg, int xy) \ + { \ + _HPyFunc_args_FOO a = { arg, xy }; \ + _HPy_CallRealFunctionFromTrampoline( \ + _ctx_for_trampolines, HPyFunc_FOO, (HPyCFunction)IMPL, &a); \ + return a.result; \ + } + + typedef struct { + cpy_PyObject *arg0; + int arg1; + cpy_PyObject * result; + } _HPyFunc_args_BAR; + + #define _HPyFunc_TRAMPOLINE_HPyFunc_BAR(SYM, IMPL) \ + static cpy_PyObject *SYM(cpy_PyObject *arg0, int arg1) \ + { \ + _HPyFunc_args_BAR a = { arg0, arg1 }; \ + _HPy_CallRealFunctionFromTrampoline( \ + _ctx_for_trampolines, HPyFunc_BAR, (HPyCFunction)IMPL, &a); \ + return a.result; \ + } + + typedef struct { + int x; + } _HPyFunc_args_PROC; + + #define _HPyFunc_TRAMPOLINE_HPyFunc_PROC(SYM, IMPL) \ + static void SYM(int x) \ + { \ + _HPyFunc_args_PROC a = { x }; \ + _HPy_CallRealFunctionFromTrampoline( \ + _ctx_for_trampolines, HPyFunc_PROC, (HPyCFunction)IMPL, &a); \ + return; \ + } + """ + assert src_equal(got, exp) + + def test_autogen_ctx_call_i(self): + api = self.parse(""" + typedef HPy (*HPyFunc_foo)(HPyContext *ctx, HPy arg, int xy); + typedef int (*HPyFunc_bar)(HPyContext *ctx); + typedef int (*HPyFunc_baz)(HPyContext *ctx, HPy, int); + typedef void (*HPyFunc_proc)(HPyContext *ctx, int x); + """) + got = autogen_ctx_call_i(api).generate() + exp = r""" + case HPyFunc_FOO: { + HPyFunc_foo f = (HPyFunc_foo)func; + _HPyFunc_args_FOO *a = (_HPyFunc_args_FOO*)args; + a->result = _h2py(f(ctx, _py2h(a->arg), a->xy)); + return; + } + case HPyFunc_BAR: { + HPyFunc_bar f = (HPyFunc_bar)func; + _HPyFunc_args_BAR *a = (_HPyFunc_args_BAR*)args; + a->result = f(ctx); + return; + } + case HPyFunc_BAZ: { + HPyFunc_baz f = (HPyFunc_baz)func; + _HPyFunc_args_BAZ *a = (_HPyFunc_args_BAZ*)args; + a->result = f(ctx, _py2h(a->arg0), a->arg1); + return; + } + case HPyFunc_PROC: { + HPyFunc_proc f = (HPyFunc_proc)func; + _HPyFunc_args_PROC *a = (_HPyFunc_args_PROC*)args; + f(ctx, a->x); + return; + } + """ + assert src_equal(got, exp) + + def test_autogen_cpython_hpyfunc_trampoline_h(self): + api = self.parse(""" + typedef HPy (*HPyFunc_foo)(HPyContext *ctx, HPy arg, int xy); + typedef HPy (*HPyFunc_bar)(HPyContext *ctx, HPy, int); + """) + got = autogen_cpython_hpyfunc_trampoline_h(api).generate() + exp = r""" + typedef HPy (*_HPyCFunction_FOO)(HPyContext *, HPy, int); + #define _HPyFunc_TRAMPOLINE_HPyFunc_FOO(SYM, IMPL) \ + static cpy_PyObject *SYM(cpy_PyObject *arg, int xy) \ + { \ + _HPyCFunction_FOO func = (_HPyCFunction_FOO)IMPL; \ + return _h2py(func(_HPyGetContext(), _py2h(arg), xy)); \ + } + typedef HPy (*_HPyCFunction_BAR)(HPyContext *, HPy, int); + #define _HPyFunc_TRAMPOLINE_HPyFunc_BAR(SYM, IMPL) \ + static cpy_PyObject *SYM(cpy_PyObject *arg0, int arg1) \ + { \ + _HPyCFunction_BAR func = (_HPyCFunction_BAR)IMPL; \ + return _h2py(func(_HPyGetContext(), _py2h(arg0), arg1)); \ + } + """ + assert src_equal(got, exp) diff --git a/graalpython/hpy/hpy/tools/autogen/trace.py b/graalpython/hpy/hpy/tools/autogen/trace.py new file mode 100644 index 0000000000..5cdc212081 --- /dev/null +++ b/graalpython/hpy/hpy/tools/autogen/trace.py @@ -0,0 +1,182 @@ +from copy import deepcopy +from pycparser import c_ast +from .autogenfile import AutoGenFile +from .parse import toC, find_typedecl, get_context_return_type, \ + maybe_make_void, make_void, get_return_constant + + +# We will call the delegate context's function but we still need to unwrap +# the context. This is in contrast to, e.g., the debug mode, where you would +# manually write a wrapper function. Here we can generate that as well. +NO_WRAPPER = { + '_HPy_CallRealFunctionFromTrampoline', + 'HPy_FatalError', +} + +class Ctx2TctxVisitor(c_ast.NodeVisitor): + """Visitor which renames all ctx to tctx""" + + def visit_TypeDecl(self, node): + if node.declname == 'ctx': + node.declname = 'tctx' + self.generic_visit(node) + +def funcnode_with_new_name(node, name): + newnode = deepcopy(node) + typedecl = find_typedecl(newnode) + typedecl.declname = name + return newnode + +def get_trace_wrapper_node(func): + newnode = funcnode_with_new_name(func.node, 'trace_%s' % func.ctx_name()) + maybe_make_void(func, newnode) + # rename ctx to tctx + visitor = Ctx2TctxVisitor() + visitor.visit(newnode) + return newnode + +class autogen_tracer_ctx_init_h(AutoGenFile): + PATH = 'hpy/trace/src/autogen_trace_ctx_init.h' + + def generate(self): + lines = [] + w = lines.append + # emit the declarations for all the trace_ctx_* functions + for func in self.api.functions: + if func.name not in NO_WRAPPER: + w(toC(get_trace_wrapper_node(func)) + ';') + n_decls = len(self.api.functions) + len(self.api.variables) + w('') + w(f'static inline void trace_ctx_init_info(HPyTraceInfo *info, HPyContext *uctx)') + w('{') + w(f' info->magic_number = HPY_TRACE_MAGIC;') + w(f' info->uctx = uctx;') + w(f' info->call_counts = (uint64_t *)calloc({n_decls}, sizeof(uint64_t));') + w(f' info->durations = (_HPyTime_t *)calloc({n_decls}, sizeof(_HPyTime_t));') + w(f' info->on_enter_func = HPy_NULL;') + w(f' info->on_exit_func = HPy_NULL;') + w('}') + w('') + w(f'static inline void trace_ctx_free_info(HPyTraceInfo *info)') + w('{') + w(f' assert(info->magic_number == HPY_TRACE_MAGIC);') + w(f' free(info->call_counts);') + w(f' free(info->durations);') + w(f' HPy_Close(info->uctx, info->on_enter_func);') + w(f' HPy_Close(info->uctx, info->on_exit_func);') + w('}') + w('') + w(f'static inline void trace_ctx_init_fields(HPyContext *tctx, HPyContext *uctx)') + w('{') + for var in self.api.variables: + name = var.name + w(f' tctx->{name} = uctx->{name};') + for func in self.api.functions: + if func.name in NO_WRAPPER: + name = func.ctx_name() + w(f' tctx->{name} = uctx->{name};') + else: + name = func.ctx_name() + w(f' tctx->{name} = &trace_{name};') + w('}') + return '\n'.join(lines) + + +class autogen_tracer_wrappers(AutoGenFile): + PATH = 'hpy/trace/src/autogen_trace_wrappers.c' + + def generate(self): + lines = [] + w = lines.append + w('#include "trace_internal.h"') + w('') + for func in self.api.functions: + debug_wrapper = self.gen_trace_wrapper(func) + if debug_wrapper: + w(debug_wrapper) + w('') + return '\n'.join(lines) + + def gen_trace_wrapper(self, func): + if func.name in NO_WRAPPER: + return + + assert not func.is_varargs() + node = get_trace_wrapper_node(func) + const_return = get_return_constant(func) + if const_return: + make_void(node) + signature = toC(node) + rettype = get_context_return_type(node, const_return) + + def get_params(): + lst = [] + for p in node.type.args.params: + if p.name == 'ctx': + lst.append('uctx') + else: + lst.append(p.name) + return ', '.join(lst) + params = get_params() + + lines = [] + w = lines.append + w(signature) + w('{') + w(f' HPyTraceInfo *info = hpy_trace_on_enter(tctx, {func.ctx_index});') + w(f' HPyContext *uctx = info->uctx;') + w(f' _HPyTime_t _ts_start, _ts_end;') + w(f' _HPyClockStatus_t r0, r1;') + w(f' r0 = get_monotonic_clock(&_ts_start);') + if rettype == 'void': + w(f' {func.name}({params});') + else: + w(f' {rettype} res = {func.name}({params});') + w(f' r1 = get_monotonic_clock(&_ts_end);') + w(f' hpy_trace_on_exit(info, {func.ctx_index}, r0, r1, &_ts_start, &_ts_end);') + if rettype != 'void': + w(f' return res;') + w('}') + return '\n'.join(lines) + + +class autogen_trace_func_table_c(AutoGenFile): + PATH = 'hpy/trace/src/autogen_trace_func_table.c' + + def generate(self): + lines = [] + w = lines.append + + n_funcs = len(self.api.functions) + n_decls = n_funcs + len(self.api.variables) + func_table = ['NO_FUNC'] * n_decls + for func in self.api.functions: + name = func.ctx_name() + func_table[func.ctx_index] = f'"{name}"' + + w('#include "trace_internal.h"') + w('') + w(f'#define TRACE_NFUNC {n_funcs}') + w('') + w('#define NO_FUNC ""') + w('static const char *trace_func_table[] = {') + for func in func_table: + w(f' {func},') + w(f' NULL /* sentinel */') + w('};') + w('') + w('int hpy_trace_get_nfunc(void)') + w('{') + w(' return TRACE_NFUNC;') + w('}') + w('') + w('const char * hpy_trace_get_func_name(int idx)') + w('{') + w(f' if (idx >= 0 && idx < {n_decls})') + w(' return trace_func_table[idx];') + w(' return NULL;') + w('}') + w('') + return '\n'.join(lines) + + diff --git a/graalpython/hpy/hpy/tools/autogen/trampolines.py b/graalpython/hpy/hpy/tools/autogen/trampolines.py new file mode 100644 index 0000000000..366f27cd4a --- /dev/null +++ b/graalpython/hpy/hpy/tools/autogen/trampolines.py @@ -0,0 +1,140 @@ +from copy import deepcopy +from .autogenfile import AutoGenFile +from .parse import toC,find_typedecl, get_context_return_type, \ + make_void, get_return_constant +from . import conf + + +class autogen_trampolines_h(AutoGenFile): + PATH = 'hpy/devel/include/hpy/universal/autogen_trampolines.h' + + def generate(self): + lines = [] + for func in self.api.functions: + trampoline = self.gen_trampoline(func) + if trampoline: + lines.append(trampoline) + lines.append('') + return '\n'.join(lines) + + def gen_trampoline(self, func): + # HPyAPI_FUNC HPy HPyModule_Create(HPyContext *ctx, HPyModuleDef *def) { + # return ctx->ctx_Module_Create ( ctx, def ); + # } + if func.name in conf.NO_TRAMPOLINES: + return None + const_return = get_return_constant(func) + rettype = get_context_return_type(func.node, const_return) + parts = [] + w = parts.append + w('HPyAPI_FUNC') + w(toC(func.node)) + w('{\n ') + + # trampolines cannot deal with varargs easily + assert not func.is_varargs() + + if rettype == 'void': + w('ctx->%s' % func.ctx_name()) + else: + w('return ctx->%s' % func.ctx_name()) + w('(') + params = [p.name for p in func.node.type.args.params] + w(', '.join(params)) + w(');') + + if const_return: + w('return %s;' % const_return) + + w('\n}') + return ' '.join(parts) + + +class cpython_autogen_api_impl_h(AutoGenFile): + PATH = 'hpy/devel/include/hpy/cpython/autogen_api_impl.h' + GENERATE_CONST_RETURN = True + + def signature(self, func, const_return): + """ + Return the C signature of the impl function. + + In CPython mode, the name it's the same as in public_api: + HPy_Add ==> HPyAPI_FUNC HPy_Add + HPyLong_FromLong ==> HPyAPI_FUNC HPyLong_FromLong + + See also universal_autogen_ctx_impl_h. + """ + sig = toC(func.node) + return 'HPyAPI_FUNC %s' % sig + + def generate(self): + lines = [] + for func in self.api.functions: + if not func.cpython_name: + continue + lines.append(self.gen_implementation(func)) + lines.append('') + return '\n'.join(lines) + + def gen_implementation(self, func): + def call(pyfunc, return_type): + # return _py2h(PyNumber_Add(_h2py(x), _h2py(y))) + args = [] + for p in func.node.type.args.params: + if toC(p.type) == 'HPyContext *': + continue + elif toC(p.type) == 'HPy': + arg = '_h2py(%s)' % p.name + elif toC(p.type) == 'HPyThreadState': + arg = '_h2threads(%s)' % p.name + else: + arg = p.name + args.append(arg) + result = '%s(%s)' % (pyfunc, ', '.join(args)) + if return_type == 'HPy': + result = '_py2h(%s)' % result + elif return_type == 'HPyThreadState': + result = '_threads2h(%s)' % result + return result + # + lines = [] + w = lines.append + pyfunc = func.cpython_name + if not pyfunc: + raise ValueError(f"Cannot generate implementation for {self}") + const_return = get_return_constant(func) + return_type = get_context_return_type(func.node, const_return) + return_stmt = '' if return_type == 'void' else 'return ' + w(self.signature(func, const_return)) + w('{') + w(' %s%s;' % (return_stmt, call(pyfunc, return_type))) + + if self.GENERATE_CONST_RETURN and const_return: + w(' return %s;' % const_return) + + w('}') + return '\n'.join(lines) + + +class universal_autogen_ctx_impl_h(cpython_autogen_api_impl_h): + PATH = 'hpy/universal/src/autogen_ctx_impl.h' + GENERATE_CONST_RETURN = False + + def signature(self, func, const_return): + """ + Return the C signature of the impl function. + + In Universal mode, the name is prefixed by ctx_: + HPy_Add ==> HPyAPI_IMPL ctx_Add + HPyLong_FromLong ==> HPyAPI_IMPL ctx_Long_FromLong + + See also cpython_autogen_api_impl_h. + """ + newnode = deepcopy(func.node) + if const_return: + make_void(newnode) + typedecl = find_typedecl(newnode) + # rename the function + typedecl.declname = func.ctx_name() + sig = toC(newnode) + return 'HPyAPI_IMPL %s' % sig diff --git a/graalpython/hpy/hpy/tools/include_path.py b/graalpython/hpy/hpy/tools/include_path.py new file mode 100644 index 0000000000..61dd06a28d --- /dev/null +++ b/graalpython/hpy/hpy/tools/include_path.py @@ -0,0 +1,4 @@ +"""Prints the include path for the current Python interpreter.""" + +from sysconfig import get_paths as gp +print(gp()['include']) diff --git a/graalpython/hpy/hpy/tools/valgrind/hpy.supp b/graalpython/hpy/hpy/tools/valgrind/hpy.supp new file mode 100644 index 0000000000..c1139552a8 --- /dev/null +++ b/graalpython/hpy/hpy/tools/valgrind/hpy.supp @@ -0,0 +1,44 @@ +{ + <_HPyModuleDef_CreatePyModuleDef_leak> + Memcheck:Leak + match-leak-kinds: definite,indirect + ... + fun:_HPyModuleDef_CreatePyModuleDef + ... +} + +{ + + Memcheck:Leak + match-leak-kinds: definite + ... + fun:ctx_Type_FromSpec + ... +} + +{ + + Memcheck:Leak + match-leak-kinds: definite + ... + fun:PyUnicode_New.part.42 + ... +} + +{ + + Memcheck:Leak + match-leak-kinds: definite + ... + fun:PyUnicode_New + ... +} + +{ + + Memcheck:Leak + match-leak-kinds: definite + ... + fun:PyLong_FromLong + ... +} diff --git a/graalpython/hpy/hpy/tools/valgrind/python.supp b/graalpython/hpy/hpy/tools/valgrind/python.supp new file mode 100644 index 0000000000..26a6f22c89 --- /dev/null +++ b/graalpython/hpy/hpy/tools/valgrind/python.supp @@ -0,0 +1,389 @@ +# +# This is a valgrind suppression file that should be used when using valgrind. +# +# Here's an example of running valgrind: +# +# cd python/dist/src +# valgrind --tool=memcheck --suppressions=Misc/valgrind-python.supp \ +# ./python -E -tt ./Lib/test/regrtest.py -u bsddb,network +# +# You must edit Objects/obmalloc.c and uncomment Py_USING_MEMORY_DEBUGGER +# to use the preferred suppressions with Py_ADDRESS_IN_RANGE. +# +# If you do not want to recompile Python, you can uncomment +# suppressions for PyObject_Free and PyObject_Realloc. +# +# See Misc/README.valgrind for more information. + +# all tool names: Addrcheck,Memcheck,cachegrind,helgrind,massif +{ + ADDRESS_IN_RANGE/Invalid read of size 4 + Memcheck:Addr4 + fun:Py_ADDRESS_IN_RANGE +} + +{ + ADDRESS_IN_RANGE/Invalid read of size 4 + Memcheck:Value4 + fun:Py_ADDRESS_IN_RANGE +} + +{ + ADDRESS_IN_RANGE/Invalid read of size 8 (x86_64 aka amd64) + Memcheck:Value8 + fun:Py_ADDRESS_IN_RANGE +} + +{ + ADDRESS_IN_RANGE/Conditional jump or move depends on uninitialised value + Memcheck:Cond + fun:Py_ADDRESS_IN_RANGE +} + +# +# Leaks (including possible leaks) +# Hmmm, I wonder if this masks some real leaks. I think it does. +# Will need to fix that. +# + +{ + Suppress leaking the GIL. Happens once per process, see comment in ceval.c. + Memcheck:Leak + fun:malloc + fun:PyThread_allocate_lock + fun:PyEval_InitThreads +} + +{ + Suppress leaking the GIL after a fork. + Memcheck:Leak + fun:malloc + fun:PyThread_allocate_lock + fun:PyEval_ReInitThreads +} + +{ + Suppress leaking the autoTLSkey. This looks like it shouldn't leak though. + Memcheck:Leak + fun:malloc + fun:PyThread_create_key + fun:_PyGILState_Init + fun:Py_InitializeEx + fun:Py_Main +} + +{ + Hmmm, is this a real leak or like the GIL? + Memcheck:Leak + fun:malloc + fun:PyThread_ReInitTLS +} + +{ + Handle PyMalloc confusing valgrind (possibly leaked) + Memcheck:Leak + fun:realloc + fun:_PyObject_GC_Resize + fun:COMMENT_THIS_LINE_TO_DISABLE_LEAK_WARNING +} + +{ + Handle PyMalloc confusing valgrind (possibly leaked) + Memcheck:Leak + fun:malloc + fun:_PyObject_GC_New + fun:COMMENT_THIS_LINE_TO_DISABLE_LEAK_WARNING +} + +{ + Handle PyMalloc confusing valgrind (possibly leaked) + Memcheck:Leak + fun:malloc + fun:_PyObject_GC_NewVar + fun:COMMENT_THIS_LINE_TO_DISABLE_LEAK_WARNING +} + +# +# Non-python specific leaks +# + +{ + Handle pthread issue (possibly leaked) + Memcheck:Leak + fun:calloc + fun:allocate_dtv + fun:_dl_allocate_tls_storage + fun:_dl_allocate_tls +} + +{ + Handle pthread issue (possibly leaked) + Memcheck:Leak + fun:memalign + fun:_dl_allocate_tls_storage + fun:_dl_allocate_tls +} + +###{ +### ADDRESS_IN_RANGE/Invalid read of size 4 +### Memcheck:Addr4 +### fun:PyObject_Free +###} +### +###{ +### ADDRESS_IN_RANGE/Invalid read of size 4 +### Memcheck:Value4 +### fun:PyObject_Free +###} +### +###{ +### ADDRESS_IN_RANGE/Conditional jump or move depends on uninitialised value +### Memcheck:Cond +### fun:PyObject_Free +###} + +###{ +### ADDRESS_IN_RANGE/Invalid read of size 4 +### Memcheck:Addr4 +### fun:PyObject_Realloc +###} +### +###{ +### ADDRESS_IN_RANGE/Invalid read of size 4 +### Memcheck:Value4 +### fun:PyObject_Realloc +###} +### +###{ +### ADDRESS_IN_RANGE/Conditional jump or move depends on uninitialised value +### Memcheck:Cond +### fun:PyObject_Realloc +###} + +### +### All the suppressions below are for errors that occur within libraries +### that Python uses. The problems to not appear to be related to Python's +### use of the libraries. +### + +{ + Generic ubuntu ld problems + Memcheck:Addr8 + obj:/lib/ld-2.4.so + obj:/lib/ld-2.4.so + obj:/lib/ld-2.4.so + obj:/lib/ld-2.4.so +} + +{ + Generic gentoo ld problems + Memcheck:Cond + obj:/lib/ld-2.3.4.so + obj:/lib/ld-2.3.4.so + obj:/lib/ld-2.3.4.so + obj:/lib/ld-2.3.4.so +} + +{ + DBM problems, see test_dbm + Memcheck:Param + write(buf) + fun:write + obj:/usr/lib/libdb1.so.2 + obj:/usr/lib/libdb1.so.2 + obj:/usr/lib/libdb1.so.2 + obj:/usr/lib/libdb1.so.2 + fun:dbm_close +} + +{ + DBM problems, see test_dbm + Memcheck:Value8 + fun:memmove + obj:/usr/lib/libdb1.so.2 + obj:/usr/lib/libdb1.so.2 + obj:/usr/lib/libdb1.so.2 + obj:/usr/lib/libdb1.so.2 + fun:dbm_store + fun:dbm_ass_sub +} + +{ + DBM problems, see test_dbm + Memcheck:Cond + obj:/usr/lib/libdb1.so.2 + obj:/usr/lib/libdb1.so.2 + obj:/usr/lib/libdb1.so.2 + fun:dbm_store + fun:dbm_ass_sub +} + +{ + DBM problems, see test_dbm + Memcheck:Cond + fun:memmove + obj:/usr/lib/libdb1.so.2 + obj:/usr/lib/libdb1.so.2 + obj:/usr/lib/libdb1.so.2 + obj:/usr/lib/libdb1.so.2 + fun:dbm_store + fun:dbm_ass_sub +} + +{ + GDBM problems, see test_gdbm + Memcheck:Param + write(buf) + fun:write + fun:gdbm_open + +} + +{ + ZLIB problems, see test_gzip + Memcheck:Cond + obj:/lib/libz.so.1.2.3 + obj:/lib/libz.so.1.2.3 + fun:deflate +} + +{ + Avoid problems w/readline doing a putenv and leaking on exit + Memcheck:Leak + fun:malloc + fun:xmalloc + fun:sh_set_lines_and_columns + fun:_rl_get_screen_size + fun:_rl_init_terminal_io + obj:/lib/libreadline.so.4.3 + fun:rl_initialize +} + +### +### These occur from somewhere within the SSL, when running +### test_socket_sll. They are too general to leave on by default. +### +###{ +### somewhere in SSL stuff +### Memcheck:Cond +### fun:memset +###} +###{ +### somewhere in SSL stuff +### Memcheck:Value4 +### fun:memset +###} +### +###{ +### somewhere in SSL stuff +### Memcheck:Cond +### fun:MD5_Update +###} +### +###{ +### somewhere in SSL stuff +### Memcheck:Value4 +### fun:MD5_Update +###} + +# +# All of these problems come from using test_socket_ssl +# +{ + from test_socket_ssl + Memcheck:Cond + fun:BN_bin2bn +} + +{ + from test_socket_ssl + Memcheck:Cond + fun:BN_num_bits_word +} + +{ + from test_socket_ssl + Memcheck:Value4 + fun:BN_num_bits_word +} + +{ + from test_socket_ssl + Memcheck:Cond + fun:BN_mod_exp_mont_word +} + +{ + from test_socket_ssl + Memcheck:Cond + fun:BN_mod_exp_mont +} + +{ + from test_socket_ssl + Memcheck:Param + write(buf) + fun:write + obj:/usr/lib/libcrypto.so.0.9.7 +} + +{ + from test_socket_ssl + Memcheck:Cond + fun:RSA_verify +} + +{ + from test_socket_ssl + Memcheck:Value4 + fun:RSA_verify +} + +{ + from test_socket_ssl + Memcheck:Value4 + fun:DES_set_key_unchecked +} + +{ + from test_socket_ssl + Memcheck:Value4 + fun:DES_encrypt2 +} + +{ + from test_socket_ssl + Memcheck:Cond + obj:/usr/lib/libssl.so.0.9.7 +} + +{ + from test_socket_ssl + Memcheck:Value4 + obj:/usr/lib/libssl.so.0.9.7 +} + +{ + from test_socket_ssl + Memcheck:Cond + fun:BUF_MEM_grow_clean +} + +{ + from test_socket_ssl + Memcheck:Cond + fun:memcpy + fun:ssl3_read_bytes +} + +{ + from test_socket_ssl + Memcheck:Cond + fun:SHA1_Update +} + +{ + from test_socket_ssl + Memcheck:Value4 + fun:SHA1_Update +} diff --git a/graalpython/hpy/hpy/trace/__init__.py b/graalpython/hpy/hpy/trace/__init__.py new file mode 100644 index 0000000000..83d5308790 --- /dev/null +++ b/graalpython/hpy/hpy/trace/__init__.py @@ -0,0 +1,6 @@ +import hpy.universal + +get_call_counts = hpy.universal._trace.get_call_counts +get_durations = hpy.universal._trace.get_durations +set_trace_functions = hpy.universal._trace.set_trace_functions +get_frequency = hpy.universal._trace.get_frequency diff --git a/graalpython/com.oracle.graal.python.jni/src/trace/_tracemod.c b/graalpython/hpy/hpy/trace/src/_tracemod.c similarity index 100% rename from graalpython/com.oracle.graal.python.jni/src/trace/_tracemod.c rename to graalpython/hpy/hpy/trace/src/_tracemod.c diff --git a/graalpython/com.oracle.graal.python.jni/src/trace/autogen_trace_ctx_init.h b/graalpython/hpy/hpy/trace/src/autogen_trace_ctx_init.h similarity index 100% rename from graalpython/com.oracle.graal.python.jni/src/trace/autogen_trace_ctx_init.h rename to graalpython/hpy/hpy/trace/src/autogen_trace_ctx_init.h diff --git a/graalpython/com.oracle.graal.python.jni/src/trace/autogen_trace_func_table.c b/graalpython/hpy/hpy/trace/src/autogen_trace_func_table.c similarity index 100% rename from graalpython/com.oracle.graal.python.jni/src/trace/autogen_trace_func_table.c rename to graalpython/hpy/hpy/trace/src/autogen_trace_func_table.c diff --git a/graalpython/com.oracle.graal.python.jni/src/trace/autogen_trace_wrappers.c b/graalpython/hpy/hpy/trace/src/autogen_trace_wrappers.c similarity index 100% rename from graalpython/com.oracle.graal.python.jni/src/trace/autogen_trace_wrappers.c rename to graalpython/hpy/hpy/trace/src/autogen_trace_wrappers.c diff --git a/graalpython/com.oracle.graal.python.jni/src/trace/hpy_trace.h b/graalpython/hpy/hpy/trace/src/include/hpy_trace.h similarity index 100% rename from graalpython/com.oracle.graal.python.jni/src/trace/hpy_trace.h rename to graalpython/hpy/hpy/trace/src/include/hpy_trace.h diff --git a/graalpython/com.oracle.graal.python.jni/src/trace/trace_ctx.c b/graalpython/hpy/hpy/trace/src/trace_ctx.c similarity index 100% rename from graalpython/com.oracle.graal.python.jni/src/trace/trace_ctx.c rename to graalpython/hpy/hpy/trace/src/trace_ctx.c diff --git a/graalpython/com.oracle.graal.python.jni/src/trace/trace_internal.h b/graalpython/hpy/hpy/trace/src/trace_internal.h similarity index 100% rename from graalpython/com.oracle.graal.python.jni/src/trace/trace_internal.h rename to graalpython/hpy/hpy/trace/src/trace_internal.h diff --git a/graalpython/hpy/hpy/universal/src/api.h b/graalpython/hpy/hpy/universal/src/api.h new file mode 100644 index 0000000000..ea4b8108eb --- /dev/null +++ b/graalpython/hpy/hpy/universal/src/api.h @@ -0,0 +1,18 @@ +#ifndef HPY_API_H +#define HPY_API_H + +#include "hpy.h" + +extern struct _HPyContext_s g_universal_ctx; + +/* declare alloca() */ +#if defined(_MSC_VER) +# include /* for alloca() */ +#else +# include +# if (defined (__SVR4) && defined (__sun)) || defined(_AIX) || defined(__hpux) +# include +# endif +#endif + +#endif /* HPY_API_H */ diff --git a/graalpython/hpy/hpy/universal/src/autogen_ctx_call.i b/graalpython/hpy/hpy/universal/src/autogen_ctx_call.i new file mode 100644 index 0000000000..7c6a759f11 --- /dev/null +++ b/graalpython/hpy/hpy/universal/src/autogen_ctx_call.i @@ -0,0 +1,180 @@ + +/* + DO NOT EDIT THIS FILE! + + This file is automatically generated by hpy.tools.autogen.hpyfunc.autogen_ctx_call_i + See also hpy.tools.autogen and hpy/tools/public_api.h + + Run this to regenerate: + make autogen + +*/ + + case HPyFunc_NOARGS: { + HPyFunc_noargs f = (HPyFunc_noargs)func; + _HPyFunc_args_NOARGS *a = (_HPyFunc_args_NOARGS*)args; + a->result = _h2py(f(ctx, _py2h(a->self))); + return; + } + case HPyFunc_O: { + HPyFunc_o f = (HPyFunc_o)func; + _HPyFunc_args_O *a = (_HPyFunc_args_O*)args; + a->result = _h2py(f(ctx, _py2h(a->self), _py2h(a->arg))); + return; + } + case HPyFunc_UNARYFUNC: { + HPyFunc_unaryfunc f = (HPyFunc_unaryfunc)func; + _HPyFunc_args_UNARYFUNC *a = (_HPyFunc_args_UNARYFUNC*)args; + a->result = _h2py(f(ctx, _py2h(a->arg0))); + return; + } + case HPyFunc_BINARYFUNC: { + HPyFunc_binaryfunc f = (HPyFunc_binaryfunc)func; + _HPyFunc_args_BINARYFUNC *a = (_HPyFunc_args_BINARYFUNC*)args; + a->result = _h2py(f(ctx, _py2h(a->arg0), _py2h(a->arg1))); + return; + } + case HPyFunc_TERNARYFUNC: { + HPyFunc_ternaryfunc f = (HPyFunc_ternaryfunc)func; + _HPyFunc_args_TERNARYFUNC *a = (_HPyFunc_args_TERNARYFUNC*)args; + a->result = _h2py(f(ctx, _py2h(a->arg0), _py2h(a->arg1), _py2h(a->arg2))); + return; + } + case HPyFunc_INQUIRY: { + HPyFunc_inquiry f = (HPyFunc_inquiry)func; + _HPyFunc_args_INQUIRY *a = (_HPyFunc_args_INQUIRY*)args; + a->result = (f(ctx, _py2h(a->arg0))); + return; + } + case HPyFunc_LENFUNC: { + HPyFunc_lenfunc f = (HPyFunc_lenfunc)func; + _HPyFunc_args_LENFUNC *a = (_HPyFunc_args_LENFUNC*)args; + a->result = (f(ctx, _py2h(a->arg0))); + return; + } + case HPyFunc_SSIZEARGFUNC: { + HPyFunc_ssizeargfunc f = (HPyFunc_ssizeargfunc)func; + _HPyFunc_args_SSIZEARGFUNC *a = (_HPyFunc_args_SSIZEARGFUNC*)args; + a->result = _h2py(f(ctx, _py2h(a->arg0), (a->arg1))); + return; + } + case HPyFunc_SSIZESSIZEARGFUNC: { + HPyFunc_ssizessizeargfunc f = (HPyFunc_ssizessizeargfunc)func; + _HPyFunc_args_SSIZESSIZEARGFUNC *a = (_HPyFunc_args_SSIZESSIZEARGFUNC*)args; + a->result = _h2py(f(ctx, _py2h(a->arg0), (a->arg1), (a->arg2))); + return; + } + case HPyFunc_SSIZEOBJARGPROC: { + HPyFunc_ssizeobjargproc f = (HPyFunc_ssizeobjargproc)func; + _HPyFunc_args_SSIZEOBJARGPROC *a = (_HPyFunc_args_SSIZEOBJARGPROC*)args; + a->result = (f(ctx, _py2h(a->arg0), (a->arg1), _py2h(a->arg2))); + return; + } + case HPyFunc_SSIZESSIZEOBJARGPROC: { + HPyFunc_ssizessizeobjargproc f = (HPyFunc_ssizessizeobjargproc)func; + _HPyFunc_args_SSIZESSIZEOBJARGPROC *a = (_HPyFunc_args_SSIZESSIZEOBJARGPROC*)args; + a->result = (f(ctx, _py2h(a->arg0), (a->arg1), (a->arg2), _py2h(a->arg3))); + return; + } + case HPyFunc_OBJOBJARGPROC: { + HPyFunc_objobjargproc f = (HPyFunc_objobjargproc)func; + _HPyFunc_args_OBJOBJARGPROC *a = (_HPyFunc_args_OBJOBJARGPROC*)args; + a->result = (f(ctx, _py2h(a->arg0), _py2h(a->arg1), _py2h(a->arg2))); + return; + } + case HPyFunc_FREEFUNC: { + HPyFunc_freefunc f = (HPyFunc_freefunc)func; + _HPyFunc_args_FREEFUNC *a = (_HPyFunc_args_FREEFUNC*)args; + f(ctx, (a->arg0)); + return; + } + case HPyFunc_GETATTRFUNC: { + HPyFunc_getattrfunc f = (HPyFunc_getattrfunc)func; + _HPyFunc_args_GETATTRFUNC *a = (_HPyFunc_args_GETATTRFUNC*)args; + a->result = _h2py(f(ctx, _py2h(a->arg0), (a->arg1))); + return; + } + case HPyFunc_GETATTROFUNC: { + HPyFunc_getattrofunc f = (HPyFunc_getattrofunc)func; + _HPyFunc_args_GETATTROFUNC *a = (_HPyFunc_args_GETATTROFUNC*)args; + a->result = _h2py(f(ctx, _py2h(a->arg0), _py2h(a->arg1))); + return; + } + case HPyFunc_SETATTRFUNC: { + HPyFunc_setattrfunc f = (HPyFunc_setattrfunc)func; + _HPyFunc_args_SETATTRFUNC *a = (_HPyFunc_args_SETATTRFUNC*)args; + a->result = (f(ctx, _py2h(a->arg0), (a->arg1), _py2h(a->arg2))); + return; + } + case HPyFunc_SETATTROFUNC: { + HPyFunc_setattrofunc f = (HPyFunc_setattrofunc)func; + _HPyFunc_args_SETATTROFUNC *a = (_HPyFunc_args_SETATTROFUNC*)args; + a->result = (f(ctx, _py2h(a->arg0), _py2h(a->arg1), _py2h(a->arg2))); + return; + } + case HPyFunc_REPRFUNC: { + HPyFunc_reprfunc f = (HPyFunc_reprfunc)func; + _HPyFunc_args_REPRFUNC *a = (_HPyFunc_args_REPRFUNC*)args; + a->result = _h2py(f(ctx, _py2h(a->arg0))); + return; + } + case HPyFunc_HASHFUNC: { + HPyFunc_hashfunc f = (HPyFunc_hashfunc)func; + _HPyFunc_args_HASHFUNC *a = (_HPyFunc_args_HASHFUNC*)args; + a->result = (f(ctx, _py2h(a->arg0))); + return; + } + case HPyFunc_RICHCMPFUNC: { + HPyFunc_richcmpfunc f = (HPyFunc_richcmpfunc)func; + _HPyFunc_args_RICHCMPFUNC *a = (_HPyFunc_args_RICHCMPFUNC*)args; + a->result = _h2py(f(ctx, _py2h(a->arg0), _py2h(a->arg1), (a->arg2))); + return; + } + case HPyFunc_GETITERFUNC: { + HPyFunc_getiterfunc f = (HPyFunc_getiterfunc)func; + _HPyFunc_args_GETITERFUNC *a = (_HPyFunc_args_GETITERFUNC*)args; + a->result = _h2py(f(ctx, _py2h(a->arg0))); + return; + } + case HPyFunc_ITERNEXTFUNC: { + HPyFunc_iternextfunc f = (HPyFunc_iternextfunc)func; + _HPyFunc_args_ITERNEXTFUNC *a = (_HPyFunc_args_ITERNEXTFUNC*)args; + a->result = _h2py(f(ctx, _py2h(a->arg0))); + return; + } + case HPyFunc_DESCRGETFUNC: { + HPyFunc_descrgetfunc f = (HPyFunc_descrgetfunc)func; + _HPyFunc_args_DESCRGETFUNC *a = (_HPyFunc_args_DESCRGETFUNC*)args; + a->result = _h2py(f(ctx, _py2h(a->arg0), _py2h(a->arg1), _py2h(a->arg2))); + return; + } + case HPyFunc_DESCRSETFUNC: { + HPyFunc_descrsetfunc f = (HPyFunc_descrsetfunc)func; + _HPyFunc_args_DESCRSETFUNC *a = (_HPyFunc_args_DESCRSETFUNC*)args; + a->result = (f(ctx, _py2h(a->arg0), _py2h(a->arg1), _py2h(a->arg2))); + return; + } + case HPyFunc_GETTER: { + HPyFunc_getter f = (HPyFunc_getter)func; + _HPyFunc_args_GETTER *a = (_HPyFunc_args_GETTER*)args; + a->result = _h2py(f(ctx, _py2h(a->arg0), (a->arg1))); + return; + } + case HPyFunc_SETTER: { + HPyFunc_setter f = (HPyFunc_setter)func; + _HPyFunc_args_SETTER *a = (_HPyFunc_args_SETTER*)args; + a->result = (f(ctx, _py2h(a->arg0), _py2h(a->arg1), (a->arg2))); + return; + } + case HPyFunc_OBJOBJPROC: { + HPyFunc_objobjproc f = (HPyFunc_objobjproc)func; + _HPyFunc_args_OBJOBJPROC *a = (_HPyFunc_args_OBJOBJPROC*)args; + a->result = (f(ctx, _py2h(a->arg0), _py2h(a->arg1))); + return; + } + case HPyFunc_DESTRUCTOR: { + HPyFunc_destructor f = (HPyFunc_destructor)func; + _HPyFunc_args_DESTRUCTOR *a = (_HPyFunc_args_DESTRUCTOR*)args; + f(ctx, _py2h(a->arg0)); + return; + } diff --git a/graalpython/hpy/hpy/universal/src/autogen_ctx_def.h b/graalpython/hpy/hpy/universal/src/autogen_ctx_def.h new file mode 100644 index 0000000000..4bdeec338e --- /dev/null +++ b/graalpython/hpy/hpy/universal/src/autogen_ctx_def.h @@ -0,0 +1,207 @@ + +/* + DO NOT EDIT THIS FILE! + + This file is automatically generated by hpy.tools.autogen.ctx.autogen_ctx_def_h + See also hpy.tools.autogen and hpy/tools/public_api.h + + Run this to regenerate: + make autogen + +*/ + +struct _HPyContext_s g_universal_ctx = { + .name = "HPy Universal ABI (CPython backend)", + ._private = NULL, + .abi_version = HPY_ABI_VERSION, + /* h_None & co. are initialized by init_universal_ctx() */ + .ctx_Dup = &ctx_Dup, + .ctx_Close = &ctx_Close, + .ctx_Long_FromInt32_t = &ctx_Long_FromInt32_t, + .ctx_Long_FromUInt32_t = &ctx_Long_FromUInt32_t, + .ctx_Long_FromInt64_t = &ctx_Long_FromInt64_t, + .ctx_Long_FromUInt64_t = &ctx_Long_FromUInt64_t, + .ctx_Long_FromSize_t = &ctx_Long_FromSize_t, + .ctx_Long_FromSsize_t = &ctx_Long_FromSsize_t, + .ctx_Long_AsInt32_t = &ctx_Long_AsInt32_t, + .ctx_Long_AsUInt32_t = &ctx_Long_AsUInt32_t, + .ctx_Long_AsUInt32_tMask = &ctx_Long_AsUInt32_tMask, + .ctx_Long_AsInt64_t = &ctx_Long_AsInt64_t, + .ctx_Long_AsUInt64_t = &ctx_Long_AsUInt64_t, + .ctx_Long_AsUInt64_tMask = &ctx_Long_AsUInt64_tMask, + .ctx_Long_AsSize_t = &ctx_Long_AsSize_t, + .ctx_Long_AsSsize_t = &ctx_Long_AsSsize_t, + .ctx_Long_AsVoidPtr = &ctx_Long_AsVoidPtr, + .ctx_Long_AsDouble = &ctx_Long_AsDouble, + .ctx_Float_FromDouble = &ctx_Float_FromDouble, + .ctx_Float_AsDouble = &ctx_Float_AsDouble, + .ctx_Bool_FromBool = &ctx_Bool_FromBool, + .ctx_Length = &ctx_Length, + .ctx_Number_Check = &ctx_Number_Check, + .ctx_Add = &ctx_Add, + .ctx_Subtract = &ctx_Subtract, + .ctx_Multiply = &ctx_Multiply, + .ctx_MatrixMultiply = &ctx_MatrixMultiply, + .ctx_FloorDivide = &ctx_FloorDivide, + .ctx_TrueDivide = &ctx_TrueDivide, + .ctx_Remainder = &ctx_Remainder, + .ctx_Divmod = &ctx_Divmod, + .ctx_Power = &ctx_Power, + .ctx_Negative = &ctx_Negative, + .ctx_Positive = &ctx_Positive, + .ctx_Absolute = &ctx_Absolute, + .ctx_Invert = &ctx_Invert, + .ctx_Lshift = &ctx_Lshift, + .ctx_Rshift = &ctx_Rshift, + .ctx_And = &ctx_And, + .ctx_Xor = &ctx_Xor, + .ctx_Or = &ctx_Or, + .ctx_Index = &ctx_Index, + .ctx_Long = &ctx_Long, + .ctx_Float = &ctx_Float, + .ctx_InPlaceAdd = &ctx_InPlaceAdd, + .ctx_InPlaceSubtract = &ctx_InPlaceSubtract, + .ctx_InPlaceMultiply = &ctx_InPlaceMultiply, + .ctx_InPlaceMatrixMultiply = &ctx_InPlaceMatrixMultiply, + .ctx_InPlaceFloorDivide = &ctx_InPlaceFloorDivide, + .ctx_InPlaceTrueDivide = &ctx_InPlaceTrueDivide, + .ctx_InPlaceRemainder = &ctx_InPlaceRemainder, + .ctx_InPlacePower = &ctx_InPlacePower, + .ctx_InPlaceLshift = &ctx_InPlaceLshift, + .ctx_InPlaceRshift = &ctx_InPlaceRshift, + .ctx_InPlaceAnd = &ctx_InPlaceAnd, + .ctx_InPlaceXor = &ctx_InPlaceXor, + .ctx_InPlaceOr = &ctx_InPlaceOr, + .ctx_Callable_Check = &ctx_Callable_Check, + .ctx_CallTupleDict = &ctx_CallTupleDict, + .ctx_Call = &ctx_Call, + .ctx_CallMethod = &ctx_CallMethod, + .ctx_GetIter = &ctx_GetIter, + .ctx_Iter_Next = &ctx_Iter_Next, + .ctx_Iter_Check = &ctx_Iter_Check, + .ctx_FatalError = &ctx_FatalError, + .ctx_Err_SetString = &ctx_Err_SetString, + .ctx_Err_SetObject = &ctx_Err_SetObject, + .ctx_Err_SetFromErrnoWithFilename = &ctx_Err_SetFromErrnoWithFilename, + .ctx_Err_SetFromErrnoWithFilenameObjects = &ctx_Err_SetFromErrnoWithFilenameObjects, + .ctx_Err_Occurred = &ctx_Err_Occurred, + .ctx_Err_ExceptionMatches = &ctx_Err_ExceptionMatches, + .ctx_Err_NoMemory = &ctx_Err_NoMemory, + .ctx_Err_Clear = &ctx_Err_Clear, + .ctx_Err_NewException = &ctx_Err_NewException, + .ctx_Err_NewExceptionWithDoc = &ctx_Err_NewExceptionWithDoc, + .ctx_Err_WarnEx = &ctx_Err_WarnEx, + .ctx_Err_WriteUnraisable = &ctx_Err_WriteUnraisable, + .ctx_IsTrue = &ctx_IsTrue, + .ctx_Type_FromSpec = &ctx_Type_FromSpec, + .ctx_Type_GenericNew = &ctx_Type_GenericNew, + .ctx_GetAttr = &ctx_GetAttr, + .ctx_GetAttr_s = &ctx_GetAttr_s, + .ctx_HasAttr = &ctx_HasAttr, + .ctx_HasAttr_s = &ctx_HasAttr_s, + .ctx_SetAttr = &ctx_SetAttr, + .ctx_SetAttr_s = &ctx_SetAttr_s, + .ctx_GetItem = &ctx_GetItem, + .ctx_GetItem_i = &ctx_GetItem_i, + .ctx_GetItem_s = &ctx_GetItem_s, + .ctx_GetSlice = &ctx_GetSlice, + .ctx_Contains = &ctx_Contains, + .ctx_SetItem = &ctx_SetItem, + .ctx_SetItem_i = &ctx_SetItem_i, + .ctx_SetItem_s = &ctx_SetItem_s, + .ctx_SetSlice = &ctx_SetSlice, + .ctx_DelItem = &ctx_DelItem, + .ctx_DelItem_i = &ctx_DelItem_i, + .ctx_DelItem_s = &ctx_DelItem_s, + .ctx_DelSlice = &ctx_DelSlice, + .ctx_Type = &ctx_Type, + .ctx_TypeCheck = &ctx_TypeCheck, + .ctx_Type_GetName = &ctx_Type_GetName, + .ctx_Type_IsSubtype = &ctx_Type_IsSubtype, + .ctx_Is = &ctx_Is, + .ctx_AsStruct_Object = &ctx_AsStruct_Object, + .ctx_AsStruct_Legacy = &ctx_AsStruct_Legacy, + .ctx_AsStruct_Type = &ctx_AsStruct_Type, + .ctx_AsStruct_Long = &ctx_AsStruct_Long, + .ctx_AsStruct_Float = &ctx_AsStruct_Float, + .ctx_AsStruct_Unicode = &ctx_AsStruct_Unicode, + .ctx_AsStruct_Tuple = &ctx_AsStruct_Tuple, + .ctx_AsStruct_List = &ctx_AsStruct_List, + .ctx_AsStruct_Dict = &ctx_AsStruct_Dict, + .ctx_Type_GetBuiltinShape = &ctx_Type_GetBuiltinShape, + .ctx_New = &ctx_New, + .ctx_Repr = &ctx_Repr, + .ctx_Str = &ctx_Str, + .ctx_ASCII = &ctx_ASCII, + .ctx_Bytes = &ctx_Bytes, + .ctx_RichCompare = &ctx_RichCompare, + .ctx_RichCompareBool = &ctx_RichCompareBool, + .ctx_Hash = &ctx_Hash, + .ctx_Bytes_Check = &ctx_Bytes_Check, + .ctx_Bytes_Size = &ctx_Bytes_Size, + .ctx_Bytes_GET_SIZE = &ctx_Bytes_GET_SIZE, + .ctx_Bytes_AsString = &ctx_Bytes_AsString, + .ctx_Bytes_AS_STRING = &ctx_Bytes_AS_STRING, + .ctx_Bytes_FromString = &ctx_Bytes_FromString, + .ctx_Bytes_FromStringAndSize = &ctx_Bytes_FromStringAndSize, + .ctx_Unicode_FromString = &ctx_Unicode_FromString, + .ctx_Unicode_Check = &ctx_Unicode_Check, + .ctx_Unicode_AsASCIIString = &ctx_Unicode_AsASCIIString, + .ctx_Unicode_AsLatin1String = &ctx_Unicode_AsLatin1String, + .ctx_Unicode_AsUTF8String = &ctx_Unicode_AsUTF8String, + .ctx_Unicode_AsUTF8AndSize = &ctx_Unicode_AsUTF8AndSize, + .ctx_Unicode_FromWideChar = &ctx_Unicode_FromWideChar, + .ctx_Unicode_DecodeFSDefault = &ctx_Unicode_DecodeFSDefault, + .ctx_Unicode_DecodeFSDefaultAndSize = &ctx_Unicode_DecodeFSDefaultAndSize, + .ctx_Unicode_EncodeFSDefault = &ctx_Unicode_EncodeFSDefault, + .ctx_Unicode_ReadChar = &ctx_Unicode_ReadChar, + .ctx_Unicode_DecodeASCII = &ctx_Unicode_DecodeASCII, + .ctx_Unicode_DecodeLatin1 = &ctx_Unicode_DecodeLatin1, + .ctx_Unicode_FromEncodedObject = &ctx_Unicode_FromEncodedObject, + .ctx_Unicode_Substring = &ctx_Unicode_Substring, + .ctx_List_Check = &ctx_List_Check, + .ctx_List_New = &ctx_List_New, + .ctx_List_Append = &ctx_List_Append, + .ctx_List_Insert = &ctx_List_Insert, + .ctx_Dict_Check = &ctx_Dict_Check, + .ctx_Dict_New = &ctx_Dict_New, + .ctx_Dict_Keys = &ctx_Dict_Keys, + .ctx_Dict_Copy = &ctx_Dict_Copy, + .ctx_Tuple_Check = &ctx_Tuple_Check, + .ctx_Tuple_FromArray = &ctx_Tuple_FromArray, + .ctx_Slice_New = &ctx_Slice_New, + .ctx_Slice_Unpack = &ctx_Slice_Unpack, + .ctx_Import_ImportModule = &ctx_Import_ImportModule, + .ctx_Capsule_New = &ctx_Capsule_New, + .ctx_Capsule_Get = &ctx_Capsule_Get, + .ctx_Capsule_IsValid = &ctx_Capsule_IsValid, + .ctx_Capsule_Set = &ctx_Capsule_Set, + .ctx_FromPyObject = &ctx_FromPyObject, + .ctx_AsPyObject = &ctx_AsPyObject, + .ctx_CallRealFunctionFromTrampoline = &ctx_CallRealFunctionFromTrampoline, + .ctx_ListBuilder_New = &ctx_ListBuilder_New, + .ctx_ListBuilder_Set = &ctx_ListBuilder_Set, + .ctx_ListBuilder_Build = &ctx_ListBuilder_Build, + .ctx_ListBuilder_Cancel = &ctx_ListBuilder_Cancel, + .ctx_TupleBuilder_New = &ctx_TupleBuilder_New, + .ctx_TupleBuilder_Set = &ctx_TupleBuilder_Set, + .ctx_TupleBuilder_Build = &ctx_TupleBuilder_Build, + .ctx_TupleBuilder_Cancel = &ctx_TupleBuilder_Cancel, + .ctx_Tracker_New = &ctx_Tracker_New, + .ctx_Tracker_Add = &ctx_Tracker_Add, + .ctx_Tracker_ForgetAll = &ctx_Tracker_ForgetAll, + .ctx_Tracker_Close = &ctx_Tracker_Close, + .ctx_Field_Store = &ctx_Field_Store, + .ctx_Field_Load = &ctx_Field_Load, + .ctx_ReenterPythonExecution = &ctx_ReenterPythonExecution, + .ctx_LeavePythonExecution = &ctx_LeavePythonExecution, + .ctx_Global_Store = &ctx_Global_Store, + .ctx_Global_Load = &ctx_Global_Load, + .ctx_Dump = &ctx_Dump, + .ctx_Compile_s = &ctx_Compile_s, + .ctx_EvalCode = &ctx_EvalCode, + .ctx_ContextVar_New = &ctx_ContextVar_New, + .ctx_ContextVar_Get = &ctx_ContextVar_Get, + .ctx_ContextVar_Set = &ctx_ContextVar_Set, + .ctx_SetCallFunction = &ctx_SetCallFunction, +}; diff --git a/graalpython/hpy/hpy/universal/src/autogen_ctx_impl.h b/graalpython/hpy/hpy/universal/src/autogen_ctx_impl.h new file mode 100644 index 0000000000..15c6e35b39 --- /dev/null +++ b/graalpython/hpy/hpy/universal/src/autogen_ctx_impl.h @@ -0,0 +1,612 @@ + +/* + DO NOT EDIT THIS FILE! + + This file is automatically generated by hpy.tools.autogen.trampolines.universal_autogen_ctx_impl_h + See also hpy.tools.autogen and hpy/tools/public_api.h + + Run this to regenerate: + make autogen + +*/ + +HPyAPI_IMPL HPy ctx_Long_FromSize_t(HPyContext *ctx, size_t value) +{ + return _py2h(PyLong_FromSize_t(value)); +} + +HPyAPI_IMPL HPy ctx_Long_FromSsize_t(HPyContext *ctx, HPy_ssize_t value) +{ + return _py2h(PyLong_FromSsize_t(value)); +} + +HPyAPI_IMPL size_t ctx_Long_AsSize_t(HPyContext *ctx, HPy h) +{ + return PyLong_AsSize_t(_h2py(h)); +} + +HPyAPI_IMPL HPy_ssize_t ctx_Long_AsSsize_t(HPyContext *ctx, HPy h) +{ + return PyLong_AsSsize_t(_h2py(h)); +} + +HPyAPI_IMPL void *ctx_Long_AsVoidPtr(HPyContext *ctx, HPy h) +{ + return PyLong_AsVoidPtr(_h2py(h)); +} + +HPyAPI_IMPL double ctx_Long_AsDouble(HPyContext *ctx, HPy h) +{ + return PyLong_AsDouble(_h2py(h)); +} + +HPyAPI_IMPL HPy ctx_Float_FromDouble(HPyContext *ctx, double v) +{ + return _py2h(PyFloat_FromDouble(v)); +} + +HPyAPI_IMPL double ctx_Float_AsDouble(HPyContext *ctx, HPy h) +{ + return PyFloat_AsDouble(_h2py(h)); +} + +HPyAPI_IMPL HPy ctx_Bool_FromBool(HPyContext *ctx, bool v) +{ + return _py2h(PyBool_FromLong(v)); +} + +HPyAPI_IMPL HPy_ssize_t ctx_Length(HPyContext *ctx, HPy h) +{ + return PyObject_Length(_h2py(h)); +} + +HPyAPI_IMPL int ctx_Number_Check(HPyContext *ctx, HPy h) +{ + return PyNumber_Check(_h2py(h)); +} + +HPyAPI_IMPL HPy ctx_Add(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_Add(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL HPy ctx_Subtract(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_Subtract(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL HPy ctx_Multiply(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_Multiply(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL HPy ctx_MatrixMultiply(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_MatrixMultiply(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL HPy ctx_FloorDivide(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_FloorDivide(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL HPy ctx_TrueDivide(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_TrueDivide(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL HPy ctx_Remainder(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_Remainder(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL HPy ctx_Divmod(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_Divmod(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL HPy ctx_Power(HPyContext *ctx, HPy h1, HPy h2, HPy h3) +{ + return _py2h(PyNumber_Power(_h2py(h1), _h2py(h2), _h2py(h3))); +} + +HPyAPI_IMPL HPy ctx_Negative(HPyContext *ctx, HPy h1) +{ + return _py2h(PyNumber_Negative(_h2py(h1))); +} + +HPyAPI_IMPL HPy ctx_Positive(HPyContext *ctx, HPy h1) +{ + return _py2h(PyNumber_Positive(_h2py(h1))); +} + +HPyAPI_IMPL HPy ctx_Absolute(HPyContext *ctx, HPy h1) +{ + return _py2h(PyNumber_Absolute(_h2py(h1))); +} + +HPyAPI_IMPL HPy ctx_Invert(HPyContext *ctx, HPy h1) +{ + return _py2h(PyNumber_Invert(_h2py(h1))); +} + +HPyAPI_IMPL HPy ctx_Lshift(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_Lshift(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL HPy ctx_Rshift(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_Rshift(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL HPy ctx_And(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_And(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL HPy ctx_Xor(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_Xor(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL HPy ctx_Or(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_Or(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL HPy ctx_Index(HPyContext *ctx, HPy h1) +{ + return _py2h(PyNumber_Index(_h2py(h1))); +} + +HPyAPI_IMPL HPy ctx_Long(HPyContext *ctx, HPy h1) +{ + return _py2h(PyNumber_Long(_h2py(h1))); +} + +HPyAPI_IMPL HPy ctx_Float(HPyContext *ctx, HPy h1) +{ + return _py2h(PyNumber_Float(_h2py(h1))); +} + +HPyAPI_IMPL HPy ctx_InPlaceAdd(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_InPlaceAdd(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL HPy ctx_InPlaceSubtract(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_InPlaceSubtract(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL HPy ctx_InPlaceMultiply(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_InPlaceMultiply(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL HPy ctx_InPlaceMatrixMultiply(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_InPlaceMatrixMultiply(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL HPy ctx_InPlaceFloorDivide(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_InPlaceFloorDivide(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL HPy ctx_InPlaceTrueDivide(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_InPlaceTrueDivide(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL HPy ctx_InPlaceRemainder(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_InPlaceRemainder(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL HPy ctx_InPlacePower(HPyContext *ctx, HPy h1, HPy h2, HPy h3) +{ + return _py2h(PyNumber_InPlacePower(_h2py(h1), _h2py(h2), _h2py(h3))); +} + +HPyAPI_IMPL HPy ctx_InPlaceLshift(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_InPlaceLshift(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL HPy ctx_InPlaceRshift(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_InPlaceRshift(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL HPy ctx_InPlaceAnd(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_InPlaceAnd(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL HPy ctx_InPlaceXor(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_InPlaceXor(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL HPy ctx_InPlaceOr(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_InPlaceOr(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL int ctx_Callable_Check(HPyContext *ctx, HPy h) +{ + return PyCallable_Check(_h2py(h)); +} + +HPyAPI_IMPL HPy ctx_GetIter(HPyContext *ctx, HPy obj) +{ + return _py2h(PyObject_GetIter(_h2py(obj))); +} + +HPyAPI_IMPL HPy ctx_Iter_Next(HPyContext *ctx, HPy obj) +{ + return _py2h(PyIter_Next(_h2py(obj))); +} + +HPyAPI_IMPL int ctx_Iter_Check(HPyContext *ctx, HPy obj) +{ + return PyIter_Check(_h2py(obj)); +} + +HPyAPI_IMPL void ctx_Err_SetString(HPyContext *ctx, HPy h_type, const char *utf8_message) +{ + PyErr_SetString(_h2py(h_type), utf8_message); +} + +HPyAPI_IMPL void ctx_Err_SetObject(HPyContext *ctx, HPy h_type, HPy h_value) +{ + PyErr_SetObject(_h2py(h_type), _h2py(h_value)); +} + +HPyAPI_IMPL HPy ctx_Err_SetFromErrnoWithFilename(HPyContext *ctx, HPy h_type, const char *filename_fsencoded) +{ + return _py2h(PyErr_SetFromErrnoWithFilename(_h2py(h_type), filename_fsencoded)); +} + +HPyAPI_IMPL void ctx_Err_SetFromErrnoWithFilenameObjects(HPyContext *ctx, HPy h_type, HPy filename1, HPy filename2) +{ + PyErr_SetFromErrnoWithFilenameObjects(_h2py(h_type), _h2py(filename1), _h2py(filename2)); +} + +HPyAPI_IMPL int ctx_Err_ExceptionMatches(HPyContext *ctx, HPy exc) +{ + return PyErr_ExceptionMatches(_h2py(exc)); +} + +HPyAPI_IMPL void ctx_Err_NoMemory(HPyContext *ctx) +{ + PyErr_NoMemory(); +} + +HPyAPI_IMPL void ctx_Err_Clear(HPyContext *ctx) +{ + PyErr_Clear(); +} + +HPyAPI_IMPL HPy ctx_Err_NewException(HPyContext *ctx, const char *utf8_name, HPy base, HPy dict) +{ + return _py2h(PyErr_NewException(utf8_name, _h2py(base), _h2py(dict))); +} + +HPyAPI_IMPL HPy ctx_Err_NewExceptionWithDoc(HPyContext *ctx, const char *utf8_name, const char *utf8_doc, HPy base, HPy dict) +{ + return _py2h(PyErr_NewExceptionWithDoc(utf8_name, utf8_doc, _h2py(base), _h2py(dict))); +} + +HPyAPI_IMPL int ctx_Err_WarnEx(HPyContext *ctx, HPy category, const char *utf8_message, HPy_ssize_t stack_level) +{ + return PyErr_WarnEx(_h2py(category), utf8_message, stack_level); +} + +HPyAPI_IMPL void ctx_Err_WriteUnraisable(HPyContext *ctx, HPy obj) +{ + PyErr_WriteUnraisable(_h2py(obj)); +} + +HPyAPI_IMPL int ctx_IsTrue(HPyContext *ctx, HPy h) +{ + return PyObject_IsTrue(_h2py(h)); +} + +HPyAPI_IMPL HPy ctx_GetAttr(HPyContext *ctx, HPy obj, HPy name) +{ + return _py2h(PyObject_GetAttr(_h2py(obj), _h2py(name))); +} + +HPyAPI_IMPL HPy ctx_GetAttr_s(HPyContext *ctx, HPy obj, const char *utf8_name) +{ + return _py2h(PyObject_GetAttrString(_h2py(obj), utf8_name)); +} + +HPyAPI_IMPL int ctx_HasAttr(HPyContext *ctx, HPy obj, HPy name) +{ + return PyObject_HasAttr(_h2py(obj), _h2py(name)); +} + +HPyAPI_IMPL int ctx_HasAttr_s(HPyContext *ctx, HPy obj, const char *utf8_name) +{ + return PyObject_HasAttrString(_h2py(obj), utf8_name); +} + +HPyAPI_IMPL int ctx_SetAttr(HPyContext *ctx, HPy obj, HPy name, HPy value) +{ + return PyObject_SetAttr(_h2py(obj), _h2py(name), _h2py(value)); +} + +HPyAPI_IMPL int ctx_SetAttr_s(HPyContext *ctx, HPy obj, const char *utf8_name, HPy value) +{ + return PyObject_SetAttrString(_h2py(obj), utf8_name, _h2py(value)); +} + +HPyAPI_IMPL HPy ctx_GetItem(HPyContext *ctx, HPy obj, HPy key) +{ + return _py2h(PyObject_GetItem(_h2py(obj), _h2py(key))); +} + +HPyAPI_IMPL HPy ctx_GetSlice(HPyContext *ctx, HPy obj, HPy_ssize_t start, HPy_ssize_t end) +{ + return _py2h(PySequence_GetSlice(_h2py(obj), start, end)); +} + +HPyAPI_IMPL int ctx_Contains(HPyContext *ctx, HPy container, HPy key) +{ + return PySequence_Contains(_h2py(container), _h2py(key)); +} + +HPyAPI_IMPL int ctx_SetItem(HPyContext *ctx, HPy obj, HPy key, HPy value) +{ + return PyObject_SetItem(_h2py(obj), _h2py(key), _h2py(value)); +} + +HPyAPI_IMPL int ctx_SetSlice(HPyContext *ctx, HPy obj, HPy_ssize_t start, HPy_ssize_t end, HPy value) +{ + return PySequence_SetSlice(_h2py(obj), start, end, _h2py(value)); +} + +HPyAPI_IMPL int ctx_DelItem(HPyContext *ctx, HPy obj, HPy key) +{ + return PyObject_DelItem(_h2py(obj), _h2py(key)); +} + +HPyAPI_IMPL int ctx_DelSlice(HPyContext *ctx, HPy obj, HPy_ssize_t start, HPy_ssize_t end) +{ + return PySequence_DelSlice(_h2py(obj), start, end); +} + +HPyAPI_IMPL HPy ctx_Repr(HPyContext *ctx, HPy obj) +{ + return _py2h(PyObject_Repr(_h2py(obj))); +} + +HPyAPI_IMPL HPy ctx_Str(HPyContext *ctx, HPy obj) +{ + return _py2h(PyObject_Str(_h2py(obj))); +} + +HPyAPI_IMPL HPy ctx_ASCII(HPyContext *ctx, HPy obj) +{ + return _py2h(PyObject_ASCII(_h2py(obj))); +} + +HPyAPI_IMPL HPy ctx_Bytes(HPyContext *ctx, HPy obj) +{ + return _py2h(PyObject_Bytes(_h2py(obj))); +} + +HPyAPI_IMPL HPy ctx_RichCompare(HPyContext *ctx, HPy v, HPy w, int op) +{ + return _py2h(PyObject_RichCompare(_h2py(v), _h2py(w), op)); +} + +HPyAPI_IMPL int ctx_RichCompareBool(HPyContext *ctx, HPy v, HPy w, int op) +{ + return PyObject_RichCompareBool(_h2py(v), _h2py(w), op); +} + +HPyAPI_IMPL HPy_hash_t ctx_Hash(HPyContext *ctx, HPy obj) +{ + return PyObject_Hash(_h2py(obj)); +} + +HPyAPI_IMPL int ctx_Bytes_Check(HPyContext *ctx, HPy h) +{ + return PyBytes_Check(_h2py(h)); +} + +HPyAPI_IMPL HPy_ssize_t ctx_Bytes_Size(HPyContext *ctx, HPy h) +{ + return PyBytes_Size(_h2py(h)); +} + +HPyAPI_IMPL HPy_ssize_t ctx_Bytes_GET_SIZE(HPyContext *ctx, HPy h) +{ + return PyBytes_GET_SIZE(_h2py(h)); +} + +HPyAPI_IMPL const char *ctx_Bytes_AsString(HPyContext *ctx, HPy h) +{ + return PyBytes_AsString(_h2py(h)); +} + +HPyAPI_IMPL const char *ctx_Bytes_AS_STRING(HPyContext *ctx, HPy h) +{ + return PyBytes_AS_STRING(_h2py(h)); +} + +HPyAPI_IMPL HPy ctx_Bytes_FromString(HPyContext *ctx, const char *bytes) +{ + return _py2h(PyBytes_FromString(bytes)); +} + +HPyAPI_IMPL HPy ctx_Unicode_FromString(HPyContext *ctx, const char *utf8) +{ + return _py2h(PyUnicode_FromString(utf8)); +} + +HPyAPI_IMPL int ctx_Unicode_Check(HPyContext *ctx, HPy h) +{ + return PyUnicode_Check(_h2py(h)); +} + +HPyAPI_IMPL HPy ctx_Unicode_AsASCIIString(HPyContext *ctx, HPy h) +{ + return _py2h(PyUnicode_AsASCIIString(_h2py(h))); +} + +HPyAPI_IMPL HPy ctx_Unicode_AsLatin1String(HPyContext *ctx, HPy h) +{ + return _py2h(PyUnicode_AsLatin1String(_h2py(h))); +} + +HPyAPI_IMPL HPy ctx_Unicode_AsUTF8String(HPyContext *ctx, HPy h) +{ + return _py2h(PyUnicode_AsUTF8String(_h2py(h))); +} + +HPyAPI_IMPL const char *ctx_Unicode_AsUTF8AndSize(HPyContext *ctx, HPy h, HPy_ssize_t *size) +{ + return PyUnicode_AsUTF8AndSize(_h2py(h), size); +} + +HPyAPI_IMPL HPy ctx_Unicode_FromWideChar(HPyContext *ctx, const wchar_t *w, HPy_ssize_t size) +{ + return _py2h(PyUnicode_FromWideChar(w, size)); +} + +HPyAPI_IMPL HPy ctx_Unicode_DecodeFSDefault(HPyContext *ctx, const char *v) +{ + return _py2h(PyUnicode_DecodeFSDefault(v)); +} + +HPyAPI_IMPL HPy ctx_Unicode_DecodeFSDefaultAndSize(HPyContext *ctx, const char *v, HPy_ssize_t size) +{ + return _py2h(PyUnicode_DecodeFSDefaultAndSize(v, size)); +} + +HPyAPI_IMPL HPy ctx_Unicode_EncodeFSDefault(HPyContext *ctx, HPy h) +{ + return _py2h(PyUnicode_EncodeFSDefault(_h2py(h))); +} + +HPyAPI_IMPL HPy_UCS4 ctx_Unicode_ReadChar(HPyContext *ctx, HPy h, HPy_ssize_t index) +{ + return PyUnicode_ReadChar(_h2py(h), index); +} + +HPyAPI_IMPL HPy ctx_Unicode_DecodeASCII(HPyContext *ctx, const char *ascii, HPy_ssize_t size, const char *errors) +{ + return _py2h(PyUnicode_DecodeASCII(ascii, size, errors)); +} + +HPyAPI_IMPL HPy ctx_Unicode_DecodeLatin1(HPyContext *ctx, const char *latin1, HPy_ssize_t size, const char *errors) +{ + return _py2h(PyUnicode_DecodeLatin1(latin1, size, errors)); +} + +HPyAPI_IMPL HPy ctx_Unicode_FromEncodedObject(HPyContext *ctx, HPy obj, const char *encoding, const char *errors) +{ + return _py2h(PyUnicode_FromEncodedObject(_h2py(obj), encoding, errors)); +} + +HPyAPI_IMPL HPy ctx_Unicode_Substring(HPyContext *ctx, HPy str, HPy_ssize_t start, HPy_ssize_t end) +{ + return _py2h(PyUnicode_Substring(_h2py(str), start, end)); +} + +HPyAPI_IMPL int ctx_List_Check(HPyContext *ctx, HPy h) +{ + return PyList_Check(_h2py(h)); +} + +HPyAPI_IMPL HPy ctx_List_New(HPyContext *ctx, HPy_ssize_t len) +{ + return _py2h(PyList_New(len)); +} + +HPyAPI_IMPL int ctx_List_Append(HPyContext *ctx, HPy h_list, HPy h_item) +{ + return PyList_Append(_h2py(h_list), _h2py(h_item)); +} + +HPyAPI_IMPL int ctx_List_Insert(HPyContext *ctx, HPy h_list, HPy_ssize_t index, HPy h_item) +{ + return PyList_Insert(_h2py(h_list), index, _h2py(h_item)); +} + +HPyAPI_IMPL int ctx_Dict_Check(HPyContext *ctx, HPy h) +{ + return PyDict_Check(_h2py(h)); +} + +HPyAPI_IMPL HPy ctx_Dict_New(HPyContext *ctx) +{ + return _py2h(PyDict_New()); +} + +HPyAPI_IMPL HPy ctx_Dict_Keys(HPyContext *ctx, HPy h) +{ + return _py2h(PyDict_Keys(_h2py(h))); +} + +HPyAPI_IMPL HPy ctx_Dict_Copy(HPyContext *ctx, HPy h) +{ + return _py2h(PyDict_Copy(_h2py(h))); +} + +HPyAPI_IMPL int ctx_Tuple_Check(HPyContext *ctx, HPy h) +{ + return PyTuple_Check(_h2py(h)); +} + +HPyAPI_IMPL HPy ctx_Slice_New(HPyContext *ctx, HPy start, HPy stop, HPy step) +{ + return _py2h(PySlice_New(_h2py(start), _h2py(stop), _h2py(step))); +} + +HPyAPI_IMPL int ctx_Slice_Unpack(HPyContext *ctx, HPy slice, HPy_ssize_t *start, HPy_ssize_t *stop, HPy_ssize_t *step) +{ + return PySlice_Unpack(_h2py(slice), start, stop, step); +} + +HPyAPI_IMPL HPy ctx_Import_ImportModule(HPyContext *ctx, const char *utf8_name) +{ + return _py2h(PyImport_ImportModule(utf8_name)); +} + +HPyAPI_IMPL int ctx_Capsule_IsValid(HPyContext *ctx, HPy capsule, const char *utf8_name) +{ + return PyCapsule_IsValid(_h2py(capsule), utf8_name); +} + +HPyAPI_IMPL void ctx_ReenterPythonExecution(HPyContext *ctx, HPyThreadState state) +{ + PyEval_RestoreThread(_h2threads(state)); +} + +HPyAPI_IMPL HPyThreadState ctx_LeavePythonExecution(HPyContext *ctx) +{ + return _threads2h(PyEval_SaveThread()); +} + +HPyAPI_IMPL HPy ctx_EvalCode(HPyContext *ctx, HPy code, HPy globals, HPy locals) +{ + return _py2h(PyEval_EvalCode(_h2py(code), _h2py(globals), _h2py(locals))); +} + +HPyAPI_IMPL HPy ctx_ContextVar_New(HPyContext *ctx, const char *name, HPy default_value) +{ + return _py2h(PyContextVar_New(name, _h2py(default_value))); +} + +HPyAPI_IMPL HPy ctx_ContextVar_Set(HPyContext *ctx, HPy context_var, HPy value) +{ + return _py2h(PyContextVar_Set(_h2py(context_var), _h2py(value))); +} + diff --git a/graalpython/hpy/hpy/universal/src/ctx.c b/graalpython/hpy/hpy/universal/src/ctx.c new file mode 100644 index 0000000000..4e9860233b --- /dev/null +++ b/graalpython/hpy/hpy/universal/src/ctx.c @@ -0,0 +1,10 @@ +#include "hpy.h" +#include "handles.h" + +#include "hpy/runtime/ctx_funcs.h" +#include "hpy/runtime/ctx_type.h" +#include "ctx_meth.h" +#include "ctx_misc.h" + +#include "autogen_ctx_impl.h" +#include "autogen_ctx_def.h" diff --git a/graalpython/hpy/hpy/universal/src/ctx_meth.c b/graalpython/hpy/hpy/universal/src/ctx_meth.c new file mode 100644 index 0000000000..923f81e23f --- /dev/null +++ b/graalpython/hpy/hpy/universal/src/ctx_meth.c @@ -0,0 +1,136 @@ +#include +#include "ctx_meth.h" +#include "hpy/runtime/ctx_type.h" +#include "hpy/runtime/ctx_module.h" +#include "handles.h" + +static void _buffer_h2py(HPyContext *ctx, const HPy_buffer *src, Py_buffer *dest) +{ + dest->buf = src->buf; + dest->obj = HPy_AsPyObject(ctx, src->obj); + dest->len = src->len; + dest->itemsize = src->itemsize; + dest->readonly = src->readonly; + dest->ndim = src->ndim; + dest->format = src->format; + dest->shape = src->shape; + dest->strides = src->strides; + dest->suboffsets = src->suboffsets; + dest->internal = src->internal; +} + +static void _buffer_py2h(HPyContext *ctx, const Py_buffer *src, HPy_buffer *dest) +{ + dest->buf = src->buf; + dest->obj = HPy_FromPyObject(ctx, src->obj); + dest->len = src->len; + dest->itemsize = src->itemsize; + dest->readonly = src->readonly; + dest->ndim = src->ndim; + dest->format = src->format; + dest->shape = src->shape; + dest->strides = src->strides; + dest->suboffsets = src->suboffsets; + dest->internal = src->internal; +} + +HPyAPI_IMPL void +ctx_CallRealFunctionFromTrampoline(HPyContext *ctx, HPyFunc_Signature sig, + HPyCFunction func, void *args) +{ + switch (sig) { + case HPyFunc_VARARGS: { + HPyFunc_varargs f = (HPyFunc_varargs)func; + _HPyFunc_args_VARARGS *a = (_HPyFunc_args_VARARGS*)args; + HPy *h_args = (HPy *)alloca(a->nargs * sizeof(HPy)); + for (HPy_ssize_t i = 0; i < a->nargs; i++) { + h_args[i] = _py2h(a->args[i]); + } + a->result = _h2py(f(ctx, _py2h(a->self), h_args, a->nargs)); + return; + } + case HPyFunc_KEYWORDS: { + HPyFunc_keywords f = (HPyFunc_keywords)func; + _HPyFunc_args_KEYWORDS *a = (_HPyFunc_args_KEYWORDS*)args; + size_t n_kwnames = a->kwnames != NULL ? PyTuple_GET_SIZE(a->kwnames) : 0; + size_t nargs = PyVectorcall_NARGS(a->nargsf); + size_t nargs_with_kw = nargs + n_kwnames; + HPy *h_args = (HPy *)alloca(nargs_with_kw * sizeof(HPy)); + for (size_t i = 0; i < nargs_with_kw; i++) { + h_args[i] = _py2h(a->args[i]); + } + a->result = _h2py(f(ctx, _py2h(a->self), h_args, nargs, _py2h(a->kwnames))); + return; + } + case HPyFunc_INITPROC: { + HPyFunc_initproc f = (HPyFunc_initproc)func; + _HPyFunc_args_INITPROC *a = (_HPyFunc_args_INITPROC*)args; + Py_ssize_t nargs = PyTuple_GET_SIZE(a->args); + HPy *h_args = (HPy *)alloca(nargs * sizeof(HPy)); + for (Py_ssize_t i = 0; i < nargs; i++) { + h_args[i] = _py2h(PyTuple_GET_ITEM(a->args, i)); + } + a->result = f(ctx, _py2h(a->self), h_args, nargs, _py2h(a->kw)); + return; + } + case HPyFunc_NEWFUNC: { + HPyFunc_newfunc f = (HPyFunc_newfunc)func; + _HPyFunc_args_NEWFUNC *a = (_HPyFunc_args_NEWFUNC*)args; + Py_ssize_t nargs = PyTuple_GET_SIZE(a->args); + HPy *h_args = (HPy *)alloca(nargs * sizeof(HPy)); + for (Py_ssize_t i = 0; i < nargs; i++) { + h_args[i] = _py2h(PyTuple_GET_ITEM(a->args, i)); + } + a->result = _h2py(f(ctx, _py2h(a->self), h_args, nargs, _py2h(a->kw))); + return; + } + case HPyFunc_GETBUFFERPROC: { + HPyFunc_getbufferproc f = (HPyFunc_getbufferproc)func; + _HPyFunc_args_GETBUFFERPROC *a = (_HPyFunc_args_GETBUFFERPROC*)args; + HPy_buffer hbuf; + a->result = f(ctx, _py2h(a->self), &hbuf, a->flags); + if (a->result < 0) { + a->view->obj = NULL; + return; + } + _buffer_h2py(ctx, &hbuf, a->view); + HPy_Close(ctx, hbuf.obj); + return; + } + case HPyFunc_RELEASEBUFFERPROC: { + HPyFunc_releasebufferproc f = (HPyFunc_releasebufferproc)func; + _HPyFunc_args_RELEASEBUFFERPROC *a = (_HPyFunc_args_RELEASEBUFFERPROC*)args; + HPy_buffer hbuf; + _buffer_py2h(ctx, a->view, &hbuf); + f(ctx, _py2h(a->self), &hbuf); + // XXX: copy back from hbuf? + HPy_Close(ctx, hbuf.obj); + return; + } + case HPyFunc_TRAVERSEPROC: { + HPyFunc_traverseproc f = (HPyFunc_traverseproc)func; + _HPyFunc_args_TRAVERSEPROC *a = (_HPyFunc_args_TRAVERSEPROC*)args; + a->result = call_traverseproc_from_trampoline(f, a->self, + a->visit, a->arg); + return; + } + case HPyFunc_CAPSULE_DESTRUCTOR: { + HPyFunc_Capsule_Destructor f = (HPyFunc_Capsule_Destructor)func; + PyObject *capsule = (PyObject *)args; + const char *name = PyCapsule_GetName(capsule); + f(name, PyCapsule_GetPointer(capsule, name), + PyCapsule_GetContext(capsule)); + return; + } + case HPyFunc_MOD_CREATE: { + HPyFunc_unaryfunc f = (HPyFunc_unaryfunc)func; + _HPyFunc_args_UNARYFUNC *a = (_HPyFunc_args_UNARYFUNC*)args; + a->result = _h2py(f(ctx, _py2h(a->arg0))); + _HPyModule_CheckCreateSlotResult(&a->result); + return; + } +#include "autogen_ctx_call.i" + default: + Py_FatalError("Unsupported HPyFunc_Signature in ctx_meth.c"); + } +} diff --git a/graalpython/hpy/hpy/universal/src/ctx_meth.h b/graalpython/hpy/hpy/universal/src/ctx_meth.h new file mode 100644 index 0000000000..16e8a4b47c --- /dev/null +++ b/graalpython/hpy/hpy/universal/src/ctx_meth.h @@ -0,0 +1,6 @@ +#include "hpy.h" +#include "api.h" + +HPyAPI_IMPL void +ctx_CallRealFunctionFromTrampoline(HPyContext *ctx, HPyFunc_Signature sig, + HPyCFunction func, void *args); diff --git a/graalpython/hpy/hpy/universal/src/ctx_misc.c b/graalpython/hpy/hpy/universal/src/ctx_misc.c new file mode 100644 index 0000000000..753354d43c --- /dev/null +++ b/graalpython/hpy/hpy/universal/src/ctx_misc.c @@ -0,0 +1,83 @@ +#include +#include "hpy.h" +#include "handles.h" +#include "ctx_misc.h" + +HPyAPI_IMPL HPy +ctx_FromPyObject(HPyContext *ctx, cpy_PyObject *obj) +{ + Py_XINCREF(obj); + return _py2h(obj); +} + +HPyAPI_IMPL cpy_PyObject * +ctx_AsPyObject(HPyContext *ctx, HPy h) +{ + PyObject *obj = _h2py(h); + Py_XINCREF(obj); + return obj; +} + +HPyAPI_IMPL void +ctx_Close(HPyContext *ctx, HPy h) +{ + PyObject *obj = _h2py(h); + Py_XDECREF(obj); +} + +HPyAPI_IMPL HPy +ctx_Dup(HPyContext *ctx, HPy h) +{ + PyObject *obj = _h2py(h); + Py_XINCREF(obj); + return _py2h(obj); +} + +HPyAPI_IMPL void +ctx_Field_Store(HPyContext *ctx, HPy target_object, HPyField *target_field, HPy h) +{ + PyObject *obj = _h2py(h); + PyObject *target_py_obj = _hf2py(*target_field); + Py_XINCREF(obj); + *target_field = _py2hf(obj); + Py_XDECREF(target_py_obj); +} + +HPyAPI_IMPL HPy +ctx_Field_Load(HPyContext *ctx, HPy source_object, HPyField source_field) +{ + PyObject *obj = _hf2py(source_field); + Py_INCREF(obj); + return _py2h(obj); +} + + +HPyAPI_IMPL void +ctx_Global_Store(HPyContext *ctx, HPyGlobal *global, HPy h) +{ + PyObject *obj = _h2py(h); + Py_XDECREF(_hg2py(*global)); + Py_XINCREF(obj); + *global = _py2hg(obj); +} + +HPyAPI_IMPL HPy +ctx_Global_Load(HPyContext *ctx, HPyGlobal global) +{ + PyObject *obj = _hg2py(global); + Py_INCREF(obj); + return _py2h(obj); +} + +HPyAPI_IMPL void +ctx_FatalError(HPyContext *ctx, const char *message) +{ + Py_FatalError(message); +} + +HPyAPI_IMPL int +ctx_Type_IsSubtype(HPyContext *ctx, HPy sub, HPy type) +{ + return PyType_IsSubtype((PyTypeObject *)_h2py(sub), + (PyTypeObject *)_h2py(type)); +} diff --git a/graalpython/hpy/hpy/universal/src/ctx_misc.h b/graalpython/hpy/hpy/universal/src/ctx_misc.h new file mode 100644 index 0000000000..deb27ec5e5 --- /dev/null +++ b/graalpython/hpy/hpy/universal/src/ctx_misc.h @@ -0,0 +1,20 @@ +#ifndef HPY_CTX_MISC_H +#define HPY_CTX_MISC_H + +#include "hpy.h" +#include "api.h" + +HPyAPI_IMPL HPy ctx_FromPyObject(HPyContext *ctx, cpy_PyObject *obj); +HPyAPI_IMPL cpy_PyObject *ctx_AsPyObject(HPyContext *ctx, HPy h); +HPyAPI_IMPL void ctx_Close(HPyContext *ctx, HPy h); +HPyAPI_IMPL HPy ctx_Dup(HPyContext *ctx, HPy h); +HPyAPI_IMPL void ctx_Field_Store(HPyContext *ctx, HPy target_object, + HPyField *target_field, HPy h); +HPyAPI_IMPL HPy ctx_Field_Load(HPyContext *ctx, HPy source_object, + HPyField source_field); +HPyAPI_IMPL void ctx_Global_Store(HPyContext *ctx, HPyGlobal *global, HPy h); +HPyAPI_IMPL HPy ctx_Global_Load(HPyContext *ctx, HPyGlobal global); +HPyAPI_IMPL void ctx_FatalError(HPyContext *ctx, const char *message); +HPyAPI_IMPL int ctx_Type_IsSubtype(HPyContext *ctx, HPy sub, HPy type); + +#endif /* HPY_CTX_MISC_H */ diff --git a/graalpython/hpy/hpy/universal/src/handles.h b/graalpython/hpy/hpy/universal/src/handles.h new file mode 100644 index 0000000000..3d0282d786 --- /dev/null +++ b/graalpython/hpy/hpy/universal/src/handles.h @@ -0,0 +1,58 @@ +#ifndef HPY_HANDLES_H +#define HPY_HANDLES_H + +#include +#include "hpy.h" + +// The main reason for +1/-1 is to make sure that if people casts HPy to +// PyObject* directly, things explode. Moreover, with this we can easily +// distinguish normal and debug handles in gdb, by only looking at the last +// bit. + +static inline HPy _py2h(PyObject *obj) { + if (obj == NULL) + return HPy_NULL; + return (HPy){(HPy_ssize_t)obj + 1}; +} + +static inline PyObject *_h2py(HPy h) { + if HPy_IsNull(h) + return NULL; + return (PyObject *)(h._i - 1); +} + +static inline HPyField _py2hf(PyObject *obj) +{ + HPy h = _py2h(obj); + return (HPyField){ ._i = h._i }; +} + +static inline PyObject * _hf2py(HPyField hf) +{ + HPy h = { ._i = hf._i }; + return _h2py(h); +} + +static inline HPyThreadState _threads2h(PyThreadState* s) +{ + return (HPyThreadState) { (intptr_t) s }; +} + +static inline PyThreadState* _h2threads(HPyThreadState h) +{ + return (PyThreadState*) h._i; +} + +static inline HPyGlobal _py2hg(PyObject *obj) +{ + HPy h = _py2h(obj); + return (HPyGlobal){ ._i = h._i }; +} + +static inline PyObject * _hg2py(HPyGlobal hf) +{ + HPy h = { ._i = hf._i }; + return _h2py(h); +} + +#endif /* HPY_HANDLES_H */ diff --git a/graalpython/hpy/hpy/universal/src/hpymodule.c b/graalpython/hpy/hpy/universal/src/hpymodule.c new file mode 100644 index 0000000000..b86e418e61 --- /dev/null +++ b/graalpython/hpy/hpy/universal/src/hpymodule.c @@ -0,0 +1,691 @@ +#define PY_SSIZE_T_CLEAN +#include +#ifdef MS_WIN32 +# include +# include "misc_win32.h" +#else +# include +// required for strncasecmp +#include +#endif +#include +#include + + +#include "api.h" +#include "handles.h" +#include "hpy/version.h" +#include "hpy_debug.h" +#include "hpy_trace.h" +#include "hpy/runtime/ctx_module.h" + +#ifdef PYPY_VERSION +# error "Cannot build hpy.universal on top of PyPy. PyPy comes with its own version of it" +#endif +#ifdef GRAALVM_PYTHON +# error "Cannot build hpy.universal on top of GraalPy. GraalPy comes with its own version of it" +#endif + +static const char *hpy_mode_names[] = { + "MODE_UNIVERSAL", + "MODE_DEBUG", + "MODE_TRACE", + // "MODE_DEBUG_TRACE", + // "MODE_TRACE_DEBUG", + NULL +}; + +typedef enum { + MODE_INVALID = -1, + MODE_UNIVERSAL = 0, + MODE_DEBUG = 1, + MODE_TRACE = 2, + /* We do currently not test the combinations of debug and trace mode, so we + do not offer them right now. This may change in future. */ + // MODE_DEBUG_TRACE = 3, + // MODE_TRACE_DEBUG = 4 +} HPyMode; + +typedef uint32_t (*VersionGetterFuncPtr)(void); +typedef HPyModuleDef* (*InitFuncPtr)(void); +typedef void (*InitContextFuncPtr)(HPyContext*); + +static const char *init_prefix = "HPyInit"; +static const char *init_ctx_prefix = "HPyInitGlobalContext_"; + +static inline int +_hpy_strncmp_ignore_case(const char *s0, const char *s1, size_t n) +{ +#ifdef MS_WIN32 + return _strnicmp(s0, s1, n); +#else + return strncasecmp(s0, s1, n); +#endif +} + +static HPyContext * get_context(HPyMode mode) +{ + switch (mode) + { + case MODE_INVALID: + return NULL; + case MODE_DEBUG: + return hpy_debug_get_ctx(&g_universal_ctx); + case MODE_TRACE: + return hpy_trace_get_ctx(&g_universal_ctx); + // case MODE_DEBUG_TRACE: + // return hpy_debug_get_ctx(hpy_trace_get_ctx(&g_universal_ctx)); + // case MODE_TRACE_DEBUG: + // return hpy_trace_get_ctx(hpy_debug_get_ctx(&g_universal_ctx)); + default: + return &g_universal_ctx; + } +} + +static PyObject * +get_encoded_name(PyObject *name) { + PyObject *tmp; + PyObject *encoded = NULL; + PyObject *modname = NULL; + Py_ssize_t name_len, lastdot; + + /* Get the short name (substring after last dot) */ + name_len = PyUnicode_GetLength(name); + lastdot = PyUnicode_FindChar(name, '.', 0, name_len, -1); + if (lastdot < -1) { + return NULL; + } else if (lastdot >= 0) { + tmp = PyUnicode_Substring(name, lastdot + 1, name_len); + if (tmp == NULL) + return NULL; + name = tmp; + /* "name" now holds a new reference to the substring */ + } else { + Py_INCREF(name); + } + + /* Encode to ASCII */ + encoded = PyUnicode_AsEncodedString(name, "ascii", NULL); + if (encoded != NULL) { + } else { + goto error; + } + + /* Replace '-' by '_' */ + modname = PyObject_CallMethod(encoded, "replace", "cc", '-', '_'); + if (modname == NULL) + goto error; + + Py_DECREF(name); + Py_DECREF(encoded); + return modname; +error: + Py_DECREF(name); + Py_XDECREF(encoded); + return NULL; +} + +static bool validate_abi_tag(const char *shortname, const char *soname, + uint32_t required_major_version, uint32_t required_minor_version) { + char *substr = strstr(soname, ".hpy"); + if (substr != NULL) { + substr += strlen(".hpy"); + if (*substr >= '0' && *substr <= '9') { + // It is a number w/o sign and whitespace, we can now parse it with atoi + uint32_t abi_tag = (uint32_t) atoi(substr); + if (abi_tag == required_major_version) { + return true; + } + PyErr_Format(PyExc_RuntimeError, + "HPy extension module '%s' at path '%s': mismatch between the " + "HPy ABI tag encoded in the filename and the major version requested " + "by the HPy extension itself. Major version tag parsed from " + "filename: %" PRIu32 ". Requested version: %" PRIu32 ".%" PRIu32 ".", + shortname, soname, abi_tag, required_major_version, required_minor_version); + return false; + } + } + PyErr_Format(PyExc_RuntimeError, + "HPy extension module '%s' at path '%s': could not find " + "HPy ABI tag encoded in the filename. The extension claims to be compiled with " + "HPy ABI version: %" PRIu32 ".%" PRIu32 ".", + shortname, soname, required_major_version, required_minor_version); + return false; +} + +static void dlsym_error(const char *soname, const char *symbol_name) { + const char *error = dlerror(); + if (error == NULL) + error = "no error message provided by the system"; + PyErr_Format(PyExc_RuntimeError, + "Error during loading of the HPy extension module at " + "path '%s' while trying to find symbol '%s'. Did you use" + "the HPy_MODINIT macro to register your module? Error " + "message from dlsym/WinAPI: %s", soname, symbol_name, error); +} + +static PyObject *do_load(PyObject *name_unicode, PyObject *path, HPyMode mode, PyObject *spec) +{ + PyObject *name = NULL; + PyObject *pathbytes = NULL; + PyModuleDef *pydef = NULL; + PyObject *py_mod = NULL; + + name = get_encoded_name(name_unicode); + if (name == NULL) { + goto error; + } + + pathbytes = PyUnicode_EncodeFSDefault(path); + if (pathbytes == NULL) + goto error; + const char *soname = PyBytes_AS_STRING(pathbytes); + + void *mylib = dlopen(soname, RTLD_NOW); // who closes this? + if (mylib == NULL) { + const char *error = dlerror(); + if (error == NULL) + error = "no error message provided by the system"; + PyErr_Format(PyExc_RuntimeError, + "Error during loading of the HPy extension module at path " + "'%s'. Error message from dlopen/WinAPI: %s", soname, error); + goto error; + } + + const char *shortname = PyBytes_AS_STRING(name); + char minor_version_symbol_name[258]; + char major_version_symbol_name[258]; + PyOS_snprintf(minor_version_symbol_name, sizeof(minor_version_symbol_name), + "get_required_hpy_minor_version_%.200s", shortname); + PyOS_snprintf(major_version_symbol_name, sizeof(major_version_symbol_name), + "get_required_hpy_major_version_%.200s", shortname); + void *minor_version_ptr = dlsym(mylib, minor_version_symbol_name); + void *major_version_ptr = dlsym(mylib, major_version_symbol_name); + if (minor_version_ptr == NULL || major_version_ptr == NULL) { + const char *error = dlerror(); + if (error == NULL) + error = "no error message provided by the system"; + PyErr_Format(PyExc_RuntimeError, + "Error during loading of the HPy extension module at path " + "'%s'. Cannot locate the required minimal HPy versions as symbols '%s' and `%s`. " + "Error message from dlopen/WinAPI: %s", + soname, minor_version_symbol_name, major_version_symbol_name, error); + goto error; + } + uint32_t required_minor_version = ((VersionGetterFuncPtr) minor_version_ptr)(); + uint32_t required_major_version = ((VersionGetterFuncPtr) major_version_ptr)(); + if (required_major_version != HPY_ABI_VERSION || required_minor_version > HPY_ABI_VERSION_MINOR) { + // For now, we have only one major version, but in the future at this + // point we would decide which HPyContext to create + PyErr_Format(PyExc_RuntimeError, + "HPy extension module '%s' requires unsupported version of the HPy runtime. " + "Requested version: %" PRIu32 ".%" PRIu32 ". Current HPy version: %" PRIu32 ".%" PRIu32 ".", + shortname, required_major_version, required_minor_version, + HPY_ABI_VERSION, HPY_ABI_VERSION_MINOR); + goto error; + } + + // Sanity check of the tag in the shared object filename + if (!validate_abi_tag(shortname, soname, required_major_version, required_minor_version)) { + goto error; + } + + HPyContext *ctx = get_context(mode); + if (ctx == NULL) + goto error; + + char init_ctx_name[258]; + PyOS_snprintf(init_ctx_name, sizeof(init_ctx_name), "%.20s_%.200s", + init_ctx_prefix, shortname); + void *initctxfn = dlsym(mylib, init_ctx_name); + if (initctxfn == NULL) { + dlsym_error(soname, init_ctx_name); + goto error; + } + ((InitContextFuncPtr)initctxfn)(ctx); + + char init_name[258]; + PyOS_snprintf(init_name, sizeof(init_name), "%.20s_%.200s", + init_prefix, shortname); + void *initfn = dlsym(mylib, init_name); + if (initfn == NULL) { + dlsym_error(soname, init_name); + goto error; + } + + HPyModuleDef* hpydef = ((InitFuncPtr)initfn)(); + if (hpydef == NULL) { + PyErr_Format(PyExc_RuntimeError, + "Error during loading of the HPy extension module at " + "path '%s'. Function '%s' returned NULL.", soname, init_name); + goto error; + } + + pydef = _HPyModuleDef_CreatePyModuleDef(hpydef); + if (pydef == NULL) { + goto error; + } + + py_mod = PyModule_FromDefAndSpec(pydef, spec); + if (py_mod == NULL) + goto error; + + if (PyModule_Check(py_mod)) { + if (PyModule_ExecDef(py_mod, pydef) != 0) + goto error; + } + + Py_XDECREF(name); + Py_XDECREF(pathbytes); + return py_mod; +error: + Py_XDECREF(py_mod); + if (pydef != NULL) + PyMem_Free(pydef); + Py_XDECREF(name); + Py_XDECREF(pathbytes); + return NULL; +} + +static PyObject *load(PyObject *self, PyObject *args, PyObject *kwargs) +{ + static char *kwlist[] = {"name", "path", "spec", "debug", "mode", NULL}; + PyObject *name_unicode; + PyObject *path; + PyObject *spec; + int debug = 0; + int mode = -1; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OOO|pi", kwlist, + &name_unicode, &path, &spec, &debug, &mode)) { + return NULL; + } + HPyMode hmode = debug ? MODE_DEBUG : MODE_UNIVERSAL; + // 'mode' just overwrites 'debug' + if (mode > 0) + { + hmode = (HPyMode) mode; + } + return do_load(name_unicode, path, hmode, spec); +} + +static HPyMode get_mode_from_string(const char *s, Py_ssize_t n) +{ + if (_hpy_strncmp_ignore_case("debug", s, n) == 0) + return MODE_DEBUG; + if (_hpy_strncmp_ignore_case("trace", s, n) == 0) + return MODE_TRACE; + // if (_hpy_strncmp_ignore_case("debug+trace", s, n) == 0) + // return MODE_DEBUG_TRACE; + // if (_hpy_strncmp_ignore_case("trace+debug", s, n) == 0) + // return MODE_TRACE_DEBUG; + if (_hpy_strncmp_ignore_case("universal", s, n) == 0) + return MODE_UNIVERSAL; + return MODE_INVALID; +} + +/* + * A little helper that does a fast-path if 'mapping' is a dict. + */ +static int mapping_get_item(PyObject *mapping, const char *skey, PyObject **value) +{ + PyObject *key = PyUnicode_FromString(skey); + if (key == NULL) + return 1; + + // fast-path if 'mapping' is a dict + if (PyDict_Check(mapping)) { + *value = PyDict_GetItem(mapping, key); + Py_DECREF(key); + /* 'NULL' means, the key is not present in the dict, so just return + 'NULL'. Since PyDict_GetItem does not set an error, we don't need to + clear any error. */ + } else { + *value = PyObject_GetItem(mapping, key); + Py_DECREF(key); + if (*value == NULL) { + if (PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Clear(); + } else { + return 1; + } + } + } + return 0; +} + +/* + * HPY_MODE := MODE | (MODULE_NAME ':' MODE { ',' MODULE_NAME ':' MODE }) + * MODULE_NAME := IDENTIFIER + * MODE := 'debug' | 'trace' | 'universal' + */ +static HPyMode get_hpy_mode_from_environ(const char *s_name, PyObject *env) +{ + PyObject *value; + Py_ssize_t size; + HPyMode res; + const char *s_value; + + if (mapping_get_item(env, "HPY", &value)) { + return MODE_INVALID; + } + + /* 'value == NULL' is not an error; this just means that the key was not + present in 'env'. */ + if (value == NULL) { + return MODE_UNIVERSAL; + } + + s_value = PyUnicode_AsUTF8AndSize(value, &size); + if (s_value == NULL) { + Py_DECREF(value); + return MODE_INVALID; + } + res = MODE_INVALID; + char *colon = strchr(s_value, ':'); + if (colon) { + // case 2: modes are specified per module + char *name_start = (char *)s_value; + char *comma; + size_t mode_len; + while (colon) { + comma = strchr(colon + 1, ','); + if (comma) { + mode_len = comma - colon - 1; + } else { + mode_len = (s_value + size) - colon - 1; + } + if (strncmp(s_name, name_start, colon - name_start) == 0) { + res = get_mode_from_string(colon + 1, mode_len); + break; + } + if (comma) { + name_start = comma + 1; + colon = strchr(name_start, ':'); + } else { + colon = NULL; + } + } + } else { + // case 1: mode was globally specified + res = get_mode_from_string(s_value, size); + } + + if (res == MODE_INVALID) + PyErr_Format(PyExc_ValueError, "invalid HPy mode: %.50s", s_value); + Py_DECREF(value); + return res; +} + +static PyObject * +load_bootstrap(PyObject *self, PyObject *args, PyObject *kwargs) +{ + static char *kwlist[] = {"name", "ext_name", "package", "file", "loader", "spec", + "env", NULL}; + PyObject *module_name, *name, *package, *file, *loader, *spec, *env; + PyObject *log_obj, *m; + HPyMode hmode; + const char *s_module_name, *log_msg; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OOOOOOO", kwlist, + &module_name, &name, &package, &file, + &loader, &spec, &env)) { + return NULL; + } + + s_module_name = PyUnicode_AsUTF8AndSize(module_name, NULL); + if (s_module_name == NULL) { + return NULL; + } + + hmode = get_hpy_mode_from_environ(s_module_name, env); + if (hmode == MODE_INVALID) + return NULL; + + if (mapping_get_item(env, "HPY_LOG", &log_obj)) + return NULL; + + if (log_obj != NULL) { + Py_DECREF(log_obj); + switch (hmode) { + case MODE_DEBUG: + log_msg = " with a debug context"; + break; + case MODE_TRACE: + log_msg = " with a trace context"; + break; + case MODE_UNIVERSAL: + log_msg = ""; + break; + default: + // that's not possible but required for the compiler + return NULL; + } + PySys_FormatStdout("Loading '%.200s' in HPy universal mode%.200s\n", + s_module_name, log_msg); + } + + m = do_load(module_name, file, hmode, spec); + if (m == NULL) + return NULL; + + PyObject_SetAttrString(m, "__file__", file); + PyObject_SetAttrString(m, "__loader__", loader); + PyObject_SetAttrString(m, "__name__", name); + PyObject_SetAttrString(m, "__package__", package); + PyObject_SetAttrString(spec, "origin", file); + PyObject_SetAttrString(m, "__spec__", spec); + + return m; +} + +static PyObject *get_version(PyObject *self, PyObject *ignored) +{ + return Py_BuildValue("ss", HPY_VERSION, HPY_GIT_REVISION); +} + +PyDoc_STRVAR(load_bootstrap_doc, "Internal function intended to be used by " + "the stub loader. This function will honor env var 'HPY' and correctly" + " set the attributes of the module."); + +static PyMethodDef HPyMethods[] = { + {"load", (PyCFunction)load, METH_VARARGS | METH_KEYWORDS, + ("Load a ." HPY_ABI_TAG ".so file")}, + {"_load_bootstrap", (PyCFunction)load_bootstrap, + METH_VARARGS | METH_KEYWORDS, load_bootstrap_doc}, + {"get_version", (PyCFunction)get_version, METH_NOARGS, + "Return a tuple ('version', 'git revision')"}, + {NULL, NULL, 0, NULL} +}; + + +static int exec_module(PyObject *mod); +static PyModuleDef_Slot hpymodule_slots[] = { + {Py_mod_exec, exec_module}, + {0, NULL}, +}; + +static struct PyModuleDef hpy_pydef = { + PyModuleDef_HEAD_INIT, + .m_name = "hpy.universal", + .m_doc = "HPy universal runtime for CPython", + .m_size = 0, + .m_methods = HPyMethods, + .m_slots = hpymodule_slots, +}; + +static int initialize_module(HPyContext *ctx, PyObject *hpy_universal, const char* name, + const char *full_name, HPyModuleDef *hpydef, + PyObject* spec_from_file_and_location, PyObject *location) +{ + PyObject *spec = NULL, *new_mod = NULL; + int result = -1; + + spec = PyObject_CallFunction(spec_from_file_and_location, "sO", full_name, location); + PyModuleDef *pydef = _HPyModuleDef_CreatePyModuleDef(hpydef); + new_mod = PyModule_FromDefAndSpec(pydef, spec); + if (new_mod == NULL) + goto cleanup; + + if (PyModule_ExecDef(new_mod, pydef) != 0) + goto cleanup; + + Py_INCREF(new_mod); // PyModule_AddObject steals the reference + if (PyModule_AddObject(hpy_universal, name, new_mod) < 0) + goto cleanup; + + result = 0; + +cleanup: + Py_XDECREF(new_mod); + Py_XDECREF(spec); + return result; +} + +// module initialization function +int exec_module(PyObject* mod) { + HPyContext *ctx = &g_universal_ctx; + + PyObject *importlib_util = PyImport_ImportModule("importlib.util"); + if (importlib_util == NULL) + return -1; + + PyObject *spec_from_file_and_location = PyObject_GetAttrString(importlib_util, "spec_from_file_location"); + Py_DecRef(importlib_util); + if (spec_from_file_and_location == NULL) + return -1; + + PyObject *current_mod_spec = PyObject_GetAttrString(mod, "__spec__"); + if (current_mod_spec == NULL) + return -1; + + PyObject *location = PyObject_GetAttrString(current_mod_spec, "origin"); + if (location == NULL) + return -1; + + HPyInitGlobalContext__debug(ctx); + int result = initialize_module(ctx, mod, "_debug", "hpy.debug._debug", + HPyInit__debug(), spec_from_file_and_location, location); + if (result != 0) + return result; + + HPyInitGlobalContext__trace(ctx); + result = initialize_module(ctx, mod, "_trace", "hpy.trace._trace", + HPyInit__trace(), spec_from_file_and_location, location); + if (result != 0) + return result; + + for (int i=0; hpy_mode_names[i]; i++) + { + if (PyModule_AddIntConstant(mod, hpy_mode_names[i], i) < 0) + return -1; + } + + return 0; +} + +static void init_universal_ctx(HPyContext *ctx) +{ + if (!HPy_IsNull(ctx->h_None)) + // already initialized + return; + + // XXX this code is basically the same as found in cpython/hpy.h. We + // should probably share and/or autogenerate both versions + /* Constants */ + ctx->h_None = _py2h(Py_None); + ctx->h_True = _py2h(Py_True); + ctx->h_False = _py2h(Py_False); + ctx->h_NotImplemented = _py2h(Py_NotImplemented); + ctx->h_Ellipsis = _py2h(Py_Ellipsis); + /* Exceptions */ + ctx->h_BaseException = _py2h(PyExc_BaseException); + ctx->h_Exception = _py2h(PyExc_Exception); + ctx->h_StopAsyncIteration = _py2h(PyExc_StopAsyncIteration); + ctx->h_StopIteration = _py2h(PyExc_StopIteration); + ctx->h_GeneratorExit = _py2h(PyExc_GeneratorExit); + ctx->h_ArithmeticError = _py2h(PyExc_ArithmeticError); + ctx->h_LookupError = _py2h(PyExc_LookupError); + ctx->h_AssertionError = _py2h(PyExc_AssertionError); + ctx->h_AttributeError = _py2h(PyExc_AttributeError); + ctx->h_BufferError = _py2h(PyExc_BufferError); + ctx->h_EOFError = _py2h(PyExc_EOFError); + ctx->h_FloatingPointError = _py2h(PyExc_FloatingPointError); + ctx->h_OSError = _py2h(PyExc_OSError); + ctx->h_ImportError = _py2h(PyExc_ImportError); + ctx->h_ModuleNotFoundError = _py2h(PyExc_ModuleNotFoundError); + ctx->h_IndexError = _py2h(PyExc_IndexError); + ctx->h_KeyError = _py2h(PyExc_KeyError); + ctx->h_KeyboardInterrupt = _py2h(PyExc_KeyboardInterrupt); + ctx->h_MemoryError = _py2h(PyExc_MemoryError); + ctx->h_NameError = _py2h(PyExc_NameError); + ctx->h_OverflowError = _py2h(PyExc_OverflowError); + ctx->h_RuntimeError = _py2h(PyExc_RuntimeError); + ctx->h_RecursionError = _py2h(PyExc_RecursionError); + ctx->h_NotImplementedError = _py2h(PyExc_NotImplementedError); + ctx->h_SyntaxError = _py2h(PyExc_SyntaxError); + ctx->h_IndentationError = _py2h(PyExc_IndentationError); + ctx->h_TabError = _py2h(PyExc_TabError); + ctx->h_ReferenceError = _py2h(PyExc_ReferenceError); + ctx->h_SystemError = _py2h(PyExc_SystemError); + ctx->h_SystemExit = _py2h(PyExc_SystemExit); + ctx->h_TypeError = _py2h(PyExc_TypeError); + ctx->h_UnboundLocalError = _py2h(PyExc_UnboundLocalError); + ctx->h_UnicodeError = _py2h(PyExc_UnicodeError); + ctx->h_UnicodeEncodeError = _py2h(PyExc_UnicodeEncodeError); + ctx->h_UnicodeDecodeError = _py2h(PyExc_UnicodeDecodeError); + ctx->h_UnicodeTranslateError = _py2h(PyExc_UnicodeTranslateError); + ctx->h_ValueError = _py2h(PyExc_ValueError); + ctx->h_ZeroDivisionError = _py2h(PyExc_ZeroDivisionError); + ctx->h_BlockingIOError = _py2h(PyExc_BlockingIOError); + ctx->h_BrokenPipeError = _py2h(PyExc_BrokenPipeError); + ctx->h_ChildProcessError = _py2h(PyExc_ChildProcessError); + ctx->h_ConnectionError = _py2h(PyExc_ConnectionError); + ctx->h_ConnectionAbortedError = _py2h(PyExc_ConnectionAbortedError); + ctx->h_ConnectionRefusedError = _py2h(PyExc_ConnectionRefusedError); + ctx->h_ConnectionResetError = _py2h(PyExc_ConnectionResetError); + ctx->h_FileExistsError = _py2h(PyExc_FileExistsError); + ctx->h_FileNotFoundError = _py2h(PyExc_FileNotFoundError); + ctx->h_InterruptedError = _py2h(PyExc_InterruptedError); + ctx->h_IsADirectoryError = _py2h(PyExc_IsADirectoryError); + ctx->h_NotADirectoryError = _py2h(PyExc_NotADirectoryError); + ctx->h_PermissionError = _py2h(PyExc_PermissionError); + ctx->h_ProcessLookupError = _py2h(PyExc_ProcessLookupError); + ctx->h_TimeoutError = _py2h(PyExc_TimeoutError); + /* Warnings */ + ctx->h_Warning = _py2h(PyExc_Warning); + ctx->h_UserWarning = _py2h(PyExc_UserWarning); + ctx->h_DeprecationWarning = _py2h(PyExc_DeprecationWarning); + ctx->h_PendingDeprecationWarning = _py2h(PyExc_PendingDeprecationWarning); + ctx->h_SyntaxWarning = _py2h(PyExc_SyntaxWarning); + ctx->h_RuntimeWarning = _py2h(PyExc_RuntimeWarning); + ctx->h_FutureWarning = _py2h(PyExc_FutureWarning); + ctx->h_ImportWarning = _py2h(PyExc_ImportWarning); + ctx->h_UnicodeWarning = _py2h(PyExc_UnicodeWarning); + ctx->h_BytesWarning = _py2h(PyExc_BytesWarning); + ctx->h_ResourceWarning = _py2h(PyExc_ResourceWarning); + /* Types */ + ctx->h_BaseObjectType = _py2h((PyObject *)&PyBaseObject_Type); + ctx->h_TypeType = _py2h((PyObject *)&PyType_Type); + ctx->h_BoolType = _py2h((PyObject *)&PyBool_Type); + ctx->h_LongType = _py2h((PyObject *)&PyLong_Type); + ctx->h_FloatType = _py2h((PyObject *)&PyFloat_Type); + ctx->h_UnicodeType = _py2h((PyObject *)&PyUnicode_Type); + ctx->h_TupleType = _py2h((PyObject *)&PyTuple_Type); + ctx->h_ListType = _py2h((PyObject *)&PyList_Type); + ctx->h_ComplexType = _py2h((PyObject *)&PyComplex_Type); + ctx->h_BytesType = _py2h((PyObject *)&PyBytes_Type); + ctx->h_MemoryViewType = _py2h((PyObject *)&PyMemoryView_Type); + ctx->h_CapsuleType = _py2h((PyObject *)&PyCapsule_Type); + ctx->h_SliceType = _py2h((PyObject *)&PySlice_Type); + ctx->h_DictType = _py2h((PyObject *)&PyDict_Type); + /* Reflection */ + ctx->h_Builtins = _py2h(PyEval_GetBuiltins()); +} + + +PyMODINIT_FUNC +PyInit_universal(void) +{ + init_universal_ctx(&g_universal_ctx); + PyObject *mod = PyModuleDef_Init(&hpy_pydef); + return mod; +} diff --git a/graalpython/hpy/hpy/universal/src/misc_win32.h b/graalpython/hpy/hpy/universal/src/misc_win32.h new file mode 100644 index 0000000000..d267e758e7 --- /dev/null +++ b/graalpython/hpy/hpy/universal/src/misc_win32.h @@ -0,0 +1,55 @@ +/************************************************************/ +/* Emulate dlopen()&co. from the Windows API */ + +#define RTLD_LAZY 0 +#define RTLD_NOW 0 +#define RTLD_GLOBAL 0 +#define RTLD_LOCAL 0 + +static void *dlopen(const char *filename, int flag) +{ + return (void *)LoadLibraryA(filename); +} + +static void *dlopenW(const wchar_t *filename) +{ + return (void *)LoadLibraryW(filename); +} + +static void *dlsym(void *handle, const char *symbol) +{ + void *address = GetProcAddress((HMODULE)handle, symbol); +#ifndef MS_WIN64 + if (!address) { + /* If 'symbol' is not found, then try '_symbol@N' for N in + (0, 4, 8, 12, ..., 124). Unlike ctypes, we try to do that + for any symbol, although in theory it should only be done + for __stdcall functions. + */ + int i; + char mangled_name[1 + strlen(symbol) + 1 + 3 + 1]; + for (i = 0; i < 32; i++) { + sprintf(mangled_name, "_%s@%d", symbol, i * 4); + address = GetProcAddress((HMODULE)handle, mangled_name); + if (address) + break; + } + } +#endif + return address; +} + +static int dlclose(void *handle) +{ + return FreeLibrary((HMODULE)handle) ? 0 : -1; +} + +static const char *dlerror(void) +{ + static char buf[32]; + DWORD dw = GetLastError(); + if (dw == 0) + return NULL; + sprintf(buf, "error 0x%x", (unsigned int)dw); + return buf; +} diff --git a/graalpython/hpy/microbench/README.md b/graalpython/hpy/microbench/README.md new file mode 100644 index 0000000000..c25916088c --- /dev/null +++ b/graalpython/hpy/microbench/README.md @@ -0,0 +1,29 @@ +To run the microbenchmarks +-------------------------- + +1. You need to have `hpy` installed in your virtuanenv. The easiest way + to do it is: + + $ cd /path/to/hpy + $ python setup.py develop + +2. Build the extension modules needed for the microbenchmarks + + $ cd /path/to/hpy/microbench + $ pip install cffi # needed to build _valgrind + $ python setup.py build_ext --inplace + +2. `py.test -v` + +3. To run only cpy or hpy tests, use -m (to select markers): + + $ py.test -v -m hpy + $ py.test -v -m cpy + +4. Step (2) build `hpy_simple` using the CPython ABI by default. If you want + to benchmark the universal mode, you need to build it explicitly: + + $ cd /path/to/hpy/microbench + $ rm *.so # make sure to delete CPython-ABI versions + $ python setup.py --hpy-abi=universal build_ext --inplace + $ py.test -v diff --git a/graalpython/hpy/microbench/_valgrind_build.py b/graalpython/hpy/microbench/_valgrind_build.py new file mode 100644 index 0000000000..4cbbc1ec7d --- /dev/null +++ b/graalpython/hpy/microbench/_valgrind_build.py @@ -0,0 +1,29 @@ +from cffi import FFI +ffibuilder = FFI() + +ffibuilder.cdef(""" + void callgrind_start(void); + void callgrind_stop(void); + int is_running_on_valgrind(void); +""") + + +ffibuilder.set_source("_valgrind", """ +#include +void callgrind_start(void) { + CALLGRIND_START_INSTRUMENTATION; +} + +void callgrind_stop(void) { + CALLGRIND_STOP_INSTRUMENTATION; + CALLGRIND_DUMP_STATS; +} + +int is_running_on_valgrind(void) { + return RUNNING_ON_VALGRIND; +} +""", + libraries=[]) + +if __name__ == "__main__": + ffibuilder.compile(verbose=True) diff --git a/graalpython/hpy/microbench/conftest.py b/graalpython/hpy/microbench/conftest.py new file mode 100644 index 0000000000..8e553ffe76 --- /dev/null +++ b/graalpython/hpy/microbench/conftest.py @@ -0,0 +1,124 @@ +import re +import time +from collections import defaultdict +import pytest +import _valgrind + +class Timer: + + def __init__(self, nodeid): + self.nodeid = nodeid + self.start = None + self.stop = None + + def __enter__(self): + if self.start is not None: + raise ValueError('You cannot use "with timer:" more than once') + _valgrind.lib.callgrind_start() + self.start = time.time() + + def __exit__(self, etype, evalue, tb): + self.stop = time.time() + _valgrind.lib.callgrind_stop() + + def __str__(self): + if self.start is None: + return '[NO TIMING]' + if self.stop is None: + return '[IN-PROGRESS]' + usec = (self.stop - self.start) * 1000 + return f'{usec:.2f} us' + + @property + def elapsed(self): + if self.start is not None and self.stop is not None: + return self.stop - self.start + return None + + +class TimerSession: + + NODEID = re.compile(r'(.*)\[(.*)\]') + + def __init__(self): + self.apis = set() # ['cpy', 'hpy', ...] + self.table = defaultdict(dict) # {shortid: {api: timer}} + self.timers = {} # nodeid -> Timer + + def new_timer(self, nodeid): + shortid, api = self.split_nodeid(nodeid) + timer = Timer(nodeid) + self.apis.add(api) + self.table[shortid][api] = timer + self.timers[nodeid] = timer + return timer + + def get_timer(self, nodeid): + return self.timers.get(nodeid) + + def split_nodeid(self, nodeid): + shortid = '::'.join(nodeid.split('::')[-2:]) # take only class::function + m = self.NODEID.match(shortid) + if not m: + return shortid, '' + return m.group(1), m.group(2) + + def format_ratio(self, reference, value): + if reference and reference.elapsed and value and value.elapsed: + ratio = value.elapsed / reference.elapsed + return f'[{ratio:.2f}]' + return '' + + def display_summary(self, tr): + w = tr.write_line + w('') + tr.write_sep('=', 'BENCHMARKS', cyan=True) + w(' '*40 + ' cpy hpy') + w(' '*40 + '---------------- -------------------') + for shortid, timings in self.table.items(): + cpy = timings.get('cpy') + hpy = timings.get('hpy') + hpy_ratio = self.format_ratio(cpy, hpy) + cpy = cpy or '' + hpy = hpy or '' + w(f'{shortid:<40} {cpy!s:>15} {hpy!s:>15} {hpy_ratio}') + w('') + + + +@pytest.fixture +def timer(request, api): + nodeid = request.node.nodeid + return request.config._timersession.new_timer(nodeid) + +def pytest_configure(config): + config._timersession = TimerSession() + config.addinivalue_line("markers", "hpy: mark modules using the HPy API") + config.addinivalue_line("markers", "cpy: mark modules using the old Python/C API") + +def pytest_addoption(parser): + parser.addoption( + "--fast", action="store_true", default=False, help="run microbench faster" + ) + parser.addoption( + "--slow", action="store_true", default=False, help="run microbench slower" + ) + + +VERBOSE_TEST_NAME_LENGTH = 90 + +@pytest.hookimpl(hookwrapper=True) +def pytest_report_teststatus(report, config): + outcome = yield + category, letter, word = outcome.get_result() + timer = config._timersession.get_timer(report.nodeid) + if category == 'passed' and timer: + L = VERBOSE_TEST_NAME_LENGTH - len(report.nodeid) + word = str(timer).rjust(L) + markup = None + if timer.elapsed is None: + markup = {'yellow': True} + outcome.force_result((category, letter, (word, markup))) + +def pytest_terminal_summary(terminalreporter, config): + config._timersession.display_summary(terminalreporter) diff --git a/graalpython/hpy/microbench/pytest_valgrind.sh b/graalpython/hpy/microbench/pytest_valgrind.sh new file mode 100755 index 0000000000..8271381d0b --- /dev/null +++ b/graalpython/hpy/microbench/pytest_valgrind.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +valgrind --tool=callgrind --instr-atstart=no python3-dbg -m pytest "$@" diff --git a/graalpython/hpy/microbench/setup.py b/graalpython/hpy/microbench/setup.py new file mode 100644 index 0000000000..16988b0aef --- /dev/null +++ b/graalpython/hpy/microbench/setup.py @@ -0,0 +1,17 @@ +from setuptools import setup, Extension + +setup( + name="hpy.microbench", + setup_requires=['cffi', 'hpy'], + ext_modules = [ + Extension('cpy_simple', + ['src/cpy_simple.c'], + extra_compile_args=['-g']) + ], + hpy_ext_modules = [ + Extension('hpy_simple', + ['src/hpy_simple.c'], + extra_compile_args=['-g']), + ], + cffi_modules=["_valgrind_build.py:ffibuilder"], +) diff --git a/graalpython/hpy/microbench/src/cpy_simple.c b/graalpython/hpy/microbench/src/cpy_simple.c new file mode 100644 index 0000000000..dc754de64e --- /dev/null +++ b/graalpython/hpy/microbench/src/cpy_simple.c @@ -0,0 +1,174 @@ +#include + +/* module-level functions */ + +static PyObject* noargs(PyObject* self, PyObject* args) +{ + Py_RETURN_NONE; +} + +static PyObject* onearg(PyObject* self, PyObject* arg) +{ + Py_RETURN_NONE; +} + +static PyObject* varargs(PyObject* self, PyObject* args) +{ + Py_RETURN_NONE; +} + +static PyObject* call_with_tuple(PyObject* self, PyObject* args) +{ + PyObject* f; + PyObject* f_args; + f = PyTuple_GetItem(args, 0); + if (f == NULL) + Py_RETURN_NONE; + f_args = PyTuple_GetItem(args, 1); + if (f_args == NULL) + Py_RETURN_NONE; + return PyObject_CallObject(f, f_args); +} + +static PyObject* call_with_tuple_and_dict(PyObject* self, PyObject* args) +{ + PyObject* f; + PyObject* f_args; + PyObject* f_kw; + f = PyTuple_GetItem(args, 0); + if (f == NULL) + Py_RETURN_NONE; + f_args = PyTuple_GetItem(args, 1); + if (f_args == NULL) + Py_RETURN_NONE; + f_kw = PyTuple_GetItem(args, 2); + if (f_kw == NULL) + Py_RETURN_NONE; + return PyObject_Call(f, f_args, f_kw); +} + +static PyObject* allocate_int(PyObject* self, PyObject* args) +{ + return PyLong_FromLong(2048); +} + +static PyObject* allocate_tuple(PyObject* self, PyObject* args) +{ + return Py_BuildValue("ii", 2048, 2049); +} + +static PyObject * Foo_getitem(PyObject *self, Py_ssize_t i) +{ + Py_RETURN_NONE; +} + +static Py_ssize_t Foo_len(PyObject *self) +{ + return 42; +} + +static PyMethodDef SimpleMethods[] = { + {"noargs", (PyCFunction)noargs, METH_NOARGS, ""}, + {"onearg", (PyCFunction)onearg, METH_O, ""}, + {"varargs", (PyCFunction)varargs, METH_VARARGS, ""}, + {"call_with_tuple", (PyCFunction)call_with_tuple, METH_VARARGS, ""}, + {"call_with_tuple_and_dict", (PyCFunction)call_with_tuple_and_dict, METH_VARARGS, ""}, + {"allocate_int", (PyCFunction)allocate_int, METH_NOARGS, ""}, + {"allocate_tuple", (PyCFunction)allocate_tuple, METH_NOARGS, ""}, + {NULL, NULL, 0, NULL} +}; + + +static PySequenceMethods FooSequence = { + .sq_length = (lenfunc)Foo_len, + .sq_item = (ssizeargfunc)Foo_getitem, + NULL, +}; + + +/* types */ + +typedef struct { + PyObject_HEAD +} FooObject; + +static PyTypeObject Foo_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + "cpy_simple.Foo", /* tp_name */ + sizeof(FooObject), /* tp_basicsize */ + 0, /* tp_itemsize */ + 0, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + &FooSequence, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + "Foo objects", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + SimpleMethods, /* tp_methods, reuse the same functions */ +}; + + +static PyType_Slot HTFoo_slots[] = { + {Py_tp_doc, "HTFoo objects"}, + {Py_tp_methods, SimpleMethods}, + {Py_sq_item, Foo_getitem}, + {Py_sq_length, Foo_len}, + {0, 0} +}; + +static PyType_Spec HTFoo_Type_spec = { + .name = "cpy_simple.Foo", + .basicsize = sizeof(FooObject), + .itemsize = 0, + .flags = Py_TPFLAGS_DEFAULT, + .slots = HTFoo_slots +}; + +static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + "cpy_simple", + "Module Doc", + -1, + SimpleMethods, + NULL, + NULL, + NULL, + NULL, +}; + +PyMODINIT_FUNC +PyInit_cpy_simple(void) +{ + PyObject* m; + Foo_Type.tp_new = PyType_GenericNew; + if (PyType_Ready(&Foo_Type) < 0) + return NULL; + m = PyModule_Create(&moduledef); + if (m == NULL) + return NULL; + Py_INCREF(&Foo_Type); + PyModule_AddObject(m, "Foo", (PyObject *)&Foo_Type); + + PyObject *HTFoo_Type = PyType_FromSpec(&HTFoo_Type_spec); + if (HTFoo_Type == NULL) + return NULL; + PyModule_AddObject(m, "HTFoo", HTFoo_Type); + + return m; +} diff --git a/graalpython/hpy/microbench/src/hpy_simple.c b/graalpython/hpy/microbench/src/hpy_simple.c new file mode 100644 index 0000000000..a5dddadd63 --- /dev/null +++ b/graalpython/hpy/microbench/src/hpy_simple.c @@ -0,0 +1,135 @@ +#include "hpy.h" + +/* module-level functions */ + +HPyDef_METH(noargs, "noargs", HPyFunc_NOARGS) +static HPy noargs_impl(HPyContext *ctx, HPy self) +{ + return HPy_Dup(ctx, ctx->h_None); +} + +HPyDef_METH(onearg, "onearg", HPyFunc_O) +static HPy onearg_impl(HPyContext *ctx, HPy self, HPy arg) +{ + return HPy_Dup(ctx, ctx->h_None); +} + +HPyDef_METH(varargs, "varargs", HPyFunc_VARARGS) +static HPy varargs_impl(HPyContext *ctx, HPy self, const HPy *args, size_t nargs) +{ + return HPy_Dup(ctx, ctx->h_None); +} + +HPyDef_METH(call_with_tuple, "call_with_tuple", HPyFunc_VARARGS) +static HPy call_with_tuple_impl(HPyContext *ctx, HPy self, const HPy *args, size_t nargs) +{ + HPy f, f_args; + if (nargs != 2) { + HPyErr_SetString(ctx, ctx->h_TypeError, "call_with_tuple requires two arguments"); + return HPy_NULL; + } + f = args[0]; + f_args = args[1]; + return HPy_CallTupleDict(ctx, f, f_args, HPy_NULL); +} + +HPyDef_METH(call_with_tuple_and_dict, "call_with_tuple_and_dict", HPyFunc_VARARGS) +static HPy call_with_tuple_and_dict_impl(HPyContext *ctx, HPy self, const HPy *args, size_t nargs) +{ + HPy f, f_args, f_kw; + if (nargs != 3) { + HPyErr_SetString(ctx, ctx->h_TypeError, "call_with_tuple_and_dict requires three arguments"); + return HPy_NULL; + } + f = args[0]; + f_args = args[1]; + f_kw = args[2]; + return HPy_CallTupleDict(ctx, f, f_args, f_kw); +} + +HPyDef_METH(allocate_int, "allocate_int", HPyFunc_NOARGS) +static HPy allocate_int_impl(HPyContext *ctx, HPy self) +{ + return HPyLong_FromLong(ctx, 2048); +} + +HPyDef_METH(allocate_tuple, "allocate_tuple", HPyFunc_NOARGS) +static HPy allocate_tuple_impl(HPyContext *ctx, HPy self) +{ + return HPy_BuildValue(ctx, "ii", 2048, 2049); +} + + +/* Foo type */ + +typedef struct { + long x; + long y; +} FooObject; + +HPyDef_SLOT(Foo_getitem, HPy_sq_item) +static HPy Foo_getitem_impl(HPyContext *ctx, HPy self, HPy_ssize_t i) +{ + return HPy_Dup(ctx, ctx->h_None); +} + +HPyDef_SLOT(Foo_len, HPy_sq_length) +static HPy_ssize_t Foo_len_impl(HPyContext *ctx, HPy self) +{ + return 42; +} + + +// note that we can reuse the same HPyDef for both module-level and type-level +// methods +static HPyDef *foo_defines[] = { + &noargs, + &onearg, + &varargs, + &allocate_int, + &allocate_tuple, + &Foo_getitem, + &Foo_len, +}; + + +static HPyType_Spec Foo_spec = { + .name = "hpy_simple.Foo", + .basicsize = sizeof(FooObject), + .flags = HPy_TPFLAGS_DEFAULT, + .defines = foo_defines +}; + + +/* Module defines */ + +HPyDef_SLOT(init_hpy_simple, HPy_mod_exec) +static int init_hpy_simple_impl(HPyContext *ctx, HPy m) +{ + HPy h_Foo = HPyType_FromSpec(ctx, &Foo_spec, NULL); + if (HPy_IsNull(h_Foo)) + return -1; + HPy_SetAttr_s(ctx, m, "Foo", h_Foo); + HPy_SetAttr_s(ctx, m, "HTFoo", h_Foo); + return 0; +} + +static HPyDef *module_defines[] = { + &noargs, + &onearg, + &varargs, + &call_with_tuple, + &call_with_tuple_and_dict, + &allocate_int, + &allocate_tuple, + &init_hpy_simple, + NULL +}; + +static HPyModuleDef moduledef = { + .doc = "HPy microbenchmarks", + .size = 0, + .defines = module_defines, +}; + +HPy_MODINIT(hpy_simple, moduledef) diff --git a/graalpython/hpy/microbench/test_microbench.py b/graalpython/hpy/microbench/test_microbench.py new file mode 100644 index 0000000000..ed59a243f0 --- /dev/null +++ b/graalpython/hpy/microbench/test_microbench.py @@ -0,0 +1,226 @@ +""" +These are not real tests, but microbenchmarks. The machinery to record the +timing and display the results is inside conftest.py +""" + +import pytest +import _valgrind + +API_PARAMS = [ + pytest.param('cpy', marks=pytest.mark.cpy), + pytest.param('hpy', marks=pytest.mark.hpy) + ] + +@pytest.fixture(params=API_PARAMS) +def api(request): + return request.param + +@pytest.fixture +def simple(request, api): + if api == 'cpy': + import cpy_simple + return cpy_simple + elif api == 'hpy': + import hpy_simple + return hpy_simple + else: + assert False, 'Unkown param: %s' % request.param + +@pytest.fixture +def N(request): + n = 10000000 + if request.config.option.fast: + n //= 100 + if request.config.option.slow: + n *= 10 + if _valgrind.lib.is_running_on_valgrind(): + n //= 100 + return n + + +class TestModule: + + def test_noargs(self, simple, timer, N): + with timer: + for i in range(N): + simple.noargs() + + def test_onearg_None(self, simple, timer, N): + with timer: + for i in range(N): + simple.onearg(None) + + def test_onearg_int(self, simple, timer, N): + with timer: + for i in range(N): + simple.onearg(i) + + def test_varargs(self, simple, timer, N): + with timer: + for i in range(N): + simple.varargs(None, None) + + def test_call_with_tuple(self, simple, timer, N): + def f(a, b): + return a + b + + with timer: + for i in range(N): + simple.call_with_tuple(f, (1, 2)) + + def test_call_with_tuple_and_dict(self, simple, timer, N): + def f(a, b): + return a + b + + with timer: + for i in range(N): + simple.call_with_tuple_and_dict(f, (1,), {"b": 2}) + + def test_allocate_int(self, simple, timer, N): + with timer: + for i in range(N): + simple.allocate_int() + + def test_allocate_tuple(self, api, simple, timer, N): + with timer: + for i in range(N): + simple.allocate_tuple() + + +class TestType: + """ Compares the performance of operations on types. + + The kinds of type used are: + + * cpy: a static type + * hpy: a heap type (HPy only has heap types) + + The type is named `simple.Foo` in both cases. + """ + + def test_allocate_obj(self, simple, timer, N): + import gc + Foo = simple.Foo + objs = [None] * N + gc.collect() + with timer: + for i in range(N): + objs[i] = Foo() + del objs + gc.collect() + + def test_method_lookup(self, simple, timer, N): + obj = simple.Foo() + with timer: + for i in range(N): + # note: here we are NOT calling it, we want to measure just + # the lookup + obj.noargs + + def test_noargs(self, simple, timer, N): + obj = simple.Foo() + with timer: + for i in range(N): + obj.noargs() + + def test_onearg_None(self, simple, timer, N): + obj = simple.Foo() + with timer: + for i in range(N): + obj.onearg(None) + + def test_onearg_int(self, simple, timer, N): + obj = simple.Foo() + with timer: + for i in range(N): + obj.onearg(i) + + def test_varargs(self, simple, timer, N): + obj = simple.Foo() + with timer: + for i in range(N): + obj.varargs(None, None) + + def test_len(self, simple, timer, N): + obj = simple.Foo() + with timer: + for i in range(N): + len(obj) + + def test_getitem(self, simple, timer, N): + obj = simple.Foo() + with timer: + for i in range(N): + obj[0] + + +class TestHeapType: + """ Compares the performance of operations on heap types. + + The type is named `simple.HTFoo` and is a heap type in all cases. + """ + + def test_allocate_obj_and_survive(self, simple, timer, N): + import gc + HTFoo = simple.HTFoo + objs = [None] * N + gc.collect() + with timer: + for i in range(N): + objs[i] = HTFoo() + del objs + gc.collect() + + def test_allocate_obj_and_die(self, simple, timer, N): + import gc + HTFoo = simple.HTFoo + gc.collect() + with timer: + for i in range(N): + obj = HTFoo() + obj.onearg(None) + gc.collect() + + def test_method_lookup(self, simple, timer, N): + obj = simple.HTFoo() + with timer: + for i in range(N): + # note: here we are NOT calling it, we want to measure just + # the lookup + obj.noargs + + def test_noargs(self, simple, timer, N): + obj = simple.HTFoo() + with timer: + for i in range(N): + obj.noargs() + + def test_onearg_None(self, simple, timer, N): + obj = simple.HTFoo() + with timer: + for i in range(N): + obj.onearg(None) + + def test_onearg_int(self, simple, timer, N): + obj = simple.HTFoo() + with timer: + for i in range(N): + obj.onearg(i) + + def test_varargs(self, simple, timer, N): + obj = simple.HTFoo() + with timer: + for i in range(N): + obj.varargs(None, None) + + def test_len(self, simple, timer, N): + obj = simple.HTFoo() + with timer: + for i in range(N): + len(obj) + + def test_getitem(self, simple, timer, N): + obj = simple.HTFoo() + with timer: + for i in range(N): + obj[0] diff --git a/graalpython/hpy/proof-of-concept/pof.c b/graalpython/hpy/proof-of-concept/pof.c new file mode 100644 index 0000000000..672a9a0199 --- /dev/null +++ b/graalpython/hpy/proof-of-concept/pof.c @@ -0,0 +1,109 @@ +#include "hpy.h" +#include + +HPyDef_METH(do_nothing, "do_nothing", HPyFunc_NOARGS) +static HPy do_nothing_impl(HPyContext *ctx, HPy self) +{ + return HPy_Dup(ctx, ctx->h_None); +} + +HPyDef_METH(double_obj, "double", HPyFunc_O) +static HPy double_obj_impl(HPyContext *ctx, HPy self, HPy obj) +{ + return HPy_Add(ctx, obj, obj); +} + +HPyDef_METH(add_ints, "add_ints", HPyFunc_VARARGS) +static HPy add_ints_impl(HPyContext *ctx, HPy self, const HPy *args, size_t nargs) +{ + long a, b; + if (!HPyArg_Parse(ctx, NULL, args, nargs, "ll", &a, &b)) + return HPy_NULL; + return HPyLong_FromLong(ctx, a+b); +} + +HPyDef_METH(add_ints_kw, "add_ints_kw", HPyFunc_KEYWORDS) +static HPy add_ints_kw_impl(HPyContext *ctx, HPy self, const HPy *args, + size_t nargs, HPy kwnames) +{ + long a, b; + const char* kwlist[] = {"a", "b", NULL}; + if (!HPyArg_ParseKeywords(ctx, NULL, args, nargs, kwnames, "ll", + kwlist, &a, &b)) + return HPy_NULL; + return HPyLong_FromLong(ctx, a+b); +} + +typedef struct { + double x; + double y; +} PointObject; + +HPyType_HELPERS(PointObject) + +HPyDef_SLOT(Point_new, HPy_tp_new) +static HPy Point_new_impl (HPyContext *ctx, HPy cls, const HPy *args, + HPy_ssize_t nargs, HPy kwnames) +{ + double x, y; + if (!HPyArg_Parse(ctx, NULL, args, nargs, "dd", &x, &y)) + return HPy_NULL; + PointObject *point; + HPy h_point = HPy_New(ctx, cls, &point); + if (HPy_IsNull(h_point)) + return HPy_NULL; + point->x = x; + point->y = y; + return h_point; +} + +HPyDef_SLOT(Point_repr, HPy_tp_repr) +static HPy Point_repr_impl(HPyContext *ctx, HPy self) +{ + PointObject *point = PointObject_AsStruct(ctx, self); + char msg[256]; + snprintf(msg, 256, "Point(%g, %g)", point->x, point->y); + return HPyUnicode_FromString(ctx, msg); + //return HPyUnicode_FromFormat("Point(%g, %g)", point->x, point->y); +} + + +static HPyDef *point_type_defines[] = { + &Point_new, + &Point_repr, + NULL +}; +static HPyType_Spec point_type_spec = { + .name = "pof.Point", + .basicsize = sizeof(PointObject), + .flags = HPy_TPFLAGS_DEFAULT, + .defines = point_type_defines +}; + +HPyDef_SLOT(mod_exec, HPy_mod_exec) +static int mod_exec_impl(HPyContext *ctx, HPy m) +{ + HPy h_point_type = HPyType_FromSpec(ctx, &point_type_spec, NULL); + if (HPy_IsNull(h_point_type)) + return -1; + HPy_SetAttr_s(ctx, m, "Point", h_point_type); + HPy_Close(ctx, h_point_type); + return 0; +} + +static HPyDef *module_defines[] = { + &do_nothing, + &double_obj, + &add_ints, + &add_ints_kw, + &mod_exec, + NULL +}; + +static HPyModuleDef moduledef = { + .doc = "HPy Proof of Concept", + .size = 0, + .defines = module_defines +}; + +HPy_MODINIT(pof, moduledef) diff --git a/graalpython/hpy/proof-of-concept/pofcpp.cpp b/graalpython/hpy/proof-of-concept/pofcpp.cpp new file mode 100644 index 0000000000..6214c5f87c --- /dev/null +++ b/graalpython/hpy/proof-of-concept/pofcpp.cpp @@ -0,0 +1,117 @@ +#include "hpy.h" +#include + +HPyDef_METH(do_nothing, "do_nothing", HPyFunc_NOARGS) +static HPy do_nothing_impl(HPyContext *ctx, HPy self) +{ + return HPy_Dup(ctx, ctx->h_None); +} + +HPyDef_METH(double_obj, "double", HPyFunc_O) +static HPy double_obj_impl(HPyContext *ctx, HPy self, HPy obj) +{ + return HPy_Add(ctx, obj, obj); +} + +HPyDef_METH(add_ints, "add_ints", HPyFunc_VARARGS) +static HPy add_ints_impl(HPyContext *ctx, HPy self, const HPy *args, size_t nargs) +{ + long a, b; + if (!HPyArg_Parse(ctx, NULL, args, nargs, "ll", &a, &b)) + return HPy_NULL; + return HPyLong_FromLong(ctx, a+b); +} + +HPyDef_METH(add_ints_kw, "add_ints_kw", HPyFunc_KEYWORDS) +static HPy add_ints_kw_impl(HPyContext *ctx, HPy self, const HPy *args, + size_t nargs, HPy kwnames) +{ + long a, b; + const char* kwlist[] = {"a", "b", NULL}; + if (!HPyArg_ParseKeywords(ctx, NULL, args, nargs, kwnames, "ll", + kwlist, &a, &b)) + return HPy_NULL; + return HPyLong_FromLong(ctx, a+b); +} + +typedef struct { + double x; + double y; +} PointObject; + +HPyType_HELPERS(PointObject) + +HPyDef_SLOT(Point_new, HPy_tp_new) +static HPy Point_new_impl (HPyContext *ctx, HPy cls, const HPy *args, + HPy_ssize_t nargs, HPy kwnames) +{ + double x, y; + if (!HPyArg_Parse(ctx, NULL, args, nargs, "dd", &x, &y)) + return HPy_NULL; + PointObject *point; + HPy h_point = HPy_New(ctx, cls, &point); + if (HPy_IsNull(h_point)) + return HPy_NULL; + point->x = x; + point->y = y; + return h_point; +} + +HPyDef_SLOT(Point_repr, HPy_tp_repr) +static HPy Point_repr_impl(HPyContext *ctx, HPy self) +{ + PointObject *point = PointObject_AsStruct(ctx, self); + char msg[256]; + snprintf(msg, 256, "Point(%g, %g)", point->x, point->y); + return HPyUnicode_FromString(ctx, msg); + //return HPyUnicode_FromFormat("Point(%g, %g)", point->x, point->y); +} + + +static HPyDef *point_type_defines[] = { + &Point_new, + &Point_repr, + NULL +}; + +static HPyType_Spec point_type_spec = { + .name = "pofcpp.Point", + .basicsize = sizeof(PointObject), + .flags = HPy_TPFLAGS_DEFAULT, + .defines = point_type_defines +}; + +HPyDef_SLOT(mod_exec, HPy_mod_exec) +static int mod_exec_impl(HPyContext *ctx, HPy m) +{ + HPy h_point_type = HPyType_FromSpec(ctx, &point_type_spec, NULL); + if (HPy_IsNull(h_point_type)) + return -1; + HPy_SetAttr_s(ctx, m, "Point", h_point_type); + HPy_Close(ctx, h_point_type); + return 0; +} + +static HPyDef *module_defines[] = { + &do_nothing, + &double_obj, + &add_ints, + &add_ints_kw, + &mod_exec, + NULL +}; +static HPyModuleDef moduledef = { + .doc = "HPy c++ Proof of Concept", + .size = 0, + .defines = module_defines +}; + +#ifdef __cplusplus +extern "C" { +#endif + +HPy_MODINIT(pofcpp, moduledef) + +#ifdef __cplusplus +} +#endif diff --git a/graalpython/hpy/proof-of-concept/pofpackage/bar.cpp b/graalpython/hpy/proof-of-concept/pofpackage/bar.cpp new file mode 100644 index 0000000000..01726d2d6f --- /dev/null +++ b/graalpython/hpy/proof-of-concept/pofpackage/bar.cpp @@ -0,0 +1,45 @@ +#include "hpy.h" + +class Bar +{ + int foo; + public: + Bar(int f) + { + foo = f; + } + + int boo(HPyContext *ctx, HPy obj) + { + return foo + HPyLong_AsLong(ctx, obj); + } +}; + +HPyDef_METH(hello, "hello", HPyFunc_O) +static HPy hello_impl(HPyContext *ctx, HPy self, HPy obj) +{ + Bar b(21); + return HPyLong_FromLong(ctx, b.boo(ctx, obj)); +} + + +static HPyDef *module_defines[] = { + &hello, + NULL +}; +static HPyModuleDef moduledef = { + .doc = "HPy C++ Proof of Concept", + .size = 0, + .defines = module_defines +}; + +#ifdef __cplusplus +extern "C" { +#endif + + +HPy_MODINIT(bar, moduledef) + +#ifdef __cplusplus +} +#endif diff --git a/graalpython/hpy/proof-of-concept/pofpackage/foo.c b/graalpython/hpy/proof-of-concept/pofpackage/foo.c new file mode 100644 index 0000000000..3a265efa25 --- /dev/null +++ b/graalpython/hpy/proof-of-concept/pofpackage/foo.c @@ -0,0 +1,20 @@ +#include "hpy.h" + +HPyDef_METH(hello, "hello", HPyFunc_NOARGS) +static HPy hello_impl(HPyContext *ctx, HPy self) +{ + return HPyUnicode_FromString(ctx, "hello from pofpackage.foo"); +} + + +static HPyDef *module_defines[] = { + &hello, + NULL +}; +static HPyModuleDef moduledef = { + .doc = "HPy Proof of Concept", + .size = 0, + .defines = module_defines +}; + +HPy_MODINIT(foo, moduledef) diff --git a/graalpython/hpy/proof-of-concept/requirements.txt b/graalpython/hpy/proof-of-concept/requirements.txt new file mode 100644 index 0000000000..2009c86f18 --- /dev/null +++ b/graalpython/hpy/proof-of-concept/requirements.txt @@ -0,0 +1 @@ +hpy diff --git a/graalpython/hpy/proof-of-concept/setup.py b/graalpython/hpy/proof-of-concept/setup.py new file mode 100644 index 0000000000..8132e1f2dc --- /dev/null +++ b/graalpython/hpy/proof-of-concept/setup.py @@ -0,0 +1,36 @@ +from setuptools import setup, Extension +import platform + +cpp_compile_extra_args = [] + +if platform.system() == "Windows": + compile_extra_args = ['/WX'] + cpp_compile_extra_args = [ + "/std:c++latest", # MSVC C7555 + ] +else: + compile_extra_args = ['-Werror'] + + +setup( + name="hpy-pof", + packages = ['pofpackage'], + zip_safe=False, + hpy_ext_modules=[ + Extension('pof', + sources=['pof.c'], + extra_compile_args=compile_extra_args), + Extension('pofpackage.foo', + sources=['pofpackage/foo.c'], + extra_compile_args=compile_extra_args), + Extension('pofcpp', + sources=['pofcpp.cpp'], + language='C++', + extra_compile_args=compile_extra_args + cpp_compile_extra_args), + Extension('pofpackage.bar', + sources=['pofpackage/bar.cpp'], + language='C++', + extra_compile_args=compile_extra_args + cpp_compile_extra_args), + ], + setup_requires=['hpy'], +) diff --git a/graalpython/hpy/proof-of-concept/test_pof.py b/graalpython/hpy/proof-of-concept/test_pof.py new file mode 100644 index 0000000000..54c6851c3f --- /dev/null +++ b/graalpython/hpy/proof-of-concept/test_pof.py @@ -0,0 +1,44 @@ +import pof +import pofpackage.foo +import pofcpp +import pofpackage.bar + +def test_do_nothing(): + assert pof.do_nothing() is None + +def test_double(): + assert pof.double(21) == 42 + +def test_add_ints(): + assert pof.add_ints(30, 12) == 42 + +def test_add_ints_kw(): + assert pof.add_ints_kw(b=30, a=12) == 42 + +def test_point(): + p = pof.Point(1, 2) + assert repr(p) == 'Point(1, 2)' # fixme when we have HPyFloat_FromDouble + +def test_pofpackage(): + assert pofpackage.foo.__name__ == "pofpackage.foo" + assert pofpackage.foo.hello() == 'hello from pofpackage.foo' + +def test_cpp_do_nothing(): + assert pofcpp.do_nothing() is None + +def test_cpp_double(): + assert pofcpp.double(21) == 42 + +def test_cpp_add_ints(): + assert pofcpp.add_ints(30, 12) == 42 + +def test_cpp_add_ints_kw(): + assert pofcpp.add_ints_kw(b=30, a=12) == 42 + +def test_cpp_point(): + p = pofcpp.Point(1, 2) + assert repr(p) == 'Point(1, 2)' # fixme when we have HPyFloat_FromDouble + +def test_cpp_pofpackage(): + assert pofpackage.bar.__name__ == "pofpackage.bar" + assert pofpackage.bar.hello(21) == 42 diff --git a/graalpython/hpy/proof-of-concept/test_pof.sh b/graalpython/hpy/proof-of-concept/test_pof.sh new file mode 100755 index 0000000000..1ac800769c --- /dev/null +++ b/graalpython/hpy/proof-of-concept/test_pof.sh @@ -0,0 +1,177 @@ +#!/bin/bash +set -e +ROOT=`pwd` # we expect this script to be run from the repo root + +# Allow the caller to override the Python runtime used +PYTHON=${PYTHON:-python3} + +_install_hpy() { + echo "Installing hpy" + # at the moment this install hpy.devel and hpy.universal. Eventually, we + # will want to split those into two separate packages + local PYTHON="$1" + pushd ${ROOT} + ${PYTHON} -m pip install -U pip + ${PYTHON} -m pip install wheel + ${PYTHON} -m pip install . + popd +} + +_test_pof() { + echo "==== testing pof ====" + # this assumes that pof is already installed, e.g. after calling + # wheel or setup_py_install + ${PYTHON} -m pip install pytest pytest-azurepipelines + cd proof-of-concept + ${PYTHON} -m pytest +} + +_build_wheel() { + HPY_ABI="$1" + local VENV="venv/wheel_builder_$HPY_ABI" + # we use this venv just to build the wheel, and then we install the wheel + # in the currently active virtualenv + echo "Create venv: $VENV" + ${PYTHON} -m venv "$VENV" + local PY_BUILDER="`pwd`/$VENV/bin/python3" + if [ -x "`pwd`/$VENV/Scripts/python.exe" ] + then + # Set the correct python executable for Windows + PY_BUILDER="`pwd`/$VENV/Scripts/python.exe" + fi + echo + echo "Installing hpy and requirements" + _install_hpy ${PY_BUILDER} + pushd proof-of-concept + ${PY_BUILDER} -m pip install -r requirements.txt + echo + echo "Building wheel" + ${PY_BUILDER} setup.py --hpy-abi="$HPY_ABI" bdist_wheel + popd +} + +_myrm() { + for path in "$@" + do + if [ -d "$path" -o -f "$path" ] + then + echo "rm $path" + rm -rf "$path" + else + echo "skipping $path" + fi + done +} + +clean() { + echo "=== cleaning up old stuff ===" + _myrm ${ROOT}/venv/wheel_builder_{cpython,universal} + _myrm ${ROOT}/venv/wheel_runner_{cpython,universal} + _myrm ${ROOT}/venv/setup_py_install_{cpython,universal} + _myrm ${ROOT}/venv/setup_py_build_ext_inplace_{cpython,universal} + _myrm ${ROOT}/build + _myrm ${ROOT}/proof-of-concept/build + _myrm ${ROOT}/proof-of-concept/dist + # remove files written by build_ext --inplace + _myrm ${ROOT}/proof-of-concept/pof*{.so,.py} + _myrm ${ROOT}/proof-of-concept/pofpackage/foo*{.so,.py} + _myrm ${ROOT}/proof-of-concept/pofcpp*{.so,.py} + _myrm ${ROOT}/proof-of-concept/pofpackage/bar*{.so,.py} + echo +} + +wheel() { + # build a wheel, install and test + HPY_ABI="$1" + local VENV="venv/wheel_runner_$HPY_ABI" + clean + echo "=== testing setup.py bdist_wheel" $HPY_ABI "===" + _build_wheel "$HPY_ABI" + WHEEL=`ls proof-of-concept/dist/*.whl` + echo "Wheel created: ${WHEEL}" + echo + echo "Create venv: $VENV" + ${PYTHON} -m venv "$VENV" + if [ -e "$VENV/bin/activate" ] ; then + source "$VENV/bin/activate" + else + source "$VENV/Scripts/activate" + fi + _install_hpy ${PYTHON} + echo "Installing wheel" + ${PYTHON} -m pip install $WHEEL + echo + _test_pof +} + +setup_py_install() { + # install proof-of-concept using setup.py install and run tests + HPY_ABI="$1" + VENV="venv/setup_py_install_$HPY_ABI" + clean + echo "=== testing setup.py --hpy-abi=$HPY_ABI install ===" + echo "Create venv: $VENV" + ${PYTHON} -m venv "$VENV" + if [ -e "$VENV/bin/activate" ] ; then + source "$VENV/bin/activate" + else + source "$VENV/Scripts/activate" + fi + _install_hpy ${PYTHON} + echo + echo "Running setup.py" + pushd proof-of-concept + ${PYTHON} setup.py --hpy-abi="$HPY_ABI" install + popd + echo + _test_pof +} + +setup_py_build_ext_inplace() { + # install proof-of-concept using setup.py install and run tests + HPY_ABI="$1" + VENV="venv/setup_py_build_ext_inplace_$HPY_ABI" + clean + echo "=== testing setup.py --hpy-abi=$HPY_ABI build_ext --inplace ===" + echo "Create venv: $VENV" + ${PYTHON} -m venv "$VENV" + if [ -e "$VENV/bin/activate" ] ; then + source "$VENV/bin/activate" + else + source "$VENV/Scripts/activate" + fi + _install_hpy ${PYTHON} + echo + echo "Running setup.py" + pushd proof-of-concept + echo python is $(which ${PYTHON}) + ${PYTHON} setup.py --hpy-abi="$HPY_ABI" build_ext --inplace + popd + echo + _test_pof +} + +# ======== main code ======= + +# validate arguments +if [[ "$#" -lt 1 || ( "$#" -lt 2 && "$1" != "clean") ]]; then + echo "Usage: $0 COMMAND [TARGET_ABI]" >&2 + echo "Commands:" >&2 + echo " wheel TARGET_ABI: build a wheel, install and test" + echo " setup_py_install TARGET_ABI: install poc using 'setup.py install' & run tests" >&2 + echo " clean: clean build artifacts" >&2 + echo "Target ABIs:" >&2 + echo " universal: Binary intended for any Python implementation" >&2 + echo " cpython : Binary optimized for CPython" >&2 + exit 1 +fi + +if [ ! -d "proof-of-concept" ] ; then + echo "Script must be run in the repo root" >&2 + exit 1 +fi + +# call the function mentioned as the first arg +COMMAND="$1" +shift +$COMMAND "$@" diff --git a/graalpython/hpy/pyproject.toml b/graalpython/hpy/pyproject.toml new file mode 100644 index 0000000000..a3194aec78 --- /dev/null +++ b/graalpython/hpy/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = [ "setuptools>=64.0", "setuptools-scm[toml]>=6.0", "wheel>=0.34.2",] +build-backend = "setuptools.build_meta" diff --git a/graalpython/hpy/requirements-autogen.txt b/graalpython/hpy/requirements-autogen.txt new file mode 100644 index 0000000000..8c9e2a7384 --- /dev/null +++ b/graalpython/hpy/requirements-autogen.txt @@ -0,0 +1,4 @@ +pycparser==2.21 +py==1.11.0 +packaging==19.2 +attrs==19.3.0 diff --git a/graalpython/hpy/setup.py b/graalpython/hpy/setup.py new file mode 100644 index 0000000000..8dd6962dd2 --- /dev/null +++ b/graalpython/hpy/setup.py @@ -0,0 +1,269 @@ +import sys +import os.path +from setuptools import setup, Extension +from setuptools.command.build_clib import build_clib +import platform + +# this package is supposed to be installed ONLY on CPython. Try to bail out +# with a meaningful error message in other cases. +if sys.implementation.name != 'cpython': + msg = 'ERROR: Cannot install and/or update hpy on this python implementation:\n' + msg += f' sys.implementation.name == {sys.implementation.name!r}\n\n' + if '_hpy_universal' in sys.builtin_module_names: + # this is a python which comes with its own hpy implementation + import _hpy_universal + if hasattr(_hpy_universal, 'get_version'): + hpy_version, git_rev = _hpy_universal.get_version() + msg += f'This python implementation comes with its own version of hpy=={hpy_version}\n' + msg += '\n' + msg += 'If you are trying to install hpy through pip, consider to put the\n' + msg += 'following in your requirements.txt, to make sure that pip will NOT\n' + msg += 'try to re-install it:\n' + msg += f' hpy=={hpy_version}' + else: + msg += 'This python implementation comes with its own version of hpy,\n' + msg += 'but the exact version could not be determined.\n' + # + else: + # this seems to be a python which does not support hpy + msg += 'This python implementation does not seem to support hpy:\n' + msg += '(built-in module _hpy_universal not found).\n' + msg += 'Please contact your vendor for more information.' + sys.exit(msg) + + +this_directory = os.path.abspath(os.path.dirname(__file__)) +with open(os.path.join(this_directory, 'README.md'), encoding='utf-8') as f: + LONG_DESCRIPTION = f.read() + +if 'HPY_DEBUG_BUILD' in os.environ: + # -fkeep-inline-functions is needed to make sure that the stubs for HPy_* + # functions are available to call inside GDB + EXTRA_COMPILE_ARGS = [ + '-g', '-O0', '-UNDEBUG', + '-fkeep-inline-functions', + # + ## these flags are useful but don't work on all + ## platforms/compilers. Uncomment temporarily if you need them. + #'-Wfatal-errors', # stop after one error (unrelated to warnings) + #'-Werror', # turn warnings into errors + ] +else: + EXTRA_COMPILE_ARGS = [] + +if '_HPY_DEBUG_FORCE_DEFAULT_MEM_PROTECT' not in os.environ: + EXTRA_COMPILE_ARGS += ['-D_HPY_DEBUG_MEM_PROTECT_USEMMAP'] + +if platform.system() == "Windows": + EXTRA_COMPILE_ARGS += ['/WX'] +else: + EXTRA_COMPILE_ARGS += ['-Werror'] + + +def get_scm_config(): + """ + We use this function as a hook to generate version.h before building. + """ + import textwrap + import subprocess + import pathlib + import setuptools_scm + + version = setuptools_scm.get_version() + try: + gitrev = subprocess.check_output('git rev-parse --short HEAD'.split(), + encoding='utf-8') + gitrev = gitrev.strip() + except subprocess.CalledProcessError: + gitrev = "__UNKNOWN__" + + version_h = pathlib.Path('.').joinpath('hpy', 'devel', 'include', 'hpy', 'version.h') + version_h.write_text(textwrap.dedent(f""" + // automatically generated by setup.py:get_scm_config() + #define HPY_VERSION "{version}" + #define HPY_GIT_REVISION "{gitrev}" + """)) + + version_py = pathlib.Path('.').joinpath('hpy', 'devel', 'version.py') + version_py.write_text(textwrap.dedent(f""" + # automatically generated by setup.py:get_scm_config() + __version__ = "{version}" + __git_revision__ = "{gitrev}" + """)) + + return {} # use the default config + +HPY_EXTRA_SOURCES = [ + 'hpy/devel/src/runtime/argparse.c', + 'hpy/devel/src/runtime/buildvalue.c', + 'hpy/devel/src/runtime/format.c', + 'hpy/devel/src/runtime/helpers.c', + 'hpy/devel/src/runtime/structseq.c', +] + +HPY_CTX_SOURCES = [ + 'hpy/devel/src/runtime/ctx_bytes.c', + 'hpy/devel/src/runtime/ctx_call.c', + 'hpy/devel/src/runtime/ctx_capsule.c', + 'hpy/devel/src/runtime/ctx_err.c', + 'hpy/devel/src/runtime/ctx_eval.c', + 'hpy/devel/src/runtime/ctx_long.c', + 'hpy/devel/src/runtime/ctx_module.c', + 'hpy/devel/src/runtime/ctx_object.c', + 'hpy/devel/src/runtime/ctx_type.c', + 'hpy/devel/src/runtime/ctx_tracker.c', + 'hpy/devel/src/runtime/ctx_listbuilder.c', + 'hpy/devel/src/runtime/ctx_tuple.c', + 'hpy/devel/src/runtime/ctx_tuplebuilder.c', + 'hpy/devel/src/runtime/ctx_contextvar.c', +] + +HPY_INCLUDE_DIRS = [ + 'hpy/devel/include', + 'hpy/universal/src', + 'hpy/debug/src/include', + 'hpy/trace/src/include', +] + +HPY_EXTRA_UNIVERSAL_LIB_NAME = "hpy-extra-universal" +HPY_EXTRA_HYBRID_LIB_NAME = "hpy-extra-hybrid" +HPY_CTX_LIB_NAME = "hpy-ctx-cpython" + +HPY_BUILD_CLIB_ABI_ATTR = "hpy_abi" + +class build_clib_hpy(build_clib): + """ Special build_clib command for building HPy's static libraries defined + by 'STATIC_LIBS' below. The behavior differs in following points: + (1) Option 'force' is set such that static libs will always be renewed. + (2) Method 'get_library_names' always returns 'None'. This is because + we only use this command to build static libraries for testing. + That means, we only use them in-place. We don't need them for + linking here. + (3) This command consumes a custom build info key + HPY_BUILD_CLIB_ABI_ATTR that is used to create separate build + temp directories for each ABI. This is necessary to avoid + incorrect sharing of (temporary) build artifacts. + (4) This command will use the include directories from command + 'build_ext'. + """ + def finalize_options(self): + super().finalize_options() + # we overwrite the include dirs and use the ones from 'build_ext' + build_ext_includes = self.get_finalized_command('build_ext').include_dirs or [] + self.include_dirs = HPY_INCLUDE_DIRS + build_ext_includes + self.force = 1 + + def get_library_names(self): + # We only build static libraries for testing. We just use them + # in-place. We don't want that our extensions (i.e. 'hpy.universal' + # etc) link to these libs. + return None + + def build_libraries(self, libraries): + # we just inherit the 'inplace' option from 'build_ext' + build_ext = self.get_finalized_command('build_ext') + inplace = build_ext.inplace + if inplace: + # the inplace option requires to find the package directory + # using the build_py command for that + build_py = self.get_finalized_command('build_py') + lib_dir = os.path.abspath(build_py.get_package_dir('hpy.devel')) + else: + lib_dir = os.path.join(build_ext.build_lib, 'hpy', 'devel') + + import pathlib + for lib in libraries: + lib_name, build_info = lib + abi = build_info.get(HPY_BUILD_CLIB_ABI_ATTR) + # Call super's build_libraries with just one library in the list + # such that we can temporarily change the 'build_temp'. + orig_build_temp = self.build_temp + orig_build_clib = self.build_clib + self.build_temp = os.path.join(orig_build_temp, 'lib', abi) + self.build_clib = os.path.join(lib_dir, 'lib', abi) + # ensure that 'build_clib' directory exists + pathlib.Path(self.build_clib).mkdir(parents=True, exist_ok=True) + try: + super().build_libraries([lib]) + finally: + self.build_temp = orig_build_temp + self.build_clib = orig_build_clib + + +STATIC_LIBS = [(HPY_EXTRA_UNIVERSAL_LIB_NAME, + {'sources': HPY_EXTRA_SOURCES, + HPY_BUILD_CLIB_ABI_ATTR: 'universal', + 'macros': [('HPY_ABI_UNIVERSAL', None)]}), + (HPY_EXTRA_HYBRID_LIB_NAME, + {'sources': HPY_EXTRA_SOURCES, + HPY_BUILD_CLIB_ABI_ATTR: 'hybrid', + 'macros': [('HPY_ABI_HYBRID', None)]}), + (HPY_CTX_LIB_NAME, + {'sources': HPY_EXTRA_SOURCES + HPY_CTX_SOURCES, + HPY_BUILD_CLIB_ABI_ATTR: 'cpython', + 'macros': [('HPY_ABI_CPYTHON', None)]})] + +EXT_MODULES = [ + Extension('hpy.universal', + ['hpy/universal/src/hpymodule.c', + 'hpy/universal/src/ctx.c', + 'hpy/universal/src/ctx_meth.c', + 'hpy/universal/src/ctx_misc.c', + 'hpy/debug/src/debug_ctx.c', + 'hpy/debug/src/debug_ctx_cpython.c', + 'hpy/debug/src/debug_handles.c', + 'hpy/debug/src/dhqueue.c', + 'hpy/debug/src/memprotect.c', + 'hpy/debug/src/stacktrace.c', + 'hpy/debug/src/_debugmod.c', + 'hpy/debug/src/autogen_debug_wrappers.c', + 'hpy/trace/src/trace_ctx.c', + 'hpy/trace/src/_tracemod.c', + 'hpy/trace/src/autogen_trace_wrappers.c', + 'hpy/trace/src/autogen_trace_func_table.c'] + + HPY_EXTRA_SOURCES + + HPY_CTX_SOURCES, + include_dirs=HPY_INCLUDE_DIRS, + extra_compile_args=[ + # so we need to enable the HYBRID ABI in order to implement + # the legacy features + '-DHPY_ABI_HYBRID', + '-DHPY_DEBUG_ENABLE_UHPY_SANITY_CHECK', + '-DHPY_EMBEDDED_MODULES', + ] + EXTRA_COMPILE_ARGS + ) + ] + +DEV_REQUIREMENTS = [ + "pytest", + "pytest-xdist", + "filelock", +] + +setup( + name="hpy", + author='The HPy team', + author_email='hpy-dev@python.org', + url='https://hpyproject.org', + license='MIT', + description='A better C API for Python', + long_description=LONG_DESCRIPTION, + long_description_content_type='text/markdown', + packages=['hpy.devel', 'hpy.debug', 'hpy.trace'], + include_package_data=True, + extras_require={ + "dev": DEV_REQUIREMENTS, + }, + libraries=STATIC_LIBS, + ext_modules=EXT_MODULES, + entry_points={ + "distutils.setup_keywords": [ + "hpy_ext_modules = hpy.devel:handle_hpy_ext_modules", + ], + }, + cmdclass={"build_clib": build_clib_hpy}, + use_scm_version=get_scm_config, + setup_requires=['setuptools_scm'], + install_requires=['setuptools>=64.0'], + python_requires='>=3.8', +) diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/__init__.py b/graalpython/hpy/test/__init__.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/__init__.py rename to graalpython/hpy/test/__init__.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/check_py27_compat.py b/graalpython/hpy/test/check_py27_compat.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/check_py27_compat.py rename to graalpython/hpy/test/check_py27_compat.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/conftest.py b/graalpython/hpy/test/conftest.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/conftest.py rename to graalpython/hpy/test/conftest.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/hpy_devel/__init__.py b/graalpython/hpy/test/debug/__init__.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/hpy_devel/__init__.py rename to graalpython/hpy/test/debug/__init__.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_builder_invalid.py b/graalpython/hpy/test/debug/test_builder_invalid.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_builder_invalid.py rename to graalpython/hpy/test/debug/test_builder_invalid.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_charptr.py b/graalpython/hpy/test/debug/test_charptr.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_charptr.py rename to graalpython/hpy/test/debug/test_charptr.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_context_reuse.py b/graalpython/hpy/test/debug/test_context_reuse.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_context_reuse.py rename to graalpython/hpy/test/debug/test_context_reuse.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_handles_invalid.py b/graalpython/hpy/test/debug/test_handles_invalid.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_handles_invalid.py rename to graalpython/hpy/test/debug/test_handles_invalid.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_handles_leak.py b/graalpython/hpy/test/debug/test_handles_leak.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_handles_leak.py rename to graalpython/hpy/test/debug/test_handles_leak.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_misc.py b/graalpython/hpy/test/debug/test_misc.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_misc.py rename to graalpython/hpy/test/debug/test_misc.py diff --git a/graalpython/hpy/test/hpy_devel/__init__.py b/graalpython/hpy/test/hpy_devel/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/hpy_devel/test_abitag.py b/graalpython/hpy/test/hpy_devel/test_abitag.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/hpy_devel/test_abitag.py rename to graalpython/hpy/test/hpy_devel/test_abitag.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/hpy_devel/test_distutils.py b/graalpython/hpy/test/hpy_devel/test_distutils.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/hpy_devel/test_distutils.py rename to graalpython/hpy/test/hpy_devel/test_distutils.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/support.py b/graalpython/hpy/test/support.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/support.py rename to graalpython/hpy/test/support.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_00_basic.py b/graalpython/hpy/test/test_00_basic.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_00_basic.py rename to graalpython/hpy/test/test_00_basic.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_argparse.py b/graalpython/hpy/test/test_argparse.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_argparse.py rename to graalpython/hpy/test/test_argparse.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_call.py b/graalpython/hpy/test/test_call.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_call.py rename to graalpython/hpy/test/test_call.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_capsule.py b/graalpython/hpy/test/test_capsule.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_capsule.py rename to graalpython/hpy/test/test_capsule.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_capsule_legacy.py b/graalpython/hpy/test/test_capsule_legacy.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_capsule_legacy.py rename to graalpython/hpy/test/test_capsule_legacy.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_contextvar.py b/graalpython/hpy/test/test_contextvar.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_contextvar.py rename to graalpython/hpy/test/test_contextvar.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_cpy_compat.py b/graalpython/hpy/test/test_cpy_compat.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_cpy_compat.py rename to graalpython/hpy/test/test_cpy_compat.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_eval.py b/graalpython/hpy/test/test_eval.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_eval.py rename to graalpython/hpy/test/test_eval.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_helpers.py b/graalpython/hpy/test/test_helpers.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_helpers.py rename to graalpython/hpy/test/test_helpers.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpybuildvalue.py b/graalpython/hpy/test/test_hpybuildvalue.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpybuildvalue.py rename to graalpython/hpy/test/test_hpybuildvalue.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpybytes.py b/graalpython/hpy/test/test_hpybytes.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpybytes.py rename to graalpython/hpy/test/test_hpybytes.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpydict.py b/graalpython/hpy/test/test_hpydict.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpydict.py rename to graalpython/hpy/test/test_hpydict.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyerr.py b/graalpython/hpy/test/test_hpyerr.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyerr.py rename to graalpython/hpy/test/test_hpyerr.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyfield.py b/graalpython/hpy/test/test_hpyfield.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyfield.py rename to graalpython/hpy/test/test_hpyfield.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyglobal.py b/graalpython/hpy/test/test_hpyglobal.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyglobal.py rename to graalpython/hpy/test/test_hpyglobal.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyimport.py b/graalpython/hpy/test/test_hpyimport.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyimport.py rename to graalpython/hpy/test/test_hpyimport.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyiter.py b/graalpython/hpy/test/test_hpyiter.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyiter.py rename to graalpython/hpy/test/test_hpyiter.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpylist.py b/graalpython/hpy/test/test_hpylist.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpylist.py rename to graalpython/hpy/test/test_hpylist.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpylong.py b/graalpython/hpy/test/test_hpylong.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpylong.py rename to graalpython/hpy/test/test_hpylong.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpymodule.py b/graalpython/hpy/test/test_hpymodule.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpymodule.py rename to graalpython/hpy/test/test_hpymodule.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyslice.py b/graalpython/hpy/test/test_hpyslice.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyslice.py rename to graalpython/hpy/test/test_hpyslice.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpystructseq.py b/graalpython/hpy/test/test_hpystructseq.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpystructseq.py rename to graalpython/hpy/test/test_hpystructseq.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpytuple.py b/graalpython/hpy/test/test_hpytuple.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpytuple.py rename to graalpython/hpy/test/test_hpytuple.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpytype.py b/graalpython/hpy/test/test_hpytype.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpytype.py rename to graalpython/hpy/test/test_hpytype.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpytype_legacy.py b/graalpython/hpy/test/test_hpytype_legacy.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpytype_legacy.py rename to graalpython/hpy/test/test_hpytype_legacy.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyunicode.py b/graalpython/hpy/test/test_hpyunicode.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyunicode.py rename to graalpython/hpy/test/test_hpyunicode.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_importing.py b/graalpython/hpy/test/test_importing.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_importing.py rename to graalpython/hpy/test/test_importing.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_legacy_forbidden.py b/graalpython/hpy/test/test_legacy_forbidden.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_legacy_forbidden.py rename to graalpython/hpy/test/test_legacy_forbidden.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_number.py b/graalpython/hpy/test/test_number.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_number.py rename to graalpython/hpy/test/test_number.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_object.py b/graalpython/hpy/test/test_object.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_object.py rename to graalpython/hpy/test/test_object.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_slots.py b/graalpython/hpy/test/test_slots.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_slots.py rename to graalpython/hpy/test/test_slots.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_slots_legacy.py b/graalpython/hpy/test/test_slots_legacy.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_slots_legacy.py rename to graalpython/hpy/test/test_slots_legacy.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_support.py b/graalpython/hpy/test/test_support.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_support.py rename to graalpython/hpy/test/test_support.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_tracker.py b/graalpython/hpy/test/test_tracker.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_tracker.py rename to graalpython/hpy/test/test_tracker.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/trace/test_trace.py b/graalpython/hpy/test/trace/test_trace.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/trace/test_trace.py rename to graalpython/hpy/test/trace/test_trace.py diff --git a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_tracker.c b/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_tracker.c deleted file mode 100644 index 17a3909f8e..0000000000 --- a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_tracker.c +++ /dev/null @@ -1,155 +0,0 @@ -/** - * A manager for HPy handles, allowing handles to be tracked - * and closed as a group. - * - * Note:: - * Calling HPyTracker_New(ctx, n) will ensure that at least n handles - * can be tracked without a call to HPyTracker_Add failing. - * - * If a call to HPyTracker_Add fails, the tracker still guarantees that - * the handle passed to it has been tracked (internally it does this by - * maintaining space for one more handle). - * - * After HPyTracker_Add fails, HPyTracker_Close should be called without - * any further calls to HPyTracker_Add. Calling HPyTracker_Close will close - * all the tracked handles, including the handled passed to the failed call - * to HPyTracker_Add. - * - * Example usage (inside an HPyDef_METH function):: - * - * long i; - * HPy key, value; - * HPyTracker ht; - * - * ht = HPyTracker_New(ctx, 0); // track the key-value pairs - * if (HPy_IsNull(ht)) - * return HPy_NULL; - * - * HPy dict = HPyDict_New(ctx); - * if (HPy_IsNull(dict)) - * goto error; - * - * for (i=0; i<5; i++) { - * key = HPyLong_FromLong(ctx, i); - * if (HPy_IsNull(key)) - * goto error; - * if (HPyTracker_Add(ctx, ht, key) < 0) - * goto error; - * value = HPyLong_FromLong(ctx, i * i); - * if (HPy_IsNull(value)) { - * goto error; - * } - * if (HPyTracker_Add(ctx, ht, value) < 0) - * goto error; - * result = HPy_SetItem(ctx, dict, key, value); - * if (result < 0) - * goto error; - * } - * - * success: - * HPyTracker_Close(ctx, ht); - * return dict; - * - * error: - * HPyTracker_Close(ctx, ht); - * HPy_Close(ctx, dict); - * // HPyErr will already have been set by the error that occurred. - * return HPy_NULL; - */ - -#include "hpy.h" - -static const HPy_ssize_t HPYTRACKER_INITIAL_CAPACITY = 5; - -typedef struct { - HPy_ssize_t capacity; // allocated handles - HPy_ssize_t length; // used handles - HPy *handles; -} _HPyTracker_s; - - -static inline _HPyTracker_s *_ht2hp(HPyTracker ht) { - return (_HPyTracker_s *) (ht)._i; -} -static inline HPyTracker _hp2ht(_HPyTracker_s *hp) { - return (HPyTracker) {(HPy_ssize_t) (hp)}; -} - - -_HPy_HIDDEN HPyTracker -ctx_Tracker_New(HPyContext *ctx, HPy_ssize_t capacity) -{ - _HPyTracker_s *hp; - if (capacity == 0) { - capacity = HPYTRACKER_INITIAL_CAPACITY; - } - capacity++; // always reserve space for an extra handle, see the docs - - hp = (_HPyTracker_s*)malloc(sizeof(_HPyTracker_s)); - if (hp == NULL) { - HPyErr_NoMemory(ctx); - return _hp2ht(0); - } - hp->handles = (HPy*)calloc(capacity, sizeof(HPy)); - if (hp->handles == NULL) { - free(hp); - HPyErr_NoMemory(ctx); - return _hp2ht(0); - } - hp->capacity = capacity; - hp->length = 0; - return _hp2ht(hp); -} - -static int -tracker_resize(HPyContext *ctx, _HPyTracker_s *hp, HPy_ssize_t capacity) -{ - HPy *new_handles; - capacity++; - - if (capacity <= hp->length) { - // refuse a resize that would either 1) lose handles or 2) not leave - // space for one new handle - HPyErr_SetString(ctx, ctx->h_ValueError, "HPyTracker resize would lose handles"); - return -1; - } - new_handles = (HPy*)realloc(hp->handles, capacity * sizeof(HPy)); - if (new_handles == NULL) { - HPyErr_NoMemory(ctx); - return -1; - } - hp->capacity = capacity; - hp->handles = new_handles; - return 0; -} - -_HPy_HIDDEN int -ctx_Tracker_Add(HPyContext *ctx, HPyTracker ht, HPy h) -{ - _HPyTracker_s *hp = _ht2hp(ht); - hp->handles[hp->length++] = h; - if (hp->capacity <= hp->length) { - if (tracker_resize(ctx, hp, hp->capacity * 2 - 1) < 0) - return -1; - } - return 0; -} - -_HPy_HIDDEN void -ctx_Tracker_ForgetAll(HPyContext *ctx, HPyTracker ht) -{ - _HPyTracker_s *hp = _ht2hp(ht); - hp->length = 0; -} - -_HPy_HIDDEN void -ctx_Tracker_Close(HPyContext *ctx, HPyTracker ht) -{ - _HPyTracker_s *hp = _ht2hp(ht); - HPy_ssize_t i; - for (i=0; ilength; i++) { - HPy_Close(ctx, hp->handles[i]); - } - free(hp->handles); - free(hp); -} diff --git a/graalpython/lib-graalpython/modules/hpy/devel/version.py b/graalpython/lib-graalpython/modules/hpy/devel/version.py deleted file mode 100644 index 0cd1dbcc22..0000000000 --- a/graalpython/lib-graalpython/modules/hpy/devel/version.py +++ /dev/null @@ -1,4 +0,0 @@ - -# automatically generated by setup.py:get_scm_config() -__version__ = "0.9.1.dev79+gb0fbdf73" -__git_revision__ = "b0fbdf73" diff --git a/graalpython/lib-graalpython/modules/hpy/trace/__init__.py b/graalpython/lib-graalpython/modules/hpy/trace/__init__.py deleted file mode 100644 index 71ece53923..0000000000 --- a/graalpython/lib-graalpython/modules/hpy/trace/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -from .leakdetector import HPyDebugError, HPyLeakError, LeakDetector - - -def set_handle_stack_trace_limit(limit): - from hpy.universal import _debug - _debug.set_handle_stack_trace_limit(limit) - - -def disable_handle_stack_traces(): - from hpy.universal import _debug - _debug.set_handle_stack_trace_limit(None) diff --git a/graalpython/lib-graalpython/modules/hpy/trace/leakdetector.py b/graalpython/lib-graalpython/modules/hpy/trace/leakdetector.py deleted file mode 100644 index 56a68d344c..0000000000 --- a/graalpython/lib-graalpython/modules/hpy/trace/leakdetector.py +++ /dev/null @@ -1,43 +0,0 @@ -from hpy.universal import _debug - -class HPyDebugError(Exception): - pass - -class HPyLeakError(HPyDebugError): - def __init__(self, leaks): - super().__init__() - self.leaks = leaks - - def __str__(self): - lines = [] - n = len(self.leaks) - s = 's' if n != 1 else '' - lines.append(f'{n} unclosed handle{s}:') - for dh in self.leaks: - lines.append(' %r' % dh) - return '\n'.join(lines) - - -class LeakDetector: - - def __init__(self): - self.generation = None - - def start(self): - if self.generation is not None: - raise ValueError('LeakDetector already started') - self.generation = _debug.new_generation() - - def stop(self): - if self.generation is None: - raise ValueError('LeakDetector not started yet') - leaks = _debug.get_open_handles(self.generation) - if leaks: - raise HPyLeakError(leaks) - - def __enter__(self): - self.start() - return self - - def __exit__(self, etype, evalue, tb): - self.stop() diff --git a/graalpython/lib-graalpython/modules/hpy/trace/pytest.py b/graalpython/lib-graalpython/modules/hpy/trace/pytest.py deleted file mode 100644 index 9a33c51343..0000000000 --- a/graalpython/lib-graalpython/modules/hpy/trace/pytest.py +++ /dev/null @@ -1,31 +0,0 @@ -# hpy.debug / pytest integration - -import pytest -from .leakdetector import LeakDetector - -# For now "hpy_debug" just does leak detection, but in the future it might -# grows extra features: that's why it's called generically "hpy_debug" instead -# of "detect_leaks". - -# NOTE: the fixture itself is currently untested :(. It turns out that testing -# that the fixture raises during the teardown is complicated and probably -# requires to write a full-fledged plugin. We might want to turn this into a -# real plugin in the future, but for now I think this is enough. - -# pypy still uses a very ancient version of pytest, 2.9.2: pytest<3.x needs to -# use @yield_fixture, which is deprecated in newer version of pytest (where -# you can just use @fixture) -if pytest.__version__ < '3': - fixture = pytest.yield_fixture -else: - fixture = pytest.fixture - -@fixture -def hpy_debug(request): - """ - pytest fixture which makes it possible to control hpy.debug from within a test. - - In particular, it automatically check that the test doesn't leak. - """ - with LeakDetector() as ld: - yield ld From 56103bff65acc629c507ccbaecc48d0d2a1b6d92 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Thu, 10 Apr 2025 13:41:37 +0200 Subject: [PATCH 3/3] Update to hpy 79571e2c558318f10be6f04040e6781ebbadb86f --- graalpython/hpy/.github/workflows/valgrind-tests.yml | 2 +- graalpython/hpy/hpy/universal/src/hpymodule.c | 3 --- graalpython/hpy/setup.py | 6 +++--- graalpython/hpy/test/conftest.py | 8 ++++++++ graalpython/hpy/test/debug/test_charptr.py | 6 +++++- graalpython/hpy/test/debug/test_context_reuse.py | 5 +++++ graalpython/hpy/test/debug/test_handles_invalid.py | 11 ++++++++++- graalpython/hpy/test/debug/test_misc.py | 8 +++++++- graalpython/hpy/test/hpy_devel/test_distutils.py | 11 +++++++---- graalpython/hpy/test/support.py | 3 +++ graalpython/hpy/test/test_cpy_compat.py | 7 ++++++- graalpython/hpy/test/test_hpyerr.py | 5 ++++- graalpython/hpy/test/test_hpyfield.py | 7 ++++++- graalpython/hpy/test/test_hpytype.py | 3 +++ graalpython/hpy/test/test_object.py | 6 ++++++ 15 files changed, 74 insertions(+), 17 deletions(-) diff --git a/graalpython/hpy/.github/workflows/valgrind-tests.yml b/graalpython/hpy/.github/workflows/valgrind-tests.yml index badbc9da6e..ab739252eb 100644 --- a/graalpython/hpy/.github/workflows/valgrind-tests.yml +++ b/graalpython/hpy/.github/workflows/valgrind-tests.yml @@ -23,7 +23,7 @@ jobs: python-version: 3.9 - name: Install / Upgrade Python dependencies - run: python -m pip install --upgrade pip wheel + run: python -m pip install --upgrade pip wheel setuptools - name: Build run: | diff --git a/graalpython/hpy/hpy/universal/src/hpymodule.c b/graalpython/hpy/hpy/universal/src/hpymodule.c index b86e418e61..6ae8b54859 100644 --- a/graalpython/hpy/hpy/universal/src/hpymodule.c +++ b/graalpython/hpy/hpy/universal/src/hpymodule.c @@ -22,9 +22,6 @@ #ifdef PYPY_VERSION # error "Cannot build hpy.universal on top of PyPy. PyPy comes with its own version of it" #endif -#ifdef GRAALVM_PYTHON -# error "Cannot build hpy.universal on top of GraalPy. GraalPy comes with its own version of it" -#endif static const char *hpy_mode_names[] = { "MODE_UNIVERSAL", diff --git a/graalpython/hpy/setup.py b/graalpython/hpy/setup.py index 8dd6962dd2..9a1cb9f2eb 100644 --- a/graalpython/hpy/setup.py +++ b/graalpython/hpy/setup.py @@ -4,9 +4,9 @@ from setuptools.command.build_clib import build_clib import platform -# this package is supposed to be installed ONLY on CPython. Try to bail out -# with a meaningful error message in other cases. -if sys.implementation.name != 'cpython': +# this package is supposed to be installed ONLY on CPython and GraalPy. Try to +# bail out with a meaningful error message in other cases. +if sys.implementation.name not in ('cpython', 'graalpy'): msg = 'ERROR: Cannot install and/or update hpy on this python implementation:\n' msg += f' sys.implementation.name == {sys.implementation.name!r}\n\n' if '_hpy_universal' in sys.builtin_module_names: diff --git a/graalpython/hpy/test/conftest.py b/graalpython/hpy/test/conftest.py index 372843459f..01c5a61103 100644 --- a/graalpython/hpy/test/conftest.py +++ b/graalpython/hpy/test/conftest.py @@ -1,4 +1,5 @@ import pytest +import sys from .support import ExtensionCompiler, DefaultExtensionTemplate,\ PythonSubprocessRunner, HPyDebugCapture, make_hpy_abi_fixture from pathlib import Path @@ -32,6 +33,13 @@ def pytest_configure(config): "markers", "syncgc: Mark tests that rely on a synchronous GC." ) + +def pytest_runtest_setup(item): + if any(item.iter_markers(name="syncgc")): + if sys.implementation.name in ("pypy", "graalpy"): + pytest.skip("requires synchronous garbage collector") + + # this is the default set of hpy_abi for all the tests. Individual files and # classes can override it. hpy_abi = make_hpy_abi_fixture('default') diff --git a/graalpython/hpy/test/debug/test_charptr.py b/graalpython/hpy/test/debug/test_charptr.py index 7df8d8f2c3..cc1fb9406f 100644 --- a/graalpython/hpy/test/debug/test_charptr.py +++ b/graalpython/hpy/test/debug/test_charptr.py @@ -1,6 +1,7 @@ import os import pytest -from ..support import SUPPORTS_SYS_EXECUTABLE, SUPPORTS_MEM_PROTECTION +import sys +from ..support import SUPPORTS_SYS_EXECUTABLE, SUPPORTS_MEM_PROTECTION, IS_GRAALPY # Tests detection of usage of char pointers associated with invalid already # closed handles. For now, the debug mode does not provide any hook for this @@ -13,6 +14,7 @@ def hpy_abi(): yield "debug" +@pytest.mark.skipif(IS_GRAALPY, reason="fails on GraalPy") @pytest.mark.skipif(not SUPPORTS_SYS_EXECUTABLE, reason="needs subprocess") def test_charptr_use_after_implicit_arg_handle_close(compiler, python_subprocess): mod = compiler.compile_module(""" @@ -70,6 +72,7 @@ def test_charptr_use_after_implicit_arg_handle_close(compiler, python_subprocess assert b"UnicodeDecodeError" in result.stderr +@pytest.mark.skipif(IS_GRAALPY, reason="fails on GraalPy") @pytest.mark.skipif(not SUPPORTS_SYS_EXECUTABLE, reason="needs subprocess") def test_charptr_use_after_handle_close(compiler, python_subprocess): mod = compiler.compile_module(""" @@ -121,6 +124,7 @@ def test_charptr_use_after_handle_close(compiler, python_subprocess): assert b"UnicodeDecodeError" in result.stderr +@pytest.mark.skipif(IS_GRAALPY, reason="transiently fails on GraalPy") @pytest.mark.skipif(not SUPPORTS_MEM_PROTECTION, reason= "Could be implemented by checking the contents on close.") @pytest.mark.skipif(not SUPPORTS_SYS_EXECUTABLE, reason="needs subprocess") diff --git a/graalpython/hpy/test/debug/test_context_reuse.py b/graalpython/hpy/test/debug/test_context_reuse.py index e3a57d3163..164505cf31 100644 --- a/graalpython/hpy/test/debug/test_context_reuse.py +++ b/graalpython/hpy/test/debug/test_context_reuse.py @@ -1,10 +1,15 @@ import pytest +import sys + +from ..support import IS_GRAALPY + @pytest.fixture def hpy_abi(): return "debug" +@pytest.mark.skipif(IS_GRAALPY, reason="Hangs on GraalPy") def test_reuse_context_from_global_variable(compiler, python_subprocess): mod = compiler.compile_module(""" #include diff --git a/graalpython/hpy/test/debug/test_handles_invalid.py b/graalpython/hpy/test/debug/test_handles_invalid.py index 36622eece9..355470c088 100644 --- a/graalpython/hpy/test/debug/test_handles_invalid.py +++ b/graalpython/hpy/test/debug/test_handles_invalid.py @@ -1,7 +1,7 @@ import pytest import sys from hpy.debug.leakdetector import LeakDetector -from ..support import SUPPORTS_SYS_EXECUTABLE, IS_PYTHON_DEBUG_BUILD +from ..support import SUPPORTS_SYS_EXECUTABLE, IS_PYTHON_DEBUG_BUILD, IS_GRAALPY from ..conftest import IS_VALGRIND_RUN @pytest.fixture @@ -12,6 +12,8 @@ def hpy_abi(): @pytest.mark.skipif(sys.implementation.name == 'pypy', reason="Cannot recover from use-after-close on pypy") +@pytest.mark.skipif(IS_GRAALPY, + reason="This corrupts process memory on GraalPy and crashes later") def test_no_invalid_handle(compiler, hpy_debug_capture): # Basic sanity check that valid code does not trigger any error reports mod = compiler.make_module(""" @@ -38,6 +40,8 @@ def test_no_invalid_handle(compiler, hpy_debug_capture): @pytest.mark.skipif(sys.implementation.name == 'pypy', reason="Cannot recover from use-after-close on pypy") +@pytest.mark.skipif(IS_GRAALPY, + reason="This corrupts process memory on GraalPy and crashes later") def test_cant_use_closed_handle(compiler, hpy_debug_capture): mod = compiler.make_module(""" HPyDef_METH(f, "f", HPyFunc_O, .doc="double close") @@ -113,6 +117,8 @@ def test_cant_use_closed_handle(compiler, hpy_debug_capture): @pytest.mark.skipif(sys.implementation.name == 'pypy', reason="Cannot recover from use-after-close on pypy") +@pytest.mark.skipif(IS_GRAALPY, + reason="This corrupts process memory on GraalPy and crashes later") def test_keeping_and_reusing_argument_handle(compiler, hpy_debug_capture): mod = compiler.make_module(""" HPy keep; @@ -142,6 +148,7 @@ def test_keeping_and_reusing_argument_handle(compiler, hpy_debug_capture): assert hpy_debug_capture.invalid_handles_count == 1 +@pytest.mark.skipif(IS_GRAALPY, reason="Crashes on GraalPy") def test_return_ctx_constant_without_dup(compiler, python_subprocess, fatal_exit_code): # Since this puts the context->h_None into an inconsistent state, we run # this test in a subprocess and check fatal error instead @@ -163,6 +170,7 @@ def test_return_ctx_constant_without_dup(compiler, python_subprocess, fatal_exit assert b"Invalid usage of already closed handle" in result.stderr +@pytest.mark.skipif(IS_GRAALPY, reason="Crashes on GraalPy") def test_close_ctx_constant(compiler, python_subprocess, fatal_exit_code): # Since this puts the context->h_True into an inconsistent state, we run # this test in a subprocess and check fatal error instead @@ -185,6 +193,7 @@ def test_close_ctx_constant(compiler, python_subprocess, fatal_exit_code): assert b"Invalid usage of already closed handle" in result.stderr +@pytest.mark.skipif(IS_GRAALPY, reason="Crashes on GraalPy") def test_invalid_handle_crashes_python_if_no_hook(compiler, python_subprocess, fatal_exit_code): if not SUPPORTS_SYS_EXECUTABLE: pytest.skip("no sys.executable") diff --git a/graalpython/hpy/test/debug/test_misc.py b/graalpython/hpy/test/debug/test_misc.py index 2994da4b3e..3640459798 100644 --- a/graalpython/hpy/test/debug/test_misc.py +++ b/graalpython/hpy/test/debug/test_misc.py @@ -1,11 +1,13 @@ import pytest -from ..support import SUPPORTS_SYS_EXECUTABLE, SUPPORTS_MEM_PROTECTION +import sys +from ..support import SUPPORTS_SYS_EXECUTABLE, SUPPORTS_MEM_PROTECTION, IS_GRAALPY @pytest.fixture def hpy_abi(): return "debug" +@pytest.mark.skipif(IS_GRAALPY, reason="hangs on GraalPy") @pytest.mark.skipif(not SUPPORTS_SYS_EXECUTABLE, reason="needs subprocess") def test_use_invalid_as_struct(compiler, python_subprocess): mod = compiler.compile_module(""" @@ -38,6 +40,7 @@ def test_use_invalid_as_struct(compiler, python_subprocess): assert "Invalid usage of _HPy_AsStruct_Object" in result.stderr.decode("utf-8") +@pytest.mark.skipif(IS_GRAALPY, reason="hangs on GraalPy") @pytest.mark.skipif(not SUPPORTS_SYS_EXECUTABLE, reason="needs subprocess") def test_typecheck(compiler, python_subprocess): mod = compiler.compile_module(""" @@ -60,6 +63,7 @@ def test_typecheck(compiler, python_subprocess): assert "HPy_TypeCheck arg 2 must be a type" in result.stderr.decode("utf-8") +@pytest.mark.skipif(IS_GRAALPY, reason="hangs on GraalPy") @pytest.mark.skipif(not SUPPORTS_MEM_PROTECTION, reason= "Could be implemented by checking the contents on close.") @pytest.mark.skipif(not SUPPORTS_SYS_EXECUTABLE, reason="needs subprocess") @@ -119,6 +123,7 @@ def test_type_getname(compiler, python_subprocess): assert result.returncode != 0 +@pytest.mark.skipif(IS_GRAALPY, reason="hangs on GraalPy") @pytest.mark.skipif(not SUPPORTS_SYS_EXECUTABLE, reason="needs subprocess") def test_type_issubtype(compiler, python_subprocess): mod = compiler.compile_module(""" @@ -143,6 +148,7 @@ def test_type_issubtype(compiler, python_subprocess): assert "HPyType_IsSubtype arg 1 must be a type" in result.stderr.decode("utf-8") +@pytest.mark.skipif(IS_GRAALPY, reason="transiently fails on GraalPy") @pytest.mark.skipif(not SUPPORTS_SYS_EXECUTABLE, reason="needs subprocess") def test_unicode_substring(compiler, python_subprocess): mod = compiler.compile_module(""" diff --git a/graalpython/hpy/test/hpy_devel/test_distutils.py b/graalpython/hpy/test/hpy_devel/test_distutils.py index f6b2daae39..62904f0957 100644 --- a/graalpython/hpy/test/hpy_devel/test_distutils.py +++ b/graalpython/hpy/test/hpy_devel/test_distutils.py @@ -16,7 +16,7 @@ import py import pytest -from ..support import atomic_run, HPY_ROOT +from ..support import atomic_run, HPY_ROOT, IS_GRAALPY # ====== IMPORTANT DEVELOPMENT TIP ===== # You can use py.test --reuse-venv to speed up local testing. @@ -43,7 +43,7 @@ def print_CalledProcessError(p): @pytest.fixture(scope='session') def venv_template(request, tmpdir_factory): - if request.config.option.reuse_venv: + if getattr(request.config.option, "reuse_venv", False): d = py.path.local('/tmp/venv-for-hpytest') if d.check(dir=True): # if it exists, we assume it's correct. If you want to recreate, @@ -59,7 +59,7 @@ def venv_template(request, tmpdir_factory): # it's just easier to use e.g. python -m pip attach_python_to_venv(d) for script in d.bin.listdir(): - if script.basename.startswith('python'): + if script.basename.startswith(('python', 'graalpy')): continue script.remove() # @@ -95,7 +95,7 @@ def initargs(self, pytestconfig, tmpdir, venv_template): self.tmpdir = tmpdir # create a fresh venv by copying the template self.venv = tmpdir.join('venv') - shutil.copytree(venv_template, self.venv) + shutil.copytree(venv_template, self.venv, symlinks=True) attach_python_to_venv(self.venv) # create the files for our test project self.hpy_test_project = tmpdir.join('hpy_test_project').ensure(dir=True) @@ -290,6 +290,7 @@ def test_hpymod_wheel(self, hpy_abi): doc = self.get_docstring('hpymod') assert doc == f'hpymod with HPy ABI: {hpy_abi}' + @pytest.mark.skipif(IS_GRAALPY, reason='not supported on GraalPy') def test_dont_mix_cpython_and_universal_abis(self): """ See issue #322 @@ -328,6 +329,8 @@ def test_dont_mix_cpython_and_universal_abis(self): def test_hpymod_legacy(self, hpy_abi): if hpy_abi == 'universal': pytest.skip('only for cpython and hybrid ABIs') + if IS_GRAALPY: + pytest.skip('not supported on GraalPy') self.gen_setup_py(""" setup(name = "hpy_test_project", hpy_ext_modules = [hpymod_legacy], diff --git a/graalpython/hpy/test/support.py b/graalpython/hpy/test/support.py index 250640548f..d102e398e7 100644 --- a/graalpython/hpy/test/support.py +++ b/graalpython/hpy/test/support.py @@ -19,6 +19,8 @@ # True if we are running on the CPython debug build IS_PYTHON_DEBUG_BUILD = hasattr(sys, 'gettotalrefcount') +IS_GRAALPY = getattr(getattr(sys, "implementation", None), "name", None) == 'graalpy' + # pytest marker to run tests only on linux ONLY_LINUX = pytest.mark.skipif(sys.platform!='linux', reason='linux only') @@ -441,6 +443,7 @@ def run(self, mod, code): @pytest.mark.usefixtures('initargs') class HPyTest: + is_graalpy = IS_GRAALPY # Exposed here for tests running as PyPy apptests ExtensionTemplate = DefaultExtensionTemplate @pytest.fixture() diff --git a/graalpython/hpy/test/test_cpy_compat.py b/graalpython/hpy/test/test_cpy_compat.py index 5a7866c335..8151be0dd9 100644 --- a/graalpython/hpy/test/test_cpy_compat.py +++ b/graalpython/hpy/test/test_cpy_compat.py @@ -1,4 +1,5 @@ -from .support import HPyTest, make_hpy_abi_fixture +import pytest +from .support import HPyTest, make_hpy_abi_fixture, IS_GRAALPY class TestCPythonCompatibility(HPyTest): @@ -30,6 +31,7 @@ def test_abi(self): expected = 'hybrid' assert hpy_abi == expected + @pytest.mark.skipif(IS_GRAALPY, reason="Crashes on GraalPy") def test_frompyobject(self): mod = self.make_module(""" #include @@ -132,6 +134,7 @@ def foo(self): obj = MyClass() assert mod.f(obj) == 1234 + @pytest.mark.skipif(IS_GRAALPY, reason="Crashes on GraalPy") def test_hpy_close(self): mod = self.make_module(""" #include @@ -156,6 +159,7 @@ def test_hpy_close(self): if self.supports_refcounts(): assert x == -1 + @pytest.mark.skipif(IS_GRAALPY, reason="Crashes on GraalPy") def test_hpy_dup(self): mod = self.make_module(""" #include @@ -182,6 +186,7 @@ def test_hpy_dup(self): if self.supports_refcounts(): assert x == +1 + @pytest.mark.skipif(IS_GRAALPY, reason="Crashes on GraalPy") def test_many_handles(self): mod = self.make_module(""" #include diff --git a/graalpython/hpy/test/test_hpyerr.py b/graalpython/hpy/test/test_hpyerr.py index 6adc8d51df..20426aaa31 100644 --- a/graalpython/hpy/test/test_hpyerr.py +++ b/graalpython/hpy/test/test_hpyerr.py @@ -1,4 +1,5 @@ -from .support import HPyTest, SUPPORTS_SYS_EXECUTABLE, trampoline +import pytest +from .support import HPyTest, SUPPORTS_SYS_EXECUTABLE, trampoline, IS_GRAALPY class TestErr(HPyTest): @@ -17,6 +18,7 @@ def test_NoMemory(self): with pytest.raises(MemoryError): mod.f() + @pytest.mark.skipif(IS_GRAALPY, reason="Fails transiently on GraalPy especially with xdist") def test_FatalError(self, python_subprocess, fatal_exit_code): mod = self.compile_module(""" HPyDef_METH(f, "f", HPyFunc_NOARGS) @@ -672,6 +674,7 @@ def do_not_raise_exception(*args): with pytest.raises(DummyException): mod.f(raise_exception, (DummyException, ), exc_types) + @pytest.mark.skipif(IS_GRAALPY, reason="Fails transiently on GraalPy especially with xdist") def test_HPyErr_WriteUnraisable(self, python_subprocess): mod = self.compile_module(""" HPyDef_METH(f, "f", HPyFunc_NOARGS) diff --git a/graalpython/hpy/test/test_hpyfield.py b/graalpython/hpy/test/test_hpyfield.py index 29ef99dba7..5e67515a13 100644 --- a/graalpython/hpy/test/test_hpyfield.py +++ b/graalpython/hpy/test/test_hpyfield.py @@ -145,7 +145,9 @@ def test_gc_track_no_gc_flag(self): assert not gc.is_tracked(p) def test_tp_traverse(self): - import sys + if self.is_graalpy: + import pytest + pytest.skip("Crashes on GraalPy") import gc mod = self.make_module(""" @DEFINE_PairObject @@ -308,6 +310,9 @@ def count_pairs(): assert count_pairs() == 0 def test_tp_finalize(self): + if self.is_graalpy: + import pytest + pytest.skip("Crashes on GraalPy") # Tests the contract of tp_finalize: what it should see # if called from within HPyField_Store mod = self.make_module(""" diff --git a/graalpython/hpy/test/test_hpytype.py b/graalpython/hpy/test/test_hpytype.py index 3b8469fd16..4d39fbcd0a 100644 --- a/graalpython/hpy/test/test_hpytype.py +++ b/graalpython/hpy/test/test_hpytype.py @@ -1594,6 +1594,9 @@ class TestPureHPyType(HPyTest): ExtensionTemplate = PointTemplate def test_builtin_shape(self): + if self.is_graalpy: + import pytest + pytest.skip("Not yet implemented on GraalPy") mod = self.make_module(""" @DEFINE_PointObject(HPyType_BuiltinShape_Long) @DEFINE_Point_xy diff --git a/graalpython/hpy/test/test_object.py b/graalpython/hpy/test/test_object.py index 8e50450457..369b1500ad 100644 --- a/graalpython/hpy/test/test_object.py +++ b/graalpython/hpy/test/test_object.py @@ -532,6 +532,8 @@ def test_getitem_s(self): def test_getslice(self): import pytest + if self.is_graalpy: + pytest.skip("Not yet implemented on GraalPy") mod = self.make_module(""" HPyDef_METH(f, "f", HPyFunc_VARARGS) static HPy f_impl(HPyContext *ctx, HPy self, const HPy *args, size_t nargs) @@ -653,6 +655,8 @@ def test_setitem_s(self): def test_setslice(self): import pytest + if self.is_graalpy: + pytest.skip("Not yet implemented on GraalPy") mod = self.make_module(""" HPyDef_METH(f, "f", HPyFunc_VARARGS) static HPy f_impl(HPyContext *ctx, HPy self, const HPy *args, size_t nargs) @@ -786,6 +790,8 @@ def test_delitem(self): def test_delslice(self): import pytest + if self.is_graalpy: + pytest.skip("Not yet implemented on GraalPy") mod = self.make_module(""" HPyDef_METH(f, "f", HPyFunc_VARARGS) static HPy f_impl(HPyContext *ctx, HPy self, const HPy *args, size_t nargs) pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy