From cb3fd406cf95c3aab22edd7baff3098f9fea7aa0 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 10 Nov 2022 14:09:01 +0000 Subject: [PATCH 01/10] Specialize calls to normal Python classes. --- Include/cpython/object.h | 1 + Include/internal/pycore_code.h | 2 + Include/internal/pycore_frame.h | 24 +++++++ Include/internal/pycore_object.h | 2 + Include/internal/pycore_opcode.h | 58 ++++++++-------- Include/opcode.h | 116 ++++++++++++++++--------------- Lib/opcode.py | 2 + Lib/test/test_sys.py | 2 +- Lib/test/test_sys_settrace.py | 23 ++++++ Objects/dictobject.c | 7 +- Objects/typeobject.c | 21 +++++- Python/bytecodes.c | 68 ++++++++++++++++++ Python/compile.c | 3 + Python/generated_cases.c.h | 69 ++++++++++++++++++ Python/opcode_targets.h | 54 +++++++------- Python/pystate.c | 1 + Python/specialize.c | 90 ++++++++++++++++++++++++ 17 files changed, 424 insertions(+), 119 deletions(-) diff --git a/Include/cpython/object.h b/Include/cpython/object.h index f4755a7b2fb852..0d53f43acb5820 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -230,6 +230,7 @@ struct _typeobject { * by code other than the specializer and interpreter. */ struct _specialization_cache { PyObject *getitem; + PyObject *init; }; /* The *real* layout of a type object when allocated on the heap */ diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 0af240ca362103..16d69b57976416 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -97,6 +97,8 @@ struct callable_cache { PyObject *len; PyObject *list_append; PyObject *object__getattribute__; + /* This is a strong reference */ + PyCodeObject *init_cleanup; }; /* "Locals plus" for a code object is the set of locals + cell vars + diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index 7fa410d288c33a..550a35034709cb 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -223,6 +223,30 @@ _PyFrame_PushUnchecked(PyThreadState *tstate, PyFunctionObject *func) return new_frame; } +/* Pushes a trampoline frame without checking for space. + * Must be guarded by _PyThreadState_HasStackSpace() */ +static inline _PyInterpreterFrame * +_PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int stackdepth, int start) +{ + CALL_STAT_INC(frames_pushed); + _PyInterpreterFrame *frame = (_PyInterpreterFrame *)tstate->datastack_top; + tstate->datastack_top += code->co_framesize; + assert(tstate->datastack_top < tstate->datastack_limit); + frame->f_funcobj = Py_NewRef(Py_None); + frame->f_code = (PyCodeObject *)Py_NewRef(code); +#ifdef Py_DEBUG + frame->f_builtins = (PyObject*)0xaaaa; + frame->f_globals = (PyObject*)0xaaab; +#endif + frame->f_locals = NULL; + frame->stacktop = code->co_nlocalsplus + stackdepth; + frame->frame_obj = NULL; + frame->prev_instr = _PyCode_CODE(code) + start; + frame->owner = FRAME_OWNED_BY_THREAD; + frame->yield_offset = 0; + return frame; +} + int _PyInterpreterFrame_GetLine(_PyInterpreterFrame *frame); static inline diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 8b78f79e950e92..be21671b1cbb8c 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -300,8 +300,10 @@ static inline int _PyType_SUPPORTS_WEAKREFS(PyTypeObject *type) { } extern PyObject* _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems); +PyObject *_PyType_NewManagedObject(PyTypeObject *type); extern int _PyObject_InitializeDict(PyObject *obj); +int _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp); extern int _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, PyObject *name, PyObject *value); PyObject * _PyObject_GetInstanceAttribute(PyObject *obj, PyDictValues *values, diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index 3f44511240aa14..37ac566e3e8700 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -87,6 +87,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [CALL_BUILTIN_FAST_WITH_KEYWORDS] = CALL, [CALL_FUNCTION_EX] = CALL_FUNCTION_EX, [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = CALL, + [CALL_NO_KW_ALLOC_AND_ENTER_INIT] = CALL, [CALL_NO_KW_BUILTIN_FAST] = CALL, [CALL_NO_KW_BUILTIN_O] = CALL, [CALL_NO_KW_ISINSTANCE] = CALL, @@ -121,6 +122,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [DICT_UPDATE] = DICT_UPDATE, [END_ASYNC_FOR] = END_ASYNC_FOR, [END_FOR] = END_FOR, + [EXIT_INIT_CHECK] = EXIT_INIT_CHECK, [EXTENDED_ARG] = EXTENDED_ARG, [FORMAT_VALUE] = FORMAT_VALUE, [FOR_ITER] = FOR_ITER, @@ -249,6 +251,7 @@ static const char *const _PyOpcode_OpName[263] = { [BINARY_OP_INPLACE_ADD_UNICODE] = "BINARY_OP_INPLACE_ADD_UNICODE", [BINARY_OP_MULTIPLY_FLOAT] = "BINARY_OP_MULTIPLY_FLOAT", [UNARY_INVERT] = "UNARY_INVERT", + [EXIT_INIT_CHECK] = "EXIT_INIT_CHECK", [BINARY_OP_MULTIPLY_INT] = "BINARY_OP_MULTIPLY_INT", [BINARY_OP_SUBTRACT_FLOAT] = "BINARY_OP_SUBTRACT_FLOAT", [BINARY_OP_SUBTRACT_INT] = "BINARY_OP_SUBTRACT_INT", @@ -257,31 +260,30 @@ static const char *const _PyOpcode_OpName[263] = { [BINARY_SUBSCR_LIST_INT] = "BINARY_SUBSCR_LIST_INT", [BINARY_SUBSCR_TUPLE_INT] = "BINARY_SUBSCR_TUPLE_INT", [CALL_PY_EXACT_ARGS] = "CALL_PY_EXACT_ARGS", - [CALL_PY_WITH_DEFAULTS] = "CALL_PY_WITH_DEFAULTS", [BINARY_SUBSCR] = "BINARY_SUBSCR", [BINARY_SLICE] = "BINARY_SLICE", [STORE_SLICE] = "STORE_SLICE", + [CALL_PY_WITH_DEFAULTS] = "CALL_PY_WITH_DEFAULTS", [CALL_BOUND_METHOD_EXACT_ARGS] = "CALL_BOUND_METHOD_EXACT_ARGS", - [CALL_BUILTIN_CLASS] = "CALL_BUILTIN_CLASS", [GET_LEN] = "GET_LEN", [MATCH_MAPPING] = "MATCH_MAPPING", [MATCH_SEQUENCE] = "MATCH_SEQUENCE", [MATCH_KEYS] = "MATCH_KEYS", - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", + [CALL_BUILTIN_CLASS] = "CALL_BUILTIN_CLASS", [PUSH_EXC_INFO] = "PUSH_EXC_INFO", [CHECK_EXC_MATCH] = "CHECK_EXC_MATCH", [CHECK_EG_MATCH] = "CHECK_EG_MATCH", + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", [CALL_NO_KW_BUILTIN_FAST] = "CALL_NO_KW_BUILTIN_FAST", [CALL_NO_KW_BUILTIN_O] = "CALL_NO_KW_BUILTIN_O", [CALL_NO_KW_ISINSTANCE] = "CALL_NO_KW_ISINSTANCE", [CALL_NO_KW_LEN] = "CALL_NO_KW_LEN", + [CALL_NO_KW_ALLOC_AND_ENTER_INIT] = "CALL_NO_KW_ALLOC_AND_ENTER_INIT", [CALL_NO_KW_LIST_APPEND] = "CALL_NO_KW_LIST_APPEND", [CALL_NO_KW_METHOD_DESCRIPTOR_FAST] = "CALL_NO_KW_METHOD_DESCRIPTOR_FAST", [CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = "CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS", [CALL_NO_KW_METHOD_DESCRIPTOR_O] = "CALL_NO_KW_METHOD_DESCRIPTOR_O", - [CALL_NO_KW_STR_1] = "CALL_NO_KW_STR_1", - [CALL_NO_KW_TUPLE_1] = "CALL_NO_KW_TUPLE_1", [WITH_EXCEPT_START] = "WITH_EXCEPT_START", [GET_AITER] = "GET_AITER", [GET_ANEXT] = "GET_ANEXT", @@ -289,37 +291,37 @@ static const char *const _PyOpcode_OpName[263] = { [BEFORE_WITH] = "BEFORE_WITH", [END_ASYNC_FOR] = "END_ASYNC_FOR", [CLEANUP_THROW] = "CLEANUP_THROW", + [CALL_NO_KW_STR_1] = "CALL_NO_KW_STR_1", + [CALL_NO_KW_TUPLE_1] = "CALL_NO_KW_TUPLE_1", [CALL_NO_KW_TYPE_1] = "CALL_NO_KW_TYPE_1", [COMPARE_OP_FLOAT_JUMP] = "COMPARE_OP_FLOAT_JUMP", - [COMPARE_OP_GENERIC] = "COMPARE_OP_GENERIC", - [COMPARE_OP_INT_JUMP] = "COMPARE_OP_INT_JUMP", [STORE_SUBSCR] = "STORE_SUBSCR", [DELETE_SUBSCR] = "DELETE_SUBSCR", - [COMPARE_OP_STR_JUMP] = "COMPARE_OP_STR_JUMP", + [COMPARE_OP_GENERIC] = "COMPARE_OP_GENERIC", [STOPITERATION_ERROR] = "STOPITERATION_ERROR", + [COMPARE_OP_INT_JUMP] = "COMPARE_OP_INT_JUMP", + [COMPARE_OP_STR_JUMP] = "COMPARE_OP_STR_JUMP", [FOR_ITER_LIST] = "FOR_ITER_LIST", [FOR_ITER_RANGE] = "FOR_ITER_RANGE", - [FOR_ITER_GEN] = "FOR_ITER_GEN", - [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", [GET_ITER] = "GET_ITER", [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", [PRINT_EXPR] = "PRINT_EXPR", [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS", - [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", - [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", + [FOR_ITER_GEN] = "FOR_ITER_GEN", + [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR", [RETURN_GENERATOR] = "RETURN_GENERATOR", + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", + [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", - [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", - [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", [LIST_TO_TUPLE] = "LIST_TO_TUPLE", [RETURN_VALUE] = "RETURN_VALUE", [IMPORT_STAR] = "IMPORT_STAR", [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", - [LOAD_ATTR_METHOD_WITH_DICT] = "LOAD_ATTR_METHOD_WITH_DICT", + [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", [ASYNC_GEN_WRAP] = "ASYNC_GEN_WRAP", [PREP_RERAISE_STAR] = "PREP_RERAISE_STAR", [POP_EXCEPT] = "POP_EXCEPT", @@ -346,7 +348,7 @@ static const char *const _PyOpcode_OpName[263] = { [JUMP_FORWARD] = "JUMP_FORWARD", [JUMP_IF_FALSE_OR_POP] = "JUMP_IF_FALSE_OR_POP", [JUMP_IF_TRUE_OR_POP] = "JUMP_IF_TRUE_OR_POP", - [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", + [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE", [POP_JUMP_IF_TRUE] = "POP_JUMP_IF_TRUE", [LOAD_GLOBAL] = "LOAD_GLOBAL", @@ -354,7 +356,7 @@ static const char *const _PyOpcode_OpName[263] = { [CONTAINS_OP] = "CONTAINS_OP", [RERAISE] = "RERAISE", [COPY] = "COPY", - [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST", + [LOAD_ATTR_METHOD_WITH_DICT] = "LOAD_ATTR_METHOD_WITH_DICT", [BINARY_OP] = "BINARY_OP", [SEND] = "SEND", [LOAD_FAST] = "LOAD_FAST", @@ -374,9 +376,9 @@ static const char *const _PyOpcode_OpName[263] = { [STORE_DEREF] = "STORE_DEREF", [DELETE_DEREF] = "DELETE_DEREF", [JUMP_BACKWARD] = "JUMP_BACKWARD", - [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", + [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX", - [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST", + [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST", [EXTENDED_ARG] = "EXTENDED_ARG", [LIST_APPEND] = "LIST_APPEND", [SET_ADD] = "SET_ADD", @@ -386,29 +388,29 @@ static const char *const _PyOpcode_OpName[263] = { [YIELD_VALUE] = "YIELD_VALUE", [RESUME] = "RESUME", [MATCH_CLASS] = "MATCH_CLASS", - [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", - [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", + [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", + [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST", [FORMAT_VALUE] = "FORMAT_VALUE", [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", [BUILD_STRING] = "BUILD_STRING", + [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", + [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", - [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", - [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST", [LIST_EXTEND] = "LIST_EXTEND", [SET_UPDATE] = "SET_UPDATE", [DICT_MERGE] = "DICT_MERGE", [DICT_UPDATE] = "DICT_UPDATE", + [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", + [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST", [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", - [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", - [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", [CALL] = "CALL", [KW_NAMES] = "KW_NAMES", + [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", + [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", - [174] = "<174>", - [175] = "<175>", [176] = "<176>", [177] = "<177>", [178] = "<178>", @@ -500,8 +502,6 @@ static const char *const _PyOpcode_OpName[263] = { #endif #define EXTRA_CASES \ - case 174: \ - case 175: \ case 176: \ case 177: \ case 178: \ diff --git a/Include/opcode.h b/Include/opcode.h index c18d9c057b849b..d33d1c9f6475ed 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -18,6 +18,7 @@ extern "C" { #define UNARY_NEGATIVE 11 #define UNARY_NOT 12 #define UNARY_INVERT 15 +#define EXIT_INIT_CHECK 16 #define BINARY_SUBSCR 25 #define BINARY_SLICE 26 #define STORE_SLICE 27 @@ -135,63 +136,64 @@ extern "C" { #define BINARY_OP_GENERIC 8 #define BINARY_OP_INPLACE_ADD_UNICODE 13 #define BINARY_OP_MULTIPLY_FLOAT 14 -#define BINARY_OP_MULTIPLY_INT 16 -#define BINARY_OP_SUBTRACT_FLOAT 17 -#define BINARY_OP_SUBTRACT_INT 18 -#define BINARY_SUBSCR_DICT 19 -#define BINARY_SUBSCR_GETITEM 20 -#define BINARY_SUBSCR_LIST_INT 21 -#define BINARY_SUBSCR_TUPLE_INT 22 -#define CALL_PY_EXACT_ARGS 23 -#define CALL_PY_WITH_DEFAULTS 24 -#define CALL_BOUND_METHOD_EXACT_ARGS 28 -#define CALL_BUILTIN_CLASS 29 -#define CALL_BUILTIN_FAST_WITH_KEYWORDS 34 -#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 38 -#define CALL_NO_KW_BUILTIN_FAST 39 -#define CALL_NO_KW_BUILTIN_O 40 -#define CALL_NO_KW_ISINSTANCE 41 -#define CALL_NO_KW_LEN 42 -#define CALL_NO_KW_LIST_APPEND 43 -#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 44 -#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 45 -#define CALL_NO_KW_METHOD_DESCRIPTOR_O 46 -#define CALL_NO_KW_STR_1 47 -#define CALL_NO_KW_TUPLE_1 48 -#define CALL_NO_KW_TYPE_1 56 -#define COMPARE_OP_FLOAT_JUMP 57 -#define COMPARE_OP_GENERIC 58 -#define COMPARE_OP_INT_JUMP 59 -#define COMPARE_OP_STR_JUMP 62 -#define FOR_ITER_LIST 64 -#define FOR_ITER_RANGE 65 -#define FOR_ITER_GEN 66 -#define LOAD_ATTR_CLASS 67 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 72 -#define LOAD_ATTR_INSTANCE_VALUE 73 -#define LOAD_ATTR_MODULE 76 -#define LOAD_ATTR_PROPERTY 77 -#define LOAD_ATTR_SLOT 78 -#define LOAD_ATTR_WITH_HINT 79 -#define LOAD_ATTR_METHOD_LAZY_DICT 80 -#define LOAD_ATTR_METHOD_NO_DICT 81 -#define LOAD_ATTR_METHOD_WITH_DICT 86 -#define LOAD_ATTR_METHOD_WITH_VALUES 113 -#define LOAD_CONST__LOAD_FAST 121 -#define LOAD_FAST__LOAD_CONST 141 -#define LOAD_FAST__LOAD_FAST 143 -#define LOAD_GLOBAL_BUILTIN 153 -#define LOAD_GLOBAL_MODULE 154 -#define STORE_ATTR_INSTANCE_VALUE 158 -#define STORE_ATTR_SLOT 159 -#define STORE_ATTR_WITH_HINT 160 -#define STORE_FAST__LOAD_FAST 161 -#define STORE_FAST__STORE_FAST 166 -#define STORE_SUBSCR_DICT 167 -#define STORE_SUBSCR_LIST_INT 168 -#define UNPACK_SEQUENCE_LIST 169 -#define UNPACK_SEQUENCE_TUPLE 170 -#define UNPACK_SEQUENCE_TWO_TUPLE 173 +#define BINARY_OP_MULTIPLY_INT 17 +#define BINARY_OP_SUBTRACT_FLOAT 18 +#define BINARY_OP_SUBTRACT_INT 19 +#define BINARY_SUBSCR_DICT 20 +#define BINARY_SUBSCR_GETITEM 21 +#define BINARY_SUBSCR_LIST_INT 22 +#define BINARY_SUBSCR_TUPLE_INT 23 +#define CALL_PY_EXACT_ARGS 24 +#define CALL_PY_WITH_DEFAULTS 28 +#define CALL_BOUND_METHOD_EXACT_ARGS 29 +#define CALL_BUILTIN_CLASS 34 +#define CALL_BUILTIN_FAST_WITH_KEYWORDS 38 +#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 39 +#define CALL_NO_KW_BUILTIN_FAST 40 +#define CALL_NO_KW_BUILTIN_O 41 +#define CALL_NO_KW_ISINSTANCE 42 +#define CALL_NO_KW_LEN 43 +#define CALL_NO_KW_ALLOC_AND_ENTER_INIT 44 +#define CALL_NO_KW_LIST_APPEND 45 +#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 46 +#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 47 +#define CALL_NO_KW_METHOD_DESCRIPTOR_O 48 +#define CALL_NO_KW_STR_1 56 +#define CALL_NO_KW_TUPLE_1 57 +#define CALL_NO_KW_TYPE_1 58 +#define COMPARE_OP_FLOAT_JUMP 59 +#define COMPARE_OP_GENERIC 62 +#define COMPARE_OP_INT_JUMP 64 +#define COMPARE_OP_STR_JUMP 65 +#define FOR_ITER_LIST 66 +#define FOR_ITER_RANGE 67 +#define FOR_ITER_GEN 72 +#define LOAD_ATTR_CLASS 73 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 76 +#define LOAD_ATTR_INSTANCE_VALUE 77 +#define LOAD_ATTR_MODULE 78 +#define LOAD_ATTR_PROPERTY 79 +#define LOAD_ATTR_SLOT 80 +#define LOAD_ATTR_WITH_HINT 81 +#define LOAD_ATTR_METHOD_LAZY_DICT 86 +#define LOAD_ATTR_METHOD_NO_DICT 113 +#define LOAD_ATTR_METHOD_WITH_DICT 121 +#define LOAD_ATTR_METHOD_WITH_VALUES 141 +#define LOAD_CONST__LOAD_FAST 143 +#define LOAD_FAST__LOAD_CONST 153 +#define LOAD_FAST__LOAD_FAST 154 +#define LOAD_GLOBAL_BUILTIN 158 +#define LOAD_GLOBAL_MODULE 159 +#define STORE_ATTR_INSTANCE_VALUE 160 +#define STORE_ATTR_SLOT 161 +#define STORE_ATTR_WITH_HINT 166 +#define STORE_FAST__LOAD_FAST 167 +#define STORE_FAST__STORE_FAST 168 +#define STORE_SUBSCR_DICT 169 +#define STORE_SUBSCR_LIST_INT 170 +#define UNPACK_SEQUENCE_LIST 173 +#define UNPACK_SEQUENCE_TUPLE 174 +#define UNPACK_SEQUENCE_TWO_TUPLE 175 #define DO_TRACING 255 #define HAS_ARG(op) ((((op) >= HAVE_ARGUMENT) && (!IS_PSEUDO_OPCODE(op)))\ diff --git a/Lib/opcode.py b/Lib/opcode.py index 0ee75958508ac7..9557fe4cb05a8f 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -87,6 +87,7 @@ def pseudo_op(name, op, real_ops): def_op('UNARY_NOT', 12) def_op('UNARY_INVERT', 15) +def_op('EXIT_INIT_CHECK', 16) def_op('BINARY_SUBSCR', 25) def_op('BINARY_SLICE', 26) @@ -306,6 +307,7 @@ def pseudo_op(name, op, real_ops): "CALL_NO_KW_BUILTIN_O", "CALL_NO_KW_ISINSTANCE", "CALL_NO_KW_LEN", + "CALL_NO_KW_ALLOC_AND_ENTER_INIT", "CALL_NO_KW_LIST_APPEND", "CALL_NO_KW_METHOD_DESCRIPTOR_FAST", "CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS", diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 2403c7c815f2c0..83bc8b454daca5 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1532,7 +1532,7 @@ def delx(self): del self.__x '10P' # PySequenceMethods '2P' # PyBufferProcs '6P' - '1P' # Specializer cache + '2P' # Specializer cache ) class newstyleclass(object): pass # Separate block for PyDictKeysObject with 8 keys and 5 entries diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index a251b2272e95eb..d02aa197a43d53 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -1569,6 +1569,29 @@ def func(): self.run_and_compare(func, EXPECTED_EVENTS) + def test_correct_tracing_quickened_call_class_init(self): + + class C: + def __init__(self): + self + + def func(): + C() + + EXPECTED_EVENTS = [ + (0, 'call'), + (1, 'line'), + (-3, 'call'), + (-2, 'line'), + (-2, 'return'), + (1, 'return')] + + self.run_and_compare(func, EXPECTED_EVENTS) + # Quicken + for _ in range(100): + func() + self.run_and_compare(func, EXPECTED_EVENTS) + def test_very_large_function(self): # There is a separate code path when the number of lines > (1 << 15). d = {} diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 97007479b1be91..3bb6ba93c5eea2 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -5324,11 +5324,10 @@ _PyDict_NewKeysForClass(void) #define CACHED_KEYS(tp) (((PyHeapTypeObject*)tp)->ht_cached_keys) -static int -init_inline_values(PyObject *obj, PyTypeObject *tp) +int +_PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp) { assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); - // assert(type->tp_dictoffset > 0); -- TO DO Update this assert. assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictKeysObject *keys = CACHED_KEYS(tp); assert(keys != NULL); @@ -5360,7 +5359,7 @@ _PyObject_InitializeDict(PyObject *obj) } if (tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) { OBJECT_STAT_INC(new_values); - return init_inline_values(obj, tp); + return _PyObject_InitInlineValues(obj, tp); } PyObject *dict; if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && CACHED_KEYS(tp)) { diff --git a/Objects/typeobject.c b/Objects/typeobject.c index e2e40b5c15297d..2c5a392a977c84 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -502,7 +502,6 @@ PyType_Modified(PyTypeObject *type) } } - type->tp_flags &= ~Py_TPFLAGS_VALID_VERSION_TAG; type->tp_version_tag = 0; /* 0 is not a valid version tag */ } @@ -1309,6 +1308,26 @@ type_call(PyTypeObject *type, PyObject *args, PyObject *kwds) return obj; } +PyObject * +_PyType_NewManagedObject(PyTypeObject *type) +{ + assert(type->tp_flags & Py_TPFLAGS_MANAGED_DICT); + assert(_PyType_IS_GC(type)); + assert(type->tp_new == PyBaseObject_Type.tp_new); + assert(type->tp_alloc == PyType_GenericAlloc); + assert(type->tp_itemsize == 0); + PyObject *obj = PyType_GenericAlloc(type, 0); + if (obj == NULL) { + return PyErr_NoMemory(); + } + _PyObject_DictOrValuesPointer(obj)->dict = NULL; + if (_PyObject_InitInlineValues(obj, type)) { + Py_DECREF(obj); + return NULL; + } + return obj; +} + PyObject * _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems) { diff --git a/Python/bytecodes.c b/Python/bytecodes.c index ed12b94e3142c1..3437fb0f036847 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3147,6 +3147,74 @@ dummy_func( CHECK_EVAL_BREAKER(); } + inst(CALL_NO_KW_ALLOC_AND_ENTER_INIT) { + /* This instruction does the following: + * 1. Creates the object (by calling ``object.__new__``) + * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) + * 3. Pushes the frame for ``__init__`` to the frame stack + * */ + assert(kwnames == NULL); + _PyCallCache *cache = (_PyCallCache *)next_instr; + DEOPT_IF(is_method(stack_pointer, oparg), CALL); + PyObject *callable = PEEK(oparg+1); + DEOPT_IF(!PyType_Check(callable), CALL); + PyHeapTypeObject *tp = (PyHeapTypeObject *)callable; + DEOPT_IF(tp->ht_type.tp_version_tag != read_u32(cache->func_version), CALL); + PyFunctionObject *init = (PyFunctionObject *)tp->_spec_cache.init; + PyCodeObject *code = (PyCodeObject *)init->func_code; + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + FRAME_SPECIALS_SIZE + 2), CALL); + STAT_INC(CALL, hit); + DEOPT_IF(code->co_argcount != oparg+1, CALL); + PyObject *self = _PyType_NewManagedObject(&tp->ht_type); + if (self == NULL) { + goto error; + } + PEEK(oparg+1) = self; + Py_DECREF(tp); + CALL_STAT_INC(inlined_py_calls); + if (_Py_EnterRecursivePy(tstate)) { + goto exit_unwind; + } + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( + tstate, interp->callable_cache.init_cleanup, 1, 1); + assert(_Py_OPCODE(*(_PyCode_CODE(shim->f_code)+ 2)) == EXIT_INIT_CHECK); + /* Push self onto stack of shim */ + Py_INCREF(self); + shim->localsplus[0] = self; + shim->previous = frame; + Py_INCREF(init); + _PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init); + /* Copy self followed by args to __init__ frame */ + STACK_SHRINK(oparg+1); + for (int i = 0; i <= oparg; i++) { + init_frame->localsplus[i] = stack_pointer[i]; + } + STACK_SHRINK(1); // Clear the NULL. + _PyFrame_SetStackPointer(frame, stack_pointer); + for (int i = oparg+1; i < code->co_nlocalsplus; i++) { + init_frame->localsplus[i] = NULL; + } + JUMPBY(INLINE_CACHE_ENTRIES_CALL); + frame->prev_instr = next_instr - 1; + init_frame->previous = shim; + frame = cframe.current_frame = init_frame; + goto start_frame; + } + + inst(EXIT_INIT_CHECK) { + assert(STACK_LEVEL() == 2); + PyObject *should_be_none = TOP(); + if (should_be_none != Py_None) { + PyErr_Format(PyExc_TypeError, + "__init__() should return None, not '%.200s'", + Py_TYPE(should_be_none)->tp_name); + goto error; + } + STACK_SHRINK(1); + _Py_DECREF_NO_DEALLOC(Py_None); + } + // stack effect: (__0, __array[oparg] -- ) inst(CALL_BUILTIN_CLASS) { int is_meth = is_method(stack_pointer, oparg); diff --git a/Python/compile.c b/Python/compile.c index c71563f81609a2..3adbfae80471a4 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1138,6 +1138,9 @@ stack_effect(int opcode, int oparg, int jump) case LOAD_GLOBAL: return (oparg & 1) + 1; + case EXIT_INIT_CHECK: + return -1; + /* Exception handling pseudo-instructions */ case SETUP_FINALLY: /* 0 in the normal flow. diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 0487851db6106e..4b830f20c7828a 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3139,6 +3139,75 @@ DISPATCH(); } + TARGET(CALL_NO_KW_ALLOC_AND_ENTER_INIT) { + /* This instruction does the following: + * 1. Creates the object (by calling ``object.__new__``) + * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) + * 3. Pushes the frame for ``__init__`` to the frame stack + * */ + assert(kwnames == NULL); + _PyCallCache *cache = (_PyCallCache *)next_instr; + DEOPT_IF(is_method(stack_pointer, oparg), CALL); + PyObject *callable = PEEK(oparg+1); + DEOPT_IF(!PyType_Check(callable), CALL); + PyHeapTypeObject *tp = (PyHeapTypeObject *)callable; + DEOPT_IF(tp->ht_type.tp_version_tag != read_u32(cache->func_version), CALL); + PyFunctionObject *init = (PyFunctionObject *)tp->_spec_cache.init; + PyCodeObject *code = (PyCodeObject *)init->func_code; + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + FRAME_SPECIALS_SIZE + 2), CALL); + STAT_INC(CALL, hit); + DEOPT_IF(code->co_argcount != oparg+1, CALL); + PyObject *self = _PyType_NewManagedObject(&tp->ht_type); + if (self == NULL) { + goto error; + } + PEEK(oparg+1) = self; + Py_DECREF(tp); + CALL_STAT_INC(inlined_py_calls); + if (_Py_EnterRecursivePy(tstate)) { + goto exit_unwind; + } + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( + tstate, interp->callable_cache.init_cleanup, 1, 1); + assert(_Py_OPCODE(*(_PyCode_CODE(shim->f_code)+ 2)) == EXIT_INIT_CHECK); + /* Push self onto stack of shim */ + Py_INCREF(self); + shim->localsplus[0] = self; + shim->previous = frame; + Py_INCREF(init); + _PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init); + /* Copy self followed by args to __init__ frame */ + STACK_SHRINK(oparg+1); + for (int i = 0; i <= oparg; i++) { + init_frame->localsplus[i] = stack_pointer[i]; + } + STACK_SHRINK(1); // Clear the NULL. + _PyFrame_SetStackPointer(frame, stack_pointer); + for (int i = oparg+1; i < code->co_nlocalsplus; i++) { + init_frame->localsplus[i] = NULL; + } + JUMPBY(INLINE_CACHE_ENTRIES_CALL); + frame->prev_instr = next_instr - 1; + init_frame->previous = shim; + frame = cframe.current_frame = init_frame; + goto start_frame; + } + + TARGET(EXIT_INIT_CHECK) { + assert(STACK_LEVEL() == 2); + PyObject *should_be_none = TOP(); + if (should_be_none != Py_None) { + PyErr_Format(PyExc_TypeError, + "__init__() should return None, not '%.200s'", + Py_TYPE(should_be_none)->tp_name); + goto error; + } + STACK_SHRINK(1); + _Py_DECREF_NO_DEALLOC(Py_None); + DISPATCH(); + } + TARGET(CALL_BUILTIN_CLASS) { int is_meth = is_method(stack_pointer, oparg); int total_args = oparg + is_meth; diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 237d3b946b1066..23ed32c7ad2476 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -15,6 +15,7 @@ static void *opcode_targets[256] = { &&TARGET_BINARY_OP_INPLACE_ADD_UNICODE, &&TARGET_BINARY_OP_MULTIPLY_FLOAT, &&TARGET_UNARY_INVERT, + &&TARGET_EXIT_INIT_CHECK, &&TARGET_BINARY_OP_MULTIPLY_INT, &&TARGET_BINARY_OP_SUBTRACT_FLOAT, &&TARGET_BINARY_OP_SUBTRACT_INT, @@ -23,31 +24,30 @@ static void *opcode_targets[256] = { &&TARGET_BINARY_SUBSCR_LIST_INT, &&TARGET_BINARY_SUBSCR_TUPLE_INT, &&TARGET_CALL_PY_EXACT_ARGS, - &&TARGET_CALL_PY_WITH_DEFAULTS, &&TARGET_BINARY_SUBSCR, &&TARGET_BINARY_SLICE, &&TARGET_STORE_SLICE, + &&TARGET_CALL_PY_WITH_DEFAULTS, &&TARGET_CALL_BOUND_METHOD_EXACT_ARGS, - &&TARGET_CALL_BUILTIN_CLASS, &&TARGET_GET_LEN, &&TARGET_MATCH_MAPPING, &&TARGET_MATCH_SEQUENCE, &&TARGET_MATCH_KEYS, - &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS, + &&TARGET_CALL_BUILTIN_CLASS, &&TARGET_PUSH_EXC_INFO, &&TARGET_CHECK_EXC_MATCH, &&TARGET_CHECK_EG_MATCH, + &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS, &&TARGET_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, &&TARGET_CALL_NO_KW_BUILTIN_FAST, &&TARGET_CALL_NO_KW_BUILTIN_O, &&TARGET_CALL_NO_KW_ISINSTANCE, &&TARGET_CALL_NO_KW_LEN, + &&TARGET_CALL_NO_KW_ALLOC_AND_ENTER_INIT, &&TARGET_CALL_NO_KW_LIST_APPEND, &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_FAST, &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_O, - &&TARGET_CALL_NO_KW_STR_1, - &&TARGET_CALL_NO_KW_TUPLE_1, &&TARGET_WITH_EXCEPT_START, &&TARGET_GET_AITER, &&TARGET_GET_ANEXT, @@ -55,37 +55,37 @@ static void *opcode_targets[256] = { &&TARGET_BEFORE_WITH, &&TARGET_END_ASYNC_FOR, &&TARGET_CLEANUP_THROW, + &&TARGET_CALL_NO_KW_STR_1, + &&TARGET_CALL_NO_KW_TUPLE_1, &&TARGET_CALL_NO_KW_TYPE_1, &&TARGET_COMPARE_OP_FLOAT_JUMP, - &&TARGET_COMPARE_OP_GENERIC, - &&TARGET_COMPARE_OP_INT_JUMP, &&TARGET_STORE_SUBSCR, &&TARGET_DELETE_SUBSCR, - &&TARGET_COMPARE_OP_STR_JUMP, + &&TARGET_COMPARE_OP_GENERIC, &&TARGET_STOPITERATION_ERROR, + &&TARGET_COMPARE_OP_INT_JUMP, + &&TARGET_COMPARE_OP_STR_JUMP, &&TARGET_FOR_ITER_LIST, &&TARGET_FOR_ITER_RANGE, - &&TARGET_FOR_ITER_GEN, - &&TARGET_LOAD_ATTR_CLASS, &&TARGET_GET_ITER, &&TARGET_GET_YIELD_FROM_ITER, &&TARGET_PRINT_EXPR, &&TARGET_LOAD_BUILD_CLASS, - &&TARGET_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, - &&TARGET_LOAD_ATTR_INSTANCE_VALUE, + &&TARGET_FOR_ITER_GEN, + &&TARGET_LOAD_ATTR_CLASS, &&TARGET_LOAD_ASSERTION_ERROR, &&TARGET_RETURN_GENERATOR, + &&TARGET_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, + &&TARGET_LOAD_ATTR_INSTANCE_VALUE, &&TARGET_LOAD_ATTR_MODULE, &&TARGET_LOAD_ATTR_PROPERTY, &&TARGET_LOAD_ATTR_SLOT, &&TARGET_LOAD_ATTR_WITH_HINT, - &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, - &&TARGET_LOAD_ATTR_METHOD_NO_DICT, &&TARGET_LIST_TO_TUPLE, &&TARGET_RETURN_VALUE, &&TARGET_IMPORT_STAR, &&TARGET_SETUP_ANNOTATIONS, - &&TARGET_LOAD_ATTR_METHOD_WITH_DICT, + &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, &&TARGET_ASYNC_GEN_WRAP, &&TARGET_PREP_RERAISE_STAR, &&TARGET_POP_EXCEPT, @@ -112,7 +112,7 @@ static void *opcode_targets[256] = { &&TARGET_JUMP_FORWARD, &&TARGET_JUMP_IF_FALSE_OR_POP, &&TARGET_JUMP_IF_TRUE_OR_POP, - &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, + &&TARGET_LOAD_ATTR_METHOD_NO_DICT, &&TARGET_POP_JUMP_IF_FALSE, &&TARGET_POP_JUMP_IF_TRUE, &&TARGET_LOAD_GLOBAL, @@ -120,7 +120,7 @@ static void *opcode_targets[256] = { &&TARGET_CONTAINS_OP, &&TARGET_RERAISE, &&TARGET_COPY, - &&TARGET_LOAD_CONST__LOAD_FAST, + &&TARGET_LOAD_ATTR_METHOD_WITH_DICT, &&TARGET_BINARY_OP, &&TARGET_SEND, &&TARGET_LOAD_FAST, @@ -140,9 +140,9 @@ static void *opcode_targets[256] = { &&TARGET_STORE_DEREF, &&TARGET_DELETE_DEREF, &&TARGET_JUMP_BACKWARD, - &&TARGET_LOAD_FAST__LOAD_CONST, + &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, &&TARGET_CALL_FUNCTION_EX, - &&TARGET_LOAD_FAST__LOAD_FAST, + &&TARGET_LOAD_CONST__LOAD_FAST, &&TARGET_EXTENDED_ARG, &&TARGET_LIST_APPEND, &&TARGET_SET_ADD, @@ -152,26 +152,28 @@ static void *opcode_targets[256] = { &&TARGET_YIELD_VALUE, &&TARGET_RESUME, &&TARGET_MATCH_CLASS, - &&TARGET_LOAD_GLOBAL_BUILTIN, - &&TARGET_LOAD_GLOBAL_MODULE, + &&TARGET_LOAD_FAST__LOAD_CONST, + &&TARGET_LOAD_FAST__LOAD_FAST, &&TARGET_FORMAT_VALUE, &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_STRING, + &&TARGET_LOAD_GLOBAL_BUILTIN, + &&TARGET_LOAD_GLOBAL_MODULE, &&TARGET_STORE_ATTR_INSTANCE_VALUE, &&TARGET_STORE_ATTR_SLOT, - &&TARGET_STORE_ATTR_WITH_HINT, - &&TARGET_STORE_FAST__LOAD_FAST, &&TARGET_LIST_EXTEND, &&TARGET_SET_UPDATE, &&TARGET_DICT_MERGE, &&TARGET_DICT_UPDATE, + &&TARGET_STORE_ATTR_WITH_HINT, + &&TARGET_STORE_FAST__LOAD_FAST, &&TARGET_STORE_FAST__STORE_FAST, &&TARGET_STORE_SUBSCR_DICT, &&TARGET_STORE_SUBSCR_LIST_INT, - &&TARGET_UNPACK_SEQUENCE_LIST, - &&TARGET_UNPACK_SEQUENCE_TUPLE, &&TARGET_CALL, &&TARGET_KW_NAMES, + &&TARGET_UNPACK_SEQUENCE_LIST, + &&TARGET_UNPACK_SEQUENCE_TUPLE, &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, &&_unknown_opcode, &&_unknown_opcode, @@ -252,7 +254,5 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_DO_TRACING }; diff --git a/Python/pystate.c b/Python/pystate.c index 04db1fb419af62..85586ca8f428a1 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -454,6 +454,7 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) PyDict_Clear(interp->builtins); Py_CLEAR(interp->sysdict); Py_CLEAR(interp->builtins); + Py_CLEAR(interp->callable_cache.init_cleanup); Py_CLEAR(interp->interpreter_trampoline); for (int i=0; i < DICT_MAX_WATCHERS; i++) { diff --git a/Python/specialize.c b/Python/specialize.c index f84596751f9186..dc1d9898c9dd05 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1436,11 +1436,45 @@ _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *ins cache->counter = adaptive_counter_cooldown(); } +static PyFunctionObject * +get_init_for_simple_managed_python_class(PyTypeObject *tp) +{ + if (tp->tp_new != PyBaseObject_Type.tp_new) { + return NULL; + } + if (tp->tp_alloc != PyType_GenericAlloc) { + return NULL; + } + if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { + return NULL; + } + if (!(tp->tp_flags & Py_TPFLAGS_HEAPTYPE)) { + return NULL; + } + PyObject *init = _PyType_Lookup(tp, &_Py_ID(__init__)); + if (init == NULL || !PyFunction_Check(init)) { + return NULL; + } + int kind = function_kind((PyCodeObject *)PyFunction_GET_CODE(init)); + if (kind != SIMPLE_FUNCTION) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_INIT_NOT_SIMPLE); + return NULL; + } + Py_CLEAR(((PyHeapTypeObject *)tp)->_spec_cache.init); + ((PyHeapTypeObject *)tp)->_spec_cache.init = init; + return (PyFunctionObject *)init; +} +static int setup_init_cleanup_code(void); + static int specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, PyObject *kwnames) { + assert(PyType_Check(callable)); PyTypeObject *tp = _PyType_CAST(callable); + if (setup_init_cleanup_code()) { + return -1; + } if (tp->tp_new == PyBaseObject_Type.tp_new) { SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_PYTHON_CLASS); return -1; @@ -1469,6 +1503,23 @@ specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, SPEC_FAIL_CALL_STR : SPEC_FAIL_CALL_CLASS_NO_VECTORCALL); return -1; } + else { + PyFunctionObject *init = get_init_for_simple_managed_python_class(tp); + if (init != NULL) { + if (((PyCodeObject *)init->func_code)->co_argcount != nargs+1) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); + return -1; + } + if (kwnames) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_KWNAMES); + return -1; + } + _PyCallCache *cache = (_PyCallCache *)(instr + 1); + write_u32(cache->func_version, tp->tp_version_tag); + _Py_SET_OPCODE(*instr, CALL_NO_KW_ALLOC_AND_ENTER_INIT); + return 0; + } + } SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_CLASS_MUTABLE); return -1; } @@ -2176,3 +2227,42 @@ _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr, int oparg) STAT_INC(FOR_ITER, success); cache->counter = adaptive_counter_cooldown(); } + +/* Code init cleanup. + * CALL_NO_KW_ALLOC_AND_ENTER_INIT will set up + * the frame to execute the EXIT_INIT_CHECK + * instruction. + * Starts with an assertion error, in case it is called + * directly. + * Ends with a RESUME so that it is not traced. + * This is used as a plain code object, not a function, + * so must not access globals or builtins. + */ +static const uint8_t INIT_CLEANUP_CODE[10] = { + LOAD_ASSERTION_ERROR, 0, + RAISE_VARARGS, 1, + EXIT_INIT_CHECK, 0, + RETURN_VALUE, 0, + RESUME, 0 +}; + +static const _PyShimCodeDef INIT_CLEANUP_DEF = { + INIT_CLEANUP_CODE, + sizeof(INIT_CLEANUP_CODE), + 2, + "type.__call__" +}; + +static int +setup_init_cleanup_code(void) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (interp->callable_cache.init_cleanup != NULL) { + return 0; + } + PyCodeObject *init = _Py_MakeShimCode(&INIT_CLEANUP_DEF); + if (init == NULL) { + return -1; + } + interp->callable_cache.init_cleanup = init; + return 0; +} From 43edcd66a82099a8d2f430598edb6df548374d6f Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 10 Nov 2022 16:03:24 +0000 Subject: [PATCH 02/10] Add news --- .../2022-11-10-13-04-35.gh-issue-91095.4E3Pwn.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-11-10-13-04-35.gh-issue-91095.4E3Pwn.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-10-13-04-35.gh-issue-91095.4E3Pwn.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-10-13-04-35.gh-issue-91095.4E3Pwn.rst new file mode 100644 index 00000000000000..5633097f4a3fdd --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-11-10-13-04-35.gh-issue-91095.4E3Pwn.rst @@ -0,0 +1,11 @@ +Specializes calls to most Python classes. Specifically, any class that +inherits from ``object``, or another Python class, and does not override +``__new__``. + +The specialized instruction does the following: + +1. Creates the object (by calling ``object.__new__``) +2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) +3. Pushes the frame for ``__init__`` to the frame stack + +Speeds up the instantiation of most Python classes. From fbeaf9bab6a38b1da6c2cf8fed38914c441d7aa4 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 9 Mar 2023 03:50:37 +0000 Subject: [PATCH 03/10] Fix up after merge --- Include/internal/pycore_frame.h | 6 ++--- Python/bytecodes.c | 16 ++++++-------- Python/ceval.c | 2 +- Python/generated_cases.c.h | 18 +++++++-------- Python/specialize.c | 39 +++++++++++++++++---------------- 5 files changed, 39 insertions(+), 42 deletions(-) diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index 8edf21447cefaa..bf5c30309f0aa3 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -255,7 +255,7 @@ _PyFrame_PushUnchecked(PyThreadState *tstate, PyFunctionObject *func, int null_l /* Pushes a trampoline frame without checking for space. * Must be guarded by _PyThreadState_HasStackSpace() */ static inline _PyInterpreterFrame * -_PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int stackdepth, int start) +_PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int stackdepth, int prev_instr) { CALL_STAT_INC(frames_pushed); _PyInterpreterFrame *frame = (_PyInterpreterFrame *)tstate->datastack_top; @@ -265,12 +265,12 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int frame->f_code = (PyCodeObject *)Py_NewRef(code); #ifdef Py_DEBUG frame->f_builtins = (PyObject*)0xaaaa; - frame->f_globals = (PyObject*)0xaaab; + frame->f_globals = NULL; #endif frame->f_locals = NULL; frame->stacktop = code->co_nlocalsplus + stackdepth; frame->frame_obj = NULL; - frame->prev_instr = _PyCode_CODE(code) + start; + frame->prev_instr = _PyCode_CODE(code) + prev_instr; frame->owner = FRAME_OWNED_BY_THREAD; frame->yield_offset = 0; return frame; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 7407e40915c5f7..390b5b5de51ebb 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2615,7 +2615,6 @@ dummy_func( if (self == NULL) { goto error; } - PEEK(oparg+1) = self; Py_DECREF(tp); CALL_STAT_INC(inlined_py_calls); if (_Py_EnterRecursivePy(tstate)) { @@ -2624,25 +2623,24 @@ dummy_func( PyInterpreterState *interp = _PyInterpreterState_GET(); _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( tstate, interp->callable_cache.init_cleanup, 1, 1); - assert(_Py_OPCODE(*(_PyCode_CODE(shim->f_code)+ 2)) == EXIT_INIT_CHECK); + assert(_PyCode_CODE(shim->f_code)[2].op.code == EXIT_INIT_CHECK); /* Push self onto stack of shim */ Py_INCREF(self); shim->localsplus[0] = self; - shim->previous = frame; Py_INCREF(init); - _PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init, code->co_argcount); + _PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init, oparg+1); /* Copy self followed by args to __init__ frame */ - for (int i = 0; i <= oparg; i++) { - init_frame->localsplus[i] = args[i]; - } - for (int i = oparg+1; i < code->co_nlocalsplus; i++) { - init_frame->localsplus[i] = NULL; + init_frame->localsplus[0] = self; + for (int i = 0; i < oparg; i++) { + init_frame->localsplus[i+1] = args[i]; } STACK_SHRINK(oparg+2); _PyFrame_SetStackPointer(frame, stack_pointer); JUMPBY(INLINE_CACHE_ENTRIES_CALL); frame->prev_instr = next_instr - 1; + /* Link frames */ init_frame->previous = shim; + shim->previous = frame; frame = cframe.current_frame = init_frame; goto start_frame; } diff --git a/Python/ceval.c b/Python/ceval.c index b422d0ed34ede3..3b265ab4bbe8c3 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -808,7 +808,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #ifdef LLTRACE { - if (frame != &entry_frame) { + if (frame != &entry_frame && GLOBALS()) { int r = PyDict_Contains(GLOBALS(), &_Py_ID(__lltrace__)); if (r < 0) { goto exit_unwind; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 9445ce2e5ec7c8..f238a4973581d4 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3273,7 +3273,6 @@ if (self == NULL) { goto error; } - PEEK(oparg+1) = self; Py_DECREF(tp); CALL_STAT_INC(inlined_py_calls); if (_Py_EnterRecursivePy(tstate)) { @@ -3282,25 +3281,24 @@ PyInterpreterState *interp = _PyInterpreterState_GET(); _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( tstate, interp->callable_cache.init_cleanup, 1, 1); - assert(_Py_OPCODE(*(_PyCode_CODE(shim->f_code)+ 2)) == EXIT_INIT_CHECK); + assert(_PyCode_CODE(shim->f_code)[2].op.code == EXIT_INIT_CHECK); /* Push self onto stack of shim */ Py_INCREF(self); shim->localsplus[0] = self; - shim->previous = frame; Py_INCREF(init); - _PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init, code->co_argcount); + _PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init, oparg+1); + /* Link frames */ + init_frame->previous = shim; + shim->previous = frame; /* Copy self followed by args to __init__ frame */ - for (int i = 0; i <= oparg; i++) { - init_frame->localsplus[i] = args[i]; - } - for (int i = oparg+1; i < code->co_nlocalsplus; i++) { - init_frame->localsplus[i] = NULL; + init_frame->localsplus[0] = self; + for (int i = 0; i < oparg; i++) { + init_frame->localsplus[i+1] = args[i]; } STACK_SHRINK(oparg+2); _PyFrame_SetStackPointer(frame, stack_pointer); JUMPBY(INLINE_CACHE_ENTRIES_CALL); frame->prev_instr = next_instr - 1; - init_frame->previous = shim; frame = cframe.current_frame = init_frame; goto start_frame; } diff --git a/Python/specialize.c b/Python/specialize.c index ed0d97f485f129..4c3207ca6b3f98 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -434,6 +434,7 @@ _PyCode_Quicken(PyCodeObject *code) #define SPEC_FAIL_CALL_KWNAMES 27 #define SPEC_FAIL_CALL_METHOD_WRAPPER 28 #define SPEC_FAIL_CALL_OPERATOR_WRAPPER 29 +#define SPEC_FAIL_CALL_INIT_NOT_SIMPLE 30 /* COMPARE_OP */ #define SPEC_FAIL_COMPARE_DIFFERENT_TYPES 12 @@ -1478,6 +1479,9 @@ _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *ins cache->counter = adaptive_counter_cooldown(); } +/* Returns a borrowed reference. + * The reference is only valid if guarded by a type version check. + */ static PyFunctionObject * get_init_for_simple_managed_python_class(PyTypeObject *tp) { @@ -1499,10 +1503,9 @@ get_init_for_simple_managed_python_class(PyTypeObject *tp) } int kind = function_kind((PyCodeObject *)PyFunction_GET_CODE(init)); if (kind != SIMPLE_FUNCTION) { - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_INIT_NOT_SIMPLE); + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_INIT_NOT_SIMPLE); return NULL; } - Py_CLEAR(((PyHeapTypeObject *)tp)->_spec_cache.init); ((PyHeapTypeObject *)tp)->_spec_cache.init = init; return (PyFunctionObject *)init; } @@ -1518,6 +1521,21 @@ specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, return -1; } if (tp->tp_new == PyBaseObject_Type.tp_new) { + PyFunctionObject *init = get_init_for_simple_managed_python_class(tp); + if (init != NULL) { + if (((PyCodeObject *)init->func_code)->co_argcount != nargs+1) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); + return -1; + } + if (kwnames) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_KWNAMES); + return -1; + } + _PyCallCache *cache = (_PyCallCache *)(instr + 1); + write_u32(cache->func_version, tp->tp_version_tag); + _Py_SET_OPCODE(*instr, CALL_NO_KW_ALLOC_AND_ENTER_INIT); + return 0; + } SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_PYTHON_CLASS); return -1; } @@ -1545,23 +1563,6 @@ specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, SPEC_FAIL_CALL_STR : SPEC_FAIL_CALL_CLASS_NO_VECTORCALL); return -1; } - else { - PyFunctionObject *init = get_init_for_simple_managed_python_class(tp); - if (init != NULL) { - if (((PyCodeObject *)init->func_code)->co_argcount != nargs+1) { - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); - return -1; - } - if (kwnames) { - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_KWNAMES); - return -1; - } - _PyCallCache *cache = (_PyCallCache *)(instr + 1); - write_u32(cache->func_version, tp->tp_version_tag); - _Py_SET_OPCODE(*instr, CALL_NO_KW_ALLOC_AND_ENTER_INIT); - return 0; - } - } SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_CLASS_MUTABLE); return -1; } From b851a2a1d6bcdb9d7ba1269dc179cbce89e3354c Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 9 Mar 2023 04:15:39 +0000 Subject: [PATCH 04/10] Update generated cases. --- Python/generated_cases.c.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index f238a4973581d4..734ed7b5ca61a6 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3287,9 +3287,6 @@ shim->localsplus[0] = self; Py_INCREF(init); _PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init, oparg+1); - /* Link frames */ - init_frame->previous = shim; - shim->previous = frame; /* Copy self followed by args to __init__ frame */ init_frame->localsplus[0] = self; for (int i = 0; i < oparg; i++) { @@ -3299,6 +3296,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); JUMPBY(INLINE_CACHE_ENTRIES_CALL); frame->prev_instr = next_instr - 1; + /* Link frames */ + init_frame->previous = shim; + shim->previous = frame; frame = cframe.current_frame = init_frame; goto start_frame; } From e46f3241efe88cb72a16a5653a9845480fe99eb1 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 9 Mar 2023 06:39:38 +0000 Subject: [PATCH 05/10] Swap order of specializations and make sure we have meaningful stats. --- Python/specialize.c | 48 +++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/Python/specialize.c b/Python/specialize.c index 4c3207ca6b3f98..866eaba6d4699b 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -425,7 +425,7 @@ _PyCode_Quicken(PyCodeObject *code) #define SPEC_FAIL_CALL_METH_DESCR_VARARGS_KEYWORDS 18 #define SPEC_FAIL_CALL_METH_DESCR_METHOD_FASTCALL_KEYWORDS 19 #define SPEC_FAIL_CALL_BAD_CALL_FLAGS 20 -#define SPEC_FAIL_CALL_PYTHON_CLASS 21 +#define SPEC_FAIL_CALL_INIT_NOT_PYTHON 21 #define SPEC_FAIL_CALL_PEP_523 22 #define SPEC_FAIL_CALL_BOUND_METHOD 23 #define SPEC_FAIL_CALL_STR 24 @@ -1485,20 +1485,23 @@ _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *ins static PyFunctionObject * get_init_for_simple_managed_python_class(PyTypeObject *tp) { - if (tp->tp_new != PyBaseObject_Type.tp_new) { - return NULL; - } + assert(tp->tp_new == PyBaseObject_Type.tp_new); if (tp->tp_alloc != PyType_GenericAlloc) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OVERRIDDEN); return NULL; } if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_NO_DICT); return NULL; } if (!(tp->tp_flags & Py_TPFLAGS_HEAPTYPE)) { + /* Is this possible? */ + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_EXPECTED_ERROR); return NULL; } PyObject *init = _PyType_Lookup(tp, &_Py_ID(__init__)); if (init == NULL || !PyFunction_Check(init)) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_INIT_NOT_PYTHON); return NULL; } int kind = function_kind((PyCodeObject *)PyFunction_GET_CODE(init)); @@ -1520,25 +1523,6 @@ specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, if (setup_init_cleanup_code()) { return -1; } - if (tp->tp_new == PyBaseObject_Type.tp_new) { - PyFunctionObject *init = get_init_for_simple_managed_python_class(tp); - if (init != NULL) { - if (((PyCodeObject *)init->func_code)->co_argcount != nargs+1) { - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); - return -1; - } - if (kwnames) { - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_KWNAMES); - return -1; - } - _PyCallCache *cache = (_PyCallCache *)(instr + 1); - write_u32(cache->func_version, tp->tp_version_tag); - _Py_SET_OPCODE(*instr, CALL_NO_KW_ALLOC_AND_ENTER_INIT); - return 0; - } - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_PYTHON_CLASS); - return -1; - } if (tp->tp_flags & Py_TPFLAGS_IMMUTABLETYPE) { int oparg = instr->op.arg; if (nargs == 1 && kwnames == NULL && oparg == 1) { @@ -1563,6 +1547,24 @@ specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, SPEC_FAIL_CALL_STR : SPEC_FAIL_CALL_CLASS_NO_VECTORCALL); return -1; } + if (tp->tp_new == PyBaseObject_Type.tp_new) { + PyFunctionObject *init = get_init_for_simple_managed_python_class(tp); + if (init != NULL) { + if (((PyCodeObject *)init->func_code)->co_argcount != nargs+1) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); + return -1; + } + if (kwnames) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_KWNAMES); + return -1; + } + _PyCallCache *cache = (_PyCallCache *)(instr + 1); + write_u32(cache->func_version, tp->tp_version_tag); + _Py_SET_OPCODE(*instr, CALL_NO_KW_ALLOC_AND_ENTER_INIT); + return 0; + } + return -1; + } SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_CLASS_MUTABLE); return -1; } From dab56e5a5a3301f8522fc71b1cebe13519997bc1 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 18 Jun 2023 09:53:28 +0100 Subject: [PATCH 06/10] Fix up new specialization to work with current interpreter generator and instrumentation. --- Include/internal/pycore_frame.h | 4 +- Python/bytecodes.c | 8 ++-- Python/generated_cases.c.h | 85 +++++++++++++++++---------------- Python/instrumentation.c | 2 +- Python/opcode_metadata.h | 2 +- Python/specialize.c | 4 +- 6 files changed, 53 insertions(+), 52 deletions(-) diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index b9f6012545da0c..952a96009dda6c 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -281,10 +281,10 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int _PyInterpreterFrame *frame = (_PyInterpreterFrame *)tstate->datastack_top; tstate->datastack_top += code->co_framesize; assert(tstate->datastack_top < tstate->datastack_limit); - frame->f_funcobj = Py_NewRef(Py_None); + frame->f_funcobj = Py_None; frame->f_executable = Py_NewRef(code); #ifdef Py_DEBUG - frame->f_builtins = (PyObject*)0xaaaa; + frame->f_builtins = NULL; frame->f_globals = NULL; #endif frame->f_locals = NULL; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 5c2a3dcbdfc0db..9970e6efbb7794 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2923,7 +2923,6 @@ dummy_func( goto error; } Py_DECREF(tp); - CALL_STAT_INC(inlined_py_calls); if (_Py_EnterRecursivePy(tstate)) { goto exit_unwind; } @@ -2940,14 +2939,16 @@ dummy_func( for (int i = 0; i < oparg; i++) { init_frame->localsplus[i+1] = args[i]; } + SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); + frame->prev_instr = next_instr - 1; + frame->return_offset = 0; STACK_SHRINK(oparg+2); _PyFrame_SetStackPointer(frame, stack_pointer); - JUMPBY(INLINE_CACHE_ENTRIES_CALL); - frame->prev_instr = next_instr - 1; /* Link frames */ init_frame->previous = shim; shim->previous = frame; frame = cframe.current_frame = init_frame; + CALL_STAT_INC(inlined_py_calls); goto start_frame; } @@ -2961,7 +2962,6 @@ dummy_func( } } - inst(CALL_BUILTIN_CLASS, (unused/1, unused/2, method, callable, args[oparg] -- res)) { int is_meth = method != NULL; int total_args = oparg; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 8bdc17e4496ba5..0ada77e13cdf50 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -4050,7 +4050,6 @@ goto error; } Py_DECREF(tp); - CALL_STAT_INC(inlined_py_calls); if (_Py_EnterRecursivePy(tstate)) { goto exit_unwind; } @@ -4067,21 +4066,23 @@ for (int i = 0; i < oparg; i++) { init_frame->localsplus[i+1] = args[i]; } + SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); + frame->prev_instr = next_instr - 1; + frame->return_offset = 0; STACK_SHRINK(oparg+2); _PyFrame_SetStackPointer(frame, stack_pointer); - JUMPBY(INLINE_CACHE_ENTRIES_CALL); - frame->prev_instr = next_instr - 1; /* Link frames */ init_frame->previous = shim; shim->previous = frame; frame = cframe.current_frame = init_frame; + CALL_STAT_INC(inlined_py_calls); goto start_frame; - #line 4080 "Python/generated_cases.c.h" + #line 4081 "Python/generated_cases.c.h" } TARGET(EXIT_INIT_CHECK) { PyObject *should_be_none = stack_pointer[-1]; - #line 2955 "Python/bytecodes.c" + #line 2956 "Python/bytecodes.c" assert(STACK_LEVEL() == 2); if (should_be_none != Py_None) { PyErr_Format(PyExc_TypeError, @@ -4089,7 +4090,7 @@ Py_TYPE(should_be_none)->tp_name); goto error; } - #line 4093 "Python/generated_cases.c.h" + #line 4094 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -4121,7 +4122,7 @@ } Py_DECREF(tp); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4125 "Python/generated_cases.c.h" + #line 4126 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4163,7 +4164,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4167 "Python/generated_cases.c.h" + #line 4168 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4209,7 +4210,7 @@ 'invalid'). In those cases an exception is set, so we must handle it. */ - #line 4213 "Python/generated_cases.c.h" + #line 4214 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4255,7 +4256,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4259 "Python/generated_cases.c.h" + #line 4260 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4294,7 +4295,7 @@ Py_DECREF(callable); Py_DECREF(arg); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4298 "Python/generated_cases.c.h" + #line 4299 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4334,7 +4335,7 @@ Py_DECREF(cls); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4338 "Python/generated_cases.c.h" + #line 4339 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4364,7 +4365,7 @@ SKIP_OVER(INLINE_CACHE_ENTRIES_CALL + 1); assert(next_instr[-1].op.code == POP_TOP); DISPATCH(); - #line 4368 "Python/generated_cases.c.h" + #line 4369 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) { @@ -4402,7 +4403,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4406 "Python/generated_cases.c.h" + #line 4407 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4444,7 +4445,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4448 "Python/generated_cases.c.h" + #line 4449 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4486,7 +4487,7 @@ Py_DECREF(self); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4490 "Python/generated_cases.c.h" + #line 4491 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4527,7 +4528,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4531 "Python/generated_cases.c.h" + #line 4532 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4539,7 +4540,7 @@ TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { #line 3298 "Python/bytecodes.c" GO_TO_INSTRUCTION(CALL_FUNCTION_EX); - #line 4543 "Python/generated_cases.c.h" + #line 4544 "Python/generated_cases.c.h" } TARGET(CALL_FUNCTION_EX) { @@ -4610,14 +4611,14 @@ } result = PyObject_Call(func, callargs, kwargs); } - #line 4614 "Python/generated_cases.c.h" + #line 4615 "Python/generated_cases.c.h" Py_DECREF(func); Py_DECREF(callargs); Py_XDECREF(kwargs); #line 3364 "Python/bytecodes.c" assert(PEEK(3 + (oparg & 1)) == NULL); if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } - #line 4621 "Python/generated_cases.c.h" + #line 4622 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 1) ? 1 : 0)); STACK_SHRINK(2); stack_pointer[-1] = result; @@ -4640,7 +4641,7 @@ func_obj->func_version = ((PyCodeObject *)codeobj)->co_version; func = (PyObject *)func_obj; - #line 4644 "Python/generated_cases.c.h" + #line 4645 "Python/generated_cases.c.h" stack_pointer[-1] = func; DISPATCH(); } @@ -4673,7 +4674,7 @@ default: Py_UNREACHABLE(); } - #line 4677 "Python/generated_cases.c.h" + #line 4678 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = func; DISPATCH(); @@ -4701,7 +4702,7 @@ frame = cframe.current_frame = prev; _PyFrame_StackPush(frame, (PyObject *)gen); goto resume_frame; - #line 4705 "Python/generated_cases.c.h" + #line 4706 "Python/generated_cases.c.h" } TARGET(BUILD_SLICE) { @@ -4711,13 +4712,13 @@ PyObject *slice; #line 3434 "Python/bytecodes.c" slice = PySlice_New(start, stop, step); - #line 4715 "Python/generated_cases.c.h" + #line 4716 "Python/generated_cases.c.h" Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); #line 3436 "Python/bytecodes.c" if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } - #line 4721 "Python/generated_cases.c.h" + #line 4722 "Python/generated_cases.c.h" STACK_SHRINK(((oparg == 3) ? 1 : 0)); STACK_SHRINK(1); stack_pointer[-1] = slice; @@ -4734,7 +4735,7 @@ result = conv_fn(value); Py_DECREF(value); if (result == NULL) goto pop_1_error; - #line 4738 "Python/generated_cases.c.h" + #line 4739 "Python/generated_cases.c.h" stack_pointer[-1] = result; DISPATCH(); } @@ -4753,7 +4754,7 @@ else { res = value; } - #line 4757 "Python/generated_cases.c.h" + #line 4758 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -4767,7 +4768,7 @@ Py_DECREF(value); Py_DECREF(fmt_spec); if (res == NULL) goto pop_2_error; - #line 4771 "Python/generated_cases.c.h" + #line 4772 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); @@ -4779,7 +4780,7 @@ #line 3469 "Python/bytecodes.c" assert(oparg > 0); top = Py_NewRef(bottom); - #line 4783 "Python/generated_cases.c.h" + #line 4784 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = top; DISPATCH(); @@ -4806,12 +4807,12 @@ assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); assert(binary_ops[oparg]); res = binary_ops[oparg](lhs, rhs); - #line 4810 "Python/generated_cases.c.h" + #line 4811 "Python/generated_cases.c.h" Py_DECREF(lhs); Py_DECREF(rhs); #line 3489 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 4815 "Python/generated_cases.c.h" + #line 4816 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -4823,7 +4824,7 @@ PyObject *bottom = stack_pointer[-(2 + (oparg-2))]; #line 3494 "Python/bytecodes.c" assert(oparg >= 2); - #line 4827 "Python/generated_cases.c.h" + #line 4828 "Python/generated_cases.c.h" stack_pointer[-1] = bottom; stack_pointer[-(2 + (oparg-2))] = top; DISPATCH(); @@ -4842,20 +4843,20 @@ assert(next_opcode > 0 && next_opcode < 256); opcode = next_opcode; DISPATCH_GOTO(); - #line 4846 "Python/generated_cases.c.h" + #line 4847 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_JUMP_FORWARD) { #line 3512 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); - #line 4852 "Python/generated_cases.c.h" + #line 4853 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_JUMP_BACKWARD) { #line 3516 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr+1-oparg, PY_MONITORING_EVENT_JUMP); - #line 4859 "Python/generated_cases.c.h" + #line 4860 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -4870,7 +4871,7 @@ assert(err == 0 || err == 1); int offset = err*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4874 "Python/generated_cases.c.h" + #line 4875 "Python/generated_cases.c.h" DISPATCH(); } @@ -4884,7 +4885,7 @@ assert(err == 0 || err == 1); int offset = (1-err)*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4888 "Python/generated_cases.c.h" + #line 4889 "Python/generated_cases.c.h" DISPATCH(); } @@ -4901,7 +4902,7 @@ offset = 0; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4905 "Python/generated_cases.c.h" + #line 4906 "Python/generated_cases.c.h" DISPATCH(); } @@ -4918,7 +4919,7 @@ offset = oparg; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4922 "Python/generated_cases.c.h" + #line 4923 "Python/generated_cases.c.h" DISPATCH(); } @@ -4929,19 +4930,19 @@ oparg = oparg << 8 | next_instr->op.arg; PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); - #line 4933 "Python/generated_cases.c.h" + #line 4934 "Python/generated_cases.c.h" } TARGET(CACHE) { #line 3579 "Python/bytecodes.c" assert(0 && "Executing a cache."); Py_UNREACHABLE(); - #line 4940 "Python/generated_cases.c.h" + #line 4941 "Python/generated_cases.c.h" } TARGET(RESERVED) { #line 3584 "Python/bytecodes.c" assert(0 && "Executing RESERVED instruction."); Py_UNREACHABLE(); - #line 4947 "Python/generated_cases.c.h" + #line 4948 "Python/generated_cases.c.h" } diff --git a/Python/instrumentation.c b/Python/instrumentation.c index fcaccc022c8b2d..524a82c076e99d 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1526,7 +1526,7 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) return 0; } /* Insert instrumentation */ - for (int i = 0; i < code_len; i+= _PyInstruction_GetLength(code, i)) { + for (int i = code->_co_firsttraceable; i < code_len; i+= _PyInstruction_GetLength(code, i)) { _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; CHECK(instr->op.code != 0); int base_opcode = _Py_GetBaseOpcode(code, i); diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index b0f759e0147584..421d9795e7da76 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -1065,7 +1065,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[512] = { [CALL_NO_KW_TYPE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG }, [CALL_NO_KW_STR_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG }, [CALL_NO_KW_TUPLE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG }, - [CALL_NO_KW_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [CALL_NO_KW_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG }, [EXIT_INIT_CHECK] = { true, INSTR_FMT_IX, 0 }, [CALL_BUILTIN_CLASS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG }, [CALL_NO_KW_BUILTIN_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG }, diff --git a/Python/specialize.c b/Python/specialize.c index f0d663fe239410..66a4e7669022da 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -2291,11 +2291,11 @@ _Py_Specialize_Send(PyObject *receiver, _Py_CODEUNIT *instr) * so must not access globals or builtins. */ -#define NO_LOC (128 | (PY_CODE_LOCATION_INFO_NONE << 3) | 7) +#define NO_LOC_4 (128 | (PY_CODE_LOCATION_INFO_NONE << 3) | 3) static PyBytesObject no_location = { PyVarObject_HEAD_INIT(&PyBytes_Type, 1) - .ob_sval = { NO_LOC } + .ob_sval = { NO_LOC_4 } }; struct _PyCode_DEF(8) _Py_InitCleanup = { From 4f4de5f3cac12a2bc0135222f8df78b0486aecb3 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 18 Jun 2023 10:26:25 +0100 Subject: [PATCH 07/10] Address review comments --- Python/bytecodes.c | 13 +-- Python/generated_cases.c.h | 163 +++++++++++++++++++------------------ 2 files changed, 89 insertions(+), 87 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 9970e6efbb7794..3ef8ed0ccaafdc 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2911,14 +2911,15 @@ dummy_func( _PyCallCache *cache = (_PyCallCache *)next_instr; DEOPT_IF(null != NULL, CALL); DEOPT_IF(!PyType_Check(callable), CALL); - PyHeapTypeObject *tp = (PyHeapTypeObject *)callable; - DEOPT_IF(tp->ht_type.tp_version_tag != read_u32(cache->func_version), CALL); - PyFunctionObject *init = (PyFunctionObject *)tp->_spec_cache.init; + PyTypeObject *tp = (PyTypeObject *)callable; + DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version), CALL); + PyHeapTypeObject *cls = (PyHeapTypeObject *)callable; + PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init; PyCodeObject *code = (PyCodeObject *)init->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + FRAME_SPECIALS_SIZE + 2), CALL); - STAT_INC(CALL, hit); DEOPT_IF(code->co_argcount != oparg+1, CALL); - PyObject *self = _PyType_NewManagedObject(&tp->ht_type); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); + STAT_INC(CALL, hit); + PyObject *self = _PyType_NewManagedObject(tp); if (self == NULL) { goto error; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 0ada77e13cdf50..155f41fc5367f6 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -4038,14 +4038,15 @@ _PyCallCache *cache = (_PyCallCache *)next_instr; DEOPT_IF(null != NULL, CALL); DEOPT_IF(!PyType_Check(callable), CALL); - PyHeapTypeObject *tp = (PyHeapTypeObject *)callable; - DEOPT_IF(tp->ht_type.tp_version_tag != read_u32(cache->func_version), CALL); - PyFunctionObject *init = (PyFunctionObject *)tp->_spec_cache.init; + PyTypeObject *tp = (PyTypeObject *)callable; + DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version), CALL); + PyHeapTypeObject *cls = (PyHeapTypeObject *)callable; + PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init; PyCodeObject *code = (PyCodeObject *)init->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + FRAME_SPECIALS_SIZE + 2), CALL); - STAT_INC(CALL, hit); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); DEOPT_IF(code->co_argcount != oparg+1, CALL); - PyObject *self = _PyType_NewManagedObject(&tp->ht_type); + STAT_INC(CALL, hit); + PyObject *self = _PyType_NewManagedObject(tp); if (self == NULL) { goto error; } @@ -4077,12 +4078,12 @@ frame = cframe.current_frame = init_frame; CALL_STAT_INC(inlined_py_calls); goto start_frame; - #line 4081 "Python/generated_cases.c.h" + #line 4082 "Python/generated_cases.c.h" } TARGET(EXIT_INIT_CHECK) { PyObject *should_be_none = stack_pointer[-1]; - #line 2956 "Python/bytecodes.c" + #line 2957 "Python/bytecodes.c" assert(STACK_LEVEL() == 2); if (should_be_none != Py_None) { PyErr_Format(PyExc_TypeError, @@ -4090,7 +4091,7 @@ Py_TYPE(should_be_none)->tp_name); goto error; } - #line 4094 "Python/generated_cases.c.h" + #line 4095 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -4100,7 +4101,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2966 "Python/bytecodes.c" + #line 2967 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -4122,7 +4123,7 @@ } Py_DECREF(tp); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4126 "Python/generated_cases.c.h" + #line 4127 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4136,7 +4137,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2991 "Python/bytecodes.c" + #line 2992 "Python/bytecodes.c" /* Builtin METH_O functions */ assert(kwnames == NULL); int is_meth = method != NULL; @@ -4164,7 +4165,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4168 "Python/generated_cases.c.h" + #line 4169 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4178,7 +4179,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3022 "Python/bytecodes.c" + #line 3023 "Python/bytecodes.c" /* Builtin METH_FASTCALL functions, without keywords */ assert(kwnames == NULL); int is_meth = method != NULL; @@ -4210,7 +4211,7 @@ 'invalid'). In those cases an exception is set, so we must handle it. */ - #line 4214 "Python/generated_cases.c.h" + #line 4215 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4224,7 +4225,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3057 "Python/bytecodes.c" + #line 3058 "Python/bytecodes.c" /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ int is_meth = method != NULL; int total_args = oparg; @@ -4256,7 +4257,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4260 "Python/generated_cases.c.h" + #line 4261 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4270,7 +4271,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3092 "Python/bytecodes.c" + #line 3093 "Python/bytecodes.c" assert(kwnames == NULL); /* len(o) */ int is_meth = method != NULL; @@ -4295,7 +4296,7 @@ Py_DECREF(callable); Py_DECREF(arg); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4299 "Python/generated_cases.c.h" + #line 4300 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4308,7 +4309,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3119 "Python/bytecodes.c" + #line 3120 "Python/bytecodes.c" assert(kwnames == NULL); /* isinstance(o, o2) */ int is_meth = method != NULL; @@ -4335,7 +4336,7 @@ Py_DECREF(cls); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4339 "Python/generated_cases.c.h" + #line 4340 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4347,7 +4348,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *self = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; - #line 3149 "Python/bytecodes.c" + #line 3150 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); assert(method != NULL); @@ -4365,14 +4366,14 @@ SKIP_OVER(INLINE_CACHE_ENTRIES_CALL + 1); assert(next_instr[-1].op.code == POP_TOP); DISPATCH(); - #line 4369 "Python/generated_cases.c.h" + #line 4370 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) { PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3169 "Python/bytecodes.c" + #line 3170 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -4403,7 +4404,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4407 "Python/generated_cases.c.h" + #line 4408 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4416,7 +4417,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3203 "Python/bytecodes.c" + #line 3204 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -4445,7 +4446,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4449 "Python/generated_cases.c.h" + #line 4450 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4458,7 +4459,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3235 "Python/bytecodes.c" + #line 3236 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 0 || oparg == 1); int is_meth = method != NULL; @@ -4487,7 +4488,7 @@ Py_DECREF(self); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4491 "Python/generated_cases.c.h" + #line 4492 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4500,7 +4501,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3267 "Python/bytecodes.c" + #line 3268 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -4528,7 +4529,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4532 "Python/generated_cases.c.h" + #line 4533 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4538,9 +4539,9 @@ } TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { - #line 3298 "Python/bytecodes.c" + #line 3299 "Python/bytecodes.c" GO_TO_INSTRUCTION(CALL_FUNCTION_EX); - #line 4544 "Python/generated_cases.c.h" + #line 4545 "Python/generated_cases.c.h" } TARGET(CALL_FUNCTION_EX) { @@ -4549,7 +4550,7 @@ PyObject *callargs = stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))]; PyObject *func = stack_pointer[-(2 + ((oparg & 1) ? 1 : 0))]; PyObject *result; - #line 3302 "Python/bytecodes.c" + #line 3303 "Python/bytecodes.c" // DICT_MERGE is called before this opcode if there are kwargs. // It converts all dict subtypes in kwargs into regular dicts. assert(kwargs == NULL || PyDict_CheckExact(kwargs)); @@ -4611,14 +4612,14 @@ } result = PyObject_Call(func, callargs, kwargs); } - #line 4615 "Python/generated_cases.c.h" + #line 4616 "Python/generated_cases.c.h" Py_DECREF(func); Py_DECREF(callargs); Py_XDECREF(kwargs); - #line 3364 "Python/bytecodes.c" + #line 3365 "Python/bytecodes.c" assert(PEEK(3 + (oparg & 1)) == NULL); if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } - #line 4622 "Python/generated_cases.c.h" + #line 4623 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 1) ? 1 : 0)); STACK_SHRINK(2); stack_pointer[-1] = result; @@ -4629,7 +4630,7 @@ TARGET(MAKE_FUNCTION) { PyObject *codeobj = stack_pointer[-1]; PyObject *func; - #line 3370 "Python/bytecodes.c" + #line 3371 "Python/bytecodes.c" PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); @@ -4641,7 +4642,7 @@ func_obj->func_version = ((PyCodeObject *)codeobj)->co_version; func = (PyObject *)func_obj; - #line 4645 "Python/generated_cases.c.h" + #line 4646 "Python/generated_cases.c.h" stack_pointer[-1] = func; DISPATCH(); } @@ -4649,7 +4650,7 @@ TARGET(SET_FUNCTION_ATTRIBUTE) { PyObject *func = stack_pointer[-1]; PyObject *attr = stack_pointer[-2]; - #line 3384 "Python/bytecodes.c" + #line 3385 "Python/bytecodes.c" assert(PyFunction_Check(func)); PyFunctionObject *func_obj = (PyFunctionObject *)func; switch(oparg) { @@ -4674,14 +4675,14 @@ default: Py_UNREACHABLE(); } - #line 4678 "Python/generated_cases.c.h" + #line 4679 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = func; DISPATCH(); } TARGET(RETURN_GENERATOR) { - #line 3411 "Python/bytecodes.c" + #line 3412 "Python/bytecodes.c" assert(PyFunction_Check(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); @@ -4702,7 +4703,7 @@ frame = cframe.current_frame = prev; _PyFrame_StackPush(frame, (PyObject *)gen); goto resume_frame; - #line 4706 "Python/generated_cases.c.h" + #line 4707 "Python/generated_cases.c.h" } TARGET(BUILD_SLICE) { @@ -4710,15 +4711,15 @@ PyObject *stop = stack_pointer[-(1 + ((oparg == 3) ? 1 : 0))]; PyObject *start = stack_pointer[-(2 + ((oparg == 3) ? 1 : 0))]; PyObject *slice; - #line 3434 "Python/bytecodes.c" + #line 3435 "Python/bytecodes.c" slice = PySlice_New(start, stop, step); - #line 4716 "Python/generated_cases.c.h" + #line 4717 "Python/generated_cases.c.h" Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); - #line 3436 "Python/bytecodes.c" + #line 3437 "Python/bytecodes.c" if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } - #line 4722 "Python/generated_cases.c.h" + #line 4723 "Python/generated_cases.c.h" STACK_SHRINK(((oparg == 3) ? 1 : 0)); STACK_SHRINK(1); stack_pointer[-1] = slice; @@ -4728,14 +4729,14 @@ TARGET(CONVERT_VALUE) { PyObject *value = stack_pointer[-1]; PyObject *result; - #line 3440 "Python/bytecodes.c" + #line 3441 "Python/bytecodes.c" convertion_func_ptr conv_fn; assert(oparg >= FVC_STR && oparg <= FVC_ASCII); conv_fn = CONVERSION_FUNCTIONS[oparg]; result = conv_fn(value); Py_DECREF(value); if (result == NULL) goto pop_1_error; - #line 4739 "Python/generated_cases.c.h" + #line 4740 "Python/generated_cases.c.h" stack_pointer[-1] = result; DISPATCH(); } @@ -4743,7 +4744,7 @@ TARGET(FORMAT_SIMPLE) { PyObject *value = stack_pointer[-1]; PyObject *res; - #line 3449 "Python/bytecodes.c" + #line 3450 "Python/bytecodes.c" /* If value is a unicode object, then we know the result * of format(value) is value itself. */ if (!PyUnicode_CheckExact(value)) { @@ -4754,7 +4755,7 @@ else { res = value; } - #line 4758 "Python/generated_cases.c.h" + #line 4759 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -4763,12 +4764,12 @@ PyObject *fmt_spec = stack_pointer[-1]; PyObject *value = stack_pointer[-2]; PyObject *res; - #line 3462 "Python/bytecodes.c" + #line 3463 "Python/bytecodes.c" res = PyObject_Format(value, fmt_spec); Py_DECREF(value); Py_DECREF(fmt_spec); if (res == NULL) goto pop_2_error; - #line 4772 "Python/generated_cases.c.h" + #line 4773 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); @@ -4777,10 +4778,10 @@ TARGET(COPY) { PyObject *bottom = stack_pointer[-(1 + (oparg-1))]; PyObject *top; - #line 3469 "Python/bytecodes.c" + #line 3470 "Python/bytecodes.c" assert(oparg > 0); top = Py_NewRef(bottom); - #line 4784 "Python/generated_cases.c.h" + #line 4785 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = top; DISPATCH(); @@ -4792,7 +4793,7 @@ PyObject *rhs = stack_pointer[-1]; PyObject *lhs = stack_pointer[-2]; PyObject *res; - #line 3474 "Python/bytecodes.c" + #line 3475 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -4807,12 +4808,12 @@ assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); assert(binary_ops[oparg]); res = binary_ops[oparg](lhs, rhs); - #line 4811 "Python/generated_cases.c.h" + #line 4812 "Python/generated_cases.c.h" Py_DECREF(lhs); Py_DECREF(rhs); - #line 3489 "Python/bytecodes.c" + #line 3490 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 4816 "Python/generated_cases.c.h" + #line 4817 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -4822,16 +4823,16 @@ TARGET(SWAP) { PyObject *top = stack_pointer[-1]; PyObject *bottom = stack_pointer[-(2 + (oparg-2))]; - #line 3494 "Python/bytecodes.c" + #line 3495 "Python/bytecodes.c" assert(oparg >= 2); - #line 4828 "Python/generated_cases.c.h" + #line 4829 "Python/generated_cases.c.h" stack_pointer[-1] = bottom; stack_pointer[-(2 + (oparg-2))] = top; DISPATCH(); } TARGET(INSTRUMENTED_INSTRUCTION) { - #line 3498 "Python/bytecodes.c" + #line 3499 "Python/bytecodes.c" int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, next_instr-1); if (next_opcode < 0) goto error; @@ -4843,26 +4844,26 @@ assert(next_opcode > 0 && next_opcode < 256); opcode = next_opcode; DISPATCH_GOTO(); - #line 4847 "Python/generated_cases.c.h" + #line 4848 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_JUMP_FORWARD) { - #line 3512 "Python/bytecodes.c" + #line 3513 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); - #line 4853 "Python/generated_cases.c.h" + #line 4854 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_JUMP_BACKWARD) { - #line 3516 "Python/bytecodes.c" + #line 3517 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr+1-oparg, PY_MONITORING_EVENT_JUMP); - #line 4860 "Python/generated_cases.c.h" + #line 4861 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { - #line 3521 "Python/bytecodes.c" + #line 3522 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); Py_DECREF(cond); @@ -4871,12 +4872,12 @@ assert(err == 0 || err == 1); int offset = err*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4875 "Python/generated_cases.c.h" + #line 4876 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { - #line 3532 "Python/bytecodes.c" + #line 3533 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); Py_DECREF(cond); @@ -4885,12 +4886,12 @@ assert(err == 0 || err == 1); int offset = (1-err)*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4889 "Python/generated_cases.c.h" + #line 4890 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { - #line 3543 "Python/bytecodes.c" + #line 3544 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4902,12 +4903,12 @@ offset = 0; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4906 "Python/generated_cases.c.h" + #line 4907 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { - #line 3557 "Python/bytecodes.c" + #line 3558 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4919,30 +4920,30 @@ offset = oparg; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4923 "Python/generated_cases.c.h" + #line 4924 "Python/generated_cases.c.h" DISPATCH(); } TARGET(EXTENDED_ARG) { - #line 3571 "Python/bytecodes.c" + #line 3572 "Python/bytecodes.c" assert(oparg); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); - #line 4934 "Python/generated_cases.c.h" + #line 4935 "Python/generated_cases.c.h" } TARGET(CACHE) { - #line 3579 "Python/bytecodes.c" + #line 3580 "Python/bytecodes.c" assert(0 && "Executing a cache."); Py_UNREACHABLE(); - #line 4941 "Python/generated_cases.c.h" + #line 4942 "Python/generated_cases.c.h" } TARGET(RESERVED) { - #line 3584 "Python/bytecodes.c" + #line 3585 "Python/bytecodes.c" assert(0 && "Executing RESERVED instruction."); Py_UNREACHABLE(); - #line 4948 "Python/generated_cases.c.h" + #line 4949 "Python/generated_cases.c.h" } From fbe0afe0c89f2c41d2484c9aa055216af94c9d41 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 18 Jun 2023 10:27:33 +0100 Subject: [PATCH 08/10] Update generated cases. --- Python/generated_cases.c.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 155f41fc5367f6..11ca535adfb19b 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -4043,8 +4043,8 @@ PyHeapTypeObject *cls = (PyHeapTypeObject *)callable; PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init; PyCodeObject *code = (PyCodeObject *)init->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); DEOPT_IF(code->co_argcount != oparg+1, CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); STAT_INC(CALL, hit); PyObject *self = _PyType_NewManagedObject(tp); if (self == NULL) { From f76ab3dfa3153d198a98485c22fbaf3cd363a432 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 18 Jun 2023 11:24:14 +0100 Subject: [PATCH 09/10] Make structs const. --- Python/ceval.c | 2 +- Python/specialize.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index e0b18c9387e169..53107018978c0b 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -641,7 +641,7 @@ static const _Py_CODEUNIT _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS[] = { { .op.code = RESUME, .op.arg = 0 } }; -extern struct _PyCode_DEF(8) _Py_InitCleanup; +extern const struct _PyCode_DEF(8) _Py_InitCleanup; /* Disable unused label warnings. They are handy for debugging, even if computed gotos aren't used. */ diff --git a/Python/specialize.c b/Python/specialize.c index 66a4e7669022da..1a630821577ca1 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -2293,12 +2293,12 @@ _Py_Specialize_Send(PyObject *receiver, _Py_CODEUNIT *instr) #define NO_LOC_4 (128 | (PY_CODE_LOCATION_INFO_NONE << 3) | 3) -static PyBytesObject no_location = { +static const PyBytesObject no_location = { PyVarObject_HEAD_INIT(&PyBytes_Type, 1) .ob_sval = { NO_LOC_4 } }; -struct _PyCode_DEF(8) _Py_InitCleanup = { +const struct _PyCode_DEF(8) _Py_InitCleanup = { _PyVarObject_HEAD_INIT(&PyCode_Type, 4) .co_consts = (PyObject *)&_Py_SINGLETON(tuple_empty), .co_names = (PyObject *)&_Py_SINGLETON(tuple_empty), From 1df353e6f533475d037af1f7a1e3ed3b6835dd8c Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 18 Jun 2023 16:15:13 +0100 Subject: [PATCH 10/10] Fix comment. --- Python/specialize.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/Python/specialize.c b/Python/specialize.c index 1a630821577ca1..0006aa733bd6cb 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -2284,8 +2284,6 @@ _Py_Specialize_Send(PyObject *receiver, _Py_CODEUNIT *instr) * CALL_NO_KW_ALLOC_AND_ENTER_INIT will set up * the frame to execute the EXIT_INIT_CHECK * instruction. - * Starts with an assertion error, in case it is called - * directly. * Ends with a RESUME so that it is not traced. * This is used as a plain code object, not a function, * so must not access globals or builtins. pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy