From 830084cf59e4e422424663a78677b9c1f4940197 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 6 Oct 2022 16:28:34 -0700 Subject: [PATCH 01/17] Specialize CALL_FUNCTION_EX --- Include/internal/pycore_call.h | 10 ++ Include/internal/pycore_code.h | 1 + Include/internal/pycore_opcode.h | 67 +++++------ Include/opcode.h | 86 +++++++------- Lib/importlib/_bootstrap_external.py | 4 +- Lib/opcode.py | 11 ++ Objects/call.c | 20 ++-- Python/ceval.c | 160 +++++++++++++++++++++++++++ Python/opcode_targets.h | 58 +++++----- Python/specialize.c | 74 +++++++++++++ 10 files changed, 375 insertions(+), 116 deletions(-) diff --git a/Include/internal/pycore_call.h b/Include/internal/pycore_call.h index 55378e3dfebf24..5d9342b562b002 100644 --- a/Include/internal/pycore_call.h +++ b/Include/internal/pycore_call.h @@ -116,6 +116,16 @@ _PyObject_FastCallTstate(PyThreadState *tstate, PyObject *func, PyObject *const return _PyObject_VectorcallTstate(tstate, func, args, (size_t)nargs, NULL); } +PyObject *const * +_PyStack_UnpackDict(PyThreadState *tstate, + PyObject *const *args, Py_ssize_t nargs, + PyObject *kwargs, PyObject **p_kwnames); + +void +_PyStack_UnpackDict_Free(PyObject *const *stack, Py_ssize_t nargs, + PyObject *kwnames); + +void _PyStack_UnpackDict_FreeNoDecRef(PyObject *const *stack, PyObject *kwnames); #ifdef __cplusplus } diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index bf5945435c1774..29caf4d524c145 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -244,6 +244,7 @@ extern int _Py_Specialize_BinarySubscr(PyObject *sub, PyObject *container, _Py_C extern int _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *instr); extern int _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, PyObject *kwnames); +extern int _Py_Specialize_CallEx(PyObject *callable, _Py_CODEUNIT *instr, PyObject *args, PyObject *kwargs); extern void _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, int oparg, PyObject **locals); extern void _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index 15925511cc1f41..88feb7d693914b 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -50,6 +50,7 @@ const uint8_t _PyOpcode_Caches[256] = { [COMPARE_OP] = 2, [LOAD_GLOBAL] = 5, [BINARY_OP] = 1, + [CALL_FUNCTION_EX] = 4, [CALL] = 4, }; @@ -88,6 +89,10 @@ const uint8_t _PyOpcode_Deopt[256] = { [CALL_BUILTIN_CLASS] = CALL, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = CALL, [CALL_FUNCTION_EX] = CALL_FUNCTION_EX, + [CALL_FUNCTION_EX_ADAPTIVE] = CALL_FUNCTION_EX, + [CALL_FUNCTION_EX_BUILTIN_PYCFUNCTIONWITHKEYWORDS] = CALL_FUNCTION_EX, + [CALL_FUNCTION_EX_PY_KWARGS] = CALL_FUNCTION_EX, + [CALL_FUNCTION_EX_PY_NO_KWARGS] = CALL_FUNCTION_EX, [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = CALL, [CALL_NO_KW_BUILTIN_FAST] = CALL, [CALL_NO_KW_BUILTIN_O] = CALL, @@ -297,36 +302,36 @@ static const char *const _PyOpcode_OpName[263] = { [END_ASYNC_FOR] = "END_ASYNC_FOR", [CLEANUP_THROW] = "CLEANUP_THROW", [CALL_NO_KW_TYPE_1] = "CALL_NO_KW_TYPE_1", + [CALL_FUNCTION_EX_ADAPTIVE] = "CALL_FUNCTION_EX_ADAPTIVE", + [CALL_FUNCTION_EX_PY_KWARGS] = "CALL_FUNCTION_EX_PY_KWARGS", + [CALL_FUNCTION_EX_PY_NO_KWARGS] = "CALL_FUNCTION_EX_PY_NO_KWARGS", + [STORE_SUBSCR] = "STORE_SUBSCR", + [DELETE_SUBSCR] = "DELETE_SUBSCR", + [CALL_FUNCTION_EX_BUILTIN_PYCFUNCTIONWITHKEYWORDS] = "CALL_FUNCTION_EX_BUILTIN_PYCFUNCTIONWITHKEYWORDS", [COMPARE_OP_ADAPTIVE] = "COMPARE_OP_ADAPTIVE", [COMPARE_OP_FLOAT_JUMP] = "COMPARE_OP_FLOAT_JUMP", [COMPARE_OP_INT_JUMP] = "COMPARE_OP_INT_JUMP", - [STORE_SUBSCR] = "STORE_SUBSCR", - [DELETE_SUBSCR] = "DELETE_SUBSCR", [COMPARE_OP_STR_JUMP] = "COMPARE_OP_STR_JUMP", [EXTENDED_ARG_QUICK] = "EXTENDED_ARG_QUICK", - [FOR_ITER_ADAPTIVE] = "FOR_ITER_ADAPTIVE", - [FOR_ITER_LIST] = "FOR_ITER_LIST", - [FOR_ITER_RANGE] = "FOR_ITER_RANGE", - [JUMP_BACKWARD_QUICK] = "JUMP_BACKWARD_QUICK", [GET_ITER] = "GET_ITER", [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", [PRINT_EXPR] = "PRINT_EXPR", [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS", - [LOAD_ATTR_ADAPTIVE] = "LOAD_ATTR_ADAPTIVE", - [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", + [FOR_ITER_ADAPTIVE] = "FOR_ITER_ADAPTIVE", + [FOR_ITER_LIST] = "FOR_ITER_LIST", [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR", [RETURN_GENERATOR] = "RETURN_GENERATOR", + [FOR_ITER_RANGE] = "FOR_ITER_RANGE", + [JUMP_BACKWARD_QUICK] = "JUMP_BACKWARD_QUICK", + [LOAD_ATTR_ADAPTIVE] = "LOAD_ATTR_ADAPTIVE", + [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", - [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", - [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", - [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", - [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", [LIST_TO_TUPLE] = "LIST_TO_TUPLE", [RETURN_VALUE] = "RETURN_VALUE", [IMPORT_STAR] = "IMPORT_STAR", [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", - [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", + [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", [ASYNC_GEN_WRAP] = "ASYNC_GEN_WRAP", [PREP_RERAISE_STAR] = "PREP_RERAISE_STAR", [POP_EXCEPT] = "POP_EXCEPT", @@ -353,7 +358,7 @@ static const char *const _PyOpcode_OpName[263] = { [JUMP_FORWARD] = "JUMP_FORWARD", [JUMP_IF_FALSE_OR_POP] = "JUMP_IF_FALSE_OR_POP", [JUMP_IF_TRUE_OR_POP] = "JUMP_IF_TRUE_OR_POP", - [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", + [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE", [POP_JUMP_IF_TRUE] = "POP_JUMP_IF_TRUE", [LOAD_GLOBAL] = "LOAD_GLOBAL", @@ -361,7 +366,7 @@ static const char *const _PyOpcode_OpName[263] = { [CONTAINS_OP] = "CONTAINS_OP", [RERAISE] = "RERAISE", [COPY] = "COPY", - [LOAD_ATTR_METHOD_WITH_DICT] = "LOAD_ATTR_METHOD_WITH_DICT", + [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", [BINARY_OP] = "BINARY_OP", [SEND] = "SEND", [LOAD_FAST] = "LOAD_FAST", @@ -381,9 +386,9 @@ static const char *const _PyOpcode_OpName[263] = { [STORE_DEREF] = "STORE_DEREF", [DELETE_DEREF] = "DELETE_DEREF", [JUMP_BACKWARD] = "JUMP_BACKWARD", - [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", + [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX", - [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST", + [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", [EXTENDED_ARG] = "EXTENDED_ARG", [LIST_APPEND] = "LIST_APPEND", [SET_ADD] = "SET_ADD", @@ -393,26 +398,30 @@ static const char *const _PyOpcode_OpName[263] = { [YIELD_VALUE] = "YIELD_VALUE", [RESUME] = "RESUME", [MATCH_CLASS] = "MATCH_CLASS", - [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", - [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST", + [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", + [LOAD_ATTR_METHOD_WITH_DICT] = "LOAD_ATTR_METHOD_WITH_DICT", [FORMAT_VALUE] = "FORMAT_VALUE", [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", [BUILD_STRING] = "BUILD_STRING", - [LOAD_GLOBAL_ADAPTIVE] = "LOAD_GLOBAL_ADAPTIVE", - [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", - [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", - [RESUME_QUICK] = "RESUME_QUICK", + [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", + [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST", + [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", + [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST", [LIST_EXTEND] = "LIST_EXTEND", [SET_UPDATE] = "SET_UPDATE", [DICT_MERGE] = "DICT_MERGE", [DICT_UPDATE] = "DICT_UPDATE", + [LOAD_GLOBAL_ADAPTIVE] = "LOAD_GLOBAL_ADAPTIVE", + [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", + [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", + [RESUME_QUICK] = "RESUME_QUICK", [STORE_ATTR_ADAPTIVE] = "STORE_ATTR_ADAPTIVE", + [CALL] = "CALL", + [KW_NAMES] = "KW_NAMES", [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST", - [CALL] = "CALL", - [KW_NAMES] = "KW_NAMES", [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", [STORE_SUBSCR_ADAPTIVE] = "STORE_SUBSCR_ADAPTIVE", [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", @@ -421,10 +430,6 @@ static const char *const _PyOpcode_OpName[263] = { [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", - [181] = "<181>", - [182] = "<182>", - [183] = "<183>", - [184] = "<184>", [185] = "<185>", [186] = "<186>", [187] = "<187>", @@ -507,10 +512,6 @@ static const char *const _PyOpcode_OpName[263] = { #endif #define EXTRA_CASES \ - case 181: \ - case 182: \ - case 183: \ - case 184: \ case 185: \ case 186: \ case 187: \ diff --git a/Include/opcode.h b/Include/opcode.h index 42825df6217b46..50958474c5acef 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -158,47 +158,51 @@ extern "C" { #define CALL_NO_KW_STR_1 47 #define CALL_NO_KW_TUPLE_1 48 #define CALL_NO_KW_TYPE_1 56 -#define COMPARE_OP_ADAPTIVE 57 -#define COMPARE_OP_FLOAT_JUMP 58 -#define COMPARE_OP_INT_JUMP 59 -#define COMPARE_OP_STR_JUMP 62 -#define EXTENDED_ARG_QUICK 63 -#define FOR_ITER_ADAPTIVE 64 -#define FOR_ITER_LIST 65 -#define FOR_ITER_RANGE 66 -#define JUMP_BACKWARD_QUICK 67 -#define LOAD_ATTR_ADAPTIVE 72 -#define LOAD_ATTR_CLASS 73 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 76 -#define LOAD_ATTR_INSTANCE_VALUE 77 -#define LOAD_ATTR_MODULE 78 -#define LOAD_ATTR_PROPERTY 79 -#define LOAD_ATTR_SLOT 80 -#define LOAD_ATTR_WITH_HINT 81 -#define LOAD_ATTR_METHOD_LAZY_DICT 86 -#define LOAD_ATTR_METHOD_NO_DICT 113 -#define LOAD_ATTR_METHOD_WITH_DICT 121 -#define LOAD_ATTR_METHOD_WITH_VALUES 141 -#define LOAD_CONST__LOAD_FAST 143 -#define LOAD_FAST__LOAD_CONST 153 -#define LOAD_FAST__LOAD_FAST 154 -#define LOAD_GLOBAL_ADAPTIVE 158 -#define LOAD_GLOBAL_BUILTIN 159 -#define LOAD_GLOBAL_MODULE 160 -#define RESUME_QUICK 161 -#define STORE_ATTR_ADAPTIVE 166 -#define STORE_ATTR_INSTANCE_VALUE 167 -#define STORE_ATTR_SLOT 168 -#define STORE_ATTR_WITH_HINT 169 -#define STORE_FAST__LOAD_FAST 170 -#define STORE_FAST__STORE_FAST 173 -#define STORE_SUBSCR_ADAPTIVE 174 -#define STORE_SUBSCR_DICT 175 -#define STORE_SUBSCR_LIST_INT 176 -#define UNPACK_SEQUENCE_ADAPTIVE 177 -#define UNPACK_SEQUENCE_LIST 178 -#define UNPACK_SEQUENCE_TUPLE 179 -#define UNPACK_SEQUENCE_TWO_TUPLE 180 +#define CALL_FUNCTION_EX_ADAPTIVE 57 +#define CALL_FUNCTION_EX_PY_KWARGS 58 +#define CALL_FUNCTION_EX_PY_NO_KWARGS 59 +#define CALL_FUNCTION_EX_BUILTIN_PYCFUNCTIONWITHKEYWORDS 62 +#define COMPARE_OP_ADAPTIVE 63 +#define COMPARE_OP_FLOAT_JUMP 64 +#define COMPARE_OP_INT_JUMP 65 +#define COMPARE_OP_STR_JUMP 66 +#define EXTENDED_ARG_QUICK 67 +#define FOR_ITER_ADAPTIVE 72 +#define FOR_ITER_LIST 73 +#define FOR_ITER_RANGE 76 +#define JUMP_BACKWARD_QUICK 77 +#define LOAD_ATTR_ADAPTIVE 78 +#define LOAD_ATTR_CLASS 79 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 80 +#define LOAD_ATTR_INSTANCE_VALUE 81 +#define LOAD_ATTR_MODULE 86 +#define LOAD_ATTR_PROPERTY 113 +#define LOAD_ATTR_SLOT 121 +#define LOAD_ATTR_WITH_HINT 141 +#define LOAD_ATTR_METHOD_LAZY_DICT 143 +#define LOAD_ATTR_METHOD_NO_DICT 153 +#define LOAD_ATTR_METHOD_WITH_DICT 154 +#define LOAD_ATTR_METHOD_WITH_VALUES 158 +#define LOAD_CONST__LOAD_FAST 159 +#define LOAD_FAST__LOAD_CONST 160 +#define LOAD_FAST__LOAD_FAST 161 +#define LOAD_GLOBAL_ADAPTIVE 166 +#define LOAD_GLOBAL_BUILTIN 167 +#define LOAD_GLOBAL_MODULE 168 +#define RESUME_QUICK 169 +#define STORE_ATTR_ADAPTIVE 170 +#define STORE_ATTR_INSTANCE_VALUE 173 +#define STORE_ATTR_SLOT 174 +#define STORE_ATTR_WITH_HINT 175 +#define STORE_FAST__LOAD_FAST 176 +#define STORE_FAST__STORE_FAST 177 +#define STORE_SUBSCR_ADAPTIVE 178 +#define STORE_SUBSCR_DICT 179 +#define STORE_SUBSCR_LIST_INT 180 +#define UNPACK_SEQUENCE_ADAPTIVE 181 +#define UNPACK_SEQUENCE_LIST 182 +#define UNPACK_SEQUENCE_TUPLE 183 +#define UNPACK_SEQUENCE_TWO_TUPLE 184 #define DO_TRACING 255 #define HAS_ARG(op) ((((op) >= HAVE_ARGUMENT) && (!IS_PSEUDO_OPCODE(op)))\ diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index b3c31b9659d849..4db6f3faf7857d 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -413,6 +413,8 @@ def _write_atomic(path, data, mode=0o666): # Python 3.12a1 3507 (Set lineno of module's RESUME to 0) # Python 3.12a1 3508 (Add CLEANUP_THROW) # Python 3.12a1 3509 (Conditional jumps only jump forward) +# Python 3.12a1 3510 (Specialization/Cache for CALL_FUNCTION_EX) + # Python 3.13 will start with 3550 @@ -425,7 +427,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3509).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3510).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c diff --git a/Lib/opcode.py b/Lib/opcode.py index 690923061418bd..bd48c594851084 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -311,6 +311,12 @@ def pseudo_op(name, op, real_ops): "CALL_NO_KW_TUPLE_1", "CALL_NO_KW_TYPE_1", ], + "CALL_FUNCTION_EX": [ + "CALL_FUNCTION_EX_ADAPTIVE", + "CALL_FUNCTION_EX_PY_KWARGS", + "CALL_FUNCTION_EX_PY_NO_KWARGS", + "CALL_FUNCTION_EX_BUILTIN_PYCFUNCTIONWITHKEYWORDS", + ], "COMPARE_OP": [ "COMPARE_OP_ADAPTIVE", "COMPARE_OP_FLOAT_JUMP", @@ -434,6 +440,11 @@ def pseudo_op(name, op, real_ops): "func_version": 2, "min_args": 1, }, + "CALL_FUNCTION_EX": { + "counter": 1, + "func_version": 2, + "min_args": 1, + }, "STORE_SUBSCR": { "counter": 1, }, diff --git a/Objects/call.c b/Objects/call.c index c2509db2a9a263..96a5b07d646a9f 100644 --- a/Objects/call.c +++ b/Objects/call.c @@ -7,16 +7,6 @@ #include "pycore_tuple.h" // _PyTuple_ITEMS() -static PyObject *const * -_PyStack_UnpackDict(PyThreadState *tstate, - PyObject *const *args, Py_ssize_t nargs, - PyObject *kwargs, PyObject **p_kwnames); - -static void -_PyStack_UnpackDict_Free(PyObject *const *stack, Py_ssize_t nargs, - PyObject *kwnames); - - static PyObject * null_error(PyThreadState *tstate) { @@ -964,7 +954,7 @@ _PyStack_AsDict(PyObject *const *values, PyObject *kwnames) The newly allocated argument vector supports PY_VECTORCALL_ARGUMENTS_OFFSET. When done, you must call _PyStack_UnpackDict_Free(stack, nargs, kwnames) */ -static PyObject *const * +PyObject *const * _PyStack_UnpackDict(PyThreadState *tstate, PyObject *const *args, Py_ssize_t nargs, PyObject *kwargs, PyObject **p_kwnames) @@ -1036,7 +1026,7 @@ _PyStack_UnpackDict(PyThreadState *tstate, return stack; } -static void +void _PyStack_UnpackDict_Free(PyObject *const *stack, Py_ssize_t nargs, PyObject *kwnames) { @@ -1044,6 +1034,12 @@ _PyStack_UnpackDict_Free(PyObject *const *stack, Py_ssize_t nargs, for (Py_ssize_t i = 0; i < n; i++) { Py_DECREF(stack[i]); } + _PyStack_UnpackDict_FreeNoDecRef(stack, kwnames); +} + +void +_PyStack_UnpackDict_FreeNoDecRef(PyObject *const *stack, PyObject *kwnames) +{ PyMem_Free((PyObject **)stack - 1); Py_DECREF(kwnames); } diff --git a/Python/ceval.c b/Python/ceval.c index 82b5422c188ea7..b0102b7cd4a7c9 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -25,6 +25,7 @@ #include "pycore_sysmodule.h" // _PySys_Audit() #include "pycore_tuple.h" // _PyTuple_ITEMS() #include "pycore_emscripten_signal.h" // _Py_CHECK_EMSCRIPTEN_SIGNALS +#include "pycore_call.h" // _PyStack_UnpackDict/_Free #include "pycore_dict.h" #include "dictobject.h" @@ -4751,6 +4752,165 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int if (result == NULL) { goto error; } + JUMPBY(INLINE_CACHE_ENTRIES_CALL); + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(CALL_FUNCTION_EX_ADAPTIVE) { + _PyCallCache *cache = (_PyCallCache *)next_instr; + PyObject *func, *callargs, *kwargs = NULL; + if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { + next_instr--; + if (oparg & 0x01) { + kwargs = TOP(); + // DICT_MERGE is called before this opcode if there are kwargs. + // It converts all dict subtypes in kwargs into regular dicts. + assert(PyDict_CheckExact(kwargs)); + callargs = SECOND(); + func = THIRD(); + } + else { + callargs = TOP(); + func = SECOND(); + } + + int err = _Py_Specialize_CallEx(func, next_instr, callargs, kwargs); + if (err < 0) { + goto error; + } + DISPATCH_SAME_OPARG(); + } + else { + STAT_INC(CALL_FUNCTION_EX, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache); + JUMP_TO_INSTRUCTION(CALL_FUNCTION_EX); + } + } + + TARGET(CALL_FUNCTION_EX_PY_KWARGS) { + assert(oparg & 0x01); + PyObject *func, *callargs, *kwargs = NULL; + kwargs = TOP(); + callargs = SECOND(); + func = THIRD(); + DEOPT_IF(!PyTuple_CheckExact(callargs), CALL_FUNCTION_EX); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + DEOPT_IF(kwargs == NULL || PyDict_GET_SIZE(kwargs) <= 0, CALL_FUNCTION_EX); + DEOPT_IF(!Py_IS_TYPE(func, &PyFunction_Type), CALL_FUNCTION_EX); + DEOPT_IF(((PyDictObject *)kwargs)->ma_keys->dk_kind != DICT_KEYS_UNICODE, CALL_FUNCTION_EX); + STAT_INC(CALL_FUNCTION_EX, hit); + STACK_SHRINK(4); /* Get rid of kwargs, callargs, func and NULL */ + + PyObject *kwnames; + PyObject *const *newargs = _PyStack_UnpackDict(tstate, + _PyTuple_ITEMS(callargs), nargs, + kwargs, &kwnames); + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + PyCodeObject *code = (PyCodeObject *)((PyFunctionObject *)func)->func_code; + CALL_STAT_INC(frames_pushed); + + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)func, locals, + newargs, nargs, kwnames + ); + Py_DECREF(kwargs); + Py_DECREF(callargs); + _PyStack_UnpackDict_FreeNoDecRef(newargs, kwnames); + if (new_frame == NULL) { + goto error; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + JUMPBY(INLINE_CACHE_ENTRIES_CALL); + frame->prev_instr = next_instr - 1; + new_frame->previous = frame; + cframe.current_frame = frame = new_frame; + CALL_STAT_INC(inlined_py_calls); + goto start_frame; + } + + TARGET(CALL_FUNCTION_EX_PY_NO_KWARGS) { + assert(!(oparg & 0x01)); + PyObject *func, *callargs; + callargs = TOP(); + func = SECOND(); + DEOPT_IF(!PyTuple_CheckExact(callargs), CALL_FUNCTION_EX); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + DEOPT_IF(!Py_IS_TYPE(func, &PyFunction_Type), CALL_FUNCTION_EX); + STACK_SHRINK(3); + STAT_INC(CALL_FUNCTION_EX, hit); + + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + PyCodeObject *code = (PyCodeObject *)((PyFunctionObject *)func)->func_code; + CALL_STAT_INC(frames_pushed); + + for (Py_ssize_t i = 0; i < nargs; ++i) { + /* We need to incref all our args since the new frame steals the references. */ + Py_INCREF(PyTuple_GET_ITEM(callargs, i)); + } + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)func, locals, + &PyTuple_GET_ITEM(callargs, 0), nargs, NULL + ); + Py_DECREF(callargs); + if (new_frame == NULL) { + goto error; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + JUMPBY(INLINE_CACHE_ENTRIES_CALL); + frame->prev_instr = next_instr - 1; + new_frame->previous = frame; + cframe.current_frame = frame = new_frame; + CALL_STAT_INC(inlined_py_calls); + goto start_frame; + } + + TARGET(CALL_FUNCTION_EX_BUILTIN_PYCFUNCTIONWITHKEYWORDS) { + PyObject *func, *callargs, *kwargs = NULL; + int has_kwargs = oparg & 0x01; + if (has_kwargs) { + kwargs = TOP(); + callargs = SECOND(); + func = THIRD(); + } + else { + callargs = TOP(); + func = SECOND(); + } + DEOPT_IF(!PyTuple_CheckExact(callargs), CALL_FUNCTION_EX); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + DEOPT_IF(!PyCFunction_CheckExact(func), CALL_FUNCTION_EX); + DEOPT_IF(PyCFunction_GET_FLAGS(func) != (METH_VARARGS | METH_KEYWORDS), CALL); + DEOPT_IF(kwargs != NULL && + ((PyDictObject *)kwargs)->ma_keys->dk_kind != DICT_KEYS_UNICODE, CALL_FUNCTION_EX); + STAT_INC(CALL_FUNCTION_EX, hit); + STACK_SHRINK(2 + has_kwargs); /* Get rid of kwargs, callargs, func */ + + // This is slower but CPython promises to check all non-vectorcall + // function calls. + if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { + goto error; + } + + /* res = func(self, args, nargs, kwnames) */ + PyCFunction cfunc = PyCFunction_GET_FUNCTION(func); + PyObject *res = ((PyCFunctionWithKeywords)(void(*)(void))cfunc)( + PyCFunction_GET_SELF(func), + callargs, + (kwargs == NULL || PyDict_GET_SIZE(kwargs) == 0) ? NULL : kwargs + ); + + Py_XDECREF(kwargs); + Py_DECREF(callargs); + _Py_LeaveRecursiveCallTstate(tstate); + assert(TOP() == NULL); + SET_TOP(res); + if (res == NULL) { + goto error; + } + JUMPBY(INLINE_CACHE_ENTRIES_CALL); CHECK_EVAL_BREAKER(); DISPATCH(); } diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index c1ff367d4fd38d..32cbf25a9929bd 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -56,36 +56,36 @@ static void *opcode_targets[256] = { &&TARGET_END_ASYNC_FOR, &&TARGET_CLEANUP_THROW, &&TARGET_CALL_NO_KW_TYPE_1, + &&TARGET_CALL_FUNCTION_EX_ADAPTIVE, + &&TARGET_CALL_FUNCTION_EX_PY_KWARGS, + &&TARGET_CALL_FUNCTION_EX_PY_NO_KWARGS, + &&TARGET_STORE_SUBSCR, + &&TARGET_DELETE_SUBSCR, + &&TARGET_CALL_FUNCTION_EX_BUILTIN_PYCFUNCTIONWITHKEYWORDS, &&TARGET_COMPARE_OP_ADAPTIVE, &&TARGET_COMPARE_OP_FLOAT_JUMP, &&TARGET_COMPARE_OP_INT_JUMP, - &&TARGET_STORE_SUBSCR, - &&TARGET_DELETE_SUBSCR, &&TARGET_COMPARE_OP_STR_JUMP, &&TARGET_EXTENDED_ARG_QUICK, - &&TARGET_FOR_ITER_ADAPTIVE, - &&TARGET_FOR_ITER_LIST, - &&TARGET_FOR_ITER_RANGE, - &&TARGET_JUMP_BACKWARD_QUICK, &&TARGET_GET_ITER, &&TARGET_GET_YIELD_FROM_ITER, &&TARGET_PRINT_EXPR, &&TARGET_LOAD_BUILD_CLASS, - &&TARGET_LOAD_ATTR_ADAPTIVE, - &&TARGET_LOAD_ATTR_CLASS, + &&TARGET_FOR_ITER_ADAPTIVE, + &&TARGET_FOR_ITER_LIST, &&TARGET_LOAD_ASSERTION_ERROR, &&TARGET_RETURN_GENERATOR, + &&TARGET_FOR_ITER_RANGE, + &&TARGET_JUMP_BACKWARD_QUICK, + &&TARGET_LOAD_ATTR_ADAPTIVE, + &&TARGET_LOAD_ATTR_CLASS, &&TARGET_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, &&TARGET_LOAD_ATTR_INSTANCE_VALUE, - &&TARGET_LOAD_ATTR_MODULE, - &&TARGET_LOAD_ATTR_PROPERTY, - &&TARGET_LOAD_ATTR_SLOT, - &&TARGET_LOAD_ATTR_WITH_HINT, &&TARGET_LIST_TO_TUPLE, &&TARGET_RETURN_VALUE, &&TARGET_IMPORT_STAR, &&TARGET_SETUP_ANNOTATIONS, - &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, + &&TARGET_LOAD_ATTR_MODULE, &&TARGET_ASYNC_GEN_WRAP, &&TARGET_PREP_RERAISE_STAR, &&TARGET_POP_EXCEPT, @@ -112,7 +112,7 @@ static void *opcode_targets[256] = { &&TARGET_JUMP_FORWARD, &&TARGET_JUMP_IF_FALSE_OR_POP, &&TARGET_JUMP_IF_TRUE_OR_POP, - &&TARGET_LOAD_ATTR_METHOD_NO_DICT, + &&TARGET_LOAD_ATTR_PROPERTY, &&TARGET_POP_JUMP_IF_FALSE, &&TARGET_POP_JUMP_IF_TRUE, &&TARGET_LOAD_GLOBAL, @@ -120,7 +120,7 @@ static void *opcode_targets[256] = { &&TARGET_CONTAINS_OP, &&TARGET_RERAISE, &&TARGET_COPY, - &&TARGET_LOAD_ATTR_METHOD_WITH_DICT, + &&TARGET_LOAD_ATTR_SLOT, &&TARGET_BINARY_OP, &&TARGET_SEND, &&TARGET_LOAD_FAST, @@ -140,9 +140,9 @@ static void *opcode_targets[256] = { &&TARGET_STORE_DEREF, &&TARGET_DELETE_DEREF, &&TARGET_JUMP_BACKWARD, - &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, + &&TARGET_LOAD_ATTR_WITH_HINT, &&TARGET_CALL_FUNCTION_EX, - &&TARGET_LOAD_CONST__LOAD_FAST, + &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, &&TARGET_EXTENDED_ARG, &&TARGET_LIST_APPEND, &&TARGET_SET_ADD, @@ -152,26 +152,30 @@ static void *opcode_targets[256] = { &&TARGET_YIELD_VALUE, &&TARGET_RESUME, &&TARGET_MATCH_CLASS, - &&TARGET_LOAD_FAST__LOAD_CONST, - &&TARGET_LOAD_FAST__LOAD_FAST, + &&TARGET_LOAD_ATTR_METHOD_NO_DICT, + &&TARGET_LOAD_ATTR_METHOD_WITH_DICT, &&TARGET_FORMAT_VALUE, &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_STRING, - &&TARGET_LOAD_GLOBAL_ADAPTIVE, - &&TARGET_LOAD_GLOBAL_BUILTIN, - &&TARGET_LOAD_GLOBAL_MODULE, - &&TARGET_RESUME_QUICK, + &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, + &&TARGET_LOAD_CONST__LOAD_FAST, + &&TARGET_LOAD_FAST__LOAD_CONST, + &&TARGET_LOAD_FAST__LOAD_FAST, &&TARGET_LIST_EXTEND, &&TARGET_SET_UPDATE, &&TARGET_DICT_MERGE, &&TARGET_DICT_UPDATE, + &&TARGET_LOAD_GLOBAL_ADAPTIVE, + &&TARGET_LOAD_GLOBAL_BUILTIN, + &&TARGET_LOAD_GLOBAL_MODULE, + &&TARGET_RESUME_QUICK, &&TARGET_STORE_ATTR_ADAPTIVE, + &&TARGET_CALL, + &&TARGET_KW_NAMES, &&TARGET_STORE_ATTR_INSTANCE_VALUE, &&TARGET_STORE_ATTR_SLOT, &&TARGET_STORE_ATTR_WITH_HINT, &&TARGET_STORE_FAST__LOAD_FAST, - &&TARGET_CALL, - &&TARGET_KW_NAMES, &&TARGET_STORE_FAST__STORE_FAST, &&TARGET_STORE_SUBSCR_ADAPTIVE, &&TARGET_STORE_SUBSCR_DICT, @@ -250,9 +254,5 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_DO_TRACING }; diff --git a/Python/specialize.c b/Python/specialize.c index b7c321e4878b98..d012ba4b119df1 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -24,6 +24,7 @@ uint8_t _PyOpcode_Adaptive[256] = { [BINARY_SUBSCR] = BINARY_SUBSCR_ADAPTIVE, [STORE_SUBSCR] = STORE_SUBSCR_ADAPTIVE, [CALL] = CALL_ADAPTIVE, + [CALL_FUNCTION_EX] = CALL_FUNCTION_EX_ADAPTIVE, [STORE_ATTR] = STORE_ATTR_ADAPTIVE, [BINARY_OP] = BINARY_OP_ADAPTIVE, [COMPARE_OP] = COMPARE_OP_ADAPTIVE, @@ -1800,6 +1801,79 @@ _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, return 0; } + +static int +specialize_py_callex(PyObject *callable, _Py_CODEUNIT *instr, PyObject *args, PyObject *kwargs, int oparg) +{ + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + if (kwargs != NULL && PyDict_GET_SIZE(kwargs) > 0) { + assert(oparg & 0x01); + if (((PyDictObject *)kwargs)->ma_keys->dk_kind != DICT_KEYS_UNICODE) { + /* kwargs keys should be unicode */ + SPECIALIZATION_FAIL(CALL_FUNCTION_EX, SPEC_FAIL_EXPECTED_ERROR); + return 1; + } + _Py_SET_OPCODE(*instr, CALL_FUNCTION_EX_PY_KWARGS); + return 0; + } + else { + if (oparg & 0x01) { + /* Bad TOS dict or something */ + SPECIALIZATION_FAIL(CALL_FUNCTION_EX, SPEC_FAIL_EXPECTED_ERROR); + return 1; + } + _Py_SET_OPCODE(*instr, CALL_FUNCTION_EX_PY_NO_KWARGS); + return 0; + } +} + + +int +_Py_Specialize_CallEx(PyObject *callable, _Py_CODEUNIT *instr, PyObject *args, PyObject *kwargs) +{ + int oparg = _Py_OPARG(*instr); + assert(_PyOpcode_Caches[CALL_FUNCTION_EX] == INLINE_CACHE_ENTRIES_CALL); + _PyCallCache *cache = (_PyCallCache *)(instr + 1); + int fail; + if (!PyTuple_CheckExact(args)) { + fail = 1; + goto fail_please; + } + if (Py_IS_TYPE(callable, &PyFunction_Type)) { + fail = specialize_py_callex(callable, instr, args, kwargs, oparg); + } + else if (PyCFunction_CheckExact(callable)) { + PyCFunctionObject *func = (PyCFunctionObject *)callable; + switch (func->m_ml->ml_flags & + (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | + METH_KEYWORDS | METH_METHOD)) { + case METH_VARARGS | METH_KEYWORDS: + _Py_SET_OPCODE(*instr, CALL_FUNCTION_EX_BUILTIN_PYCFUNCTIONWITHKEYWORDS); + break; + default: + fail = 1; + } + } + else { + SPECIALIZATION_FAIL(CALL_FUNCTION_EX, call_fail_kind(callable)); + fail = 1; + } + +fail_please: + if (fail) { + STAT_INC(CALL_FUNCTION_EX, failure); + assert(!PyErr_Occurred()); + cache->counter = adaptive_counter_backoff(cache->counter); + } + else { + STAT_INC(CALL_FUNCTION_EX, success); + assert(!PyErr_Occurred()); + cache->counter = miss_counter_start(); + } + return 0; +} + + #ifdef Py_STATS static int binary_op_fail_kind(int oparg, PyObject *lhs, PyObject *rhs) From d9fbb11db91157cbc9c02823a5d1cfcd0454dde0 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Thu, 6 Oct 2022 23:32:12 +0000 Subject: [PATCH 02/17] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst new file mode 100644 index 00000000000000..a6dc08f6092ba3 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst @@ -0,0 +1 @@ +Specialize complex calls. From e7246663df1b3bfa32e92791b8c081aada995258 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 6 Oct 2022 16:50:13 -0700 Subject: [PATCH 03/17] fix refleak --- Python/ceval.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Python/ceval.c b/Python/ceval.c index b0102b7cd4a7c9..dfdad3501296ad 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -4904,6 +4904,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int Py_XDECREF(kwargs); Py_DECREF(callargs); + Py_DECREF(func); _Py_LeaveRecursiveCallTstate(tstate); assert(TOP() == NULL); SET_TOP(res); From 92fead82dbf0680eeacaa970aed957a892e97d31 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 6 Oct 2022 16:52:38 -0700 Subject: [PATCH 04/17] re-order STACK_SHRINK to fix error case --- Python/ceval.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index dfdad3501296ad..9e4b3746e33cbb 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -4838,7 +4838,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int DEOPT_IF(!PyTuple_CheckExact(callargs), CALL_FUNCTION_EX); Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); DEOPT_IF(!Py_IS_TYPE(func, &PyFunction_Type), CALL_FUNCTION_EX); - STACK_SHRINK(3); + STACK_SHRINK(3); /* Get rid of callargs, func and NULL */ STAT_INC(CALL_FUNCTION_EX, hit); int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; @@ -4886,7 +4886,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int DEOPT_IF(kwargs != NULL && ((PyDictObject *)kwargs)->ma_keys->dk_kind != DICT_KEYS_UNICODE, CALL_FUNCTION_EX); STAT_INC(CALL_FUNCTION_EX, hit); - STACK_SHRINK(2 + has_kwargs); /* Get rid of kwargs, callargs, func */ // This is slower but CPython promises to check all non-vectorcall // function calls. @@ -4894,6 +4893,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int goto error; } + STACK_SHRINK(2 + has_kwargs); /* Get rid of kwargs, callargs, func */ /* res = func(self, args, nargs, kwnames) */ PyCFunction cfunc = PyCFunction_GET_FUNCTION(func); PyObject *res = ((PyCFunctionWithKeywords)(void(*)(void))cfunc)( From 84d0f8b737cca775a08bfcff2455ab8debae59fa Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 7 Oct 2022 15:04:47 -0700 Subject: [PATCH 05/17] Inline instead of specializing --- Include/internal/pycore_opcode.h | 67 +++--- Include/opcode.h | 86 ++++---- Lib/importlib/_bootstrap_external.py | 3 +- Lib/opcode.py | 11 - ...2-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst | 3 +- Python/ceval.c | 205 ++++-------------- Python/opcode_targets.h | 58 ++--- Python/specialize.c | 74 ------- 8 files changed, 152 insertions(+), 355 deletions(-) diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index 88feb7d693914b..15925511cc1f41 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -50,7 +50,6 @@ const uint8_t _PyOpcode_Caches[256] = { [COMPARE_OP] = 2, [LOAD_GLOBAL] = 5, [BINARY_OP] = 1, - [CALL_FUNCTION_EX] = 4, [CALL] = 4, }; @@ -89,10 +88,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [CALL_BUILTIN_CLASS] = CALL, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = CALL, [CALL_FUNCTION_EX] = CALL_FUNCTION_EX, - [CALL_FUNCTION_EX_ADAPTIVE] = CALL_FUNCTION_EX, - [CALL_FUNCTION_EX_BUILTIN_PYCFUNCTIONWITHKEYWORDS] = CALL_FUNCTION_EX, - [CALL_FUNCTION_EX_PY_KWARGS] = CALL_FUNCTION_EX, - [CALL_FUNCTION_EX_PY_NO_KWARGS] = CALL_FUNCTION_EX, [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = CALL, [CALL_NO_KW_BUILTIN_FAST] = CALL, [CALL_NO_KW_BUILTIN_O] = CALL, @@ -302,36 +297,36 @@ static const char *const _PyOpcode_OpName[263] = { [END_ASYNC_FOR] = "END_ASYNC_FOR", [CLEANUP_THROW] = "CLEANUP_THROW", [CALL_NO_KW_TYPE_1] = "CALL_NO_KW_TYPE_1", - [CALL_FUNCTION_EX_ADAPTIVE] = "CALL_FUNCTION_EX_ADAPTIVE", - [CALL_FUNCTION_EX_PY_KWARGS] = "CALL_FUNCTION_EX_PY_KWARGS", - [CALL_FUNCTION_EX_PY_NO_KWARGS] = "CALL_FUNCTION_EX_PY_NO_KWARGS", - [STORE_SUBSCR] = "STORE_SUBSCR", - [DELETE_SUBSCR] = "DELETE_SUBSCR", - [CALL_FUNCTION_EX_BUILTIN_PYCFUNCTIONWITHKEYWORDS] = "CALL_FUNCTION_EX_BUILTIN_PYCFUNCTIONWITHKEYWORDS", [COMPARE_OP_ADAPTIVE] = "COMPARE_OP_ADAPTIVE", [COMPARE_OP_FLOAT_JUMP] = "COMPARE_OP_FLOAT_JUMP", [COMPARE_OP_INT_JUMP] = "COMPARE_OP_INT_JUMP", + [STORE_SUBSCR] = "STORE_SUBSCR", + [DELETE_SUBSCR] = "DELETE_SUBSCR", [COMPARE_OP_STR_JUMP] = "COMPARE_OP_STR_JUMP", [EXTENDED_ARG_QUICK] = "EXTENDED_ARG_QUICK", - [GET_ITER] = "GET_ITER", - [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", - [PRINT_EXPR] = "PRINT_EXPR", - [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS", [FOR_ITER_ADAPTIVE] = "FOR_ITER_ADAPTIVE", [FOR_ITER_LIST] = "FOR_ITER_LIST", - [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR", - [RETURN_GENERATOR] = "RETURN_GENERATOR", [FOR_ITER_RANGE] = "FOR_ITER_RANGE", [JUMP_BACKWARD_QUICK] = "JUMP_BACKWARD_QUICK", + [GET_ITER] = "GET_ITER", + [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", + [PRINT_EXPR] = "PRINT_EXPR", + [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS", [LOAD_ATTR_ADAPTIVE] = "LOAD_ATTR_ADAPTIVE", [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", + [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR", + [RETURN_GENERATOR] = "RETURN_GENERATOR", [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", + [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", + [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", + [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", + [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", [LIST_TO_TUPLE] = "LIST_TO_TUPLE", [RETURN_VALUE] = "RETURN_VALUE", [IMPORT_STAR] = "IMPORT_STAR", [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", - [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", + [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", [ASYNC_GEN_WRAP] = "ASYNC_GEN_WRAP", [PREP_RERAISE_STAR] = "PREP_RERAISE_STAR", [POP_EXCEPT] = "POP_EXCEPT", @@ -358,7 +353,7 @@ static const char *const _PyOpcode_OpName[263] = { [JUMP_FORWARD] = "JUMP_FORWARD", [JUMP_IF_FALSE_OR_POP] = "JUMP_IF_FALSE_OR_POP", [JUMP_IF_TRUE_OR_POP] = "JUMP_IF_TRUE_OR_POP", - [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", + [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE", [POP_JUMP_IF_TRUE] = "POP_JUMP_IF_TRUE", [LOAD_GLOBAL] = "LOAD_GLOBAL", @@ -366,7 +361,7 @@ static const char *const _PyOpcode_OpName[263] = { [CONTAINS_OP] = "CONTAINS_OP", [RERAISE] = "RERAISE", [COPY] = "COPY", - [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", + [LOAD_ATTR_METHOD_WITH_DICT] = "LOAD_ATTR_METHOD_WITH_DICT", [BINARY_OP] = "BINARY_OP", [SEND] = "SEND", [LOAD_FAST] = "LOAD_FAST", @@ -386,9 +381,9 @@ static const char *const _PyOpcode_OpName[263] = { [STORE_DEREF] = "STORE_DEREF", [DELETE_DEREF] = "DELETE_DEREF", [JUMP_BACKWARD] = "JUMP_BACKWARD", - [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", + [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX", - [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", + [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST", [EXTENDED_ARG] = "EXTENDED_ARG", [LIST_APPEND] = "LIST_APPEND", [SET_ADD] = "SET_ADD", @@ -398,30 +393,26 @@ static const char *const _PyOpcode_OpName[263] = { [YIELD_VALUE] = "YIELD_VALUE", [RESUME] = "RESUME", [MATCH_CLASS] = "MATCH_CLASS", - [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", - [LOAD_ATTR_METHOD_WITH_DICT] = "LOAD_ATTR_METHOD_WITH_DICT", + [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", + [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST", [FORMAT_VALUE] = "FORMAT_VALUE", [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", [BUILD_STRING] = "BUILD_STRING", - [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", - [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST", - [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", - [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST", - [LIST_EXTEND] = "LIST_EXTEND", - [SET_UPDATE] = "SET_UPDATE", - [DICT_MERGE] = "DICT_MERGE", - [DICT_UPDATE] = "DICT_UPDATE", [LOAD_GLOBAL_ADAPTIVE] = "LOAD_GLOBAL_ADAPTIVE", [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", [RESUME_QUICK] = "RESUME_QUICK", + [LIST_EXTEND] = "LIST_EXTEND", + [SET_UPDATE] = "SET_UPDATE", + [DICT_MERGE] = "DICT_MERGE", + [DICT_UPDATE] = "DICT_UPDATE", [STORE_ATTR_ADAPTIVE] = "STORE_ATTR_ADAPTIVE", - [CALL] = "CALL", - [KW_NAMES] = "KW_NAMES", [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST", + [CALL] = "CALL", + [KW_NAMES] = "KW_NAMES", [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", [STORE_SUBSCR_ADAPTIVE] = "STORE_SUBSCR_ADAPTIVE", [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", @@ -430,6 +421,10 @@ static const char *const _PyOpcode_OpName[263] = { [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", + [181] = "<181>", + [182] = "<182>", + [183] = "<183>", + [184] = "<184>", [185] = "<185>", [186] = "<186>", [187] = "<187>", @@ -512,6 +507,10 @@ static const char *const _PyOpcode_OpName[263] = { #endif #define EXTRA_CASES \ + case 181: \ + case 182: \ + case 183: \ + case 184: \ case 185: \ case 186: \ case 187: \ diff --git a/Include/opcode.h b/Include/opcode.h index 50958474c5acef..42825df6217b46 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -158,51 +158,47 @@ extern "C" { #define CALL_NO_KW_STR_1 47 #define CALL_NO_KW_TUPLE_1 48 #define CALL_NO_KW_TYPE_1 56 -#define CALL_FUNCTION_EX_ADAPTIVE 57 -#define CALL_FUNCTION_EX_PY_KWARGS 58 -#define CALL_FUNCTION_EX_PY_NO_KWARGS 59 -#define CALL_FUNCTION_EX_BUILTIN_PYCFUNCTIONWITHKEYWORDS 62 -#define COMPARE_OP_ADAPTIVE 63 -#define COMPARE_OP_FLOAT_JUMP 64 -#define COMPARE_OP_INT_JUMP 65 -#define COMPARE_OP_STR_JUMP 66 -#define EXTENDED_ARG_QUICK 67 -#define FOR_ITER_ADAPTIVE 72 -#define FOR_ITER_LIST 73 -#define FOR_ITER_RANGE 76 -#define JUMP_BACKWARD_QUICK 77 -#define LOAD_ATTR_ADAPTIVE 78 -#define LOAD_ATTR_CLASS 79 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 80 -#define LOAD_ATTR_INSTANCE_VALUE 81 -#define LOAD_ATTR_MODULE 86 -#define LOAD_ATTR_PROPERTY 113 -#define LOAD_ATTR_SLOT 121 -#define LOAD_ATTR_WITH_HINT 141 -#define LOAD_ATTR_METHOD_LAZY_DICT 143 -#define LOAD_ATTR_METHOD_NO_DICT 153 -#define LOAD_ATTR_METHOD_WITH_DICT 154 -#define LOAD_ATTR_METHOD_WITH_VALUES 158 -#define LOAD_CONST__LOAD_FAST 159 -#define LOAD_FAST__LOAD_CONST 160 -#define LOAD_FAST__LOAD_FAST 161 -#define LOAD_GLOBAL_ADAPTIVE 166 -#define LOAD_GLOBAL_BUILTIN 167 -#define LOAD_GLOBAL_MODULE 168 -#define RESUME_QUICK 169 -#define STORE_ATTR_ADAPTIVE 170 -#define STORE_ATTR_INSTANCE_VALUE 173 -#define STORE_ATTR_SLOT 174 -#define STORE_ATTR_WITH_HINT 175 -#define STORE_FAST__LOAD_FAST 176 -#define STORE_FAST__STORE_FAST 177 -#define STORE_SUBSCR_ADAPTIVE 178 -#define STORE_SUBSCR_DICT 179 -#define STORE_SUBSCR_LIST_INT 180 -#define UNPACK_SEQUENCE_ADAPTIVE 181 -#define UNPACK_SEQUENCE_LIST 182 -#define UNPACK_SEQUENCE_TUPLE 183 -#define UNPACK_SEQUENCE_TWO_TUPLE 184 +#define COMPARE_OP_ADAPTIVE 57 +#define COMPARE_OP_FLOAT_JUMP 58 +#define COMPARE_OP_INT_JUMP 59 +#define COMPARE_OP_STR_JUMP 62 +#define EXTENDED_ARG_QUICK 63 +#define FOR_ITER_ADAPTIVE 64 +#define FOR_ITER_LIST 65 +#define FOR_ITER_RANGE 66 +#define JUMP_BACKWARD_QUICK 67 +#define LOAD_ATTR_ADAPTIVE 72 +#define LOAD_ATTR_CLASS 73 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 76 +#define LOAD_ATTR_INSTANCE_VALUE 77 +#define LOAD_ATTR_MODULE 78 +#define LOAD_ATTR_PROPERTY 79 +#define LOAD_ATTR_SLOT 80 +#define LOAD_ATTR_WITH_HINT 81 +#define LOAD_ATTR_METHOD_LAZY_DICT 86 +#define LOAD_ATTR_METHOD_NO_DICT 113 +#define LOAD_ATTR_METHOD_WITH_DICT 121 +#define LOAD_ATTR_METHOD_WITH_VALUES 141 +#define LOAD_CONST__LOAD_FAST 143 +#define LOAD_FAST__LOAD_CONST 153 +#define LOAD_FAST__LOAD_FAST 154 +#define LOAD_GLOBAL_ADAPTIVE 158 +#define LOAD_GLOBAL_BUILTIN 159 +#define LOAD_GLOBAL_MODULE 160 +#define RESUME_QUICK 161 +#define STORE_ATTR_ADAPTIVE 166 +#define STORE_ATTR_INSTANCE_VALUE 167 +#define STORE_ATTR_SLOT 168 +#define STORE_ATTR_WITH_HINT 169 +#define STORE_FAST__LOAD_FAST 170 +#define STORE_FAST__STORE_FAST 173 +#define STORE_SUBSCR_ADAPTIVE 174 +#define STORE_SUBSCR_DICT 175 +#define STORE_SUBSCR_LIST_INT 176 +#define UNPACK_SEQUENCE_ADAPTIVE 177 +#define UNPACK_SEQUENCE_LIST 178 +#define UNPACK_SEQUENCE_TUPLE 179 +#define UNPACK_SEQUENCE_TWO_TUPLE 180 #define DO_TRACING 255 #define HAS_ARG(op) ((((op) >= HAVE_ARGUMENT) && (!IS_PSEUDO_OPCODE(op)))\ diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 4db6f3faf7857d..734fe2dce9fb51 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -413,7 +413,6 @@ def _write_atomic(path, data, mode=0o666): # Python 3.12a1 3507 (Set lineno of module's RESUME to 0) # Python 3.12a1 3508 (Add CLEANUP_THROW) # Python 3.12a1 3509 (Conditional jumps only jump forward) -# Python 3.12a1 3510 (Specialization/Cache for CALL_FUNCTION_EX) # Python 3.13 will start with 3550 @@ -427,7 +426,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3510).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3509).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c diff --git a/Lib/opcode.py b/Lib/opcode.py index bd48c594851084..690923061418bd 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -311,12 +311,6 @@ def pseudo_op(name, op, real_ops): "CALL_NO_KW_TUPLE_1", "CALL_NO_KW_TYPE_1", ], - "CALL_FUNCTION_EX": [ - "CALL_FUNCTION_EX_ADAPTIVE", - "CALL_FUNCTION_EX_PY_KWARGS", - "CALL_FUNCTION_EX_PY_NO_KWARGS", - "CALL_FUNCTION_EX_BUILTIN_PYCFUNCTIONWITHKEYWORDS", - ], "COMPARE_OP": [ "COMPARE_OP_ADAPTIVE", "COMPARE_OP_FLOAT_JUMP", @@ -440,11 +434,6 @@ def pseudo_op(name, op, real_ops): "func_version": 2, "min_args": 1, }, - "CALL_FUNCTION_EX": { - "counter": 1, - "func_version": 2, - "min_args": 1, - }, "STORE_SUBSCR": { "counter": 1, }, diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst index a6dc08f6092ba3..c0764d245132e7 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst @@ -1 +1,2 @@ -Specialize complex calls. +Complex function calls are now significantly faster and consume no C stack +space. \ No newline at end of file diff --git a/Python/ceval.c b/Python/ceval.c index 9e4b3746e33cbb..36cb0c993180de 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -4741,177 +4741,64 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int } assert(PyTuple_CheckExact(callargs)); - result = do_call_core(tstate, func, callargs, kwargs, cframe.use_tracing); - Py_DECREF(func); - Py_DECREF(callargs); - Py_XDECREF(kwargs); - - STACK_SHRINK(1); - assert(TOP() == NULL); - SET_TOP(result); - if (result == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); - CHECK_EVAL_BREAKER(); - DISPATCH(); - } - - TARGET(CALL_FUNCTION_EX_ADAPTIVE) { - _PyCallCache *cache = (_PyCallCache *)next_instr; - PyObject *func, *callargs, *kwargs = NULL; - if (ADAPTIVE_COUNTER_IS_ZERO(cache)) { - next_instr--; - if (oparg & 0x01) { - kwargs = TOP(); - // DICT_MERGE is called before this opcode if there are kwargs. - // It converts all dict subtypes in kwargs into regular dicts. - assert(PyDict_CheckExact(kwargs)); - callargs = SECOND(); - func = THIRD(); + if (Py_IS_TYPE(func, &PyFunction_Type) && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + assert(PyTuple_CheckExact(callargs)); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + PyCodeObject *code = (PyCodeObject *)((PyFunctionObject *)func)->func_code; + + bool has_dict = (kwargs != NULL && PyDict_GET_SIZE(kwargs) > 0); + PyObject *kwnames = NULL; + PyObject *const *newargs = has_dict + ? _PyStack_UnpackDict(tstate, _PyTuple_ITEMS(callargs), + nargs, kwargs, &kwnames) + : &PyTuple_GET_ITEM(callargs, 0); + if (newargs == NULL) { + goto error; } - else { - callargs = TOP(); - func = SECOND(); + if (!has_dict) { + /* We need to incref all our args since the new frame steals the references. */ + for (Py_ssize_t i = 0; i < nargs; ++i) { + Py_INCREF(PyTuple_GET_ITEM(callargs, i)); + } } - - int err = _Py_Specialize_CallEx(func, next_instr, callargs, kwargs); - if (err < 0) { + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)func, locals, + newargs, nargs, kwnames + ); + STACK_SHRINK(2); /* get rid of func and NULL */ + Py_DECREF(callargs); + Py_XDECREF(kwargs); + if (has_dict) { + _PyStack_UnpackDict_FreeNoDecRef(newargs, kwnames); + } + if (new_frame == NULL) { goto error; } - DISPATCH_SAME_OPARG(); - } - else { - STAT_INC(CALL_FUNCTION_EX, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache); - JUMP_TO_INSTRUCTION(CALL_FUNCTION_EX); - } - } - - TARGET(CALL_FUNCTION_EX_PY_KWARGS) { - assert(oparg & 0x01); - PyObject *func, *callargs, *kwargs = NULL; - kwargs = TOP(); - callargs = SECOND(); - func = THIRD(); - DEOPT_IF(!PyTuple_CheckExact(callargs), CALL_FUNCTION_EX); - Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); - DEOPT_IF(kwargs == NULL || PyDict_GET_SIZE(kwargs) <= 0, CALL_FUNCTION_EX); - DEOPT_IF(!Py_IS_TYPE(func, &PyFunction_Type), CALL_FUNCTION_EX); - DEOPT_IF(((PyDictObject *)kwargs)->ma_keys->dk_kind != DICT_KEYS_UNICODE, CALL_FUNCTION_EX); - STAT_INC(CALL_FUNCTION_EX, hit); - STACK_SHRINK(4); /* Get rid of kwargs, callargs, func and NULL */ - - PyObject *kwnames; - PyObject *const *newargs = _PyStack_UnpackDict(tstate, - _PyTuple_ITEMS(callargs), nargs, - kwargs, &kwnames); - int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - PyCodeObject *code = (PyCodeObject *)((PyFunctionObject *)func)->func_code; - CALL_STAT_INC(frames_pushed); - - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)func, locals, - newargs, nargs, kwnames - ); - Py_DECREF(kwargs); - Py_DECREF(callargs); - _PyStack_UnpackDict_FreeNoDecRef(newargs, kwnames); - if (new_frame == NULL) { - goto error; - } - _PyFrame_SetStackPointer(frame, stack_pointer); - JUMPBY(INLINE_CACHE_ENTRIES_CALL); - frame->prev_instr = next_instr - 1; - new_frame->previous = frame; - cframe.current_frame = frame = new_frame; - CALL_STAT_INC(inlined_py_calls); - goto start_frame; - } - - TARGET(CALL_FUNCTION_EX_PY_NO_KWARGS) { - assert(!(oparg & 0x01)); - PyObject *func, *callargs; - callargs = TOP(); - func = SECOND(); - DEOPT_IF(!PyTuple_CheckExact(callargs), CALL_FUNCTION_EX); - Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); - DEOPT_IF(!Py_IS_TYPE(func, &PyFunction_Type), CALL_FUNCTION_EX); - STACK_SHRINK(3); /* Get rid of callargs, func and NULL */ - STAT_INC(CALL_FUNCTION_EX, hit); - - int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - PyCodeObject *code = (PyCodeObject *)((PyFunctionObject *)func)->func_code; - CALL_STAT_INC(frames_pushed); - - for (Py_ssize_t i = 0; i < nargs; ++i) { - /* We need to incref all our args since the new frame steals the references. */ - Py_INCREF(PyTuple_GET_ITEM(callargs, i)); - } - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)func, locals, - &PyTuple_GET_ITEM(callargs, 0), nargs, NULL - ); - Py_DECREF(callargs); - if (new_frame == NULL) { - goto error; - } - _PyFrame_SetStackPointer(frame, stack_pointer); - JUMPBY(INLINE_CACHE_ENTRIES_CALL); - frame->prev_instr = next_instr - 1; - new_frame->previous = frame; - cframe.current_frame = frame = new_frame; - CALL_STAT_INC(inlined_py_calls); - goto start_frame; - } - - TARGET(CALL_FUNCTION_EX_BUILTIN_PYCFUNCTIONWITHKEYWORDS) { - PyObject *func, *callargs, *kwargs = NULL; - int has_kwargs = oparg & 0x01; - if (has_kwargs) { - kwargs = TOP(); - callargs = SECOND(); - func = THIRD(); + _PyFrame_SetStackPointer(frame, stack_pointer); + frame->prev_instr = next_instr - 1; + new_frame->previous = frame; + cframe.current_frame = frame = new_frame; + CALL_STAT_INC(inlined_py_calls); + goto start_frame; } else { - callargs = TOP(); - func = SECOND(); + result = do_call_core(tstate, func, callargs, kwargs, cframe.use_tracing); + Py_DECREF(func); + Py_DECREF(callargs); + Py_XDECREF(kwargs); } - DEOPT_IF(!PyTuple_CheckExact(callargs), CALL_FUNCTION_EX); - Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); - DEOPT_IF(!PyCFunction_CheckExact(func), CALL_FUNCTION_EX); - DEOPT_IF(PyCFunction_GET_FLAGS(func) != (METH_VARARGS | METH_KEYWORDS), CALL); - DEOPT_IF(kwargs != NULL && - ((PyDictObject *)kwargs)->ma_keys->dk_kind != DICT_KEYS_UNICODE, CALL_FUNCTION_EX); - STAT_INC(CALL_FUNCTION_EX, hit); - // This is slower but CPython promises to check all non-vectorcall - // function calls. - if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { - goto error; - } - - STACK_SHRINK(2 + has_kwargs); /* Get rid of kwargs, callargs, func */ - /* res = func(self, args, nargs, kwnames) */ - PyCFunction cfunc = PyCFunction_GET_FUNCTION(func); - PyObject *res = ((PyCFunctionWithKeywords)(void(*)(void))cfunc)( - PyCFunction_GET_SELF(func), - callargs, - (kwargs == NULL || PyDict_GET_SIZE(kwargs) == 0) ? NULL : kwargs - ); - Py_XDECREF(kwargs); - Py_DECREF(callargs); - Py_DECREF(func); - _Py_LeaveRecursiveCallTstate(tstate); + STACK_SHRINK(1); assert(TOP() == NULL); - SET_TOP(res); - if (res == NULL) { + SET_TOP(result); + if (result == NULL) { goto error; } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); CHECK_EVAL_BREAKER(); DISPATCH(); } diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 32cbf25a9929bd..c1ff367d4fd38d 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -56,36 +56,36 @@ static void *opcode_targets[256] = { &&TARGET_END_ASYNC_FOR, &&TARGET_CLEANUP_THROW, &&TARGET_CALL_NO_KW_TYPE_1, - &&TARGET_CALL_FUNCTION_EX_ADAPTIVE, - &&TARGET_CALL_FUNCTION_EX_PY_KWARGS, - &&TARGET_CALL_FUNCTION_EX_PY_NO_KWARGS, - &&TARGET_STORE_SUBSCR, - &&TARGET_DELETE_SUBSCR, - &&TARGET_CALL_FUNCTION_EX_BUILTIN_PYCFUNCTIONWITHKEYWORDS, &&TARGET_COMPARE_OP_ADAPTIVE, &&TARGET_COMPARE_OP_FLOAT_JUMP, &&TARGET_COMPARE_OP_INT_JUMP, + &&TARGET_STORE_SUBSCR, + &&TARGET_DELETE_SUBSCR, &&TARGET_COMPARE_OP_STR_JUMP, &&TARGET_EXTENDED_ARG_QUICK, - &&TARGET_GET_ITER, - &&TARGET_GET_YIELD_FROM_ITER, - &&TARGET_PRINT_EXPR, - &&TARGET_LOAD_BUILD_CLASS, &&TARGET_FOR_ITER_ADAPTIVE, &&TARGET_FOR_ITER_LIST, - &&TARGET_LOAD_ASSERTION_ERROR, - &&TARGET_RETURN_GENERATOR, &&TARGET_FOR_ITER_RANGE, &&TARGET_JUMP_BACKWARD_QUICK, + &&TARGET_GET_ITER, + &&TARGET_GET_YIELD_FROM_ITER, + &&TARGET_PRINT_EXPR, + &&TARGET_LOAD_BUILD_CLASS, &&TARGET_LOAD_ATTR_ADAPTIVE, &&TARGET_LOAD_ATTR_CLASS, + &&TARGET_LOAD_ASSERTION_ERROR, + &&TARGET_RETURN_GENERATOR, &&TARGET_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, &&TARGET_LOAD_ATTR_INSTANCE_VALUE, + &&TARGET_LOAD_ATTR_MODULE, + &&TARGET_LOAD_ATTR_PROPERTY, + &&TARGET_LOAD_ATTR_SLOT, + &&TARGET_LOAD_ATTR_WITH_HINT, &&TARGET_LIST_TO_TUPLE, &&TARGET_RETURN_VALUE, &&TARGET_IMPORT_STAR, &&TARGET_SETUP_ANNOTATIONS, - &&TARGET_LOAD_ATTR_MODULE, + &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, &&TARGET_ASYNC_GEN_WRAP, &&TARGET_PREP_RERAISE_STAR, &&TARGET_POP_EXCEPT, @@ -112,7 +112,7 @@ static void *opcode_targets[256] = { &&TARGET_JUMP_FORWARD, &&TARGET_JUMP_IF_FALSE_OR_POP, &&TARGET_JUMP_IF_TRUE_OR_POP, - &&TARGET_LOAD_ATTR_PROPERTY, + &&TARGET_LOAD_ATTR_METHOD_NO_DICT, &&TARGET_POP_JUMP_IF_FALSE, &&TARGET_POP_JUMP_IF_TRUE, &&TARGET_LOAD_GLOBAL, @@ -120,7 +120,7 @@ static void *opcode_targets[256] = { &&TARGET_CONTAINS_OP, &&TARGET_RERAISE, &&TARGET_COPY, - &&TARGET_LOAD_ATTR_SLOT, + &&TARGET_LOAD_ATTR_METHOD_WITH_DICT, &&TARGET_BINARY_OP, &&TARGET_SEND, &&TARGET_LOAD_FAST, @@ -140,9 +140,9 @@ static void *opcode_targets[256] = { &&TARGET_STORE_DEREF, &&TARGET_DELETE_DEREF, &&TARGET_JUMP_BACKWARD, - &&TARGET_LOAD_ATTR_WITH_HINT, + &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, &&TARGET_CALL_FUNCTION_EX, - &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, + &&TARGET_LOAD_CONST__LOAD_FAST, &&TARGET_EXTENDED_ARG, &&TARGET_LIST_APPEND, &&TARGET_SET_ADD, @@ -152,30 +152,26 @@ static void *opcode_targets[256] = { &&TARGET_YIELD_VALUE, &&TARGET_RESUME, &&TARGET_MATCH_CLASS, - &&TARGET_LOAD_ATTR_METHOD_NO_DICT, - &&TARGET_LOAD_ATTR_METHOD_WITH_DICT, + &&TARGET_LOAD_FAST__LOAD_CONST, + &&TARGET_LOAD_FAST__LOAD_FAST, &&TARGET_FORMAT_VALUE, &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_STRING, - &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, - &&TARGET_LOAD_CONST__LOAD_FAST, - &&TARGET_LOAD_FAST__LOAD_CONST, - &&TARGET_LOAD_FAST__LOAD_FAST, - &&TARGET_LIST_EXTEND, - &&TARGET_SET_UPDATE, - &&TARGET_DICT_MERGE, - &&TARGET_DICT_UPDATE, &&TARGET_LOAD_GLOBAL_ADAPTIVE, &&TARGET_LOAD_GLOBAL_BUILTIN, &&TARGET_LOAD_GLOBAL_MODULE, &&TARGET_RESUME_QUICK, + &&TARGET_LIST_EXTEND, + &&TARGET_SET_UPDATE, + &&TARGET_DICT_MERGE, + &&TARGET_DICT_UPDATE, &&TARGET_STORE_ATTR_ADAPTIVE, - &&TARGET_CALL, - &&TARGET_KW_NAMES, &&TARGET_STORE_ATTR_INSTANCE_VALUE, &&TARGET_STORE_ATTR_SLOT, &&TARGET_STORE_ATTR_WITH_HINT, &&TARGET_STORE_FAST__LOAD_FAST, + &&TARGET_CALL, + &&TARGET_KW_NAMES, &&TARGET_STORE_FAST__STORE_FAST, &&TARGET_STORE_SUBSCR_ADAPTIVE, &&TARGET_STORE_SUBSCR_DICT, @@ -254,5 +250,9 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, &&TARGET_DO_TRACING }; diff --git a/Python/specialize.c b/Python/specialize.c index d012ba4b119df1..b7c321e4878b98 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -24,7 +24,6 @@ uint8_t _PyOpcode_Adaptive[256] = { [BINARY_SUBSCR] = BINARY_SUBSCR_ADAPTIVE, [STORE_SUBSCR] = STORE_SUBSCR_ADAPTIVE, [CALL] = CALL_ADAPTIVE, - [CALL_FUNCTION_EX] = CALL_FUNCTION_EX_ADAPTIVE, [STORE_ATTR] = STORE_ATTR_ADAPTIVE, [BINARY_OP] = BINARY_OP_ADAPTIVE, [COMPARE_OP] = COMPARE_OP_ADAPTIVE, @@ -1801,79 +1800,6 @@ _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, return 0; } - -static int -specialize_py_callex(PyObject *callable, _Py_CODEUNIT *instr, PyObject *args, PyObject *kwargs, int oparg) -{ - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - if (kwargs != NULL && PyDict_GET_SIZE(kwargs) > 0) { - assert(oparg & 0x01); - if (((PyDictObject *)kwargs)->ma_keys->dk_kind != DICT_KEYS_UNICODE) { - /* kwargs keys should be unicode */ - SPECIALIZATION_FAIL(CALL_FUNCTION_EX, SPEC_FAIL_EXPECTED_ERROR); - return 1; - } - _Py_SET_OPCODE(*instr, CALL_FUNCTION_EX_PY_KWARGS); - return 0; - } - else { - if (oparg & 0x01) { - /* Bad TOS dict or something */ - SPECIALIZATION_FAIL(CALL_FUNCTION_EX, SPEC_FAIL_EXPECTED_ERROR); - return 1; - } - _Py_SET_OPCODE(*instr, CALL_FUNCTION_EX_PY_NO_KWARGS); - return 0; - } -} - - -int -_Py_Specialize_CallEx(PyObject *callable, _Py_CODEUNIT *instr, PyObject *args, PyObject *kwargs) -{ - int oparg = _Py_OPARG(*instr); - assert(_PyOpcode_Caches[CALL_FUNCTION_EX] == INLINE_CACHE_ENTRIES_CALL); - _PyCallCache *cache = (_PyCallCache *)(instr + 1); - int fail; - if (!PyTuple_CheckExact(args)) { - fail = 1; - goto fail_please; - } - if (Py_IS_TYPE(callable, &PyFunction_Type)) { - fail = specialize_py_callex(callable, instr, args, kwargs, oparg); - } - else if (PyCFunction_CheckExact(callable)) { - PyCFunctionObject *func = (PyCFunctionObject *)callable; - switch (func->m_ml->ml_flags & - (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | - METH_KEYWORDS | METH_METHOD)) { - case METH_VARARGS | METH_KEYWORDS: - _Py_SET_OPCODE(*instr, CALL_FUNCTION_EX_BUILTIN_PYCFUNCTIONWITHKEYWORDS); - break; - default: - fail = 1; - } - } - else { - SPECIALIZATION_FAIL(CALL_FUNCTION_EX, call_fail_kind(callable)); - fail = 1; - } - -fail_please: - if (fail) { - STAT_INC(CALL_FUNCTION_EX, failure); - assert(!PyErr_Occurred()); - cache->counter = adaptive_counter_backoff(cache->counter); - } - else { - STAT_INC(CALL_FUNCTION_EX, success); - assert(!PyErr_Occurred()); - cache->counter = miss_counter_start(); - } - return 0; -} - - #ifdef Py_STATS static int binary_op_fail_kind(int oparg, PyObject *lhs, PyObject *rhs) From 1955f19e0156cf032fe0d8e3c6273998d5c82eb9 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 7 Oct 2022 15:05:27 -0700 Subject: [PATCH 06/17] reduce diff --- Lib/importlib/_bootstrap_external.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 734fe2dce9fb51..b3c31b9659d849 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -414,7 +414,6 @@ def _write_atomic(path, data, mode=0o666): # Python 3.12a1 3508 (Add CLEANUP_THROW) # Python 3.12a1 3509 (Conditional jumps only jump forward) - # Python 3.13 will start with 3550 # MAGIC must change whenever the bytecode emitted by the compiler may no From eb25c1bceedc495cb9dac882d55cd3ec047f7248 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 7 Oct 2022 15:32:54 -0700 Subject: [PATCH 07/17] temper expectations --- .../2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst index c0764d245132e7..efa56f87c8be8c 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst @@ -1,2 +1,2 @@ -Complex function calls are now significantly faster and consume no C stack -space. \ No newline at end of file +Complex function calls are now faster and consume no C stack +space. From 3f409da31683fa7b7d2fe68731a742bd26db3a26 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 7 Oct 2022 16:20:54 -0700 Subject: [PATCH 08/17] Add newline --- .../2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst index efa56f87c8be8c..f9e71bc1344bb3 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst @@ -1,2 +1,3 @@ Complex function calls are now faster and consume no C stack space. + From 09b25281645858127a073f931d9eada81eaedce7 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 11 Oct 2022 11:27:26 +0800 Subject: [PATCH 09/17] remove unused variable --- Python/ceval.c | 1 - 1 file changed, 1 deletion(-) diff --git a/Python/ceval.c b/Python/ceval.c index 36cb0c993180de..08f23e14682b67 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -4748,7 +4748,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - PyCodeObject *code = (PyCodeObject *)((PyFunctionObject *)func)->func_code; bool has_dict = (kwargs != NULL && PyDict_GET_SIZE(kwargs) > 0); PyObject *kwnames = NULL; From f0d77dc3080c9c9914e4fd1e43b8c0ad1eaa3c8a Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 24 Nov 2022 23:52:59 +0800 Subject: [PATCH 10/17] remove unused function prototype --- Include/internal/pycore_code.h | 1 - 1 file changed, 1 deletion(-) diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 29caf4d524c145..bf5945435c1774 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -244,7 +244,6 @@ extern int _Py_Specialize_BinarySubscr(PyObject *sub, PyObject *container, _Py_C extern int _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *instr); extern int _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, PyObject *kwnames); -extern int _Py_Specialize_CallEx(PyObject *callable, _Py_CODEUNIT *instr, PyObject *args, PyObject *kwargs); extern void _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, int oparg, PyObject **locals); extern void _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, From b933f6e0ea4d5eef05bf7b90c8d8f8f8af061c09 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 24 Nov 2022 23:59:02 +0800 Subject: [PATCH 11/17] Use new case regenerator --- Python/bytecodes.c | 53 +++++++++++++++++++++++++++++++++++--- Python/generated_cases.c.h | 53 +++++++++++++++++++++++++++++++++++--- 2 files changed, 98 insertions(+), 8 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index a1f910da8ed54a..d39c529ad37020 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3475,10 +3475,55 @@ dummy_func( } assert(PyTuple_CheckExact(callargs)); - result = do_call_core(tstate, func, callargs, kwargs, cframe.use_tracing); - Py_DECREF(func); - Py_DECREF(callargs); - Py_XDECREF(kwargs); + if (Py_IS_TYPE(func, &PyFunction_Type) && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + assert(PyTuple_CheckExact(callargs)); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + + bool has_dict = (kwargs != NULL && PyDict_GET_SIZE(kwargs) > 0); + PyObject *kwnames = NULL; + PyObject *const *newargs = has_dict + ? _PyStack_UnpackDict(tstate, _PyTuple_ITEMS(callargs), + nargs, kwargs, &kwnames) + : &PyTuple_GET_ITEM(callargs, 0); + if (newargs == NULL) { + goto error; + } + if (!has_dict) { + /* We need to incref all our args since the new frame steals the references. */ + for (Py_ssize_t i = 0; i < nargs; ++i) { + Py_INCREF(PyTuple_GET_ITEM(callargs, i)); + } + } + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)func, locals, + newargs, nargs, kwnames + ); + STACK_SHRINK(2); /* get rid of func and NULL */ + Py_DECREF(callargs); + Py_XDECREF(kwargs); + if (has_dict) { + _PyStack_UnpackDict_FreeNoDecRef(newargs, kwnames); + } + if (new_frame == NULL) { + goto error; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + frame->prev_instr = next_instr - 1; + new_frame->previous = frame; + cframe.current_frame = frame = new_frame; + CALL_STAT_INC(inlined_py_calls); + goto start_frame; + } + else { + result = do_call_core(tstate, func, callargs, kwargs, cframe.use_tracing); + Py_DECREF(func); + Py_DECREF(callargs); + Py_XDECREF(kwargs); + } STACK_SHRINK(1); assert(TOP() == NULL); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index ae8fdd5e99c3dc..9139a716ad8f3f 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3499,10 +3499,55 @@ } assert(PyTuple_CheckExact(callargs)); - result = do_call_core(tstate, func, callargs, kwargs, cframe.use_tracing); - Py_DECREF(func); - Py_DECREF(callargs); - Py_XDECREF(kwargs); + if (Py_IS_TYPE(func, &PyFunction_Type) && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + assert(PyTuple_CheckExact(callargs)); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + + bool has_dict = (kwargs != NULL && PyDict_GET_SIZE(kwargs) > 0); + PyObject *kwnames = NULL; + PyObject *const *newargs = has_dict + ? _PyStack_UnpackDict(tstate, _PyTuple_ITEMS(callargs), + nargs, kwargs, &kwnames) + : &PyTuple_GET_ITEM(callargs, 0); + if (newargs == NULL) { + goto error; + } + if (!has_dict) { + /* We need to incref all our args since the new frame steals the references. */ + for (Py_ssize_t i = 0; i < nargs; ++i) { + Py_INCREF(PyTuple_GET_ITEM(callargs, i)); + } + } + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)func, locals, + newargs, nargs, kwnames + ); + STACK_SHRINK(2); /* get rid of func and NULL */ + Py_DECREF(callargs); + Py_XDECREF(kwargs); + if (has_dict) { + _PyStack_UnpackDict_FreeNoDecRef(newargs, kwnames); + } + if (new_frame == NULL) { + goto error; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + frame->prev_instr = next_instr - 1; + new_frame->previous = frame; + cframe.current_frame = frame = new_frame; + CALL_STAT_INC(inlined_py_calls); + goto start_frame; + } + else { + result = do_call_core(tstate, func, callargs, kwargs, cframe.use_tracing); + Py_DECREF(func); + Py_DECREF(callargs); + Py_XDECREF(kwargs); + } STACK_SHRINK(1); assert(TOP() == NULL); From 06ddc00a8f5063fde60ba0c548c4a3a69a162ec6 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 1 Dec 2022 01:47:32 +0800 Subject: [PATCH 12/17] Apply Mark's and Brandt's suggestion --- .../pycore_global_objects_fini_generated.h | 1 + Include/internal/pycore_global_strings.h | 1 + .../internal/pycore_runtime_init_generated.h | 1 + .../internal/pycore_unicodeobject_generated.h | 2 + Python/bytecodes.c | 34 ++------------ Python/ceval.c | 45 +++++++++++++++++++ Python/generated_cases.c.h | 34 ++------------ 7 files changed, 58 insertions(+), 60 deletions(-) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 494bcf293cdb7b..0919fdd668b858 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -1218,6 +1218,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(x)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(year)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(zdict)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(zipimporter)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(strings).ascii[0]); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(strings).ascii[1]); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(strings).ascii[2]); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index b0cb8365933e77..8abb561c053505 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -704,6 +704,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(x) STRUCT_FOR_ID(year) STRUCT_FOR_ID(zdict) + STRUCT_FOR_ID(zipimporter) } identifiers; struct { PyASCIIObject _ascii; diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 4b128da54555b7..5c28d04b5f5768 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -1210,6 +1210,7 @@ extern "C" { INIT_ID(x), \ INIT_ID(year), \ INIT_ID(zdict), \ + INIT_ID(zipimporter), \ } #define _Py_str_ascii_INIT { \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 7ef1f7e94ddead..68c757a180c73c 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -1314,6 +1314,8 @@ _PyUnicode_InitStaticStrings(void) { PyUnicode_InternInPlace(&string); string = &_Py_ID(zdict); PyUnicode_InternInPlace(&string); + string = &_Py_ID(zipimporter); + PyUnicode_InternInPlace(&string); } /* End auto-generated code */ #ifdef __cplusplus diff --git a/Python/bytecodes.c b/Python/bytecodes.c index d39c529ad37020..7866f24eab20cf 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3483,40 +3483,14 @@ dummy_func( int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - bool has_dict = (kwargs != NULL && PyDict_GET_SIZE(kwargs) > 0); - PyObject *kwnames = NULL; - PyObject *const *newargs = has_dict - ? _PyStack_UnpackDict(tstate, _PyTuple_ITEMS(callargs), - nargs, kwargs, &kwnames) - : &PyTuple_GET_ITEM(callargs, 0); - if (newargs == NULL) { - goto error; - } - if (!has_dict) { - /* We need to incref all our args since the new frame steals the references. */ - for (Py_ssize_t i = 0; i < nargs; ++i) { - Py_INCREF(PyTuple_GET_ITEM(callargs, i)); - } - } - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)func, locals, - newargs, nargs, kwnames - ); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, + (PyFunctionObject *)func, locals, + nargs, callargs, kwargs); STACK_SHRINK(2); /* get rid of func and NULL */ - Py_DECREF(callargs); - Py_XDECREF(kwargs); - if (has_dict) { - _PyStack_UnpackDict_FreeNoDecRef(newargs, kwnames); - } if (new_frame == NULL) { goto error; } - _PyFrame_SetStackPointer(frame, stack_pointer); - frame->prev_instr = next_instr - 1; - new_frame->previous = frame; - cframe.current_frame = frame = new_frame; - CALL_STAT_INC(inlined_py_calls); - goto start_frame; + DISPATCH_INLINED(new_frame); } else { result = do_call_core(tstate, func, callargs, kwargs, cframe.use_tracing); diff --git a/Python/ceval.c b/Python/ceval.c index 80bfa21ad0b6f0..4e13b2f45e1714 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -217,6 +217,9 @@ static _PyInterpreterFrame * _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, PyObject *locals, PyObject* const* args, size_t argcount, PyObject *kwnames); +static _PyInterpreterFrame * +_PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func, + PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs); static void _PyEvalFrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame); @@ -2005,6 +2008,48 @@ _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, return NULL; } +/* Same as _PyEvalFramePushAndInit but takes an args tuple and kwargs dict. + Steals references to func, callargs and kwargs. +*/ +static _PyInterpreterFrame * +_PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func, + PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs) +{ + bool has_dict = (kwargs != NULL && PyDict_GET_SIZE(kwargs) > 0); + PyObject *kwnames = NULL; + PyObject *const *newargs = has_dict + ? _PyStack_UnpackDict(tstate, _PyTuple_ITEMS(callargs), + nargs, kwargs, &kwnames) + : &PyTuple_GET_ITEM(callargs, 0); + if (newargs == NULL) { + Py_DECREF(func); + goto error; + } + if (!has_dict) { + /* We need to incref all our args since the new frame steals the references. */ + for (Py_ssize_t i = 0; i < nargs; ++i) { + Py_INCREF(PyTuple_GET_ITEM(callargs, i)); + } + } + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)func, locals, + newargs, nargs, kwnames + ); + if (has_dict) { + _PyStack_UnpackDict_FreeNoDecRef(newargs, kwnames); + } + /* No need to decref func here because the reference has been stolen by + _PyEvalFramePushAndInit. + */ + Py_DECREF(callargs); + Py_XDECREF(kwargs); + return new_frame; +error: + Py_DECREF(callargs); + Py_XDECREF(kwargs); + return NULL; +} + static void clear_thread_frame(PyThreadState *tstate, _PyInterpreterFrame * frame) { diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 9139a716ad8f3f..8ac5e0ffa0ee73 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3507,40 +3507,14 @@ int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - bool has_dict = (kwargs != NULL && PyDict_GET_SIZE(kwargs) > 0); - PyObject *kwnames = NULL; - PyObject *const *newargs = has_dict - ? _PyStack_UnpackDict(tstate, _PyTuple_ITEMS(callargs), - nargs, kwargs, &kwnames) - : &PyTuple_GET_ITEM(callargs, 0); - if (newargs == NULL) { - goto error; - } - if (!has_dict) { - /* We need to incref all our args since the new frame steals the references. */ - for (Py_ssize_t i = 0; i < nargs; ++i) { - Py_INCREF(PyTuple_GET_ITEM(callargs, i)); - } - } - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)func, locals, - newargs, nargs, kwnames - ); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, + (PyFunctionObject *)func, locals, + nargs, callargs, kwargs); STACK_SHRINK(2); /* get rid of func and NULL */ - Py_DECREF(callargs); - Py_XDECREF(kwargs); - if (has_dict) { - _PyStack_UnpackDict_FreeNoDecRef(newargs, kwnames); - } if (new_frame == NULL) { goto error; } - _PyFrame_SetStackPointer(frame, stack_pointer); - frame->prev_instr = next_instr - 1; - new_frame->previous = frame; - cframe.current_frame = frame = new_frame; - CALL_STAT_INC(inlined_py_calls); - goto start_frame; + DISPATCH_INLINED(new_frame); } else { result = do_call_core(tstate, func, callargs, kwargs, cframe.use_tracing); From bfab4dcffa6e8cf3012fa07e70a8db1b2a9e983a Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 1 Dec 2022 23:09:58 +0800 Subject: [PATCH 13/17] Run --regen --- Include/internal/pycore_global_objects_fini_generated.h | 1 - Include/internal/pycore_global_strings.h | 1 - Include/internal/pycore_runtime_init_generated.h | 1 - Include/internal/pycore_unicodeobject_generated.h | 2 -- 4 files changed, 5 deletions(-) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 0919fdd668b858..494bcf293cdb7b 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -1218,7 +1218,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(x)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(year)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(zdict)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(zipimporter)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(strings).ascii[0]); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(strings).ascii[1]); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(strings).ascii[2]); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 8abb561c053505..b0cb8365933e77 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -704,7 +704,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(x) STRUCT_FOR_ID(year) STRUCT_FOR_ID(zdict) - STRUCT_FOR_ID(zipimporter) } identifiers; struct { PyASCIIObject _ascii; diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 5c28d04b5f5768..4b128da54555b7 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -1210,7 +1210,6 @@ extern "C" { INIT_ID(x), \ INIT_ID(year), \ INIT_ID(zdict), \ - INIT_ID(zipimporter), \ } #define _Py_str_ascii_INIT { \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 68c757a180c73c..7ef1f7e94ddead 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -1314,8 +1314,6 @@ _PyUnicode_InitStaticStrings(void) { PyUnicode_InternInPlace(&string); string = &_Py_ID(zdict); PyUnicode_InternInPlace(&string); - string = &_Py_ID(zipimporter); - PyUnicode_InternInPlace(&string); } /* End auto-generated code */ #ifdef __cplusplus From dfd6b01e99fbb4830d015a8e3e018e5c920ba486 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 27 Apr 2023 17:34:38 +0800 Subject: [PATCH 14/17] Fix merge problems Co-Authored-By: Brandt Bucher --- Python/bytecodes.c | 4 +++- Python/ceval.c | 1 + Python/generated_cases.c.h | 3 ++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index cfce9da3d77376..968e912545f1e6 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3114,10 +3114,12 @@ dummy_func( _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, (PyFunctionObject *)func, locals, nargs, callargs, kwargs); - STACK_SHRINK(2); /* get rid of func and NULL */ + // Need to manually shrinkg the stack since we exit with DISPATCH_INLINED. + STACK_SHRINK(oparg + 3); if (new_frame == NULL) { goto error; } + frame->return_offset = 0; DISPATCH_INLINED(new_frame); } result = PyObject_Call(func, callargs, kwargs); diff --git a/Python/ceval.c b/Python/ceval.c index 5597ff4a67219b..958689debc87f8 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1513,6 +1513,7 @@ _PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func, { bool has_dict = (kwargs != NULL && PyDict_GET_SIZE(kwargs) > 0); PyObject *kwnames = NULL; + PyObject *const *newargs; if (has_dict) { newargs = _PyStack_UnpackDict(tstate, _PyTuple_ITEMS(callargs), nargs, kwargs, &kwnames); if (newargs == NULL) { diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 026b27c1f4106a..d19de3aed2e538 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3860,10 +3860,11 @@ _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, (PyFunctionObject *)func, locals, nargs, callargs, kwargs); - STACK_SHRINK(2); /* get rid of func and NULL */ + STACK_SHRINK(oparg + 3); if (new_frame == NULL) { goto error; } + frame->return_offset = 0; DISPATCH_INLINED(new_frame); } result = PyObject_Call(func, callargs, kwargs); From 1afff6dc586b596db6a9884a37f75c4da937715e Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 28 Apr 2023 21:15:15 +0800 Subject: [PATCH 15/17] Update generated_cases.c.h --- Python/generated_cases.c.h | 491 +++++++++++++++++++++++++++++++++++++ 1 file changed, 491 insertions(+) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index d19de3aed2e538..153a6325534ac2 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -8,6 +8,7 @@ } TARGET(RESUME) { + #line 137 "Python/bytecodes.c" assert(tstate->cframe == &cframe); assert(frame == cframe.current_frame); /* Possibly combine this with eval breaker */ @@ -19,10 +20,12 @@ else if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { goto handle_eval_breaker; } + #line 24 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_RESUME) { + #line 151 "Python/bytecodes.c" /* Possible performance enhancement: * We need to check the eval breaker anyway, can we * combine the instrument verison check and the eval breaker test? @@ -48,15 +51,18 @@ goto handle_eval_breaker; } } + #line 55 "Python/generated_cases.c.h" DISPATCH(); } TARGET(LOAD_CLOSURE) { PyObject *value; + #line 179 "Python/bytecodes.c" /* We keep LOAD_CLOSURE so that the bytecode stays more readable. */ value = GETLOCAL(oparg); if (value == NULL) goto unbound_local_error; Py_INCREF(value); + #line 66 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -64,9 +70,11 @@ TARGET(LOAD_FAST_CHECK) { PyObject *value; + #line 186 "Python/bytecodes.c" value = GETLOCAL(oparg); if (value == NULL) goto unbound_local_error; Py_INCREF(value); + #line 78 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -74,9 +82,11 @@ TARGET(LOAD_FAST) { PyObject *value; + #line 192 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); + #line 90 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -85,8 +95,10 @@ TARGET(LOAD_CONST) { PREDICTED(LOAD_CONST); PyObject *value; + #line 198 "Python/bytecodes.c" value = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(value); + #line 102 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -94,7 +106,9 @@ TARGET(STORE_FAST) { PyObject *value = stack_pointer[-1]; + #line 203 "Python/bytecodes.c" SETLOCAL(oparg, value); + #line 112 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -104,17 +118,21 @@ PyObject *_tmp_2; { PyObject *value; + #line 192 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); + #line 126 "Python/generated_cases.c.h" _tmp_2 = value; } oparg = (next_instr++)->op.arg; { PyObject *value; + #line 192 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); + #line 136 "Python/generated_cases.c.h" _tmp_1 = value; } STACK_GROW(2); @@ -128,16 +146,20 @@ PyObject *_tmp_2; { PyObject *value; + #line 192 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); + #line 154 "Python/generated_cases.c.h" _tmp_2 = value; } oparg = (next_instr++)->op.arg; { PyObject *value; + #line 198 "Python/bytecodes.c" value = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(value); + #line 163 "Python/generated_cases.c.h" _tmp_1 = value; } STACK_GROW(2); @@ -150,14 +172,18 @@ PyObject *_tmp_1 = stack_pointer[-1]; { PyObject *value = _tmp_1; + #line 203 "Python/bytecodes.c" SETLOCAL(oparg, value); + #line 178 "Python/generated_cases.c.h" } oparg = (next_instr++)->op.arg; { PyObject *value; + #line 192 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); + #line 187 "Python/generated_cases.c.h" _tmp_1 = value; } stack_pointer[-1] = _tmp_1; @@ -169,12 +195,16 @@ PyObject *_tmp_2 = stack_pointer[-2]; { PyObject *value = _tmp_1; + #line 203 "Python/bytecodes.c" SETLOCAL(oparg, value); + #line 201 "Python/generated_cases.c.h" } oparg = (next_instr++)->op.arg; { PyObject *value = _tmp_2; + #line 203 "Python/bytecodes.c" SETLOCAL(oparg, value); + #line 208 "Python/generated_cases.c.h" } STACK_SHRINK(2); DISPATCH(); @@ -185,16 +215,20 @@ PyObject *_tmp_2; { PyObject *value; + #line 198 "Python/bytecodes.c" value = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(value); + #line 222 "Python/generated_cases.c.h" _tmp_2 = value; } oparg = (next_instr++)->op.arg; { PyObject *value; + #line 192 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); + #line 232 "Python/generated_cases.c.h" _tmp_1 = value; } STACK_GROW(2); @@ -205,6 +239,8 @@ TARGET(POP_TOP) { PyObject *value = stack_pointer[-1]; + #line 213 "Python/bytecodes.c" + #line 244 "Python/generated_cases.c.h" Py_DECREF(value); STACK_SHRINK(1); DISPATCH(); @@ -212,7 +248,9 @@ TARGET(PUSH_NULL) { PyObject *res; + #line 217 "Python/bytecodes.c" res = NULL; + #line 254 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); @@ -223,10 +261,14 @@ PyObject *_tmp_2 = stack_pointer[-2]; { PyObject *value = _tmp_1; + #line 213 "Python/bytecodes.c" + #line 266 "Python/generated_cases.c.h" Py_DECREF(value); } { PyObject *value = _tmp_2; + #line 213 "Python/bytecodes.c" + #line 272 "Python/generated_cases.c.h" Py_DECREF(value); } STACK_SHRINK(2); @@ -236,6 +278,7 @@ TARGET(INSTRUMENTED_END_FOR) { PyObject *value = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; + #line 223 "Python/bytecodes.c" /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ if (PyGen_Check(receiver)) { @@ -245,6 +288,7 @@ } PyErr_SetRaisedException(NULL); } + #line 292 "Python/generated_cases.c.h" Py_DECREF(receiver); Py_DECREF(value); STACK_SHRINK(2); @@ -254,7 +298,9 @@ TARGET(END_SEND) { PyObject *value = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; + #line 236 "Python/bytecodes.c" Py_DECREF(receiver); + #line 304 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = value; DISPATCH(); @@ -263,6 +309,7 @@ TARGET(INSTRUMENTED_END_SEND) { PyObject *value = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; + #line 240 "Python/bytecodes.c" if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); if (monitor_stop_iteration(tstate, frame, next_instr-1)) { @@ -271,6 +318,7 @@ PyErr_SetRaisedException(NULL); } Py_DECREF(receiver); + #line 322 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = value; DISPATCH(); @@ -279,9 +327,13 @@ TARGET(UNARY_NEGATIVE) { PyObject *value = stack_pointer[-1]; PyObject *res; + #line 251 "Python/bytecodes.c" res = PyNumber_Negative(value); + #line 333 "Python/generated_cases.c.h" Py_DECREF(value); + #line 253 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; + #line 337 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -289,8 +341,11 @@ TARGET(UNARY_NOT) { PyObject *value = stack_pointer[-1]; PyObject *res; + #line 257 "Python/bytecodes.c" int err = PyObject_IsTrue(value); + #line 347 "Python/generated_cases.c.h" Py_DECREF(value); + #line 259 "Python/bytecodes.c" if (err < 0) goto pop_1_error; if (err == 0) { res = Py_True; @@ -299,6 +354,7 @@ res = Py_False; } Py_INCREF(res); + #line 358 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -306,9 +362,13 @@ TARGET(UNARY_INVERT) { PyObject *value = stack_pointer[-1]; PyObject *res; + #line 270 "Python/bytecodes.c" res = PyNumber_Invert(value); + #line 368 "Python/generated_cases.c.h" Py_DECREF(value); + #line 272 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; + #line 372 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -317,6 +377,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *prod; + #line 289 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -324,6 +385,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (prod == NULL) goto pop_2_error; + #line 389 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = prod; next_instr += 1; @@ -334,12 +396,14 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *prod; + #line 299 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); double dprod = ((PyFloatObject *)left)->ob_fval * ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dprod, prod); + #line 407 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = prod; next_instr += 1; @@ -350,6 +414,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *sub; + #line 308 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -357,6 +422,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (sub == NULL) goto pop_2_error; + #line 426 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = sub; next_instr += 1; @@ -367,11 +433,13 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *sub; + #line 318 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); double dsub = ((PyFloatObject *)left)->ob_fval - ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dsub, sub); + #line 443 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = sub; next_instr += 1; @@ -382,6 +450,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; + #line 326 "Python/bytecodes.c" DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -389,6 +458,7 @@ _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); if (res == NULL) goto pop_2_error; + #line 462 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -398,6 +468,7 @@ TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; + #line 342 "Python/bytecodes.c" DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); _Py_CODEUNIT true_next = next_instr[INLINE_CACHE_ENTRIES_BINARY_OP]; @@ -424,6 +495,7 @@ if (*target_local == NULL) goto pop_2_error; // The STORE_FAST is already done. JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP + 1); + #line 499 "Python/generated_cases.c.h" STACK_SHRINK(2); DISPATCH(); } @@ -432,12 +504,14 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *sum; + #line 371 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); double dsum = ((PyFloatObject *)left)->ob_fval + ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dsum, sum); + #line 515 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = sum; next_instr += 1; @@ -448,6 +522,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *sum; + #line 380 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -455,6 +530,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (sum == NULL) goto pop_2_error; + #line 534 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = sum; next_instr += 1; @@ -467,6 +543,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *container = stack_pointer[-2]; PyObject *res; + #line 398 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -478,9 +555,12 @@ DECREMENT_ADAPTIVE_COUNTER(cache->counter); #endif /* ENABLE_SPECIALIZATION */ res = PyObject_GetItem(container, sub); + #line 559 "Python/generated_cases.c.h" Py_DECREF(container); Py_DECREF(sub); + #line 410 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; + #line 564 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -492,6 +572,7 @@ PyObject *start = stack_pointer[-2]; PyObject *container = stack_pointer[-3]; PyObject *res; + #line 414 "Python/bytecodes.c" PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); // Can't use ERROR_IF() here, because we haven't // DECREF'ed container yet, and we still own slice. @@ -504,6 +585,7 @@ } Py_DECREF(container); if (res == NULL) goto pop_3_error; + #line 589 "Python/generated_cases.c.h" STACK_SHRINK(2); stack_pointer[-1] = res; DISPATCH(); @@ -514,6 +596,7 @@ PyObject *start = stack_pointer[-2]; PyObject *container = stack_pointer[-3]; PyObject *v = stack_pointer[-4]; + #line 429 "Python/bytecodes.c" PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); int err; if (slice == NULL) { @@ -526,6 +609,7 @@ Py_DECREF(v); Py_DECREF(container); if (err) goto pop_4_error; + #line 613 "Python/generated_cases.c.h" STACK_SHRINK(4); DISPATCH(); } @@ -534,6 +618,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *list = stack_pointer[-2]; PyObject *res; + #line 444 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); @@ -547,6 +632,7 @@ Py_INCREF(res); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(list); + #line 636 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -557,6 +643,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *tuple = stack_pointer[-2]; PyObject *res; + #line 460 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); @@ -570,6 +657,7 @@ Py_INCREF(res); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(tuple); + #line 661 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -580,6 +668,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *dict = stack_pointer[-2]; PyObject *res; + #line 476 "Python/bytecodes.c" DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); res = PyDict_GetItemWithError(dict, sub); @@ -587,11 +676,14 @@ if (!_PyErr_Occurred(tstate)) { _PyErr_SetKeyError(sub); } + #line 680 "Python/generated_cases.c.h" Py_DECREF(dict); Py_DECREF(sub); + #line 484 "Python/bytecodes.c" if (true) goto pop_2_error; } Py_INCREF(res); // Do this before DECREF'ing dict, sub + #line 687 "Python/generated_cases.c.h" Py_DECREF(dict); Py_DECREF(sub); STACK_SHRINK(1); @@ -603,6 +695,7 @@ TARGET(BINARY_SUBSCR_GETITEM) { PyObject *sub = stack_pointer[-1]; PyObject *container = stack_pointer[-2]; + #line 491 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(container); DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; @@ -624,12 +717,15 @@ JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR); frame->return_offset = 0; DISPATCH_INLINED(new_frame); + #line 721 "Python/generated_cases.c.h" } TARGET(LIST_APPEND) { PyObject *v = stack_pointer[-1]; PyObject *list = stack_pointer[-(2 + (oparg-1))]; + #line 515 "Python/bytecodes.c" if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; + #line 729 "Python/generated_cases.c.h" STACK_SHRINK(1); PREDICT(JUMP_BACKWARD); DISPATCH(); @@ -638,9 +734,13 @@ TARGET(SET_ADD) { PyObject *v = stack_pointer[-1]; PyObject *set = stack_pointer[-(2 + (oparg-1))]; + #line 520 "Python/bytecodes.c" int err = PySet_Add(set, v); + #line 740 "Python/generated_cases.c.h" Py_DECREF(v); + #line 522 "Python/bytecodes.c" if (err) goto pop_1_error; + #line 744 "Python/generated_cases.c.h" STACK_SHRINK(1); PREDICT(JUMP_BACKWARD); DISPATCH(); @@ -653,6 +753,7 @@ PyObject *container = stack_pointer[-2]; PyObject *v = stack_pointer[-3]; uint16_t counter = read_u16(&next_instr[0].cache); + #line 533 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr--; @@ -667,10 +768,13 @@ #endif /* ENABLE_SPECIALIZATION */ /* container[sub] = v */ int err = PyObject_SetItem(container, sub, v); + #line 772 "Python/generated_cases.c.h" Py_DECREF(v); Py_DECREF(container); Py_DECREF(sub); + #line 548 "Python/bytecodes.c" if (err) goto pop_3_error; + #line 778 "Python/generated_cases.c.h" STACK_SHRINK(3); next_instr += 1; DISPATCH(); @@ -680,6 +784,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *list = stack_pointer[-2]; PyObject *value = stack_pointer[-3]; + #line 552 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); @@ -696,6 +801,7 @@ Py_DECREF(old_value); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(list); + #line 805 "Python/generated_cases.c.h" STACK_SHRINK(3); next_instr += 1; DISPATCH(); @@ -705,11 +811,13 @@ PyObject *sub = stack_pointer[-1]; PyObject *dict = stack_pointer[-2]; PyObject *value = stack_pointer[-3]; + #line 571 "Python/bytecodes.c" DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); Py_DECREF(dict); if (err) goto pop_3_error; + #line 821 "Python/generated_cases.c.h" STACK_SHRINK(3); next_instr += 1; DISPATCH(); @@ -718,11 +826,15 @@ TARGET(DELETE_SUBSCR) { PyObject *sub = stack_pointer[-1]; PyObject *container = stack_pointer[-2]; + #line 579 "Python/bytecodes.c" /* del container[sub] */ int err = PyObject_DelItem(container, sub); + #line 833 "Python/generated_cases.c.h" Py_DECREF(container); Py_DECREF(sub); + #line 582 "Python/bytecodes.c" if (err) goto pop_2_error; + #line 838 "Python/generated_cases.c.h" STACK_SHRINK(2); DISPATCH(); } @@ -730,10 +842,14 @@ TARGET(CALL_INTRINSIC_1) { PyObject *value = stack_pointer[-1]; PyObject *res; + #line 586 "Python/bytecodes.c" assert(oparg <= MAX_INTRINSIC_1); res = _PyIntrinsics_UnaryFunctions[oparg](tstate, value); + #line 849 "Python/generated_cases.c.h" Py_DECREF(value); + #line 589 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; + #line 853 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -742,11 +858,15 @@ PyObject *value1 = stack_pointer[-1]; PyObject *value2 = stack_pointer[-2]; PyObject *res; + #line 593 "Python/bytecodes.c" assert(oparg <= MAX_INTRINSIC_2); res = _PyIntrinsics_BinaryFunctions[oparg](tstate, value2, value1); + #line 865 "Python/generated_cases.c.h" Py_DECREF(value2); Py_DECREF(value1); + #line 596 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; + #line 870 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); @@ -754,6 +874,7 @@ TARGET(RAISE_VARARGS) { PyObject **args = (stack_pointer - oparg); + #line 600 "Python/bytecodes.c" PyObject *cause = NULL, *exc = NULL; switch (oparg) { case 2: @@ -771,10 +892,12 @@ break; } if (true) { STACK_SHRINK(oparg); goto error; } + #line 896 "Python/generated_cases.c.h" } TARGET(INTERPRETER_EXIT) { PyObject *retval = stack_pointer[-1]; + #line 620 "Python/bytecodes.c" assert(frame == &entry_frame); assert(_PyFrame_IsIncomplete(frame)); STACK_SHRINK(1); // Since we're not going to DISPATCH() @@ -785,10 +908,12 @@ assert(!_PyErr_Occurred(tstate)); _Py_LeaveRecursiveCallTstate(tstate); return retval; + #line 912 "Python/generated_cases.c.h" } TARGET(RETURN_VALUE) { PyObject *retval = stack_pointer[-1]; + #line 633 "Python/bytecodes.c" STACK_SHRINK(1); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -801,10 +926,12 @@ frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); goto resume_frame; + #line 930 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_RETURN_VALUE) { PyObject *retval = stack_pointer[-1]; + #line 648 "Python/bytecodes.c" int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, next_instr-1, retval); @@ -821,9 +948,11 @@ frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); goto resume_frame; + #line 952 "Python/generated_cases.c.h" } TARGET(RETURN_CONST) { + #line 667 "Python/bytecodes.c" PyObject *retval = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(retval); assert(EMPTY()); @@ -837,9 +966,11 @@ frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); goto resume_frame; + #line 970 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_RETURN_CONST) { + #line 683 "Python/bytecodes.c" PyObject *retval = GETITEM(frame->f_code->co_consts, oparg); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, @@ -857,11 +988,13 @@ frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); goto resume_frame; + #line 992 "Python/generated_cases.c.h" } TARGET(GET_AITER) { PyObject *obj = stack_pointer[-1]; PyObject *iter; + #line 703 "Python/bytecodes.c" unaryfunc getter = NULL; PyTypeObject *type = Py_TYPE(obj); @@ -874,12 +1007,16 @@ "'async for' requires an object with " "__aiter__ method, got %.100s", type->tp_name); + #line 1011 "Python/generated_cases.c.h" Py_DECREF(obj); + #line 716 "Python/bytecodes.c" if (true) goto pop_1_error; } iter = (*getter)(obj); + #line 1018 "Python/generated_cases.c.h" Py_DECREF(obj); + #line 721 "Python/bytecodes.c" if (iter == NULL) goto pop_1_error; if (Py_TYPE(iter)->tp_as_async == NULL || @@ -892,6 +1029,7 @@ Py_DECREF(iter); if (true) goto pop_1_error; } + #line 1033 "Python/generated_cases.c.h" stack_pointer[-1] = iter; DISPATCH(); } @@ -899,6 +1037,7 @@ TARGET(GET_ANEXT) { PyObject *aiter = stack_pointer[-1]; PyObject *awaitable; + #line 736 "Python/bytecodes.c" unaryfunc getter = NULL; PyObject *next_iter = NULL; PyTypeObject *type = Py_TYPE(aiter); @@ -942,6 +1081,7 @@ } } + #line 1085 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = awaitable; PREDICT(LOAD_CONST); @@ -952,13 +1092,16 @@ PREDICTED(GET_AWAITABLE); PyObject *iterable = stack_pointer[-1]; PyObject *iter; + #line 783 "Python/bytecodes.c" iter = _PyCoro_GetAwaitableIter(iterable); if (iter == NULL) { format_awaitable_error(tstate, Py_TYPE(iterable), oparg); } + #line 1103 "Python/generated_cases.c.h" Py_DECREF(iterable); + #line 790 "Python/bytecodes.c" if (iter != NULL && PyCoro_CheckExact(iter)) { PyObject *yf = _PyGen_yf((PyGenObject*)iter); @@ -976,6 +1119,7 @@ if (iter == NULL) goto pop_1_error; + #line 1123 "Python/generated_cases.c.h" stack_pointer[-1] = iter; PREDICT(LOAD_CONST); DISPATCH(); @@ -986,6 +1130,7 @@ PyObject *v = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; PyObject *retval; + #line 816 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PySendCache *cache = (_PySendCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1031,6 +1176,7 @@ } } Py_DECREF(v); + #line 1180 "Python/generated_cases.c.h" stack_pointer[-1] = retval; next_instr += 1; DISPATCH(); @@ -1039,6 +1185,7 @@ TARGET(SEND_GEN) { PyObject *v = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; + #line 864 "Python/bytecodes.c" PyGenObject *gen = (PyGenObject *)receiver; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); @@ -1053,10 +1200,12 @@ tstate->exc_info = &gen->gi_exc_state; JUMPBY(INLINE_CACHE_ENTRIES_SEND); DISPATCH_INLINED(gen_frame); + #line 1204 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_YIELD_VALUE) { PyObject *retval = stack_pointer[-1]; + #line 881 "Python/bytecodes.c" assert(frame != &entry_frame); PyGenObject *gen = _PyFrame_GetGenerator(frame); gen->gi_frame_state = FRAME_SUSPENDED; @@ -1073,10 +1222,12 @@ gen_frame->previous = NULL; _PyFrame_StackPush(frame, retval); goto resume_frame; + #line 1226 "Python/generated_cases.c.h" } TARGET(YIELD_VALUE) { PyObject *retval = stack_pointer[-1]; + #line 900 "Python/bytecodes.c" // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. @@ -1092,12 +1243,15 @@ gen_frame->previous = NULL; _PyFrame_StackPush(frame, retval); goto resume_frame; + #line 1247 "Python/generated_cases.c.h" } TARGET(POP_EXCEPT) { PyObject *exc_value = stack_pointer[-1]; + #line 918 "Python/bytecodes.c" _PyErr_StackItem *exc_info = tstate->exc_info; Py_XSETREF(exc_info->exc_value, exc_value); + #line 1255 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -1105,6 +1259,7 @@ TARGET(RERAISE) { PyObject *exc = stack_pointer[-1]; PyObject **values = (stack_pointer - (1 + oparg)); + #line 923 "Python/bytecodes.c" assert(oparg >= 0 && oparg <= 2); if (oparg) { PyObject *lasti = values[0]; @@ -1122,21 +1277,26 @@ Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); goto exception_unwind; + #line 1281 "Python/generated_cases.c.h" } TARGET(END_ASYNC_FOR) { PyObject *exc = stack_pointer[-1]; PyObject *awaitable = stack_pointer[-2]; + #line 943 "Python/bytecodes.c" assert(exc && PyExceptionInstance_Check(exc)); if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { + #line 1290 "Python/generated_cases.c.h" Py_DECREF(awaitable); Py_DECREF(exc); + #line 946 "Python/bytecodes.c" } else { Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); goto exception_unwind; } + #line 1300 "Python/generated_cases.c.h" STACK_SHRINK(2); DISPATCH(); } @@ -1147,19 +1307,23 @@ PyObject *sub_iter = stack_pointer[-3]; PyObject *none; PyObject *value; + #line 955 "Python/bytecodes.c" assert(throwflag); assert(exc_value && PyExceptionInstance_Check(exc_value)); if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { value = Py_NewRef(((PyStopIterationObject *)exc_value)->value); + #line 1316 "Python/generated_cases.c.h" Py_DECREF(sub_iter); Py_DECREF(last_sent_val); Py_DECREF(exc_value); + #line 960 "Python/bytecodes.c" none = Py_NewRef(Py_None); } else { _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); goto exception_unwind; } + #line 1327 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = value; stack_pointer[-2] = none; @@ -1168,7 +1332,9 @@ TARGET(LOAD_ASSERTION_ERROR) { PyObject *value; + #line 969 "Python/bytecodes.c" value = Py_NewRef(PyExc_AssertionError); + #line 1338 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -1176,6 +1342,7 @@ TARGET(LOAD_BUILD_CLASS) { PyObject *bc; + #line 973 "Python/bytecodes.c" if (PyDict_CheckExact(BUILTINS())) { bc = _PyDict_GetItemWithError(BUILTINS(), &_Py_ID(__build_class__)); @@ -1197,6 +1364,7 @@ if (true) goto error; } } + #line 1368 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = bc; DISPATCH(); @@ -1204,26 +1372,33 @@ TARGET(STORE_NAME) { PyObject *v = stack_pointer[-1]; + #line 997 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); PyObject *ns = LOCALS(); int err; if (ns == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals found when storing %R", name); + #line 1383 "Python/generated_cases.c.h" Py_DECREF(v); + #line 1004 "Python/bytecodes.c" if (true) goto pop_1_error; } if (PyDict_CheckExact(ns)) err = PyDict_SetItem(ns, name, v); else err = PyObject_SetItem(ns, name, v); + #line 1392 "Python/generated_cases.c.h" Py_DECREF(v); + #line 1011 "Python/bytecodes.c" if (err) goto pop_1_error; + #line 1396 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(DELETE_NAME) { + #line 1015 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); PyObject *ns = LOCALS(); int err; @@ -1240,6 +1415,7 @@ name); goto error; } + #line 1419 "Python/generated_cases.c.h" DISPATCH(); } @@ -1247,6 +1423,7 @@ PREDICTED(UNPACK_SEQUENCE); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); PyObject *seq = stack_pointer[-1]; + #line 1041 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1259,8 +1436,11 @@ #endif /* ENABLE_SPECIALIZATION */ PyObject **top = stack_pointer + oparg - 1; int res = unpack_iterable(tstate, seq, oparg, -1, top); + #line 1440 "Python/generated_cases.c.h" Py_DECREF(seq); + #line 1054 "Python/bytecodes.c" if (res == 0) goto pop_1_error; + #line 1444 "Python/generated_cases.c.h" STACK_SHRINK(1); STACK_GROW(oparg); next_instr += 1; @@ -1270,12 +1450,14 @@ TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { PyObject *seq = stack_pointer[-1]; PyObject **values = stack_pointer - (1); + #line 1058 "Python/bytecodes.c" DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); assert(oparg == 2); STAT_INC(UNPACK_SEQUENCE, hit); values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); + #line 1461 "Python/generated_cases.c.h" Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); @@ -1286,6 +1468,7 @@ TARGET(UNPACK_SEQUENCE_TUPLE) { PyObject *seq = stack_pointer[-1]; PyObject **values = stack_pointer - (1); + #line 1068 "Python/bytecodes.c" DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); @@ -1293,6 +1476,7 @@ for (int i = oparg; --i >= 0; ) { *values++ = Py_NewRef(items[i]); } + #line 1480 "Python/generated_cases.c.h" Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); @@ -1303,6 +1487,7 @@ TARGET(UNPACK_SEQUENCE_LIST) { PyObject *seq = stack_pointer[-1]; PyObject **values = stack_pointer - (1); + #line 1079 "Python/bytecodes.c" DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); @@ -1310,6 +1495,7 @@ for (int i = oparg; --i >= 0; ) { *values++ = Py_NewRef(items[i]); } + #line 1499 "Python/generated_cases.c.h" Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); @@ -1319,11 +1505,15 @@ TARGET(UNPACK_EX) { PyObject *seq = stack_pointer[-1]; + #line 1090 "Python/bytecodes.c" int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); PyObject **top = stack_pointer + totalargs - 1; int res = unpack_iterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); + #line 1513 "Python/generated_cases.c.h" Py_DECREF(seq); + #line 1094 "Python/bytecodes.c" if (res == 0) goto pop_1_error; + #line 1517 "Python/generated_cases.c.h" STACK_GROW((oparg & 0xFF) + (oparg >> 8)); DISPATCH(); } @@ -1334,6 +1524,7 @@ PyObject *owner = stack_pointer[-1]; PyObject *v = stack_pointer[-2]; uint16_t counter = read_u16(&next_instr[0].cache); + #line 1105 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { PyObject *name = GETITEM(frame->f_code->co_names, oparg); @@ -1349,9 +1540,12 @@ #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyObject_SetAttr(owner, name, v); + #line 1544 "Python/generated_cases.c.h" Py_DECREF(v); Py_DECREF(owner); + #line 1121 "Python/bytecodes.c" if (err) goto pop_2_error; + #line 1549 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -1359,25 +1553,34 @@ TARGET(DELETE_ATTR) { PyObject *owner = stack_pointer[-1]; + #line 1125 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyObject_SetAttr(owner, name, (PyObject *)NULL); + #line 1560 "Python/generated_cases.c.h" Py_DECREF(owner); + #line 1128 "Python/bytecodes.c" if (err) goto pop_1_error; + #line 1564 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(STORE_GLOBAL) { PyObject *v = stack_pointer[-1]; + #line 1132 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyDict_SetItem(GLOBALS(), name, v); + #line 1574 "Python/generated_cases.c.h" Py_DECREF(v); + #line 1135 "Python/bytecodes.c" if (err) goto pop_1_error; + #line 1578 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(DELETE_GLOBAL) { + #line 1139 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err; err = PyDict_DelItem(GLOBALS(), name); @@ -1389,11 +1592,13 @@ } goto error; } + #line 1596 "Python/generated_cases.c.h" DISPATCH(); } TARGET(LOAD_NAME) { PyObject *v; + #line 1153 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); PyObject *locals = LOCALS(); if (locals == NULL) { @@ -1452,6 +1657,7 @@ } } } + #line 1661 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = v; DISPATCH(); @@ -1462,6 +1668,7 @@ static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyObject *null = NULL; PyObject *v; + #line 1220 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1513,6 +1720,7 @@ } } null = NULL; + #line 1724 "Python/generated_cases.c.h" STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = v; @@ -1526,6 +1734,7 @@ PyObject *res; uint16_t index = read_u16(&next_instr[1].cache); uint16_t version = read_u16(&next_instr[2].cache); + #line 1274 "Python/bytecodes.c" DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); PyDictObject *dict = (PyDictObject *)GLOBALS(); DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); @@ -1536,6 +1745,7 @@ Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; + #line 1749 "Python/generated_cases.c.h" STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -1550,6 +1760,7 @@ uint16_t index = read_u16(&next_instr[1].cache); uint16_t mod_version = read_u16(&next_instr[2].cache); uint16_t bltn_version = read_u16(&next_instr[3].cache); + #line 1287 "Python/bytecodes.c" DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); DEOPT_IF(!PyDict_CheckExact(BUILTINS()), LOAD_GLOBAL); PyDictObject *mdict = (PyDictObject *)GLOBALS(); @@ -1564,6 +1775,7 @@ Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; + #line 1779 "Python/generated_cases.c.h" STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -1573,13 +1785,16 @@ } TARGET(DELETE_FAST) { + #line 1304 "Python/bytecodes.c" PyObject *v = GETLOCAL(oparg); if (v == NULL) goto unbound_local_error; SETLOCAL(oparg, NULL); + #line 1793 "Python/generated_cases.c.h" DISPATCH(); } TARGET(MAKE_CELL) { + #line 1310 "Python/bytecodes.c" // "initial" is probably NULL but not if it's an arg (or set // via PyFrame_LocalsToFast() before MAKE_CELL has run). PyObject *initial = GETLOCAL(oparg); @@ -1588,10 +1803,12 @@ goto resume_with_error; } SETLOCAL(oparg, cell); + #line 1807 "Python/generated_cases.c.h" DISPATCH(); } TARGET(DELETE_DEREF) { + #line 1321 "Python/bytecodes.c" PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); // Can't use ERROR_IF here. @@ -1602,11 +1819,13 @@ } PyCell_SET(cell, NULL); Py_DECREF(oldobj); + #line 1823 "Python/generated_cases.c.h" DISPATCH(); } TARGET(LOAD_CLASSDEREF) { PyObject *value; + #line 1334 "Python/bytecodes.c" PyObject *name, *locals = LOCALS(); assert(locals); assert(oparg >= 0 && oparg < frame->f_code->co_nlocalsplus); @@ -1638,6 +1857,7 @@ } Py_INCREF(value); } + #line 1861 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -1645,6 +1865,7 @@ TARGET(LOAD_DEREF) { PyObject *value; + #line 1368 "Python/bytecodes.c" PyObject *cell = GETLOCAL(oparg); value = PyCell_GET(cell); if (value == NULL) { @@ -1652,6 +1873,7 @@ if (true) goto error; } Py_INCREF(value); + #line 1877 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -1659,15 +1881,18 @@ TARGET(STORE_DEREF) { PyObject *v = stack_pointer[-1]; + #line 1378 "Python/bytecodes.c" PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); PyCell_SET(cell, v); Py_XDECREF(oldobj); + #line 1890 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(COPY_FREE_VARS) { + #line 1385 "Python/bytecodes.c" /* Copy closure variables to free variables */ PyCodeObject *co = frame->f_code; assert(PyFunction_Check(frame->f_funcobj)); @@ -1678,17 +1903,22 @@ PyObject *o = PyTuple_GET_ITEM(closure, i); frame->localsplus[offset + i] = Py_NewRef(o); } + #line 1907 "Python/generated_cases.c.h" DISPATCH(); } TARGET(BUILD_STRING) { PyObject **pieces = (stack_pointer - oparg); PyObject *str; + #line 1398 "Python/bytecodes.c" str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); + #line 1916 "Python/generated_cases.c.h" for (int _i = oparg; --_i >= 0;) { Py_DECREF(pieces[_i]); } + #line 1400 "Python/bytecodes.c" if (str == NULL) { STACK_SHRINK(oparg); goto error; } + #line 1922 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = str; @@ -1698,8 +1928,10 @@ TARGET(BUILD_TUPLE) { PyObject **values = (stack_pointer - oparg); PyObject *tup; + #line 1404 "Python/bytecodes.c" tup = _PyTuple_FromArraySteal(values, oparg); if (tup == NULL) { STACK_SHRINK(oparg); goto error; } + #line 1935 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = tup; @@ -1709,8 +1941,10 @@ TARGET(BUILD_LIST) { PyObject **values = (stack_pointer - oparg); PyObject *list; + #line 1409 "Python/bytecodes.c" list = _PyList_FromArraySteal(values, oparg); if (list == NULL) { STACK_SHRINK(oparg); goto error; } + #line 1948 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = list; @@ -1720,6 +1954,7 @@ TARGET(LIST_EXTEND) { PyObject *iterable = stack_pointer[-1]; PyObject *list = stack_pointer[-(2 + (oparg-1))]; + #line 1414 "Python/bytecodes.c" PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); if (none_val == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && @@ -1730,10 +1965,13 @@ "Value after * must be an iterable, not %.200s", Py_TYPE(iterable)->tp_name); } + #line 1969 "Python/generated_cases.c.h" Py_DECREF(iterable); + #line 1425 "Python/bytecodes.c" if (true) goto pop_1_error; } Py_DECREF(none_val); + #line 1975 "Python/generated_cases.c.h" Py_DECREF(iterable); STACK_SHRINK(1); DISPATCH(); @@ -1742,9 +1980,13 @@ TARGET(SET_UPDATE) { PyObject *iterable = stack_pointer[-1]; PyObject *set = stack_pointer[-(2 + (oparg-1))]; + #line 1432 "Python/bytecodes.c" int err = _PySet_Update(set, iterable); + #line 1986 "Python/generated_cases.c.h" Py_DECREF(iterable); + #line 1434 "Python/bytecodes.c" if (err < 0) goto pop_1_error; + #line 1990 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -1752,6 +1994,7 @@ TARGET(BUILD_SET) { PyObject **values = (stack_pointer - oparg); PyObject *set; + #line 1438 "Python/bytecodes.c" set = PySet_New(NULL); if (set == NULL) goto error; @@ -1766,6 +2009,7 @@ Py_DECREF(set); if (true) { STACK_SHRINK(oparg); goto error; } } + #line 2013 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = set; @@ -1775,6 +2019,7 @@ TARGET(BUILD_MAP) { PyObject **values = (stack_pointer - oparg*2); PyObject *map; + #line 1455 "Python/bytecodes.c" map = _PyDict_FromItems( values, 2, values+1, 2, @@ -1782,10 +2027,13 @@ if (map == NULL) goto error; + #line 2031 "Python/generated_cases.c.h" for (int _i = oparg*2; --_i >= 0;) { Py_DECREF(values[_i]); } + #line 1463 "Python/bytecodes.c" if (map == NULL) { STACK_SHRINK(oparg*2); goto error; } + #line 2037 "Python/generated_cases.c.h" STACK_SHRINK(oparg*2); STACK_GROW(1); stack_pointer[-1] = map; @@ -1793,6 +2041,7 @@ } TARGET(SETUP_ANNOTATIONS) { + #line 1467 "Python/bytecodes.c" int err; PyObject *ann_dict; if (LOCALS() == NULL) { @@ -1832,6 +2081,7 @@ Py_DECREF(ann_dict); } } + #line 2085 "Python/generated_cases.c.h" DISPATCH(); } @@ -1839,6 +2089,7 @@ PyObject *keys = stack_pointer[-1]; PyObject **values = (stack_pointer - (1 + oparg)); PyObject *map; + #line 1509 "Python/bytecodes.c" if (!PyTuple_CheckExact(keys) || PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { _PyErr_SetString(tstate, PyExc_SystemError, @@ -1848,11 +2099,14 @@ map = _PyDict_FromItems( &PyTuple_GET_ITEM(keys, 0), 1, values, 1, oparg); + #line 2103 "Python/generated_cases.c.h" for (int _i = oparg; --_i >= 0;) { Py_DECREF(values[_i]); } Py_DECREF(keys); + #line 1519 "Python/bytecodes.c" if (map == NULL) { STACK_SHRINK(oparg); goto pop_1_error; } + #line 2110 "Python/generated_cases.c.h" STACK_SHRINK(oparg); stack_pointer[-1] = map; DISPATCH(); @@ -1860,6 +2114,7 @@ TARGET(DICT_UPDATE) { PyObject *update = stack_pointer[-1]; + #line 1523 "Python/bytecodes.c" PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (PyDict_Update(dict, update) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { @@ -1867,9 +2122,12 @@ "'%.200s' object is not a mapping", Py_TYPE(update)->tp_name); } + #line 2126 "Python/generated_cases.c.h" Py_DECREF(update); + #line 1531 "Python/bytecodes.c" if (true) goto pop_1_error; } + #line 2131 "Python/generated_cases.c.h" Py_DECREF(update); STACK_SHRINK(1); DISPATCH(); @@ -1877,13 +2135,17 @@ TARGET(DICT_MERGE) { PyObject *update = stack_pointer[-1]; + #line 1537 "Python/bytecodes.c" PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (_PyDict_MergeEx(dict, update, 2) < 0) { format_kwargs_error(tstate, PEEK(3 + oparg), update); + #line 2144 "Python/generated_cases.c.h" Py_DECREF(update); + #line 1542 "Python/bytecodes.c" if (true) goto pop_1_error; } + #line 2149 "Python/generated_cases.c.h" Py_DECREF(update); STACK_SHRINK(1); PREDICT(CALL_FUNCTION_EX); @@ -1893,11 +2155,13 @@ TARGET(MAP_ADD) { PyObject *value = stack_pointer[-1]; PyObject *key = stack_pointer[-2]; + #line 1549 "Python/bytecodes.c" PyObject *dict = PEEK(oparg + 2); // key, value are still on the stack assert(PyDict_CheckExact(dict)); /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; + #line 2165 "Python/generated_cases.c.h" STACK_SHRINK(2); PREDICT(JUMP_BACKWARD); DISPATCH(); @@ -1911,6 +2175,7 @@ PyObject *global_super = stack_pointer[-3]; PyObject *res2 = NULL; PyObject *res; + #line 1563 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg >> 2); int load_method = oparg & 1; #if ENABLE_SPECIALIZATION @@ -1928,13 +2193,16 @@ // handle any case whose performance we care about PyObject *stack[] = {class, self}; PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); + #line 2197 "Python/generated_cases.c.h" Py_DECREF(global_super); Py_DECREF(class); Py_DECREF(self); + #line 1581 "Python/bytecodes.c" if (super == NULL) goto pop_3_error; res = PyObject_GetAttr(super, name); Py_DECREF(super); if (res == NULL) goto pop_3_error; + #line 2206 "Python/generated_cases.c.h" STACK_SHRINK(2); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -1952,6 +2220,7 @@ uint32_t class_version = read_u32(&next_instr[1].cache); uint32_t self_type_version = read_u32(&next_instr[3].cache); PyObject *method = read_obj(&next_instr[5].cache); + #line 1588 "Python/bytecodes.c" DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); DEOPT_IF(((PyTypeObject *)class)->tp_version_tag != class_version, LOAD_SUPER_ATTR); @@ -1962,6 +2231,7 @@ Py_INCREF(res2); Py_DECREF(global_super); Py_DECREF(class); + #line 2235 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; stack_pointer[-2] = res2; @@ -1975,6 +2245,7 @@ PyObject *owner = stack_pointer[-1]; PyObject *res2 = NULL; PyObject *res; + #line 1615 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -2008,7 +2279,9 @@ NULL | meth | arg1 | ... | argN */ + #line 2283 "Python/generated_cases.c.h" Py_DECREF(owner); + #line 1649 "Python/bytecodes.c" if (meth == NULL) goto pop_1_error; res2 = NULL; res = meth; @@ -2017,9 +2290,12 @@ else { /* Classic, pushes one value. */ res = PyObject_GetAttr(owner, name); + #line 2294 "Python/generated_cases.c.h" Py_DECREF(owner); + #line 1658 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; } + #line 2299 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -2033,6 +2309,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); + #line 1663 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2045,6 +2322,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; + #line 2326 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2059,6 +2337,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); + #line 1679 "Python/bytecodes.c" DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict != NULL); @@ -2071,6 +2350,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; + #line 2354 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2085,6 +2365,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); + #line 1695 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2111,6 +2392,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; + #line 2396 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2125,6 +2407,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); + #line 1725 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2134,6 +2417,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; + #line 2421 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2148,6 +2432,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); + #line 1738 "Python/bytecodes.c" DEOPT_IF(!PyType_Check(cls), LOAD_ATTR); DEOPT_IF(((PyTypeObject *)cls)->tp_version_tag != type_version, @@ -2159,6 +2444,7 @@ res = descr; assert(res != NULL); Py_INCREF(res); + #line 2448 "Python/generated_cases.c.h" Py_DECREF(cls); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2172,6 +2458,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t func_version = read_u32(&next_instr[3].cache); PyObject *fget = read_obj(&next_instr[5].cache); + #line 1753 "Python/bytecodes.c" DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); @@ -2195,6 +2482,7 @@ JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); frame->return_offset = 0; DISPATCH_INLINED(new_frame); + #line 2486 "Python/generated_cases.c.h" } TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { @@ -2202,6 +2490,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t func_version = read_u32(&next_instr[3].cache); PyObject *getattribute = read_obj(&next_instr[5].cache); + #line 1779 "Python/bytecodes.c" DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); @@ -2227,6 +2516,7 @@ JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); frame->return_offset = 0; DISPATCH_INLINED(new_frame); + #line 2520 "Python/generated_cases.c.h" } TARGET(STORE_ATTR_INSTANCE_VALUE) { @@ -2234,6 +2524,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); + #line 1807 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2251,6 +2542,7 @@ Py_DECREF(old_value); } Py_DECREF(owner); + #line 2546 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2261,6 +2553,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t hint = read_u16(&next_instr[3].cache); + #line 1827 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2299,6 +2592,7 @@ /* PEP 509 */ dict->ma_version_tag = new_version; Py_DECREF(owner); + #line 2596 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2309,6 +2603,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); + #line 1868 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2318,6 +2613,7 @@ *(PyObject **)addr = value; Py_XDECREF(old_value); Py_DECREF(owner); + #line 2617 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2329,6 +2625,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; + #line 1887 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -2341,9 +2638,12 @@ #endif /* ENABLE_SPECIALIZATION */ assert((oparg >> 4) <= Py_GE); res = PyObject_RichCompare(left, right, oparg>>4); + #line 2642 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); + #line 1900 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; + #line 2647 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2354,6 +2654,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; + #line 1904 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); @@ -2365,6 +2666,7 @@ _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); res = (sign_ish & oparg) ? Py_True : Py_False; Py_INCREF(res); + #line 2670 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2375,6 +2677,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; + #line 1919 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); @@ -2390,6 +2693,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); res = (sign_ish & oparg) ? Py_True : Py_False; Py_INCREF(res); + #line 2697 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2400,6 +2704,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; + #line 1938 "Python/bytecodes.c" DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); @@ -2412,6 +2717,7 @@ assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; Py_INCREF(res); + #line 2721 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2422,10 +2728,14 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; + #line 1953 "Python/bytecodes.c" int res = Py_Is(left, right) ^ oparg; + #line 2734 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); + #line 1955 "Python/bytecodes.c" b = Py_NewRef(res ? Py_True : Py_False); + #line 2739 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = b; DISPATCH(); @@ -2435,11 +2745,15 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; + #line 1959 "Python/bytecodes.c" int res = PySequence_Contains(right, left); + #line 2751 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); + #line 1961 "Python/bytecodes.c" if (res < 0) goto pop_2_error; b = Py_NewRef((res^oparg) ? Py_True : Py_False); + #line 2757 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = b; DISPATCH(); @@ -2450,9 +2764,12 @@ PyObject *exc_value = stack_pointer[-2]; PyObject *rest; PyObject *match; + #line 1966 "Python/bytecodes.c" if (check_except_star_type_valid(tstate, match_type) < 0) { + #line 2770 "Python/generated_cases.c.h" Py_DECREF(exc_value); Py_DECREF(match_type); + #line 1968 "Python/bytecodes.c" if (true) goto pop_2_error; } @@ -2460,8 +2777,10 @@ rest = NULL; int res = exception_group_match(exc_value, match_type, &match, &rest); + #line 2781 "Python/generated_cases.c.h" Py_DECREF(exc_value); Py_DECREF(match_type); + #line 1976 "Python/bytecodes.c" if (res < 0) goto pop_2_error; assert((match == NULL) == (rest == NULL)); @@ -2470,6 +2789,7 @@ if (!Py_IsNone(match)) { PyErr_SetHandledException(match); } + #line 2793 "Python/generated_cases.c.h" stack_pointer[-1] = match; stack_pointer[-2] = rest; DISPATCH(); @@ -2479,15 +2799,21 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; + #line 1987 "Python/bytecodes.c" assert(PyExceptionInstance_Check(left)); if (check_except_type_valid(tstate, right) < 0) { + #line 2806 "Python/generated_cases.c.h" Py_DECREF(right); + #line 1990 "Python/bytecodes.c" if (true) goto pop_1_error; } int res = PyErr_GivenExceptionMatches(left, right); + #line 2813 "Python/generated_cases.c.h" Py_DECREF(right); + #line 1995 "Python/bytecodes.c" b = Py_NewRef(res ? Py_True : Py_False); + #line 2817 "Python/generated_cases.c.h" stack_pointer[-1] = b; DISPATCH(); } @@ -2496,11 +2822,15 @@ PyObject *fromlist = stack_pointer[-1]; PyObject *level = stack_pointer[-2]; PyObject *res; + #line 1999 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_name(tstate, frame, name, fromlist, level); + #line 2829 "Python/generated_cases.c.h" Py_DECREF(level); Py_DECREF(fromlist); + #line 2002 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; + #line 2834 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); @@ -2509,23 +2839,29 @@ TARGET(IMPORT_FROM) { PyObject *from = stack_pointer[-1]; PyObject *res; + #line 2006 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_from(tstate, from, name); if (res == NULL) goto error; + #line 2847 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); } TARGET(JUMP_FORWARD) { + #line 2012 "Python/bytecodes.c" JUMPBY(oparg); + #line 2856 "Python/generated_cases.c.h" DISPATCH(); } TARGET(JUMP_BACKWARD) { PREDICTED(JUMP_BACKWARD); + #line 2016 "Python/bytecodes.c" assert(oparg < INSTR_OFFSET()); JUMPBY(-oparg); + #line 2865 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -2533,6 +2869,7 @@ TARGET(POP_JUMP_IF_FALSE) { PREDICTED(POP_JUMP_IF_FALSE); PyObject *cond = stack_pointer[-1]; + #line 2022 "Python/bytecodes.c" if (Py_IsTrue(cond)) { _Py_DECREF_NO_DEALLOC(cond); } @@ -2542,7 +2879,9 @@ } else { int err = PyObject_IsTrue(cond); + #line 2883 "Python/generated_cases.c.h" Py_DECREF(cond); + #line 2032 "Python/bytecodes.c" if (err == 0) { JUMPBY(oparg); } @@ -2550,12 +2889,14 @@ if (err < 0) goto pop_1_error; } } + #line 2893 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_TRUE) { PyObject *cond = stack_pointer[-1]; + #line 2042 "Python/bytecodes.c" if (Py_IsFalse(cond)) { _Py_DECREF_NO_DEALLOC(cond); } @@ -2565,7 +2906,9 @@ } else { int err = PyObject_IsTrue(cond); + #line 2910 "Python/generated_cases.c.h" Py_DECREF(cond); + #line 2052 "Python/bytecodes.c" if (err > 0) { JUMPBY(oparg); } @@ -2573,54 +2916,67 @@ if (err < 0) goto pop_1_error; } } + #line 2920 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_NOT_NONE) { PyObject *value = stack_pointer[-1]; + #line 2062 "Python/bytecodes.c" if (!Py_IsNone(value)) { + #line 2929 "Python/generated_cases.c.h" Py_DECREF(value); + #line 2064 "Python/bytecodes.c" JUMPBY(oparg); } else { _Py_DECREF_NO_DEALLOC(value); } + #line 2937 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_NONE) { PyObject *value = stack_pointer[-1]; + #line 2072 "Python/bytecodes.c" if (Py_IsNone(value)) { _Py_DECREF_NO_DEALLOC(value); JUMPBY(oparg); } else { + #line 2950 "Python/generated_cases.c.h" Py_DECREF(value); + #line 2078 "Python/bytecodes.c" } + #line 2954 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(JUMP_BACKWARD_NO_INTERRUPT) { + #line 2082 "Python/bytecodes.c" /* This bytecode is used in the `yield from` or `await` loop. * If there is an interrupt, we want it handled in the innermost * generator or coroutine, so we deliberately do not check it here. * (see bpo-30039). */ JUMPBY(-oparg); + #line 2967 "Python/generated_cases.c.h" DISPATCH(); } TARGET(GET_LEN) { PyObject *obj = stack_pointer[-1]; PyObject *len_o; + #line 2091 "Python/bytecodes.c" // PUSH(len(TOS)) Py_ssize_t len_i = PyObject_Length(obj); if (len_i < 0) goto error; len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) goto error; + #line 2980 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = len_o; DISPATCH(); @@ -2631,13 +2987,16 @@ PyObject *type = stack_pointer[-2]; PyObject *subject = stack_pointer[-3]; PyObject *attrs; + #line 2099 "Python/bytecodes.c" // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. assert(PyTuple_CheckExact(names)); attrs = match_class(tstate, subject, type, oparg, names); + #line 2996 "Python/generated_cases.c.h" Py_DECREF(subject); Py_DECREF(type); Py_DECREF(names); + #line 2104 "Python/bytecodes.c" if (attrs) { assert(PyTuple_CheckExact(attrs)); // Success! } @@ -2645,6 +3004,7 @@ if (_PyErr_Occurred(tstate)) goto pop_3_error; attrs = Py_NewRef(Py_None); // Failure! } + #line 3008 "Python/generated_cases.c.h" STACK_SHRINK(2); stack_pointer[-1] = attrs; DISPATCH(); @@ -2653,8 +3013,10 @@ TARGET(MATCH_MAPPING) { PyObject *subject = stack_pointer[-1]; PyObject *res; + #line 2114 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; res = Py_NewRef(match ? Py_True : Py_False); + #line 3020 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; PREDICT(POP_JUMP_IF_FALSE); @@ -2664,8 +3026,10 @@ TARGET(MATCH_SEQUENCE) { PyObject *subject = stack_pointer[-1]; PyObject *res; + #line 2120 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; res = Py_NewRef(match ? Py_True : Py_False); + #line 3033 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; PREDICT(POP_JUMP_IF_FALSE); @@ -2676,9 +3040,11 @@ PyObject *keys = stack_pointer[-1]; PyObject *subject = stack_pointer[-2]; PyObject *values_or_none; + #line 2126 "Python/bytecodes.c" // On successful match, PUSH(values). Otherwise, PUSH(None). values_or_none = match_keys(tstate, subject, keys); if (values_or_none == NULL) goto error; + #line 3048 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = values_or_none; DISPATCH(); @@ -2687,10 +3053,14 @@ TARGET(GET_ITER) { PyObject *iterable = stack_pointer[-1]; PyObject *iter; + #line 2132 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ iter = PyObject_GetIter(iterable); + #line 3060 "Python/generated_cases.c.h" Py_DECREF(iterable); + #line 2135 "Python/bytecodes.c" if (iter == NULL) goto pop_1_error; + #line 3064 "Python/generated_cases.c.h" stack_pointer[-1] = iter; DISPATCH(); } @@ -2698,6 +3068,7 @@ TARGET(GET_YIELD_FROM_ITER) { PyObject *iterable = stack_pointer[-1]; PyObject *iter; + #line 2139 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ if (PyCoro_CheckExact(iterable)) { /* `iterable` is a coroutine */ @@ -2720,8 +3091,11 @@ if (iter == NULL) { goto error; } + #line 3095 "Python/generated_cases.c.h" Py_DECREF(iterable); + #line 2162 "Python/bytecodes.c" } + #line 3099 "Python/generated_cases.c.h" stack_pointer[-1] = iter; PREDICT(LOAD_CONST); DISPATCH(); @@ -2732,6 +3106,7 @@ static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); PyObject *iter = stack_pointer[-1]; PyObject *next; + #line 2181 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyForIterCache *cache = (_PyForIterCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -2762,6 +3137,7 @@ DISPATCH(); } // Common case: no jump, leave it to the code generator + #line 3141 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -2769,6 +3145,7 @@ } TARGET(INSTRUMENTED_FOR_ITER) { + #line 2214 "Python/bytecodes.c" _Py_CODEUNIT *here = next_instr-1; _Py_CODEUNIT *target; PyObject *iter = TOP(); @@ -2794,12 +3171,14 @@ target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1; } INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); + #line 3175 "Python/generated_cases.c.h" DISPATCH(); } TARGET(FOR_ITER_LIST) { PyObject *iter = stack_pointer[-1]; PyObject *next; + #line 2242 "Python/bytecodes.c" DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); _PyListIterObject *it = (_PyListIterObject *)iter; STAT_INC(FOR_ITER, hit); @@ -2819,6 +3198,7 @@ DISPATCH(); end_for_iter_list: // Common case: no jump, leave it to the code generator + #line 3202 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -2828,6 +3208,7 @@ TARGET(FOR_ITER_TUPLE) { PyObject *iter = stack_pointer[-1]; PyObject *next; + #line 2264 "Python/bytecodes.c" _PyTupleIterObject *it = (_PyTupleIterObject *)iter; DEOPT_IF(Py_TYPE(it) != &PyTupleIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -2847,6 +3228,7 @@ DISPATCH(); end_for_iter_tuple: // Common case: no jump, leave it to the code generator + #line 3232 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -2856,6 +3238,7 @@ TARGET(FOR_ITER_RANGE) { PyObject *iter = stack_pointer[-1]; PyObject *next; + #line 2286 "Python/bytecodes.c" _PyRangeIterObject *r = (_PyRangeIterObject *)iter; DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -2873,6 +3256,7 @@ if (next == NULL) { goto error; } + #line 3260 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -2881,6 +3265,7 @@ TARGET(FOR_ITER_GEN) { PyObject *iter = stack_pointer[-1]; + #line 2306 "Python/bytecodes.c" PyGenObject *gen = (PyGenObject *)iter; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); @@ -2895,12 +3280,14 @@ assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); DISPATCH_INLINED(gen_frame); + #line 3284 "Python/generated_cases.c.h" } TARGET(BEFORE_ASYNC_WITH) { PyObject *mgr = stack_pointer[-1]; PyObject *exit; PyObject *res; + #line 2323 "Python/bytecodes.c" PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); if (enter == NULL) { if (!_PyErr_Occurred(tstate)) { @@ -2923,13 +3310,16 @@ Py_DECREF(enter); goto error; } + #line 3314 "Python/generated_cases.c.h" Py_DECREF(mgr); + #line 2346 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); if (true) goto pop_1_error; } + #line 3323 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -2941,6 +3331,7 @@ PyObject *mgr = stack_pointer[-1]; PyObject *exit; PyObject *res; + #line 2356 "Python/bytecodes.c" /* pop the context manager, push its __exit__ and the * value returned from calling its __enter__ */ @@ -2966,13 +3357,16 @@ Py_DECREF(enter); goto error; } + #line 3361 "Python/generated_cases.c.h" Py_DECREF(mgr); + #line 2382 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); if (true) goto pop_1_error; } + #line 3370 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -2984,6 +3378,7 @@ PyObject *lasti = stack_pointer[-3]; PyObject *exit_func = stack_pointer[-4]; PyObject *res; + #line 2391 "Python/bytecodes.c" /* At the top of the stack are 4 values: - val: TOP = exc_info() - unused: SECOND = previous exception @@ -3004,6 +3399,7 @@ res = PyObject_Vectorcall(exit_func, stack + 1, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); if (res == NULL) goto error; + #line 3403 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); @@ -3012,6 +3408,7 @@ TARGET(PUSH_EXC_INFO) { PyObject *new_exc = stack_pointer[-1]; PyObject *prev_exc; + #line 2414 "Python/bytecodes.c" _PyErr_StackItem *exc_info = tstate->exc_info; if (exc_info->exc_value != NULL) { prev_exc = exc_info->exc_value; @@ -3021,6 +3418,7 @@ } assert(PyExceptionInstance_Check(new_exc)); exc_info->exc_value = Py_NewRef(new_exc); + #line 3422 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = new_exc; stack_pointer[-2] = prev_exc; @@ -3034,6 +3432,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t keys_version = read_u32(&next_instr[3].cache); PyObject *descr = read_obj(&next_instr[5].cache); + #line 2426 "Python/bytecodes.c" /* Cached method object */ PyTypeObject *self_cls = Py_TYPE(self); assert(type_version != 0); @@ -3050,6 +3449,7 @@ assert(_PyType_HasFeature(Py_TYPE(res2), Py_TPFLAGS_METHOD_DESCRIPTOR)); res = self; assert(oparg & 1); + #line 3453 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3063,6 +3463,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); + #line 2445 "Python/bytecodes.c" PyTypeObject *self_cls = Py_TYPE(self); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); assert(self_cls->tp_dictoffset == 0); @@ -3072,6 +3473,7 @@ res2 = Py_NewRef(descr); res = self; assert(oparg & 1); + #line 3477 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3085,6 +3487,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); + #line 2457 "Python/bytecodes.c" PyTypeObject *self_cls = Py_TYPE(self); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); Py_ssize_t dictoffset = self_cls->tp_dictoffset; @@ -3098,6 +3501,7 @@ res2 = Py_NewRef(descr); res = self; assert(oparg & 1); + #line 3505 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3106,13 +3510,16 @@ } TARGET(KW_NAMES) { + #line 2473 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg < PyTuple_GET_SIZE(frame->f_code->co_consts)); kwnames = GETITEM(frame->f_code->co_consts, oparg); + #line 3518 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_CALL) { + #line 2479 "Python/bytecodes.c" int is_meth = PEEK(oparg+2) != NULL; int total_args = oparg + is_meth; PyObject *function = PEEK(total_args + 1); @@ -3125,6 +3532,7 @@ _PyCallCache *cache = (_PyCallCache *)next_instr; INCREMENT_ADAPTIVE_COUNTER(cache->counter); GO_TO_INSTRUCTION(CALL); + #line 3536 "Python/generated_cases.c.h" } TARGET(CALL) { @@ -3134,6 +3542,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; + #line 2524 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3215,6 +3624,7 @@ Py_DECREF(args[i]); } if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + #line 3628 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3226,6 +3636,7 @@ TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; + #line 2612 "Python/bytecodes.c" DEOPT_IF(method != NULL, CALL); DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); STAT_INC(CALL, hit); @@ -3235,6 +3646,7 @@ PEEK(oparg + 2) = Py_NewRef(meth); // method Py_DECREF(callable); GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); + #line 3650 "Python/generated_cases.c.h" } TARGET(CALL_PY_EXACT_ARGS) { @@ -3243,6 +3655,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); + #line 2624 "Python/bytecodes.c" assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; @@ -3268,6 +3681,7 @@ JUMPBY(INLINE_CACHE_ENTRIES_CALL); frame->return_offset = 0; DISPATCH_INLINED(new_frame); + #line 3685 "Python/generated_cases.c.h" } TARGET(CALL_PY_WITH_DEFAULTS) { @@ -3275,6 +3689,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); + #line 2652 "Python/bytecodes.c" assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; @@ -3310,6 +3725,7 @@ JUMPBY(INLINE_CACHE_ENTRIES_CALL); frame->return_offset = 0; DISPATCH_INLINED(new_frame); + #line 3729 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_TYPE_1) { @@ -3317,6 +3733,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; + #line 2690 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3326,6 +3743,7 @@ res = Py_NewRef(Py_TYPE(obj)); Py_DECREF(obj); Py_DECREF(&PyType_Type); // I.e., callable + #line 3747 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3338,6 +3756,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; + #line 2702 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3348,6 +3767,7 @@ Py_DECREF(arg); Py_DECREF(&PyUnicode_Type); // I.e., callable if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + #line 3771 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3361,6 +3781,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; + #line 2716 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3371,6 +3792,7 @@ Py_DECREF(arg); Py_DECREF(&PyTuple_Type); // I.e., tuple if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + #line 3796 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3384,6 +3806,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; + #line 2730 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3405,6 +3828,7 @@ } Py_DECREF(tp); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + #line 3832 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3418,6 +3842,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; + #line 2755 "Python/bytecodes.c" /* Builtin METH_O functions */ assert(kwnames == NULL); int is_meth = method != NULL; @@ -3445,6 +3870,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + #line 3874 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3458,6 +3884,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; + #line 2786 "Python/bytecodes.c" /* Builtin METH_FASTCALL functions, without keywords */ assert(kwnames == NULL); int is_meth = method != NULL; @@ -3489,6 +3916,7 @@ 'invalid'). In those cases an exception is set, so we must handle it. */ + #line 3920 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3502,6 +3930,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; + #line 2821 "Python/bytecodes.c" /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ int is_meth = method != NULL; int total_args = oparg; @@ -3533,6 +3962,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + #line 3966 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3546,6 +3976,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; + #line 2856 "Python/bytecodes.c" assert(kwnames == NULL); /* len(o) */ int is_meth = method != NULL; @@ -3570,6 +4001,7 @@ Py_DECREF(callable); Py_DECREF(arg); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + #line 4005 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3582,6 +4014,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; + #line 2883 "Python/bytecodes.c" assert(kwnames == NULL); /* isinstance(o, o2) */ int is_meth = method != NULL; @@ -3608,6 +4041,7 @@ Py_DECREF(cls); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + #line 4045 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3619,6 +4053,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *self = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; + #line 2913 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); assert(method != NULL); @@ -3636,12 +4071,14 @@ JUMPBY(INLINE_CACHE_ENTRIES_CALL + 1); assert(next_instr[-1].op.code == POP_TOP); DISPATCH(); + #line 4075 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) { PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; + #line 2933 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -3672,6 +4109,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + #line 4113 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3684,6 +4122,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; + #line 2967 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3712,6 +4151,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + #line 4155 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3724,6 +4164,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; + #line 2999 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 0 || oparg == 1); int is_meth = method != NULL; @@ -3752,6 +4193,7 @@ Py_DECREF(self); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + #line 4197 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3764,6 +4206,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; + #line 3031 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -3791,6 +4234,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + #line 4238 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3800,7 +4244,9 @@ } TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { + #line 3062 "Python/bytecodes.c" GO_TO_INSTRUCTION(CALL_FUNCTION_EX); + #line 4250 "Python/generated_cases.c.h" } TARGET(CALL_FUNCTION_EX) { @@ -3809,6 +4255,7 @@ PyObject *callargs = stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))]; PyObject *func = stack_pointer[-(2 + ((oparg & 1) ? 1 : 0))]; PyObject *result; + #line 3066 "Python/bytecodes.c" // DICT_MERGE is called before this opcode if there are kwargs. // It converts all dict subtypes in kwargs into regular dicts. assert(kwargs == NULL || PyDict_CheckExact(kwargs)); @@ -3860,6 +4307,7 @@ _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, (PyFunctionObject *)func, locals, nargs, callargs, kwargs); + // Need to manually shrinkg the stack since we exit with DISPATCH_INLINED. STACK_SHRINK(oparg + 3); if (new_frame == NULL) { goto error; @@ -3869,11 +4317,14 @@ } result = PyObject_Call(func, callargs, kwargs); } + #line 4321 "Python/generated_cases.c.h" Py_DECREF(func); Py_DECREF(callargs); Py_XDECREF(kwargs); + #line 3128 "Python/bytecodes.c" assert(PEEK(3 + (oparg & 1)) == NULL); if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } + #line 4328 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 1) ? 1 : 0)); STACK_SHRINK(2); stack_pointer[-1] = result; @@ -3888,6 +4339,7 @@ PyObject *kwdefaults = (oparg & 0x02) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0))] : NULL; PyObject *defaults = (oparg & 0x01) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x01) ? 1 : 0))] : NULL; PyObject *func; + #line 3138 "Python/bytecodes.c" PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); @@ -3916,12 +4368,14 @@ func_obj->func_version = ((PyCodeObject *)codeobj)->co_version; func = (PyObject *)func_obj; + #line 4372 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 0x01) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x08) ? 1 : 0)); stack_pointer[-1] = func; DISPATCH(); } TARGET(RETURN_GENERATOR) { + #line 3169 "Python/bytecodes.c" assert(PyFunction_Check(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); @@ -3942,6 +4396,7 @@ frame = cframe.current_frame = prev; _PyFrame_StackPush(frame, (PyObject *)gen); goto resume_frame; + #line 4400 "Python/generated_cases.c.h" } TARGET(BUILD_SLICE) { @@ -3949,11 +4404,15 @@ PyObject *stop = stack_pointer[-(1 + ((oparg == 3) ? 1 : 0))]; PyObject *start = stack_pointer[-(2 + ((oparg == 3) ? 1 : 0))]; PyObject *slice; + #line 3192 "Python/bytecodes.c" slice = PySlice_New(start, stop, step); + #line 4410 "Python/generated_cases.c.h" Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); + #line 3194 "Python/bytecodes.c" if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } + #line 4416 "Python/generated_cases.c.h" STACK_SHRINK(((oparg == 3) ? 1 : 0)); STACK_SHRINK(1); stack_pointer[-1] = slice; @@ -3964,6 +4423,7 @@ PyObject *fmt_spec = ((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? stack_pointer[-((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))] : NULL; PyObject *value = stack_pointer[-(1 + (((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))]; PyObject *result; + #line 3198 "Python/bytecodes.c" /* Handles f-string value formatting. */ PyObject *(*conv_fn)(PyObject *); int which_conversion = oparg & FVC_MASK; @@ -3998,6 +4458,7 @@ Py_DECREF(value); Py_XDECREF(fmt_spec); if (result == NULL) { STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); goto pop_1_error; } + #line 4462 "Python/generated_cases.c.h" STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); stack_pointer[-1] = result; DISPATCH(); @@ -4006,8 +4467,10 @@ TARGET(COPY) { PyObject *bottom = stack_pointer[-(1 + (oparg-1))]; PyObject *top; + #line 3235 "Python/bytecodes.c" assert(oparg > 0); top = Py_NewRef(bottom); + #line 4474 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = top; DISPATCH(); @@ -4019,6 +4482,7 @@ PyObject *rhs = stack_pointer[-1]; PyObject *lhs = stack_pointer[-2]; PyObject *res; + #line 3240 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -4033,9 +4497,12 @@ assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); assert(binary_ops[oparg]); res = binary_ops[oparg](lhs, rhs); + #line 4501 "Python/generated_cases.c.h" Py_DECREF(lhs); Py_DECREF(rhs); + #line 3255 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; + #line 4506 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -4045,13 +4512,16 @@ TARGET(SWAP) { PyObject *top = stack_pointer[-1]; PyObject *bottom = stack_pointer[-(2 + (oparg-2))]; + #line 3260 "Python/bytecodes.c" assert(oparg >= 2); + #line 4518 "Python/generated_cases.c.h" stack_pointer[-1] = bottom; stack_pointer[-(2 + (oparg-2))] = top; DISPATCH(); } TARGET(INSTRUMENTED_LINE) { + #line 3264 "Python/bytecodes.c" _Py_CODEUNIT *here = next_instr-1; _PyFrame_SetStackPointer(frame, stack_pointer); int original_opcode = _Py_call_instrumentation_line( @@ -4071,9 +4541,11 @@ } opcode = original_opcode; DISPATCH_GOTO(); + #line 4545 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_INSTRUCTION) { + #line 3286 "Python/bytecodes.c" int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, next_instr-1); if (next_opcode < 0) goto error; @@ -4085,20 +4557,26 @@ assert(next_opcode > 0 && next_opcode < 256); opcode = next_opcode; DISPATCH_GOTO(); + #line 4561 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_JUMP_FORWARD) { + #line 3300 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); + #line 4567 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_JUMP_BACKWARD) { + #line 3304 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr-oparg, PY_MONITORING_EVENT_JUMP); + #line 4574 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { + #line 3309 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); Py_DECREF(cond); @@ -4107,10 +4585,12 @@ assert(err == 0 || err == 1); int offset = err*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + #line 4589 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { + #line 3320 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); Py_DECREF(cond); @@ -4119,10 +4599,12 @@ assert(err == 0 || err == 1); int offset = (1-err)*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + #line 4603 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { + #line 3331 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4135,10 +4617,12 @@ offset = 0; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + #line 4621 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { + #line 3346 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4151,23 +4635,30 @@ offset = oparg; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + #line 4639 "Python/generated_cases.c.h" DISPATCH(); } TARGET(EXTENDED_ARG) { + #line 3361 "Python/bytecodes.c" assert(oparg); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); + #line 4650 "Python/generated_cases.c.h" } TARGET(CACHE) { + #line 3369 "Python/bytecodes.c" assert(0 && "Executing a cache."); Py_UNREACHABLE(); + #line 4657 "Python/generated_cases.c.h" } TARGET(RESERVED) { + #line 3374 "Python/bytecodes.c" assert(0 && "Executing RESERVED instruction."); Py_UNREACHABLE(); + #line 4664 "Python/generated_cases.c.h" } From f1a94659953a8674e1a7533dd02338ceb102985a Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Sat, 29 Apr 2023 12:21:11 +0800 Subject: [PATCH 16/17] Update Python/bytecodes.c Co-authored-by: Carl Meyer --- Python/bytecodes.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 968e912545f1e6..e83894e8902872 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3114,7 +3114,7 @@ dummy_func( _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, (PyFunctionObject *)func, locals, nargs, callargs, kwargs); - // Need to manually shrinkg the stack since we exit with DISPATCH_INLINED. + // Need to manually shrink the stack since we exit with DISPATCH_INLINED. STACK_SHRINK(oparg + 3); if (new_frame == NULL) { goto error; From 94782c385e19183a8511334da97bb81ad5d9c0c1 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 29 Apr 2023 15:53:10 +0800 Subject: [PATCH 17/17] Update generated_cases.c.h --- Python/generated_cases.c.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 153a6325534ac2..069a7ced0a4c25 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -4307,7 +4307,7 @@ _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, (PyFunctionObject *)func, locals, nargs, callargs, kwargs); - // Need to manually shrinkg the stack since we exit with DISPATCH_INLINED. + // Need to manually shrink the stack since we exit with DISPATCH_INLINED. STACK_SHRINK(oparg + 3); if (new_frame == NULL) { goto error; pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy