From b19152888c39dcb29c7ff6cdc365476310f52436 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 21 Jan 2025 11:38:37 +0800 Subject: [PATCH 1/5] Move GO_TO_INSTRUCTION and PREDICT to cases generator --- Python/ceval_macros.h | 33 +-------------- Python/generated_cases.c.h | 49 +++++++++++----------- Tools/cases_generator/generators_common.py | 18 ++++++++ Tools/cases_generator/tier1_generator.py | 2 +- 4 files changed, 45 insertions(+), 57 deletions(-) diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index c37e1cf3afa60e..eb572e984ddb50 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -165,35 +165,6 @@ GETITEM(PyObject *v, Py_ssize_t i) { #define JUMPBY(x) (next_instr += (x)) #define SKIP_OVER(x) (next_instr += (x)) -/* OpCode prediction macros - Some opcodes tend to come in pairs thus making it possible to - predict the second code when the first is run. For example, - COMPARE_OP is often followed by POP_JUMP_IF_FALSE or POP_JUMP_IF_TRUE. - - Verifying the prediction costs a single high-speed test of a register - variable against a constant. If the pairing was good, then the - processor's own internal branch predication has a high likelihood of - success, resulting in a nearly zero-overhead transition to the - next opcode. A successful prediction saves a trip through the eval-loop - including its unpredictable switch-case branch. Combined with the - processor's internal branch prediction, a successful PREDICT has the - effect of making the two opcodes run as if they were a single new opcode - with the bodies combined. - - If collecting opcode statistics, your choices are to either keep the - predictions turned-on and interpret the results as if some opcodes - had been combined or turn-off predictions so that the opcode frequency - counter updates for both opcodes. - - Opcode prediction is disabled with threaded code, since the latter allows - the CPU to record separate branch prediction information for each - opcode. - -*/ - -#define PREDICT_ID(op) PRED_##op -#define PREDICTED(op) PREDICT_ID(op): - /* Stack manipulation macros */ @@ -259,8 +230,6 @@ GETITEM(PyObject *v, Py_ssize_t i) { GETLOCAL(i) = value; \ PyStackRef_XCLOSE(tmp); } while (0) -#define GO_TO_INSTRUCTION(op) goto PREDICT_ID(op) - #ifdef Py_STATS #define UPDATE_MISS_STATS(INSTNAME) \ do { \ @@ -280,7 +249,7 @@ GETITEM(PyObject *v, Py_ssize_t i) { /* This is only a single jump on release builds! */ \ UPDATE_MISS_STATS((INSTNAME)); \ assert(_PyOpcode_Deopt[opcode] == (INSTNAME)); \ - GO_TO_INSTRUCTION(INSTNAME); \ + goto PRED_##INSTNAME; \ } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index ec3397e1cf6e5f..661b6c5a270e83 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -13,7 +13,7 @@ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP); - PREDICTED(BINARY_OP); + PRED_BINARY_OP: _Py_CODEUNIT* const this_instr = next_instr - 6; (void)this_instr; _PyStackRef lhs; @@ -484,7 +484,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR); - PREDICTED(BINARY_SUBSCR); + PRED_BINARY_SUBSCR: _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef container; @@ -931,7 +931,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL); - PREDICTED(CALL); + PRED_CALL: _Py_CODEUNIT* const this_instr = next_instr - 4; (void)this_instr; _PyStackRef *callable; @@ -1707,7 +1707,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CALL_FUNCTION_EX); - PREDICTED(CALL_FUNCTION_EX); + PRED_CALL_FUNCTION_EX: _Py_CODEUNIT* const this_instr = next_instr - 1; (void)this_instr; _PyStackRef func; @@ -1960,7 +1960,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW); - PREDICTED(CALL_KW); + PRED_CALL_KW: _Py_CODEUNIT* const this_instr = next_instr - 4; (void)this_instr; _PyStackRef *callable; @@ -3299,7 +3299,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP); - PREDICTED(COMPARE_OP); + PRED_COMPARE_OP: _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef left; @@ -3479,7 +3479,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(CONTAINS_OP); - PREDICTED(CONTAINS_OP); + PRED_CONTAINS_OP: _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef left; @@ -4000,7 +4000,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER); - PREDICTED(FOR_ITER); + PRED_FOR_ITER: _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef iter; @@ -4631,7 +4631,8 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); - GO_TO_INSTRUCTION(CALL_FUNCTION_EX); + + goto PRED_CALL_FUNCTION_EX; } TARGET(INSTRUMENTED_CALL_KW) { @@ -4655,7 +4656,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (err) goto error; PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - GO_TO_INSTRUCTION(CALL_KW); + goto PRED_CALL_KW; } TARGET(INSTRUMENTED_END_FOR) { @@ -4845,7 +4846,7 @@ // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we // don't want to specialize instrumented instructions PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); + goto PRED_LOAD_SUPER_ATTR; } TARGET(INSTRUMENTED_LOAD_SUPER_METHOD) { @@ -4857,7 +4858,7 @@ // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we // don't want to specialize instrumented instructions PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - GO_TO_INSTRUCTION(LOAD_SUPER_METHOD); + goto PRED_LOAD_SUPER_METHOD; } TARGET(INSTRUMENTED_NOT_TAKEN) { @@ -5312,7 +5313,7 @@ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR); - PREDICTED(LOAD_ATTR); + PRED_LOAD_ATTR: _Py_CODEUNIT* const this_instr = next_instr - 10; (void)this_instr; _PyStackRef owner; @@ -5829,7 +5830,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_CONST); - PREDICTED(LOAD_CONST); + PRED_LOAD_CONST: _Py_CODEUNIT* const this_instr = next_instr - 1; (void)this_instr; _PyStackRef value; @@ -6072,7 +6073,7 @@ frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL); - PREDICTED(LOAD_GLOBAL); + PRED_LOAD_GLOBAL: _Py_CODEUNIT* const this_instr = next_instr - 5; (void)this_instr; _PyStackRef *res; @@ -6222,7 +6223,7 @@ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_METHOD); - PREDICTED(LOAD_METHOD); + PRED_LOAD_METHOD: _Py_CODEUNIT* const this_instr = next_instr - 10; (void)this_instr; _PyStackRef owner; @@ -6482,7 +6483,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR); - PREDICTED(LOAD_SUPER_ATTR); + PRED_LOAD_SUPER_ATTR: _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef global_super_st; @@ -6612,7 +6613,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_METHOD); - PREDICTED(LOAD_SUPER_METHOD); + PRED_LOAD_SUPER_METHOD: _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef global_super_st; @@ -7182,7 +7183,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESUME); - PREDICTED(RESUME); + PRED_RESUME: _Py_CODEUNIT* const this_instr = next_instr - 1; (void)this_instr; // _LOAD_BYTECODE @@ -7335,7 +7336,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(SEND); - PREDICTED(SEND); + PRED_SEND: _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef receiver; @@ -7576,7 +7577,7 @@ frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR); - PREDICTED(STORE_ATTR); + PRED_STORE_ATTR: _Py_CODEUNIT* const this_instr = next_instr - 5; (void)this_instr; _PyStackRef owner; @@ -7940,7 +7941,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR); - PREDICTED(STORE_SUBSCR); + PRED_STORE_SUBSCR: _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef container; @@ -8070,7 +8071,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL); - PREDICTED(TO_BOOL); + PRED_TO_BOOL: _Py_CODEUNIT* const this_instr = next_instr - 4; (void)this_instr; _PyStackRef value; @@ -8306,7 +8307,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE); - PREDICTED(UNPACK_SEQUENCE); + PRED_UNPACK_SEQUENCE: _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef seq; diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index c441569b7e70dc..71eec9a2d32667 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -127,6 +127,7 @@ def __init__(self, out: CWriter): "DISPATCH": self.dispatch, "INSTRUCTION_SIZE": self.instruction_size, "POP_INPUT": self.pop_input, + "GO_TO_INSTRUCTION": self.go_to_instruction, } self.out = out @@ -402,6 +403,23 @@ def sync_sp( self._print_storage(storage) return True + def go_to_instruction( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: Uop, + storage: Storage, + inst: Instruction | None, + ) -> bool: + next(tkn_iter) + name = next(tkn_iter) + next(tkn_iter) + next(tkn_iter) + assert name.kind == "IDENTIFIER" + self.emit("\n") + self.emit(f"goto PRED_{name.text};\n") + return True + def emit_save(self, storage: Storage) -> None: storage.save(self.out) self._print_storage(storage) diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 95876c387bd745..f3836cac9f8ad9 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -158,7 +158,7 @@ def generate_tier1( out.emit(f"next_instr += {inst.size};\n") out.emit(f"INSTRUCTION_STATS({name});\n") if inst.is_target: - out.emit(f"PREDICTED({name});\n") + out.emit(f"PRED_{name}:\n") if needs_this: out.emit(f"_Py_CODEUNIT* const this_instr = next_instr - {inst.size};\n") out.emit(unused_guard) From 47f973b649eb3d1a1b2038f473f242d9ab000d5c Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 21 Jan 2025 11:49:34 +0800 Subject: [PATCH 2/5] Try fix on macOS --- Python/generated_cases.c.h | 2284 +++++++++++----------- Tools/cases_generator/tier1_generator.py | 5 + 2 files changed, 1167 insertions(+), 1122 deletions(-) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 661b6c5a270e83..29572ee6a45fd7 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -14,48 +14,50 @@ next_instr += 6; INSTRUCTION_STATS(BINARY_OP); PRED_BINARY_OP: - _Py_CODEUNIT* const this_instr = next_instr - 6; - (void)this_instr; - _PyStackRef lhs; - _PyStackRef rhs; - _PyStackRef res; - // _SPECIALIZE_BINARY_OP - { - rhs = stack_pointer[-1]; - lhs = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; + { + _Py_CODEUNIT* const this_instr = next_instr - 6; + (void)this_instr; + _PyStackRef lhs; + _PyStackRef rhs; + _PyStackRef res; + // _SPECIALIZE_BINARY_OP + { + rhs = stack_pointer[-1]; + lhs = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(BINARY_OP); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + assert(NB_ADD <= oparg); + assert(oparg <= NB_INPLACE_XOR); + } + /* Skip 4 cache entries */ + // _BINARY_OP + { + PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs); + PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs); + assert(_PyEval_BinaryOps[oparg]); _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); + PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + PyStackRef_CLOSE(lhs); + PyStackRef_CLOSE(rhs); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); } - OPCODE_DEFERRED_INC(BINARY_OP); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - assert(NB_ADD <= oparg); - assert(oparg <= NB_INPLACE_XOR); - } - /* Skip 4 cache entries */ - // _BINARY_OP - { - PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs); - PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs); - assert(_PyEval_BinaryOps[oparg]); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(lhs); - PyStackRef_CLOSE(rhs); - if (res_o == NULL) goto pop_2_error; - res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -485,45 +487,47 @@ next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR); PRED_BINARY_SUBSCR: - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef container; - _PyStackRef sub; - _PyStackRef res; - // _SPECIALIZE_BINARY_SUBSCR { - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - assert(frame->stackpointer == NULL); - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef container; + _PyStackRef sub; + _PyStackRef res; + // _SPECIALIZE_BINARY_SUBSCR + { + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + assert(frame->stackpointer == NULL); + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_BinarySubscr(container, sub, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(BINARY_SUBSCR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _BINARY_SUBSCR + { + PyObject *container_o = PyStackRef_AsPyObjectBorrow(container); + PyObject *sub_o = PyStackRef_AsPyObjectBorrow(sub); _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_BinarySubscr(container, sub, next_instr); + PyObject *res_o = PyObject_GetItem(container_o, sub_o); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + PyStackRef_CLOSE(container); + PyStackRef_CLOSE(sub); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); } - OPCODE_DEFERRED_INC(BINARY_SUBSCR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _BINARY_SUBSCR - { - PyObject *container_o = PyStackRef_AsPyObjectBorrow(container); - PyObject *sub_o = PyStackRef_AsPyObjectBorrow(sub); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_GetItem(container_o, sub_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(container); - PyStackRef_CLOSE(sub); - if (res_o == NULL) goto pop_2_error; - res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -932,159 +936,161 @@ next_instr += 4; INSTRUCTION_STATS(CALL); PRED_CALL: - _Py_CODEUNIT* const this_instr = next_instr - 4; - (void)this_instr; - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef *func; - _PyStackRef *maybe_self; - _PyStackRef res; - // _SPECIALIZE_CALL - { - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_Call(callable[0], next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(CALL); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 2 cache entries */ - // _MAYBE_EXPAND_METHOD { - args = &stack_pointer[-oparg]; - func = &stack_pointer[-2 - oparg]; - maybe_self = &stack_pointer[-1 - oparg]; - if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - maybe_self[0] = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - _PyStackRef temp = callable[0]; - func[0] = PyStackRef_FromPyObjectNew(method); - PyStackRef_CLOSE(temp); + _Py_CODEUNIT* const this_instr = next_instr - 4; + (void)this_instr; + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef *func; + _PyStackRef *maybe_self; + _PyStackRef res; + // _SPECIALIZE_CALL + { + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_Call(callable[0], next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(CALL); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ } - } - // _DO_CALL - { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - // oparg counts all of the args, but *not* self: - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null[0])) { - arguments--; - total_args++; + /* Skip 2 cache entries */ + // _MAYBE_EXPAND_METHOD + { + args = &stack_pointer[-oparg]; + func = &stack_pointer[-2 - oparg]; + maybe_self = &stack_pointer[-1 - oparg]; + if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + maybe_self[0] = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable[0]; + func[0] = PyStackRef_FromPyObjectNew(method); + PyStackRef_CLOSE(temp); + } } - // Check if the call can be inlined or not - if (Py_TYPE(callable_o) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + // _DO_CALL { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null[0])) { + arguments--; + total_args++; + } + // Check if the call can be inlined or not + if (Py_TYPE(callable_o) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable[0], locals, + arguments, total_args, NULL, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + // Manipulate stack directly since we leave using DISPATCH_INLINED(). + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + if (new_frame == NULL) { + goto error; + } + frame->return_offset = 4 ; + DISPATCH_INLINED(new_frame); + } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, callable[0], locals, - arguments, total_args, NULL, frame - ); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); stack_pointer = _PyFrame_GetStackPointer(frame); - // Manipulate stack directly since we leave using DISPATCH_INLINED(). - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - if (new_frame == NULL) { - goto error; + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); + if (res_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + Py_CLEAR(res_o); + } + } } - frame->return_offset = 4 ; - DISPATCH_INLINED(new_frame); - } - /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); PyStackRef_CLOSE(callable[0]); PyStackRef_XCLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } - { + if (res_o == NULL) { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); goto error; } + res = PyStackRef_FromPyObjectSteal(res_o); } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - if (opcode == INSTRUMENTED_CALL) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); - if (res_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable_o, arg); + int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - Py_CLEAR(res_o); - } + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); } } - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - if (res_o == NULL) { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -1708,161 +1714,163 @@ next_instr += 1; INSTRUCTION_STATS(CALL_FUNCTION_EX); PRED_CALL_FUNCTION_EX: - _Py_CODEUNIT* const this_instr = next_instr - 1; - (void)this_instr; - _PyStackRef func; - _PyStackRef callargs; - _PyStackRef kwargs_in; - _PyStackRef tuple; - _PyStackRef kwargs_out; - _PyStackRef func_st; - _PyStackRef callargs_st; - _PyStackRef kwargs_st; - _PyStackRef result; - // _MAKE_CALLARGS_A_TUPLE { - kwargs_in = stack_pointer[-1]; - callargs = stack_pointer[-2]; - func = stack_pointer[-4]; - PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); - if (PyTuple_CheckExact(callargs_o)) { - tuple = callargs; - kwargs_out = kwargs_in; - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - goto error; - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *tuple_o = PySequence_Tuple(callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (tuple_o == NULL) { - goto error; - } - kwargs_out = kwargs_in; - PyStackRef_CLOSE(callargs); - tuple = PyStackRef_FromPyObjectSteal(tuple_o); - } - } - // _DO_CALL_FUNCTION_EX - { - kwargs_st = kwargs_out; - callargs_st = tuple; - func_st = func; - PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); - // DICT_MERGE is called before this opcode if there are kwargs. - // It converts all dict subtypes in kwargs into regular dicts. - EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); - PyObject *result_o; - assert(!_PyErr_Occurred(tstate)); - if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { - PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); - PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - assert(PyTuple_CheckExact(callargs)); - PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? - PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; - stack_pointer[-2] = callargs_st; - stack_pointer[-1] = kwargs_st; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - goto error; + _Py_CODEUNIT* const this_instr = next_instr - 1; + (void)this_instr; + _PyStackRef func; + _PyStackRef callargs; + _PyStackRef kwargs_in; + _PyStackRef tuple; + _PyStackRef kwargs_out; + _PyStackRef func_st; + _PyStackRef callargs_st; + _PyStackRef kwargs_st; + _PyStackRef result; + // _MAKE_CALLARGS_A_TUPLE + { + kwargs_in = stack_pointer[-1]; + callargs = stack_pointer[-2]; + func = stack_pointer[-4]; + PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); + if (PyTuple_CheckExact(callargs_o)) { + tuple = callargs; + kwargs_out = kwargs_in; } - _PyFrame_SetStackPointer(frame, stack_pointer); - result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!PyFunction_Check(func) && !PyMethod_Check(func)) { - if (result_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + goto error; } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - Py_CLEAR(result_o); - } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *tuple_o = PySequence_Tuple(callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (tuple_o == NULL) { + goto error; } + kwargs_out = kwargs_in; + PyStackRef_CLOSE(callargs); + tuple = PyStackRef_FromPyObjectSteal(tuple_o); } } - else { - if (Py_TYPE(func) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { - PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); - assert(PyTuple_CheckExact(callargs)); - PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); + // _DO_CALL_FUNCTION_EX + { + kwargs_st = kwargs_out; + callargs_st = tuple; + func_st = func; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + // DICT_MERGE is called before this opcode if there are kwargs. + // It converts all dict subtypes in kwargs into regular dicts. + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); + PyObject *result_o; + assert(!_PyErr_Occurred(tstate)); + if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); - int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + assert(PyTuple_CheckExact(callargs)); + PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? + PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; + stack_pointer[-2] = callargs_st; + stack_pointer[-1] = kwargs_st; _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( - tstate, func_st, locals, - nargs, callargs, kwargs, frame); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, func, arg); stack_pointer = _PyFrame_GetStackPointer(frame); - // Need to sync the stack since we exit with DISPATCH_INLINED. - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - if (new_frame == NULL) { + if (err) { goto error; } - assert( 1 == 1); - frame->return_offset = 1; - DISPATCH_INLINED(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + result_o = PyObject_Call(func, callargs, kwargs); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!PyFunction_Check(func) && !PyMethod_Check(func)) { + if (result_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + Py_CLEAR(result_o); + } + } + } + } + else { + if (Py_TYPE(func) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( + tstate, func_st, locals, + nargs, callargs, kwargs, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + // Need to sync the stack since we exit with DISPATCH_INLINED. + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + if (new_frame == NULL) { + goto error; + } + assert( 1 == 1); + frame->return_offset = 1; + DISPATCH_INLINED(new_frame); + } + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + stack_pointer[-2] = callargs_st; + stack_pointer[-1] = kwargs_st; + _PyFrame_SetStackPointer(frame, stack_pointer); + result_o = PyObject_Call(func, callargs, kwargs); + stack_pointer = _PyFrame_GetStackPointer(frame); } - PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); - assert(PyTuple_CheckExact(callargs)); - PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - stack_pointer[-2] = callargs_st; - stack_pointer[-1] = kwargs_st; _PyFrame_SetStackPointer(frame, stack_pointer); - result_o = PyObject_Call(func, callargs, kwargs); + PyStackRef_XCLOSE(kwargs_st); stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(callargs_st); + PyStackRef_CLOSE(func_st); + if (result_o == NULL) goto pop_4_error; + result = PyStackRef_FromPyObjectSteal(result_o); } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(kwargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(callargs_st); - PyStackRef_CLOSE(func_st); - if (result_o == NULL) goto pop_4_error; - result = PyStackRef_FromPyObjectSteal(result_o); - } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-4] = result; - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; - stack_pointer += 3; - assert(WITHIN_STACK_BOUNDS()); + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-4] = result; + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 3; + assert(WITHIN_STACK_BOUNDS()); + } } + stack_pointer[-4] = result; + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-4] = result; - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -1961,156 +1969,158 @@ next_instr += 4; INSTRUCTION_STATS(CALL_KW); PRED_CALL_KW: - _Py_CODEUNIT* const this_instr = next_instr - 4; - (void)this_instr; - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef kwnames; - _PyStackRef kwnames_in; - _PyStackRef *func; - _PyStackRef *maybe_self; - _PyStackRef kwnames_out; - _PyStackRef res; - // _SPECIALIZE_CALL_KW - { - self_or_null = &stack_pointer[-2 - oparg]; - callable = &stack_pointer[-3 - oparg]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_CallKw(callable[0], next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(CALL_KW); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 2 cache entries */ - // _MAYBE_EXPAND_METHOD_KW { - kwnames_in = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - func = &stack_pointer[-3 - oparg]; - maybe_self = &stack_pointer[-2 - oparg]; - if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - maybe_self[0] = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - _PyStackRef temp = callable[0]; - func[0] = PyStackRef_FromPyObjectNew(method); - PyStackRef_CLOSE(temp); + _Py_CODEUNIT* const this_instr = next_instr - 4; + (void)this_instr; + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef kwnames_in; + _PyStackRef *func; + _PyStackRef *maybe_self; + _PyStackRef kwnames_out; + _PyStackRef res; + // _SPECIALIZE_CALL_KW + { + self_or_null = &stack_pointer[-2 - oparg]; + callable = &stack_pointer[-3 - oparg]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_CallKw(callable[0], next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(CALL_KW); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ } - kwnames_out = kwnames_in; - } - // _DO_CALL_KW - { - kwnames = kwnames_out; - args = &stack_pointer[-1 - oparg]; - self_or_null = &stack_pointer[-2 - oparg]; - callable = &stack_pointer[-3 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - // oparg counts all of the args, but *not* self: - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null[0])) { - arguments--; - total_args++; + /* Skip 2 cache entries */ + // _MAYBE_EXPAND_METHOD_KW + { + kwnames_in = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + func = &stack_pointer[-3 - oparg]; + maybe_self = &stack_pointer[-2 - oparg]; + if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + maybe_self[0] = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable[0]; + func[0] = PyStackRef_FromPyObjectNew(method); + PyStackRef_CLOSE(temp); + } + kwnames_out = kwnames_in; } - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - // Check if the call can be inlined or not - if (Py_TYPE(callable_o) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + // _DO_CALL_KW { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + kwnames = kwnames_out; + args = &stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; + callable = &stack_pointer[-3 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null[0])) { + arguments--; + total_args++; + } + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + // Check if the call can be inlined or not + if (Py_TYPE(callable_o) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + stack_pointer[-1] = kwnames; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable[0], locals, + arguments, positional_args, kwnames_o, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(kwnames); + // Sync stack explicitly since we leave using DISPATCH_INLINED(). + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + if (new_frame == NULL) { + goto error; + } + assert( 4 == 1 + INLINE_CACHE_ENTRIES_CALL_KW); + frame->return_offset = 4 ; + DISPATCH_INLINED(new_frame); + } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + PyStackRef_CLOSE(kwnames); + { + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + } stack_pointer[-1] = kwnames; _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, callable[0], locals, - arguments, positional_args, kwnames_o, frame - ); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames_o); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(kwnames); - // Sync stack explicitly since we leave using DISPATCH_INLINED(). - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - if (new_frame == NULL) { - goto error; + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL_KW) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); + if (res_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + Py_CLEAR(res_o); + } + } } - assert( 4 == 1 + INLINE_CACHE_ENTRIES_CALL_KW); - frame->return_offset = 4 ; - DISPATCH_INLINED(new_frame); - } - /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); PyStackRef_XCLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } PyStackRef_CLOSE(kwnames); - { + if (res_o == NULL) { stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); goto error; } + res = PyStackRef_FromPyObjectSteal(res_o); } - stack_pointer[-1] = kwnames; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - if (opcode == INSTRUMENTED_CALL_KW) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); - if (res_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - Py_CLEAR(res_o); - } - } - } - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - PyStackRef_CLOSE(kwnames); - if (res_o == NULL) { - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-3 - oparg] = res; - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -3300,59 +3310,61 @@ next_instr += 2; INSTRUCTION_STATS(COMPARE_OP); PRED_COMPARE_OP: - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _SPECIALIZE_COMPARE_OP { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_CompareOp(left, right, next_instr, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _SPECIALIZE_COMPARE_OP + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_CompareOp(left, right, next_instr, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(COMPARE_OP); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ } - OPCODE_DEFERRED_INC(COMPARE_OP); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION */ - } - // _COMPARE_OP - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert((oparg >> 5) <= Py_GE); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); - if (res_o == NULL) goto pop_2_error; - if (oparg & 16) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + // _COMPARE_OP + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert((oparg >> 5) <= Py_GE); _PyFrame_SetStackPointer(frame, stack_pointer); - int res_bool = PyObject_IsTrue(res_o); - Py_DECREF(res_o); + PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_bool < 0) goto error; - res = res_bool ? PyStackRef_True : PyStackRef_False; - } - else { - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + if (res_o == NULL) goto pop_2_error; + if (oparg & 16) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res_bool = PyObject_IsTrue(res_o); + Py_DECREF(res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_bool < 0) goto error; + res = res_bool ? PyStackRef_True : PyStackRef_False; + } + else { + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + } } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -3480,44 +3492,46 @@ next_instr += 2; INSTRUCTION_STATS(CONTAINS_OP); PRED_CONTAINS_OP: - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef left; - _PyStackRef right; - _PyStackRef b; - // _SPECIALIZE_CONTAINS_OP { - right = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_ContainsOp(right, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(CONTAINS_OP); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _CONTAINS_OP - { - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = PySequence_Contains(right_o, left_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); - if (res < 0) goto pop_2_error; - b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + // _SPECIALIZE_CONTAINS_OP + { + right = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_ContainsOp(right, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(CONTAINS_OP); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _CONTAINS_OP + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = PySequence_Contains(right_o, left_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + if (res < 0) goto pop_2_error; + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + } + stack_pointer[-2] = b; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-2] = b; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4001,60 +4015,62 @@ next_instr += 2; INSTRUCTION_STATS(FOR_ITER); PRED_FOR_ITER: - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef iter; - _PyStackRef next; - // _SPECIALIZE_FOR_ITER { - iter = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_ForIter(iter, next_instr, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(FOR_ITER); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION */ - } - // _FOR_ITER - { - /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (next_o == NULL) { - if (_PyErr_Occurred(tstate)) { + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef iter; + _PyStackRef next; + // _SPECIALIZE_FOR_ITER + { + iter = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + _Py_Specialize_ForIter(iter, next_instr, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - if (!matches) { - goto error; + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(FOR_ITER); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + } + // _FOR_ITER + { + /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (next_o == NULL) { + if (_PyErr_Occurred(tstate)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!matches) { + goto error; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_MonitorRaise(tstate, frame, this_instr); + _PyErr_Clear(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_MonitorRaise(tstate, frame, this_instr); - _PyErr_Clear(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + /* iterator ended normally */ + assert(next_instr[oparg].op.code == END_FOR || + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + /* Jump forward oparg, then skip following END_FOR */ + JUMPBY(oparg + 1); + DISPATCH(); } - /* iterator ended normally */ - assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - /* Jump forward oparg, then skip following END_FOR */ - JUMPBY(oparg + 1); - DISPATCH(); + next = PyStackRef_FromPyObjectSteal(next_o); + // Common case: no jump, leave it to the code generator } - next = PyStackRef_FromPyObjectSteal(next_o); - // Common case: no jump, leave it to the code generator + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -5314,40 +5330,42 @@ next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR); PRED_LOAD_ATTR: - _Py_CODEUNIT* const this_instr = next_instr - 10; - (void)this_instr; - _PyStackRef owner; - _PyStackRef attr; - // _SPECIALIZE_LOAD_ATTR { - owner = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + _Py_CODEUNIT* const this_instr = next_instr - 10; + (void)this_instr; + _PyStackRef owner; + _PyStackRef attr; + // _SPECIALIZE_LOAD_ATTR + { + owner = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_LoadAttr(owner, next_instr, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(LOAD_ATTR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 8 cache entries */ + // _LOAD_ATTR + { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_LoadAttr(owner, next_instr, name); + PyObject *attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + PyStackRef_CLOSE(owner); + if (attr_o == NULL) goto pop_1_error; + attr = PyStackRef_FromPyObjectSteal(attr_o); } - OPCODE_DEFERRED_INC(LOAD_ATTR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ + stack_pointer[-1] = attr; } - /* Skip 8 cache entries */ - // _LOAD_ATTR - { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(owner); - if (attr_o == NULL) goto pop_1_error; - attr = PyStackRef_FromPyObjectSteal(attr_o); - } - stack_pointer[-1] = attr; DISPATCH(); } @@ -5831,21 +5849,23 @@ next_instr += 1; INSTRUCTION_STATS(LOAD_CONST); PRED_LOAD_CONST: - _Py_CODEUNIT* const this_instr = next_instr - 1; - (void)this_instr; - _PyStackRef value; - /* We can't do this in the bytecode compiler as - * marshalling can intern strings and make them immortal. */ - PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); - value = PyStackRef_FromPyObjectNew(obj); - #if ENABLE_SPECIALIZATION - if (this_instr->op.code == LOAD_CONST) { - this_instr->op.code = _Py_IsImmortal(obj) ? LOAD_CONST_IMMORTAL : LOAD_CONST_MORTAL; + { + _Py_CODEUNIT* const this_instr = next_instr - 1; + (void)this_instr; + _PyStackRef value; + /* We can't do this in the bytecode compiler as + * marshalling can intern strings and make them immortal. */ + PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); + value = PyStackRef_FromPyObjectNew(obj); + #if ENABLE_SPECIALIZATION + if (this_instr->op.code == LOAD_CONST) { + this_instr->op.code = _Py_IsImmortal(obj) ? LOAD_CONST_IMMORTAL : LOAD_CONST_MORTAL; + } + #endif + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - #endif - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -6074,40 +6094,42 @@ next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL); PRED_LOAD_GLOBAL: - _Py_CODEUNIT* const this_instr = next_instr - 5; - (void)this_instr; - _PyStackRef *res; - // _SPECIALIZE_LOAD_GLOBAL { - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + _Py_CODEUNIT* const this_instr = next_instr - 5; + (void)this_instr; + _PyStackRef *res; + // _SPECIALIZE_LOAD_GLOBAL + { + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(LOAD_GLOBAL); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 1 cache entry */ + /* Skip 1 cache entry */ + /* Skip 1 cache entry */ + // _LOAD_GLOBAL + { + res = &stack_pointer[0]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); + _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + if (PyStackRef_IsNull(*res)) goto error; } - OPCODE_DEFERRED_INC(LOAD_GLOBAL); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 1 cache entry */ - /* Skip 1 cache entry */ - /* Skip 1 cache entry */ - // _LOAD_GLOBAL - { - res = &stack_pointer[0]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (PyStackRef_IsNull(*res)) goto error; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -6224,64 +6246,66 @@ next_instr += 10; INSTRUCTION_STATS(LOAD_METHOD); PRED_LOAD_METHOD: - _Py_CODEUNIT* const this_instr = next_instr - 10; - (void)this_instr; - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self_or_null; - // _SPECIALIZE_LOAD_METHOD { - owner = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + _Py_CODEUNIT* const this_instr = next_instr - 10; + (void)this_instr; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self_or_null; + // _SPECIALIZE_LOAD_METHOD + { + owner = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_LoadMethod(owner, next_instr, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(LOAD_METHOD); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 8 cache entries */ + // _LOAD_METHOD + { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - next_instr = this_instr; + PyObject *attr_o; + /* Designed to work in tandem with CALL, pushes two values. */ + attr_o = NULL; _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_LoadMethod(owner, next_instr, name); + int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(LOAD_METHOD); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 8 cache entries */ - // _LOAD_METHOD - { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *attr_o; - /* Designed to work in tandem with CALL, pushes two values. */ - attr_o = NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); - int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (is_meth) { - /* We can bypass temporary bound method object. - meth is unbound method and obj is self. - meth | self | arg1 | ... | argN - */ - assert(attr_o != NULL); // No errors on this branch - self_or_null = owner; // Transfer ownership - } - else { - /* meth is not an unbound method (but a regular attr, or - something was returned by a descriptor protocol). Set - the second element of the stack to NULL, to signal - CALL that it's not a method call. - meth | NULL | arg1 | ... | argN - */ - PyStackRef_CLOSE(owner); - if (attr_o == NULL) goto pop_1_error; - self_or_null = PyStackRef_NULL; + if (is_meth) { + /* We can bypass temporary bound method object. + meth is unbound method and obj is self. + meth | self | arg1 | ... | argN + */ + assert(attr_o != NULL); // No errors on this branch + self_or_null = owner; // Transfer ownership + } + else { + /* meth is not an unbound method (but a regular attr, or + something was returned by a descriptor protocol). Set + the second element of the stack to NULL, to signal + CALL that it's not a method call. + meth | NULL | arg1 | ... | argN + */ + PyStackRef_CLOSE(owner); + if (attr_o == NULL) goto pop_1_error; + self_or_null = PyStackRef_NULL; + } + attr = PyStackRef_FromPyObjectSteal(attr_o); } - attr = PyStackRef_FromPyObjectSteal(attr_o); + stack_pointer[-1] = attr; + stack_pointer[0] = self_or_null; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-1] = attr; - stack_pointer[0] = self_or_null; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -6484,93 +6508,95 @@ next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR); PRED_LOAD_SUPER_ATTR: - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef global_super_st; - _PyStackRef class_st; - _PyStackRef self_st; - _PyStackRef attr; - // _SPECIALIZE_LOAD_SUPER_ATTR - { - class_st = stack_pointer[-2]; - global_super_st = stack_pointer[-3]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, 0); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(LOAD_SUPER_ATTR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _LOAD_SUPER_ATTR { - self_st = stack_pointer[-1]; - PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); - PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); - PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - if (opcode >= MIN_INSTRUMENTED_OPCODE) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); - PyStackRef_CLOSE(self_st); - goto pop_3_error; - } - } - // we make no attempt to optimize here; specializations should - // handle any case whose performance we care about - PyObject *stack[] = {class, self}; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (opcode >= MIN_INSTRUMENTED_OPCODE) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - if (super == NULL) { + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr; + // _SPECIALIZE_LOAD_SUPER_ATTR + { + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, global_super, arg); + _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, 0); stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); } - else { + OPCODE_DEFERRED_INC(LOAD_SUPER_ATTR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _LOAD_SUPER_ATTR + { + self_st = stack_pointer[-1]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + if (opcode >= MIN_INSTRUMENTED_OPCODE) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, + tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, global_super, arg); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - Py_CLEAR(super); + if (err) { + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + PyStackRef_CLOSE(self_st); + goto pop_3_error; + } + } + // we make no attempt to optimize here; specializations should + // handle any case whose performance we care about + PyObject *stack[] = {class, self}; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (opcode >= MIN_INSTRUMENTED_OPCODE) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + if (super == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + Py_CLEAR(super); + } } } + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + PyStackRef_CLOSE(self_st); + if (super == NULL) goto pop_3_error; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = PyObject_GetAttr(super, name); + Py_DECREF(super); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (attr_o == NULL) goto error; + attr = PyStackRef_FromPyObjectSteal(attr_o); } - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); - PyStackRef_CLOSE(self_st); - if (super == NULL) goto pop_3_error; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - stack_pointer += -3; + stack_pointer[0] = attr; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = PyObject_GetAttr(super, name); - Py_DECREF(super); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (attr_o == NULL) goto error; - attr = PyStackRef_FromPyObjectSteal(attr_o); } - stack_pointer[0] = attr; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -6614,99 +6640,101 @@ next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_METHOD); PRED_LOAD_SUPER_METHOD: - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef global_super_st; - _PyStackRef class_st; - _PyStackRef self_st; - _PyStackRef attr; - _PyStackRef null; - // _SPECIALIZE_LOAD_SUPER_METHOD - { - class_st = stack_pointer[-2]; - global_super_st = stack_pointer[-3]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, 1); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(LOAD_SUPER_METHOD); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _LOAD_SUPER_ATTR { - self_st = stack_pointer[-1]; - PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); - PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); - PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - if (opcode >= MIN_INSTRUMENTED_OPCODE) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); - PyStackRef_CLOSE(self_st); - goto pop_3_error; - } - } - // we make no attempt to optimize here; specializations should - // handle any case whose performance we care about - PyObject *stack[] = {class, self}; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (opcode >= MIN_INSTRUMENTED_OPCODE) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - if (super == NULL) { + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr; + _PyStackRef null; + // _SPECIALIZE_LOAD_SUPER_METHOD + { + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, global_super, arg); + _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, 1); stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); } - else { + OPCODE_DEFERRED_INC(LOAD_SUPER_METHOD); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _LOAD_SUPER_ATTR + { + self_st = stack_pointer[-1]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + if (opcode >= MIN_INSTRUMENTED_OPCODE) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, + tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, global_super, arg); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - Py_CLEAR(super); + if (err) { + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + PyStackRef_CLOSE(self_st); + goto pop_3_error; + } + } + // we make no attempt to optimize here; specializations should + // handle any case whose performance we care about + PyObject *stack[] = {class, self}; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (opcode >= MIN_INSTRUMENTED_OPCODE) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + if (super == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + Py_CLEAR(super); + } } } + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + PyStackRef_CLOSE(self_st); + if (super == NULL) goto pop_3_error; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = PyObject_GetAttr(super, name); + Py_DECREF(super); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (attr_o == NULL) goto error; + attr = PyStackRef_FromPyObjectSteal(attr_o); } - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); - PyStackRef_CLOSE(self_st); - if (super == NULL) goto pop_3_error; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - stack_pointer += -3; + // _PUSH_NULL + { + null = PyStackRef_NULL; + } + stack_pointer[0] = attr; + stack_pointer[1] = null; + stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = PyObject_GetAttr(super, name); - Py_DECREF(super); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (attr_o == NULL) goto error; - attr = PyStackRef_FromPyObjectSteal(attr_o); - } - // _PUSH_NULL - { - null = PyStackRef_NULL; } - stack_pointer[0] = attr; - stack_pointer[1] = null; - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -7184,65 +7212,67 @@ next_instr += 1; INSTRUCTION_STATS(RESUME); PRED_RESUME: - _Py_CODEUNIT* const this_instr = next_instr - 1; - (void)this_instr; - // _LOAD_BYTECODE - { - #ifdef Py_GIL_DISABLED - if (frame->tlbc_index != - ((_PyThreadStateImpl *)tstate)->tlbc_index) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_CODEUNIT *bytecode = - _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (bytecode == NULL) goto error; - _PyFrame_SetStackPointer(frame, stack_pointer); - ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); - stack_pointer = _PyFrame_GetStackPointer(frame); - frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; - frame->instr_ptr = bytecode + off; - // Make sure this_instr gets reset correctley for any uops that - // follow - next_instr = frame->instr_ptr; - DISPATCH(); - } - #endif - } - // _MAYBE_INSTRUMENT { - if (tstate->tracing == 0) { - uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); - if (code_version != global_version) { + _Py_CODEUNIT* const this_instr = next_instr - 1; + (void)this_instr; + // _LOAD_BYTECODE + { + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + _Py_CODEUNIT *bytecode = + _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - goto error; - } - next_instr = this_instr; + if (bytecode == NULL) goto error; + _PyFrame_SetStackPointer(frame, stack_pointer); + ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; + frame->instr_ptr = bytecode + off; + // Make sure this_instr gets reset correctley for any uops that + // follow + next_instr = frame->instr_ptr; DISPATCH(); } + #endif } - } - // _QUICKEN_RESUME - { - #if ENABLE_SPECIALIZATION_FT - if (tstate->tracing == 0 && this_instr->op.code == RESUME) { - FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); + // _MAYBE_INSTRUMENT + { + if (tstate->tracing == 0) { + uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + if (code_version != global_version) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + goto error; + } + next_instr = this_instr; + DISPATCH(); + } + } } - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _CHECK_PERIODIC_IF_NOT_YIELD_FROM - { - if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); \ - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; + // _QUICKEN_RESUME + { + #if ENABLE_SPECIALIZATION_FT + if (tstate->tracing == 0 && this_instr->op.code == RESUME) { + FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); + } + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _CHECK_PERIODIC_IF_NOT_YIELD_FROM + { + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); \ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + } } } } @@ -7337,88 +7367,90 @@ next_instr += 2; INSTRUCTION_STATS(SEND); PRED_SEND: - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef receiver; - _PyStackRef v; - _PyStackRef retval; - // _SPECIALIZE_SEND { - receiver = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_Send(receiver, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(SEND); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _SEND - { - v = stack_pointer[-1]; - PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); - PyObject *retval_o; - assert(frame != &entry_frame); - if ((tstate->interp->eval_frame == NULL) && - (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) && - ((PyGenObject *)receiver_o)->gi_frame_state < FRAME_EXECUTING) + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef receiver; + _PyStackRef v; + _PyStackRef retval; + // _SPECIALIZE_SEND { - PyGenObject *gen = (PyGenObject *)receiver_o; - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - STACK_SHRINK(1); - _PyFrame_StackPush(gen_frame, v); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - assert( 2 + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)( 2 + oparg); - assert(gen_frame->previous == NULL); - gen_frame->previous = frame; - DISPATCH_INLINED(gen_frame); - } - if (PyStackRef_IsNone(v) && PyIter_Check(receiver_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - retval_o = PyObject_CallMethodOneArg(receiver_o, - &_Py_ID(send), - PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - if (retval_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { + receiver = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_MonitorRaise(tstate, frame, this_instr); + _Py_Specialize_Send(receiver, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); } - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyGen_FetchStopIterationValue(&retval_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err == 0) { - assert(retval_o != NULL); - JUMPBY(oparg); + OPCODE_DEFERRED_INC(SEND); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _SEND + { + v = stack_pointer[-1]; + PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); + PyObject *retval_o; + assert(frame != &entry_frame); + if ((tstate->interp->eval_frame == NULL) && + (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) && + ((PyGenObject *)receiver_o)->gi_frame_state < FRAME_EXECUTING) + { + PyGenObject *gen = (PyGenObject *)receiver_o; + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + STACK_SHRINK(1); + _PyFrame_StackPush(gen_frame, v); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert( 2 + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)( 2 + oparg); + assert(gen_frame->previous == NULL); + gen_frame->previous = frame; + DISPATCH_INLINED(gen_frame); + } + if (PyStackRef_IsNone(v) && PyIter_Check(receiver_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o); + stack_pointer = _PyFrame_GetStackPointer(frame); } else { - PyStackRef_CLOSE(v); - goto pop_1_error; + _PyFrame_SetStackPointer(frame, stack_pointer); + retval_o = PyObject_CallMethodOneArg(receiver_o, + &_Py_ID(send), + PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); } + if (retval_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_MonitorRaise(tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyGen_FetchStopIterationValue(&retval_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err == 0) { + assert(retval_o != NULL); + JUMPBY(oparg); + } + else { + PyStackRef_CLOSE(v); + goto pop_1_error; + } + } + PyStackRef_CLOSE(v); + retval = PyStackRef_FromPyObjectSteal(retval_o); } - PyStackRef_CLOSE(v); - retval = PyStackRef_FromPyObjectSteal(retval_o); + stack_pointer[-1] = retval; } - stack_pointer[-1] = retval; DISPATCH(); } @@ -7578,43 +7610,45 @@ next_instr += 5; INSTRUCTION_STATS(STORE_ATTR); PRED_STORE_ATTR: - _Py_CODEUNIT* const this_instr = next_instr - 5; - (void)this_instr; - _PyStackRef owner; - _PyStackRef v; - // _SPECIALIZE_STORE_ATTR { - owner = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + _Py_CODEUNIT* const this_instr = next_instr - 5; + (void)this_instr; + _PyStackRef owner; + _PyStackRef v; + // _SPECIALIZE_STORE_ATTR + { + owner = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_StoreAttr(owner, next_instr, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(STORE_ATTR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 3 cache entries */ + // _STORE_ATTR + { + v = stack_pointer[-2]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_StoreAttr(owner, next_instr, name); + int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), + name, PyStackRef_AsPyObjectBorrow(v)); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + PyStackRef_CLOSE(v); + PyStackRef_CLOSE(owner); + if (err) goto pop_2_error; } - OPCODE_DEFERRED_INC(STORE_ATTR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 3 cache entries */ - // _STORE_ATTR - { - v = stack_pointer[-2]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), - name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(v); - PyStackRef_CLOSE(owner); - if (err) goto pop_2_error; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -7942,43 +7976,45 @@ next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR); PRED_STORE_SUBSCR: - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef container; - _PyStackRef sub; - _PyStackRef v; - // _SPECIALIZE_STORE_SUBSCR { - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef container; + _PyStackRef sub; + _PyStackRef v; + // _SPECIALIZE_STORE_SUBSCR + { + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_StoreSubscr(container, sub, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(STORE_SUBSCR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _STORE_SUBSCR + { + v = stack_pointer[-3]; + /* container[sub] = v */ _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_StoreSubscr(container, sub, next_instr); + int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + PyStackRef_CLOSE(v); + PyStackRef_CLOSE(container); + PyStackRef_CLOSE(sub); + if (err) goto pop_3_error; } - OPCODE_DEFERRED_INC(STORE_SUBSCR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _STORE_SUBSCR - { - v = stack_pointer[-3]; - /* container[sub] = v */ - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(v); - PyStackRef_CLOSE(container); - PyStackRef_CLOSE(sub); - if (err) goto pop_3_error; + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -8072,38 +8108,40 @@ next_instr += 4; INSTRUCTION_STATS(TO_BOOL); PRED_TO_BOOL: - _Py_CODEUNIT* const this_instr = next_instr - 4; - (void)this_instr; - _PyStackRef value; - _PyStackRef res; - // _SPECIALIZE_TO_BOOL { - value = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; + _Py_CODEUNIT* const this_instr = next_instr - 4; + (void)this_instr; + _PyStackRef value; + _PyStackRef res; + // _SPECIALIZE_TO_BOOL + { + value = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_ToBool(value, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(TO_BOOL); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 2 cache entries */ + // _TO_BOOL + { _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_ToBool(value, next_instr); + int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + PyStackRef_CLOSE(value); + if (err < 0) goto pop_1_error; + res = err ? PyStackRef_True : PyStackRef_False; } - OPCODE_DEFERRED_INC(TO_BOOL); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ + stack_pointer[-1] = res; } - /* Skip 2 cache entries */ - // _TO_BOOL - { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(value); - if (err < 0) goto pop_1_error; - res = err ? PyStackRef_True : PyStackRef_False; - } - stack_pointer[-1] = res; DISPATCH(); } @@ -8308,41 +8346,43 @@ next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE); PRED_UNPACK_SEQUENCE: - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef seq; - _PyStackRef *output; - // _SPECIALIZE_UNPACK_SEQUENCE - { - seq = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; + { + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef seq; + _PyStackRef *output; + // _SPECIALIZE_UNPACK_SEQUENCE + { + seq = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_UnpackSequence(seq, next_instr, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(UNPACK_SEQUENCE); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + (void)seq; + (void)counter; + } + // _UNPACK_SEQUENCE + { + output = &stack_pointer[-1]; + _PyStackRef *top = output + oparg; _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_UnpackSequence(seq, next_instr, oparg); + int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg, -1, top); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + PyStackRef_CLOSE(seq); + if (res == 0) goto pop_1_error; } - OPCODE_DEFERRED_INC(UNPACK_SEQUENCE); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - (void)seq; - (void)counter; - } - // _UNPACK_SEQUENCE - { - output = &stack_pointer[-1]; - _PyStackRef *top = output + oparg; - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg, -1, top); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(seq); - if (res == 0) goto pop_1_error; + stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer += -1 + oparg; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index f3836cac9f8ad9..8290a09eb287c7 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -159,6 +159,9 @@ def generate_tier1( out.emit(f"INSTRUCTION_STATS({name});\n") if inst.is_target: out.emit(f"PRED_{name}:\n") + # We need to wrap the code in a block after the label so that declarations are allowed. + # This is a limitation of some compilers. + out.emit("{\n") if needs_this: out.emit(f"_Py_CODEUNIT* const this_instr = next_instr - {inst.size};\n") out.emit(unused_guard) @@ -177,6 +180,8 @@ def generate_tier1( out.start_line() stack.flush(out) + if inst.is_target: + out.emit("}\n") # PRED_{ID}: if not inst.parts[-1].properties.always_exits: out.emit("DISPATCH();\n") out.start_line() From c2c68de0ef2aa11798dff553a318f1e7513b40b7 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 21 Jan 2025 11:57:06 +0800 Subject: [PATCH 3/5] Update test_generated_cases.py --- Lib/test/test_generated_cases.py | 68 +++++++++++++++++--------------- 1 file changed, 36 insertions(+), 32 deletions(-) diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 4b17bd5c797375..0a2f43b49ac7b0 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -412,10 +412,12 @@ def test_predictions(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP1); - PREDICTED(OP1); - _PyStackRef res; - res = Py_None; - stack_pointer[-1] = res; + PRED_OP1: + { + _PyStackRef res; + res = Py_None; + stack_pointer[-1] = res; + } DISPATCH(); } @@ -646,36 +648,38 @@ def test_macro_instruction(self): frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(OP); - PREDICTED(OP); - _Py_CODEUNIT* const this_instr = next_instr - 6; - (void)this_instr; - _PyStackRef left; - _PyStackRef right; - _PyStackRef arg2; - _PyStackRef res; - // _OP1 + PRED_OP: { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - _PyFrame_SetStackPointer(frame, stack_pointer); - op1(left, right); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - /* Skip 2 cache entries */ - // OP2 - { - arg2 = stack_pointer[-3]; - uint32_t extra = read_u32(&this_instr[4].cache); - (void)extra; - _PyFrame_SetStackPointer(frame, stack_pointer); - res = op2(arg2, left, right); - stack_pointer = _PyFrame_GetStackPointer(frame); + _Py_CODEUNIT* const this_instr = next_instr - 6; + (void)this_instr; + _PyStackRef left; + _PyStackRef right; + _PyStackRef arg2; + _PyStackRef res; + // _OP1 + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + _PyFrame_SetStackPointer(frame, stack_pointer); + op1(left, right); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + /* Skip 2 cache entries */ + // OP2 + { + arg2 = stack_pointer[-3]; + uint32_t extra = read_u32(&this_instr[4].cache); + (void)extra; + _PyFrame_SetStackPointer(frame, stack_pointer); + res = op2(arg2, left, right); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-3] = res; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } From 36408f29bae511da49fa059fd3349e4035a8832e Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 21 Jan 2025 18:50:58 +0800 Subject: [PATCH 4/5] Address review --- Lib/test/test_generated_cases.py | 68 +- Python/ceval_macros.h | 2 +- Python/generated_cases.c.h | 2350 ++++++++++---------- Tools/cases_generator/generators_common.py | 2 +- Tools/cases_generator/tier1_generator.py | 7 +- 5 files changed, 1190 insertions(+), 1239 deletions(-) diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 0a2f43b49ac7b0..c93d6fc3d237a2 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -412,12 +412,10 @@ def test_predictions(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP1); - PRED_OP1: - { - _PyStackRef res; - res = Py_None; - stack_pointer[-1] = res; - } + PREDICTED_OP1:; + _PyStackRef res; + res = Py_None; + stack_pointer[-1] = res; DISPATCH(); } @@ -648,38 +646,36 @@ def test_macro_instruction(self): frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(OP); - PRED_OP: + PREDICTED_OP:; + _Py_CODEUNIT* const this_instr = next_instr - 6; + (void)this_instr; + _PyStackRef left; + _PyStackRef right; + _PyStackRef arg2; + _PyStackRef res; + // _OP1 { - _Py_CODEUNIT* const this_instr = next_instr - 6; - (void)this_instr; - _PyStackRef left; - _PyStackRef right; - _PyStackRef arg2; - _PyStackRef res; - // _OP1 - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - _PyFrame_SetStackPointer(frame, stack_pointer); - op1(left, right); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - /* Skip 2 cache entries */ - // OP2 - { - arg2 = stack_pointer[-3]; - uint32_t extra = read_u32(&this_instr[4].cache); - (void)extra; - _PyFrame_SetStackPointer(frame, stack_pointer); - res = op2(arg2, left, right); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - stack_pointer[-3] = res; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + right = stack_pointer[-1]; + left = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + _PyFrame_SetStackPointer(frame, stack_pointer); + op1(left, right); + stack_pointer = _PyFrame_GetStackPointer(frame); } + /* Skip 2 cache entries */ + // OP2 + { + arg2 = stack_pointer[-3]; + uint32_t extra = read_u32(&this_instr[4].cache); + (void)extra; + _PyFrame_SetStackPointer(frame, stack_pointer); + res = op2(arg2, left, right); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index eb572e984ddb50..a1dde9047fe368 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -249,7 +249,7 @@ GETITEM(PyObject *v, Py_ssize_t i) { /* This is only a single jump on release builds! */ \ UPDATE_MISS_STATS((INSTNAME)); \ assert(_PyOpcode_Deopt[opcode] == (INSTNAME)); \ - goto PRED_##INSTNAME; \ + goto PREDICTED_##INSTNAME; \ } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 29572ee6a45fd7..4eba3b297e24af 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -13,51 +13,49 @@ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP); - PRED_BINARY_OP: - { - _Py_CODEUNIT* const this_instr = next_instr - 6; - (void)this_instr; - _PyStackRef lhs; - _PyStackRef rhs; - _PyStackRef res; - // _SPECIALIZE_BINARY_OP - { - rhs = stack_pointer[-1]; - lhs = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(BINARY_OP); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - assert(NB_ADD <= oparg); - assert(oparg <= NB_INPLACE_XOR); - } - /* Skip 4 cache entries */ - // _BINARY_OP - { - PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs); - PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs); - assert(_PyEval_BinaryOps[oparg]); + PREDICTED_BINARY_OP:; + _Py_CODEUNIT* const this_instr = next_instr - 6; + (void)this_instr; + _PyStackRef lhs; + _PyStackRef rhs; + _PyStackRef res; + // _SPECIALIZE_BINARY_OP + { + rhs = stack_pointer[-1]; + lhs = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); + _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(lhs); - PyStackRef_CLOSE(rhs); - if (res_o == NULL) goto pop_2_error; - res = PyStackRef_FromPyObjectSteal(res_o); + DISPATCH_SAME_OPARG(); } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + OPCODE_DEFERRED_INC(BINARY_OP); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + assert(NB_ADD <= oparg); + assert(oparg <= NB_INPLACE_XOR); + } + /* Skip 4 cache entries */ + // _BINARY_OP + { + PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs); + PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs); + assert(_PyEval_BinaryOps[oparg]); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(lhs); + PyStackRef_CLOSE(rhs); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -486,48 +484,46 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR); - PRED_BINARY_SUBSCR: - { - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef container; - _PyStackRef sub; - _PyStackRef res; - // _SPECIALIZE_BINARY_SUBSCR - { - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - assert(frame->stackpointer == NULL); - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_BinarySubscr(container, sub, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(BINARY_SUBSCR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _BINARY_SUBSCR - { - PyObject *container_o = PyStackRef_AsPyObjectBorrow(container); - PyObject *sub_o = PyStackRef_AsPyObjectBorrow(sub); + PREDICTED_BINARY_SUBSCR:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef container; + _PyStackRef sub; + _PyStackRef res; + // _SPECIALIZE_BINARY_SUBSCR + { + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + assert(frame->stackpointer == NULL); + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_GetItem(container_o, sub_o); + _Py_Specialize_BinarySubscr(container, sub, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(container); - PyStackRef_CLOSE(sub); - if (res_o == NULL) goto pop_2_error; - res = PyStackRef_FromPyObjectSteal(res_o); + DISPATCH_SAME_OPARG(); } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + OPCODE_DEFERRED_INC(BINARY_SUBSCR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ } + // _BINARY_SUBSCR + { + PyObject *container_o = PyStackRef_AsPyObjectBorrow(container); + PyObject *sub_o = PyStackRef_AsPyObjectBorrow(sub); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_GetItem(container_o, sub_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(container); + PyStackRef_CLOSE(sub); + if (res_o == NULL) goto pop_2_error; + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -935,162 +931,160 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL); - PRED_CALL: - { - _Py_CODEUNIT* const this_instr = next_instr - 4; - (void)this_instr; - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef *func; - _PyStackRef *maybe_self; - _PyStackRef res; - // _SPECIALIZE_CALL - { - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_Call(callable[0], next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(CALL); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ + PREDICTED_CALL:; + _Py_CODEUNIT* const this_instr = next_instr - 4; + (void)this_instr; + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef *func; + _PyStackRef *maybe_self; + _PyStackRef res; + // _SPECIALIZE_CALL + { + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_Call(callable[0], next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); } - /* Skip 2 cache entries */ - // _MAYBE_EXPAND_METHOD - { - args = &stack_pointer[-oparg]; - func = &stack_pointer[-2 - oparg]; - maybe_self = &stack_pointer[-1 - oparg]; - if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - maybe_self[0] = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - _PyStackRef temp = callable[0]; - func[0] = PyStackRef_FromPyObjectNew(method); - PyStackRef_CLOSE(temp); - } + OPCODE_DEFERRED_INC(CALL); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 2 cache entries */ + // _MAYBE_EXPAND_METHOD + { + args = &stack_pointer[-oparg]; + func = &stack_pointer[-2 - oparg]; + maybe_self = &stack_pointer[-1 - oparg]; + if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + maybe_self[0] = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable[0]; + func[0] = PyStackRef_FromPyObjectNew(method); + PyStackRef_CLOSE(temp); + } + } + // _DO_CALL + { + args = &stack_pointer[-oparg]; + self_or_null = &stack_pointer[-1 - oparg]; + callable = &stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null[0])) { + arguments--; + total_args++; } - // _DO_CALL + // Check if the call can be inlined or not + if (Py_TYPE(callable_o) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) { - args = &stack_pointer[-oparg]; - self_or_null = &stack_pointer[-1 - oparg]; - callable = &stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - // oparg counts all of the args, but *not* self: - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null[0])) { - arguments--; - total_args++; - } - // Check if the call can be inlined or not - if (Py_TYPE(callable_o) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) - { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, callable[0], locals, - arguments, total_args, NULL, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - // Manipulate stack directly since we leave using DISPATCH_INLINED(). - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - if (new_frame == NULL) { - goto error; - } - frame->return_offset = 4 ; - DISPATCH_INLINED(new_frame); - } - /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - { - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - } + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable[0], locals, + arguments, total_args, NULL, frame + ); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - if (opcode == INSTRUMENTED_CALL) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); - if (res_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - Py_CLEAR(res_o); - } - } + // Manipulate stack directly since we leave using DISPATCH_INLINED(). + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + if (new_frame == NULL) { + goto error; } - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + frame->return_offset = 4 ; + DISPATCH_INLINED(new_frame); + } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); PyStackRef_XCLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } - if (res_o == NULL) { + { stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); goto error; } - res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); + if (res_o == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; - stack_pointer += 1 + oparg; - assert(WITHIN_STACK_BOUNDS()); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + Py_CLEAR(res_o); + } } } - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + if (res_o == NULL) { + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + } } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -1713,164 +1707,162 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CALL_FUNCTION_EX); - PRED_CALL_FUNCTION_EX: - { - _Py_CODEUNIT* const this_instr = next_instr - 1; - (void)this_instr; - _PyStackRef func; - _PyStackRef callargs; - _PyStackRef kwargs_in; - _PyStackRef tuple; - _PyStackRef kwargs_out; - _PyStackRef func_st; - _PyStackRef callargs_st; - _PyStackRef kwargs_st; - _PyStackRef result; - // _MAKE_CALLARGS_A_TUPLE - { - kwargs_in = stack_pointer[-1]; - callargs = stack_pointer[-2]; - func = stack_pointer[-4]; - PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); - if (PyTuple_CheckExact(callargs_o)) { - tuple = callargs; - kwargs_out = kwargs_in; - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - goto error; - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *tuple_o = PySequence_Tuple(callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (tuple_o == NULL) { - goto error; - } - kwargs_out = kwargs_in; - PyStackRef_CLOSE(callargs); - tuple = PyStackRef_FromPyObjectSteal(tuple_o); - } + PREDICTED_CALL_FUNCTION_EX:; + _Py_CODEUNIT* const this_instr = next_instr - 1; + (void)this_instr; + _PyStackRef func; + _PyStackRef callargs; + _PyStackRef kwargs_in; + _PyStackRef tuple; + _PyStackRef kwargs_out; + _PyStackRef func_st; + _PyStackRef callargs_st; + _PyStackRef kwargs_st; + _PyStackRef result; + // _MAKE_CALLARGS_A_TUPLE + { + kwargs_in = stack_pointer[-1]; + callargs = stack_pointer[-2]; + func = stack_pointer[-4]; + PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); + if (PyTuple_CheckExact(callargs_o)) { + tuple = callargs; + kwargs_out = kwargs_in; } - // _DO_CALL_FUNCTION_EX - { - kwargs_st = kwargs_out; - callargs_st = tuple; - func_st = func; - PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); - // DICT_MERGE is called before this opcode if there are kwargs. - // It converts all dict subtypes in kwargs into regular dicts. - EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); - PyObject *result_o; - assert(!_PyErr_Occurred(tstate)); - if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { - PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); - PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - assert(PyTuple_CheckExact(callargs)); - PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? - PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; - stack_pointer[-2] = callargs_st; - stack_pointer[-1] = kwargs_st; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - goto error; - } - _PyFrame_SetStackPointer(frame, stack_pointer); - result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!PyFunction_Check(func) && !PyMethod_Check(func)) { - if (result_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - Py_CLEAR(result_o); - } - } - } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + goto error; } - else { - if (Py_TYPE(func) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { - PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); - assert(PyTuple_CheckExact(callargs)); - PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); - int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( - tstate, func_st, locals, - nargs, callargs, kwargs, frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *tuple_o = PySequence_Tuple(callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (tuple_o == NULL) { + goto error; + } + kwargs_out = kwargs_in; + PyStackRef_CLOSE(callargs); + tuple = PyStackRef_FromPyObjectSteal(tuple_o); + } + } + // _DO_CALL_FUNCTION_EX + { + kwargs_st = kwargs_out; + callargs_st = tuple; + func_st = func; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + // DICT_MERGE is called before this opcode if there are kwargs. + // It converts all dict subtypes in kwargs into regular dicts. + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); + PyObject *result_o; + assert(!_PyErr_Occurred(tstate)); + if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + assert(PyTuple_CheckExact(callargs)); + PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? + PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; + stack_pointer[-2] = callargs_st; + stack_pointer[-1] = kwargs_st; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + goto error; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + result_o = PyObject_Call(func, callargs, kwargs); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!PyFunction_Check(func) && !PyMethod_Check(func)) { + if (result_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, func, arg); stack_pointer = _PyFrame_GetStackPointer(frame); - // Need to sync the stack since we exit with DISPATCH_INLINED. - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - if (new_frame == NULL) { - goto error; + if (err < 0) { + Py_CLEAR(result_o); } - assert( 1 == 1); - frame->return_offset = 1; - DISPATCH_INLINED(new_frame); } - PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); - assert(PyTuple_CheckExact(callargs)); - PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - stack_pointer[-2] = callargs_st; - stack_pointer[-1] = kwargs_st; - _PyFrame_SetStackPointer(frame, stack_pointer); - result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(kwargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(callargs_st); - PyStackRef_CLOSE(func_st); - if (result_o == NULL) goto pop_4_error; - result = PyStackRef_FromPyObjectSteal(result_o); } - // _CHECK_PERIODIC - { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[-4] = result; + else { + if (Py_TYPE(func) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( + tstate, func_st, locals, + nargs, callargs, kwargs, frame); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; - stack_pointer += 3; + // Need to sync the stack since we exit with DISPATCH_INLINED. + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + if (new_frame == NULL) { + goto error; + } + assert( 1 == 1); + frame->return_offset = 1; + DISPATCH_INLINED(new_frame); } + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + stack_pointer[-2] = callargs_st; + stack_pointer[-1] = kwargs_st; + _PyFrame_SetStackPointer(frame, stack_pointer); + result_o = PyObject_Call(func, callargs, kwargs); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(kwargs_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(callargs_st); + PyStackRef_CLOSE(func_st); + if (result_o == NULL) goto pop_4_error; + result = PyStackRef_FromPyObjectSteal(result_o); + } + // _CHECK_PERIODIC + { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + stack_pointer[-4] = result; + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; + stack_pointer += 3; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[-4] = result; - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); } + stack_pointer[-4] = result; + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -1968,159 +1960,157 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW); - PRED_CALL_KW: - { - _Py_CODEUNIT* const this_instr = next_instr - 4; - (void)this_instr; - _PyStackRef *callable; - _PyStackRef *self_or_null; - _PyStackRef *args; - _PyStackRef kwnames; - _PyStackRef kwnames_in; - _PyStackRef *func; - _PyStackRef *maybe_self; - _PyStackRef kwnames_out; - _PyStackRef res; - // _SPECIALIZE_CALL_KW - { - self_or_null = &stack_pointer[-2 - oparg]; - callable = &stack_pointer[-3 - oparg]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_CallKw(callable[0], next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(CALL_KW); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ + PREDICTED_CALL_KW:; + _Py_CODEUNIT* const this_instr = next_instr - 4; + (void)this_instr; + _PyStackRef *callable; + _PyStackRef *self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef kwnames_in; + _PyStackRef *func; + _PyStackRef *maybe_self; + _PyStackRef kwnames_out; + _PyStackRef res; + // _SPECIALIZE_CALL_KW + { + self_or_null = &stack_pointer[-2 - oparg]; + callable = &stack_pointer[-3 - oparg]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_CallKw(callable[0], next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); } - /* Skip 2 cache entries */ - // _MAYBE_EXPAND_METHOD_KW - { - kwnames_in = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - func = &stack_pointer[-3 - oparg]; - maybe_self = &stack_pointer[-2 - oparg]; - if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - maybe_self[0] = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - _PyStackRef temp = callable[0]; - func[0] = PyStackRef_FromPyObjectNew(method); - PyStackRef_CLOSE(temp); - } - kwnames_out = kwnames_in; + OPCODE_DEFERRED_INC(CALL_KW); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 2 cache entries */ + // _MAYBE_EXPAND_METHOD_KW + { + kwnames_in = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + func = &stack_pointer[-3 - oparg]; + maybe_self = &stack_pointer[-2 - oparg]; + if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + maybe_self[0] = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable[0]; + func[0] = PyStackRef_FromPyObjectNew(method); + PyStackRef_CLOSE(temp); + } + kwnames_out = kwnames_in; + } + // _DO_CALL_KW + { + kwnames = kwnames_out; + args = &stack_pointer[-1 - oparg]; + self_or_null = &stack_pointer[-2 - oparg]; + callable = &stack_pointer[-3 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + // oparg counts all of the args, but *not* self: + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null[0])) { + arguments--; + total_args++; } - // _DO_CALL_KW + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + // Check if the call can be inlined or not + if (Py_TYPE(callable_o) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) { - kwnames = kwnames_out; - args = &stack_pointer[-1 - oparg]; - self_or_null = &stack_pointer[-2 - oparg]; - callable = &stack_pointer[-3 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]); - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - // oparg counts all of the args, but *not* self: - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null[0])) { - arguments--; - total_args++; - } - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - // Check if the call can be inlined or not - if (Py_TYPE(callable_o) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) - { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - stack_pointer[-1] = kwnames; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, callable[0], locals, - arguments, positional_args, kwnames_o, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(kwnames); - // Sync stack explicitly since we leave using DISPATCH_INLINED(). - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - if (new_frame == NULL) { - goto error; - } - assert( 4 == 1 + INLINE_CACHE_ENTRIES_CALL_KW); - frame->return_offset = 4 ; - DISPATCH_INLINED(new_frame); - } - /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - PyStackRef_CLOSE(callable[0]); - PyStackRef_XCLOSE(self_or_null[0]); - for (int _i = oparg; --_i >= 0;) { - PyStackRef_CLOSE(args[_i]); - } - PyStackRef_CLOSE(kwnames); - { - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - goto error; - } - } + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); stack_pointer[-1] = kwnames; _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames_o); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable[0], locals, + arguments, positional_args, kwnames_o, frame + ); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - if (opcode == INSTRUMENTED_CALL_KW) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); - if (res_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - Py_CLEAR(res_o); - } - } + PyStackRef_CLOSE(kwnames); + // Sync stack explicitly since we leave using DISPATCH_INLINED(). + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + if (new_frame == NULL) { + goto error; } + assert( 4 == 1 + INLINE_CACHE_ENTRIES_CALL_KW); + frame->return_offset = 4 ; + DISPATCH_INLINED(new_frame); + } + /* Callable is not a normal Python function */ + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable[0]); PyStackRef_XCLOSE(self_or_null[0]); for (int _i = oparg; --_i >= 0;) { PyStackRef_CLOSE(args[_i]); } PyStackRef_CLOSE(kwnames); - if (res_o == NULL) { + { stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); goto error; } - res = PyStackRef_FromPyObjectSteal(res_o); } - stack_pointer[-3 - oparg] = res; - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-1] = kwnames; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL_KW) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); + if (res_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + Py_CLEAR(res_o); + } + } + } + PyStackRef_CLOSE(callable[0]); + PyStackRef_XCLOSE(self_or_null[0]); + for (int _i = oparg; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + PyStackRef_CLOSE(kwnames); + if (res_o == NULL) { + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + res = PyStackRef_FromPyObjectSteal(res_o); } + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -3309,62 +3299,60 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP); - PRED_COMPARE_OP: - { - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _SPECIALIZE_COMPARE_OP - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_CompareOp(left, right, next_instr, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(COMPARE_OP); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION */ + PREDICTED_COMPARE_OP:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _SPECIALIZE_COMPARE_OP + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_CompareOp(left, right, next_instr, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); } - // _COMPARE_OP - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert((oparg >> 5) <= Py_GE); + OPCODE_DEFERRED_INC(COMPARE_OP); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + } + // _COMPARE_OP + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert((oparg >> 5) <= Py_GE); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + if (res_o == NULL) goto pop_2_error; + if (oparg & 16) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); + int res_bool = PyObject_IsTrue(res_o); + Py_DECREF(res_o); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); - if (res_o == NULL) goto pop_2_error; - if (oparg & 16) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res_bool = PyObject_IsTrue(res_o); - Py_DECREF(res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_bool < 0) goto error; - res = res_bool ? PyStackRef_True : PyStackRef_False; - } - else { - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - } + if (res_bool < 0) goto error; + res = res_bool ? PyStackRef_True : PyStackRef_False; + } + else { + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -3491,47 +3479,45 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(CONTAINS_OP); - PRED_CONTAINS_OP: - { - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef left; - _PyStackRef right; - _PyStackRef b; - // _SPECIALIZE_CONTAINS_OP - { - right = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_ContainsOp(right, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(CONTAINS_OP); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _CONTAINS_OP - { - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + PREDICTED_CONTAINS_OP:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + // _SPECIALIZE_CONTAINS_OP + { + right = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - int res = PySequence_Contains(right_o, left_o); + _Py_Specialize_ContainsOp(right, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(left); - PyStackRef_CLOSE(right); - if (res < 0) goto pop_2_error; - b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + DISPATCH_SAME_OPARG(); } - stack_pointer[-2] = b; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + OPCODE_DEFERRED_INC(CONTAINS_OP); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _CONTAINS_OP + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = PySequence_Contains(right_o, left_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(left); + PyStackRef_CLOSE(right); + if (res < 0) goto pop_2_error; + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; } + stack_pointer[-2] = b; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4014,63 +4000,61 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER); - PRED_FOR_ITER: + PREDICTED_FOR_ITER:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef iter; + _PyStackRef next; + // _SPECIALIZE_FOR_ITER { - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef iter; - _PyStackRef next; - // _SPECIALIZE_FOR_ITER - { - iter = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_ForIter(iter, next_instr, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(FOR_ITER); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION */ - } - // _FOR_ITER - { - /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + iter = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); + _Py_Specialize_ForIter(iter, next_instr, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - if (next_o == NULL) { - if (_PyErr_Occurred(tstate)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!matches) { - goto error; - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_MonitorRaise(tstate, frame, this_instr); - _PyErr_Clear(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(FOR_ITER); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + } + // _FOR_ITER + { + /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (next_o == NULL) { + if (_PyErr_Occurred(tstate)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!matches) { + goto error; } - /* iterator ended normally */ - assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - /* Jump forward oparg, then skip following END_FOR */ - JUMPBY(oparg + 1); - DISPATCH(); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_MonitorRaise(tstate, frame, this_instr); + _PyErr_Clear(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); } - next = PyStackRef_FromPyObjectSteal(next_o); - // Common case: no jump, leave it to the code generator + /* iterator ended normally */ + assert(next_instr[oparg].op.code == END_FOR || + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + /* Jump forward oparg, then skip following END_FOR */ + JUMPBY(oparg + 1); + DISPATCH(); } - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + next = PyStackRef_FromPyObjectSteal(next_o); + // Common case: no jump, leave it to the code generator } + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4648,7 +4632,7 @@ next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); - goto PRED_CALL_FUNCTION_EX; + goto PREDICTED_CALL_FUNCTION_EX; } TARGET(INSTRUMENTED_CALL_KW) { @@ -4672,7 +4656,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (err) goto error; PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - goto PRED_CALL_KW; + goto PREDICTED_CALL_KW; } TARGET(INSTRUMENTED_END_FOR) { @@ -4862,7 +4846,7 @@ // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we // don't want to specialize instrumented instructions PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - goto PRED_LOAD_SUPER_ATTR; + goto PREDICTED_LOAD_SUPER_ATTR; } TARGET(INSTRUMENTED_LOAD_SUPER_METHOD) { @@ -4874,7 +4858,7 @@ // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we // don't want to specialize instrumented instructions PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - goto PRED_LOAD_SUPER_METHOD; + goto PREDICTED_LOAD_SUPER_METHOD; } TARGET(INSTRUMENTED_NOT_TAKEN) { @@ -5329,43 +5313,41 @@ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR); - PRED_LOAD_ATTR: + PREDICTED_LOAD_ATTR:; + _Py_CODEUNIT* const this_instr = next_instr - 10; + (void)this_instr; + _PyStackRef owner; + _PyStackRef attr; + // _SPECIALIZE_LOAD_ATTR { - _Py_CODEUNIT* const this_instr = next_instr - 10; - (void)this_instr; - _PyStackRef owner; - _PyStackRef attr; - // _SPECIALIZE_LOAD_ATTR - { - owner = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_LoadAttr(owner, next_instr, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(LOAD_ATTR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 8 cache entries */ - // _LOAD_ATTR - { + owner = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); + _Py_Specialize_LoadAttr(owner, next_instr, name); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(owner); - if (attr_o == NULL) goto pop_1_error; - attr = PyStackRef_FromPyObjectSteal(attr_o); + DISPATCH_SAME_OPARG(); } - stack_pointer[-1] = attr; + OPCODE_DEFERRED_INC(LOAD_ATTR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ } + /* Skip 8 cache entries */ + // _LOAD_ATTR + { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(owner); + if (attr_o == NULL) goto pop_1_error; + attr = PyStackRef_FromPyObjectSteal(attr_o); + } + stack_pointer[-1] = attr; DISPATCH(); } @@ -5848,24 +5830,22 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_CONST); - PRED_LOAD_CONST: - { - _Py_CODEUNIT* const this_instr = next_instr - 1; - (void)this_instr; - _PyStackRef value; - /* We can't do this in the bytecode compiler as - * marshalling can intern strings and make them immortal. */ - PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); - value = PyStackRef_FromPyObjectNew(obj); - #if ENABLE_SPECIALIZATION - if (this_instr->op.code == LOAD_CONST) { - this_instr->op.code = _Py_IsImmortal(obj) ? LOAD_CONST_IMMORTAL : LOAD_CONST_MORTAL; - } - #endif - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + PREDICTED_LOAD_CONST:; + _Py_CODEUNIT* const this_instr = next_instr - 1; + (void)this_instr; + _PyStackRef value; + /* We can't do this in the bytecode compiler as + * marshalling can intern strings and make them immortal. */ + PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); + value = PyStackRef_FromPyObjectNew(obj); + #if ENABLE_SPECIALIZATION + if (this_instr->op.code == LOAD_CONST) { + this_instr->op.code = _Py_IsImmortal(obj) ? LOAD_CONST_IMMORTAL : LOAD_CONST_MORTAL; } + #endif + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -6093,43 +6073,41 @@ frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL); - PRED_LOAD_GLOBAL: + PREDICTED_LOAD_GLOBAL:; + _Py_CODEUNIT* const this_instr = next_instr - 5; + (void)this_instr; + _PyStackRef *res; + // _SPECIALIZE_LOAD_GLOBAL { - _Py_CODEUNIT* const this_instr = next_instr - 5; - (void)this_instr; - _PyStackRef *res; - // _SPECIALIZE_LOAD_GLOBAL - { - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(LOAD_GLOBAL); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 1 cache entry */ - /* Skip 1 cache entry */ - /* Skip 1 cache entry */ - // _LOAD_GLOBAL - { - res = &stack_pointer[0]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); + _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); stack_pointer = _PyFrame_GetStackPointer(frame); - if (PyStackRef_IsNull(*res)) goto error; + DISPATCH_SAME_OPARG(); } - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + OPCODE_DEFERRED_INC(LOAD_GLOBAL); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 1 cache entry */ + /* Skip 1 cache entry */ + /* Skip 1 cache entry */ + // _LOAD_GLOBAL + { + res = &stack_pointer[0]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (PyStackRef_IsNull(*res)) goto error; } + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -6245,67 +6223,65 @@ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_METHOD); - PRED_LOAD_METHOD: - { - _Py_CODEUNIT* const this_instr = next_instr - 10; - (void)this_instr; - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self_or_null; - // _SPECIALIZE_LOAD_METHOD - { - owner = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_LoadMethod(owner, next_instr, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(LOAD_METHOD); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 8 cache entries */ - // _LOAD_METHOD - { + PREDICTED_LOAD_METHOD:; + _Py_CODEUNIT* const this_instr = next_instr - 10; + (void)this_instr; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self_or_null; + // _SPECIALIZE_LOAD_METHOD + { + owner = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *attr_o; - /* Designed to work in tandem with CALL, pushes two values. */ - attr_o = NULL; + next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); + _Py_Specialize_LoadMethod(owner, next_instr, name); stack_pointer = _PyFrame_GetStackPointer(frame); - if (is_meth) { - /* We can bypass temporary bound method object. - meth is unbound method and obj is self. - meth | self | arg1 | ... | argN - */ - assert(attr_o != NULL); // No errors on this branch - self_or_null = owner; // Transfer ownership - } - else { - /* meth is not an unbound method (but a regular attr, or - something was returned by a descriptor protocol). Set - the second element of the stack to NULL, to signal - CALL that it's not a method call. - meth | NULL | arg1 | ... | argN - */ - PyStackRef_CLOSE(owner); - if (attr_o == NULL) goto pop_1_error; - self_or_null = PyStackRef_NULL; - } - attr = PyStackRef_FromPyObjectSteal(attr_o); + DISPATCH_SAME_OPARG(); } - stack_pointer[-1] = attr; - stack_pointer[0] = self_or_null; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + OPCODE_DEFERRED_INC(LOAD_METHOD); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 8 cache entries */ + // _LOAD_METHOD + { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *attr_o; + /* Designed to work in tandem with CALL, pushes two values. */ + attr_o = NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); + int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (is_meth) { + /* We can bypass temporary bound method object. + meth is unbound method and obj is self. + meth | self | arg1 | ... | argN + */ + assert(attr_o != NULL); // No errors on this branch + self_or_null = owner; // Transfer ownership + } + else { + /* meth is not an unbound method (but a regular attr, or + something was returned by a descriptor protocol). Set + the second element of the stack to NULL, to signal + CALL that it's not a method call. + meth | NULL | arg1 | ... | argN + */ + PyStackRef_CLOSE(owner); + if (attr_o == NULL) goto pop_1_error; + self_or_null = PyStackRef_NULL; + } + attr = PyStackRef_FromPyObjectSteal(attr_o); } + stack_pointer[-1] = attr; + stack_pointer[0] = self_or_null; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -6507,96 +6483,94 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR); - PRED_LOAD_SUPER_ATTR: - { - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef global_super_st; - _PyStackRef class_st; - _PyStackRef self_st; - _PyStackRef attr; - // _SPECIALIZE_LOAD_SUPER_ATTR - { - class_st = stack_pointer[-2]; - global_super_st = stack_pointer[-3]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; + PREDICTED_LOAD_SUPER_ATTR:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr; + // _SPECIALIZE_LOAD_SUPER_ATTR + { + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, 0); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(LOAD_SUPER_ATTR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _LOAD_SUPER_ATTR + { + self_st = stack_pointer[-1]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + if (opcode >= MIN_INSTRUMENTED_OPCODE) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + PyStackRef_CLOSE(self_st); + goto pop_3_error; + } + } + // we make no attempt to optimize here; specializations should + // handle any case whose performance we care about + PyObject *stack[] = {class, self}; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (opcode >= MIN_INSTRUMENTED_OPCODE) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + if (super == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, 0); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, global_super, arg); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); } - OPCODE_DEFERRED_INC(LOAD_SUPER_ATTR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _LOAD_SUPER_ATTR - { - self_st = stack_pointer[-1]; - PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); - PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); - PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - if (opcode >= MIN_INSTRUMENTED_OPCODE) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + else { _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, + tstate, PY_MONITORING_EVENT_C_RETURN, frame, this_instr, global_super, arg); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); - PyStackRef_CLOSE(self_st); - goto pop_3_error; - } - } - // we make no attempt to optimize here; specializations should - // handle any case whose performance we care about - PyObject *stack[] = {class, self}; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (opcode >= MIN_INSTRUMENTED_OPCODE) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - if (super == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - Py_CLEAR(super); - } + if (err < 0) { + Py_CLEAR(super); } } - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); - PyStackRef_CLOSE(self_st); - if (super == NULL) goto pop_3_error; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = PyObject_GetAttr(super, name); - Py_DECREF(super); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (attr_o == NULL) goto error; - attr = PyStackRef_FromPyObjectSteal(attr_o); } - stack_pointer[0] = attr; - stack_pointer += 1; + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + PyStackRef_CLOSE(self_st); + if (super == NULL) goto pop_3_error; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = PyObject_GetAttr(super, name); + Py_DECREF(super); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (attr_o == NULL) goto error; + attr = PyStackRef_FromPyObjectSteal(attr_o); } + stack_pointer[0] = attr; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -6639,102 +6613,100 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_METHOD); - PRED_LOAD_SUPER_METHOD: - { - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef global_super_st; - _PyStackRef class_st; - _PyStackRef self_st; - _PyStackRef attr; - _PyStackRef null; - // _SPECIALIZE_LOAD_SUPER_METHOD - { - class_st = stack_pointer[-2]; - global_super_st = stack_pointer[-3]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; + PREDICTED_LOAD_SUPER_METHOD:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr; + _PyStackRef null; + // _SPECIALIZE_LOAD_SUPER_METHOD + { + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, 1); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(LOAD_SUPER_METHOD); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _LOAD_SUPER_ATTR + { + self_st = stack_pointer[-1]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + if (opcode >= MIN_INSTRUMENTED_OPCODE) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + PyStackRef_CLOSE(self_st); + goto pop_3_error; + } + } + // we make no attempt to optimize here; specializations should + // handle any case whose performance we care about + PyObject *stack[] = {class, self}; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (opcode >= MIN_INSTRUMENTED_OPCODE) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + if (super == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, 1); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, global_super, arg); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); } - OPCODE_DEFERRED_INC(LOAD_SUPER_METHOD); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _LOAD_SUPER_ATTR - { - self_st = stack_pointer[-1]; - PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); - PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); - PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - if (opcode >= MIN_INSTRUMENTED_OPCODE) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + else { _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, + tstate, PY_MONITORING_EVENT_C_RETURN, frame, this_instr, global_super, arg); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); - PyStackRef_CLOSE(self_st); - goto pop_3_error; - } - } - // we make no attempt to optimize here; specializations should - // handle any case whose performance we care about - PyObject *stack[] = {class, self}; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (opcode >= MIN_INSTRUMENTED_OPCODE) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - if (super == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - Py_CLEAR(super); - } + if (err < 0) { + Py_CLEAR(super); } } - PyStackRef_CLOSE(global_super_st); - PyStackRef_CLOSE(class_st); - PyStackRef_CLOSE(self_st); - if (super == NULL) goto pop_3_error; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = PyObject_GetAttr(super, name); - Py_DECREF(super); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (attr_o == NULL) goto error; - attr = PyStackRef_FromPyObjectSteal(attr_o); - } - // _PUSH_NULL - { - null = PyStackRef_NULL; } - stack_pointer[0] = attr; - stack_pointer[1] = null; - stack_pointer += 2; + PyStackRef_CLOSE(global_super_st); + PyStackRef_CLOSE(class_st); + PyStackRef_CLOSE(self_st); + if (super == NULL) goto pop_3_error; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = PyObject_GetAttr(super, name); + Py_DECREF(super); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (attr_o == NULL) goto error; + attr = PyStackRef_FromPyObjectSteal(attr_o); + } + // _PUSH_NULL + { + null = PyStackRef_NULL; } + stack_pointer[0] = attr; + stack_pointer[1] = null; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -7211,68 +7183,66 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESUME); - PRED_RESUME: + PREDICTED_RESUME:; + _Py_CODEUNIT* const this_instr = next_instr - 1; + (void)this_instr; + // _LOAD_BYTECODE { - _Py_CODEUNIT* const this_instr = next_instr - 1; - (void)this_instr; - // _LOAD_BYTECODE - { - #ifdef Py_GIL_DISABLED - if (frame->tlbc_index != - ((_PyThreadStateImpl *)tstate)->tlbc_index) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_CODEUNIT *bytecode = - _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (bytecode == NULL) goto error; + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_CODEUNIT *bytecode = + _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (bytecode == NULL) goto error; + _PyFrame_SetStackPointer(frame, stack_pointer); + ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; + frame->instr_ptr = bytecode + off; + // Make sure this_instr gets reset correctley for any uops that + // follow + next_instr = frame->instr_ptr; + DISPATCH(); + } + #endif + } + // _MAYBE_INSTRUMENT + { + if (tstate->tracing == 0) { + uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + if (code_version != global_version) { _PyFrame_SetStackPointer(frame, stack_pointer); - ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); + int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); stack_pointer = _PyFrame_GetStackPointer(frame); - frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; - frame->instr_ptr = bytecode + off; - // Make sure this_instr gets reset correctley for any uops that - // follow - next_instr = frame->instr_ptr; - DISPATCH(); - } - #endif - } - // _MAYBE_INSTRUMENT - { - if (tstate->tracing == 0) { - uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); - if (code_version != global_version) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - goto error; - } - next_instr = this_instr; - DISPATCH(); + if (err) { + goto error; } + next_instr = this_instr; + DISPATCH(); } } - // _QUICKEN_RESUME - { - #if ENABLE_SPECIALIZATION_FT - if (tstate->tracing == 0 && this_instr->op.code == RESUME) { - FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); - } - #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _QUICKEN_RESUME + { + #if ENABLE_SPECIALIZATION_FT + if (tstate->tracing == 0 && this_instr->op.code == RESUME) { + FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); } - // _CHECK_PERIODIC_IF_NOT_YIELD_FROM - { - if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); \ - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) goto error; - } + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _CHECK_PERIODIC_IF_NOT_YIELD_FROM + { + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); \ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) goto error; } } } @@ -7366,91 +7336,89 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(SEND); - PRED_SEND: - { - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef receiver; - _PyStackRef v; - _PyStackRef retval; - // _SPECIALIZE_SEND - { - receiver = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_Send(receiver, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(SEND); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ + PREDICTED_SEND:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef receiver; + _PyStackRef v; + _PyStackRef retval; + // _SPECIALIZE_SEND + { + receiver = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_Send(receiver, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); } - // _SEND + OPCODE_DEFERRED_INC(SEND); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _SEND + { + v = stack_pointer[-1]; + PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); + PyObject *retval_o; + assert(frame != &entry_frame); + if ((tstate->interp->eval_frame == NULL) && + (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) && + ((PyGenObject *)receiver_o)->gi_frame_state < FRAME_EXECUTING) { - v = stack_pointer[-1]; - PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); - PyObject *retval_o; - assert(frame != &entry_frame); - if ((tstate->interp->eval_frame == NULL) && - (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) && - ((PyGenObject *)receiver_o)->gi_frame_state < FRAME_EXECUTING) - { - PyGenObject *gen = (PyGenObject *)receiver_o; - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - STACK_SHRINK(1); - _PyFrame_StackPush(gen_frame, v); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - assert( 2 + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)( 2 + oparg); - assert(gen_frame->previous == NULL); - gen_frame->previous = frame; - DISPATCH_INLINED(gen_frame); - } - if (PyStackRef_IsNone(v) && PyIter_Check(receiver_o)) { + PyGenObject *gen = (PyGenObject *)receiver_o; + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + STACK_SHRINK(1); + _PyFrame_StackPush(gen_frame, v); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert( 2 + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)( 2 + oparg); + assert(gen_frame->previous == NULL); + gen_frame->previous = frame; + DISPATCH_INLINED(gen_frame); + } + if (PyStackRef_IsNone(v) && PyIter_Check(receiver_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + retval_o = PyObject_CallMethodOneArg(receiver_o, + &_Py_ID(send), + PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + if (retval_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { _PyFrame_SetStackPointer(frame, stack_pointer); - retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o); + _PyEval_MonitorRaise(tstate, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - retval_o = PyObject_CallMethodOneArg(receiver_o, - &_Py_ID(send), - PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyGen_FetchStopIterationValue(&retval_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err == 0) { + assert(retval_o != NULL); + JUMPBY(oparg); } - if (retval_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_MonitorRaise(tstate, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyGen_FetchStopIterationValue(&retval_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err == 0) { - assert(retval_o != NULL); - JUMPBY(oparg); - } - else { - PyStackRef_CLOSE(v); - goto pop_1_error; - } + else { + PyStackRef_CLOSE(v); + goto pop_1_error; } - PyStackRef_CLOSE(v); - retval = PyStackRef_FromPyObjectSteal(retval_o); } - stack_pointer[-1] = retval; + PyStackRef_CLOSE(v); + retval = PyStackRef_FromPyObjectSteal(retval_o); } + stack_pointer[-1] = retval; DISPATCH(); } @@ -7609,46 +7577,44 @@ frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR); - PRED_STORE_ATTR: + PREDICTED_STORE_ATTR:; + _Py_CODEUNIT* const this_instr = next_instr - 5; + (void)this_instr; + _PyStackRef owner; + _PyStackRef v; + // _SPECIALIZE_STORE_ATTR { - _Py_CODEUNIT* const this_instr = next_instr - 5; - (void)this_instr; - _PyStackRef owner; - _PyStackRef v; - // _SPECIALIZE_STORE_ATTR - { - owner = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_StoreAttr(owner, next_instr, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(STORE_ATTR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 3 cache entries */ - // _STORE_ATTR - { - v = stack_pointer[-2]; + owner = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), - name, PyStackRef_AsPyObjectBorrow(v)); + _Py_Specialize_StoreAttr(owner, next_instr, name); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(v); - PyStackRef_CLOSE(owner); - if (err) goto pop_2_error; + DISPATCH_SAME_OPARG(); } - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + OPCODE_DEFERRED_INC(STORE_ATTR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ } + /* Skip 3 cache entries */ + // _STORE_ATTR + { + v = stack_pointer[-2]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), + name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(v); + PyStackRef_CLOSE(owner); + if (err) goto pop_2_error; + } + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -7975,46 +7941,44 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR); - PRED_STORE_SUBSCR: - { - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef container; - _PyStackRef sub; - _PyStackRef v; - // _SPECIALIZE_STORE_SUBSCR - { - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_StoreSubscr(container, sub, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(STORE_SUBSCR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _STORE_SUBSCR - { - v = stack_pointer[-3]; - /* container[sub] = v */ + PREDICTED_STORE_SUBSCR:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef container; + _PyStackRef sub; + _PyStackRef v; + // _SPECIALIZE_STORE_SUBSCR + { + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); + _Py_Specialize_StoreSubscr(container, sub, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(v); - PyStackRef_CLOSE(container); - PyStackRef_CLOSE(sub); - if (err) goto pop_3_error; + DISPATCH_SAME_OPARG(); } - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + OPCODE_DEFERRED_INC(STORE_SUBSCR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _STORE_SUBSCR + { + v = stack_pointer[-3]; + /* container[sub] = v */ + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(v); + PyStackRef_CLOSE(container); + PyStackRef_CLOSE(sub); + if (err) goto pop_3_error; } + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -8107,41 +8071,39 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL); - PRED_TO_BOOL: + PREDICTED_TO_BOOL:; + _Py_CODEUNIT* const this_instr = next_instr - 4; + (void)this_instr; + _PyStackRef value; + _PyStackRef res; + // _SPECIALIZE_TO_BOOL { - _Py_CODEUNIT* const this_instr = next_instr - 4; - (void)this_instr; - _PyStackRef value; - _PyStackRef res; - // _SPECIALIZE_TO_BOOL - { - value = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_ToBool(value, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(TO_BOOL); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 2 cache entries */ - // _TO_BOOL - { + value = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); + _Py_Specialize_ToBool(value, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(value); - if (err < 0) goto pop_1_error; - res = err ? PyStackRef_True : PyStackRef_False; + DISPATCH_SAME_OPARG(); } - stack_pointer[-1] = res; + OPCODE_DEFERRED_INC(TO_BOOL); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ } + /* Skip 2 cache entries */ + // _TO_BOOL + { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(value); + if (err < 0) goto pop_1_error; + res = err ? PyStackRef_True : PyStackRef_False; + } + stack_pointer[-1] = res; DISPATCH(); } @@ -8345,44 +8307,42 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE); - PRED_UNPACK_SEQUENCE: - { - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - _PyStackRef seq; - _PyStackRef *output; - // _SPECIALIZE_UNPACK_SEQUENCE - { - seq = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_UnpackSequence(seq, next_instr, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(UNPACK_SEQUENCE); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - (void)seq; - (void)counter; - } - // _UNPACK_SEQUENCE - { - output = &stack_pointer[-1]; - _PyStackRef *top = output + oparg; + PREDICTED_UNPACK_SEQUENCE:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef seq; + _PyStackRef *output; + // _SPECIALIZE_UNPACK_SEQUENCE + { + seq = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg, -1, top); + _Py_Specialize_UnpackSequence(seq, next_instr, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - PyStackRef_CLOSE(seq); - if (res == 0) goto pop_1_error; + DISPATCH_SAME_OPARG(); } - stack_pointer += -1 + oparg; - assert(WITHIN_STACK_BOUNDS()); + OPCODE_DEFERRED_INC(UNPACK_SEQUENCE); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + (void)seq; + (void)counter; } + // _UNPACK_SEQUENCE + { + output = &stack_pointer[-1]; + _PyStackRef *top = output + oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PyEval_UnpackIterableStackRef(tstate, seq, oparg, -1, top); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE(seq); + if (res == 0) goto pop_1_error; + } + stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 71eec9a2d32667..9edf3d4898eb4d 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -417,7 +417,7 @@ def go_to_instruction( next(tkn_iter) assert name.kind == "IDENTIFIER" self.emit("\n") - self.emit(f"goto PRED_{name.text};\n") + self.emit(f"goto PREDICTED_{name.text};\n") return True def emit_save(self, storage: Storage) -> None: diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 8290a09eb287c7..f4409da3d83ccd 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -158,10 +158,7 @@ def generate_tier1( out.emit(f"next_instr += {inst.size};\n") out.emit(f"INSTRUCTION_STATS({name});\n") if inst.is_target: - out.emit(f"PRED_{name}:\n") - # We need to wrap the code in a block after the label so that declarations are allowed. - # This is a limitation of some compilers. - out.emit("{\n") + out.emit(f"PREDICTED_{name}:;\n") if needs_this: out.emit(f"_Py_CODEUNIT* const this_instr = next_instr - {inst.size};\n") out.emit(unused_guard) @@ -180,8 +177,6 @@ def generate_tier1( out.start_line() stack.flush(out) - if inst.is_target: - out.emit("}\n") # PRED_{ID}: if not inst.parts[-1].properties.always_exits: out.emit("DISPATCH();\n") out.start_line() From 76e8b58c81c89f2ca7bfe43b0433731b27fb37d8 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 21 Jan 2025 21:59:08 +0800 Subject: [PATCH 5/5] alignment --- Python/ceval_macros.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index a1dde9047fe368..c0b7c0c637a4c8 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -249,7 +249,7 @@ GETITEM(PyObject *v, Py_ssize_t i) { /* This is only a single jump on release builds! */ \ UPDATE_MISS_STATS((INSTNAME)); \ assert(_PyOpcode_Deopt[opcode] == (INSTNAME)); \ - goto PREDICTED_##INSTNAME; \ + goto PREDICTED_##INSTNAME; \ } pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy