diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h index cf830b6066f4ab..1728c576f5cc41 100644 --- a/Include/cpython/pystats.h +++ b/Include/cpython/pystats.h @@ -29,7 +29,7 @@ # error "this header file must not be included directly" #endif -#define PYSTATS_MAX_UOP_ID 1024 +#define PYSTATS_MAX_UOP_ID 1500 #define SPECIALIZATION_FAILURE_KINDS 60 diff --git a/Include/internal/pycore_jit.h b/Include/internal/pycore_jit.h index 8a88cbf607ba4b..ba8790213c4e52 100644 --- a/Include/internal/pycore_jit.h +++ b/Include/internal/pycore_jit.h @@ -15,7 +15,7 @@ extern "C" { #ifdef _Py_JIT -typedef _Py_CODEUNIT *(*jit_func)(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate); +typedef _Py_CODEUNIT *(*jit_func)(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _PyStackRef _tos_cache0, _PyStackRef _tos_cache1, _PyStackRef _tos_cache2); int _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size_t length); void _PyJIT_Free(_PyExecutorObject *executor); diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index dd1bf2d1d2b51a..a5eed2c8eafd14 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1029,7 +1029,7 @@ enum InstructionFormat { #define HAS_EXIT_FLAG (1024) #define HAS_PURE_FLAG (2048) #define HAS_PASSTHROUGH_FLAG (4096) -#define HAS_OPARG_AND_1_FLAG (8192) +#define HAS_SYNC_SP_FLAG (8192) #define HAS_ERROR_NO_POP_FLAG (16384) #define HAS_NO_SAVE_IP_FLAG (32768) #define OPCODE_HAS_ARG(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ARG_FLAG)) @@ -1045,7 +1045,7 @@ enum InstructionFormat { #define OPCODE_HAS_EXIT(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_EXIT_FLAG)) #define OPCODE_HAS_PURE(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PURE_FLAG)) #define OPCODE_HAS_PASSTHROUGH(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PASSTHROUGH_FLAG)) -#define OPCODE_HAS_OPARG_AND_1(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_OPARG_AND_1_FLAG)) +#define OPCODE_HAS_SYNC_SP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_SYNC_SP_FLAG)) #define OPCODE_HAS_ERROR_NO_POP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ERROR_NO_POP_FLAG)) #define OPCODE_HAS_NO_SAVE_IP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NO_SAVE_IP_FLAG)) @@ -1079,7 +1079,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG }, [BINARY_OP_SUBSCR_DICT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG }, + [BINARY_OP_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_SYNC_SP_FLAG }, [BINARY_OP_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_SUBSCR_LIST_SLICE] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_SUBSCR_STR_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, @@ -1096,22 +1096,22 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [BUILD_TEMPLATE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BUILD_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, [CACHE] = { true, INSTR_FMT_IX, 0 }, - [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [CALL_BOUND_METHOD_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG }, + [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG }, + [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG }, + [CALL_BOUND_METHOD_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG }, [CALL_BUILTIN_CLASS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_BUILTIN_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_BUILTIN_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG }, [CALL_INTRINSIC_1] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_INTRINSIC_2] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_ISINSTANCE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_KW_BOUND_METHOD] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG }, + [CALL_KW_BOUND_METHOD] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG }, [CALL_KW_NON_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_KW_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_KW_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG }, [CALL_LEN] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1119,8 +1119,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [CALL_METHOD_DESCRIPTOR_NOARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_METHOD_DESCRIPTOR_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_NON_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [CALL_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_SYNC_SP_FLAG }, + [CALL_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG }, [CALL_STR_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_TUPLE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_TYPE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, @@ -1154,7 +1154,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [FORMAT_SIMPLE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [FORMAT_WITH_SPEC] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_SYNC_SP_FLAG }, [FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG }, @@ -1166,9 +1166,9 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [GET_YIELD_FROM_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [IMPORT_FROM] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [IMPORT_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG }, + [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG }, + [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG }, [INSTRUMENTED_END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG }, [INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, @@ -1176,7 +1176,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [INSTRUMENTED_INSTRUCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [INSTRUMENTED_LINE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, + [INSTRUMENTED_LINE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG }, [INSTRUMENTED_LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_NOT_TAKEN] = { true, INSTR_FMT_IX, 0 }, [INSTRUMENTED_POP_ITER] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, @@ -1199,7 +1199,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [LOAD_ATTR] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG }, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_SYNC_SP_FLAG }, [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG }, @@ -1207,7 +1207,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_SYNC_SP_FLAG }, [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_BUILD_CLASS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1257,8 +1257,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [RESUME_CHECK] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, - [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG }, + [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_SYNC_SP_FLAG }, [SETUP_ANNOTATIONS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [SET_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [SET_FUNCTION_ATTRIBUTE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 8b7f12bf03d624..5778bc6a23550c 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -119,7 +119,7 @@ PyAPI_FUNC(void) _Py_Executors_InvalidateCold(PyInterpreterState *interp); #define JIT_CLEANUP_THRESHOLD 100000 // This is the length of the trace we project initially. -#define UOP_MAX_TRACE_LENGTH 800 +#define UOP_MAX_TRACE_LENGTH 1600 #define TRACE_STACK_SIZE 5 diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index 6bf82d8322f508..91df73daf6000a 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -70,6 +70,9 @@ static const _PyStackRef PyStackRef_ERROR = { .index = 2 }; #define PyStackRef_False ((_PyStackRef){ .index = 6 }) #define PyStackRef_True ((_PyStackRef){ .index = 8 }) +#define PyStackRef_ZERO_BITS PyStackRef_NULL + + #define INITIAL_STACKREF_INDEX 10 static inline int @@ -264,6 +267,10 @@ PyStackRef_IsNullOrInt(_PyStackRef ref); static const _PyStackRef PyStackRef_ERROR = { .bits = Py_TAG_INVALID }; +/* For use in the JIT to clear an unused value. + * PyStackRef_ZERO_BITS has no meaning and should not be used other than by the JIT. */ +static const _PyStackRef PyStackRef_ZERO_BITS = { .bits = 0 }; + /* Wrap a pointer in a stack ref. * The resulting stack reference is not safe and should only be used * in the interpreter to pass values from one uop to another. diff --git a/Include/internal/pycore_stats.h b/Include/internal/pycore_stats.h index ab649574f33dbf..51ceea87ba3a15 100644 --- a/Include/internal/pycore_stats.h +++ b/Include/internal/pycore_stats.h @@ -28,7 +28,7 @@ extern "C" { #define GC_STAT_ADD(gen, name, n) do { if (_Py_stats) _Py_stats->gc_stats[(gen)].name += (n); } while (0) #define OPT_STAT_INC(name) do { if (_Py_stats) _Py_stats->optimization_stats.name++; } while (0) #define OPT_STAT_ADD(name, n) do { if (_Py_stats) _Py_stats->optimization_stats.name += (n); } while (0) -#define UOP_STAT_INC(opname, name) do { if (_Py_stats) { assert(opname < 512); _Py_stats->optimization_stats.opcode[opname].name++; } } while (0) +#define UOP_STAT_INC(opname, name) do { if (_Py_stats) { _Py_stats->optimization_stats.opcode[opname].name++; } } while (0) #define UOP_PAIR_INC(uopcode, lastuop) \ do { \ if (lastuop && _Py_stats) { \ diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index a9432401525ebb..68d56aa427933b 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -239,9 +239,7 @@ extern "C" { #define _LOAD_FAST_BORROW_5 474 #define _LOAD_FAST_BORROW_6 475 #define _LOAD_FAST_BORROW_7 476 -#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW LOAD_FAST_BORROW_LOAD_FAST_BORROW #define _LOAD_FAST_CHECK LOAD_FAST_CHECK -#define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS #define _LOAD_GLOBAL 477 @@ -310,50 +308,677 @@ extern "C" { #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 519 -#define _STORE_ATTR 520 -#define _STORE_ATTR_INSTANCE_VALUE 521 -#define _STORE_ATTR_SLOT 522 -#define _STORE_ATTR_WITH_HINT 523 +#define _SPILL_OR_RELOAD 519 +#define _START_EXECUTOR 520 +#define _STORE_ATTR 521 +#define _STORE_ATTR_INSTANCE_VALUE 522 +#define _STORE_ATTR_SLOT 523 +#define _STORE_ATTR_WITH_HINT 524 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 524 -#define _STORE_FAST_0 525 -#define _STORE_FAST_1 526 -#define _STORE_FAST_2 527 -#define _STORE_FAST_3 528 -#define _STORE_FAST_4 529 -#define _STORE_FAST_5 530 -#define _STORE_FAST_6 531 -#define _STORE_FAST_7 532 -#define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST -#define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST +#define _STORE_FAST 525 +#define _STORE_FAST_0 526 +#define _STORE_FAST_1 527 +#define _STORE_FAST_2 528 +#define _STORE_FAST_3 529 +#define _STORE_FAST_4 530 +#define _STORE_FAST_5 531 +#define _STORE_FAST_6 532 +#define _STORE_FAST_7 533 #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 533 -#define _STORE_SUBSCR 534 -#define _STORE_SUBSCR_DICT 535 -#define _STORE_SUBSCR_LIST_INT 536 -#define _SWAP 537 -#define _SWAP_2 538 -#define _SWAP_3 539 -#define _TIER2_RESUME_CHECK 540 -#define _TO_BOOL 541 +#define _STORE_SLICE 534 +#define _STORE_SUBSCR 535 +#define _STORE_SUBSCR_DICT 536 +#define _STORE_SUBSCR_LIST_INT 537 +#define _SWAP 538 +#define _SWAP_2 539 +#define _SWAP_3 540 +#define _TIER2_RESUME_CHECK 541 +#define _TO_BOOL 542 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT -#define _TO_BOOL_LIST 542 +#define _TO_BOOL_LIST 543 #define _TO_BOOL_NONE TO_BOOL_NONE -#define _TO_BOOL_STR 543 +#define _TO_BOOL_STR 544 #define _UNARY_INVERT UNARY_INVERT #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 544 -#define _UNPACK_SEQUENCE_LIST 545 -#define _UNPACK_SEQUENCE_TUPLE 546 -#define _UNPACK_SEQUENCE_TWO_TUPLE 547 +#define _UNPACK_SEQUENCE 545 +#define _UNPACK_SEQUENCE_LIST 546 +#define _UNPACK_SEQUENCE_TUPLE 547 +#define _UNPACK_SEQUENCE_TWO_TUPLE 548 #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 547 +#define MAX_UOP_ID 548 +#define _BINARY_OP_r01 549 +#define _BINARY_OP_ADD_FLOAT_r21 550 +#define _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS_r21 551 +#define _BINARY_OP_ADD_INT_r21 552 +#define _BINARY_OP_ADD_UNICODE_r21 553 +#define _BINARY_OP_EXTEND_r01 554 +#define _BINARY_OP_INPLACE_ADD_UNICODE_r00 555 +#define _BINARY_OP_MULTIPLY_FLOAT_r21 556 +#define _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS_r21 557 +#define _BINARY_OP_MULTIPLY_INT_r21 558 +#define _BINARY_OP_SUBSCR_CHECK_FUNC_r23 559 +#define _BINARY_OP_SUBSCR_DICT_r01 560 +#define _BINARY_OP_SUBSCR_INIT_CALL_r31 561 +#define _BINARY_OP_SUBSCR_LIST_INT_r01 562 +#define _BINARY_OP_SUBSCR_LIST_SLICE_r01 563 +#define _BINARY_OP_SUBSCR_STR_INT_r21 564 +#define _BINARY_OP_SUBSCR_TUPLE_INT_r21 565 +#define _BINARY_OP_SUBTRACT_FLOAT_r21 566 +#define _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS_r21 567 +#define _BINARY_OP_SUBTRACT_INT_r21 568 +#define _BINARY_SLICE_r01 569 +#define _BUILD_INTERPOLATION_r01 570 +#define _BUILD_LIST_r01 571 +#define _BUILD_MAP_r01 572 +#define _BUILD_SET_r01 573 +#define _BUILD_SLICE_r01 574 +#define _BUILD_STRING_r01 575 +#define _BUILD_TEMPLATE_r01 576 +#define _BUILD_TUPLE_r01 577 +#define _CALL_BUILTIN_CLASS_r01 578 +#define _CALL_BUILTIN_FAST_r01 579 +#define _CALL_BUILTIN_FAST_WITH_KEYWORDS_r01 580 +#define _CALL_BUILTIN_O_r01 581 +#define _CALL_INTRINSIC_1_r01 582 +#define _CALL_INTRINSIC_2_r01 583 +#define _CALL_ISINSTANCE_r01 584 +#define _CALL_KW_NON_PY_r01 585 +#define _CALL_LEN_r01 586 +#define _CALL_LIST_APPEND_r30 587 +#define _CALL_METHOD_DESCRIPTOR_FAST_r01 588 +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r01 589 +#define _CALL_METHOD_DESCRIPTOR_NOARGS_r01 590 +#define _CALL_METHOD_DESCRIPTOR_O_r01 591 +#define _CALL_NON_PY_GENERAL_r01 592 +#define _CALL_STR_1_r01 593 +#define _CALL_TUPLE_1_r01 594 +#define _CALL_TYPE_1_r31 595 +#define _CHECK_AND_ALLOCATE_OBJECT_r00 596 +#define _CHECK_ATTR_CLASS_r00 597 +#define _CHECK_ATTR_CLASS_r11 598 +#define _CHECK_ATTR_CLASS_r22 599 +#define _CHECK_ATTR_CLASS_r33 600 +#define _CHECK_ATTR_METHOD_LAZY_DICT_r00 601 +#define _CHECK_ATTR_METHOD_LAZY_DICT_r11 602 +#define _CHECK_ATTR_METHOD_LAZY_DICT_r22 603 +#define _CHECK_ATTR_METHOD_LAZY_DICT_r33 604 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS_r00 605 +#define _CHECK_EG_MATCH_r02 606 +#define _CHECK_EXC_MATCH_r01 607 +#define _CHECK_FUNCTION_r00 608 +#define _CHECK_FUNCTION_r11 609 +#define _CHECK_FUNCTION_r22 610 +#define _CHECK_FUNCTION_r33 611 +#define _CHECK_FUNCTION_EXACT_ARGS_r00 612 +#define _CHECK_FUNCTION_VERSION_r00 613 +#define _CHECK_FUNCTION_VERSION_INLINE_r00 614 +#define _CHECK_FUNCTION_VERSION_INLINE_r11 615 +#define _CHECK_FUNCTION_VERSION_INLINE_r22 616 +#define _CHECK_FUNCTION_VERSION_INLINE_r33 617 +#define _CHECK_FUNCTION_VERSION_KW_r11 618 +#define _CHECK_IS_NOT_PY_CALLABLE_r00 619 +#define _CHECK_IS_NOT_PY_CALLABLE_KW_r11 620 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r00 621 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r11 622 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r22 623 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r33 624 +#define _CHECK_METHOD_VERSION_r00 625 +#define _CHECK_METHOD_VERSION_KW_r11 626 +#define _CHECK_PEP_523_r00 627 +#define _CHECK_PEP_523_r11 628 +#define _CHECK_PEP_523_r22 629 +#define _CHECK_PEP_523_r33 630 +#define _CHECK_PERIODIC_r00 631 +#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM_r00 632 +#define _CHECK_RECURSION_REMAINING_r00 633 +#define _CHECK_RECURSION_REMAINING_r11 634 +#define _CHECK_RECURSION_REMAINING_r22 635 +#define _CHECK_RECURSION_REMAINING_r33 636 +#define _CHECK_STACK_SPACE_r00 637 +#define _CHECK_STACK_SPACE_OPERAND_r00 638 +#define _CHECK_STACK_SPACE_OPERAND_r11 639 +#define _CHECK_STACK_SPACE_OPERAND_r22 640 +#define _CHECK_STACK_SPACE_OPERAND_r33 641 +#define _CHECK_VALIDITY_r00 642 +#define _CHECK_VALIDITY_r11 643 +#define _CHECK_VALIDITY_r22 644 +#define _CHECK_VALIDITY_r33 645 +#define _COMPARE_OP_r01 646 +#define _COMPARE_OP_FLOAT_r21 647 +#define _COMPARE_OP_INT_r21 648 +#define _COMPARE_OP_STR_r21 649 +#define _CONTAINS_OP_r01 650 +#define _CONTAINS_OP_DICT_r01 651 +#define _CONTAINS_OP_SET_r01 652 +#define _CONVERT_VALUE_r01 653 +#define _COPY_r01 654 +#define _COPY_1_r01 655 +#define _COPY_1_r12 656 +#define _COPY_1_r23 657 +#define _COPY_2_r23 658 +#define _COPY_3_r33 659 +#define _COPY_FREE_VARS_r00 660 +#define _CREATE_INIT_FRAME_r01 661 +#define _DELETE_ATTR_r00 662 +#define _DELETE_DEREF_r00 663 +#define _DELETE_FAST_r00 664 +#define _DELETE_GLOBAL_r00 665 +#define _DELETE_NAME_r00 666 +#define _DELETE_SUBSCR_r00 667 +#define _DEOPT_r00 668 +#define _DEOPT_r10 669 +#define _DEOPT_r20 670 +#define _DEOPT_r30 671 +#define _DICT_MERGE_r00 672 +#define _DICT_UPDATE_r00 673 +#define _DO_CALL_r01 674 +#define _DO_CALL_FUNCTION_EX_r01 675 +#define _DO_CALL_KW_r01 676 +#define _END_FOR_r10 677 +#define _END_SEND_r21 678 +#define _ERROR_POP_N_r00 679 +#define _EXIT_INIT_CHECK_r00 680 +#define _EXIT_TRACE_r00 681 +#define _EXIT_TRACE_r10 682 +#define _EXIT_TRACE_r20 683 +#define _EXIT_TRACE_r30 684 +#define _EXPAND_METHOD_r00 685 +#define _EXPAND_METHOD_KW_r00 686 +#define _FATAL_ERROR_r00 687 +#define _FATAL_ERROR_r11 688 +#define _FATAL_ERROR_r22 689 +#define _FATAL_ERROR_r33 690 +#define _FORMAT_SIMPLE_r01 691 +#define _FORMAT_WITH_SPEC_r01 692 +#define _FOR_ITER_r01 693 +#define _FOR_ITER_GEN_FRAME_r23 694 +#define _FOR_ITER_TIER_TWO_r01 695 +#define _GET_AITER_r01 696 +#define _GET_ANEXT_r01 697 +#define _GET_AWAITABLE_r01 698 +#define _GET_ITER_r02 699 +#define _GET_LEN_r01 700 +#define _GET_YIELD_FROM_ITER_r01 701 +#define _GUARD_BINARY_OP_EXTEND_r00 702 +#define _GUARD_CALLABLE_ISINSTANCE_r33 703 +#define _GUARD_CALLABLE_LEN_r33 704 +#define _GUARD_CALLABLE_LIST_APPEND_r33 705 +#define _GUARD_CALLABLE_STR_1_r33 706 +#define _GUARD_CALLABLE_TUPLE_1_r33 707 +#define _GUARD_CALLABLE_TYPE_1_r33 708 +#define _GUARD_DORV_NO_DICT_r00 709 +#define _GUARD_DORV_NO_DICT_r11 710 +#define _GUARD_DORV_NO_DICT_r22 711 +#define _GUARD_DORV_NO_DICT_r33 712 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r00 713 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r11 714 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r22 715 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r33 716 +#define _GUARD_GLOBALS_VERSION_r00 717 +#define _GUARD_GLOBALS_VERSION_r11 718 +#define _GUARD_GLOBALS_VERSION_r22 719 +#define _GUARD_GLOBALS_VERSION_r33 720 +#define _GUARD_IS_FALSE_POP_r10 721 +#define _GUARD_IS_NONE_POP_r10 722 +#define _GUARD_IS_NOT_NONE_POP_r10 723 +#define _GUARD_IS_TRUE_POP_r10 724 +#define _GUARD_KEYS_VERSION_r00 725 +#define _GUARD_KEYS_VERSION_r11 726 +#define _GUARD_KEYS_VERSION_r22 727 +#define _GUARD_KEYS_VERSION_r33 728 +#define _GUARD_NOS_DICT_r00 729 +#define _GUARD_NOS_DICT_r11 730 +#define _GUARD_NOS_DICT_r22 731 +#define _GUARD_NOS_DICT_r33 732 +#define _GUARD_NOS_FLOAT_r00 733 +#define _GUARD_NOS_FLOAT_r11 734 +#define _GUARD_NOS_FLOAT_r22 735 +#define _GUARD_NOS_FLOAT_r33 736 +#define _GUARD_NOS_INT_r00 737 +#define _GUARD_NOS_INT_r11 738 +#define _GUARD_NOS_INT_r22 739 +#define _GUARD_NOS_INT_r33 740 +#define _GUARD_NOS_LIST_r00 741 +#define _GUARD_NOS_LIST_r11 742 +#define _GUARD_NOS_LIST_r22 743 +#define _GUARD_NOS_LIST_r33 744 +#define _GUARD_NOS_NOT_NULL_r00 745 +#define _GUARD_NOS_NOT_NULL_r11 746 +#define _GUARD_NOS_NOT_NULL_r22 747 +#define _GUARD_NOS_NOT_NULL_r33 748 +#define _GUARD_NOS_NULL_r00 749 +#define _GUARD_NOS_NULL_r11 750 +#define _GUARD_NOS_NULL_r22 751 +#define _GUARD_NOS_NULL_r33 752 +#define _GUARD_NOS_OVERFLOWED_r00 753 +#define _GUARD_NOS_OVERFLOWED_r11 754 +#define _GUARD_NOS_OVERFLOWED_r22 755 +#define _GUARD_NOS_OVERFLOWED_r33 756 +#define _GUARD_NOS_TUPLE_r00 757 +#define _GUARD_NOS_TUPLE_r11 758 +#define _GUARD_NOS_TUPLE_r22 759 +#define _GUARD_NOS_TUPLE_r33 760 +#define _GUARD_NOS_UNICODE_r00 761 +#define _GUARD_NOS_UNICODE_r11 762 +#define _GUARD_NOS_UNICODE_r22 763 +#define _GUARD_NOS_UNICODE_r33 764 +#define _GUARD_NOT_EXHAUSTED_LIST_r00 765 +#define _GUARD_NOT_EXHAUSTED_LIST_r11 766 +#define _GUARD_NOT_EXHAUSTED_LIST_r22 767 +#define _GUARD_NOT_EXHAUSTED_LIST_r33 768 +#define _GUARD_NOT_EXHAUSTED_RANGE_r00 769 +#define _GUARD_NOT_EXHAUSTED_RANGE_r11 770 +#define _GUARD_NOT_EXHAUSTED_RANGE_r22 771 +#define _GUARD_NOT_EXHAUSTED_RANGE_r33 772 +#define _GUARD_NOT_EXHAUSTED_TUPLE_r00 773 +#define _GUARD_NOT_EXHAUSTED_TUPLE_r11 774 +#define _GUARD_NOT_EXHAUSTED_TUPLE_r22 775 +#define _GUARD_NOT_EXHAUSTED_TUPLE_r33 776 +#define _GUARD_THIRD_NULL_r33 777 +#define _GUARD_TOS_ANY_SET_r00 778 +#define _GUARD_TOS_ANY_SET_r11 779 +#define _GUARD_TOS_ANY_SET_r22 780 +#define _GUARD_TOS_ANY_SET_r33 781 +#define _GUARD_TOS_DICT_r00 782 +#define _GUARD_TOS_DICT_r11 783 +#define _GUARD_TOS_DICT_r22 784 +#define _GUARD_TOS_DICT_r33 785 +#define _GUARD_TOS_FLOAT_r00 786 +#define _GUARD_TOS_FLOAT_r11 787 +#define _GUARD_TOS_FLOAT_r22 788 +#define _GUARD_TOS_FLOAT_r33 789 +#define _GUARD_TOS_INT_r00 790 +#define _GUARD_TOS_INT_r11 791 +#define _GUARD_TOS_INT_r22 792 +#define _GUARD_TOS_INT_r33 793 +#define _GUARD_TOS_LIST_r00 794 +#define _GUARD_TOS_LIST_r11 795 +#define _GUARD_TOS_LIST_r22 796 +#define _GUARD_TOS_LIST_r33 797 +#define _GUARD_TOS_OVERFLOWED_r00 798 +#define _GUARD_TOS_OVERFLOWED_r11 799 +#define _GUARD_TOS_OVERFLOWED_r22 800 +#define _GUARD_TOS_OVERFLOWED_r33 801 +#define _GUARD_TOS_SLICE_r00 802 +#define _GUARD_TOS_SLICE_r11 803 +#define _GUARD_TOS_SLICE_r22 804 +#define _GUARD_TOS_SLICE_r33 805 +#define _GUARD_TOS_TUPLE_r00 806 +#define _GUARD_TOS_TUPLE_r11 807 +#define _GUARD_TOS_TUPLE_r22 808 +#define _GUARD_TOS_TUPLE_r33 809 +#define _GUARD_TOS_UNICODE_r00 810 +#define _GUARD_TOS_UNICODE_r11 811 +#define _GUARD_TOS_UNICODE_r22 812 +#define _GUARD_TOS_UNICODE_r33 813 +#define _GUARD_TYPE_VERSION_r00 814 +#define _GUARD_TYPE_VERSION_r11 815 +#define _GUARD_TYPE_VERSION_r22 816 +#define _GUARD_TYPE_VERSION_r33 817 +#define _GUARD_TYPE_VERSION_AND_LOCK_r00 818 +#define _GUARD_TYPE_VERSION_AND_LOCK_r11 819 +#define _GUARD_TYPE_VERSION_AND_LOCK_r22 820 +#define _GUARD_TYPE_VERSION_AND_LOCK_r33 821 +#define _IMPORT_FROM_r01 822 +#define _IMPORT_NAME_r01 823 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS_r00 824 +#define _INIT_CALL_PY_EXACT_ARGS_r01 825 +#define _INIT_CALL_PY_EXACT_ARGS_0_r01 826 +#define _INIT_CALL_PY_EXACT_ARGS_1_r01 827 +#define _INIT_CALL_PY_EXACT_ARGS_2_r01 828 +#define _INIT_CALL_PY_EXACT_ARGS_3_r01 829 +#define _INIT_CALL_PY_EXACT_ARGS_4_r01 830 +#define _INSERT_NULL_r10 831 +#define _INSTRUMENTED_FOR_ITER_r01 832 +#define _INSTRUMENTED_INSTRUCTION_r00 833 +#define _INSTRUMENTED_JUMP_FORWARD_r00 834 +#define _INSTRUMENTED_JUMP_FORWARD_r11 835 +#define _INSTRUMENTED_JUMP_FORWARD_r22 836 +#define _INSTRUMENTED_JUMP_FORWARD_r33 837 +#define _INSTRUMENTED_LINE_r00 838 +#define _INSTRUMENTED_NOT_TAKEN_r00 839 +#define _INSTRUMENTED_NOT_TAKEN_r11 840 +#define _INSTRUMENTED_NOT_TAKEN_r22 841 +#define _INSTRUMENTED_NOT_TAKEN_r33 842 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r10 843 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r21 844 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r32 845 +#define _INSTRUMENTED_POP_JUMP_IF_NONE_r10 846 +#define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE_r10 847 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r10 848 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r21 849 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r32 850 +#define _IS_NONE_r11 851 +#define _IS_OP_r21 852 +#define _ITER_CHECK_LIST_r00 853 +#define _ITER_CHECK_LIST_r11 854 +#define _ITER_CHECK_LIST_r22 855 +#define _ITER_CHECK_LIST_r33 856 +#define _ITER_CHECK_RANGE_r00 857 +#define _ITER_CHECK_RANGE_r11 858 +#define _ITER_CHECK_RANGE_r22 859 +#define _ITER_CHECK_RANGE_r33 860 +#define _ITER_CHECK_TUPLE_r00 861 +#define _ITER_CHECK_TUPLE_r11 862 +#define _ITER_CHECK_TUPLE_r22 863 +#define _ITER_CHECK_TUPLE_r33 864 +#define _ITER_JUMP_LIST_r00 865 +#define _ITER_JUMP_LIST_r11 866 +#define _ITER_JUMP_LIST_r22 867 +#define _ITER_JUMP_LIST_r33 868 +#define _ITER_JUMP_RANGE_r00 869 +#define _ITER_JUMP_RANGE_r11 870 +#define _ITER_JUMP_RANGE_r22 871 +#define _ITER_JUMP_RANGE_r33 872 +#define _ITER_JUMP_TUPLE_r00 873 +#define _ITER_JUMP_TUPLE_r11 874 +#define _ITER_JUMP_TUPLE_r22 875 +#define _ITER_JUMP_TUPLE_r33 876 +#define _ITER_NEXT_LIST_r01 877 +#define _ITER_NEXT_LIST_TIER_TWO_r01 878 +#define _ITER_NEXT_RANGE_r23 879 +#define _ITER_NEXT_TUPLE_r23 880 +#define _JUMP_TO_TOP_r00 881 +#define _LIST_APPEND_r10 882 +#define _LIST_EXTEND_r00 883 +#define _LOAD_ATTR_r00 884 +#define _LOAD_ATTR_CLASS_r11 885 +#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_r11 886 +#define _LOAD_ATTR_INSTANCE_VALUE_r11 887 +#define _LOAD_ATTR_METHOD_LAZY_DICT_r01 888 +#define _LOAD_ATTR_METHOD_LAZY_DICT_r12 889 +#define _LOAD_ATTR_METHOD_LAZY_DICT_r23 890 +#define _LOAD_ATTR_METHOD_NO_DICT_r01 891 +#define _LOAD_ATTR_METHOD_NO_DICT_r12 892 +#define _LOAD_ATTR_METHOD_NO_DICT_r23 893 +#define _LOAD_ATTR_METHOD_WITH_VALUES_r01 894 +#define _LOAD_ATTR_METHOD_WITH_VALUES_r12 895 +#define _LOAD_ATTR_METHOD_WITH_VALUES_r23 896 +#define _LOAD_ATTR_MODULE_r11 897 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11 898 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11 899 +#define _LOAD_ATTR_PROPERTY_FRAME_r11 900 +#define _LOAD_ATTR_SLOT_r11 901 +#define _LOAD_ATTR_WITH_HINT_r11 902 +#define _LOAD_BUILD_CLASS_r01 903 +#define _LOAD_BYTECODE_r00 904 +#define _LOAD_COMMON_CONSTANT_r01 905 +#define _LOAD_COMMON_CONSTANT_r12 906 +#define _LOAD_COMMON_CONSTANT_r23 907 +#define _LOAD_CONST_r01 908 +#define _LOAD_CONST_r12 909 +#define _LOAD_CONST_r23 910 +#define _LOAD_CONST_INLINE_r01 911 +#define _LOAD_CONST_INLINE_r12 912 +#define _LOAD_CONST_INLINE_r23 913 +#define _LOAD_CONST_INLINE_BORROW_r01 914 +#define _LOAD_CONST_INLINE_BORROW_r12 915 +#define _LOAD_CONST_INLINE_BORROW_r23 916 +#define _LOAD_CONST_UNDER_INLINE_r01 917 +#define _LOAD_CONST_UNDER_INLINE_r12 918 +#define _LOAD_CONST_UNDER_INLINE_r23 919 +#define _LOAD_CONST_UNDER_INLINE_BORROW_r01 920 +#define _LOAD_CONST_UNDER_INLINE_BORROW_r12 921 +#define _LOAD_CONST_UNDER_INLINE_BORROW_r23 922 +#define _LOAD_DEREF_r01 923 +#define _LOAD_FAST_r01 924 +#define _LOAD_FAST_r12 925 +#define _LOAD_FAST_r23 926 +#define _LOAD_FAST_0_r01 927 +#define _LOAD_FAST_0_r12 928 +#define _LOAD_FAST_0_r23 929 +#define _LOAD_FAST_1_r01 930 +#define _LOAD_FAST_1_r12 931 +#define _LOAD_FAST_1_r23 932 +#define _LOAD_FAST_2_r01 933 +#define _LOAD_FAST_2_r12 934 +#define _LOAD_FAST_2_r23 935 +#define _LOAD_FAST_3_r01 936 +#define _LOAD_FAST_3_r12 937 +#define _LOAD_FAST_3_r23 938 +#define _LOAD_FAST_4_r01 939 +#define _LOAD_FAST_4_r12 940 +#define _LOAD_FAST_4_r23 941 +#define _LOAD_FAST_5_r01 942 +#define _LOAD_FAST_5_r12 943 +#define _LOAD_FAST_5_r23 944 +#define _LOAD_FAST_6_r01 945 +#define _LOAD_FAST_6_r12 946 +#define _LOAD_FAST_6_r23 947 +#define _LOAD_FAST_7_r01 948 +#define _LOAD_FAST_7_r12 949 +#define _LOAD_FAST_7_r23 950 +#define _LOAD_FAST_AND_CLEAR_r01 951 +#define _LOAD_FAST_AND_CLEAR_r12 952 +#define _LOAD_FAST_AND_CLEAR_r23 953 +#define _LOAD_FAST_BORROW_r01 954 +#define _LOAD_FAST_BORROW_r12 955 +#define _LOAD_FAST_BORROW_r23 956 +#define _LOAD_FAST_BORROW_0_r01 957 +#define _LOAD_FAST_BORROW_0_r12 958 +#define _LOAD_FAST_BORROW_0_r23 959 +#define _LOAD_FAST_BORROW_1_r01 960 +#define _LOAD_FAST_BORROW_1_r12 961 +#define _LOAD_FAST_BORROW_1_r23 962 +#define _LOAD_FAST_BORROW_2_r01 963 +#define _LOAD_FAST_BORROW_2_r12 964 +#define _LOAD_FAST_BORROW_2_r23 965 +#define _LOAD_FAST_BORROW_3_r01 966 +#define _LOAD_FAST_BORROW_3_r12 967 +#define _LOAD_FAST_BORROW_3_r23 968 +#define _LOAD_FAST_BORROW_4_r01 969 +#define _LOAD_FAST_BORROW_4_r12 970 +#define _LOAD_FAST_BORROW_4_r23 971 +#define _LOAD_FAST_BORROW_5_r01 972 +#define _LOAD_FAST_BORROW_5_r12 973 +#define _LOAD_FAST_BORROW_5_r23 974 +#define _LOAD_FAST_BORROW_6_r01 975 +#define _LOAD_FAST_BORROW_6_r12 976 +#define _LOAD_FAST_BORROW_6_r23 977 +#define _LOAD_FAST_BORROW_7_r01 978 +#define _LOAD_FAST_BORROW_7_r12 979 +#define _LOAD_FAST_BORROW_7_r23 980 +#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r02 981 +#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r13 982 +#define _LOAD_FAST_CHECK_r01 983 +#define _LOAD_FAST_LOAD_FAST_r02 984 +#define _LOAD_FAST_LOAD_FAST_r13 985 +#define _LOAD_FROM_DICT_OR_DEREF_r01 986 +#define _LOAD_FROM_DICT_OR_GLOBALS_r01 987 +#define _LOAD_GLOBAL_r00 988 +#define _LOAD_GLOBAL_BUILTINS_r01 989 +#define _LOAD_GLOBAL_MODULE_r01 990 +#define _LOAD_LOCALS_r01 991 +#define _LOAD_NAME_r01 992 +#define _LOAD_SMALL_INT_r01 993 +#define _LOAD_SMALL_INT_r12 994 +#define _LOAD_SMALL_INT_r23 995 +#define _LOAD_SMALL_INT_0_r01 996 +#define _LOAD_SMALL_INT_0_r12 997 +#define _LOAD_SMALL_INT_0_r23 998 +#define _LOAD_SMALL_INT_1_r01 999 +#define _LOAD_SMALL_INT_1_r12 1000 +#define _LOAD_SMALL_INT_1_r23 1001 +#define _LOAD_SMALL_INT_2_r01 1002 +#define _LOAD_SMALL_INT_2_r12 1003 +#define _LOAD_SMALL_INT_2_r23 1004 +#define _LOAD_SMALL_INT_3_r01 1005 +#define _LOAD_SMALL_INT_3_r12 1006 +#define _LOAD_SMALL_INT_3_r23 1007 +#define _LOAD_SPECIAL_r00 1008 +#define _LOAD_SUPER_ATTR_ATTR_r01 1009 +#define _LOAD_SUPER_ATTR_METHOD_r02 1010 +#define _MAKE_CALLARGS_A_TUPLE_r00 1011 +#define _MAKE_CELL_r00 1012 +#define _MAKE_FUNCTION_r01 1013 +#define _MAKE_WARM_r00 1014 +#define _MAKE_WARM_r11 1015 +#define _MAKE_WARM_r22 1016 +#define _MAKE_WARM_r33 1017 +#define _MAP_ADD_r00 1018 +#define _MATCH_CLASS_r01 1019 +#define _MATCH_KEYS_r01 1020 +#define _MATCH_MAPPING_r01 1021 +#define _MATCH_MAPPING_r12 1022 +#define _MATCH_MAPPING_r23 1023 +#define _MATCH_SEQUENCE_r01 1024 +#define _MATCH_SEQUENCE_r12 1025 +#define _MATCH_SEQUENCE_r23 1026 +#define _MAYBE_EXPAND_METHOD_r00 1027 +#define _MAYBE_EXPAND_METHOD_KW_r00 1028 +#define _MONITOR_CALL_r00 1029 +#define _MONITOR_CALL_KW_r00 1030 +#define _MONITOR_JUMP_BACKWARD_r00 1031 +#define _MONITOR_JUMP_BACKWARD_r11 1032 +#define _MONITOR_JUMP_BACKWARD_r22 1033 +#define _MONITOR_JUMP_BACKWARD_r33 1034 +#define _MONITOR_RESUME_r00 1035 +#define _NOP_r00 1036 +#define _NOP_r11 1037 +#define _NOP_r22 1038 +#define _NOP_r33 1039 +#define _POP_CALL_r20 1040 +#define _POP_CALL_LOAD_CONST_INLINE_BORROW_r21 1041 +#define _POP_CALL_ONE_r30 1042 +#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW_r31 1043 +#define _POP_CALL_TWO_r30 1044 +#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW_r31 1045 +#define _POP_EXCEPT_r00 1046 +#define _POP_ITER_r20 1047 +#define _POP_JUMP_IF_FALSE_r10 1048 +#define _POP_JUMP_IF_FALSE_r21 1049 +#define _POP_JUMP_IF_FALSE_r32 1050 +#define _POP_JUMP_IF_TRUE_r10 1051 +#define _POP_JUMP_IF_TRUE_r21 1052 +#define _POP_JUMP_IF_TRUE_r32 1053 +#define _POP_TOP_r10 1054 +#define _POP_TOP_FLOAT_r10 1055 +#define _POP_TOP_FLOAT_r21 1056 +#define _POP_TOP_FLOAT_r32 1057 +#define _POP_TOP_INT_r10 1058 +#define _POP_TOP_INT_r21 1059 +#define _POP_TOP_INT_r32 1060 +#define _POP_TOP_LOAD_CONST_INLINE_r11 1061 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW_r11 1062 +#define _POP_TOP_NOP_r10 1063 +#define _POP_TOP_NOP_r21 1064 +#define _POP_TOP_NOP_r32 1065 +#define _POP_TOP_UNICODE_r10 1066 +#define _POP_TOP_UNICODE_r21 1067 +#define _POP_TOP_UNICODE_r32 1068 +#define _POP_TWO_r20 1069 +#define _POP_TWO_LOAD_CONST_INLINE_BORROW_r21 1070 +#define _PUSH_EXC_INFO_r01 1071 +#define _PUSH_EXC_INFO_r12 1072 +#define _PUSH_EXC_INFO_r23 1073 +#define _PUSH_FRAME_r10 1074 +#define _PUSH_NULL_r01 1075 +#define _PUSH_NULL_r12 1076 +#define _PUSH_NULL_r23 1077 +#define _PUSH_NULL_CONDITIONAL_r00 1078 +#define _PY_FRAME_GENERAL_r01 1079 +#define _PY_FRAME_KW_r01 1080 +#define _QUICKEN_RESUME_r00 1081 +#define _QUICKEN_RESUME_r11 1082 +#define _QUICKEN_RESUME_r22 1083 +#define _QUICKEN_RESUME_r33 1084 +#define _REPLACE_WITH_TRUE_r11 1085 +#define _RESUME_CHECK_r00 1086 +#define _RESUME_CHECK_r11 1087 +#define _RESUME_CHECK_r22 1088 +#define _RESUME_CHECK_r33 1089 +#define _RETURN_GENERATOR_r01 1090 +#define _RETURN_VALUE_r01 1091 +#define _SAVE_RETURN_OFFSET_r00 1092 +#define _SAVE_RETURN_OFFSET_r11 1093 +#define _SAVE_RETURN_OFFSET_r22 1094 +#define _SAVE_RETURN_OFFSET_r33 1095 +#define _SEND_r01 1096 +#define _SEND_GEN_FRAME_r22 1097 +#define _SETUP_ANNOTATIONS_r00 1098 +#define _SET_ADD_r00 1099 +#define _SET_FUNCTION_ATTRIBUTE_r21 1100 +#define _SET_IP_r00 1101 +#define _SET_IP_r11 1102 +#define _SET_IP_r22 1103 +#define _SET_IP_r33 1104 +#define _SET_UPDATE_r00 1105 +#define _SPILL_OR_RELOAD_r01 1106 +#define _SPILL_OR_RELOAD_r02 1107 +#define _SPILL_OR_RELOAD_r03 1108 +#define _SPILL_OR_RELOAD_r10 1109 +#define _SPILL_OR_RELOAD_r12 1110 +#define _SPILL_OR_RELOAD_r13 1111 +#define _SPILL_OR_RELOAD_r20 1112 +#define _SPILL_OR_RELOAD_r21 1113 +#define _SPILL_OR_RELOAD_r23 1114 +#define _SPILL_OR_RELOAD_r30 1115 +#define _SPILL_OR_RELOAD_r31 1116 +#define _SPILL_OR_RELOAD_r32 1117 +#define _START_EXECUTOR_r00 1118 +#define _STORE_ATTR_r00 1119 +#define _STORE_ATTR_INSTANCE_VALUE_r20 1120 +#define _STORE_ATTR_SLOT_r20 1121 +#define _STORE_ATTR_WITH_HINT_r00 1122 +#define _STORE_DEREF_r00 1123 +#define _STORE_FAST_r10 1124 +#define _STORE_FAST_0_r10 1125 +#define _STORE_FAST_1_r10 1126 +#define _STORE_FAST_2_r10 1127 +#define _STORE_FAST_3_r10 1128 +#define _STORE_FAST_4_r10 1129 +#define _STORE_FAST_5_r10 1130 +#define _STORE_FAST_6_r10 1131 +#define _STORE_FAST_7_r10 1132 +#define _STORE_FAST_LOAD_FAST_r11 1133 +#define _STORE_FAST_STORE_FAST_r20 1134 +#define _STORE_GLOBAL_r00 1135 +#define _STORE_NAME_r00 1136 +#define _STORE_SLICE_r00 1137 +#define _STORE_SUBSCR_r00 1138 +#define _STORE_SUBSCR_DICT_r00 1139 +#define _STORE_SUBSCR_LIST_INT_r30 1140 +#define _SWAP_r11 1141 +#define _SWAP_2_r00 1142 +#define _SWAP_2_r11 1143 +#define _SWAP_2_r22 1144 +#define _SWAP_2_r33 1145 +#define _SWAP_3_r33 1146 +#define _TIER2_RESUME_CHECK_r00 1147 +#define _TIER2_RESUME_CHECK_r11 1148 +#define _TIER2_RESUME_CHECK_r22 1149 +#define _TIER2_RESUME_CHECK_r33 1150 +#define _TO_BOOL_r01 1151 +#define _TO_BOOL_BOOL_r00 1152 +#define _TO_BOOL_BOOL_r11 1153 +#define _TO_BOOL_BOOL_r22 1154 +#define _TO_BOOL_BOOL_r33 1155 +#define _TO_BOOL_INT_r11 1156 +#define _TO_BOOL_LIST_r11 1157 +#define _TO_BOOL_NONE_r00 1158 +#define _TO_BOOL_NONE_r11 1159 +#define _TO_BOOL_NONE_r22 1160 +#define _TO_BOOL_NONE_r33 1161 +#define _TO_BOOL_STR_r11 1162 +#define _UNARY_INVERT_r01 1163 +#define _UNARY_NEGATIVE_r01 1164 +#define _UNARY_NOT_r00 1165 +#define _UNARY_NOT_r11 1166 +#define _UNARY_NOT_r22 1167 +#define _UNARY_NOT_r33 1168 +#define _UNPACK_EX_r00 1169 +#define _UNPACK_SEQUENCE_r00 1170 +#define _UNPACK_SEQUENCE_LIST_r10 1171 +#define _UNPACK_SEQUENCE_TUPLE_r10 1172 +#define _UNPACK_SEQUENCE_TWO_TUPLE_r12 1173 +#define _WITH_EXCEPT_START_r01 1174 +#define _YIELD_VALUE_r11 1175 +#define MAX_UOP_REGS_ID 1175 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index ff7e800aa9bb1a..82bebf38f60f87 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -14,10 +14,19 @@ extern "C" { extern const uint16_t _PyUop_Flags[MAX_UOP_ID+1]; typedef struct _rep_range { uint8_t start; uint8_t stop; } ReplicationRange; extern const ReplicationRange _PyUop_Replication[MAX_UOP_ID+1]; -extern const char * const _PyOpcode_uop_name[MAX_UOP_ID+1]; +extern const char * const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1]; extern int _PyUop_num_popped(int opcode, int oparg); +typedef struct _pyuop_info { + int8_t min_input; int8_t max_input; int8_t delta; + int8_t exit_depth_is_output; uint16_t opcodes[4]; +} _PyUopCachingInfo; +extern const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1]; + +extern const uint16_t _PyUop_SpillsAndReloads[4][4]; +extern const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1]; + #ifdef NEED_OPCODE_METADATA const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_NOP] = HAS_PURE_FLAG, @@ -44,8 +53,6 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_FAST_BORROW_7] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, [_LOAD_FAST_BORROW] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG, [_LOAD_FAST_AND_CLEAR] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, - [_LOAD_FAST_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, - [_LOAD_FAST_BORROW_LOAD_FAST_BORROW] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_LOAD_CONST] = HAS_ARG_FLAG | HAS_CONST_FLAG, [_LOAD_SMALL_INT_0] = 0, [_LOAD_SMALL_INT_1] = 0, @@ -61,8 +68,6 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_STORE_FAST_6] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, [_STORE_FAST_7] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, [_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, - [_STORE_FAST_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, - [_STORE_FAST_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, [_POP_TOP] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_POP_TOP_NOP] = 0, [_POP_TOP_INT] = 0, @@ -237,7 +242,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CHECK_ATTR_METHOD_LAZY_DICT] = HAS_DEOPT_FLAG, [_LOAD_ATTR_METHOD_LAZY_DICT] = HAS_ARG_FLAG, [_MAYBE_EXPAND_METHOD] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, - [_PY_FRAME_GENERAL] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_PY_FRAME_GENERAL] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG, [_CHECK_FUNCTION_VERSION] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_CHECK_FUNCTION_VERSION_INLINE] = HAS_EXIT_FLAG, [_CHECK_METHOD_VERSION] = HAS_ARG_FLAG | HAS_EXIT_FLAG, @@ -256,7 +261,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_INIT_CALL_PY_EXACT_ARGS_3] = HAS_PURE_FLAG, [_INIT_CALL_PY_EXACT_ARGS_4] = HAS_PURE_FLAG, [_INIT_CALL_PY_EXACT_ARGS] = HAS_ARG_FLAG | HAS_PURE_FLAG, - [_PUSH_FRAME] = 0, + [_PUSH_FRAME] = HAS_SYNC_SP_FLAG, [_GUARD_NOS_NULL] = HAS_DEOPT_FLAG, [_GUARD_NOS_NOT_NULL] = HAS_EXIT_FLAG, [_GUARD_THIRD_NULL] = HAS_DEOPT_FLAG, @@ -267,7 +272,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_GUARD_CALLABLE_TUPLE_1] = HAS_DEOPT_FLAG, [_CALL_TUPLE_1] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CHECK_AND_ALLOCATE_OBJECT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_CREATE_INIT_FRAME] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CREATE_INIT_FRAME] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG, [_EXIT_INIT_CHECK] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_CALL_BUILTIN_CLASS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_BUILTIN_O] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -284,7 +289,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CALL_METHOD_DESCRIPTOR_NOARGS] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_MAYBE_EXPAND_METHOD_KW] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, - [_PY_FRAME_KW] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_PY_FRAME_KW] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG, [_CHECK_FUNCTION_VERSION_KW] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_CHECK_METHOD_VERSION_KW] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_EXPAND_METHOD_KW] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, @@ -306,15 +311,15 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_SWAP_2] = HAS_PURE_FLAG, [_SWAP_3] = HAS_PURE_FLAG, [_SWAP] = HAS_ARG_FLAG | HAS_PURE_FLAG, - [_GUARD_IS_TRUE_POP] = HAS_EXIT_FLAG, - [_GUARD_IS_FALSE_POP] = HAS_EXIT_FLAG, - [_GUARD_IS_NONE_POP] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, - [_GUARD_IS_NOT_NONE_POP] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_IS_TRUE_POP] = HAS_EXIT_FLAG | HAS_SYNC_SP_FLAG, + [_GUARD_IS_FALSE_POP] = HAS_EXIT_FLAG | HAS_SYNC_SP_FLAG, + [_GUARD_IS_NONE_POP] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG, + [_GUARD_IS_NOT_NONE_POP] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG, [_JUMP_TO_TOP] = 0, [_SET_IP] = 0, [_CHECK_STACK_SPACE_OPERAND] = HAS_DEOPT_FLAG, [_SAVE_RETURN_OFFSET] = HAS_ARG_FLAG, - [_EXIT_TRACE] = HAS_ESCAPES_FLAG, + [_EXIT_TRACE] = HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG, [_CHECK_VALIDITY] = HAS_DEOPT_FLAG, [_LOAD_CONST_INLINE] = HAS_PURE_FLAG, [_POP_TOP_LOAD_CONST_INLINE] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, @@ -333,9 +338,10 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_START_EXECUTOR] = 0, [_MAKE_WARM] = 0, [_FATAL_ERROR] = 0, - [_DEOPT] = 0, - [_ERROR_POP_N] = HAS_ARG_FLAG, + [_DEOPT] = HAS_SYNC_SP_FLAG, + [_ERROR_POP_N] = HAS_ARG_FLAG | HAS_SYNC_SP_FLAG, [_TIER2_RESUME_CHECK] = HAS_DEOPT_FLAG, + [_SPILL_OR_RELOAD] = 0, }; const ReplicationRange _PyUop_Replication[MAX_UOP_ID+1] = { @@ -348,323 +354,1781 @@ const ReplicationRange _PyUop_Replication[MAX_UOP_ID+1] = { [_SWAP] = { 2, 4 }, }; -const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { +const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { + [_NOP] = { 0, 3, 0, 1, { _NOP_r00, _NOP_r11, _NOP_r22, _NOP_r33 } }, + [_CHECK_PERIODIC] = { 0, 0, 0, 1, { _CHECK_PERIODIC_r00, 0, 0, 0 } }, + [_CHECK_PERIODIC_IF_NOT_YIELD_FROM] = { 0, 0, 0, 1, { _CHECK_PERIODIC_IF_NOT_YIELD_FROM_r00, 0, 0, 0 } }, + [_RESUME_CHECK] = { 0, 3, 0, 1, { _RESUME_CHECK_r00, _RESUME_CHECK_r11, _RESUME_CHECK_r22, _RESUME_CHECK_r33 } }, + [_LOAD_FAST_CHECK] = { 0, 0, 1, 0, { _LOAD_FAST_CHECK_r01, 0, 0, 0 } }, + [_LOAD_FAST_0] = { 0, 2, 1, 0, { _LOAD_FAST_0_r01, _LOAD_FAST_0_r12, _LOAD_FAST_0_r23, 0 } }, + [_LOAD_FAST_1] = { 0, 2, 1, 0, { _LOAD_FAST_1_r01, _LOAD_FAST_1_r12, _LOAD_FAST_1_r23, 0 } }, + [_LOAD_FAST_2] = { 0, 2, 1, 0, { _LOAD_FAST_2_r01, _LOAD_FAST_2_r12, _LOAD_FAST_2_r23, 0 } }, + [_LOAD_FAST_3] = { 0, 2, 1, 0, { _LOAD_FAST_3_r01, _LOAD_FAST_3_r12, _LOAD_FAST_3_r23, 0 } }, + [_LOAD_FAST_4] = { 0, 2, 1, 0, { _LOAD_FAST_4_r01, _LOAD_FAST_4_r12, _LOAD_FAST_4_r23, 0 } }, + [_LOAD_FAST_5] = { 0, 2, 1, 0, { _LOAD_FAST_5_r01, _LOAD_FAST_5_r12, _LOAD_FAST_5_r23, 0 } }, + [_LOAD_FAST_6] = { 0, 2, 1, 0, { _LOAD_FAST_6_r01, _LOAD_FAST_6_r12, _LOAD_FAST_6_r23, 0 } }, + [_LOAD_FAST_7] = { 0, 2, 1, 0, { _LOAD_FAST_7_r01, _LOAD_FAST_7_r12, _LOAD_FAST_7_r23, 0 } }, + [_LOAD_FAST] = { 0, 2, 1, 0, { _LOAD_FAST_r01, _LOAD_FAST_r12, _LOAD_FAST_r23, 0 } }, + [_LOAD_FAST_BORROW_0] = { 0, 2, 1, 0, { _LOAD_FAST_BORROW_0_r01, _LOAD_FAST_BORROW_0_r12, _LOAD_FAST_BORROW_0_r23, 0 } }, + [_LOAD_FAST_BORROW_1] = { 0, 2, 1, 0, { _LOAD_FAST_BORROW_1_r01, _LOAD_FAST_BORROW_1_r12, _LOAD_FAST_BORROW_1_r23, 0 } }, + [_LOAD_FAST_BORROW_2] = { 0, 2, 1, 0, { _LOAD_FAST_BORROW_2_r01, _LOAD_FAST_BORROW_2_r12, _LOAD_FAST_BORROW_2_r23, 0 } }, + [_LOAD_FAST_BORROW_3] = { 0, 2, 1, 0, { _LOAD_FAST_BORROW_3_r01, _LOAD_FAST_BORROW_3_r12, _LOAD_FAST_BORROW_3_r23, 0 } }, + [_LOAD_FAST_BORROW_4] = { 0, 2, 1, 0, { _LOAD_FAST_BORROW_4_r01, _LOAD_FAST_BORROW_4_r12, _LOAD_FAST_BORROW_4_r23, 0 } }, + [_LOAD_FAST_BORROW_5] = { 0, 2, 1, 0, { _LOAD_FAST_BORROW_5_r01, _LOAD_FAST_BORROW_5_r12, _LOAD_FAST_BORROW_5_r23, 0 } }, + [_LOAD_FAST_BORROW_6] = { 0, 2, 1, 0, { _LOAD_FAST_BORROW_6_r01, _LOAD_FAST_BORROW_6_r12, _LOAD_FAST_BORROW_6_r23, 0 } }, + [_LOAD_FAST_BORROW_7] = { 0, 2, 1, 0, { _LOAD_FAST_BORROW_7_r01, _LOAD_FAST_BORROW_7_r12, _LOAD_FAST_BORROW_7_r23, 0 } }, + [_LOAD_FAST_BORROW] = { 0, 2, 1, 0, { _LOAD_FAST_BORROW_r01, _LOAD_FAST_BORROW_r12, _LOAD_FAST_BORROW_r23, 0 } }, + [_LOAD_FAST_AND_CLEAR] = { 0, 2, 1, 0, { _LOAD_FAST_AND_CLEAR_r01, _LOAD_FAST_AND_CLEAR_r12, _LOAD_FAST_AND_CLEAR_r23, 0 } }, + [_LOAD_CONST] = { 0, 2, 1, 0, { _LOAD_CONST_r01, _LOAD_CONST_r12, _LOAD_CONST_r23, 0 } }, + [_LOAD_SMALL_INT_0] = { 0, 2, 1, 0, { _LOAD_SMALL_INT_0_r01, _LOAD_SMALL_INT_0_r12, _LOAD_SMALL_INT_0_r23, 0 } }, + [_LOAD_SMALL_INT_1] = { 0, 2, 1, 0, { _LOAD_SMALL_INT_1_r01, _LOAD_SMALL_INT_1_r12, _LOAD_SMALL_INT_1_r23, 0 } }, + [_LOAD_SMALL_INT_2] = { 0, 2, 1, 0, { _LOAD_SMALL_INT_2_r01, _LOAD_SMALL_INT_2_r12, _LOAD_SMALL_INT_2_r23, 0 } }, + [_LOAD_SMALL_INT_3] = { 0, 2, 1, 0, { _LOAD_SMALL_INT_3_r01, _LOAD_SMALL_INT_3_r12, _LOAD_SMALL_INT_3_r23, 0 } }, + [_LOAD_SMALL_INT] = { 0, 2, 1, 0, { _LOAD_SMALL_INT_r01, _LOAD_SMALL_INT_r12, _LOAD_SMALL_INT_r23, 0 } }, + [_STORE_FAST_0] = { 1, 1, -1, 0, { 0, _STORE_FAST_0_r10, 0, 0 } }, + [_STORE_FAST_1] = { 1, 1, -1, 0, { 0, _STORE_FAST_1_r10, 0, 0 } }, + [_STORE_FAST_2] = { 1, 1, -1, 0, { 0, _STORE_FAST_2_r10, 0, 0 } }, + [_STORE_FAST_3] = { 1, 1, -1, 0, { 0, _STORE_FAST_3_r10, 0, 0 } }, + [_STORE_FAST_4] = { 1, 1, -1, 0, { 0, _STORE_FAST_4_r10, 0, 0 } }, + [_STORE_FAST_5] = { 1, 1, -1, 0, { 0, _STORE_FAST_5_r10, 0, 0 } }, + [_STORE_FAST_6] = { 1, 1, -1, 0, { 0, _STORE_FAST_6_r10, 0, 0 } }, + [_STORE_FAST_7] = { 1, 1, -1, 0, { 0, _STORE_FAST_7_r10, 0, 0 } }, + [_STORE_FAST] = { 1, 1, -1, 0, { 0, _STORE_FAST_r10, 0, 0 } }, + [_POP_TOP] = { 1, 1, -1, 0, { 0, _POP_TOP_r10, 0, 0 } }, + [_POP_TOP_NOP] = { 1, 3, -1, 0, { 0, _POP_TOP_NOP_r10, _POP_TOP_NOP_r21, _POP_TOP_NOP_r32 } }, + [_POP_TOP_INT] = { 1, 3, -1, 0, { 0, _POP_TOP_INT_r10, _POP_TOP_INT_r21, _POP_TOP_INT_r32 } }, + [_POP_TOP_FLOAT] = { 1, 3, -1, 0, { 0, _POP_TOP_FLOAT_r10, _POP_TOP_FLOAT_r21, _POP_TOP_FLOAT_r32 } }, + [_POP_TOP_UNICODE] = { 1, 3, -1, 0, { 0, _POP_TOP_UNICODE_r10, _POP_TOP_UNICODE_r21, _POP_TOP_UNICODE_r32 } }, + [_POP_TWO] = { 2, 2, -2, 0, { 0, 0, _POP_TWO_r20, 0 } }, + [_PUSH_NULL] = { 0, 2, 1, 0, { _PUSH_NULL_r01, _PUSH_NULL_r12, _PUSH_NULL_r23, 0 } }, + [_END_FOR] = { 1, 1, -1, 0, { 0, _END_FOR_r10, 0, 0 } }, + [_POP_ITER] = { 2, 2, -2, 0, { 0, 0, _POP_ITER_r20, 0 } }, + [_END_SEND] = { 2, 2, -1, 0, { 0, 0, _END_SEND_r21, 0 } }, + [_UNARY_NEGATIVE] = { 0, 0, 1, 0, { _UNARY_NEGATIVE_r01, 0, 0, 0 } }, + [_UNARY_NOT] = { 0, 3, 0, 1, { _UNARY_NOT_r00, _UNARY_NOT_r11, _UNARY_NOT_r22, _UNARY_NOT_r33 } }, + [_TO_BOOL] = { 0, 0, 1, 0, { _TO_BOOL_r01, 0, 0, 0 } }, + [_TO_BOOL_BOOL] = { 0, 3, 0, 1, { _TO_BOOL_BOOL_r00, _TO_BOOL_BOOL_r11, _TO_BOOL_BOOL_r22, _TO_BOOL_BOOL_r33 } }, + [_TO_BOOL_INT] = { 1, 1, 0, 1, { 0, _TO_BOOL_INT_r11, 0, 0 } }, + [_GUARD_NOS_LIST] = { 0, 3, 0, 1, { _GUARD_NOS_LIST_r00, _GUARD_NOS_LIST_r11, _GUARD_NOS_LIST_r22, _GUARD_NOS_LIST_r33 } }, + [_GUARD_TOS_LIST] = { 0, 3, 0, 1, { _GUARD_TOS_LIST_r00, _GUARD_TOS_LIST_r11, _GUARD_TOS_LIST_r22, _GUARD_TOS_LIST_r33 } }, + [_GUARD_TOS_SLICE] = { 0, 3, 0, 1, { _GUARD_TOS_SLICE_r00, _GUARD_TOS_SLICE_r11, _GUARD_TOS_SLICE_r22, _GUARD_TOS_SLICE_r33 } }, + [_TO_BOOL_LIST] = { 1, 1, 0, 1, { 0, _TO_BOOL_LIST_r11, 0, 0 } }, + [_TO_BOOL_NONE] = { 0, 3, 0, 1, { _TO_BOOL_NONE_r00, _TO_BOOL_NONE_r11, _TO_BOOL_NONE_r22, _TO_BOOL_NONE_r33 } }, + [_GUARD_NOS_UNICODE] = { 0, 3, 0, 1, { _GUARD_NOS_UNICODE_r00, _GUARD_NOS_UNICODE_r11, _GUARD_NOS_UNICODE_r22, _GUARD_NOS_UNICODE_r33 } }, + [_GUARD_TOS_UNICODE] = { 0, 3, 0, 1, { _GUARD_TOS_UNICODE_r00, _GUARD_TOS_UNICODE_r11, _GUARD_TOS_UNICODE_r22, _GUARD_TOS_UNICODE_r33 } }, + [_TO_BOOL_STR] = { 1, 1, 0, 1, { 0, _TO_BOOL_STR_r11, 0, 0 } }, + [_REPLACE_WITH_TRUE] = { 1, 1, 0, 1, { 0, _REPLACE_WITH_TRUE_r11, 0, 0 } }, + [_UNARY_INVERT] = { 0, 0, 1, 0, { _UNARY_INVERT_r01, 0, 0, 0 } }, + [_GUARD_NOS_INT] = { 0, 3, 0, 1, { _GUARD_NOS_INT_r00, _GUARD_NOS_INT_r11, _GUARD_NOS_INT_r22, _GUARD_NOS_INT_r33 } }, + [_GUARD_TOS_INT] = { 0, 3, 0, 1, { _GUARD_TOS_INT_r00, _GUARD_TOS_INT_r11, _GUARD_TOS_INT_r22, _GUARD_TOS_INT_r33 } }, + [_GUARD_NOS_OVERFLOWED] = { 0, 3, 0, 1, { _GUARD_NOS_OVERFLOWED_r00, _GUARD_NOS_OVERFLOWED_r11, _GUARD_NOS_OVERFLOWED_r22, _GUARD_NOS_OVERFLOWED_r33 } }, + [_GUARD_TOS_OVERFLOWED] = { 0, 3, 0, 1, { _GUARD_TOS_OVERFLOWED_r00, _GUARD_TOS_OVERFLOWED_r11, _GUARD_TOS_OVERFLOWED_r22, _GUARD_TOS_OVERFLOWED_r33 } }, + [_BINARY_OP_MULTIPLY_INT] = { 2, 2, -1, 0, { 0, 0, _BINARY_OP_MULTIPLY_INT_r21, 0 } }, + [_BINARY_OP_ADD_INT] = { 2, 2, -1, 0, { 0, 0, _BINARY_OP_ADD_INT_r21, 0 } }, + [_BINARY_OP_SUBTRACT_INT] = { 2, 2, -1, 0, { 0, 0, _BINARY_OP_SUBTRACT_INT_r21, 0 } }, + [_GUARD_NOS_FLOAT] = { 0, 3, 0, 1, { _GUARD_NOS_FLOAT_r00, _GUARD_NOS_FLOAT_r11, _GUARD_NOS_FLOAT_r22, _GUARD_NOS_FLOAT_r33 } }, + [_GUARD_TOS_FLOAT] = { 0, 3, 0, 1, { _GUARD_TOS_FLOAT_r00, _GUARD_TOS_FLOAT_r11, _GUARD_TOS_FLOAT_r22, _GUARD_TOS_FLOAT_r33 } }, + [_BINARY_OP_MULTIPLY_FLOAT] = { 2, 2, -1, 0, { 0, 0, _BINARY_OP_MULTIPLY_FLOAT_r21, 0 } }, + [_BINARY_OP_ADD_FLOAT] = { 2, 2, -1, 0, { 0, 0, _BINARY_OP_ADD_FLOAT_r21, 0 } }, + [_BINARY_OP_SUBTRACT_FLOAT] = { 2, 2, -1, 0, { 0, 0, _BINARY_OP_SUBTRACT_FLOAT_r21, 0 } }, + [_BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS] = { 2, 2, -1, 0, { 0, 0, _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS_r21, 0 } }, + [_BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS] = { 2, 2, -1, 0, { 0, 0, _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS_r21, 0 } }, + [_BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS] = { 2, 2, -1, 0, { 0, 0, _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS_r21, 0 } }, + [_BINARY_OP_ADD_UNICODE] = { 2, 2, -1, 0, { 0, 0, _BINARY_OP_ADD_UNICODE_r21, 0 } }, + [_BINARY_OP_INPLACE_ADD_UNICODE] = { 0, 0, 0, 1, { _BINARY_OP_INPLACE_ADD_UNICODE_r00, 0, 0, 0 } }, + [_GUARD_BINARY_OP_EXTEND] = { 0, 0, 0, 1, { _GUARD_BINARY_OP_EXTEND_r00, 0, 0, 0 } }, + [_BINARY_OP_EXTEND] = { 0, 0, 1, 0, { _BINARY_OP_EXTEND_r01, 0, 0, 0 } }, + [_BINARY_SLICE] = { 0, 0, 1, 0, { _BINARY_SLICE_r01, 0, 0, 0 } }, + [_STORE_SLICE] = { 0, 0, 0, 1, { _STORE_SLICE_r00, 0, 0, 0 } }, + [_BINARY_OP_SUBSCR_LIST_INT] = { 0, 0, 1, 0, { _BINARY_OP_SUBSCR_LIST_INT_r01, 0, 0, 0 } }, + [_BINARY_OP_SUBSCR_LIST_SLICE] = { 0, 0, 1, 0, { _BINARY_OP_SUBSCR_LIST_SLICE_r01, 0, 0, 0 } }, + [_BINARY_OP_SUBSCR_STR_INT] = { 2, 2, -1, 0, { 0, 0, _BINARY_OP_SUBSCR_STR_INT_r21, 0 } }, + [_GUARD_NOS_TUPLE] = { 0, 3, 0, 1, { _GUARD_NOS_TUPLE_r00, _GUARD_NOS_TUPLE_r11, _GUARD_NOS_TUPLE_r22, _GUARD_NOS_TUPLE_r33 } }, + [_GUARD_TOS_TUPLE] = { 0, 3, 0, 1, { _GUARD_TOS_TUPLE_r00, _GUARD_TOS_TUPLE_r11, _GUARD_TOS_TUPLE_r22, _GUARD_TOS_TUPLE_r33 } }, + [_BINARY_OP_SUBSCR_TUPLE_INT] = { 2, 2, -1, 0, { 0, 0, _BINARY_OP_SUBSCR_TUPLE_INT_r21, 0 } }, + [_GUARD_NOS_DICT] = { 0, 3, 0, 1, { _GUARD_NOS_DICT_r00, _GUARD_NOS_DICT_r11, _GUARD_NOS_DICT_r22, _GUARD_NOS_DICT_r33 } }, + [_GUARD_TOS_DICT] = { 0, 3, 0, 1, { _GUARD_TOS_DICT_r00, _GUARD_TOS_DICT_r11, _GUARD_TOS_DICT_r22, _GUARD_TOS_DICT_r33 } }, + [_BINARY_OP_SUBSCR_DICT] = { 0, 0, 1, 0, { _BINARY_OP_SUBSCR_DICT_r01, 0, 0, 0 } }, + [_BINARY_OP_SUBSCR_CHECK_FUNC] = { 2, 2, 1, 0, { 0, 0, _BINARY_OP_SUBSCR_CHECK_FUNC_r23, 0 } }, + [_BINARY_OP_SUBSCR_INIT_CALL] = { 3, 3, -2, 0, { 0, 0, 0, _BINARY_OP_SUBSCR_INIT_CALL_r31 } }, + [_LIST_APPEND] = { 1, 1, -1, 0, { 0, _LIST_APPEND_r10, 0, 0 } }, + [_SET_ADD] = { 0, 0, 0, 1, { _SET_ADD_r00, 0, 0, 0 } }, + [_STORE_SUBSCR] = { 0, 0, 0, 1, { _STORE_SUBSCR_r00, 0, 0, 0 } }, + [_STORE_SUBSCR_LIST_INT] = { 3, 3, -3, 0, { 0, 0, 0, _STORE_SUBSCR_LIST_INT_r30 } }, + [_STORE_SUBSCR_DICT] = { 0, 0, 0, 1, { _STORE_SUBSCR_DICT_r00, 0, 0, 0 } }, + [_DELETE_SUBSCR] = { 0, 0, 0, 1, { _DELETE_SUBSCR_r00, 0, 0, 0 } }, + [_CALL_INTRINSIC_1] = { 0, 0, 1, 0, { _CALL_INTRINSIC_1_r01, 0, 0, 0 } }, + [_CALL_INTRINSIC_2] = { 0, 0, 1, 0, { _CALL_INTRINSIC_2_r01, 0, 0, 0 } }, + [_RETURN_VALUE] = { 0, 0, 1, 0, { _RETURN_VALUE_r01, 0, 0, 0 } }, + [_GET_AITER] = { 0, 0, 1, 0, { _GET_AITER_r01, 0, 0, 0 } }, + [_GET_ANEXT] = { 0, 0, 1, 0, { _GET_ANEXT_r01, 0, 0, 0 } }, + [_GET_AWAITABLE] = { 0, 0, 1, 0, { _GET_AWAITABLE_r01, 0, 0, 0 } }, + [_SEND_GEN_FRAME] = { 2, 2, 0, 1, { 0, 0, _SEND_GEN_FRAME_r22, 0 } }, + [_YIELD_VALUE] = { 1, 1, 0, 1, { 0, _YIELD_VALUE_r11, 0, 0 } }, + [_POP_EXCEPT] = { 0, 0, 0, 1, { _POP_EXCEPT_r00, 0, 0, 0 } }, + [_LOAD_COMMON_CONSTANT] = { 0, 2, 1, 0, { _LOAD_COMMON_CONSTANT_r01, _LOAD_COMMON_CONSTANT_r12, _LOAD_COMMON_CONSTANT_r23, 0 } }, + [_LOAD_BUILD_CLASS] = { 0, 0, 1, 0, { _LOAD_BUILD_CLASS_r01, 0, 0, 0 } }, + [_STORE_NAME] = { 0, 0, 0, 1, { _STORE_NAME_r00, 0, 0, 0 } }, + [_DELETE_NAME] = { 0, 0, 0, 1, { _DELETE_NAME_r00, 0, 0, 0 } }, + [_UNPACK_SEQUENCE] = { 0, 0, 0, 1, { _UNPACK_SEQUENCE_r00, 0, 0, 0 } }, + [_UNPACK_SEQUENCE_TWO_TUPLE] = { 1, 1, 1, 0, { 0, _UNPACK_SEQUENCE_TWO_TUPLE_r12, 0, 0 } }, + [_UNPACK_SEQUENCE_TUPLE] = { 1, 1, -1, 0, { 0, _UNPACK_SEQUENCE_TUPLE_r10, 0, 0 } }, + [_UNPACK_SEQUENCE_LIST] = { 1, 1, -1, 0, { 0, _UNPACK_SEQUENCE_LIST_r10, 0, 0 } }, + [_UNPACK_EX] = { 0, 0, 0, 1, { _UNPACK_EX_r00, 0, 0, 0 } }, + [_STORE_ATTR] = { 0, 0, 0, 1, { _STORE_ATTR_r00, 0, 0, 0 } }, + [_DELETE_ATTR] = { 0, 0, 0, 1, { _DELETE_ATTR_r00, 0, 0, 0 } }, + [_STORE_GLOBAL] = { 0, 0, 0, 1, { _STORE_GLOBAL_r00, 0, 0, 0 } }, + [_DELETE_GLOBAL] = { 0, 0, 0, 1, { _DELETE_GLOBAL_r00, 0, 0, 0 } }, + [_LOAD_LOCALS] = { 0, 0, 1, 0, { _LOAD_LOCALS_r01, 0, 0, 0 } }, + [_LOAD_NAME] = { 0, 0, 1, 0, { _LOAD_NAME_r01, 0, 0, 0 } }, + [_LOAD_GLOBAL] = { 0, 0, 0, 1, { _LOAD_GLOBAL_r00, 0, 0, 0 } }, + [_PUSH_NULL_CONDITIONAL] = { 0, 0, 0, 1, { _PUSH_NULL_CONDITIONAL_r00, 0, 0, 0 } }, + [_GUARD_GLOBALS_VERSION] = { 0, 3, 0, 1, { _GUARD_GLOBALS_VERSION_r00, _GUARD_GLOBALS_VERSION_r11, _GUARD_GLOBALS_VERSION_r22, _GUARD_GLOBALS_VERSION_r33 } }, + [_LOAD_GLOBAL_MODULE] = { 0, 0, 1, 0, { _LOAD_GLOBAL_MODULE_r01, 0, 0, 0 } }, + [_LOAD_GLOBAL_BUILTINS] = { 0, 0, 1, 0, { _LOAD_GLOBAL_BUILTINS_r01, 0, 0, 0 } }, + [_DELETE_FAST] = { 0, 0, 0, 1, { _DELETE_FAST_r00, 0, 0, 0 } }, + [_MAKE_CELL] = { 0, 0, 0, 1, { _MAKE_CELL_r00, 0, 0, 0 } }, + [_DELETE_DEREF] = { 0, 0, 0, 1, { _DELETE_DEREF_r00, 0, 0, 0 } }, + [_LOAD_FROM_DICT_OR_DEREF] = { 0, 0, 1, 0, { _LOAD_FROM_DICT_OR_DEREF_r01, 0, 0, 0 } }, + [_LOAD_DEREF] = { 0, 0, 1, 0, { _LOAD_DEREF_r01, 0, 0, 0 } }, + [_STORE_DEREF] = { 0, 0, 0, 1, { _STORE_DEREF_r00, 0, 0, 0 } }, + [_COPY_FREE_VARS] = { 0, 0, 0, 1, { _COPY_FREE_VARS_r00, 0, 0, 0 } }, + [_BUILD_STRING] = { 0, 0, 1, 0, { _BUILD_STRING_r01, 0, 0, 0 } }, + [_BUILD_INTERPOLATION] = { 0, 0, 1, 0, { _BUILD_INTERPOLATION_r01, 0, 0, 0 } }, + [_BUILD_TEMPLATE] = { 0, 0, 1, 0, { _BUILD_TEMPLATE_r01, 0, 0, 0 } }, + [_BUILD_TUPLE] = { 0, 0, 1, 0, { _BUILD_TUPLE_r01, 0, 0, 0 } }, + [_BUILD_LIST] = { 0, 0, 1, 0, { _BUILD_LIST_r01, 0, 0, 0 } }, + [_LIST_EXTEND] = { 0, 0, 0, 1, { _LIST_EXTEND_r00, 0, 0, 0 } }, + [_SET_UPDATE] = { 0, 0, 0, 1, { _SET_UPDATE_r00, 0, 0, 0 } }, + [_BUILD_SET] = { 0, 0, 1, 0, { _BUILD_SET_r01, 0, 0, 0 } }, + [_BUILD_MAP] = { 0, 0, 1, 0, { _BUILD_MAP_r01, 0, 0, 0 } }, + [_SETUP_ANNOTATIONS] = { 0, 0, 0, 1, { _SETUP_ANNOTATIONS_r00, 0, 0, 0 } }, + [_DICT_UPDATE] = { 0, 0, 0, 1, { _DICT_UPDATE_r00, 0, 0, 0 } }, + [_DICT_MERGE] = { 0, 0, 0, 1, { _DICT_MERGE_r00, 0, 0, 0 } }, + [_MAP_ADD] = { 0, 0, 0, 1, { _MAP_ADD_r00, 0, 0, 0 } }, + [_LOAD_SUPER_ATTR_ATTR] = { 0, 0, 1, 0, { _LOAD_SUPER_ATTR_ATTR_r01, 0, 0, 0 } }, + [_LOAD_SUPER_ATTR_METHOD] = { 0, 0, 2, 0, { _LOAD_SUPER_ATTR_METHOD_r02, 0, 0, 0 } }, + [_LOAD_ATTR] = { 0, 0, 0, 1, { _LOAD_ATTR_r00, 0, 0, 0 } }, + [_GUARD_TYPE_VERSION] = { 0, 3, 0, 1, { _GUARD_TYPE_VERSION_r00, _GUARD_TYPE_VERSION_r11, _GUARD_TYPE_VERSION_r22, _GUARD_TYPE_VERSION_r33 } }, + [_GUARD_TYPE_VERSION_AND_LOCK] = { 0, 3, 0, 1, { _GUARD_TYPE_VERSION_AND_LOCK_r00, _GUARD_TYPE_VERSION_AND_LOCK_r11, _GUARD_TYPE_VERSION_AND_LOCK_r22, _GUARD_TYPE_VERSION_AND_LOCK_r33 } }, + [_CHECK_MANAGED_OBJECT_HAS_VALUES] = { 0, 3, 0, 1, { _CHECK_MANAGED_OBJECT_HAS_VALUES_r00, _CHECK_MANAGED_OBJECT_HAS_VALUES_r11, _CHECK_MANAGED_OBJECT_HAS_VALUES_r22, _CHECK_MANAGED_OBJECT_HAS_VALUES_r33 } }, + [_LOAD_ATTR_INSTANCE_VALUE] = { 1, 1, 0, 1, { 0, _LOAD_ATTR_INSTANCE_VALUE_r11, 0, 0 } }, + [_LOAD_ATTR_MODULE] = { 1, 1, 0, 1, { 0, _LOAD_ATTR_MODULE_r11, 0, 0 } }, + [_LOAD_ATTR_WITH_HINT] = { 1, 1, 0, 1, { 0, _LOAD_ATTR_WITH_HINT_r11, 0, 0 } }, + [_LOAD_ATTR_SLOT] = { 1, 1, 0, 1, { 0, _LOAD_ATTR_SLOT_r11, 0, 0 } }, + [_CHECK_ATTR_CLASS] = { 0, 3, 0, 1, { _CHECK_ATTR_CLASS_r00, _CHECK_ATTR_CLASS_r11, _CHECK_ATTR_CLASS_r22, _CHECK_ATTR_CLASS_r33 } }, + [_LOAD_ATTR_CLASS] = { 1, 1, 0, 1, { 0, _LOAD_ATTR_CLASS_r11, 0, 0 } }, + [_LOAD_ATTR_PROPERTY_FRAME] = { 1, 1, 0, 1, { 0, _LOAD_ATTR_PROPERTY_FRAME_r11, 0, 0 } }, + [_GUARD_DORV_NO_DICT] = { 0, 3, 0, 1, { _GUARD_DORV_NO_DICT_r00, _GUARD_DORV_NO_DICT_r11, _GUARD_DORV_NO_DICT_r22, _GUARD_DORV_NO_DICT_r33 } }, + [_STORE_ATTR_INSTANCE_VALUE] = { 2, 2, -2, 0, { 0, 0, _STORE_ATTR_INSTANCE_VALUE_r20, 0 } }, + [_STORE_ATTR_WITH_HINT] = { 0, 0, 0, 1, { _STORE_ATTR_WITH_HINT_r00, 0, 0, 0 } }, + [_STORE_ATTR_SLOT] = { 2, 2, -2, 0, { 0, 0, _STORE_ATTR_SLOT_r20, 0 } }, + [_COMPARE_OP] = { 0, 0, 1, 0, { _COMPARE_OP_r01, 0, 0, 0 } }, + [_COMPARE_OP_FLOAT] = { 2, 2, -1, 0, { 0, 0, _COMPARE_OP_FLOAT_r21, 0 } }, + [_COMPARE_OP_INT] = { 2, 2, -1, 0, { 0, 0, _COMPARE_OP_INT_r21, 0 } }, + [_COMPARE_OP_STR] = { 2, 2, -1, 0, { 0, 0, _COMPARE_OP_STR_r21, 0 } }, + [_IS_OP] = { 2, 2, -1, 0, { 0, 0, _IS_OP_r21, 0 } }, + [_CONTAINS_OP] = { 0, 0, 1, 0, { _CONTAINS_OP_r01, 0, 0, 0 } }, + [_GUARD_TOS_ANY_SET] = { 0, 3, 0, 1, { _GUARD_TOS_ANY_SET_r00, _GUARD_TOS_ANY_SET_r11, _GUARD_TOS_ANY_SET_r22, _GUARD_TOS_ANY_SET_r33 } }, + [_CONTAINS_OP_SET] = { 0, 0, 1, 0, { _CONTAINS_OP_SET_r01, 0, 0, 0 } }, + [_CONTAINS_OP_DICT] = { 0, 0, 1, 0, { _CONTAINS_OP_DICT_r01, 0, 0, 0 } }, + [_CHECK_EG_MATCH] = { 0, 0, 2, 0, { _CHECK_EG_MATCH_r02, 0, 0, 0 } }, + [_CHECK_EXC_MATCH] = { 0, 0, 1, 0, { _CHECK_EXC_MATCH_r01, 0, 0, 0 } }, + [_IMPORT_NAME] = { 0, 0, 1, 0, { _IMPORT_NAME_r01, 0, 0, 0 } }, + [_IMPORT_FROM] = { 0, 0, 1, 0, { _IMPORT_FROM_r01, 0, 0, 0 } }, + [_IS_NONE] = { 1, 1, 0, 1, { 0, _IS_NONE_r11, 0, 0 } }, + [_GET_LEN] = { 0, 0, 1, 0, { _GET_LEN_r01, 0, 0, 0 } }, + [_MATCH_CLASS] = { 0, 0, 1, 0, { _MATCH_CLASS_r01, 0, 0, 0 } }, + [_MATCH_MAPPING] = { 0, 2, 1, 0, { _MATCH_MAPPING_r01, _MATCH_MAPPING_r12, _MATCH_MAPPING_r23, 0 } }, + [_MATCH_SEQUENCE] = { 0, 2, 1, 0, { _MATCH_SEQUENCE_r01, _MATCH_SEQUENCE_r12, _MATCH_SEQUENCE_r23, 0 } }, + [_MATCH_KEYS] = { 0, 0, 1, 0, { _MATCH_KEYS_r01, 0, 0, 0 } }, + [_GET_ITER] = { 0, 0, 2, 0, { _GET_ITER_r02, 0, 0, 0 } }, + [_GET_YIELD_FROM_ITER] = { 0, 0, 1, 0, { _GET_YIELD_FROM_ITER_r01, 0, 0, 0 } }, + [_FOR_ITER_TIER_TWO] = { 0, 0, 1, 0, { _FOR_ITER_TIER_TWO_r01, 0, 0, 0 } }, + [_ITER_CHECK_LIST] = { 0, 3, 0, 1, { _ITER_CHECK_LIST_r00, _ITER_CHECK_LIST_r11, _ITER_CHECK_LIST_r22, _ITER_CHECK_LIST_r33 } }, + [_GUARD_NOT_EXHAUSTED_LIST] = { 0, 3, 0, 1, { _GUARD_NOT_EXHAUSTED_LIST_r00, _GUARD_NOT_EXHAUSTED_LIST_r11, _GUARD_NOT_EXHAUSTED_LIST_r22, _GUARD_NOT_EXHAUSTED_LIST_r33 } }, + [_ITER_NEXT_LIST_TIER_TWO] = { 0, 0, 1, 0, { _ITER_NEXT_LIST_TIER_TWO_r01, 0, 0, 0 } }, + [_ITER_CHECK_TUPLE] = { 0, 3, 0, 1, { _ITER_CHECK_TUPLE_r00, _ITER_CHECK_TUPLE_r11, _ITER_CHECK_TUPLE_r22, _ITER_CHECK_TUPLE_r33 } }, + [_GUARD_NOT_EXHAUSTED_TUPLE] = { 0, 3, 0, 1, { _GUARD_NOT_EXHAUSTED_TUPLE_r00, _GUARD_NOT_EXHAUSTED_TUPLE_r11, _GUARD_NOT_EXHAUSTED_TUPLE_r22, _GUARD_NOT_EXHAUSTED_TUPLE_r33 } }, + [_ITER_NEXT_TUPLE] = { 2, 2, 1, 0, { 0, 0, _ITER_NEXT_TUPLE_r23, 0 } }, + [_ITER_CHECK_RANGE] = { 0, 3, 0, 1, { _ITER_CHECK_RANGE_r00, _ITER_CHECK_RANGE_r11, _ITER_CHECK_RANGE_r22, _ITER_CHECK_RANGE_r33 } }, + [_GUARD_NOT_EXHAUSTED_RANGE] = { 0, 3, 0, 1, { _GUARD_NOT_EXHAUSTED_RANGE_r00, _GUARD_NOT_EXHAUSTED_RANGE_r11, _GUARD_NOT_EXHAUSTED_RANGE_r22, _GUARD_NOT_EXHAUSTED_RANGE_r33 } }, + [_ITER_NEXT_RANGE] = { 2, 2, 1, 0, { 0, 0, _ITER_NEXT_RANGE_r23, 0 } }, + [_FOR_ITER_GEN_FRAME] = { 2, 2, 1, 0, { 0, 0, _FOR_ITER_GEN_FRAME_r23, 0 } }, + [_INSERT_NULL] = { 1, 1, -1, 0, { 0, _INSERT_NULL_r10, 0, 0 } }, + [_LOAD_SPECIAL] = { 0, 0, 0, 1, { _LOAD_SPECIAL_r00, 0, 0, 0 } }, + [_WITH_EXCEPT_START] = { 0, 0, 1, 0, { _WITH_EXCEPT_START_r01, 0, 0, 0 } }, + [_PUSH_EXC_INFO] = { 0, 2, 1, 0, { _PUSH_EXC_INFO_r01, _PUSH_EXC_INFO_r12, _PUSH_EXC_INFO_r23, 0 } }, + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = { 0, 3, 0, 1, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r00, _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r11, _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r22, _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r33 } }, + [_GUARD_KEYS_VERSION] = { 0, 3, 0, 1, { _GUARD_KEYS_VERSION_r00, _GUARD_KEYS_VERSION_r11, _GUARD_KEYS_VERSION_r22, _GUARD_KEYS_VERSION_r33 } }, + [_LOAD_ATTR_METHOD_WITH_VALUES] = { 0, 2, 1, 0, { _LOAD_ATTR_METHOD_WITH_VALUES_r01, _LOAD_ATTR_METHOD_WITH_VALUES_r12, _LOAD_ATTR_METHOD_WITH_VALUES_r23, 0 } }, + [_LOAD_ATTR_METHOD_NO_DICT] = { 0, 2, 1, 0, { _LOAD_ATTR_METHOD_NO_DICT_r01, _LOAD_ATTR_METHOD_NO_DICT_r12, _LOAD_ATTR_METHOD_NO_DICT_r23, 0 } }, + [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { 1, 1, 0, 1, { 0, _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11, 0, 0 } }, + [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { 1, 1, 0, 1, { 0, _LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11, 0, 0 } }, + [_CHECK_ATTR_METHOD_LAZY_DICT] = { 0, 3, 0, 1, { _CHECK_ATTR_METHOD_LAZY_DICT_r00, _CHECK_ATTR_METHOD_LAZY_DICT_r11, _CHECK_ATTR_METHOD_LAZY_DICT_r22, _CHECK_ATTR_METHOD_LAZY_DICT_r33 } }, + [_LOAD_ATTR_METHOD_LAZY_DICT] = { 0, 2, 1, 0, { _LOAD_ATTR_METHOD_LAZY_DICT_r01, _LOAD_ATTR_METHOD_LAZY_DICT_r12, _LOAD_ATTR_METHOD_LAZY_DICT_r23, 0 } }, + [_MAYBE_EXPAND_METHOD] = { 0, 0, 0, 1, { _MAYBE_EXPAND_METHOD_r00, 0, 0, 0 } }, + [_PY_FRAME_GENERAL] = { 0, 0, 1, 0, { _PY_FRAME_GENERAL_r01, 0, 0, 0 } }, + [_CHECK_FUNCTION_VERSION] = { 0, 0, 0, 1, { _CHECK_FUNCTION_VERSION_r00, 0, 0, 0 } }, + [_CHECK_FUNCTION_VERSION_INLINE] = { 0, 3, 0, 1, { _CHECK_FUNCTION_VERSION_INLINE_r00, _CHECK_FUNCTION_VERSION_INLINE_r11, _CHECK_FUNCTION_VERSION_INLINE_r22, _CHECK_FUNCTION_VERSION_INLINE_r33 } }, + [_CHECK_METHOD_VERSION] = { 0, 0, 0, 1, { _CHECK_METHOD_VERSION_r00, 0, 0, 0 } }, + [_EXPAND_METHOD] = { 0, 0, 0, 1, { _EXPAND_METHOD_r00, 0, 0, 0 } }, + [_CHECK_IS_NOT_PY_CALLABLE] = { 0, 0, 0, 1, { _CHECK_IS_NOT_PY_CALLABLE_r00, 0, 0, 0 } }, + [_CALL_NON_PY_GENERAL] = { 0, 0, 1, 0, { _CALL_NON_PY_GENERAL_r01, 0, 0, 0 } }, + [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = { 0, 0, 0, 1, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS_r00, 0, 0, 0 } }, + [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = { 0, 0, 0, 1, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS_r00, 0, 0, 0 } }, + [_CHECK_PEP_523] = { 0, 3, 0, 1, { _CHECK_PEP_523_r00, _CHECK_PEP_523_r11, _CHECK_PEP_523_r22, _CHECK_PEP_523_r33 } }, + [_CHECK_FUNCTION_EXACT_ARGS] = { 0, 0, 0, 1, { _CHECK_FUNCTION_EXACT_ARGS_r00, 0, 0, 0 } }, + [_CHECK_STACK_SPACE] = { 0, 0, 0, 1, { _CHECK_STACK_SPACE_r00, 0, 0, 0 } }, + [_CHECK_RECURSION_REMAINING] = { 0, 3, 0, 1, { _CHECK_RECURSION_REMAINING_r00, _CHECK_RECURSION_REMAINING_r11, _CHECK_RECURSION_REMAINING_r22, _CHECK_RECURSION_REMAINING_r33 } }, + [_INIT_CALL_PY_EXACT_ARGS_0] = { 0, 0, 1, 0, { _INIT_CALL_PY_EXACT_ARGS_0_r01, 0, 0, 0 } }, + [_INIT_CALL_PY_EXACT_ARGS_1] = { 0, 0, 1, 0, { _INIT_CALL_PY_EXACT_ARGS_1_r01, 0, 0, 0 } }, + [_INIT_CALL_PY_EXACT_ARGS_2] = { 0, 0, 1, 0, { _INIT_CALL_PY_EXACT_ARGS_2_r01, 0, 0, 0 } }, + [_INIT_CALL_PY_EXACT_ARGS_3] = { 0, 0, 1, 0, { _INIT_CALL_PY_EXACT_ARGS_3_r01, 0, 0, 0 } }, + [_INIT_CALL_PY_EXACT_ARGS_4] = { 0, 0, 1, 0, { _INIT_CALL_PY_EXACT_ARGS_4_r01, 0, 0, 0 } }, + [_INIT_CALL_PY_EXACT_ARGS] = { 0, 0, 1, 0, { _INIT_CALL_PY_EXACT_ARGS_r01, 0, 0, 0 } }, + [_PUSH_FRAME] = { 1, 1, -1, 1, { 0, _PUSH_FRAME_r10, 0, 0 } }, + [_GUARD_NOS_NULL] = { 0, 3, 0, 1, { _GUARD_NOS_NULL_r00, _GUARD_NOS_NULL_r11, _GUARD_NOS_NULL_r22, _GUARD_NOS_NULL_r33 } }, + [_GUARD_NOS_NOT_NULL] = { 0, 3, 0, 1, { _GUARD_NOS_NOT_NULL_r00, _GUARD_NOS_NOT_NULL_r11, _GUARD_NOS_NOT_NULL_r22, _GUARD_NOS_NOT_NULL_r33 } }, + [_GUARD_THIRD_NULL] = { 3, 3, 0, 1, { 0, 0, 0, _GUARD_THIRD_NULL_r33 } }, + [_GUARD_CALLABLE_TYPE_1] = { 3, 3, 0, 1, { 0, 0, 0, _GUARD_CALLABLE_TYPE_1_r33 } }, + [_CALL_TYPE_1] = { 3, 3, -2, 0, { 0, 0, 0, _CALL_TYPE_1_r31 } }, + [_GUARD_CALLABLE_STR_1] = { 3, 3, 0, 1, { 0, 0, 0, _GUARD_CALLABLE_STR_1_r33 } }, + [_CALL_STR_1] = { 0, 0, 1, 0, { _CALL_STR_1_r01, 0, 0, 0 } }, + [_GUARD_CALLABLE_TUPLE_1] = { 3, 3, 0, 1, { 0, 0, 0, _GUARD_CALLABLE_TUPLE_1_r33 } }, + [_CALL_TUPLE_1] = { 0, 0, 1, 0, { _CALL_TUPLE_1_r01, 0, 0, 0 } }, + [_CHECK_AND_ALLOCATE_OBJECT] = { 0, 0, 0, 1, { _CHECK_AND_ALLOCATE_OBJECT_r00, 0, 0, 0 } }, + [_CREATE_INIT_FRAME] = { 0, 0, 1, 0, { _CREATE_INIT_FRAME_r01, 0, 0, 0 } }, + [_EXIT_INIT_CHECK] = { 0, 0, 0, 1, { _EXIT_INIT_CHECK_r00, 0, 0, 0 } }, + [_CALL_BUILTIN_CLASS] = { 0, 0, 1, 0, { _CALL_BUILTIN_CLASS_r01, 0, 0, 0 } }, + [_CALL_BUILTIN_O] = { 0, 0, 1, 0, { _CALL_BUILTIN_O_r01, 0, 0, 0 } }, + [_CALL_BUILTIN_FAST] = { 0, 0, 1, 0, { _CALL_BUILTIN_FAST_r01, 0, 0, 0 } }, + [_CALL_BUILTIN_FAST_WITH_KEYWORDS] = { 0, 0, 1, 0, { _CALL_BUILTIN_FAST_WITH_KEYWORDS_r01, 0, 0, 0 } }, + [_GUARD_CALLABLE_LEN] = { 3, 3, 0, 1, { 0, 0, 0, _GUARD_CALLABLE_LEN_r33 } }, + [_CALL_LEN] = { 0, 0, 1, 0, { _CALL_LEN_r01, 0, 0, 0 } }, + [_GUARD_CALLABLE_ISINSTANCE] = { 3, 3, 0, 1, { 0, 0, 0, _GUARD_CALLABLE_ISINSTANCE_r33 } }, + [_CALL_ISINSTANCE] = { 0, 0, 1, 0, { _CALL_ISINSTANCE_r01, 0, 0, 0 } }, + [_GUARD_CALLABLE_LIST_APPEND] = { 3, 3, 0, 1, { 0, 0, 0, _GUARD_CALLABLE_LIST_APPEND_r33 } }, + [_CALL_LIST_APPEND] = { 3, 3, -3, 0, { 0, 0, 0, _CALL_LIST_APPEND_r30 } }, + [_CALL_METHOD_DESCRIPTOR_O] = { 0, 0, 1, 0, { _CALL_METHOD_DESCRIPTOR_O_r01, 0, 0, 0 } }, + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { 0, 0, 1, 0, { _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r01, 0, 0, 0 } }, + [_CALL_METHOD_DESCRIPTOR_NOARGS] = { 0, 0, 1, 0, { _CALL_METHOD_DESCRIPTOR_NOARGS_r01, 0, 0, 0 } }, + [_CALL_METHOD_DESCRIPTOR_FAST] = { 0, 0, 1, 0, { _CALL_METHOD_DESCRIPTOR_FAST_r01, 0, 0, 0 } }, + [_MAYBE_EXPAND_METHOD_KW] = { 0, 0, 0, 1, { _MAYBE_EXPAND_METHOD_KW_r00, 0, 0, 0 } }, + [_PY_FRAME_KW] = { 0, 0, 1, 0, { _PY_FRAME_KW_r01, 0, 0, 0 } }, + [_CHECK_FUNCTION_VERSION_KW] = { 1, 1, 0, 1, { 0, _CHECK_FUNCTION_VERSION_KW_r11, 0, 0 } }, + [_CHECK_METHOD_VERSION_KW] = { 1, 1, 0, 1, { 0, _CHECK_METHOD_VERSION_KW_r11, 0, 0 } }, + [_EXPAND_METHOD_KW] = { 0, 0, 0, 1, { _EXPAND_METHOD_KW_r00, 0, 0, 0 } }, + [_CHECK_IS_NOT_PY_CALLABLE_KW] = { 1, 1, 0, 1, { 0, _CHECK_IS_NOT_PY_CALLABLE_KW_r11, 0, 0 } }, + [_CALL_KW_NON_PY] = { 0, 0, 1, 0, { _CALL_KW_NON_PY_r01, 0, 0, 0 } }, + [_MAKE_CALLARGS_A_TUPLE] = { 0, 0, 0, 1, { _MAKE_CALLARGS_A_TUPLE_r00, 0, 0, 0 } }, + [_MAKE_FUNCTION] = { 0, 0, 1, 0, { _MAKE_FUNCTION_r01, 0, 0, 0 } }, + [_SET_FUNCTION_ATTRIBUTE] = { 2, 2, -1, 0, { 0, 0, _SET_FUNCTION_ATTRIBUTE_r21, 0 } }, + [_RETURN_GENERATOR] = { 0, 0, 1, 0, { _RETURN_GENERATOR_r01, 0, 0, 0 } }, + [_BUILD_SLICE] = { 0, 0, 1, 0, { _BUILD_SLICE_r01, 0, 0, 0 } }, + [_CONVERT_VALUE] = { 0, 0, 1, 0, { _CONVERT_VALUE_r01, 0, 0, 0 } }, + [_FORMAT_SIMPLE] = { 0, 0, 1, 0, { _FORMAT_SIMPLE_r01, 0, 0, 0 } }, + [_FORMAT_WITH_SPEC] = { 0, 0, 1, 0, { _FORMAT_WITH_SPEC_r01, 0, 0, 0 } }, + [_COPY_1] = { 0, 2, 1, 0, { _COPY_1_r01, _COPY_1_r12, _COPY_1_r23, 0 } }, + [_COPY_2] = { 2, 2, 1, 0, { 0, 0, _COPY_2_r23, 0 } }, + [_COPY_3] = { 3, 3, 0, 1, { 0, 0, 0, _COPY_3_r33 } }, + [_COPY] = { 0, 0, 1, 0, { _COPY_r01, 0, 0, 0 } }, + [_BINARY_OP] = { 0, 0, 1, 0, { _BINARY_OP_r01, 0, 0, 0 } }, + [_SWAP_2] = { 0, 3, 0, 1, { _SWAP_2_r00, _SWAP_2_r11, _SWAP_2_r22, _SWAP_2_r33 } }, + [_SWAP_3] = { 3, 3, 0, 1, { 0, 0, 0, _SWAP_3_r33 } }, + [_SWAP] = { 1, 1, 0, 1, { 0, _SWAP_r11, 0, 0 } }, + [_GUARD_IS_TRUE_POP] = { 1, 1, -1, 1, { 0, _GUARD_IS_TRUE_POP_r10, 0, 0 } }, + [_GUARD_IS_FALSE_POP] = { 1, 1, -1, 1, { 0, _GUARD_IS_FALSE_POP_r10, 0, 0 } }, + [_GUARD_IS_NONE_POP] = { 1, 1, -1, 1, { 0, _GUARD_IS_NONE_POP_r10, 0, 0 } }, + [_GUARD_IS_NOT_NONE_POP] = { 1, 1, -1, 1, { 0, _GUARD_IS_NOT_NONE_POP_r10, 0, 0 } }, + [_JUMP_TO_TOP] = { 0, 0, 0, 1, { _JUMP_TO_TOP_r00, 0, 0, 0 } }, + [_SET_IP] = { 0, 3, 0, 1, { _SET_IP_r00, _SET_IP_r11, _SET_IP_r22, _SET_IP_r33 } }, + [_CHECK_STACK_SPACE_OPERAND] = { 0, 3, 0, 1, { _CHECK_STACK_SPACE_OPERAND_r00, _CHECK_STACK_SPACE_OPERAND_r11, _CHECK_STACK_SPACE_OPERAND_r22, _CHECK_STACK_SPACE_OPERAND_r33 } }, + [_SAVE_RETURN_OFFSET] = { 0, 3, 0, 1, { _SAVE_RETURN_OFFSET_r00, _SAVE_RETURN_OFFSET_r11, _SAVE_RETURN_OFFSET_r22, _SAVE_RETURN_OFFSET_r33 } }, + [_EXIT_TRACE] = { 0, 3, -3, 0, { _EXIT_TRACE_r00, _EXIT_TRACE_r10, _EXIT_TRACE_r20, _EXIT_TRACE_r30 } }, + [_CHECK_VALIDITY] = { 0, 3, 0, 1, { _CHECK_VALIDITY_r00, _CHECK_VALIDITY_r11, _CHECK_VALIDITY_r22, _CHECK_VALIDITY_r33 } }, + [_LOAD_CONST_INLINE] = { 0, 2, 1, 0, { _LOAD_CONST_INLINE_r01, _LOAD_CONST_INLINE_r12, _LOAD_CONST_INLINE_r23, 0 } }, + [_POP_TOP_LOAD_CONST_INLINE] = { 1, 1, 0, 1, { 0, _POP_TOP_LOAD_CONST_INLINE_r11, 0, 0 } }, + [_LOAD_CONST_INLINE_BORROW] = { 0, 2, 1, 0, { _LOAD_CONST_INLINE_BORROW_r01, _LOAD_CONST_INLINE_BORROW_r12, _LOAD_CONST_INLINE_BORROW_r23, 0 } }, + [_POP_CALL] = { 2, 2, -2, 0, { 0, 0, _POP_CALL_r20, 0 } }, + [_POP_CALL_ONE] = { 3, 3, -3, 0, { 0, 0, 0, _POP_CALL_ONE_r30 } }, + [_POP_CALL_TWO] = { 3, 3, -3, 0, { 0, 0, 0, _POP_CALL_TWO_r30 } }, + [_POP_TOP_LOAD_CONST_INLINE_BORROW] = { 1, 1, 0, 1, { 0, _POP_TOP_LOAD_CONST_INLINE_BORROW_r11, 0, 0 } }, + [_POP_TWO_LOAD_CONST_INLINE_BORROW] = { 2, 2, -1, 0, { 0, 0, _POP_TWO_LOAD_CONST_INLINE_BORROW_r21, 0 } }, + [_POP_CALL_LOAD_CONST_INLINE_BORROW] = { 2, 2, -1, 0, { 0, 0, _POP_CALL_LOAD_CONST_INLINE_BORROW_r21, 0 } }, + [_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW] = { 3, 3, -2, 0, { 0, 0, 0, _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW_r31 } }, + [_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW] = { 3, 3, -2, 0, { 0, 0, 0, _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW_r31 } }, + [_LOAD_CONST_UNDER_INLINE] = { 0, 2, 1, 0, { _LOAD_CONST_UNDER_INLINE_r01, _LOAD_CONST_UNDER_INLINE_r12, _LOAD_CONST_UNDER_INLINE_r23, 0 } }, + [_LOAD_CONST_UNDER_INLINE_BORROW] = { 0, 2, 1, 0, { _LOAD_CONST_UNDER_INLINE_BORROW_r01, _LOAD_CONST_UNDER_INLINE_BORROW_r12, _LOAD_CONST_UNDER_INLINE_BORROW_r23, 0 } }, + [_CHECK_FUNCTION] = { 0, 3, 0, 1, { _CHECK_FUNCTION_r00, _CHECK_FUNCTION_r11, _CHECK_FUNCTION_r22, _CHECK_FUNCTION_r33 } }, + [_START_EXECUTOR] = { 0, 0, 0, 1, { _START_EXECUTOR_r00, 0, 0, 0 } }, + [_MAKE_WARM] = { 0, 3, 0, 1, { _MAKE_WARM_r00, _MAKE_WARM_r11, _MAKE_WARM_r22, _MAKE_WARM_r33 } }, + [_FATAL_ERROR] = { 0, 3, 0, 1, { _FATAL_ERROR_r00, _FATAL_ERROR_r11, _FATAL_ERROR_r22, _FATAL_ERROR_r33 } }, + [_DEOPT] = { 0, 3, -3, 0, { _DEOPT_r00, _DEOPT_r10, _DEOPT_r20, _DEOPT_r30 } }, + [_ERROR_POP_N] = { 0, 0, 0, 1, { _ERROR_POP_N_r00, 0, 0, 0 } }, + [_TIER2_RESUME_CHECK] = { 0, 3, 0, 1, { _TIER2_RESUME_CHECK_r00, _TIER2_RESUME_CHECK_r11, _TIER2_RESUME_CHECK_r22, _TIER2_RESUME_CHECK_r33 } }, +}; + +const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { + [_NOP_r00] = _NOP, + [_NOP_r11] = _NOP, + [_NOP_r22] = _NOP, + [_NOP_r33] = _NOP, + [_CHECK_PERIODIC_r00] = _CHECK_PERIODIC, + [_CHECK_PERIODIC_IF_NOT_YIELD_FROM_r00] = _CHECK_PERIODIC_IF_NOT_YIELD_FROM, + [_RESUME_CHECK_r00] = _RESUME_CHECK, + [_RESUME_CHECK_r11] = _RESUME_CHECK, + [_RESUME_CHECK_r22] = _RESUME_CHECK, + [_RESUME_CHECK_r33] = _RESUME_CHECK, + [_LOAD_FAST_CHECK_r01] = _LOAD_FAST_CHECK, + [_LOAD_FAST_0_r01] = _LOAD_FAST_0, + [_LOAD_FAST_0_r12] = _LOAD_FAST_0, + [_LOAD_FAST_0_r23] = _LOAD_FAST_0, + [_LOAD_FAST_1_r01] = _LOAD_FAST_1, + [_LOAD_FAST_1_r12] = _LOAD_FAST_1, + [_LOAD_FAST_1_r23] = _LOAD_FAST_1, + [_LOAD_FAST_2_r01] = _LOAD_FAST_2, + [_LOAD_FAST_2_r12] = _LOAD_FAST_2, + [_LOAD_FAST_2_r23] = _LOAD_FAST_2, + [_LOAD_FAST_3_r01] = _LOAD_FAST_3, + [_LOAD_FAST_3_r12] = _LOAD_FAST_3, + [_LOAD_FAST_3_r23] = _LOAD_FAST_3, + [_LOAD_FAST_4_r01] = _LOAD_FAST_4, + [_LOAD_FAST_4_r12] = _LOAD_FAST_4, + [_LOAD_FAST_4_r23] = _LOAD_FAST_4, + [_LOAD_FAST_5_r01] = _LOAD_FAST_5, + [_LOAD_FAST_5_r12] = _LOAD_FAST_5, + [_LOAD_FAST_5_r23] = _LOAD_FAST_5, + [_LOAD_FAST_6_r01] = _LOAD_FAST_6, + [_LOAD_FAST_6_r12] = _LOAD_FAST_6, + [_LOAD_FAST_6_r23] = _LOAD_FAST_6, + [_LOAD_FAST_7_r01] = _LOAD_FAST_7, + [_LOAD_FAST_7_r12] = _LOAD_FAST_7, + [_LOAD_FAST_7_r23] = _LOAD_FAST_7, + [_LOAD_FAST_r01] = _LOAD_FAST, + [_LOAD_FAST_r12] = _LOAD_FAST, + [_LOAD_FAST_r23] = _LOAD_FAST, + [_LOAD_FAST_BORROW_0_r01] = _LOAD_FAST_BORROW_0, + [_LOAD_FAST_BORROW_0_r12] = _LOAD_FAST_BORROW_0, + [_LOAD_FAST_BORROW_0_r23] = _LOAD_FAST_BORROW_0, + [_LOAD_FAST_BORROW_1_r01] = _LOAD_FAST_BORROW_1, + [_LOAD_FAST_BORROW_1_r12] = _LOAD_FAST_BORROW_1, + [_LOAD_FAST_BORROW_1_r23] = _LOAD_FAST_BORROW_1, + [_LOAD_FAST_BORROW_2_r01] = _LOAD_FAST_BORROW_2, + [_LOAD_FAST_BORROW_2_r12] = _LOAD_FAST_BORROW_2, + [_LOAD_FAST_BORROW_2_r23] = _LOAD_FAST_BORROW_2, + [_LOAD_FAST_BORROW_3_r01] = _LOAD_FAST_BORROW_3, + [_LOAD_FAST_BORROW_3_r12] = _LOAD_FAST_BORROW_3, + [_LOAD_FAST_BORROW_3_r23] = _LOAD_FAST_BORROW_3, + [_LOAD_FAST_BORROW_4_r01] = _LOAD_FAST_BORROW_4, + [_LOAD_FAST_BORROW_4_r12] = _LOAD_FAST_BORROW_4, + [_LOAD_FAST_BORROW_4_r23] = _LOAD_FAST_BORROW_4, + [_LOAD_FAST_BORROW_5_r01] = _LOAD_FAST_BORROW_5, + [_LOAD_FAST_BORROW_5_r12] = _LOAD_FAST_BORROW_5, + [_LOAD_FAST_BORROW_5_r23] = _LOAD_FAST_BORROW_5, + [_LOAD_FAST_BORROW_6_r01] = _LOAD_FAST_BORROW_6, + [_LOAD_FAST_BORROW_6_r12] = _LOAD_FAST_BORROW_6, + [_LOAD_FAST_BORROW_6_r23] = _LOAD_FAST_BORROW_6, + [_LOAD_FAST_BORROW_7_r01] = _LOAD_FAST_BORROW_7, + [_LOAD_FAST_BORROW_7_r12] = _LOAD_FAST_BORROW_7, + [_LOAD_FAST_BORROW_7_r23] = _LOAD_FAST_BORROW_7, + [_LOAD_FAST_BORROW_r01] = _LOAD_FAST_BORROW, + [_LOAD_FAST_BORROW_r12] = _LOAD_FAST_BORROW, + [_LOAD_FAST_BORROW_r23] = _LOAD_FAST_BORROW, + [_LOAD_FAST_AND_CLEAR_r01] = _LOAD_FAST_AND_CLEAR, + [_LOAD_FAST_AND_CLEAR_r12] = _LOAD_FAST_AND_CLEAR, + [_LOAD_FAST_AND_CLEAR_r23] = _LOAD_FAST_AND_CLEAR, + [_LOAD_CONST_r01] = _LOAD_CONST, + [_LOAD_CONST_r12] = _LOAD_CONST, + [_LOAD_CONST_r23] = _LOAD_CONST, + [_LOAD_SMALL_INT_0_r01] = _LOAD_SMALL_INT_0, + [_LOAD_SMALL_INT_0_r12] = _LOAD_SMALL_INT_0, + [_LOAD_SMALL_INT_0_r23] = _LOAD_SMALL_INT_0, + [_LOAD_SMALL_INT_1_r01] = _LOAD_SMALL_INT_1, + [_LOAD_SMALL_INT_1_r12] = _LOAD_SMALL_INT_1, + [_LOAD_SMALL_INT_1_r23] = _LOAD_SMALL_INT_1, + [_LOAD_SMALL_INT_2_r01] = _LOAD_SMALL_INT_2, + [_LOAD_SMALL_INT_2_r12] = _LOAD_SMALL_INT_2, + [_LOAD_SMALL_INT_2_r23] = _LOAD_SMALL_INT_2, + [_LOAD_SMALL_INT_3_r01] = _LOAD_SMALL_INT_3, + [_LOAD_SMALL_INT_3_r12] = _LOAD_SMALL_INT_3, + [_LOAD_SMALL_INT_3_r23] = _LOAD_SMALL_INT_3, + [_LOAD_SMALL_INT_r01] = _LOAD_SMALL_INT, + [_LOAD_SMALL_INT_r12] = _LOAD_SMALL_INT, + [_LOAD_SMALL_INT_r23] = _LOAD_SMALL_INT, + [_STORE_FAST_0_r10] = _STORE_FAST_0, + [_STORE_FAST_1_r10] = _STORE_FAST_1, + [_STORE_FAST_2_r10] = _STORE_FAST_2, + [_STORE_FAST_3_r10] = _STORE_FAST_3, + [_STORE_FAST_4_r10] = _STORE_FAST_4, + [_STORE_FAST_5_r10] = _STORE_FAST_5, + [_STORE_FAST_6_r10] = _STORE_FAST_6, + [_STORE_FAST_7_r10] = _STORE_FAST_7, + [_STORE_FAST_r10] = _STORE_FAST, + [_POP_TOP_r10] = _POP_TOP, + [_POP_TOP_NOP_r10] = _POP_TOP_NOP, + [_POP_TOP_NOP_r21] = _POP_TOP_NOP, + [_POP_TOP_NOP_r32] = _POP_TOP_NOP, + [_POP_TOP_INT_r10] = _POP_TOP_INT, + [_POP_TOP_INT_r21] = _POP_TOP_INT, + [_POP_TOP_INT_r32] = _POP_TOP_INT, + [_POP_TOP_FLOAT_r10] = _POP_TOP_FLOAT, + [_POP_TOP_FLOAT_r21] = _POP_TOP_FLOAT, + [_POP_TOP_FLOAT_r32] = _POP_TOP_FLOAT, + [_POP_TOP_UNICODE_r10] = _POP_TOP_UNICODE, + [_POP_TOP_UNICODE_r21] = _POP_TOP_UNICODE, + [_POP_TOP_UNICODE_r32] = _POP_TOP_UNICODE, + [_POP_TWO_r20] = _POP_TWO, + [_PUSH_NULL_r01] = _PUSH_NULL, + [_PUSH_NULL_r12] = _PUSH_NULL, + [_PUSH_NULL_r23] = _PUSH_NULL, + [_END_FOR_r10] = _END_FOR, + [_POP_ITER_r20] = _POP_ITER, + [_END_SEND_r21] = _END_SEND, + [_UNARY_NEGATIVE_r01] = _UNARY_NEGATIVE, + [_UNARY_NOT_r11] = _UNARY_NOT, + [_UNARY_NOT_r00] = _UNARY_NOT, + [_UNARY_NOT_r22] = _UNARY_NOT, + [_UNARY_NOT_r33] = _UNARY_NOT, + [_TO_BOOL_r01] = _TO_BOOL, + [_TO_BOOL_BOOL_r11] = _TO_BOOL_BOOL, + [_TO_BOOL_BOOL_r00] = _TO_BOOL_BOOL, + [_TO_BOOL_BOOL_r22] = _TO_BOOL_BOOL, + [_TO_BOOL_BOOL_r33] = _TO_BOOL_BOOL, + [_TO_BOOL_INT_r11] = _TO_BOOL_INT, + [_GUARD_NOS_LIST_r22] = _GUARD_NOS_LIST, + [_GUARD_NOS_LIST_r00] = _GUARD_NOS_LIST, + [_GUARD_NOS_LIST_r11] = _GUARD_NOS_LIST, + [_GUARD_NOS_LIST_r33] = _GUARD_NOS_LIST, + [_GUARD_TOS_LIST_r11] = _GUARD_TOS_LIST, + [_GUARD_TOS_LIST_r00] = _GUARD_TOS_LIST, + [_GUARD_TOS_LIST_r22] = _GUARD_TOS_LIST, + [_GUARD_TOS_LIST_r33] = _GUARD_TOS_LIST, + [_GUARD_TOS_SLICE_r11] = _GUARD_TOS_SLICE, + [_GUARD_TOS_SLICE_r00] = _GUARD_TOS_SLICE, + [_GUARD_TOS_SLICE_r22] = _GUARD_TOS_SLICE, + [_GUARD_TOS_SLICE_r33] = _GUARD_TOS_SLICE, + [_TO_BOOL_LIST_r11] = _TO_BOOL_LIST, + [_TO_BOOL_NONE_r11] = _TO_BOOL_NONE, + [_TO_BOOL_NONE_r00] = _TO_BOOL_NONE, + [_TO_BOOL_NONE_r22] = _TO_BOOL_NONE, + [_TO_BOOL_NONE_r33] = _TO_BOOL_NONE, + [_GUARD_NOS_UNICODE_r22] = _GUARD_NOS_UNICODE, + [_GUARD_NOS_UNICODE_r00] = _GUARD_NOS_UNICODE, + [_GUARD_NOS_UNICODE_r11] = _GUARD_NOS_UNICODE, + [_GUARD_NOS_UNICODE_r33] = _GUARD_NOS_UNICODE, + [_GUARD_TOS_UNICODE_r11] = _GUARD_TOS_UNICODE, + [_GUARD_TOS_UNICODE_r00] = _GUARD_TOS_UNICODE, + [_GUARD_TOS_UNICODE_r22] = _GUARD_TOS_UNICODE, + [_GUARD_TOS_UNICODE_r33] = _GUARD_TOS_UNICODE, + [_TO_BOOL_STR_r11] = _TO_BOOL_STR, + [_REPLACE_WITH_TRUE_r11] = _REPLACE_WITH_TRUE, + [_UNARY_INVERT_r01] = _UNARY_INVERT, + [_GUARD_NOS_INT_r22] = _GUARD_NOS_INT, + [_GUARD_NOS_INT_r00] = _GUARD_NOS_INT, + [_GUARD_NOS_INT_r11] = _GUARD_NOS_INT, + [_GUARD_NOS_INT_r33] = _GUARD_NOS_INT, + [_GUARD_TOS_INT_r11] = _GUARD_TOS_INT, + [_GUARD_TOS_INT_r00] = _GUARD_TOS_INT, + [_GUARD_TOS_INT_r22] = _GUARD_TOS_INT, + [_GUARD_TOS_INT_r33] = _GUARD_TOS_INT, + [_GUARD_NOS_OVERFLOWED_r22] = _GUARD_NOS_OVERFLOWED, + [_GUARD_NOS_OVERFLOWED_r00] = _GUARD_NOS_OVERFLOWED, + [_GUARD_NOS_OVERFLOWED_r11] = _GUARD_NOS_OVERFLOWED, + [_GUARD_NOS_OVERFLOWED_r33] = _GUARD_NOS_OVERFLOWED, + [_GUARD_TOS_OVERFLOWED_r11] = _GUARD_TOS_OVERFLOWED, + [_GUARD_TOS_OVERFLOWED_r00] = _GUARD_TOS_OVERFLOWED, + [_GUARD_TOS_OVERFLOWED_r22] = _GUARD_TOS_OVERFLOWED, + [_GUARD_TOS_OVERFLOWED_r33] = _GUARD_TOS_OVERFLOWED, + [_BINARY_OP_MULTIPLY_INT_r21] = _BINARY_OP_MULTIPLY_INT, + [_BINARY_OP_ADD_INT_r21] = _BINARY_OP_ADD_INT, + [_BINARY_OP_SUBTRACT_INT_r21] = _BINARY_OP_SUBTRACT_INT, + [_GUARD_NOS_FLOAT_r22] = _GUARD_NOS_FLOAT, + [_GUARD_NOS_FLOAT_r00] = _GUARD_NOS_FLOAT, + [_GUARD_NOS_FLOAT_r11] = _GUARD_NOS_FLOAT, + [_GUARD_NOS_FLOAT_r33] = _GUARD_NOS_FLOAT, + [_GUARD_TOS_FLOAT_r11] = _GUARD_TOS_FLOAT, + [_GUARD_TOS_FLOAT_r00] = _GUARD_TOS_FLOAT, + [_GUARD_TOS_FLOAT_r22] = _GUARD_TOS_FLOAT, + [_GUARD_TOS_FLOAT_r33] = _GUARD_TOS_FLOAT, + [_BINARY_OP_MULTIPLY_FLOAT_r21] = _BINARY_OP_MULTIPLY_FLOAT, + [_BINARY_OP_ADD_FLOAT_r21] = _BINARY_OP_ADD_FLOAT, + [_BINARY_OP_SUBTRACT_FLOAT_r21] = _BINARY_OP_SUBTRACT_FLOAT, + [_BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS_r21] = _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS, + [_BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS_r21] = _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS, + [_BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS_r21] = _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS, + [_BINARY_OP_ADD_UNICODE_r21] = _BINARY_OP_ADD_UNICODE, + [_BINARY_OP_INPLACE_ADD_UNICODE_r00] = _BINARY_OP_INPLACE_ADD_UNICODE, + [_GUARD_BINARY_OP_EXTEND_r00] = _GUARD_BINARY_OP_EXTEND, + [_BINARY_OP_EXTEND_r01] = _BINARY_OP_EXTEND, + [_BINARY_SLICE_r01] = _BINARY_SLICE, + [_STORE_SLICE_r00] = _STORE_SLICE, + [_BINARY_OP_SUBSCR_LIST_INT_r01] = _BINARY_OP_SUBSCR_LIST_INT, + [_BINARY_OP_SUBSCR_LIST_SLICE_r01] = _BINARY_OP_SUBSCR_LIST_SLICE, + [_BINARY_OP_SUBSCR_STR_INT_r21] = _BINARY_OP_SUBSCR_STR_INT, + [_GUARD_NOS_TUPLE_r22] = _GUARD_NOS_TUPLE, + [_GUARD_NOS_TUPLE_r00] = _GUARD_NOS_TUPLE, + [_GUARD_NOS_TUPLE_r11] = _GUARD_NOS_TUPLE, + [_GUARD_NOS_TUPLE_r33] = _GUARD_NOS_TUPLE, + [_GUARD_TOS_TUPLE_r11] = _GUARD_TOS_TUPLE, + [_GUARD_TOS_TUPLE_r00] = _GUARD_TOS_TUPLE, + [_GUARD_TOS_TUPLE_r22] = _GUARD_TOS_TUPLE, + [_GUARD_TOS_TUPLE_r33] = _GUARD_TOS_TUPLE, + [_BINARY_OP_SUBSCR_TUPLE_INT_r21] = _BINARY_OP_SUBSCR_TUPLE_INT, + [_GUARD_NOS_DICT_r22] = _GUARD_NOS_DICT, + [_GUARD_NOS_DICT_r00] = _GUARD_NOS_DICT, + [_GUARD_NOS_DICT_r11] = _GUARD_NOS_DICT, + [_GUARD_NOS_DICT_r33] = _GUARD_NOS_DICT, + [_GUARD_TOS_DICT_r11] = _GUARD_TOS_DICT, + [_GUARD_TOS_DICT_r00] = _GUARD_TOS_DICT, + [_GUARD_TOS_DICT_r22] = _GUARD_TOS_DICT, + [_GUARD_TOS_DICT_r33] = _GUARD_TOS_DICT, + [_BINARY_OP_SUBSCR_DICT_r01] = _BINARY_OP_SUBSCR_DICT, + [_BINARY_OP_SUBSCR_CHECK_FUNC_r23] = _BINARY_OP_SUBSCR_CHECK_FUNC, + [_BINARY_OP_SUBSCR_INIT_CALL_r31] = _BINARY_OP_SUBSCR_INIT_CALL, + [_LIST_APPEND_r10] = _LIST_APPEND, + [_SET_ADD_r00] = _SET_ADD, + [_STORE_SUBSCR_r00] = _STORE_SUBSCR, + [_STORE_SUBSCR_LIST_INT_r30] = _STORE_SUBSCR_LIST_INT, + [_STORE_SUBSCR_DICT_r00] = _STORE_SUBSCR_DICT, + [_DELETE_SUBSCR_r00] = _DELETE_SUBSCR, + [_CALL_INTRINSIC_1_r01] = _CALL_INTRINSIC_1, + [_CALL_INTRINSIC_2_r01] = _CALL_INTRINSIC_2, + [_RETURN_VALUE_r01] = _RETURN_VALUE, + [_GET_AITER_r01] = _GET_AITER, + [_GET_ANEXT_r01] = _GET_ANEXT, + [_GET_AWAITABLE_r01] = _GET_AWAITABLE, + [_SEND_GEN_FRAME_r22] = _SEND_GEN_FRAME, + [_YIELD_VALUE_r11] = _YIELD_VALUE, + [_POP_EXCEPT_r00] = _POP_EXCEPT, + [_LOAD_COMMON_CONSTANT_r01] = _LOAD_COMMON_CONSTANT, + [_LOAD_COMMON_CONSTANT_r12] = _LOAD_COMMON_CONSTANT, + [_LOAD_COMMON_CONSTANT_r23] = _LOAD_COMMON_CONSTANT, + [_LOAD_BUILD_CLASS_r01] = _LOAD_BUILD_CLASS, + [_STORE_NAME_r00] = _STORE_NAME, + [_DELETE_NAME_r00] = _DELETE_NAME, + [_UNPACK_SEQUENCE_r00] = _UNPACK_SEQUENCE, + [_UNPACK_SEQUENCE_TWO_TUPLE_r12] = _UNPACK_SEQUENCE_TWO_TUPLE, + [_UNPACK_SEQUENCE_TUPLE_r10] = _UNPACK_SEQUENCE_TUPLE, + [_UNPACK_SEQUENCE_LIST_r10] = _UNPACK_SEQUENCE_LIST, + [_UNPACK_EX_r00] = _UNPACK_EX, + [_STORE_ATTR_r00] = _STORE_ATTR, + [_DELETE_ATTR_r00] = _DELETE_ATTR, + [_STORE_GLOBAL_r00] = _STORE_GLOBAL, + [_DELETE_GLOBAL_r00] = _DELETE_GLOBAL, + [_LOAD_LOCALS_r01] = _LOAD_LOCALS, + [_LOAD_NAME_r01] = _LOAD_NAME, + [_LOAD_GLOBAL_r00] = _LOAD_GLOBAL, + [_PUSH_NULL_CONDITIONAL_r00] = _PUSH_NULL_CONDITIONAL, + [_GUARD_GLOBALS_VERSION_r00] = _GUARD_GLOBALS_VERSION, + [_GUARD_GLOBALS_VERSION_r11] = _GUARD_GLOBALS_VERSION, + [_GUARD_GLOBALS_VERSION_r22] = _GUARD_GLOBALS_VERSION, + [_GUARD_GLOBALS_VERSION_r33] = _GUARD_GLOBALS_VERSION, + [_LOAD_GLOBAL_MODULE_r01] = _LOAD_GLOBAL_MODULE, + [_LOAD_GLOBAL_BUILTINS_r01] = _LOAD_GLOBAL_BUILTINS, + [_DELETE_FAST_r00] = _DELETE_FAST, + [_MAKE_CELL_r00] = _MAKE_CELL, + [_DELETE_DEREF_r00] = _DELETE_DEREF, + [_LOAD_FROM_DICT_OR_DEREF_r01] = _LOAD_FROM_DICT_OR_DEREF, + [_LOAD_DEREF_r01] = _LOAD_DEREF, + [_STORE_DEREF_r00] = _STORE_DEREF, + [_COPY_FREE_VARS_r00] = _COPY_FREE_VARS, + [_BUILD_STRING_r01] = _BUILD_STRING, + [_BUILD_INTERPOLATION_r01] = _BUILD_INTERPOLATION, + [_BUILD_TEMPLATE_r01] = _BUILD_TEMPLATE, + [_BUILD_TUPLE_r01] = _BUILD_TUPLE, + [_BUILD_LIST_r01] = _BUILD_LIST, + [_LIST_EXTEND_r00] = _LIST_EXTEND, + [_SET_UPDATE_r00] = _SET_UPDATE, + [_BUILD_SET_r01] = _BUILD_SET, + [_BUILD_MAP_r01] = _BUILD_MAP, + [_SETUP_ANNOTATIONS_r00] = _SETUP_ANNOTATIONS, + [_DICT_UPDATE_r00] = _DICT_UPDATE, + [_DICT_MERGE_r00] = _DICT_MERGE, + [_MAP_ADD_r00] = _MAP_ADD, + [_LOAD_SUPER_ATTR_ATTR_r01] = _LOAD_SUPER_ATTR_ATTR, + [_LOAD_SUPER_ATTR_METHOD_r02] = _LOAD_SUPER_ATTR_METHOD, + [_LOAD_ATTR_r00] = _LOAD_ATTR, + [_GUARD_TYPE_VERSION_r11] = _GUARD_TYPE_VERSION, + [_GUARD_TYPE_VERSION_r00] = _GUARD_TYPE_VERSION, + [_GUARD_TYPE_VERSION_r22] = _GUARD_TYPE_VERSION, + [_GUARD_TYPE_VERSION_r33] = _GUARD_TYPE_VERSION, + [_GUARD_TYPE_VERSION_AND_LOCK_r11] = _GUARD_TYPE_VERSION_AND_LOCK, + [_GUARD_TYPE_VERSION_AND_LOCK_r00] = _GUARD_TYPE_VERSION_AND_LOCK, + [_GUARD_TYPE_VERSION_AND_LOCK_r22] = _GUARD_TYPE_VERSION_AND_LOCK, + [_GUARD_TYPE_VERSION_AND_LOCK_r33] = _GUARD_TYPE_VERSION_AND_LOCK, + [_CHECK_MANAGED_OBJECT_HAS_VALUES_r11] = _CHECK_MANAGED_OBJECT_HAS_VALUES, + [_CHECK_MANAGED_OBJECT_HAS_VALUES_r00] = _CHECK_MANAGED_OBJECT_HAS_VALUES, + [_CHECK_MANAGED_OBJECT_HAS_VALUES_r22] = _CHECK_MANAGED_OBJECT_HAS_VALUES, + [_CHECK_MANAGED_OBJECT_HAS_VALUES_r33] = _CHECK_MANAGED_OBJECT_HAS_VALUES, + [_LOAD_ATTR_INSTANCE_VALUE_r11] = _LOAD_ATTR_INSTANCE_VALUE, + [_LOAD_ATTR_MODULE_r11] = _LOAD_ATTR_MODULE, + [_LOAD_ATTR_WITH_HINT_r11] = _LOAD_ATTR_WITH_HINT, + [_LOAD_ATTR_SLOT_r11] = _LOAD_ATTR_SLOT, + [_CHECK_ATTR_CLASS_r11] = _CHECK_ATTR_CLASS, + [_CHECK_ATTR_CLASS_r00] = _CHECK_ATTR_CLASS, + [_CHECK_ATTR_CLASS_r22] = _CHECK_ATTR_CLASS, + [_CHECK_ATTR_CLASS_r33] = _CHECK_ATTR_CLASS, + [_LOAD_ATTR_CLASS_r11] = _LOAD_ATTR_CLASS, + [_LOAD_ATTR_PROPERTY_FRAME_r11] = _LOAD_ATTR_PROPERTY_FRAME, + [_GUARD_DORV_NO_DICT_r11] = _GUARD_DORV_NO_DICT, + [_GUARD_DORV_NO_DICT_r00] = _GUARD_DORV_NO_DICT, + [_GUARD_DORV_NO_DICT_r22] = _GUARD_DORV_NO_DICT, + [_GUARD_DORV_NO_DICT_r33] = _GUARD_DORV_NO_DICT, + [_STORE_ATTR_INSTANCE_VALUE_r20] = _STORE_ATTR_INSTANCE_VALUE, + [_STORE_ATTR_WITH_HINT_r00] = _STORE_ATTR_WITH_HINT, + [_STORE_ATTR_SLOT_r20] = _STORE_ATTR_SLOT, + [_COMPARE_OP_r01] = _COMPARE_OP, + [_COMPARE_OP_FLOAT_r21] = _COMPARE_OP_FLOAT, + [_COMPARE_OP_INT_r21] = _COMPARE_OP_INT, + [_COMPARE_OP_STR_r21] = _COMPARE_OP_STR, + [_IS_OP_r21] = _IS_OP, + [_CONTAINS_OP_r01] = _CONTAINS_OP, + [_GUARD_TOS_ANY_SET_r11] = _GUARD_TOS_ANY_SET, + [_GUARD_TOS_ANY_SET_r00] = _GUARD_TOS_ANY_SET, + [_GUARD_TOS_ANY_SET_r22] = _GUARD_TOS_ANY_SET, + [_GUARD_TOS_ANY_SET_r33] = _GUARD_TOS_ANY_SET, + [_CONTAINS_OP_SET_r01] = _CONTAINS_OP_SET, + [_CONTAINS_OP_DICT_r01] = _CONTAINS_OP_DICT, + [_CHECK_EG_MATCH_r02] = _CHECK_EG_MATCH, + [_CHECK_EXC_MATCH_r01] = _CHECK_EXC_MATCH, + [_IMPORT_NAME_r01] = _IMPORT_NAME, + [_IMPORT_FROM_r01] = _IMPORT_FROM, + [_IS_NONE_r11] = _IS_NONE, + [_GET_LEN_r01] = _GET_LEN, + [_MATCH_CLASS_r01] = _MATCH_CLASS, + [_MATCH_MAPPING_r12] = _MATCH_MAPPING, + [_MATCH_MAPPING_r01] = _MATCH_MAPPING, + [_MATCH_MAPPING_r23] = _MATCH_MAPPING, + [_MATCH_SEQUENCE_r12] = _MATCH_SEQUENCE, + [_MATCH_SEQUENCE_r01] = _MATCH_SEQUENCE, + [_MATCH_SEQUENCE_r23] = _MATCH_SEQUENCE, + [_MATCH_KEYS_r01] = _MATCH_KEYS, + [_GET_ITER_r02] = _GET_ITER, + [_GET_YIELD_FROM_ITER_r01] = _GET_YIELD_FROM_ITER, + [_FOR_ITER_TIER_TWO_r01] = _FOR_ITER_TIER_TWO, + [_ITER_CHECK_LIST_r22] = _ITER_CHECK_LIST, + [_ITER_CHECK_LIST_r00] = _ITER_CHECK_LIST, + [_ITER_CHECK_LIST_r11] = _ITER_CHECK_LIST, + [_ITER_CHECK_LIST_r33] = _ITER_CHECK_LIST, + [_GUARD_NOT_EXHAUSTED_LIST_r22] = _GUARD_NOT_EXHAUSTED_LIST, + [_GUARD_NOT_EXHAUSTED_LIST_r00] = _GUARD_NOT_EXHAUSTED_LIST, + [_GUARD_NOT_EXHAUSTED_LIST_r11] = _GUARD_NOT_EXHAUSTED_LIST, + [_GUARD_NOT_EXHAUSTED_LIST_r33] = _GUARD_NOT_EXHAUSTED_LIST, + [_ITER_NEXT_LIST_TIER_TWO_r01] = _ITER_NEXT_LIST_TIER_TWO, + [_ITER_CHECK_TUPLE_r22] = _ITER_CHECK_TUPLE, + [_ITER_CHECK_TUPLE_r00] = _ITER_CHECK_TUPLE, + [_ITER_CHECK_TUPLE_r11] = _ITER_CHECK_TUPLE, + [_ITER_CHECK_TUPLE_r33] = _ITER_CHECK_TUPLE, + [_GUARD_NOT_EXHAUSTED_TUPLE_r22] = _GUARD_NOT_EXHAUSTED_TUPLE, + [_GUARD_NOT_EXHAUSTED_TUPLE_r00] = _GUARD_NOT_EXHAUSTED_TUPLE, + [_GUARD_NOT_EXHAUSTED_TUPLE_r11] = _GUARD_NOT_EXHAUSTED_TUPLE, + [_GUARD_NOT_EXHAUSTED_TUPLE_r33] = _GUARD_NOT_EXHAUSTED_TUPLE, + [_ITER_NEXT_TUPLE_r23] = _ITER_NEXT_TUPLE, + [_ITER_CHECK_RANGE_r22] = _ITER_CHECK_RANGE, + [_ITER_CHECK_RANGE_r00] = _ITER_CHECK_RANGE, + [_ITER_CHECK_RANGE_r11] = _ITER_CHECK_RANGE, + [_ITER_CHECK_RANGE_r33] = _ITER_CHECK_RANGE, + [_GUARD_NOT_EXHAUSTED_RANGE_r22] = _GUARD_NOT_EXHAUSTED_RANGE, + [_GUARD_NOT_EXHAUSTED_RANGE_r00] = _GUARD_NOT_EXHAUSTED_RANGE, + [_GUARD_NOT_EXHAUSTED_RANGE_r11] = _GUARD_NOT_EXHAUSTED_RANGE, + [_GUARD_NOT_EXHAUSTED_RANGE_r33] = _GUARD_NOT_EXHAUSTED_RANGE, + [_ITER_NEXT_RANGE_r23] = _ITER_NEXT_RANGE, + [_FOR_ITER_GEN_FRAME_r23] = _FOR_ITER_GEN_FRAME, + [_INSERT_NULL_r10] = _INSERT_NULL, + [_LOAD_SPECIAL_r00] = _LOAD_SPECIAL, + [_WITH_EXCEPT_START_r01] = _WITH_EXCEPT_START, + [_PUSH_EXC_INFO_r12] = _PUSH_EXC_INFO, + [_PUSH_EXC_INFO_r01] = _PUSH_EXC_INFO, + [_PUSH_EXC_INFO_r23] = _PUSH_EXC_INFO, + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r11] = _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r00] = _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r22] = _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r33] = _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, + [_GUARD_KEYS_VERSION_r11] = _GUARD_KEYS_VERSION, + [_GUARD_KEYS_VERSION_r00] = _GUARD_KEYS_VERSION, + [_GUARD_KEYS_VERSION_r22] = _GUARD_KEYS_VERSION, + [_GUARD_KEYS_VERSION_r33] = _GUARD_KEYS_VERSION, + [_LOAD_ATTR_METHOD_WITH_VALUES_r12] = _LOAD_ATTR_METHOD_WITH_VALUES, + [_LOAD_ATTR_METHOD_WITH_VALUES_r01] = _LOAD_ATTR_METHOD_WITH_VALUES, + [_LOAD_ATTR_METHOD_WITH_VALUES_r23] = _LOAD_ATTR_METHOD_WITH_VALUES, + [_LOAD_ATTR_METHOD_NO_DICT_r12] = _LOAD_ATTR_METHOD_NO_DICT, + [_LOAD_ATTR_METHOD_NO_DICT_r01] = _LOAD_ATTR_METHOD_NO_DICT, + [_LOAD_ATTR_METHOD_NO_DICT_r23] = _LOAD_ATTR_METHOD_NO_DICT, + [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11] = _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, + [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11] = _LOAD_ATTR_NONDESCRIPTOR_NO_DICT, + [_CHECK_ATTR_METHOD_LAZY_DICT_r11] = _CHECK_ATTR_METHOD_LAZY_DICT, + [_CHECK_ATTR_METHOD_LAZY_DICT_r00] = _CHECK_ATTR_METHOD_LAZY_DICT, + [_CHECK_ATTR_METHOD_LAZY_DICT_r22] = _CHECK_ATTR_METHOD_LAZY_DICT, + [_CHECK_ATTR_METHOD_LAZY_DICT_r33] = _CHECK_ATTR_METHOD_LAZY_DICT, + [_LOAD_ATTR_METHOD_LAZY_DICT_r12] = _LOAD_ATTR_METHOD_LAZY_DICT, + [_LOAD_ATTR_METHOD_LAZY_DICT_r01] = _LOAD_ATTR_METHOD_LAZY_DICT, + [_LOAD_ATTR_METHOD_LAZY_DICT_r23] = _LOAD_ATTR_METHOD_LAZY_DICT, + [_MAYBE_EXPAND_METHOD_r00] = _MAYBE_EXPAND_METHOD, + [_PY_FRAME_GENERAL_r01] = _PY_FRAME_GENERAL, + [_CHECK_FUNCTION_VERSION_r00] = _CHECK_FUNCTION_VERSION, + [_CHECK_FUNCTION_VERSION_INLINE_r00] = _CHECK_FUNCTION_VERSION_INLINE, + [_CHECK_FUNCTION_VERSION_INLINE_r11] = _CHECK_FUNCTION_VERSION_INLINE, + [_CHECK_FUNCTION_VERSION_INLINE_r22] = _CHECK_FUNCTION_VERSION_INLINE, + [_CHECK_FUNCTION_VERSION_INLINE_r33] = _CHECK_FUNCTION_VERSION_INLINE, + [_CHECK_METHOD_VERSION_r00] = _CHECK_METHOD_VERSION, + [_EXPAND_METHOD_r00] = _EXPAND_METHOD, + [_CHECK_IS_NOT_PY_CALLABLE_r00] = _CHECK_IS_NOT_PY_CALLABLE, + [_CALL_NON_PY_GENERAL_r01] = _CALL_NON_PY_GENERAL, + [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS_r00] = _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, + [_INIT_CALL_BOUND_METHOD_EXACT_ARGS_r00] = _INIT_CALL_BOUND_METHOD_EXACT_ARGS, + [_CHECK_PEP_523_r00] = _CHECK_PEP_523, + [_CHECK_PEP_523_r11] = _CHECK_PEP_523, + [_CHECK_PEP_523_r22] = _CHECK_PEP_523, + [_CHECK_PEP_523_r33] = _CHECK_PEP_523, + [_CHECK_FUNCTION_EXACT_ARGS_r00] = _CHECK_FUNCTION_EXACT_ARGS, + [_CHECK_STACK_SPACE_r00] = _CHECK_STACK_SPACE, + [_CHECK_RECURSION_REMAINING_r00] = _CHECK_RECURSION_REMAINING, + [_CHECK_RECURSION_REMAINING_r11] = _CHECK_RECURSION_REMAINING, + [_CHECK_RECURSION_REMAINING_r22] = _CHECK_RECURSION_REMAINING, + [_CHECK_RECURSION_REMAINING_r33] = _CHECK_RECURSION_REMAINING, + [_INIT_CALL_PY_EXACT_ARGS_0_r01] = _INIT_CALL_PY_EXACT_ARGS_0, + [_INIT_CALL_PY_EXACT_ARGS_1_r01] = _INIT_CALL_PY_EXACT_ARGS_1, + [_INIT_CALL_PY_EXACT_ARGS_2_r01] = _INIT_CALL_PY_EXACT_ARGS_2, + [_INIT_CALL_PY_EXACT_ARGS_3_r01] = _INIT_CALL_PY_EXACT_ARGS_3, + [_INIT_CALL_PY_EXACT_ARGS_4_r01] = _INIT_CALL_PY_EXACT_ARGS_4, + [_INIT_CALL_PY_EXACT_ARGS_r01] = _INIT_CALL_PY_EXACT_ARGS, + [_PUSH_FRAME_r10] = _PUSH_FRAME, + [_GUARD_NOS_NULL_r22] = _GUARD_NOS_NULL, + [_GUARD_NOS_NULL_r00] = _GUARD_NOS_NULL, + [_GUARD_NOS_NULL_r11] = _GUARD_NOS_NULL, + [_GUARD_NOS_NULL_r33] = _GUARD_NOS_NULL, + [_GUARD_NOS_NOT_NULL_r22] = _GUARD_NOS_NOT_NULL, + [_GUARD_NOS_NOT_NULL_r00] = _GUARD_NOS_NOT_NULL, + [_GUARD_NOS_NOT_NULL_r11] = _GUARD_NOS_NOT_NULL, + [_GUARD_NOS_NOT_NULL_r33] = _GUARD_NOS_NOT_NULL, + [_GUARD_THIRD_NULL_r33] = _GUARD_THIRD_NULL, + [_GUARD_CALLABLE_TYPE_1_r33] = _GUARD_CALLABLE_TYPE_1, + [_CALL_TYPE_1_r31] = _CALL_TYPE_1, + [_GUARD_CALLABLE_STR_1_r33] = _GUARD_CALLABLE_STR_1, + [_CALL_STR_1_r01] = _CALL_STR_1, + [_GUARD_CALLABLE_TUPLE_1_r33] = _GUARD_CALLABLE_TUPLE_1, + [_CALL_TUPLE_1_r01] = _CALL_TUPLE_1, + [_CHECK_AND_ALLOCATE_OBJECT_r00] = _CHECK_AND_ALLOCATE_OBJECT, + [_CREATE_INIT_FRAME_r01] = _CREATE_INIT_FRAME, + [_EXIT_INIT_CHECK_r00] = _EXIT_INIT_CHECK, + [_CALL_BUILTIN_CLASS_r01] = _CALL_BUILTIN_CLASS, + [_CALL_BUILTIN_O_r01] = _CALL_BUILTIN_O, + [_CALL_BUILTIN_FAST_r01] = _CALL_BUILTIN_FAST, + [_CALL_BUILTIN_FAST_WITH_KEYWORDS_r01] = _CALL_BUILTIN_FAST_WITH_KEYWORDS, + [_GUARD_CALLABLE_LEN_r33] = _GUARD_CALLABLE_LEN, + [_CALL_LEN_r01] = _CALL_LEN, + [_GUARD_CALLABLE_ISINSTANCE_r33] = _GUARD_CALLABLE_ISINSTANCE, + [_CALL_ISINSTANCE_r01] = _CALL_ISINSTANCE, + [_GUARD_CALLABLE_LIST_APPEND_r33] = _GUARD_CALLABLE_LIST_APPEND, + [_CALL_LIST_APPEND_r30] = _CALL_LIST_APPEND, + [_CALL_METHOD_DESCRIPTOR_O_r01] = _CALL_METHOD_DESCRIPTOR_O, + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r01] = _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, + [_CALL_METHOD_DESCRIPTOR_NOARGS_r01] = _CALL_METHOD_DESCRIPTOR_NOARGS, + [_CALL_METHOD_DESCRIPTOR_FAST_r01] = _CALL_METHOD_DESCRIPTOR_FAST, + [_MAYBE_EXPAND_METHOD_KW_r00] = _MAYBE_EXPAND_METHOD_KW, + [_PY_FRAME_KW_r01] = _PY_FRAME_KW, + [_CHECK_FUNCTION_VERSION_KW_r11] = _CHECK_FUNCTION_VERSION_KW, + [_CHECK_METHOD_VERSION_KW_r11] = _CHECK_METHOD_VERSION_KW, + [_EXPAND_METHOD_KW_r00] = _EXPAND_METHOD_KW, + [_CHECK_IS_NOT_PY_CALLABLE_KW_r11] = _CHECK_IS_NOT_PY_CALLABLE_KW, + [_CALL_KW_NON_PY_r01] = _CALL_KW_NON_PY, + [_MAKE_CALLARGS_A_TUPLE_r00] = _MAKE_CALLARGS_A_TUPLE, + [_MAKE_FUNCTION_r01] = _MAKE_FUNCTION, + [_SET_FUNCTION_ATTRIBUTE_r21] = _SET_FUNCTION_ATTRIBUTE, + [_RETURN_GENERATOR_r01] = _RETURN_GENERATOR, + [_BUILD_SLICE_r01] = _BUILD_SLICE, + [_CONVERT_VALUE_r01] = _CONVERT_VALUE, + [_FORMAT_SIMPLE_r01] = _FORMAT_SIMPLE, + [_FORMAT_WITH_SPEC_r01] = _FORMAT_WITH_SPEC, + [_COPY_1_r12] = _COPY_1, + [_COPY_1_r01] = _COPY_1, + [_COPY_1_r23] = _COPY_1, + [_COPY_2_r23] = _COPY_2, + [_COPY_3_r33] = _COPY_3, + [_COPY_r01] = _COPY, + [_BINARY_OP_r01] = _BINARY_OP, + [_SWAP_2_r22] = _SWAP_2, + [_SWAP_2_r00] = _SWAP_2, + [_SWAP_2_r11] = _SWAP_2, + [_SWAP_2_r33] = _SWAP_2, + [_SWAP_3_r33] = _SWAP_3, + [_SWAP_r11] = _SWAP, + [_GUARD_IS_TRUE_POP_r10] = _GUARD_IS_TRUE_POP, + [_GUARD_IS_FALSE_POP_r10] = _GUARD_IS_FALSE_POP, + [_GUARD_IS_NONE_POP_r10] = _GUARD_IS_NONE_POP, + [_GUARD_IS_NOT_NONE_POP_r10] = _GUARD_IS_NOT_NONE_POP, + [_JUMP_TO_TOP_r00] = _JUMP_TO_TOP, + [_SET_IP_r00] = _SET_IP, + [_SET_IP_r11] = _SET_IP, + [_SET_IP_r22] = _SET_IP, + [_SET_IP_r33] = _SET_IP, + [_CHECK_STACK_SPACE_OPERAND_r00] = _CHECK_STACK_SPACE_OPERAND, + [_CHECK_STACK_SPACE_OPERAND_r11] = _CHECK_STACK_SPACE_OPERAND, + [_CHECK_STACK_SPACE_OPERAND_r22] = _CHECK_STACK_SPACE_OPERAND, + [_CHECK_STACK_SPACE_OPERAND_r33] = _CHECK_STACK_SPACE_OPERAND, + [_SAVE_RETURN_OFFSET_r00] = _SAVE_RETURN_OFFSET, + [_SAVE_RETURN_OFFSET_r11] = _SAVE_RETURN_OFFSET, + [_SAVE_RETURN_OFFSET_r22] = _SAVE_RETURN_OFFSET, + [_SAVE_RETURN_OFFSET_r33] = _SAVE_RETURN_OFFSET, + [_EXIT_TRACE_r00] = _EXIT_TRACE, + [_EXIT_TRACE_r10] = _EXIT_TRACE, + [_EXIT_TRACE_r20] = _EXIT_TRACE, + [_EXIT_TRACE_r30] = _EXIT_TRACE, + [_CHECK_VALIDITY_r00] = _CHECK_VALIDITY, + [_CHECK_VALIDITY_r11] = _CHECK_VALIDITY, + [_CHECK_VALIDITY_r22] = _CHECK_VALIDITY, + [_CHECK_VALIDITY_r33] = _CHECK_VALIDITY, + [_LOAD_CONST_INLINE_r01] = _LOAD_CONST_INLINE, + [_LOAD_CONST_INLINE_r12] = _LOAD_CONST_INLINE, + [_LOAD_CONST_INLINE_r23] = _LOAD_CONST_INLINE, + [_POP_TOP_LOAD_CONST_INLINE_r11] = _POP_TOP_LOAD_CONST_INLINE, + [_LOAD_CONST_INLINE_BORROW_r01] = _LOAD_CONST_INLINE_BORROW, + [_LOAD_CONST_INLINE_BORROW_r12] = _LOAD_CONST_INLINE_BORROW, + [_LOAD_CONST_INLINE_BORROW_r23] = _LOAD_CONST_INLINE_BORROW, + [_POP_CALL_r20] = _POP_CALL, + [_POP_CALL_ONE_r30] = _POP_CALL_ONE, + [_POP_CALL_TWO_r30] = _POP_CALL_TWO, + [_POP_TOP_LOAD_CONST_INLINE_BORROW_r11] = _POP_TOP_LOAD_CONST_INLINE_BORROW, + [_POP_TWO_LOAD_CONST_INLINE_BORROW_r21] = _POP_TWO_LOAD_CONST_INLINE_BORROW, + [_POP_CALL_LOAD_CONST_INLINE_BORROW_r21] = _POP_CALL_LOAD_CONST_INLINE_BORROW, + [_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW_r31] = _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW, + [_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW_r31] = _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, + [_LOAD_CONST_UNDER_INLINE_r12] = _LOAD_CONST_UNDER_INLINE, + [_LOAD_CONST_UNDER_INLINE_r01] = _LOAD_CONST_UNDER_INLINE, + [_LOAD_CONST_UNDER_INLINE_r23] = _LOAD_CONST_UNDER_INLINE, + [_LOAD_CONST_UNDER_INLINE_BORROW_r12] = _LOAD_CONST_UNDER_INLINE_BORROW, + [_LOAD_CONST_UNDER_INLINE_BORROW_r01] = _LOAD_CONST_UNDER_INLINE_BORROW, + [_LOAD_CONST_UNDER_INLINE_BORROW_r23] = _LOAD_CONST_UNDER_INLINE_BORROW, + [_CHECK_FUNCTION_r00] = _CHECK_FUNCTION, + [_CHECK_FUNCTION_r11] = _CHECK_FUNCTION, + [_CHECK_FUNCTION_r22] = _CHECK_FUNCTION, + [_CHECK_FUNCTION_r33] = _CHECK_FUNCTION, + [_START_EXECUTOR_r00] = _START_EXECUTOR, + [_MAKE_WARM_r00] = _MAKE_WARM, + [_MAKE_WARM_r11] = _MAKE_WARM, + [_MAKE_WARM_r22] = _MAKE_WARM, + [_MAKE_WARM_r33] = _MAKE_WARM, + [_FATAL_ERROR_r00] = _FATAL_ERROR, + [_FATAL_ERROR_r11] = _FATAL_ERROR, + [_FATAL_ERROR_r22] = _FATAL_ERROR, + [_FATAL_ERROR_r33] = _FATAL_ERROR, + [_DEOPT_r00] = _DEOPT, + [_DEOPT_r10] = _DEOPT, + [_DEOPT_r20] = _DEOPT, + [_DEOPT_r30] = _DEOPT, + [_ERROR_POP_N_r00] = _ERROR_POP_N, + [_TIER2_RESUME_CHECK_r00] = _TIER2_RESUME_CHECK, + [_TIER2_RESUME_CHECK_r11] = _TIER2_RESUME_CHECK, + [_TIER2_RESUME_CHECK_r22] = _TIER2_RESUME_CHECK, + [_TIER2_RESUME_CHECK_r33] = _TIER2_RESUME_CHECK, + [_SPILL_OR_RELOAD_r01] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r02] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r03] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r10] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r12] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r13] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r20] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r21] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r23] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r30] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r31] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r32] = _SPILL_OR_RELOAD, +}; + +const uint16_t _PyUop_SpillsAndReloads[4][4] = { + [0][1] = _SPILL_OR_RELOAD_r01, + [0][2] = _SPILL_OR_RELOAD_r02, + [0][3] = _SPILL_OR_RELOAD_r03, + [1][0] = _SPILL_OR_RELOAD_r10, + [1][2] = _SPILL_OR_RELOAD_r12, + [1][3] = _SPILL_OR_RELOAD_r13, + [2][0] = _SPILL_OR_RELOAD_r20, + [2][1] = _SPILL_OR_RELOAD_r21, + [2][3] = _SPILL_OR_RELOAD_r23, + [3][0] = _SPILL_OR_RELOAD_r30, + [3][1] = _SPILL_OR_RELOAD_r31, + [3][2] = _SPILL_OR_RELOAD_r32, +}; + +const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_BINARY_OP] = "_BINARY_OP", + [_BINARY_OP_r01] = "_BINARY_OP_r01", [_BINARY_OP_ADD_FLOAT] = "_BINARY_OP_ADD_FLOAT", + [_BINARY_OP_ADD_FLOAT_r21] = "_BINARY_OP_ADD_FLOAT_r21", [_BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS] = "_BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS", + [_BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS_r21] = "_BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS_r21", [_BINARY_OP_ADD_INT] = "_BINARY_OP_ADD_INT", + [_BINARY_OP_ADD_INT_r21] = "_BINARY_OP_ADD_INT_r21", [_BINARY_OP_ADD_UNICODE] = "_BINARY_OP_ADD_UNICODE", + [_BINARY_OP_ADD_UNICODE_r21] = "_BINARY_OP_ADD_UNICODE_r21", [_BINARY_OP_EXTEND] = "_BINARY_OP_EXTEND", + [_BINARY_OP_EXTEND_r01] = "_BINARY_OP_EXTEND_r01", [_BINARY_OP_INPLACE_ADD_UNICODE] = "_BINARY_OP_INPLACE_ADD_UNICODE", + [_BINARY_OP_INPLACE_ADD_UNICODE_r00] = "_BINARY_OP_INPLACE_ADD_UNICODE_r00", [_BINARY_OP_MULTIPLY_FLOAT] = "_BINARY_OP_MULTIPLY_FLOAT", + [_BINARY_OP_MULTIPLY_FLOAT_r21] = "_BINARY_OP_MULTIPLY_FLOAT_r21", [_BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS] = "_BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS", + [_BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS_r21] = "_BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS_r21", [_BINARY_OP_MULTIPLY_INT] = "_BINARY_OP_MULTIPLY_INT", + [_BINARY_OP_MULTIPLY_INT_r21] = "_BINARY_OP_MULTIPLY_INT_r21", [_BINARY_OP_SUBSCR_CHECK_FUNC] = "_BINARY_OP_SUBSCR_CHECK_FUNC", + [_BINARY_OP_SUBSCR_CHECK_FUNC_r23] = "_BINARY_OP_SUBSCR_CHECK_FUNC_r23", [_BINARY_OP_SUBSCR_DICT] = "_BINARY_OP_SUBSCR_DICT", + [_BINARY_OP_SUBSCR_DICT_r01] = "_BINARY_OP_SUBSCR_DICT_r01", [_BINARY_OP_SUBSCR_INIT_CALL] = "_BINARY_OP_SUBSCR_INIT_CALL", + [_BINARY_OP_SUBSCR_INIT_CALL_r31] = "_BINARY_OP_SUBSCR_INIT_CALL_r31", [_BINARY_OP_SUBSCR_LIST_INT] = "_BINARY_OP_SUBSCR_LIST_INT", + [_BINARY_OP_SUBSCR_LIST_INT_r01] = "_BINARY_OP_SUBSCR_LIST_INT_r01", [_BINARY_OP_SUBSCR_LIST_SLICE] = "_BINARY_OP_SUBSCR_LIST_SLICE", + [_BINARY_OP_SUBSCR_LIST_SLICE_r01] = "_BINARY_OP_SUBSCR_LIST_SLICE_r01", [_BINARY_OP_SUBSCR_STR_INT] = "_BINARY_OP_SUBSCR_STR_INT", + [_BINARY_OP_SUBSCR_STR_INT_r21] = "_BINARY_OP_SUBSCR_STR_INT_r21", [_BINARY_OP_SUBSCR_TUPLE_INT] = "_BINARY_OP_SUBSCR_TUPLE_INT", + [_BINARY_OP_SUBSCR_TUPLE_INT_r21] = "_BINARY_OP_SUBSCR_TUPLE_INT_r21", [_BINARY_OP_SUBTRACT_FLOAT] = "_BINARY_OP_SUBTRACT_FLOAT", + [_BINARY_OP_SUBTRACT_FLOAT_r21] = "_BINARY_OP_SUBTRACT_FLOAT_r21", [_BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS] = "_BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS", + [_BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS_r21] = "_BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS_r21", [_BINARY_OP_SUBTRACT_INT] = "_BINARY_OP_SUBTRACT_INT", + [_BINARY_OP_SUBTRACT_INT_r21] = "_BINARY_OP_SUBTRACT_INT_r21", [_BINARY_SLICE] = "_BINARY_SLICE", + [_BINARY_SLICE_r01] = "_BINARY_SLICE_r01", [_BUILD_INTERPOLATION] = "_BUILD_INTERPOLATION", + [_BUILD_INTERPOLATION_r01] = "_BUILD_INTERPOLATION_r01", [_BUILD_LIST] = "_BUILD_LIST", + [_BUILD_LIST_r01] = "_BUILD_LIST_r01", [_BUILD_MAP] = "_BUILD_MAP", + [_BUILD_MAP_r01] = "_BUILD_MAP_r01", [_BUILD_SET] = "_BUILD_SET", + [_BUILD_SET_r01] = "_BUILD_SET_r01", [_BUILD_SLICE] = "_BUILD_SLICE", + [_BUILD_SLICE_r01] = "_BUILD_SLICE_r01", [_BUILD_STRING] = "_BUILD_STRING", + [_BUILD_STRING_r01] = "_BUILD_STRING_r01", [_BUILD_TEMPLATE] = "_BUILD_TEMPLATE", + [_BUILD_TEMPLATE_r01] = "_BUILD_TEMPLATE_r01", [_BUILD_TUPLE] = "_BUILD_TUPLE", + [_BUILD_TUPLE_r01] = "_BUILD_TUPLE_r01", [_CALL_BUILTIN_CLASS] = "_CALL_BUILTIN_CLASS", + [_CALL_BUILTIN_CLASS_r01] = "_CALL_BUILTIN_CLASS_r01", [_CALL_BUILTIN_FAST] = "_CALL_BUILTIN_FAST", + [_CALL_BUILTIN_FAST_r01] = "_CALL_BUILTIN_FAST_r01", [_CALL_BUILTIN_FAST_WITH_KEYWORDS] = "_CALL_BUILTIN_FAST_WITH_KEYWORDS", + [_CALL_BUILTIN_FAST_WITH_KEYWORDS_r01] = "_CALL_BUILTIN_FAST_WITH_KEYWORDS_r01", [_CALL_BUILTIN_O] = "_CALL_BUILTIN_O", + [_CALL_BUILTIN_O_r01] = "_CALL_BUILTIN_O_r01", [_CALL_INTRINSIC_1] = "_CALL_INTRINSIC_1", + [_CALL_INTRINSIC_1_r01] = "_CALL_INTRINSIC_1_r01", [_CALL_INTRINSIC_2] = "_CALL_INTRINSIC_2", + [_CALL_INTRINSIC_2_r01] = "_CALL_INTRINSIC_2_r01", [_CALL_ISINSTANCE] = "_CALL_ISINSTANCE", + [_CALL_ISINSTANCE_r01] = "_CALL_ISINSTANCE_r01", [_CALL_KW_NON_PY] = "_CALL_KW_NON_PY", + [_CALL_KW_NON_PY_r01] = "_CALL_KW_NON_PY_r01", [_CALL_LEN] = "_CALL_LEN", + [_CALL_LEN_r01] = "_CALL_LEN_r01", [_CALL_LIST_APPEND] = "_CALL_LIST_APPEND", + [_CALL_LIST_APPEND_r30] = "_CALL_LIST_APPEND_r30", [_CALL_METHOD_DESCRIPTOR_FAST] = "_CALL_METHOD_DESCRIPTOR_FAST", + [_CALL_METHOD_DESCRIPTOR_FAST_r01] = "_CALL_METHOD_DESCRIPTOR_FAST_r01", [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r01] = "_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r01", [_CALL_METHOD_DESCRIPTOR_NOARGS] = "_CALL_METHOD_DESCRIPTOR_NOARGS", + [_CALL_METHOD_DESCRIPTOR_NOARGS_r01] = "_CALL_METHOD_DESCRIPTOR_NOARGS_r01", [_CALL_METHOD_DESCRIPTOR_O] = "_CALL_METHOD_DESCRIPTOR_O", + [_CALL_METHOD_DESCRIPTOR_O_r01] = "_CALL_METHOD_DESCRIPTOR_O_r01", [_CALL_NON_PY_GENERAL] = "_CALL_NON_PY_GENERAL", + [_CALL_NON_PY_GENERAL_r01] = "_CALL_NON_PY_GENERAL_r01", [_CALL_STR_1] = "_CALL_STR_1", + [_CALL_STR_1_r01] = "_CALL_STR_1_r01", [_CALL_TUPLE_1] = "_CALL_TUPLE_1", + [_CALL_TUPLE_1_r01] = "_CALL_TUPLE_1_r01", [_CALL_TYPE_1] = "_CALL_TYPE_1", + [_CALL_TYPE_1_r31] = "_CALL_TYPE_1_r31", [_CHECK_AND_ALLOCATE_OBJECT] = "_CHECK_AND_ALLOCATE_OBJECT", + [_CHECK_AND_ALLOCATE_OBJECT_r00] = "_CHECK_AND_ALLOCATE_OBJECT_r00", [_CHECK_ATTR_CLASS] = "_CHECK_ATTR_CLASS", + [_CHECK_ATTR_CLASS_r11] = "_CHECK_ATTR_CLASS_r11", + [_CHECK_ATTR_CLASS_r00] = "_CHECK_ATTR_CLASS_r00", + [_CHECK_ATTR_CLASS_r22] = "_CHECK_ATTR_CLASS_r22", + [_CHECK_ATTR_CLASS_r33] = "_CHECK_ATTR_CLASS_r33", [_CHECK_ATTR_METHOD_LAZY_DICT] = "_CHECK_ATTR_METHOD_LAZY_DICT", + [_CHECK_ATTR_METHOD_LAZY_DICT_r11] = "_CHECK_ATTR_METHOD_LAZY_DICT_r11", + [_CHECK_ATTR_METHOD_LAZY_DICT_r00] = "_CHECK_ATTR_METHOD_LAZY_DICT_r00", + [_CHECK_ATTR_METHOD_LAZY_DICT_r22] = "_CHECK_ATTR_METHOD_LAZY_DICT_r22", + [_CHECK_ATTR_METHOD_LAZY_DICT_r33] = "_CHECK_ATTR_METHOD_LAZY_DICT_r33", [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = "_CHECK_CALL_BOUND_METHOD_EXACT_ARGS", + [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS_r00] = "_CHECK_CALL_BOUND_METHOD_EXACT_ARGS_r00", [_CHECK_EG_MATCH] = "_CHECK_EG_MATCH", + [_CHECK_EG_MATCH_r02] = "_CHECK_EG_MATCH_r02", [_CHECK_EXC_MATCH] = "_CHECK_EXC_MATCH", + [_CHECK_EXC_MATCH_r01] = "_CHECK_EXC_MATCH_r01", [_CHECK_FUNCTION] = "_CHECK_FUNCTION", + [_CHECK_FUNCTION_r00] = "_CHECK_FUNCTION_r00", + [_CHECK_FUNCTION_r11] = "_CHECK_FUNCTION_r11", + [_CHECK_FUNCTION_r22] = "_CHECK_FUNCTION_r22", + [_CHECK_FUNCTION_r33] = "_CHECK_FUNCTION_r33", [_CHECK_FUNCTION_EXACT_ARGS] = "_CHECK_FUNCTION_EXACT_ARGS", + [_CHECK_FUNCTION_EXACT_ARGS_r00] = "_CHECK_FUNCTION_EXACT_ARGS_r00", [_CHECK_FUNCTION_VERSION] = "_CHECK_FUNCTION_VERSION", + [_CHECK_FUNCTION_VERSION_r00] = "_CHECK_FUNCTION_VERSION_r00", [_CHECK_FUNCTION_VERSION_INLINE] = "_CHECK_FUNCTION_VERSION_INLINE", + [_CHECK_FUNCTION_VERSION_INLINE_r00] = "_CHECK_FUNCTION_VERSION_INLINE_r00", + [_CHECK_FUNCTION_VERSION_INLINE_r11] = "_CHECK_FUNCTION_VERSION_INLINE_r11", + [_CHECK_FUNCTION_VERSION_INLINE_r22] = "_CHECK_FUNCTION_VERSION_INLINE_r22", + [_CHECK_FUNCTION_VERSION_INLINE_r33] = "_CHECK_FUNCTION_VERSION_INLINE_r33", [_CHECK_FUNCTION_VERSION_KW] = "_CHECK_FUNCTION_VERSION_KW", + [_CHECK_FUNCTION_VERSION_KW_r11] = "_CHECK_FUNCTION_VERSION_KW_r11", [_CHECK_IS_NOT_PY_CALLABLE] = "_CHECK_IS_NOT_PY_CALLABLE", + [_CHECK_IS_NOT_PY_CALLABLE_r00] = "_CHECK_IS_NOT_PY_CALLABLE_r00", [_CHECK_IS_NOT_PY_CALLABLE_KW] = "_CHECK_IS_NOT_PY_CALLABLE_KW", + [_CHECK_IS_NOT_PY_CALLABLE_KW_r11] = "_CHECK_IS_NOT_PY_CALLABLE_KW_r11", [_CHECK_MANAGED_OBJECT_HAS_VALUES] = "_CHECK_MANAGED_OBJECT_HAS_VALUES", + [_CHECK_MANAGED_OBJECT_HAS_VALUES_r11] = "_CHECK_MANAGED_OBJECT_HAS_VALUES_r11", + [_CHECK_MANAGED_OBJECT_HAS_VALUES_r00] = "_CHECK_MANAGED_OBJECT_HAS_VALUES_r00", + [_CHECK_MANAGED_OBJECT_HAS_VALUES_r22] = "_CHECK_MANAGED_OBJECT_HAS_VALUES_r22", + [_CHECK_MANAGED_OBJECT_HAS_VALUES_r33] = "_CHECK_MANAGED_OBJECT_HAS_VALUES_r33", [_CHECK_METHOD_VERSION] = "_CHECK_METHOD_VERSION", + [_CHECK_METHOD_VERSION_r00] = "_CHECK_METHOD_VERSION_r00", [_CHECK_METHOD_VERSION_KW] = "_CHECK_METHOD_VERSION_KW", + [_CHECK_METHOD_VERSION_KW_r11] = "_CHECK_METHOD_VERSION_KW_r11", [_CHECK_PEP_523] = "_CHECK_PEP_523", + [_CHECK_PEP_523_r00] = "_CHECK_PEP_523_r00", + [_CHECK_PEP_523_r11] = "_CHECK_PEP_523_r11", + [_CHECK_PEP_523_r22] = "_CHECK_PEP_523_r22", + [_CHECK_PEP_523_r33] = "_CHECK_PEP_523_r33", [_CHECK_PERIODIC] = "_CHECK_PERIODIC", + [_CHECK_PERIODIC_r00] = "_CHECK_PERIODIC_r00", [_CHECK_PERIODIC_IF_NOT_YIELD_FROM] = "_CHECK_PERIODIC_IF_NOT_YIELD_FROM", + [_CHECK_PERIODIC_IF_NOT_YIELD_FROM_r00] = "_CHECK_PERIODIC_IF_NOT_YIELD_FROM_r00", [_CHECK_RECURSION_REMAINING] = "_CHECK_RECURSION_REMAINING", + [_CHECK_RECURSION_REMAINING_r00] = "_CHECK_RECURSION_REMAINING_r00", + [_CHECK_RECURSION_REMAINING_r11] = "_CHECK_RECURSION_REMAINING_r11", + [_CHECK_RECURSION_REMAINING_r22] = "_CHECK_RECURSION_REMAINING_r22", + [_CHECK_RECURSION_REMAINING_r33] = "_CHECK_RECURSION_REMAINING_r33", [_CHECK_STACK_SPACE] = "_CHECK_STACK_SPACE", + [_CHECK_STACK_SPACE_r00] = "_CHECK_STACK_SPACE_r00", [_CHECK_STACK_SPACE_OPERAND] = "_CHECK_STACK_SPACE_OPERAND", + [_CHECK_STACK_SPACE_OPERAND_r00] = "_CHECK_STACK_SPACE_OPERAND_r00", + [_CHECK_STACK_SPACE_OPERAND_r11] = "_CHECK_STACK_SPACE_OPERAND_r11", + [_CHECK_STACK_SPACE_OPERAND_r22] = "_CHECK_STACK_SPACE_OPERAND_r22", + [_CHECK_STACK_SPACE_OPERAND_r33] = "_CHECK_STACK_SPACE_OPERAND_r33", [_CHECK_VALIDITY] = "_CHECK_VALIDITY", + [_CHECK_VALIDITY_r00] = "_CHECK_VALIDITY_r00", + [_CHECK_VALIDITY_r11] = "_CHECK_VALIDITY_r11", + [_CHECK_VALIDITY_r22] = "_CHECK_VALIDITY_r22", + [_CHECK_VALIDITY_r33] = "_CHECK_VALIDITY_r33", [_COMPARE_OP] = "_COMPARE_OP", + [_COMPARE_OP_r01] = "_COMPARE_OP_r01", [_COMPARE_OP_FLOAT] = "_COMPARE_OP_FLOAT", + [_COMPARE_OP_FLOAT_r21] = "_COMPARE_OP_FLOAT_r21", [_COMPARE_OP_INT] = "_COMPARE_OP_INT", + [_COMPARE_OP_INT_r21] = "_COMPARE_OP_INT_r21", [_COMPARE_OP_STR] = "_COMPARE_OP_STR", + [_COMPARE_OP_STR_r21] = "_COMPARE_OP_STR_r21", [_CONTAINS_OP] = "_CONTAINS_OP", + [_CONTAINS_OP_r01] = "_CONTAINS_OP_r01", [_CONTAINS_OP_DICT] = "_CONTAINS_OP_DICT", + [_CONTAINS_OP_DICT_r01] = "_CONTAINS_OP_DICT_r01", [_CONTAINS_OP_SET] = "_CONTAINS_OP_SET", + [_CONTAINS_OP_SET_r01] = "_CONTAINS_OP_SET_r01", [_CONVERT_VALUE] = "_CONVERT_VALUE", + [_CONVERT_VALUE_r01] = "_CONVERT_VALUE_r01", [_COPY] = "_COPY", + [_COPY_r01] = "_COPY_r01", [_COPY_1] = "_COPY_1", + [_COPY_1_r12] = "_COPY_1_r12", + [_COPY_1_r01] = "_COPY_1_r01", + [_COPY_1_r23] = "_COPY_1_r23", [_COPY_2] = "_COPY_2", + [_COPY_2_r23] = "_COPY_2_r23", [_COPY_3] = "_COPY_3", + [_COPY_3_r33] = "_COPY_3_r33", [_COPY_FREE_VARS] = "_COPY_FREE_VARS", + [_COPY_FREE_VARS_r00] = "_COPY_FREE_VARS_r00", [_CREATE_INIT_FRAME] = "_CREATE_INIT_FRAME", + [_CREATE_INIT_FRAME_r01] = "_CREATE_INIT_FRAME_r01", [_DELETE_ATTR] = "_DELETE_ATTR", + [_DELETE_ATTR_r00] = "_DELETE_ATTR_r00", [_DELETE_DEREF] = "_DELETE_DEREF", + [_DELETE_DEREF_r00] = "_DELETE_DEREF_r00", [_DELETE_FAST] = "_DELETE_FAST", + [_DELETE_FAST_r00] = "_DELETE_FAST_r00", [_DELETE_GLOBAL] = "_DELETE_GLOBAL", + [_DELETE_GLOBAL_r00] = "_DELETE_GLOBAL_r00", [_DELETE_NAME] = "_DELETE_NAME", + [_DELETE_NAME_r00] = "_DELETE_NAME_r00", [_DELETE_SUBSCR] = "_DELETE_SUBSCR", + [_DELETE_SUBSCR_r00] = "_DELETE_SUBSCR_r00", [_DEOPT] = "_DEOPT", + [_DEOPT_r00] = "_DEOPT_r00", + [_DEOPT_r10] = "_DEOPT_r10", + [_DEOPT_r20] = "_DEOPT_r20", + [_DEOPT_r30] = "_DEOPT_r30", [_DICT_MERGE] = "_DICT_MERGE", + [_DICT_MERGE_r00] = "_DICT_MERGE_r00", [_DICT_UPDATE] = "_DICT_UPDATE", + [_DICT_UPDATE_r00] = "_DICT_UPDATE_r00", [_END_FOR] = "_END_FOR", + [_END_FOR_r10] = "_END_FOR_r10", [_END_SEND] = "_END_SEND", + [_END_SEND_r21] = "_END_SEND_r21", [_ERROR_POP_N] = "_ERROR_POP_N", + [_ERROR_POP_N_r00] = "_ERROR_POP_N_r00", [_EXIT_INIT_CHECK] = "_EXIT_INIT_CHECK", + [_EXIT_INIT_CHECK_r00] = "_EXIT_INIT_CHECK_r00", [_EXIT_TRACE] = "_EXIT_TRACE", + [_EXIT_TRACE_r00] = "_EXIT_TRACE_r00", + [_EXIT_TRACE_r10] = "_EXIT_TRACE_r10", + [_EXIT_TRACE_r20] = "_EXIT_TRACE_r20", + [_EXIT_TRACE_r30] = "_EXIT_TRACE_r30", [_EXPAND_METHOD] = "_EXPAND_METHOD", + [_EXPAND_METHOD_r00] = "_EXPAND_METHOD_r00", [_EXPAND_METHOD_KW] = "_EXPAND_METHOD_KW", + [_EXPAND_METHOD_KW_r00] = "_EXPAND_METHOD_KW_r00", [_FATAL_ERROR] = "_FATAL_ERROR", + [_FATAL_ERROR_r00] = "_FATAL_ERROR_r00", + [_FATAL_ERROR_r11] = "_FATAL_ERROR_r11", + [_FATAL_ERROR_r22] = "_FATAL_ERROR_r22", + [_FATAL_ERROR_r33] = "_FATAL_ERROR_r33", [_FORMAT_SIMPLE] = "_FORMAT_SIMPLE", + [_FORMAT_SIMPLE_r01] = "_FORMAT_SIMPLE_r01", [_FORMAT_WITH_SPEC] = "_FORMAT_WITH_SPEC", + [_FORMAT_WITH_SPEC_r01] = "_FORMAT_WITH_SPEC_r01", [_FOR_ITER_GEN_FRAME] = "_FOR_ITER_GEN_FRAME", + [_FOR_ITER_GEN_FRAME_r23] = "_FOR_ITER_GEN_FRAME_r23", [_FOR_ITER_TIER_TWO] = "_FOR_ITER_TIER_TWO", + [_FOR_ITER_TIER_TWO_r01] = "_FOR_ITER_TIER_TWO_r01", [_GET_AITER] = "_GET_AITER", + [_GET_AITER_r01] = "_GET_AITER_r01", [_GET_ANEXT] = "_GET_ANEXT", + [_GET_ANEXT_r01] = "_GET_ANEXT_r01", [_GET_AWAITABLE] = "_GET_AWAITABLE", + [_GET_AWAITABLE_r01] = "_GET_AWAITABLE_r01", [_GET_ITER] = "_GET_ITER", + [_GET_ITER_r02] = "_GET_ITER_r02", [_GET_LEN] = "_GET_LEN", + [_GET_LEN_r01] = "_GET_LEN_r01", [_GET_YIELD_FROM_ITER] = "_GET_YIELD_FROM_ITER", + [_GET_YIELD_FROM_ITER_r01] = "_GET_YIELD_FROM_ITER_r01", [_GUARD_BINARY_OP_EXTEND] = "_GUARD_BINARY_OP_EXTEND", + [_GUARD_BINARY_OP_EXTEND_r00] = "_GUARD_BINARY_OP_EXTEND_r00", [_GUARD_CALLABLE_ISINSTANCE] = "_GUARD_CALLABLE_ISINSTANCE", + [_GUARD_CALLABLE_ISINSTANCE_r33] = "_GUARD_CALLABLE_ISINSTANCE_r33", [_GUARD_CALLABLE_LEN] = "_GUARD_CALLABLE_LEN", + [_GUARD_CALLABLE_LEN_r33] = "_GUARD_CALLABLE_LEN_r33", [_GUARD_CALLABLE_LIST_APPEND] = "_GUARD_CALLABLE_LIST_APPEND", + [_GUARD_CALLABLE_LIST_APPEND_r33] = "_GUARD_CALLABLE_LIST_APPEND_r33", [_GUARD_CALLABLE_STR_1] = "_GUARD_CALLABLE_STR_1", + [_GUARD_CALLABLE_STR_1_r33] = "_GUARD_CALLABLE_STR_1_r33", [_GUARD_CALLABLE_TUPLE_1] = "_GUARD_CALLABLE_TUPLE_1", + [_GUARD_CALLABLE_TUPLE_1_r33] = "_GUARD_CALLABLE_TUPLE_1_r33", [_GUARD_CALLABLE_TYPE_1] = "_GUARD_CALLABLE_TYPE_1", + [_GUARD_CALLABLE_TYPE_1_r33] = "_GUARD_CALLABLE_TYPE_1_r33", [_GUARD_DORV_NO_DICT] = "_GUARD_DORV_NO_DICT", + [_GUARD_DORV_NO_DICT_r11] = "_GUARD_DORV_NO_DICT_r11", + [_GUARD_DORV_NO_DICT_r00] = "_GUARD_DORV_NO_DICT_r00", + [_GUARD_DORV_NO_DICT_r22] = "_GUARD_DORV_NO_DICT_r22", + [_GUARD_DORV_NO_DICT_r33] = "_GUARD_DORV_NO_DICT_r33", [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT", + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r11] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r11", + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r00] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r00", + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r22] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r22", + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r33] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r33", [_GUARD_GLOBALS_VERSION] = "_GUARD_GLOBALS_VERSION", + [_GUARD_GLOBALS_VERSION_r00] = "_GUARD_GLOBALS_VERSION_r00", + [_GUARD_GLOBALS_VERSION_r11] = "_GUARD_GLOBALS_VERSION_r11", + [_GUARD_GLOBALS_VERSION_r22] = "_GUARD_GLOBALS_VERSION_r22", + [_GUARD_GLOBALS_VERSION_r33] = "_GUARD_GLOBALS_VERSION_r33", [_GUARD_IS_FALSE_POP] = "_GUARD_IS_FALSE_POP", + [_GUARD_IS_FALSE_POP_r10] = "_GUARD_IS_FALSE_POP_r10", [_GUARD_IS_NONE_POP] = "_GUARD_IS_NONE_POP", + [_GUARD_IS_NONE_POP_r10] = "_GUARD_IS_NONE_POP_r10", [_GUARD_IS_NOT_NONE_POP] = "_GUARD_IS_NOT_NONE_POP", + [_GUARD_IS_NOT_NONE_POP_r10] = "_GUARD_IS_NOT_NONE_POP_r10", [_GUARD_IS_TRUE_POP] = "_GUARD_IS_TRUE_POP", + [_GUARD_IS_TRUE_POP_r10] = "_GUARD_IS_TRUE_POP_r10", [_GUARD_KEYS_VERSION] = "_GUARD_KEYS_VERSION", + [_GUARD_KEYS_VERSION_r11] = "_GUARD_KEYS_VERSION_r11", + [_GUARD_KEYS_VERSION_r00] = "_GUARD_KEYS_VERSION_r00", + [_GUARD_KEYS_VERSION_r22] = "_GUARD_KEYS_VERSION_r22", + [_GUARD_KEYS_VERSION_r33] = "_GUARD_KEYS_VERSION_r33", [_GUARD_NOS_DICT] = "_GUARD_NOS_DICT", + [_GUARD_NOS_DICT_r22] = "_GUARD_NOS_DICT_r22", + [_GUARD_NOS_DICT_r00] = "_GUARD_NOS_DICT_r00", + [_GUARD_NOS_DICT_r11] = "_GUARD_NOS_DICT_r11", + [_GUARD_NOS_DICT_r33] = "_GUARD_NOS_DICT_r33", [_GUARD_NOS_FLOAT] = "_GUARD_NOS_FLOAT", + [_GUARD_NOS_FLOAT_r22] = "_GUARD_NOS_FLOAT_r22", + [_GUARD_NOS_FLOAT_r00] = "_GUARD_NOS_FLOAT_r00", + [_GUARD_NOS_FLOAT_r11] = "_GUARD_NOS_FLOAT_r11", + [_GUARD_NOS_FLOAT_r33] = "_GUARD_NOS_FLOAT_r33", [_GUARD_NOS_INT] = "_GUARD_NOS_INT", + [_GUARD_NOS_INT_r22] = "_GUARD_NOS_INT_r22", + [_GUARD_NOS_INT_r00] = "_GUARD_NOS_INT_r00", + [_GUARD_NOS_INT_r11] = "_GUARD_NOS_INT_r11", + [_GUARD_NOS_INT_r33] = "_GUARD_NOS_INT_r33", [_GUARD_NOS_LIST] = "_GUARD_NOS_LIST", + [_GUARD_NOS_LIST_r22] = "_GUARD_NOS_LIST_r22", + [_GUARD_NOS_LIST_r00] = "_GUARD_NOS_LIST_r00", + [_GUARD_NOS_LIST_r11] = "_GUARD_NOS_LIST_r11", + [_GUARD_NOS_LIST_r33] = "_GUARD_NOS_LIST_r33", [_GUARD_NOS_NOT_NULL] = "_GUARD_NOS_NOT_NULL", + [_GUARD_NOS_NOT_NULL_r22] = "_GUARD_NOS_NOT_NULL_r22", + [_GUARD_NOS_NOT_NULL_r00] = "_GUARD_NOS_NOT_NULL_r00", + [_GUARD_NOS_NOT_NULL_r11] = "_GUARD_NOS_NOT_NULL_r11", + [_GUARD_NOS_NOT_NULL_r33] = "_GUARD_NOS_NOT_NULL_r33", [_GUARD_NOS_NULL] = "_GUARD_NOS_NULL", + [_GUARD_NOS_NULL_r22] = "_GUARD_NOS_NULL_r22", + [_GUARD_NOS_NULL_r00] = "_GUARD_NOS_NULL_r00", + [_GUARD_NOS_NULL_r11] = "_GUARD_NOS_NULL_r11", + [_GUARD_NOS_NULL_r33] = "_GUARD_NOS_NULL_r33", [_GUARD_NOS_OVERFLOWED] = "_GUARD_NOS_OVERFLOWED", + [_GUARD_NOS_OVERFLOWED_r22] = "_GUARD_NOS_OVERFLOWED_r22", + [_GUARD_NOS_OVERFLOWED_r00] = "_GUARD_NOS_OVERFLOWED_r00", + [_GUARD_NOS_OVERFLOWED_r11] = "_GUARD_NOS_OVERFLOWED_r11", + [_GUARD_NOS_OVERFLOWED_r33] = "_GUARD_NOS_OVERFLOWED_r33", [_GUARD_NOS_TUPLE] = "_GUARD_NOS_TUPLE", + [_GUARD_NOS_TUPLE_r22] = "_GUARD_NOS_TUPLE_r22", + [_GUARD_NOS_TUPLE_r00] = "_GUARD_NOS_TUPLE_r00", + [_GUARD_NOS_TUPLE_r11] = "_GUARD_NOS_TUPLE_r11", + [_GUARD_NOS_TUPLE_r33] = "_GUARD_NOS_TUPLE_r33", [_GUARD_NOS_UNICODE] = "_GUARD_NOS_UNICODE", + [_GUARD_NOS_UNICODE_r22] = "_GUARD_NOS_UNICODE_r22", + [_GUARD_NOS_UNICODE_r00] = "_GUARD_NOS_UNICODE_r00", + [_GUARD_NOS_UNICODE_r11] = "_GUARD_NOS_UNICODE_r11", + [_GUARD_NOS_UNICODE_r33] = "_GUARD_NOS_UNICODE_r33", [_GUARD_NOT_EXHAUSTED_LIST] = "_GUARD_NOT_EXHAUSTED_LIST", + [_GUARD_NOT_EXHAUSTED_LIST_r22] = "_GUARD_NOT_EXHAUSTED_LIST_r22", + [_GUARD_NOT_EXHAUSTED_LIST_r00] = "_GUARD_NOT_EXHAUSTED_LIST_r00", + [_GUARD_NOT_EXHAUSTED_LIST_r11] = "_GUARD_NOT_EXHAUSTED_LIST_r11", + [_GUARD_NOT_EXHAUSTED_LIST_r33] = "_GUARD_NOT_EXHAUSTED_LIST_r33", [_GUARD_NOT_EXHAUSTED_RANGE] = "_GUARD_NOT_EXHAUSTED_RANGE", + [_GUARD_NOT_EXHAUSTED_RANGE_r22] = "_GUARD_NOT_EXHAUSTED_RANGE_r22", + [_GUARD_NOT_EXHAUSTED_RANGE_r00] = "_GUARD_NOT_EXHAUSTED_RANGE_r00", + [_GUARD_NOT_EXHAUSTED_RANGE_r11] = "_GUARD_NOT_EXHAUSTED_RANGE_r11", + [_GUARD_NOT_EXHAUSTED_RANGE_r33] = "_GUARD_NOT_EXHAUSTED_RANGE_r33", [_GUARD_NOT_EXHAUSTED_TUPLE] = "_GUARD_NOT_EXHAUSTED_TUPLE", + [_GUARD_NOT_EXHAUSTED_TUPLE_r22] = "_GUARD_NOT_EXHAUSTED_TUPLE_r22", + [_GUARD_NOT_EXHAUSTED_TUPLE_r00] = "_GUARD_NOT_EXHAUSTED_TUPLE_r00", + [_GUARD_NOT_EXHAUSTED_TUPLE_r11] = "_GUARD_NOT_EXHAUSTED_TUPLE_r11", + [_GUARD_NOT_EXHAUSTED_TUPLE_r33] = "_GUARD_NOT_EXHAUSTED_TUPLE_r33", [_GUARD_THIRD_NULL] = "_GUARD_THIRD_NULL", + [_GUARD_THIRD_NULL_r33] = "_GUARD_THIRD_NULL_r33", [_GUARD_TOS_ANY_SET] = "_GUARD_TOS_ANY_SET", + [_GUARD_TOS_ANY_SET_r11] = "_GUARD_TOS_ANY_SET_r11", + [_GUARD_TOS_ANY_SET_r00] = "_GUARD_TOS_ANY_SET_r00", + [_GUARD_TOS_ANY_SET_r22] = "_GUARD_TOS_ANY_SET_r22", + [_GUARD_TOS_ANY_SET_r33] = "_GUARD_TOS_ANY_SET_r33", [_GUARD_TOS_DICT] = "_GUARD_TOS_DICT", + [_GUARD_TOS_DICT_r11] = "_GUARD_TOS_DICT_r11", + [_GUARD_TOS_DICT_r00] = "_GUARD_TOS_DICT_r00", + [_GUARD_TOS_DICT_r22] = "_GUARD_TOS_DICT_r22", + [_GUARD_TOS_DICT_r33] = "_GUARD_TOS_DICT_r33", [_GUARD_TOS_FLOAT] = "_GUARD_TOS_FLOAT", + [_GUARD_TOS_FLOAT_r11] = "_GUARD_TOS_FLOAT_r11", + [_GUARD_TOS_FLOAT_r00] = "_GUARD_TOS_FLOAT_r00", + [_GUARD_TOS_FLOAT_r22] = "_GUARD_TOS_FLOAT_r22", + [_GUARD_TOS_FLOAT_r33] = "_GUARD_TOS_FLOAT_r33", [_GUARD_TOS_INT] = "_GUARD_TOS_INT", + [_GUARD_TOS_INT_r11] = "_GUARD_TOS_INT_r11", + [_GUARD_TOS_INT_r00] = "_GUARD_TOS_INT_r00", + [_GUARD_TOS_INT_r22] = "_GUARD_TOS_INT_r22", + [_GUARD_TOS_INT_r33] = "_GUARD_TOS_INT_r33", [_GUARD_TOS_LIST] = "_GUARD_TOS_LIST", + [_GUARD_TOS_LIST_r11] = "_GUARD_TOS_LIST_r11", + [_GUARD_TOS_LIST_r00] = "_GUARD_TOS_LIST_r00", + [_GUARD_TOS_LIST_r22] = "_GUARD_TOS_LIST_r22", + [_GUARD_TOS_LIST_r33] = "_GUARD_TOS_LIST_r33", [_GUARD_TOS_OVERFLOWED] = "_GUARD_TOS_OVERFLOWED", + [_GUARD_TOS_OVERFLOWED_r11] = "_GUARD_TOS_OVERFLOWED_r11", + [_GUARD_TOS_OVERFLOWED_r00] = "_GUARD_TOS_OVERFLOWED_r00", + [_GUARD_TOS_OVERFLOWED_r22] = "_GUARD_TOS_OVERFLOWED_r22", + [_GUARD_TOS_OVERFLOWED_r33] = "_GUARD_TOS_OVERFLOWED_r33", [_GUARD_TOS_SLICE] = "_GUARD_TOS_SLICE", + [_GUARD_TOS_SLICE_r11] = "_GUARD_TOS_SLICE_r11", + [_GUARD_TOS_SLICE_r00] = "_GUARD_TOS_SLICE_r00", + [_GUARD_TOS_SLICE_r22] = "_GUARD_TOS_SLICE_r22", + [_GUARD_TOS_SLICE_r33] = "_GUARD_TOS_SLICE_r33", [_GUARD_TOS_TUPLE] = "_GUARD_TOS_TUPLE", + [_GUARD_TOS_TUPLE_r11] = "_GUARD_TOS_TUPLE_r11", + [_GUARD_TOS_TUPLE_r00] = "_GUARD_TOS_TUPLE_r00", + [_GUARD_TOS_TUPLE_r22] = "_GUARD_TOS_TUPLE_r22", + [_GUARD_TOS_TUPLE_r33] = "_GUARD_TOS_TUPLE_r33", [_GUARD_TOS_UNICODE] = "_GUARD_TOS_UNICODE", + [_GUARD_TOS_UNICODE_r11] = "_GUARD_TOS_UNICODE_r11", + [_GUARD_TOS_UNICODE_r00] = "_GUARD_TOS_UNICODE_r00", + [_GUARD_TOS_UNICODE_r22] = "_GUARD_TOS_UNICODE_r22", + [_GUARD_TOS_UNICODE_r33] = "_GUARD_TOS_UNICODE_r33", [_GUARD_TYPE_VERSION] = "_GUARD_TYPE_VERSION", + [_GUARD_TYPE_VERSION_r11] = "_GUARD_TYPE_VERSION_r11", + [_GUARD_TYPE_VERSION_r00] = "_GUARD_TYPE_VERSION_r00", + [_GUARD_TYPE_VERSION_r22] = "_GUARD_TYPE_VERSION_r22", + [_GUARD_TYPE_VERSION_r33] = "_GUARD_TYPE_VERSION_r33", [_GUARD_TYPE_VERSION_AND_LOCK] = "_GUARD_TYPE_VERSION_AND_LOCK", + [_GUARD_TYPE_VERSION_AND_LOCK_r11] = "_GUARD_TYPE_VERSION_AND_LOCK_r11", + [_GUARD_TYPE_VERSION_AND_LOCK_r00] = "_GUARD_TYPE_VERSION_AND_LOCK_r00", + [_GUARD_TYPE_VERSION_AND_LOCK_r22] = "_GUARD_TYPE_VERSION_AND_LOCK_r22", + [_GUARD_TYPE_VERSION_AND_LOCK_r33] = "_GUARD_TYPE_VERSION_AND_LOCK_r33", [_IMPORT_FROM] = "_IMPORT_FROM", + [_IMPORT_FROM_r01] = "_IMPORT_FROM_r01", [_IMPORT_NAME] = "_IMPORT_NAME", + [_IMPORT_NAME_r01] = "_IMPORT_NAME_r01", [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = "_INIT_CALL_BOUND_METHOD_EXACT_ARGS", + [_INIT_CALL_BOUND_METHOD_EXACT_ARGS_r00] = "_INIT_CALL_BOUND_METHOD_EXACT_ARGS_r00", [_INIT_CALL_PY_EXACT_ARGS] = "_INIT_CALL_PY_EXACT_ARGS", + [_INIT_CALL_PY_EXACT_ARGS_r01] = "_INIT_CALL_PY_EXACT_ARGS_r01", [_INIT_CALL_PY_EXACT_ARGS_0] = "_INIT_CALL_PY_EXACT_ARGS_0", + [_INIT_CALL_PY_EXACT_ARGS_0_r01] = "_INIT_CALL_PY_EXACT_ARGS_0_r01", [_INIT_CALL_PY_EXACT_ARGS_1] = "_INIT_CALL_PY_EXACT_ARGS_1", + [_INIT_CALL_PY_EXACT_ARGS_1_r01] = "_INIT_CALL_PY_EXACT_ARGS_1_r01", [_INIT_CALL_PY_EXACT_ARGS_2] = "_INIT_CALL_PY_EXACT_ARGS_2", + [_INIT_CALL_PY_EXACT_ARGS_2_r01] = "_INIT_CALL_PY_EXACT_ARGS_2_r01", [_INIT_CALL_PY_EXACT_ARGS_3] = "_INIT_CALL_PY_EXACT_ARGS_3", + [_INIT_CALL_PY_EXACT_ARGS_3_r01] = "_INIT_CALL_PY_EXACT_ARGS_3_r01", [_INIT_CALL_PY_EXACT_ARGS_4] = "_INIT_CALL_PY_EXACT_ARGS_4", + [_INIT_CALL_PY_EXACT_ARGS_4_r01] = "_INIT_CALL_PY_EXACT_ARGS_4_r01", [_INSERT_NULL] = "_INSERT_NULL", + [_INSERT_NULL_r10] = "_INSERT_NULL_r10", [_IS_NONE] = "_IS_NONE", + [_IS_NONE_r11] = "_IS_NONE_r11", [_IS_OP] = "_IS_OP", + [_IS_OP_r21] = "_IS_OP_r21", [_ITER_CHECK_LIST] = "_ITER_CHECK_LIST", + [_ITER_CHECK_LIST_r22] = "_ITER_CHECK_LIST_r22", + [_ITER_CHECK_LIST_r00] = "_ITER_CHECK_LIST_r00", + [_ITER_CHECK_LIST_r11] = "_ITER_CHECK_LIST_r11", + [_ITER_CHECK_LIST_r33] = "_ITER_CHECK_LIST_r33", [_ITER_CHECK_RANGE] = "_ITER_CHECK_RANGE", + [_ITER_CHECK_RANGE_r22] = "_ITER_CHECK_RANGE_r22", + [_ITER_CHECK_RANGE_r00] = "_ITER_CHECK_RANGE_r00", + [_ITER_CHECK_RANGE_r11] = "_ITER_CHECK_RANGE_r11", + [_ITER_CHECK_RANGE_r33] = "_ITER_CHECK_RANGE_r33", [_ITER_CHECK_TUPLE] = "_ITER_CHECK_TUPLE", + [_ITER_CHECK_TUPLE_r22] = "_ITER_CHECK_TUPLE_r22", + [_ITER_CHECK_TUPLE_r00] = "_ITER_CHECK_TUPLE_r00", + [_ITER_CHECK_TUPLE_r11] = "_ITER_CHECK_TUPLE_r11", + [_ITER_CHECK_TUPLE_r33] = "_ITER_CHECK_TUPLE_r33", [_ITER_NEXT_LIST_TIER_TWO] = "_ITER_NEXT_LIST_TIER_TWO", + [_ITER_NEXT_LIST_TIER_TWO_r01] = "_ITER_NEXT_LIST_TIER_TWO_r01", [_ITER_NEXT_RANGE] = "_ITER_NEXT_RANGE", + [_ITER_NEXT_RANGE_r23] = "_ITER_NEXT_RANGE_r23", [_ITER_NEXT_TUPLE] = "_ITER_NEXT_TUPLE", + [_ITER_NEXT_TUPLE_r23] = "_ITER_NEXT_TUPLE_r23", [_JUMP_TO_TOP] = "_JUMP_TO_TOP", + [_JUMP_TO_TOP_r00] = "_JUMP_TO_TOP_r00", [_LIST_APPEND] = "_LIST_APPEND", + [_LIST_APPEND_r10] = "_LIST_APPEND_r10", [_LIST_EXTEND] = "_LIST_EXTEND", + [_LIST_EXTEND_r00] = "_LIST_EXTEND_r00", [_LOAD_ATTR] = "_LOAD_ATTR", + [_LOAD_ATTR_r00] = "_LOAD_ATTR_r00", [_LOAD_ATTR_CLASS] = "_LOAD_ATTR_CLASS", + [_LOAD_ATTR_CLASS_r11] = "_LOAD_ATTR_CLASS_r11", [_LOAD_ATTR_INSTANCE_VALUE] = "_LOAD_ATTR_INSTANCE_VALUE", + [_LOAD_ATTR_INSTANCE_VALUE_r11] = "_LOAD_ATTR_INSTANCE_VALUE_r11", [_LOAD_ATTR_METHOD_LAZY_DICT] = "_LOAD_ATTR_METHOD_LAZY_DICT", + [_LOAD_ATTR_METHOD_LAZY_DICT_r12] = "_LOAD_ATTR_METHOD_LAZY_DICT_r12", + [_LOAD_ATTR_METHOD_LAZY_DICT_r01] = "_LOAD_ATTR_METHOD_LAZY_DICT_r01", + [_LOAD_ATTR_METHOD_LAZY_DICT_r23] = "_LOAD_ATTR_METHOD_LAZY_DICT_r23", [_LOAD_ATTR_METHOD_NO_DICT] = "_LOAD_ATTR_METHOD_NO_DICT", + [_LOAD_ATTR_METHOD_NO_DICT_r12] = "_LOAD_ATTR_METHOD_NO_DICT_r12", + [_LOAD_ATTR_METHOD_NO_DICT_r01] = "_LOAD_ATTR_METHOD_NO_DICT_r01", + [_LOAD_ATTR_METHOD_NO_DICT_r23] = "_LOAD_ATTR_METHOD_NO_DICT_r23", [_LOAD_ATTR_METHOD_WITH_VALUES] = "_LOAD_ATTR_METHOD_WITH_VALUES", + [_LOAD_ATTR_METHOD_WITH_VALUES_r12] = "_LOAD_ATTR_METHOD_WITH_VALUES_r12", + [_LOAD_ATTR_METHOD_WITH_VALUES_r01] = "_LOAD_ATTR_METHOD_WITH_VALUES_r01", + [_LOAD_ATTR_METHOD_WITH_VALUES_r23] = "_LOAD_ATTR_METHOD_WITH_VALUES_r23", [_LOAD_ATTR_MODULE] = "_LOAD_ATTR_MODULE", + [_LOAD_ATTR_MODULE_r11] = "_LOAD_ATTR_MODULE_r11", [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = "_LOAD_ATTR_NONDESCRIPTOR_NO_DICT", + [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11] = "_LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11", [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = "_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", + [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11] = "_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11", [_LOAD_ATTR_PROPERTY_FRAME] = "_LOAD_ATTR_PROPERTY_FRAME", + [_LOAD_ATTR_PROPERTY_FRAME_r11] = "_LOAD_ATTR_PROPERTY_FRAME_r11", [_LOAD_ATTR_SLOT] = "_LOAD_ATTR_SLOT", + [_LOAD_ATTR_SLOT_r11] = "_LOAD_ATTR_SLOT_r11", [_LOAD_ATTR_WITH_HINT] = "_LOAD_ATTR_WITH_HINT", + [_LOAD_ATTR_WITH_HINT_r11] = "_LOAD_ATTR_WITH_HINT_r11", [_LOAD_BUILD_CLASS] = "_LOAD_BUILD_CLASS", + [_LOAD_BUILD_CLASS_r01] = "_LOAD_BUILD_CLASS_r01", [_LOAD_COMMON_CONSTANT] = "_LOAD_COMMON_CONSTANT", + [_LOAD_COMMON_CONSTANT_r01] = "_LOAD_COMMON_CONSTANT_r01", + [_LOAD_COMMON_CONSTANT_r12] = "_LOAD_COMMON_CONSTANT_r12", + [_LOAD_COMMON_CONSTANT_r23] = "_LOAD_COMMON_CONSTANT_r23", [_LOAD_CONST] = "_LOAD_CONST", + [_LOAD_CONST_r01] = "_LOAD_CONST_r01", + [_LOAD_CONST_r12] = "_LOAD_CONST_r12", + [_LOAD_CONST_r23] = "_LOAD_CONST_r23", [_LOAD_CONST_INLINE] = "_LOAD_CONST_INLINE", + [_LOAD_CONST_INLINE_r01] = "_LOAD_CONST_INLINE_r01", + [_LOAD_CONST_INLINE_r12] = "_LOAD_CONST_INLINE_r12", + [_LOAD_CONST_INLINE_r23] = "_LOAD_CONST_INLINE_r23", [_LOAD_CONST_INLINE_BORROW] = "_LOAD_CONST_INLINE_BORROW", + [_LOAD_CONST_INLINE_BORROW_r01] = "_LOAD_CONST_INLINE_BORROW_r01", + [_LOAD_CONST_INLINE_BORROW_r12] = "_LOAD_CONST_INLINE_BORROW_r12", + [_LOAD_CONST_INLINE_BORROW_r23] = "_LOAD_CONST_INLINE_BORROW_r23", [_LOAD_CONST_UNDER_INLINE] = "_LOAD_CONST_UNDER_INLINE", + [_LOAD_CONST_UNDER_INLINE_r12] = "_LOAD_CONST_UNDER_INLINE_r12", + [_LOAD_CONST_UNDER_INLINE_r01] = "_LOAD_CONST_UNDER_INLINE_r01", + [_LOAD_CONST_UNDER_INLINE_r23] = "_LOAD_CONST_UNDER_INLINE_r23", [_LOAD_CONST_UNDER_INLINE_BORROW] = "_LOAD_CONST_UNDER_INLINE_BORROW", + [_LOAD_CONST_UNDER_INLINE_BORROW_r12] = "_LOAD_CONST_UNDER_INLINE_BORROW_r12", + [_LOAD_CONST_UNDER_INLINE_BORROW_r01] = "_LOAD_CONST_UNDER_INLINE_BORROW_r01", + [_LOAD_CONST_UNDER_INLINE_BORROW_r23] = "_LOAD_CONST_UNDER_INLINE_BORROW_r23", [_LOAD_DEREF] = "_LOAD_DEREF", + [_LOAD_DEREF_r01] = "_LOAD_DEREF_r01", [_LOAD_FAST] = "_LOAD_FAST", + [_LOAD_FAST_r01] = "_LOAD_FAST_r01", + [_LOAD_FAST_r12] = "_LOAD_FAST_r12", + [_LOAD_FAST_r23] = "_LOAD_FAST_r23", [_LOAD_FAST_0] = "_LOAD_FAST_0", + [_LOAD_FAST_0_r01] = "_LOAD_FAST_0_r01", + [_LOAD_FAST_0_r12] = "_LOAD_FAST_0_r12", + [_LOAD_FAST_0_r23] = "_LOAD_FAST_0_r23", [_LOAD_FAST_1] = "_LOAD_FAST_1", + [_LOAD_FAST_1_r01] = "_LOAD_FAST_1_r01", + [_LOAD_FAST_1_r12] = "_LOAD_FAST_1_r12", + [_LOAD_FAST_1_r23] = "_LOAD_FAST_1_r23", [_LOAD_FAST_2] = "_LOAD_FAST_2", + [_LOAD_FAST_2_r01] = "_LOAD_FAST_2_r01", + [_LOAD_FAST_2_r12] = "_LOAD_FAST_2_r12", + [_LOAD_FAST_2_r23] = "_LOAD_FAST_2_r23", [_LOAD_FAST_3] = "_LOAD_FAST_3", + [_LOAD_FAST_3_r01] = "_LOAD_FAST_3_r01", + [_LOAD_FAST_3_r12] = "_LOAD_FAST_3_r12", + [_LOAD_FAST_3_r23] = "_LOAD_FAST_3_r23", [_LOAD_FAST_4] = "_LOAD_FAST_4", + [_LOAD_FAST_4_r01] = "_LOAD_FAST_4_r01", + [_LOAD_FAST_4_r12] = "_LOAD_FAST_4_r12", + [_LOAD_FAST_4_r23] = "_LOAD_FAST_4_r23", [_LOAD_FAST_5] = "_LOAD_FAST_5", + [_LOAD_FAST_5_r01] = "_LOAD_FAST_5_r01", + [_LOAD_FAST_5_r12] = "_LOAD_FAST_5_r12", + [_LOAD_FAST_5_r23] = "_LOAD_FAST_5_r23", [_LOAD_FAST_6] = "_LOAD_FAST_6", + [_LOAD_FAST_6_r01] = "_LOAD_FAST_6_r01", + [_LOAD_FAST_6_r12] = "_LOAD_FAST_6_r12", + [_LOAD_FAST_6_r23] = "_LOAD_FAST_6_r23", [_LOAD_FAST_7] = "_LOAD_FAST_7", + [_LOAD_FAST_7_r01] = "_LOAD_FAST_7_r01", + [_LOAD_FAST_7_r12] = "_LOAD_FAST_7_r12", + [_LOAD_FAST_7_r23] = "_LOAD_FAST_7_r23", [_LOAD_FAST_AND_CLEAR] = "_LOAD_FAST_AND_CLEAR", + [_LOAD_FAST_AND_CLEAR_r01] = "_LOAD_FAST_AND_CLEAR_r01", + [_LOAD_FAST_AND_CLEAR_r12] = "_LOAD_FAST_AND_CLEAR_r12", + [_LOAD_FAST_AND_CLEAR_r23] = "_LOAD_FAST_AND_CLEAR_r23", [_LOAD_FAST_BORROW] = "_LOAD_FAST_BORROW", + [_LOAD_FAST_BORROW_r01] = "_LOAD_FAST_BORROW_r01", + [_LOAD_FAST_BORROW_r12] = "_LOAD_FAST_BORROW_r12", + [_LOAD_FAST_BORROW_r23] = "_LOAD_FAST_BORROW_r23", [_LOAD_FAST_BORROW_0] = "_LOAD_FAST_BORROW_0", + [_LOAD_FAST_BORROW_0_r01] = "_LOAD_FAST_BORROW_0_r01", + [_LOAD_FAST_BORROW_0_r12] = "_LOAD_FAST_BORROW_0_r12", + [_LOAD_FAST_BORROW_0_r23] = "_LOAD_FAST_BORROW_0_r23", [_LOAD_FAST_BORROW_1] = "_LOAD_FAST_BORROW_1", + [_LOAD_FAST_BORROW_1_r01] = "_LOAD_FAST_BORROW_1_r01", + [_LOAD_FAST_BORROW_1_r12] = "_LOAD_FAST_BORROW_1_r12", + [_LOAD_FAST_BORROW_1_r23] = "_LOAD_FAST_BORROW_1_r23", [_LOAD_FAST_BORROW_2] = "_LOAD_FAST_BORROW_2", + [_LOAD_FAST_BORROW_2_r01] = "_LOAD_FAST_BORROW_2_r01", + [_LOAD_FAST_BORROW_2_r12] = "_LOAD_FAST_BORROW_2_r12", + [_LOAD_FAST_BORROW_2_r23] = "_LOAD_FAST_BORROW_2_r23", [_LOAD_FAST_BORROW_3] = "_LOAD_FAST_BORROW_3", + [_LOAD_FAST_BORROW_3_r01] = "_LOAD_FAST_BORROW_3_r01", + [_LOAD_FAST_BORROW_3_r12] = "_LOAD_FAST_BORROW_3_r12", + [_LOAD_FAST_BORROW_3_r23] = "_LOAD_FAST_BORROW_3_r23", [_LOAD_FAST_BORROW_4] = "_LOAD_FAST_BORROW_4", + [_LOAD_FAST_BORROW_4_r01] = "_LOAD_FAST_BORROW_4_r01", + [_LOAD_FAST_BORROW_4_r12] = "_LOAD_FAST_BORROW_4_r12", + [_LOAD_FAST_BORROW_4_r23] = "_LOAD_FAST_BORROW_4_r23", [_LOAD_FAST_BORROW_5] = "_LOAD_FAST_BORROW_5", + [_LOAD_FAST_BORROW_5_r01] = "_LOAD_FAST_BORROW_5_r01", + [_LOAD_FAST_BORROW_5_r12] = "_LOAD_FAST_BORROW_5_r12", + [_LOAD_FAST_BORROW_5_r23] = "_LOAD_FAST_BORROW_5_r23", [_LOAD_FAST_BORROW_6] = "_LOAD_FAST_BORROW_6", + [_LOAD_FAST_BORROW_6_r01] = "_LOAD_FAST_BORROW_6_r01", + [_LOAD_FAST_BORROW_6_r12] = "_LOAD_FAST_BORROW_6_r12", + [_LOAD_FAST_BORROW_6_r23] = "_LOAD_FAST_BORROW_6_r23", [_LOAD_FAST_BORROW_7] = "_LOAD_FAST_BORROW_7", - [_LOAD_FAST_BORROW_LOAD_FAST_BORROW] = "_LOAD_FAST_BORROW_LOAD_FAST_BORROW", + [_LOAD_FAST_BORROW_7_r01] = "_LOAD_FAST_BORROW_7_r01", + [_LOAD_FAST_BORROW_7_r12] = "_LOAD_FAST_BORROW_7_r12", + [_LOAD_FAST_BORROW_7_r23] = "_LOAD_FAST_BORROW_7_r23", [_LOAD_FAST_CHECK] = "_LOAD_FAST_CHECK", - [_LOAD_FAST_LOAD_FAST] = "_LOAD_FAST_LOAD_FAST", + [_LOAD_FAST_CHECK_r01] = "_LOAD_FAST_CHECK_r01", [_LOAD_FROM_DICT_OR_DEREF] = "_LOAD_FROM_DICT_OR_DEREF", + [_LOAD_FROM_DICT_OR_DEREF_r01] = "_LOAD_FROM_DICT_OR_DEREF_r01", [_LOAD_GLOBAL] = "_LOAD_GLOBAL", + [_LOAD_GLOBAL_r00] = "_LOAD_GLOBAL_r00", [_LOAD_GLOBAL_BUILTINS] = "_LOAD_GLOBAL_BUILTINS", + [_LOAD_GLOBAL_BUILTINS_r01] = "_LOAD_GLOBAL_BUILTINS_r01", [_LOAD_GLOBAL_MODULE] = "_LOAD_GLOBAL_MODULE", + [_LOAD_GLOBAL_MODULE_r01] = "_LOAD_GLOBAL_MODULE_r01", [_LOAD_LOCALS] = "_LOAD_LOCALS", + [_LOAD_LOCALS_r01] = "_LOAD_LOCALS_r01", [_LOAD_NAME] = "_LOAD_NAME", + [_LOAD_NAME_r01] = "_LOAD_NAME_r01", [_LOAD_SMALL_INT] = "_LOAD_SMALL_INT", + [_LOAD_SMALL_INT_r01] = "_LOAD_SMALL_INT_r01", + [_LOAD_SMALL_INT_r12] = "_LOAD_SMALL_INT_r12", + [_LOAD_SMALL_INT_r23] = "_LOAD_SMALL_INT_r23", [_LOAD_SMALL_INT_0] = "_LOAD_SMALL_INT_0", + [_LOAD_SMALL_INT_0_r01] = "_LOAD_SMALL_INT_0_r01", + [_LOAD_SMALL_INT_0_r12] = "_LOAD_SMALL_INT_0_r12", + [_LOAD_SMALL_INT_0_r23] = "_LOAD_SMALL_INT_0_r23", [_LOAD_SMALL_INT_1] = "_LOAD_SMALL_INT_1", + [_LOAD_SMALL_INT_1_r01] = "_LOAD_SMALL_INT_1_r01", + [_LOAD_SMALL_INT_1_r12] = "_LOAD_SMALL_INT_1_r12", + [_LOAD_SMALL_INT_1_r23] = "_LOAD_SMALL_INT_1_r23", [_LOAD_SMALL_INT_2] = "_LOAD_SMALL_INT_2", + [_LOAD_SMALL_INT_2_r01] = "_LOAD_SMALL_INT_2_r01", + [_LOAD_SMALL_INT_2_r12] = "_LOAD_SMALL_INT_2_r12", + [_LOAD_SMALL_INT_2_r23] = "_LOAD_SMALL_INT_2_r23", [_LOAD_SMALL_INT_3] = "_LOAD_SMALL_INT_3", + [_LOAD_SMALL_INT_3_r01] = "_LOAD_SMALL_INT_3_r01", + [_LOAD_SMALL_INT_3_r12] = "_LOAD_SMALL_INT_3_r12", + [_LOAD_SMALL_INT_3_r23] = "_LOAD_SMALL_INT_3_r23", [_LOAD_SPECIAL] = "_LOAD_SPECIAL", + [_LOAD_SPECIAL_r00] = "_LOAD_SPECIAL_r00", [_LOAD_SUPER_ATTR_ATTR] = "_LOAD_SUPER_ATTR_ATTR", + [_LOAD_SUPER_ATTR_ATTR_r01] = "_LOAD_SUPER_ATTR_ATTR_r01", [_LOAD_SUPER_ATTR_METHOD] = "_LOAD_SUPER_ATTR_METHOD", + [_LOAD_SUPER_ATTR_METHOD_r02] = "_LOAD_SUPER_ATTR_METHOD_r02", [_MAKE_CALLARGS_A_TUPLE] = "_MAKE_CALLARGS_A_TUPLE", + [_MAKE_CALLARGS_A_TUPLE_r00] = "_MAKE_CALLARGS_A_TUPLE_r00", [_MAKE_CELL] = "_MAKE_CELL", + [_MAKE_CELL_r00] = "_MAKE_CELL_r00", [_MAKE_FUNCTION] = "_MAKE_FUNCTION", + [_MAKE_FUNCTION_r01] = "_MAKE_FUNCTION_r01", [_MAKE_WARM] = "_MAKE_WARM", + [_MAKE_WARM_r00] = "_MAKE_WARM_r00", + [_MAKE_WARM_r11] = "_MAKE_WARM_r11", + [_MAKE_WARM_r22] = "_MAKE_WARM_r22", + [_MAKE_WARM_r33] = "_MAKE_WARM_r33", [_MAP_ADD] = "_MAP_ADD", + [_MAP_ADD_r00] = "_MAP_ADD_r00", [_MATCH_CLASS] = "_MATCH_CLASS", + [_MATCH_CLASS_r01] = "_MATCH_CLASS_r01", [_MATCH_KEYS] = "_MATCH_KEYS", + [_MATCH_KEYS_r01] = "_MATCH_KEYS_r01", [_MATCH_MAPPING] = "_MATCH_MAPPING", + [_MATCH_MAPPING_r12] = "_MATCH_MAPPING_r12", + [_MATCH_MAPPING_r01] = "_MATCH_MAPPING_r01", + [_MATCH_MAPPING_r23] = "_MATCH_MAPPING_r23", [_MATCH_SEQUENCE] = "_MATCH_SEQUENCE", + [_MATCH_SEQUENCE_r12] = "_MATCH_SEQUENCE_r12", + [_MATCH_SEQUENCE_r01] = "_MATCH_SEQUENCE_r01", + [_MATCH_SEQUENCE_r23] = "_MATCH_SEQUENCE_r23", [_MAYBE_EXPAND_METHOD] = "_MAYBE_EXPAND_METHOD", + [_MAYBE_EXPAND_METHOD_r00] = "_MAYBE_EXPAND_METHOD_r00", [_MAYBE_EXPAND_METHOD_KW] = "_MAYBE_EXPAND_METHOD_KW", + [_MAYBE_EXPAND_METHOD_KW_r00] = "_MAYBE_EXPAND_METHOD_KW_r00", [_NOP] = "_NOP", + [_NOP_r00] = "_NOP_r00", + [_NOP_r11] = "_NOP_r11", + [_NOP_r22] = "_NOP_r22", + [_NOP_r33] = "_NOP_r33", [_POP_CALL] = "_POP_CALL", + [_POP_CALL_r20] = "_POP_CALL_r20", [_POP_CALL_LOAD_CONST_INLINE_BORROW] = "_POP_CALL_LOAD_CONST_INLINE_BORROW", + [_POP_CALL_LOAD_CONST_INLINE_BORROW_r21] = "_POP_CALL_LOAD_CONST_INLINE_BORROW_r21", [_POP_CALL_ONE] = "_POP_CALL_ONE", + [_POP_CALL_ONE_r30] = "_POP_CALL_ONE_r30", [_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW] = "_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW", + [_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW_r31] = "_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW_r31", [_POP_CALL_TWO] = "_POP_CALL_TWO", + [_POP_CALL_TWO_r30] = "_POP_CALL_TWO_r30", [_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW] = "_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW", + [_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW_r31] = "_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW_r31", [_POP_EXCEPT] = "_POP_EXCEPT", + [_POP_EXCEPT_r00] = "_POP_EXCEPT_r00", [_POP_ITER] = "_POP_ITER", + [_POP_ITER_r20] = "_POP_ITER_r20", [_POP_TOP] = "_POP_TOP", + [_POP_TOP_r10] = "_POP_TOP_r10", [_POP_TOP_FLOAT] = "_POP_TOP_FLOAT", + [_POP_TOP_FLOAT_r10] = "_POP_TOP_FLOAT_r10", + [_POP_TOP_FLOAT_r21] = "_POP_TOP_FLOAT_r21", + [_POP_TOP_FLOAT_r32] = "_POP_TOP_FLOAT_r32", [_POP_TOP_INT] = "_POP_TOP_INT", + [_POP_TOP_INT_r10] = "_POP_TOP_INT_r10", + [_POP_TOP_INT_r21] = "_POP_TOP_INT_r21", + [_POP_TOP_INT_r32] = "_POP_TOP_INT_r32", [_POP_TOP_LOAD_CONST_INLINE] = "_POP_TOP_LOAD_CONST_INLINE", + [_POP_TOP_LOAD_CONST_INLINE_r11] = "_POP_TOP_LOAD_CONST_INLINE_r11", [_POP_TOP_LOAD_CONST_INLINE_BORROW] = "_POP_TOP_LOAD_CONST_INLINE_BORROW", + [_POP_TOP_LOAD_CONST_INLINE_BORROW_r11] = "_POP_TOP_LOAD_CONST_INLINE_BORROW_r11", [_POP_TOP_NOP] = "_POP_TOP_NOP", + [_POP_TOP_NOP_r10] = "_POP_TOP_NOP_r10", + [_POP_TOP_NOP_r21] = "_POP_TOP_NOP_r21", + [_POP_TOP_NOP_r32] = "_POP_TOP_NOP_r32", [_POP_TOP_UNICODE] = "_POP_TOP_UNICODE", + [_POP_TOP_UNICODE_r10] = "_POP_TOP_UNICODE_r10", + [_POP_TOP_UNICODE_r21] = "_POP_TOP_UNICODE_r21", + [_POP_TOP_UNICODE_r32] = "_POP_TOP_UNICODE_r32", [_POP_TWO] = "_POP_TWO", + [_POP_TWO_r20] = "_POP_TWO_r20", [_POP_TWO_LOAD_CONST_INLINE_BORROW] = "_POP_TWO_LOAD_CONST_INLINE_BORROW", + [_POP_TWO_LOAD_CONST_INLINE_BORROW_r21] = "_POP_TWO_LOAD_CONST_INLINE_BORROW_r21", [_PUSH_EXC_INFO] = "_PUSH_EXC_INFO", + [_PUSH_EXC_INFO_r12] = "_PUSH_EXC_INFO_r12", + [_PUSH_EXC_INFO_r01] = "_PUSH_EXC_INFO_r01", + [_PUSH_EXC_INFO_r23] = "_PUSH_EXC_INFO_r23", [_PUSH_FRAME] = "_PUSH_FRAME", + [_PUSH_FRAME_r10] = "_PUSH_FRAME_r10", [_PUSH_NULL] = "_PUSH_NULL", + [_PUSH_NULL_r01] = "_PUSH_NULL_r01", + [_PUSH_NULL_r12] = "_PUSH_NULL_r12", + [_PUSH_NULL_r23] = "_PUSH_NULL_r23", [_PUSH_NULL_CONDITIONAL] = "_PUSH_NULL_CONDITIONAL", + [_PUSH_NULL_CONDITIONAL_r00] = "_PUSH_NULL_CONDITIONAL_r00", [_PY_FRAME_GENERAL] = "_PY_FRAME_GENERAL", + [_PY_FRAME_GENERAL_r01] = "_PY_FRAME_GENERAL_r01", [_PY_FRAME_KW] = "_PY_FRAME_KW", + [_PY_FRAME_KW_r01] = "_PY_FRAME_KW_r01", [_REPLACE_WITH_TRUE] = "_REPLACE_WITH_TRUE", + [_REPLACE_WITH_TRUE_r11] = "_REPLACE_WITH_TRUE_r11", [_RESUME_CHECK] = "_RESUME_CHECK", + [_RESUME_CHECK_r00] = "_RESUME_CHECK_r00", + [_RESUME_CHECK_r11] = "_RESUME_CHECK_r11", + [_RESUME_CHECK_r22] = "_RESUME_CHECK_r22", + [_RESUME_CHECK_r33] = "_RESUME_CHECK_r33", [_RETURN_GENERATOR] = "_RETURN_GENERATOR", + [_RETURN_GENERATOR_r01] = "_RETURN_GENERATOR_r01", [_RETURN_VALUE] = "_RETURN_VALUE", + [_RETURN_VALUE_r01] = "_RETURN_VALUE_r01", [_SAVE_RETURN_OFFSET] = "_SAVE_RETURN_OFFSET", + [_SAVE_RETURN_OFFSET_r00] = "_SAVE_RETURN_OFFSET_r00", + [_SAVE_RETURN_OFFSET_r11] = "_SAVE_RETURN_OFFSET_r11", + [_SAVE_RETURN_OFFSET_r22] = "_SAVE_RETURN_OFFSET_r22", + [_SAVE_RETURN_OFFSET_r33] = "_SAVE_RETURN_OFFSET_r33", [_SEND_GEN_FRAME] = "_SEND_GEN_FRAME", + [_SEND_GEN_FRAME_r22] = "_SEND_GEN_FRAME_r22", [_SETUP_ANNOTATIONS] = "_SETUP_ANNOTATIONS", + [_SETUP_ANNOTATIONS_r00] = "_SETUP_ANNOTATIONS_r00", [_SET_ADD] = "_SET_ADD", + [_SET_ADD_r00] = "_SET_ADD_r00", [_SET_FUNCTION_ATTRIBUTE] = "_SET_FUNCTION_ATTRIBUTE", + [_SET_FUNCTION_ATTRIBUTE_r21] = "_SET_FUNCTION_ATTRIBUTE_r21", [_SET_IP] = "_SET_IP", + [_SET_IP_r00] = "_SET_IP_r00", + [_SET_IP_r11] = "_SET_IP_r11", + [_SET_IP_r22] = "_SET_IP_r22", + [_SET_IP_r33] = "_SET_IP_r33", [_SET_UPDATE] = "_SET_UPDATE", + [_SET_UPDATE_r00] = "_SET_UPDATE_r00", + [_SPILL_OR_RELOAD] = "_SPILL_OR_RELOAD", + [_SPILL_OR_RELOAD_r01] = "_SPILL_OR_RELOAD_r01", + [_SPILL_OR_RELOAD_r02] = "_SPILL_OR_RELOAD_r02", + [_SPILL_OR_RELOAD_r03] = "_SPILL_OR_RELOAD_r03", + [_SPILL_OR_RELOAD_r10] = "_SPILL_OR_RELOAD_r10", + [_SPILL_OR_RELOAD_r12] = "_SPILL_OR_RELOAD_r12", + [_SPILL_OR_RELOAD_r13] = "_SPILL_OR_RELOAD_r13", + [_SPILL_OR_RELOAD_r20] = "_SPILL_OR_RELOAD_r20", + [_SPILL_OR_RELOAD_r21] = "_SPILL_OR_RELOAD_r21", + [_SPILL_OR_RELOAD_r23] = "_SPILL_OR_RELOAD_r23", + [_SPILL_OR_RELOAD_r30] = "_SPILL_OR_RELOAD_r30", + [_SPILL_OR_RELOAD_r31] = "_SPILL_OR_RELOAD_r31", + [_SPILL_OR_RELOAD_r32] = "_SPILL_OR_RELOAD_r32", [_START_EXECUTOR] = "_START_EXECUTOR", + [_START_EXECUTOR_r00] = "_START_EXECUTOR_r00", [_STORE_ATTR] = "_STORE_ATTR", + [_STORE_ATTR_r00] = "_STORE_ATTR_r00", [_STORE_ATTR_INSTANCE_VALUE] = "_STORE_ATTR_INSTANCE_VALUE", + [_STORE_ATTR_INSTANCE_VALUE_r20] = "_STORE_ATTR_INSTANCE_VALUE_r20", [_STORE_ATTR_SLOT] = "_STORE_ATTR_SLOT", + [_STORE_ATTR_SLOT_r20] = "_STORE_ATTR_SLOT_r20", [_STORE_ATTR_WITH_HINT] = "_STORE_ATTR_WITH_HINT", + [_STORE_ATTR_WITH_HINT_r00] = "_STORE_ATTR_WITH_HINT_r00", [_STORE_DEREF] = "_STORE_DEREF", + [_STORE_DEREF_r00] = "_STORE_DEREF_r00", [_STORE_FAST] = "_STORE_FAST", + [_STORE_FAST_r10] = "_STORE_FAST_r10", [_STORE_FAST_0] = "_STORE_FAST_0", + [_STORE_FAST_0_r10] = "_STORE_FAST_0_r10", [_STORE_FAST_1] = "_STORE_FAST_1", + [_STORE_FAST_1_r10] = "_STORE_FAST_1_r10", [_STORE_FAST_2] = "_STORE_FAST_2", + [_STORE_FAST_2_r10] = "_STORE_FAST_2_r10", [_STORE_FAST_3] = "_STORE_FAST_3", + [_STORE_FAST_3_r10] = "_STORE_FAST_3_r10", [_STORE_FAST_4] = "_STORE_FAST_4", + [_STORE_FAST_4_r10] = "_STORE_FAST_4_r10", [_STORE_FAST_5] = "_STORE_FAST_5", + [_STORE_FAST_5_r10] = "_STORE_FAST_5_r10", [_STORE_FAST_6] = "_STORE_FAST_6", + [_STORE_FAST_6_r10] = "_STORE_FAST_6_r10", [_STORE_FAST_7] = "_STORE_FAST_7", - [_STORE_FAST_LOAD_FAST] = "_STORE_FAST_LOAD_FAST", - [_STORE_FAST_STORE_FAST] = "_STORE_FAST_STORE_FAST", + [_STORE_FAST_7_r10] = "_STORE_FAST_7_r10", [_STORE_GLOBAL] = "_STORE_GLOBAL", + [_STORE_GLOBAL_r00] = "_STORE_GLOBAL_r00", [_STORE_NAME] = "_STORE_NAME", + [_STORE_NAME_r00] = "_STORE_NAME_r00", [_STORE_SLICE] = "_STORE_SLICE", + [_STORE_SLICE_r00] = "_STORE_SLICE_r00", [_STORE_SUBSCR] = "_STORE_SUBSCR", + [_STORE_SUBSCR_r00] = "_STORE_SUBSCR_r00", [_STORE_SUBSCR_DICT] = "_STORE_SUBSCR_DICT", + [_STORE_SUBSCR_DICT_r00] = "_STORE_SUBSCR_DICT_r00", [_STORE_SUBSCR_LIST_INT] = "_STORE_SUBSCR_LIST_INT", + [_STORE_SUBSCR_LIST_INT_r30] = "_STORE_SUBSCR_LIST_INT_r30", [_SWAP] = "_SWAP", + [_SWAP_r11] = "_SWAP_r11", [_SWAP_2] = "_SWAP_2", + [_SWAP_2_r22] = "_SWAP_2_r22", + [_SWAP_2_r00] = "_SWAP_2_r00", + [_SWAP_2_r11] = "_SWAP_2_r11", + [_SWAP_2_r33] = "_SWAP_2_r33", [_SWAP_3] = "_SWAP_3", + [_SWAP_3_r33] = "_SWAP_3_r33", [_TIER2_RESUME_CHECK] = "_TIER2_RESUME_CHECK", + [_TIER2_RESUME_CHECK_r00] = "_TIER2_RESUME_CHECK_r00", + [_TIER2_RESUME_CHECK_r11] = "_TIER2_RESUME_CHECK_r11", + [_TIER2_RESUME_CHECK_r22] = "_TIER2_RESUME_CHECK_r22", + [_TIER2_RESUME_CHECK_r33] = "_TIER2_RESUME_CHECK_r33", [_TO_BOOL] = "_TO_BOOL", + [_TO_BOOL_r01] = "_TO_BOOL_r01", [_TO_BOOL_BOOL] = "_TO_BOOL_BOOL", + [_TO_BOOL_BOOL_r11] = "_TO_BOOL_BOOL_r11", + [_TO_BOOL_BOOL_r00] = "_TO_BOOL_BOOL_r00", + [_TO_BOOL_BOOL_r22] = "_TO_BOOL_BOOL_r22", + [_TO_BOOL_BOOL_r33] = "_TO_BOOL_BOOL_r33", [_TO_BOOL_INT] = "_TO_BOOL_INT", + [_TO_BOOL_INT_r11] = "_TO_BOOL_INT_r11", [_TO_BOOL_LIST] = "_TO_BOOL_LIST", + [_TO_BOOL_LIST_r11] = "_TO_BOOL_LIST_r11", [_TO_BOOL_NONE] = "_TO_BOOL_NONE", + [_TO_BOOL_NONE_r11] = "_TO_BOOL_NONE_r11", + [_TO_BOOL_NONE_r00] = "_TO_BOOL_NONE_r00", + [_TO_BOOL_NONE_r22] = "_TO_BOOL_NONE_r22", + [_TO_BOOL_NONE_r33] = "_TO_BOOL_NONE_r33", [_TO_BOOL_STR] = "_TO_BOOL_STR", + [_TO_BOOL_STR_r11] = "_TO_BOOL_STR_r11", [_UNARY_INVERT] = "_UNARY_INVERT", + [_UNARY_INVERT_r01] = "_UNARY_INVERT_r01", [_UNARY_NEGATIVE] = "_UNARY_NEGATIVE", + [_UNARY_NEGATIVE_r01] = "_UNARY_NEGATIVE_r01", [_UNARY_NOT] = "_UNARY_NOT", + [_UNARY_NOT_r11] = "_UNARY_NOT_r11", + [_UNARY_NOT_r00] = "_UNARY_NOT_r00", + [_UNARY_NOT_r22] = "_UNARY_NOT_r22", + [_UNARY_NOT_r33] = "_UNARY_NOT_r33", [_UNPACK_EX] = "_UNPACK_EX", + [_UNPACK_EX_r00] = "_UNPACK_EX_r00", [_UNPACK_SEQUENCE] = "_UNPACK_SEQUENCE", + [_UNPACK_SEQUENCE_r00] = "_UNPACK_SEQUENCE_r00", [_UNPACK_SEQUENCE_LIST] = "_UNPACK_SEQUENCE_LIST", + [_UNPACK_SEQUENCE_LIST_r10] = "_UNPACK_SEQUENCE_LIST_r10", [_UNPACK_SEQUENCE_TUPLE] = "_UNPACK_SEQUENCE_TUPLE", + [_UNPACK_SEQUENCE_TUPLE_r10] = "_UNPACK_SEQUENCE_TUPLE_r10", [_UNPACK_SEQUENCE_TWO_TUPLE] = "_UNPACK_SEQUENCE_TWO_TUPLE", + [_UNPACK_SEQUENCE_TWO_TUPLE_r12] = "_UNPACK_SEQUENCE_TWO_TUPLE_r12", [_WITH_EXCEPT_START] = "_WITH_EXCEPT_START", + [_WITH_EXCEPT_START_r01] = "_WITH_EXCEPT_START_r01", [_YIELD_VALUE] = "_YIELD_VALUE", + [_YIELD_VALUE_r11] = "_YIELD_VALUE_r11", }; int _PyUop_num_popped(int opcode, int oparg) { @@ -717,10 +2181,6 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _LOAD_FAST_AND_CLEAR: return 0; - case _LOAD_FAST_LOAD_FAST: - return 0; - case _LOAD_FAST_BORROW_LOAD_FAST_BORROW: - return 0; case _LOAD_CONST: return 0; case _LOAD_SMALL_INT_0: @@ -751,10 +2211,6 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _STORE_FAST: return 1; - case _STORE_FAST_LOAD_FAST: - return 1; - case _STORE_FAST_STORE_FAST: - return 2; case _POP_TOP: return 1; case _POP_TOP_NOP: @@ -1301,6 +2757,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _TIER2_RESUME_CHECK: return 0; + case _SPILL_OR_RELOAD: + return 0; default: return -1; } diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-20-16-03-59.gh-issue-135379.eDg89T.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-20-16-03-59.gh-issue-135379.eDg89T.rst new file mode 100644 index 00000000000000..7060b9b004cc25 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-20-16-03-59.gh-issue-135379.eDg89T.rst @@ -0,0 +1,6 @@ +Implement a limited form of register allocation known as "top of stack +caching" in the JIT. It works by keeping 0-3 of the top items in the stack +in registers. The code generator generates multiple versions of those uops +that do not escape and are relatively small. During JIT compilation, the +copy that produces the least memory traffic is selected, spilling or +reloading values when needed. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index d9abc4c53d1f50..2df107b8824e62 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5259,7 +5259,9 @@ dummy_func( _Py_BackoffCounter temperature = exit->temperature; if (!backoff_counter_triggers(temperature)) { exit->temperature = advance_backoff_counter(temperature); + SYNC_SP(); GOTO_TIER_ONE(target); + Py_UNREACHABLE(); } _PyExecutorObject *executor; if (target->op.code == ENTER_EXECUTOR) { @@ -5271,12 +5273,18 @@ dummy_func( int optimized = _PyOptimizer_Optimize(frame, target, &executor, chain_depth); if (optimized <= 0) { exit->temperature = restart_backoff_counter(temperature); + SYNC_SP(); GOTO_TIER_ONE(optimized < 0 ? NULL : target); + Py_UNREACHABLE(); } exit->temperature = initial_temperature_backoff_counter(); } exit->executor = executor; } + /* In future we might want to avoid spilling + * on side exits, so we might not sync the stack + * here and start the side trace with N cached registers */ + SYNC_SP(); GOTO_TIER_TWO(exit->executor); } @@ -5392,6 +5400,7 @@ dummy_func( } tier2 op(_DEOPT, (--)) { + SYNC_SP(); GOTO_TIER_ONE(_PyFrame_GetBytecode(frame) + CURRENT_TARGET()); } @@ -5414,6 +5423,9 @@ dummy_func( assert(tstate->tracing || eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version)); } + tier2 op(_SPILL_OR_RELOAD, (--)) { + } + label(pop_2_error) { stack_pointer -= 2; assert(WITHIN_STACK_BOUNDS()); diff --git a/Python/ceval.c b/Python/ceval.c index 291e753dec0ce5..6e02f50659a7ad 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -164,13 +164,26 @@ dump_item(_PyStackRef item) printf("%" PRId64, (int64_t)PyStackRef_UntagInt(item)); return; } - PyObject *obj = PyStackRef_AsPyObjectBorrow(item); - if (obj == NULL) { - printf(""); - return; + if (PyStackRef_IsValid(item)) { + PyObject *obj = PyStackRef_AsPyObjectBorrow(item); + if (obj == NULL) { + printf(""); + return; + } + // Don't call __repr__(), it might recurse into the interpreter. + printf("<%s at %p>", Py_TYPE(obj)->tp_name, (void *)obj); + } + else { + /* Already handled NULL */ + if (PyStackRef_IsError(item)) { + printf("ERROR"); + } + else { + // Wrapped item + void *ptr = PyStackRef_Unwrap(item); + printf("Wrapped(pointer %p)", ptr); + } } - // Don't call __repr__(), it might recurse into the interpreter. - printf("<%s at %p>", Py_TYPE(obj)->tp_name, (void *)obj); } static void @@ -1154,14 +1167,21 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int uint64_t trace_uop_execution_counter = 0; #endif - assert(next_uop->opcode == _START_EXECUTOR); +#ifdef Py_DEBUG + int current_cached_values = 0; +#endif + _PyStackRef _tos_cache0 = PyStackRef_NULL; + _PyStackRef _tos_cache1 = PyStackRef_NULL; + _PyStackRef _tos_cache2 = PyStackRef_NULL; + + assert(next_uop->opcode == _START_EXECUTOR_r00); tier2_dispatch: for (;;) { uopcode = next_uop->opcode; #ifdef Py_DEBUG if (frame->lltrace >= 3) { dump_stack(frame, stack_pointer); - if (next_uop->opcode == _START_EXECUTOR) { + if (next_uop->opcode == _START_EXECUTOR_r00) { printf("%4d uop: ", 0); } else { diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 187ec8fdd26584..dd1a97ff8b893e 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -363,7 +363,7 @@ do { \ jit_func jitted = _executor->jit_code; \ /* Keep the shim frame alive via the executor: */ \ Py_INCREF(_executor); \ - next_instr = jitted(frame, stack_pointer, tstate); \ + next_instr = jitted(frame, stack_pointer, tstate, PyStackRef_NULL, PyStackRef_NULL, PyStackRef_NULL); \ Py_DECREF(_executor); \ frame = tstate->current_frame; \ stack_pointer = _PyFrame_GetStackPointer(frame); \ @@ -380,7 +380,7 @@ do { \ _PyExecutorObject *_executor = (EXECUTOR); \ tstate->current_executor = (PyObject *)_executor; \ next_uop = _executor->trace; \ - assert(next_uop->opcode == _START_EXECUTOR); \ + assert(next_uop->opcode == _START_EXECUTOR_r00); \ goto enter_tier_two; \ } while (0) #endif @@ -425,3 +425,11 @@ do { \ _PyObjectArray_Free(NAME - 1, NAME##_temp); #define CONVERSION_FAILED(NAME) ((NAME) == NULL) + +#if defined(Py_DEBUG) && !defined(_Py_JIT) +#define SET_CURRENT_CACHED_VALUES(N) current_cached_values = (N) +#define CHECK_CURRENT_CACHED_VALUES(N) assert(current_cached_values == (N)) +#else +#define SET_CURRENT_CACHED_VALUES(N) ((void)0) +#define CHECK_CURRENT_CACHED_VALUES(N) ((void)0) +#endif diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index e152865e4ec9e8..305e24518ec683 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -8,11 +8,44 @@ #endif #define TIER_TWO 2 - case _NOP: { + case _NOP_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + SET_CURRENT_CACHED_VALUES(0); break; } - case _CHECK_PERIODIC: { + case _NOP_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef _stack_item_0 = _tos_cache0; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _NOP_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _NOP_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _CHECK_PERIODIC_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); QSBR_QUIESCENT_STATE(tstate); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { @@ -20,13 +53,19 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } - case _CHECK_PERIODIC_IF_NOT_YIELD_FROM: { + case _CHECK_PERIODIC_IF_NOT_YIELD_FROM_r00: { + CHECK_CURRENT_CACHED_VALUES(0); oparg = CURRENT_OPARG(); if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); @@ -36,10 +75,15 @@ int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } } } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } @@ -47,10 +91,111 @@ /* _LOAD_BYTECODE is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ - case _RESUME_CHECK: { + case _RESUME_CHECK_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + #if defined(__EMSCRIPTEN__) + if (_Py_emscripten_signal_clock == 0) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + #endif + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + assert((version & _PY_EVAL_EVENTS_MASK) == 0); + if (eval_breaker != version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + #endif + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _RESUME_CHECK_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef _stack_item_0 = _tos_cache0; + #if defined(__EMSCRIPTEN__) + if (_Py_emscripten_signal_clock == 0) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + #endif + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + assert((version & _PY_EVAL_EVENTS_MASK) == 0); + if (eval_breaker != version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + #endif + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _RESUME_CHECK_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + #if defined(__EMSCRIPTEN__) + if (_Py_emscripten_signal_clock == 0) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + #endif + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + assert((version & _PY_EVAL_EVENTS_MASK) == 0); + if (eval_breaker != version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + #endif + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _RESUME_CHECK_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; #if defined(__EMSCRIPTEN__) if (_Py_emscripten_signal_clock == 0) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; @@ -60,21 +205,28 @@ assert((version & _PY_EVAL_EVENTS_MASK) == 0); if (eval_breaker != version) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } #ifdef Py_GIL_DISABLED if (frame->tlbc_index != ((_PyThreadStateImpl *)tstate)->tlbc_index) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } #endif + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); break; } /* _MONITOR_RESUME is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ - case _LOAD_FAST_CHECK: { + case _LOAD_FAST_CHECK_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef value; oparg = CURRENT_OPARG(); _PyStackRef value_s = GETLOCAL(oparg); @@ -85,751 +237,2186 @@ PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } value = PyStackRef_DUP(value_s); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = value; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _LOAD_FAST_0: { + case _LOAD_FAST_0_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef value; oparg = 0; assert(oparg == CURRENT_OPARG()); assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_DUP(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); break; } - case _LOAD_FAST_1: { + case _LOAD_FAST_0_r12: { + CHECK_CURRENT_CACHED_VALUES(1); _PyStackRef value; - oparg = 1; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 0; assert(oparg == CURRENT_OPARG()); assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_DUP(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); break; } - case _LOAD_FAST_2: { + case _LOAD_FAST_0_r23: { + CHECK_CURRENT_CACHED_VALUES(2); _PyStackRef value; - oparg = 2; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 0; assert(oparg == CURRENT_OPARG()); assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_DUP(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); break; } - case _LOAD_FAST_3: { + case _LOAD_FAST_1_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef value; - oparg = 3; + oparg = 1; assert(oparg == CURRENT_OPARG()); assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_DUP(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); break; } - case _LOAD_FAST_4: { + case _LOAD_FAST_1_r12: { + CHECK_CURRENT_CACHED_VALUES(1); _PyStackRef value; - oparg = 4; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 1; assert(oparg == CURRENT_OPARG()); assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_DUP(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); break; } - case _LOAD_FAST_5: { + case _LOAD_FAST_1_r23: { + CHECK_CURRENT_CACHED_VALUES(2); _PyStackRef value; - oparg = 5; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 1; assert(oparg == CURRENT_OPARG()); assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_DUP(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); break; } - case _LOAD_FAST_6: { + case _LOAD_FAST_2_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef value; - oparg = 6; + oparg = 2; assert(oparg == CURRENT_OPARG()); assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_DUP(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); break; } - case _LOAD_FAST_7: { + case _LOAD_FAST_2_r12: { + CHECK_CURRENT_CACHED_VALUES(1); _PyStackRef value; - oparg = 7; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 2; assert(oparg == CURRENT_OPARG()); assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_DUP(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); break; } - case _LOAD_FAST: { + case _LOAD_FAST_2_r23: { + CHECK_CURRENT_CACHED_VALUES(2); _PyStackRef value; - oparg = CURRENT_OPARG(); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 2; + assert(oparg == CURRENT_OPARG()); assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_DUP(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); break; } - case _LOAD_FAST_BORROW_0: { + case _LOAD_FAST_3_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef value; - oparg = 0; + oparg = 3; assert(oparg == CURRENT_OPARG()); assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); break; } - case _LOAD_FAST_BORROW_1: { + case _LOAD_FAST_3_r12: { + CHECK_CURRENT_CACHED_VALUES(1); _PyStackRef value; - oparg = 1; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 3; assert(oparg == CURRENT_OPARG()); assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); break; } - case _LOAD_FAST_BORROW_2: { + case _LOAD_FAST_3_r23: { + CHECK_CURRENT_CACHED_VALUES(2); _PyStackRef value; - oparg = 2; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 3; assert(oparg == CURRENT_OPARG()); assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); break; } - case _LOAD_FAST_BORROW_3: { + case _LOAD_FAST_4_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef value; - oparg = 3; + oparg = 4; assert(oparg == CURRENT_OPARG()); assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); break; } - case _LOAD_FAST_BORROW_4: { + case _LOAD_FAST_4_r12: { + CHECK_CURRENT_CACHED_VALUES(1); _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; oparg = 4; assert(oparg == CURRENT_OPARG()); assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); break; } - case _LOAD_FAST_BORROW_5: { + case _LOAD_FAST_4_r23: { + CHECK_CURRENT_CACHED_VALUES(2); _PyStackRef value; - oparg = 5; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 4; assert(oparg == CURRENT_OPARG()); assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); break; } - case _LOAD_FAST_BORROW_6: { + case _LOAD_FAST_5_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef value; - oparg = 6; + oparg = 5; assert(oparg == CURRENT_OPARG()); assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); break; } - case _LOAD_FAST_BORROW_7: { + case _LOAD_FAST_5_r12: { + CHECK_CURRENT_CACHED_VALUES(1); _PyStackRef value; - oparg = 7; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 5; assert(oparg == CURRENT_OPARG()); assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); break; } - case _LOAD_FAST_BORROW: { + case _LOAD_FAST_5_r23: { + CHECK_CURRENT_CACHED_VALUES(2); _PyStackRef value; - oparg = CURRENT_OPARG(); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 5; + assert(oparg == CURRENT_OPARG()); assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); break; } - case _LOAD_FAST_AND_CLEAR: { + case _LOAD_FAST_6_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef value; - oparg = CURRENT_OPARG(); - value = GETLOCAL(oparg); - GETLOCAL(oparg) = PyStackRef_NULL; - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + oparg = 6; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); break; } - case _LOAD_CONST: { + case _LOAD_FAST_6_r12: { + CHECK_CURRENT_CACHED_VALUES(1); _PyStackRef value; - oparg = CURRENT_OPARG(); - PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); - value = PyStackRef_FromPyObjectBorrow(obj); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 6; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); break; } - case _LOAD_SMALL_INT_0: { + case _LOAD_FAST_6_r23: { + CHECK_CURRENT_CACHED_VALUES(2); _PyStackRef value; - oparg = 0; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 6; assert(oparg == CURRENT_OPARG()); - assert(oparg < _PY_NSMALLPOSINTS); - PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; - value = PyStackRef_FromPyObjectBorrow(obj); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); break; } - case _LOAD_SMALL_INT_1: { + case _LOAD_FAST_7_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef value; - oparg = 1; + oparg = 7; assert(oparg == CURRENT_OPARG()); - assert(oparg < _PY_NSMALLPOSINTS); - PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; - value = PyStackRef_FromPyObjectBorrow(obj); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); break; } - case _LOAD_SMALL_INT_2: { + case _LOAD_FAST_7_r12: { + CHECK_CURRENT_CACHED_VALUES(1); _PyStackRef value; - oparg = 2; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 7; assert(oparg == CURRENT_OPARG()); - assert(oparg < _PY_NSMALLPOSINTS); - PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; - value = PyStackRef_FromPyObjectBorrow(obj); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); break; } - case _LOAD_SMALL_INT_3: { + case _LOAD_FAST_7_r23: { + CHECK_CURRENT_CACHED_VALUES(2); _PyStackRef value; - oparg = 3; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 7; assert(oparg == CURRENT_OPARG()); - assert(oparg < _PY_NSMALLPOSINTS); - PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; - value = PyStackRef_FromPyObjectBorrow(obj); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); break; } - case _LOAD_SMALL_INT: { + case _LOAD_FAST_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef value; oparg = CURRENT_OPARG(); - assert(oparg < _PY_NSMALLPOSINTS); - PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; - value = PyStackRef_FromPyObjectBorrow(obj); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _LOAD_FAST_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _LOAD_FAST_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); break; } - case _STORE_FAST_0: { + case _LOAD_FAST_BORROW_0_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef value; oparg = 0; assert(oparg == CURRENT_OPARG()); - value = stack_pointer[-1]; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _LOAD_FAST_BORROW_0_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 0; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _LOAD_FAST_BORROW_0_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 0; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); break; } - case _STORE_FAST_1: { + case _LOAD_FAST_BORROW_1_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef value; oparg = 1; assert(oparg == CURRENT_OPARG()); - value = stack_pointer[-1]; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); break; } - case _STORE_FAST_2: { + case _LOAD_FAST_BORROW_1_r12: { + CHECK_CURRENT_CACHED_VALUES(1); _PyStackRef value; - oparg = 2; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 1; assert(oparg == CURRENT_OPARG()); - value = stack_pointer[-1]; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); break; } - case _STORE_FAST_3: { + case _LOAD_FAST_BORROW_1_r23: { + CHECK_CURRENT_CACHED_VALUES(2); _PyStackRef value; - oparg = 3; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 1; assert(oparg == CURRENT_OPARG()); - value = stack_pointer[-1]; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); break; } - case _STORE_FAST_4: { + case _LOAD_FAST_BORROW_2_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef value; - oparg = 4; + oparg = 2; assert(oparg == CURRENT_OPARG()); - value = stack_pointer[-1]; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); break; } - case _STORE_FAST_5: { + case _LOAD_FAST_BORROW_2_r12: { + CHECK_CURRENT_CACHED_VALUES(1); _PyStackRef value; - oparg = 5; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 2; assert(oparg == CURRENT_OPARG()); - value = stack_pointer[-1]; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); break; } - case _STORE_FAST_6: { + case _LOAD_FAST_BORROW_2_r23: { + CHECK_CURRENT_CACHED_VALUES(2); _PyStackRef value; - oparg = 6; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 2; assert(oparg == CURRENT_OPARG()); - value = stack_pointer[-1]; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); break; } - case _STORE_FAST_7: { + case _LOAD_FAST_BORROW_3_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef value; - oparg = 7; + oparg = 3; assert(oparg == CURRENT_OPARG()); - value = stack_pointer[-1]; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); break; } - case _STORE_FAST: { + case _LOAD_FAST_BORROW_3_r12: { + CHECK_CURRENT_CACHED_VALUES(1); _PyStackRef value; - oparg = CURRENT_OPARG(); - value = stack_pointer[-1]; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 3; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); break; } - case _POP_TOP: { + case _LOAD_FAST_BORROW_3_r23: { + CHECK_CURRENT_CACHED_VALUES(2); _PyStackRef value; - value = stack_pointer[-1]; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 3; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); break; } - case _POP_TOP_NOP: { + case _LOAD_FAST_BORROW_4_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef value; - value = stack_pointer[-1]; - assert(PyStackRef_IsNull(value) || (!PyStackRef_RefcountOnObject(value)) || - _Py_IsImmortal((PyStackRef_AsPyObjectBorrow(value)))); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + oparg = 4; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); break; } - case _POP_TOP_INT: { + case _LOAD_FAST_BORROW_4_r12: { + CHECK_CURRENT_CACHED_VALUES(1); _PyStackRef value; - value = stack_pointer[-1]; - assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); - PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 4; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); break; } - case _POP_TOP_FLOAT: { + case _LOAD_FAST_BORROW_4_r23: { + CHECK_CURRENT_CACHED_VALUES(2); _PyStackRef value; - value = stack_pointer[-1]; - assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); - PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 4; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); break; } - case _POP_TOP_UNICODE: { + case _LOAD_FAST_BORROW_5_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef value; - value = stack_pointer[-1]; - assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(value))); - PyStackRef_CLOSE_SPECIALIZED(value, _PyUnicode_ExactDealloc); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + oparg = 5; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); break; } - case _POP_TWO: { - _PyStackRef tos; - _PyStackRef nos; - tos = stack_pointer[-1]; - nos = stack_pointer[-2]; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(tos); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(nos); - stack_pointer = _PyFrame_GetStackPointer(frame); + case _LOAD_FAST_BORROW_5_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 5; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); break; } - case _PUSH_NULL: { - _PyStackRef res; - res = PyStackRef_NULL; - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + case _LOAD_FAST_BORROW_5_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 5; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); break; } - case _END_FOR: { + case _LOAD_FAST_BORROW_6_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef value; - value = stack_pointer[-1]; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + oparg = 6; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); break; } - case _POP_ITER: { - _PyStackRef index_or_null; - _PyStackRef iter; - index_or_null = stack_pointer[-1]; - iter = stack_pointer[-2]; - (void)index_or_null; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iter); - stack_pointer = _PyFrame_GetStackPointer(frame); + case _LOAD_FAST_BORROW_6_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 6; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); break; } - case _END_SEND: { + case _LOAD_FAST_BORROW_6_r23: { + CHECK_CURRENT_CACHED_VALUES(2); _PyStackRef value; - _PyStackRef receiver; - _PyStackRef val; - value = stack_pointer[-1]; - receiver = stack_pointer[-2]; - val = value; - stack_pointer[-2] = val; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(receiver); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 6; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); break; } - case _UNARY_NEGATIVE: { + case _LOAD_FAST_BORROW_7_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; + oparg = 7; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _LOAD_FAST_BORROW_7_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 7; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _LOAD_FAST_BORROW_7_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 7; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _LOAD_FAST_BORROW_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + _PyStackRef value; + oparg = CURRENT_OPARG(); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _LOAD_FAST_BORROW_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _LOAD_FAST_BORROW_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _LOAD_FAST_AND_CLEAR_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + _PyStackRef value; + oparg = CURRENT_OPARG(); + value = GETLOCAL(oparg); + GETLOCAL(oparg) = PyStackRef_NULL; + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _LOAD_FAST_AND_CLEAR_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + value = GETLOCAL(oparg); + GETLOCAL(oparg) = PyStackRef_NULL; + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _LOAD_FAST_AND_CLEAR_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + value = GETLOCAL(oparg); + GETLOCAL(oparg) = PyStackRef_NULL; + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _LOAD_CONST_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + _PyStackRef value; + oparg = CURRENT_OPARG(); + PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _LOAD_CONST_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _LOAD_CONST_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _LOAD_SMALL_INT_0_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + _PyStackRef value; + oparg = 0; + assert(oparg == CURRENT_OPARG()); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _LOAD_SMALL_INT_0_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 0; + assert(oparg == CURRENT_OPARG()); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _LOAD_SMALL_INT_0_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 0; + assert(oparg == CURRENT_OPARG()); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _LOAD_SMALL_INT_1_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + _PyStackRef value; + oparg = 1; + assert(oparg == CURRENT_OPARG()); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _LOAD_SMALL_INT_1_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 1; + assert(oparg == CURRENT_OPARG()); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _LOAD_SMALL_INT_1_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 1; + assert(oparg == CURRENT_OPARG()); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _LOAD_SMALL_INT_2_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + _PyStackRef value; + oparg = 2; + assert(oparg == CURRENT_OPARG()); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _LOAD_SMALL_INT_2_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 2; + assert(oparg == CURRENT_OPARG()); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _LOAD_SMALL_INT_2_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 2; + assert(oparg == CURRENT_OPARG()); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _LOAD_SMALL_INT_3_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + _PyStackRef value; + oparg = 3; + assert(oparg == CURRENT_OPARG()); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _LOAD_SMALL_INT_3_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 3; + assert(oparg == CURRENT_OPARG()); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _LOAD_SMALL_INT_3_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 3; + assert(oparg == CURRENT_OPARG()); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _LOAD_SMALL_INT_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + _PyStackRef value; + oparg = CURRENT_OPARG(); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _LOAD_SMALL_INT_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _LOAD_SMALL_INT_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _STORE_FAST_0_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 0; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_0; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _STORE_FAST_1_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 1; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_0; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _STORE_FAST_2_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 2; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_0; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _STORE_FAST_3_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 3; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_0; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _STORE_FAST_4_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 4; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_0; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _STORE_FAST_5_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 5; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_0; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _STORE_FAST_6_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 6; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_0; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _STORE_FAST_7_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 7; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_0; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _STORE_FAST_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + value = _stack_item_0; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _POP_TOP_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _POP_TOP_NOP_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + assert(PyStackRef_IsNull(value) || (!PyStackRef_RefcountOnObject(value)) || + _Py_IsImmortal((PyStackRef_AsPyObjectBorrow(value)))); + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _POP_TOP_NOP_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + value = _stack_item_1; + assert(PyStackRef_IsNull(value) || (!PyStackRef_RefcountOnObject(value)) || + _Py_IsImmortal((PyStackRef_AsPyObjectBorrow(value)))); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _POP_TOP_NOP_r32: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + value = _stack_item_2; + assert(PyStackRef_IsNull(value) || (!PyStackRef_RefcountOnObject(value)) || + _Py_IsImmortal((PyStackRef_AsPyObjectBorrow(value)))); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _POP_TOP_INT_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _POP_TOP_INT_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + value = _stack_item_1; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _POP_TOP_INT_r32: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + value = _stack_item_2; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _POP_TOP_FLOAT_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _POP_TOP_FLOAT_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + value = _stack_item_1; + assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _POP_TOP_FLOAT_r32: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + value = _stack_item_2; + assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _POP_TOP_UNICODE_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyUnicode_ExactDealloc); + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _POP_TOP_UNICODE_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + value = _stack_item_1; + assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyUnicode_ExactDealloc); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _POP_TOP_UNICODE_r32: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + value = _stack_item_2; + assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyUnicode_ExactDealloc); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _POP_TWO_r20: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef tos; + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + tos = _stack_item_1; + nos = _stack_item_0; + stack_pointer[0] = nos; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(tos); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(nos); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _PUSH_NULL_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + _PyStackRef res; + res = PyStackRef_NULL; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _PUSH_NULL_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + res = PyStackRef_NULL; + _tos_cache1 = res; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _PUSH_NULL_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + res = PyStackRef_NULL; + _tos_cache2 = res; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _END_FOR_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _POP_ITER_r20: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef index_or_null; + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + index_or_null = _stack_item_1; + iter = _stack_item_0; + (void)index_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iter); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _END_SEND_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef value; + _PyStackRef receiver; + _PyStackRef val; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + value = _stack_item_1; + receiver = _stack_item_0; + val = value; + stack_pointer[0] = val; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(receiver); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = val; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _UNARY_NEGATIVE_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(res_o); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _UNARY_NOT_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef value; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + assert(PyStackRef_BoolCheck(value)); + res = PyStackRef_IsFalse(value) + ? PyStackRef_True : PyStackRef_False; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _UNARY_NOT_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(value)); + res = PyStackRef_IsFalse(value) + ? PyStackRef_True : PyStackRef_False; + stack_pointer[-1] = res; + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _UNARY_NOT_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef value; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + value = _stack_item_1; + assert(PyStackRef_BoolCheck(value)); + res = PyStackRef_IsFalse(value) + ? PyStackRef_True : PyStackRef_False; + _tos_cache1 = res; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _UNARY_NOT_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef value; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + value = _stack_item_2; + assert(PyStackRef_BoolCheck(value)); + res = PyStackRef_IsFalse(value) + ? PyStackRef_True : PyStackRef_False; + _tos_cache2 = res; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _TO_BOOL_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = err ? PyStackRef_True : PyStackRef_False; + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _TO_BOOL_BOOL_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + if (!PyStackRef_BoolCheck(value)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(TO_BOOL, hit); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _TO_BOOL_BOOL_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + _PyStackRef value; + value = stack_pointer[-1]; + if (!PyStackRef_BoolCheck(value)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(TO_BOOL, hit); + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _TO_BOOL_BOOL_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + value = _stack_item_1; + if (!PyStackRef_BoolCheck(value)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(TO_BOOL, hit); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _TO_BOOL_BOOL_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + value = _stack_item_2; + if (!PyStackRef_BoolCheck(value)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(TO_BOOL, hit); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _TO_BOOL_INT_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef value; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyLong_CheckExact(value_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(TO_BOOL, hit); + if (_PyLong_IsZero((PyLongObject *)value_o)) { + assert(_Py_IsImmortal(value_o)); + res = PyStackRef_False; + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_True; + } + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _GUARD_NOS_LIST_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + nos = _stack_item_0; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyList_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _GUARD_NOS_LIST_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + _PyStackRef nos; + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyList_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _GUARD_NOS_LIST_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + nos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyList_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _GUARD_NOS_LIST_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + nos = _stack_item_1; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyList_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = nos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _GUARD_TOS_LIST_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + tos = _stack_item_0; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyList_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = tos; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _GUARD_TOS_LIST_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + _PyStackRef tos; + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyList_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _GUARD_TOS_LIST_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + tos = _stack_item_1; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyList_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = tos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _GUARD_TOS_LIST_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + tos = _stack_item_2; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyList_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = tos; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _GUARD_TOS_SLICE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + tos = _stack_item_0; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PySlice_Check(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = tos; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _GUARD_TOS_SLICE_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + _PyStackRef tos; + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PySlice_Check(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _GUARD_TOS_SLICE_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + tos = _stack_item_1; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PySlice_Check(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = tos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _GUARD_TOS_SLICE_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + tos = _stack_item_2; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PySlice_Check(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = tos; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _TO_BOOL_LIST_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef value; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + assert(PyList_CheckExact(value_o)); + STAT_INC(TO_BOOL, hit); + res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; + stack_pointer[0] = value; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); + _PyStackRef tmp = value; + value = res; + stack_pointer[-1] = value; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - JUMP_TO_ERROR(); - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; + _tos_cache0 = res; + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _UNARY_NOT: { + case _TO_BOOL_NONE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); _PyStackRef value; _PyStackRef res; - value = stack_pointer[-1]; - assert(PyStackRef_BoolCheck(value)); - res = PyStackRef_IsFalse(value) - ? PyStackRef_True : PyStackRef_False; - stack_pointer[-1] = res; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + if (!PyStackRef_IsNone(value)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(TO_BOOL, hit); + res = PyStackRef_False; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(1); break; } - case _TO_BOOL: { + case _TO_BOOL_NONE_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef value; _PyStackRef res; value = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - JUMP_TO_ERROR(); + if (!PyStackRef_IsNone(value)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); } - res = err ? PyStackRef_True : PyStackRef_False; - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + STAT_INC(TO_BOOL, hit); + res = PyStackRef_False; + stack_pointer[-1] = res; + SET_CURRENT_CACHED_VALUES(0); break; } - case _TO_BOOL_BOOL: { + case _TO_BOOL_NONE_r22: { + CHECK_CURRENT_CACHED_VALUES(2); _PyStackRef value; - value = stack_pointer[-1]; - if (!PyStackRef_BoolCheck(value)) { + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + value = _stack_item_1; + if (!PyStackRef_IsNone(value)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } STAT_INC(TO_BOOL, hit); + res = PyStackRef_False; + _tos_cache1 = res; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); break; } - case _TO_BOOL_INT: { + case _TO_BOOL_NONE_r33: { + CHECK_CURRENT_CACHED_VALUES(3); _PyStackRef value; _PyStackRef res; - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyLong_CheckExact(value_o)) { + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + value = _stack_item_2; + if (!PyStackRef_IsNone(value)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } STAT_INC(TO_BOOL, hit); - if (_PyLong_IsZero((PyLongObject *)value_o)) { - assert(_Py_IsImmortal(value_o)); - res = PyStackRef_False; - } - else { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_True; - stack_pointer += 1; + res = PyStackRef_False; + _tos_cache2 = res; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _GUARD_NOS_UNICODE_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + nos = _stack_item_0; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyUnicode_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); } - stack_pointer[-1] = res; + _tos_cache1 = _stack_item_1; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); break; } - case _GUARD_NOS_LIST: { + case _GUARD_NOS_UNICODE_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef nos; nos = stack_pointer[-2]; PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyList_CheckExact(o)) { + if (!PyUnicode_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } + SET_CURRENT_CACHED_VALUES(0); break; } - case _GUARD_TOS_LIST: { - _PyStackRef tos; - tos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyList_CheckExact(o)) { + case _GUARD_NOS_UNICODE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + nos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyUnicode_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); break; } - case _GUARD_TOS_SLICE: { - _PyStackRef tos; - tos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PySlice_Check(o)) { + case _GUARD_NOS_UNICODE_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + nos = _stack_item_1; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyUnicode_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } + _tos_cache2 = _stack_item_2; + _tos_cache1 = nos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); break; } - case _TO_BOOL_LIST: { + case _GUARD_TOS_UNICODE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - assert(PyList_CheckExact(value_o)); - STAT_INC(TO_BOOL, hit); - res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = value; - value = res; - stack_pointer[-1] = value; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + if (!PyUnicode_CheckExact(value_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); break; } - case _TO_BOOL_NONE: { + case _GUARD_TOS_UNICODE_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef value; - _PyStackRef res; value = stack_pointer[-1]; - if (!PyStackRef_IsNone(value)) { + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyUnicode_CheckExact(value_o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - STAT_INC(TO_BOOL, hit); - res = PyStackRef_False; - stack_pointer[-1] = res; + SET_CURRENT_CACHED_VALUES(0); break; } - case _GUARD_NOS_UNICODE: { - _PyStackRef nos; - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyUnicode_CheckExact(o)) { + case _GUARD_TOS_UNICODE_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + value = _stack_item_1; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyUnicode_CheckExact(value_o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); break; } - case _GUARD_TOS_UNICODE: { + case _GUARD_TOS_UNICODE_r33: { + CHECK_CURRENT_CACHED_VALUES(3); _PyStackRef value; - value = stack_pointer[-1]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + value = _stack_item_2; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); if (!PyUnicode_CheckExact(value_o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); break; } - case _TO_BOOL_STR: { + case _TO_BOOL_STR_r11: { + CHECK_CURRENT_CACHED_VALUES(1); _PyStackRef value; _PyStackRef res; - value = stack_pointer[-1]; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; STAT_INC(TO_BOOL, hit); PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); if (value_o == &_Py_STR(empty)) { @@ -838,35 +2425,37 @@ } else { assert(Py_SIZE(value_o)); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_True; - stack_pointer += 1; } - stack_pointer[-1] = res; + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _REPLACE_WITH_TRUE: { + case _REPLACE_WITH_TRUE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); _PyStackRef value; _PyStackRef res; - value = stack_pointer[-1]; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_True; - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _UNARY_INVERT: { + case _UNARY_INVERT_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef value; _PyStackRef res; value = stack_pointer[-1]; @@ -879,67 +2468,306 @@ PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _GUARD_NOS_INT_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!_PyLong_CheckExactAndCompact(left_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); break; } - case _GUARD_NOS_INT: { + case _GUARD_NOS_INT_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef left; left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); if (!_PyLong_CheckExactAndCompact(left_o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _GUARD_NOS_INT_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + left = stack_pointer[-1]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!_PyLong_CheckExactAndCompact(left_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _GUARD_NOS_INT_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + left = _stack_item_1; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!_PyLong_CheckExactAndCompact(left_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = left; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _GUARD_TOS_INT_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); break; } - case _GUARD_TOS_INT: { + case _GUARD_TOS_INT_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef value; value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); if (!_PyLong_CheckExactAndCompact(value_o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _GUARD_TOS_INT_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + value = _stack_item_1; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _GUARD_TOS_INT_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + value = _stack_item_2; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _GUARD_NOS_OVERFLOWED_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + assert(Py_TYPE(left_o) == &PyLong_Type); + if (!_PyLong_IsCompact((PyLongObject *)left_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } + _tos_cache1 = _stack_item_1; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); break; } - case _GUARD_NOS_OVERFLOWED: { + case _GUARD_NOS_OVERFLOWED_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef left; left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); assert(Py_TYPE(left_o) == &PyLong_Type); if (!_PyLong_IsCompact((PyLongObject *)left_o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _GUARD_NOS_OVERFLOWED_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + left = stack_pointer[-1]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + assert(Py_TYPE(left_o) == &PyLong_Type); + if (!_PyLong_IsCompact((PyLongObject *)left_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _GUARD_NOS_OVERFLOWED_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + left = _stack_item_1; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + assert(Py_TYPE(left_o) == &PyLong_Type); + if (!_PyLong_IsCompact((PyLongObject *)left_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = left; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _GUARD_TOS_OVERFLOWED_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + assert(Py_TYPE(value_o) == &PyLong_Type); + if (!_PyLong_IsCompact((PyLongObject *)value_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); break; } - case _GUARD_TOS_OVERFLOWED: { + case _GUARD_TOS_OVERFLOWED_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef value; value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); assert(Py_TYPE(value_o) == &PyLong_Type); if (!_PyLong_IsCompact((PyLongObject *)value_o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _GUARD_TOS_OVERFLOWED_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + value = _stack_item_1; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + assert(Py_TYPE(value_o) == &PyLong_Type); + if (!_PyLong_IsCompact((PyLongObject *)value_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _GUARD_TOS_OVERFLOWED_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + value = _stack_item_2; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + assert(Py_TYPE(value_o) == &PyLong_Type); + if (!_PyLong_IsCompact((PyLongObject *)value_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); break; } - case _BINARY_OP_MULTIPLY_INT: { + case _BINARY_OP_MULTIPLY_INT_r21: { + CHECK_CURRENT_CACHED_VALUES(2); _PyStackRef right; _PyStackRef left; - _PyStackRef res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); @@ -949,22 +2777,27 @@ res = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); if (PyStackRef_IsNull(res)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _BINARY_OP_ADD_INT: { + case _BINARY_OP_ADD_INT_r21: { + CHECK_CURRENT_CACHED_VALUES(2); _PyStackRef right; _PyStackRef left; _PyStackRef res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); @@ -974,22 +2807,27 @@ res = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); if (PyStackRef_IsNull(res)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _BINARY_OP_SUBTRACT_INT: { + case _BINARY_OP_SUBTRACT_INT_r21: { + CHECK_CURRENT_CACHED_VALUES(2); _PyStackRef right; _PyStackRef left; _PyStackRef res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); @@ -999,44 +2837,163 @@ res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); if (PyStackRef_IsNull(res)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _GUARD_NOS_FLOAT_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyFloat_CheckExact(left_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); break; } - case _GUARD_NOS_FLOAT: { + case _GUARD_NOS_FLOAT_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef left; left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); if (!PyFloat_CheckExact(left_o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _GUARD_NOS_FLOAT_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + left = stack_pointer[-1]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyFloat_CheckExact(left_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _GUARD_NOS_FLOAT_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + left = _stack_item_1; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyFloat_CheckExact(left_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = left; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _GUARD_TOS_FLOAT_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(value_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); break; } - case _GUARD_TOS_FLOAT: { + case _GUARD_TOS_FLOAT_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef value; value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); if (!PyFloat_CheckExact(value_o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _GUARD_TOS_FLOAT_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + value = _stack_item_1; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(value_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _GUARD_TOS_FLOAT_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + value = _stack_item_2; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(value_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); break; } - case _BINARY_OP_MULTIPLY_FLOAT: { + case _BINARY_OP_MULTIPLY_FLOAT_r21: { + CHECK_CURRENT_CACHED_VALUES(2); _PyStackRef right; _PyStackRef left; _PyStackRef res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyFloat_CheckExact(left_o)); @@ -1047,23 +3004,25 @@ ((PyFloatObject *)right_o)->ob_fval; res = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); if (PyStackRef_IsNull(res)) { - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _BINARY_OP_ADD_FLOAT: { + case _BINARY_OP_ADD_FLOAT_r21: { + CHECK_CURRENT_CACHED_VALUES(2); _PyStackRef right; _PyStackRef left; _PyStackRef res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyFloat_CheckExact(left_o)); @@ -1074,23 +3033,25 @@ ((PyFloatObject *)right_o)->ob_fval; res = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); if (PyStackRef_IsNull(res)) { - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _BINARY_OP_SUBTRACT_FLOAT: { + case _BINARY_OP_SUBTRACT_FLOAT_r21: { + CHECK_CURRENT_CACHED_VALUES(2); _PyStackRef right; _PyStackRef left; _PyStackRef res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyFloat_CheckExact(left_o)); @@ -1101,23 +3062,25 @@ ((PyFloatObject *)right_o)->ob_fval; res = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); if (PyStackRef_IsNull(res)) { - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS: { + case _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS_r21: { + CHECK_CURRENT_CACHED_VALUES(2); _PyStackRef right; _PyStackRef left; _PyStackRef res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyFloat_CheckExact(left_o)); @@ -1128,23 +3091,25 @@ ((PyFloatObject *)right_o)->ob_fval; res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); if (PyStackRef_IsNull(res)) { - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS: { + case _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS_r21: { + CHECK_CURRENT_CACHED_VALUES(2); _PyStackRef right; _PyStackRef left; _PyStackRef res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyFloat_CheckExact(left_o)); @@ -1155,23 +3120,25 @@ ((PyFloatObject *)right_o)->ob_fval; res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); if (PyStackRef_IsNull(res)) { - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS: { + case _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS_r21: { + CHECK_CURRENT_CACHED_VALUES(2); _PyStackRef right; _PyStackRef left; _PyStackRef res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyFloat_CheckExact(left_o)); @@ -1182,23 +3149,25 @@ ((PyFloatObject *)right_o)->ob_fval; res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); if (PyStackRef_IsNull(res)) { - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _BINARY_OP_ADD_UNICODE: { + case _BINARY_OP_ADD_UNICODE_r21: { + CHECK_CURRENT_CACHED_VALUES(2); _PyStackRef right; _PyStackRef left; _PyStackRef res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyUnicode_CheckExact(left_o)); @@ -1208,18 +3177,19 @@ PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); if (res_o == NULL) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _BINARY_OP_INPLACE_ADD_UNICODE: { + case _BINARY_OP_INPLACE_ADD_UNICODE_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef right; _PyStackRef left; right = stack_pointer[-1]; @@ -1238,6 +3208,7 @@ assert(PyUnicode_CheckExact(left_o)); if (PyStackRef_AsPyObjectBorrow(*target_local) != left_o) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } STAT_INC(BINARY_OP, hit); @@ -1255,6 +3226,7 @@ Py_DECREF(right_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (PyStackRef_IsNull(*target_local)) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } #if TIER_ONE @@ -1262,10 +3234,15 @@ assert(next_instr->op.code == STORE_FAST); SKIP_OVER(1); #endif + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } - case _GUARD_BINARY_OP_EXTEND: { + case _GUARD_BINARY_OP_EXTEND_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef right; _PyStackRef left; right = stack_pointer[-1]; @@ -1281,12 +3258,18 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (!res) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } - case _BINARY_OP_EXTEND: { + case _BINARY_OP_EXTEND_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -1312,13 +3295,15 @@ stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _BINARY_SLICE: { + case _BINARY_SLICE_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef stop; _PyStackRef start; _PyStackRef container; @@ -1349,16 +3334,19 @@ PyStackRef_CLOSE(container); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _STORE_SLICE: { + case _STORE_SLICE_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef stop; _PyStackRef start; _PyStackRef container; @@ -1397,12 +3385,18 @@ stack_pointer += -4; assert(WITHIN_STACK_BOUNDS()); if (err) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } - case _BINARY_OP_SUBSCR_LIST_INT: { + case _BINARY_OP_SUBSCR_LIST_INT_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef sub_st; _PyStackRef list_st; _PyStackRef res; @@ -1414,6 +3408,7 @@ assert(PyList_CheckExact(list)); if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; @@ -1423,6 +3418,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } STAT_INC(BINARY_OP, hit); @@ -1430,6 +3426,7 @@ #else if (index >= PyList_GET_SIZE(list)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } STAT_INC(BINARY_OP, hit); @@ -1450,10 +3447,17 @@ stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _BINARY_OP_SUBSCR_LIST_SLICE: { + case _BINARY_OP_SUBSCR_LIST_SLICE_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef sub_st; _PyStackRef list_st; _PyStackRef res; @@ -1480,93 +3484,219 @@ stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _BINARY_OP_SUBSCR_STR_INT: { + case _BINARY_OP_SUBSCR_STR_INT_r21: { + CHECK_CURRENT_CACHED_VALUES(2); _PyStackRef sub_st; _PyStackRef str_st; _PyStackRef res; - sub_st = stack_pointer[-1]; - str_st = stack_pointer[-2]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + sub_st = _stack_item_1; + str_st = _stack_item_0; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); assert(PyLong_CheckExact(sub)); assert(PyUnicode_CheckExact(str)); if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; if (PyUnicode_GET_LENGTH(str) <= index) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } Py_UCS4 c = PyUnicode_READ_CHAR(str, index); if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } STAT_INC(BINARY_OP, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(str_st); stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_FromPyObjectBorrow(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _GUARD_NOS_TUPLE_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + nos = _stack_item_0; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyTuple_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); break; } - case _GUARD_NOS_TUPLE: { + case _GUARD_NOS_TUPLE_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef nos; nos = stack_pointer[-2]; PyObject *o = PyStackRef_AsPyObjectBorrow(nos); if (!PyTuple_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _GUARD_NOS_TUPLE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + nos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyTuple_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _GUARD_NOS_TUPLE_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + nos = _stack_item_1; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyTuple_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = nos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _GUARD_TOS_TUPLE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + tos = _stack_item_0; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyTuple_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } + _tos_cache0 = tos; + SET_CURRENT_CACHED_VALUES(1); break; } - case _GUARD_TOS_TUPLE: { + case _GUARD_TOS_TUPLE_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef tos; tos = stack_pointer[-1]; PyObject *o = PyStackRef_AsPyObjectBorrow(tos); if (!PyTuple_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } + SET_CURRENT_CACHED_VALUES(0); break; } - case _BINARY_OP_SUBSCR_TUPLE_INT: { + case _GUARD_TOS_TUPLE_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + tos = _stack_item_1; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyTuple_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = tos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _GUARD_TOS_TUPLE_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + tos = _stack_item_2; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyTuple_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = tos; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _BINARY_OP_SUBSCR_TUPLE_INT_r21: { + CHECK_CURRENT_CACHED_VALUES(2); _PyStackRef sub_st; _PyStackRef tuple_st; _PyStackRef res; - sub_st = stack_pointer[-1]; - tuple_st = stack_pointer[-2]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + sub_st = _stack_item_1; + tuple_st = _stack_item_0; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); assert(PyLong_CheckExact(sub)); assert(PyTuple_CheckExact(tuple)); if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; if (index >= PyTuple_GET_SIZE(tuple)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } STAT_INC(BINARY_OP, hit); @@ -1574,7 +3704,8 @@ assert(res_o != NULL); PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); res = PyStackRef_FromPyObjectNew(res_o); - stack_pointer += -1; + stack_pointer[0] = tuple_st; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef tmp = tuple_st; @@ -1582,32 +3713,153 @@ stack_pointer[-1] = tuple_st; PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _GUARD_NOS_DICT_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + nos = _stack_item_0; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyDict_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); break; } - case _GUARD_NOS_DICT: { + case _GUARD_NOS_DICT_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef nos; nos = stack_pointer[-2]; PyObject *o = PyStackRef_AsPyObjectBorrow(nos); if (!PyDict_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } + SET_CURRENT_CACHED_VALUES(0); break; } - case _GUARD_TOS_DICT: { + case _GUARD_NOS_DICT_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + nos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyDict_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _GUARD_NOS_DICT_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + nos = _stack_item_1; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyDict_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = nos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _GUARD_TOS_DICT_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + tos = _stack_item_0; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyDict_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = tos; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _GUARD_TOS_DICT_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef tos; tos = stack_pointer[-1]; PyObject *o = PyStackRef_AsPyObjectBorrow(tos); if (!PyDict_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _GUARD_TOS_DICT_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + tos = _stack_item_1; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyDict_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = tos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _GUARD_TOS_DICT_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + tos = _stack_item_2; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyDict_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } + _tos_cache2 = tos; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); break; } - case _BINARY_OP_SUBSCR_DICT: { + case _BINARY_OP_SUBSCR_DICT_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef sub_st; _PyStackRef dict_st; _PyStackRef res; @@ -1639,88 +3891,102 @@ stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); if (rc <= 0) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _BINARY_OP_SUBSCR_CHECK_FUNC: { + case _BINARY_OP_SUBSCR_CHECK_FUNC_r23: { + CHECK_CURRENT_CACHED_VALUES(2); _PyStackRef container; _PyStackRef getitem; - container = stack_pointer[-2]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + container = _stack_item_0; PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); if (!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); if (getitem_o == NULL) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } assert(PyFunction_Check(getitem_o)); uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); if (((PyFunctionObject *)getitem_o)->func_version != cached_version) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); assert(code->co_argcount == 2); if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } getitem = PyStackRef_FromPyObjectNew(getitem_o); STAT_INC(BINARY_OP, hit); - stack_pointer[0] = getitem; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache2 = getitem; + _tos_cache1 = _stack_item_1; + _tos_cache0 = container; + SET_CURRENT_CACHED_VALUES(3); break; } - case _BINARY_OP_SUBSCR_INIT_CALL: { + case _BINARY_OP_SUBSCR_INIT_CALL_r31: { + CHECK_CURRENT_CACHED_VALUES(3); _PyStackRef getitem; _PyStackRef sub; _PyStackRef container; _PyStackRef new_frame; - getitem = stack_pointer[-1]; - sub = stack_pointer[-2]; - container = stack_pointer[-3]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + getitem = _stack_item_2; + sub = _stack_item_1; + container = _stack_item_0; _PyInterpreterFrame* pushed_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); pushed_frame->localsplus[0] = container; pushed_frame->localsplus[1] = sub; frame->return_offset = 6 ; new_frame = PyStackRef_Wrap(pushed_frame); - stack_pointer[-3] = new_frame; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = new_frame; + SET_CURRENT_CACHED_VALUES(1); break; } - case _LIST_APPEND: { + case _LIST_APPEND_r10: { + CHECK_CURRENT_CACHED_VALUES(1); _PyStackRef v; _PyStackRef list; + _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); - v = stack_pointer[-1]; - list = stack_pointer[-2 - (oparg-1)]; + v = _stack_item_0; + list = stack_pointer[-1 - (oparg-1)]; int err = _PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), PyStackRef_AsPyObjectSteal(v)); if (err < 0) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); break; } - case _SET_ADD: { + case _SET_ADD_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef v; _PyStackRef set; oparg = CURRENT_OPARG(); @@ -1733,14 +3999,20 @@ if (err) { stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } - case _STORE_SUBSCR: { + case _STORE_SUBSCR_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef sub; _PyStackRef container; _PyStackRef v; @@ -1765,35 +4037,47 @@ stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); if (err) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } - case _STORE_SUBSCR_LIST_INT: { + case _STORE_SUBSCR_LIST_INT_r30: { + CHECK_CURRENT_CACHED_VALUES(3); _PyStackRef sub_st; _PyStackRef list_st; _PyStackRef value; - sub_st = stack_pointer[-1]; - list_st = stack_pointer[-2]; - value = stack_pointer[-3]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + sub_st = _stack_item_2; + list_st = _stack_item_1; + value = _stack_item_0; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); assert(PyLong_CheckExact(sub)); assert(PyList_CheckExact(list)); if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; if (!LOCK_OBJECT(list)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } if (index >= PyList_GET_SIZE(list)) { UNLOCK_OBJECT(list); if (true) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } } @@ -1804,16 +4088,19 @@ assert(old_value != NULL); UNLOCK_OBJECT(list); PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(list_st); Py_DECREF(old_value); stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } - case _STORE_SUBSCR_DICT: { + case _STORE_SUBSCR_DICT_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef sub; _PyStackRef dict_st; _PyStackRef value; @@ -1834,12 +4121,18 @@ PyStackRef_CLOSE(dict_st); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } - case _DELETE_SUBSCR: { + case _DELETE_SUBSCR_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef sub; _PyStackRef container; sub = stack_pointer[-1]; @@ -1859,12 +4152,18 @@ stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); if (err) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } - case _CALL_INTRINSIC_1: { + case _CALL_INTRINSIC_1_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef value; _PyStackRef res; oparg = CURRENT_OPARG(); @@ -1879,16 +4178,19 @@ PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _CALL_INTRINSIC_2: { + case _CALL_INTRINSIC_2_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef value1_st; _PyStackRef value2_st; _PyStackRef res; @@ -1912,16 +4214,19 @@ stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _RETURN_VALUE: { + case _RETURN_VALUE_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef retval; _PyStackRef res; retval = stack_pointer[-1]; @@ -1939,13 +4244,15 @@ LOAD_IP(frame->return_offset); res = temp; LLTRACE_RESUME_FRAME(); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _GET_AITER: { + case _GET_AITER_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef obj; _PyStackRef iter; obj = stack_pointer[-1]; @@ -1968,6 +4275,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(obj); stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } _PyFrame_SetStackPointer(frame, stack_pointer); @@ -1979,6 +4287,7 @@ PyStackRef_CLOSE(obj); stack_pointer = _PyFrame_GetStackPointer(frame); if (iter_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } if (Py_TYPE(iter_o)->tp_as_async == NULL || @@ -1990,16 +4299,19 @@ Py_TYPE(iter_o)->tp_name); Py_DECREF(iter_o); stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } iter = PyStackRef_FromPyObjectSteal(iter_o); - stack_pointer[0] = iter; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = iter; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _GET_ANEXT: { + case _GET_ANEXT_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef aiter; _PyStackRef awaitable; aiter = stack_pointer[-1]; @@ -2007,16 +4319,19 @@ PyObject *awaitable_o = _PyEval_GetANext(PyStackRef_AsPyObjectBorrow(aiter)); stack_pointer = _PyFrame_GetStackPointer(frame); if (awaitable_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); - stack_pointer[0] = awaitable; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = awaitable; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _GET_AWAITABLE: { + case _GET_AWAITABLE_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef iterable; _PyStackRef iter; oparg = CURRENT_OPARG(); @@ -2030,31 +4345,38 @@ PyStackRef_CLOSE(iterable); stack_pointer = _PyFrame_GetStackPointer(frame); if (iter_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } iter = PyStackRef_FromPyObjectSteal(iter_o); - stack_pointer[0] = iter; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = iter; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } /* _SEND is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ - case _SEND_GEN_FRAME: { + case _SEND_GEN_FRAME_r22: { + CHECK_CURRENT_CACHED_VALUES(2); _PyStackRef v; _PyStackRef receiver; _PyStackRef gen_frame; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; oparg = CURRENT_OPARG(); - v = stack_pointer[-1]; - receiver = stack_pointer[-2]; + v = _stack_item_1; + receiver = _stack_item_0; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } if (gen->gi_frame_state >= FRAME_EXECUTING) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } STAT_INC(SEND, hit); @@ -2067,15 +4389,20 @@ frame->return_offset = (uint16_t)( 2 + oparg); pushed_frame->previous = frame; gen_frame = PyStackRef_Wrap(pushed_frame); - stack_pointer[-1] = gen_frame; + _tos_cache1 = gen_frame; + _tos_cache0 = receiver; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(2); break; } - case _YIELD_VALUE: { + case _YIELD_VALUE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); _PyStackRef retval; _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); - retval = stack_pointer[-1]; + retval = _stack_item_0; assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); frame->instr_ptr++; PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); @@ -2083,8 +4410,6 @@ assert(oparg == 0 || oparg == 1); gen->gi_frame_state = FRAME_SUSPENDED + oparg; _PyStackRef temp = retval; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; @@ -2105,13 +4430,15 @@ LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); value = PyStackRef_MakeHeapSafe(temp); LLTRACE_RESUME_FRAME(); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = value; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _POP_EXCEPT: { + case _POP_EXCEPT_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef exc_value; exc_value = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; @@ -2122,27 +4449,61 @@ stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } - case _LOAD_COMMON_CONSTANT: { + case _LOAD_COMMON_CONSTANT_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef value; oparg = CURRENT_OPARG(); assert(oparg < NUM_COMMON_CONSTANTS); value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _LOAD_COMMON_CONSTANT_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + assert(oparg < NUM_COMMON_CONSTANTS); + value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); break; } - case _LOAD_BUILD_CLASS: { + case _LOAD_COMMON_CONSTANT_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + assert(oparg < NUM_COMMON_CONSTANTS); + value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _LOAD_BUILD_CLASS_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef bc; PyObject *bc_o; _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } if (bc_o == NULL) { @@ -2150,16 +4511,19 @@ _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } bc = PyStackRef_FromPyObjectSteal(bc_o); - stack_pointer[0] = bc; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = bc; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _STORE_NAME: { + case _STORE_NAME_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef v; oparg = CURRENT_OPARG(); v = stack_pointer[-1]; @@ -2176,6 +4540,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(v); stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } if (PyDict_CheckExact(ns)) { @@ -2194,12 +4559,18 @@ PyStackRef_CLOSE(v); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } - case _DELETE_NAME: { + case _DELETE_NAME_r00: { + CHECK_CURRENT_CACHED_VALUES(0); oparg = CURRENT_OPARG(); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *ns = LOCALS(); @@ -2209,6 +4580,7 @@ _PyErr_Format(tstate, PyExc_SystemError, "no locals when deleting %R", name); stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } _PyFrame_SetStackPointer(frame, stack_pointer); @@ -2220,12 +4592,18 @@ NAME_ERROR_MSG, name); stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } - case _UNPACK_SEQUENCE: { + case _UNPACK_SEQUENCE_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef seq; _PyStackRef *top; oparg = CURRENT_OPARG(); @@ -2239,49 +4617,66 @@ Py_DECREF(seq_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (res == 0) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } stack_pointer += oparg; assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } - case _UNPACK_SEQUENCE_TWO_TUPLE: { + case _UNPACK_SEQUENCE_TWO_TUPLE_r12: { + CHECK_CURRENT_CACHED_VALUES(1); _PyStackRef seq; _PyStackRef val1; _PyStackRef val0; + _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); - seq = stack_pointer[-1]; + seq = _stack_item_0; assert(oparg == 2); PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); assert(PyTuple_CheckExact(seq_o)); if (PyTuple_GET_SIZE(seq_o) != 2) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } STAT_INC(UNPACK_SEQUENCE, hit); val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); - stack_pointer[-1] = val1; - stack_pointer[0] = val0; - stack_pointer += 1; + stack_pointer[0] = val1; + stack_pointer[1] = val0; + stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(seq); stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache1 = val0; + _tos_cache0 = val1; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(2); break; } - case _UNPACK_SEQUENCE_TUPLE: { + case _UNPACK_SEQUENCE_TUPLE_r10: { + CHECK_CURRENT_CACHED_VALUES(1); _PyStackRef seq; _PyStackRef *values; + _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); - seq = stack_pointer[-1]; - values = &stack_pointer[-1]; + seq = _stack_item_0; + values = &stack_pointer[0]; PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); assert(PyTuple_CheckExact(seq_o)); if (PyTuple_GET_SIZE(seq_o) != oparg) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } STAT_INC(UNPACK_SEQUENCE, hit); @@ -2289,30 +4684,38 @@ for (int i = oparg; --i >= 0; ) { *values++ = PyStackRef_FromPyObjectNew(items[i]); } - stack_pointer += -1 + oparg; + stack_pointer += oparg; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(seq); stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } - case _UNPACK_SEQUENCE_LIST: { + case _UNPACK_SEQUENCE_LIST_r10: { + CHECK_CURRENT_CACHED_VALUES(1); _PyStackRef seq; _PyStackRef *values; + _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); - seq = stack_pointer[-1]; - values = &stack_pointer[-1]; + seq = _stack_item_0; + values = &stack_pointer[0]; PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); assert(PyList_CheckExact(seq_o)); if (!LOCK_OBJECT(seq_o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } if (PyList_GET_SIZE(seq_o) != oparg) { UNLOCK_OBJECT(seq_o); if (true) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } } @@ -2322,15 +4725,20 @@ *values++ = PyStackRef_FromPyObjectNew(items[i]); } UNLOCK_OBJECT(seq_o); - stack_pointer += -1 + oparg; + stack_pointer += oparg; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(seq); stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } - case _UNPACK_EX: { + case _UNPACK_EX_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef seq; _PyStackRef *top; oparg = CURRENT_OPARG(); @@ -2344,14 +4752,20 @@ Py_DECREF(seq_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (res == 0) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } stack_pointer += 1 + (oparg & 0xFF) + (oparg >> 8); assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } - case _STORE_ATTR: { + case _STORE_ATTR_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef owner; _PyStackRef v; oparg = CURRENT_OPARG(); @@ -2373,12 +4787,18 @@ stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); if (err) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } - case _DELETE_ATTR: { + case _DELETE_ATTR_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef owner; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; @@ -2392,12 +4812,18 @@ PyStackRef_CLOSE(owner); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } - case _STORE_GLOBAL: { + case _STORE_GLOBAL_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef v; oparg = CURRENT_OPARG(); v = stack_pointer[-1]; @@ -2411,18 +4837,25 @@ PyStackRef_CLOSE(v); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } - case _DELETE_GLOBAL: { + case _DELETE_GLOBAL_r00: { + CHECK_CURRENT_CACHED_VALUES(0); oparg = CURRENT_OPARG(); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyDict_Pop(GLOBALS(), name, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } if (err == 0) { @@ -2430,12 +4863,18 @@ _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } - case _LOAD_LOCALS: { + case _LOAD_LOCALS_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef locals; PyObject *l = LOCALS(); if (l == NULL) { @@ -2443,18 +4882,21 @@ _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } locals = PyStackRef_FromPyObjectNew(l); - stack_pointer[0] = locals; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = locals; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } /* _LOAD_FROM_DICT_OR_GLOBALS is not a viable micro-op for tier 2 because it has both popping and not-popping errors */ - case _LOAD_NAME: { + case _LOAD_NAME_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef v; oparg = CURRENT_OPARG(); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); @@ -2462,16 +4904,19 @@ PyObject *v_o = _PyEval_LoadName(tstate, frame, name); stack_pointer = _PyFrame_GetStackPointer(frame); if (v_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } v = PyStackRef_FromPyObjectSteal(v_o); - stack_pointer[0] = v; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = v; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _LOAD_GLOBAL: { + case _LOAD_GLOBAL_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef *res; oparg = CURRENT_OPARG(); res = &stack_pointer[0]; @@ -2480,14 +4925,20 @@ _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); stack_pointer = _PyFrame_GetStackPointer(frame); if (PyStackRef_IsNull(*res)) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } - case _PUSH_NULL_CONDITIONAL: { + case _PUSH_NULL_CONDITIONAL_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef *null; oparg = CURRENT_OPARG(); null = &stack_pointer[0]; @@ -2496,37 +4947,117 @@ } stack_pointer += (oparg & 1); assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _GUARD_GLOBALS_VERSION_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + uint16_t version = (uint16_t)CURRENT_OPERAND0(); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + if (!PyDict_CheckExact(dict)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + assert(DK_IS_UNICODE(keys)); + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _GUARD_GLOBALS_VERSION_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef _stack_item_0 = _tos_cache0; + uint16_t version = (uint16_t)CURRENT_OPERAND0(); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + if (!PyDict_CheckExact(dict)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + assert(DK_IS_UNICODE(keys)); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _GUARD_GLOBALS_VERSION_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + uint16_t version = (uint16_t)CURRENT_OPERAND0(); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + if (!PyDict_CheckExact(dict)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + assert(DK_IS_UNICODE(keys)); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); break; } - case _GUARD_GLOBALS_VERSION: { + case _GUARD_GLOBALS_VERSION_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; uint16_t version = (uint16_t)CURRENT_OPERAND0(); PyDictObject *dict = (PyDictObject *)GLOBALS(); if (!PyDict_CheckExact(dict)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } assert(DK_IS_UNICODE(keys)); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); break; } - case _LOAD_GLOBAL_MODULE: { + case _LOAD_GLOBAL_MODULE_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef res; uint16_t version = (uint16_t)CURRENT_OPERAND0(); uint16_t index = (uint16_t)CURRENT_OPERAND1(); PyDictObject *dict = (PyDictObject *)GLOBALS(); if (!PyDict_CheckExact(dict)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } assert(DK_IS_UNICODE(keys)); @@ -2535,36 +5066,42 @@ PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); if (res_o == NULL) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } #if Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); if (!increfed) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } #else res = PyStackRef_FromPyObjectNew(res_o); #endif STAT_INC(LOAD_GLOBAL, hit); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _LOAD_GLOBAL_BUILTINS: { + case _LOAD_GLOBAL_BUILTINS_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef res; uint16_t version = (uint16_t)CURRENT_OPERAND0(); uint16_t index = (uint16_t)CURRENT_OPERAND1(); PyDictObject *dict = (PyDictObject *)BUILTINS(); if (!PyDict_CheckExact(dict)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } assert(DK_IS_UNICODE(keys)); @@ -2572,25 +5109,29 @@ PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); if (res_o == NULL) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } #if Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); if (!increfed) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } #else res = PyStackRef_FromPyObjectNew(res_o); #endif STAT_INC(LOAD_GLOBAL, hit); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _DELETE_FAST: { + case _DELETE_FAST_r00: { + CHECK_CURRENT_CACHED_VALUES(0); oparg = CURRENT_OPARG(); _PyStackRef v = GETLOCAL(oparg); if (PyStackRef_IsNull(v)) { @@ -2600,6 +5141,7 @@ PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } _PyStackRef tmp = GETLOCAL(oparg); @@ -2607,14 +5149,20 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } - case _MAKE_CELL: { + case _MAKE_CELL_r00: { + CHECK_CURRENT_CACHED_VALUES(0); oparg = CURRENT_OPARG(); PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); PyObject *cell = PyCell_New(initial); if (cell == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } _PyStackRef tmp = GETLOCAL(oparg); @@ -2622,10 +5170,15 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } - case _DELETE_DEREF: { + case _DELETE_DEREF_r00: { + CHECK_CURRENT_CACHED_VALUES(0); oparg = CURRENT_OPARG(); PyObject *cell = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); @@ -2633,15 +5186,21 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(oldobj); stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } - case _LOAD_FROM_DICT_OR_DEREF: { + case _LOAD_FROM_DICT_OR_DEREF_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef class_dict_st; _PyStackRef value; oparg = CURRENT_OPARG(); @@ -2656,6 +5215,7 @@ int err = PyMapping_GetOptionalItem(class_dict, name, &value_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } if (!value_o) { @@ -2665,6 +5225,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } } @@ -2674,13 +5235,15 @@ PyStackRef_CLOSE(class_dict_st); stack_pointer = _PyFrame_GetStackPointer(frame); value = PyStackRef_FromPyObjectSteal(value_o); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = value; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _LOAD_DEREF: { + case _LOAD_DEREF_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef value; oparg = CURRENT_OPARG(); PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); @@ -2694,15 +5257,18 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = value; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _STORE_DEREF: { + case _STORE_DEREF_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef v; oparg = CURRENT_OPARG(); v = stack_pointer[-1]; @@ -2712,10 +5278,15 @@ stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } - case _COPY_FREE_VARS: { + case _COPY_FREE_VARS_r00: { + CHECK_CURRENT_CACHED_VALUES(0); oparg = CURRENT_OPARG(); PyCodeObject *co = _PyFrame_GetCode(frame); assert(PyStackRef_FunctionCheck(frame->f_funcobj)); @@ -2727,10 +5298,15 @@ PyObject *o = PyTuple_GET_ITEM(closure, i); frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o); } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } - case _BUILD_STRING: { + case _BUILD_STRING_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef *pieces; _PyStackRef str; oparg = CURRENT_OPARG(); @@ -2747,6 +5323,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); @@ -2762,16 +5339,19 @@ stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); if (str_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } str = PyStackRef_FromPyObjectSteal(str_o); - stack_pointer[0] = str; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = str; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _BUILD_INTERPOLATION: { + case _BUILD_INTERPOLATION_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef *format; _PyStackRef str; _PyStackRef value; @@ -2814,16 +5394,19 @@ PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); if (interpolation_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } interpolation = PyStackRef_FromPyObjectSteal(interpolation_o); - stack_pointer[0] = interpolation; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = interpolation; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _BUILD_TEMPLATE: { + case _BUILD_TEMPLATE_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef interpolations; _PyStackRef strings; _PyStackRef template; @@ -2845,32 +5428,38 @@ PyStackRef_CLOSE(strings); stack_pointer = _PyFrame_GetStackPointer(frame); if (template_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } template = PyStackRef_FromPyObjectSteal(template_o); - stack_pointer[0] = template; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = template; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _BUILD_TUPLE: { + case _BUILD_TUPLE_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef *values; _PyStackRef tup; oparg = CURRENT_OPARG(); values = &stack_pointer[-oparg]; PyObject *tup_o = _PyTuple_FromStackRefStealOnSuccess(values, oparg); if (tup_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } tup = PyStackRef_FromPyObjectStealMortal(tup_o); - stack_pointer[-oparg] = tup; - stack_pointer += 1 - oparg; + _tos_cache0 = tup; + stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(1); break; } - case _BUILD_LIST: { + case _BUILD_LIST_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef *values; _PyStackRef list; oparg = CURRENT_OPARG(); @@ -2879,16 +5468,21 @@ PyObject *list_o = _PyList_FromStackRefStealOnSuccess(values, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); if (list_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } list = PyStackRef_FromPyObjectStealMortal(list_o); - stack_pointer[-oparg] = list; - stack_pointer += 1 - oparg; + _tos_cache0 = list; + stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _LIST_EXTEND: { + case _LIST_EXTEND_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef iterable_st; _PyStackRef list_st; oparg = CURRENT_OPARG(); @@ -2918,6 +5512,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iterable_st); stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } assert(Py_IsNone(none_val)); @@ -2926,10 +5521,15 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iterable_st); stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } - case _SET_UPDATE: { + case _SET_UPDATE_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef iterable; _PyStackRef set; oparg = CURRENT_OPARG(); @@ -2945,12 +5545,18 @@ PyStackRef_CLOSE(iterable); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } - case _BUILD_SET: { + case _BUILD_SET_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef *values; _PyStackRef set; oparg = CURRENT_OPARG(); @@ -2969,6 +5575,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } int err = 0; @@ -2992,16 +5599,21 @@ _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(set_o); stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } set = PyStackRef_FromPyObjectStealMortal(set_o); - stack_pointer[-oparg] = set; - stack_pointer += 1 - oparg; + _tos_cache0 = set; + stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _BUILD_MAP: { + case _BUILD_MAP_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef *values; _PyStackRef map; oparg = CURRENT_OPARG(); @@ -3018,6 +5630,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -oparg*2; assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } _PyFrame_SetStackPointer(frame, stack_pointer); @@ -3038,28 +5651,33 @@ stack_pointer += -oparg*2; assert(WITHIN_STACK_BOUNDS()); if (map_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } map = PyStackRef_FromPyObjectStealMortal(map_o); - stack_pointer[0] = map; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = map; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _SETUP_ANNOTATIONS: { + case _SETUP_ANNOTATIONS_r00: { + CHECK_CURRENT_CACHED_VALUES(0); PyObject *ann_dict; if (LOCALS() == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_Format(tstate, PyExc_SystemError, "no locals found when setting up annotations"); stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } if (ann_dict == NULL) { @@ -3067,6 +5685,7 @@ ann_dict = PyDict_New(); stack_pointer = _PyFrame_GetStackPointer(frame); if (ann_dict == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } _PyFrame_SetStackPointer(frame, stack_pointer); @@ -3075,6 +5694,7 @@ Py_DECREF(ann_dict); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } } @@ -3083,10 +5703,15 @@ Py_DECREF(ann_dict); stack_pointer = _PyFrame_GetStackPointer(frame); } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } - case _DICT_UPDATE: { + case _DICT_UPDATE_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef update; _PyStackRef dict; oparg = CURRENT_OPARG(); @@ -3113,6 +5738,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(update); stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } stack_pointer += -1; @@ -3120,10 +5746,15 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(update); stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } - case _DICT_MERGE: { + case _DICT_MERGE_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef update; _PyStackRef dict; _PyStackRef callable; @@ -3146,6 +5777,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(update); stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } stack_pointer += -1; @@ -3153,10 +5785,15 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(update); stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } - case _MAP_ADD: { + case _MAP_ADD_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef value; _PyStackRef key; _PyStackRef dict_st; @@ -3176,14 +5813,20 @@ if (err != 0) { stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } - case _LOAD_SUPER_ATTR_ATTR: { + case _LOAD_SUPER_ATTR_ATTR_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef self_st; _PyStackRef class_st; _PyStackRef global_super_st; @@ -3198,10 +5841,12 @@ assert(!(oparg & 1)); if (global_super != (PyObject *)&PySuper_Type) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } if (!PyType_Check(class)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } STAT_INC(LOAD_SUPER_ATTR, hit); @@ -3224,16 +5869,19 @@ stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); if (attr == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } attr_st = PyStackRef_FromPyObjectSteal(attr); - stack_pointer[0] = attr_st; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = attr_st; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _LOAD_SUPER_ATTR_METHOD: { + case _LOAD_SUPER_ATTR_METHOD_r02: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef self_st; _PyStackRef class_st; _PyStackRef global_super_st; @@ -3249,10 +5897,12 @@ assert(oparg & 1); if (global_super != (PyObject *)&PySuper_Type) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } if (!PyType_Check(class)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } STAT_INC(LOAD_SUPER_ATTR, hit); @@ -3264,6 +5914,7 @@ Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); stack_pointer = _PyFrame_GetStackPointer(frame); if (attr_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } if (method_found) { @@ -3292,14 +5943,15 @@ stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); attr = PyStackRef_FromPyObjectSteal(attr_o); - stack_pointer[0] = attr; - stack_pointer[1] = self_or_null; - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache1 = self_or_null; + _tos_cache0 = attr; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(2); break; } - case _LOAD_ATTR: { + case _LOAD_ATTR_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef owner; _PyStackRef *attr; _PyStackRef *self_or_null; @@ -3324,6 +5976,7 @@ PyStackRef_CLOSE(owner); stack_pointer = _PyFrame_GetStackPointer(frame); if (PyStackRef_IsNull(*attr)) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } self_or_null[0] = PyStackRef_NULL; @@ -3340,6 +5993,7 @@ PyStackRef_CLOSE(owner); stack_pointer = _PyFrame_GetStackPointer(frame); if (attr_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } *attr = PyStackRef_FromPyObjectSteal(attr_o); @@ -3347,10 +6001,33 @@ } stack_pointer += (oparg&1); assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _GUARD_TYPE_VERSION_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + owner = _stack_item_0; + uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); break; } - case _GUARD_TYPE_VERSION: { + case _GUARD_TYPE_VERSION_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef owner; owner = stack_pointer[-1]; uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); @@ -3358,55 +6035,256 @@ assert(type_version != 0); if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _GUARD_TYPE_VERSION_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + owner = _stack_item_1; + uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _GUARD_TYPE_VERSION_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + owner = _stack_item_2; + uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = owner; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _GUARD_TYPE_VERSION_AND_LOCK_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + owner = _stack_item_0; + uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(type_version != 0); + if (!LOCK_OBJECT(owner_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + PyTypeObject *tp = Py_TYPE(owner_o); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UNLOCK_OBJECT(owner_o); + if (true) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + } + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _GUARD_TYPE_VERSION_AND_LOCK_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + _PyStackRef owner; + owner = stack_pointer[-1]; + uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(type_version != 0); + if (!LOCK_OBJECT(owner_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + PyTypeObject *tp = Py_TYPE(owner_o); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UNLOCK_OBJECT(owner_o); + if (true) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + } + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _GUARD_TYPE_VERSION_AND_LOCK_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + owner = _stack_item_1; + uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(type_version != 0); + if (!LOCK_OBJECT(owner_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + PyTypeObject *tp = Py_TYPE(owner_o); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UNLOCK_OBJECT(owner_o); + if (true) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + } + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _GUARD_TYPE_VERSION_AND_LOCK_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + owner = _stack_item_2; + uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(type_version != 0); + if (!LOCK_OBJECT(owner_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + PyTypeObject *tp = Py_TYPE(owner_o); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UNLOCK_OBJECT(owner_o); + if (true) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + } + _tos_cache2 = owner; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _CHECK_MANAGED_OBJECT_HAS_VALUES_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + owner = _stack_item_0; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _CHECK_MANAGED_OBJECT_HAS_VALUES_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + _PyStackRef owner; + owner = stack_pointer[-1]; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } + SET_CURRENT_CACHED_VALUES(0); break; } - case _GUARD_TYPE_VERSION_AND_LOCK: { + case _CHECK_MANAGED_OBJECT_HAS_VALUES_r22: { + CHECK_CURRENT_CACHED_VALUES(2); _PyStackRef owner; - owner = stack_pointer[-1]; - uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + owner = _stack_item_1; PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(type_version != 0); - if (!LOCK_OBJECT(owner_o)) { + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } - PyTypeObject *tp = Py_TYPE(owner_o); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UNLOCK_OBJECT(owner_o); - if (true) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - } + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); break; } - case _CHECK_MANAGED_OBJECT_HAS_VALUES: { + case _CHECK_MANAGED_OBJECT_HAS_VALUES_r33: { + CHECK_CURRENT_CACHED_VALUES(3); _PyStackRef owner; - owner = stack_pointer[-1]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + owner = _stack_item_2; PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } + _tos_cache2 = owner; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); break; } - case _LOAD_ATTR_INSTANCE_VALUE: { + case _LOAD_ATTR_INSTANCE_VALUE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); _PyStackRef owner; _PyStackRef attr; - owner = stack_pointer[-1]; + _PyStackRef _stack_item_0 = _tos_cache0; + owner = _stack_item_0; uint16_t offset = (uint16_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); if (attr_o == NULL) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } #ifdef Py_GIL_DISABLED @@ -3414,6 +6292,7 @@ if (!increfed) { if (true) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } } @@ -3421,22 +6300,33 @@ attr = PyStackRef_FromPyObjectNew(attr_o); #endif STAT_INC(LOAD_ATTR, hit); - stack_pointer[-1] = attr; + stack_pointer[0] = attr; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = attr; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _LOAD_ATTR_MODULE: { + case _LOAD_ATTR_MODULE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); _PyStackRef owner; _PyStackRef attr; - owner = stack_pointer[-1]; + _PyStackRef _stack_item_0 = _tos_cache0; + owner = _stack_item_0; uint32_t dict_version = (uint32_t)CURRENT_OPERAND0(); uint16_t index = (uint16_t)CURRENT_OPERAND1(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); if (Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; @@ -3444,6 +6334,7 @@ PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } assert(keys->dk_kind == DICT_KEYS_UNICODE); @@ -3452,6 +6343,7 @@ PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); if (attr_o == NULL) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } #ifdef Py_GIL_DISABLED @@ -3459,6 +6351,7 @@ if (!increfed) { if (true) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } } @@ -3466,24 +6359,35 @@ attr = PyStackRef_FromPyObjectNew(attr_o); #endif STAT_INC(LOAD_ATTR, hit); - stack_pointer[-1] = attr; + stack_pointer[0] = attr; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = attr; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _LOAD_ATTR_WITH_HINT: { + case _LOAD_ATTR_WITH_HINT_r11: { + CHECK_CURRENT_CACHED_VALUES(1); _PyStackRef owner; _PyStackRef attr; + _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); - owner = stack_pointer[-1]; + owner = _stack_item_0; uint16_t hint = (uint16_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictObject *dict = _PyObject_GetManagedDict(owner_o); if (dict == NULL) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } PyDictKeysObject *dk = FT_ATOMIC_LOAD_PTR(dict->ma_keys); @@ -3491,6 +6395,7 @@ #ifdef Py_GIL_DISABLED if (!_Py_IsOwnedByCurrentThread((PyObject *)dict) && !_PyObject_GC_IS_SHARED(dict)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } #endif @@ -3498,6 +6403,7 @@ if (hint >= (size_t)FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_nentries)) { if (true) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } } @@ -3505,6 +6411,7 @@ if (dk->dk_kind != DICT_KEYS_UNICODE) { if (true) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } } @@ -3512,6 +6419,7 @@ if (FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_key) != name) { if (true) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } } @@ -3519,6 +6427,7 @@ if (attr_o == NULL) { if (true) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } } @@ -3528,89 +6437,203 @@ if (!increfed) { if (true) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } } #else attr = PyStackRef_FromPyObjectNew(attr_o); #endif - stack_pointer[-1] = attr; + stack_pointer[0] = attr; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = attr; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _LOAD_ATTR_SLOT: { + case _LOAD_ATTR_SLOT_r11: { + CHECK_CURRENT_CACHED_VALUES(1); _PyStackRef owner; _PyStackRef attr; - owner = stack_pointer[-1]; + _PyStackRef _stack_item_0 = _tos_cache0; + owner = _stack_item_0; uint16_t index = (uint16_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **addr = (PyObject **)((char *)owner_o + index); PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); if (attr_o == NULL) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); if (!increfed) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } #else attr = PyStackRef_FromPyObjectNew(attr_o); #endif STAT_INC(LOAD_ATTR, hit); + stack_pointer[0] = owner; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef tmp = owner; owner = attr; stack_pointer[-1] = owner; PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = attr; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _CHECK_ATTR_CLASS_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + owner = _stack_item_0; + uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (!PyType_Check(owner_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); break; } - case _CHECK_ATTR_CLASS: { + case _CHECK_ATTR_CLASS_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef owner; owner = stack_pointer[-1]; uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); if (!PyType_Check(owner_o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _CHECK_ATTR_CLASS_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + owner = _stack_item_1; + uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (!PyType_Check(owner_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _CHECK_ATTR_CLASS_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + owner = _stack_item_2; + uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (!PyType_Check(owner_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } assert(type_version != 0); if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } + _tos_cache2 = owner; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); break; } - case _LOAD_ATTR_CLASS: { + case _LOAD_ATTR_CLASS_r11: { + CHECK_CURRENT_CACHED_VALUES(1); _PyStackRef owner; _PyStackRef attr; - owner = stack_pointer[-1]; + _PyStackRef _stack_item_0 = _tos_cache0; + owner = _stack_item_0; PyObject *descr = (PyObject *)CURRENT_OPERAND0(); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); + stack_pointer[0] = owner; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef tmp = owner; owner = attr; stack_pointer[-1] = owner; PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = attr; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _LOAD_ATTR_PROPERTY_FRAME: { + case _LOAD_ATTR_PROPERTY_FRAME_r11: { + CHECK_CURRENT_CACHED_VALUES(1); _PyStackRef owner; _PyStackRef new_frame; + _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); - owner = stack_pointer[-1]; + owner = _stack_item_0; PyObject *fget = (PyObject *)CURRENT_OPERAND0(); assert((oparg & 1) == 0); assert(Py_IS_TYPE(fget, &PyFunction_Type)); @@ -3618,31 +6641,61 @@ PyCodeObject *code = (PyCodeObject *)f->func_code; if ((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } if (code->co_kwonlyargcount) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } if (code->co_argcount != 1) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } STAT_INC(LOAD_ATTR, hit); _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); pushed_frame->localsplus[0] = owner; new_frame = PyStackRef_Wrap(pushed_frame); - stack_pointer[-1] = new_frame; + _tos_cache0 = new_frame; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } /* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 because it has too many cache entries */ - case _GUARD_DORV_NO_DICT: { + case _GUARD_DORV_NO_DICT_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + owner = _stack_item_0; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (_PyObject_GetManagedDict(owner_o) || + !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UNLOCK_OBJECT(owner_o); + if (true) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + } + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _GUARD_DORV_NO_DICT_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef owner; owner = stack_pointer[-1]; PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); @@ -3653,17 +6706,72 @@ UNLOCK_OBJECT(owner_o); if (true) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + } + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _GUARD_DORV_NO_DICT_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + owner = _stack_item_1; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (_PyObject_GetManagedDict(owner_o) || + !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UNLOCK_OBJECT(owner_o); + if (true) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + } + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _GUARD_DORV_NO_DICT_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + owner = _stack_item_2; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (_PyObject_GetManagedDict(owner_o) || + !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UNLOCK_OBJECT(owner_o); + if (true) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } } + _tos_cache2 = owner; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); break; } - case _STORE_ATTR_INSTANCE_VALUE: { + case _STORE_ATTR_INSTANCE_VALUE_r20: { + CHECK_CURRENT_CACHED_VALUES(2); _PyStackRef owner; _PyStackRef value; - owner = stack_pointer[-1]; - value = stack_pointer[-2]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + owner = _stack_item_1; + value = _stack_item_0; uint16_t offset = (uint16_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); STAT_INC(STORE_ATTR, hit); @@ -3677,16 +6785,19 @@ _PyDictValues_AddToInsertionOrder(values, index); } UNLOCK_OBJECT(owner_o); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); Py_XDECREF(old_value); stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } - case _STORE_ATTR_WITH_HINT: { + case _STORE_ATTR_WITH_HINT_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef owner; _PyStackRef value; oparg = CURRENT_OPARG(); @@ -3698,10 +6809,12 @@ PyDictObject *dict = _PyObject_GetManagedDict(owner_o); if (dict == NULL) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } if (!LOCK_OBJECT(dict)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } assert(PyDict_CheckExact((PyObject *)dict)); @@ -3711,6 +6824,7 @@ UNLOCK_OBJECT(dict); if (true) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } } @@ -3719,6 +6833,7 @@ UNLOCK_OBJECT(dict); if (true) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } } @@ -3727,6 +6842,7 @@ UNLOCK_OBJECT(dict); if (true) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } } @@ -3742,18 +6858,26 @@ PyStackRef_CLOSE(owner); Py_XDECREF(old_value); stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } - case _STORE_ATTR_SLOT: { + case _STORE_ATTR_SLOT_r20: { + CHECK_CURRENT_CACHED_VALUES(2); _PyStackRef owner; _PyStackRef value; - owner = stack_pointer[-1]; - value = stack_pointer[-2]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + owner = _stack_item_1; + value = _stack_item_0; uint16_t index = (uint16_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); if (!LOCK_OBJECT(owner_o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } char *addr = (char *)owner_o + index; @@ -3761,16 +6885,19 @@ PyObject *old_value = *(PyObject **)addr; FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); UNLOCK_OBJECT(owner_o); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); Py_XDECREF(old_value); stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } - case _COMPARE_OP: { + case _COMPARE_OP_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -3794,6 +6921,7 @@ stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } if (oparg & 16) { @@ -3802,6 +6930,7 @@ Py_DECREF(res_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_bool < 0) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } res = res_bool ? PyStackRef_True : PyStackRef_False; @@ -3809,19 +6938,23 @@ else { res = PyStackRef_FromPyObjectSteal(res_o); } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _COMPARE_OP_FLOAT: { + case _COMPARE_OP_FLOAT_r21: { + CHECK_CURRENT_CACHED_VALUES(2); _PyStackRef right; _PyStackRef left; _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; oparg = CURRENT_OPARG(); - right = stack_pointer[-1]; - left = stack_pointer[-2]; + right = _stack_item_1; + left = _stack_item_0; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(COMPARE_OP, hit); @@ -3831,19 +6964,23 @@ PyStackRef_CLOSE_SPECIALIZED(left, _PyFloat_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(right, _PyFloat_ExactDealloc); res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _COMPARE_OP_INT: { + case _COMPARE_OP_INT_r21: { + CHECK_CURRENT_CACHED_VALUES(2); _PyStackRef right; _PyStackRef left; _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; oparg = CURRENT_OPARG(); - right = stack_pointer[-1]; - left = stack_pointer[-2]; + right = _stack_item_1; + left = _stack_item_0; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(_PyLong_IsCompact((PyLongObject *)left_o)); @@ -3857,19 +6994,23 @@ PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _COMPARE_OP_STR: { + case _COMPARE_OP_STR_r21: { + CHECK_CURRENT_CACHED_VALUES(2); _PyStackRef right; _PyStackRef left; _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; oparg = CURRENT_OPARG(); - right = stack_pointer[-1]; - left = stack_pointer[-2]; + right = _stack_item_1; + left = _stack_item_0; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(COMPARE_OP, hit); @@ -3881,40 +7022,45 @@ assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? PyStackRef_True : PyStackRef_False; - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _IS_OP: { + case _IS_OP_r21: { + CHECK_CURRENT_CACHED_VALUES(2); _PyStackRef right; _PyStackRef left; _PyStackRef b; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; oparg = CURRENT_OPARG(); - right = stack_pointer[-1]; - left = stack_pointer[-2]; + right = _stack_item_1; + left = _stack_item_0; int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef tmp = right; right = PyStackRef_NULL; - stack_pointer[-1] = right; + stack_pointer[0] = left; + stack_pointer[1] = right; PyStackRef_CLOSE(tmp); tmp = left; left = PyStackRef_NULL; - stack_pointer[-2] = left; + stack_pointer[0] = left; PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); b = res ? PyStackRef_True : PyStackRef_False; - stack_pointer[0] = b; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = b; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _CONTAINS_OP: { + case _CONTAINS_OP_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef right; _PyStackRef left; _PyStackRef b; @@ -3937,27 +7083,87 @@ stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); if (res < 0) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; - stack_pointer[0] = b; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = b; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _GUARD_TOS_ANY_SET_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + tos = _stack_item_0; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyAnySet_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = tos; + SET_CURRENT_CACHED_VALUES(1); break; } - case _GUARD_TOS_ANY_SET: { + case _GUARD_TOS_ANY_SET_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef tos; tos = stack_pointer[-1]; PyObject *o = PyStackRef_AsPyObjectBorrow(tos); if (!PyAnySet_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _GUARD_TOS_ANY_SET_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + tos = _stack_item_1; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyAnySet_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = tos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _GUARD_TOS_ANY_SET_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + tos = _stack_item_2; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyAnySet_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } + _tos_cache2 = tos; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); break; } - case _CONTAINS_OP_SET: { + case _CONTAINS_OP_SET_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef right; _PyStackRef left; _PyStackRef b; @@ -3982,16 +7188,19 @@ stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); if (res < 0) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; - stack_pointer[0] = b; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = b; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _CONTAINS_OP_DICT: { + case _CONTAINS_OP_DICT_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef right; _PyStackRef left; _PyStackRef b; @@ -4016,16 +7225,19 @@ stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); if (res < 0) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; - stack_pointer[0] = b; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = b; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _CHECK_EG_MATCH: { + case _CHECK_EG_MATCH_r02: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef match_type_st; _PyStackRef exc_value_st; _PyStackRef rest; @@ -4050,6 +7262,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } PyObject *match_o = NULL; @@ -4069,10 +7282,12 @@ stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); if (res < 0) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } assert((match_o == NULL) == (rest_o == NULL)); if (match_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } if (!Py_IsNone(match_o)) { @@ -4082,14 +7297,15 @@ } rest = PyStackRef_FromPyObjectSteal(rest_o); match = PyStackRef_FromPyObjectSteal(match_o); - stack_pointer[0] = rest; - stack_pointer[1] = match; - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache1 = match; + _tos_cache0 = rest; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(2); break; } - case _CHECK_EXC_MATCH: { + case _CHECK_EXC_MATCH_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef right; _PyStackRef left; _PyStackRef b; @@ -4102,6 +7318,7 @@ int err = _PyEval_CheckExceptTypeValid(tstate, right_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } _PyFrame_SetStackPointer(frame, stack_pointer); @@ -4113,13 +7330,15 @@ PyStackRef_CLOSE(right); stack_pointer = _PyFrame_GetStackPointer(frame); b = res ? PyStackRef_True : PyStackRef_False; - stack_pointer[0] = b; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = b; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _IMPORT_NAME: { + case _IMPORT_NAME_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef fromlist; _PyStackRef level; _PyStackRef res; @@ -4143,16 +7362,19 @@ stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _IMPORT_FROM: { + case _IMPORT_FROM_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef from; _PyStackRef res; oparg = CURRENT_OPARG(); @@ -4162,12 +7384,14 @@ PyObject *res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } @@ -4175,27 +7399,37 @@ /* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 because it is replaced */ - case _IS_NONE: { + case _IS_NONE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); _PyStackRef value; _PyStackRef b; - value = stack_pointer[-1]; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; if (PyStackRef_IsNone(value)) { b = PyStackRef_True; } else { b = PyStackRef_False; + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef tmp = value; value = b; stack_pointer[-1] = value; PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; } - stack_pointer[-1] = b; + _tos_cache0 = b; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _GET_LEN: { + case _GET_LEN_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef obj; _PyStackRef len; obj = stack_pointer[-1]; @@ -4203,20 +7437,24 @@ Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); stack_pointer = _PyFrame_GetStackPointer(frame); if (len_i < 0) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } PyObject *len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } len = PyStackRef_FromPyObjectSteal(len_o); - stack_pointer[0] = len; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = len; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _MATCH_CLASS: { + case _MATCH_CLASS_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef names; _PyStackRef type; _PyStackRef subject; @@ -4252,41 +7490,104 @@ } else { if (_PyErr_Occurred(tstate)) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } attrs = PyStackRef_None; } - stack_pointer[0] = attrs; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = attrs; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _MATCH_MAPPING_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef subject; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + subject = _stack_item_0; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; + res = match ? PyStackRef_True : PyStackRef_False; + _tos_cache1 = res; + _tos_cache0 = subject; + SET_CURRENT_CACHED_VALUES(2); break; } - case _MATCH_MAPPING: { + case _MATCH_MAPPING_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef subject; _PyStackRef res; subject = stack_pointer[-1]; int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; res = match ? PyStackRef_True : PyStackRef_False; - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _MATCH_MAPPING_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef subject; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + subject = _stack_item_1; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; + res = match ? PyStackRef_True : PyStackRef_False; + _tos_cache2 = res; + _tos_cache1 = subject; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _MATCH_SEQUENCE_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef subject; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + subject = _stack_item_0; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; + res = match ? PyStackRef_True : PyStackRef_False; + _tos_cache1 = res; + _tos_cache0 = subject; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _MATCH_SEQUENCE_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + _PyStackRef subject; + _PyStackRef res; + subject = stack_pointer[-1]; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; + res = match ? PyStackRef_True : PyStackRef_False; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(1); break; } - case _MATCH_SEQUENCE: { + case _MATCH_SEQUENCE_r23: { + CHECK_CURRENT_CACHED_VALUES(2); _PyStackRef subject; _PyStackRef res; - subject = stack_pointer[-1]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + subject = _stack_item_1; int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; res = match ? PyStackRef_True : PyStackRef_False; - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache2 = res; + _tos_cache1 = subject; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); break; } - case _MATCH_KEYS: { + case _MATCH_KEYS_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef keys; _PyStackRef subject; _PyStackRef values_or_none; @@ -4297,16 +7598,19 @@ PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); stack_pointer = _PyFrame_GetStackPointer(frame); if (values_or_none_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); - stack_pointer[0] = values_or_none; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = values_or_none; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _GET_ITER: { + case _GET_ITER_r02: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef iterable; _PyStackRef iter; _PyStackRef index_or_null; @@ -4332,20 +7636,24 @@ PyStackRef_CLOSE(iterable); stack_pointer = _PyFrame_GetStackPointer(frame); if (iter_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } iter = PyStackRef_FromPyObjectSteal(iter_o); index_or_null = PyStackRef_NULL; stack_pointer += 1; } - stack_pointer[-1] = iter; - stack_pointer[0] = index_or_null; - stack_pointer += 1; + _tos_cache1 = index_or_null; + _tos_cache0 = iter; + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(2); break; } - case _GET_YIELD_FROM_ITER: { + case _GET_YIELD_FROM_ITER_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef iterable; _PyStackRef iter; iterable = stack_pointer[-1]; @@ -4357,6 +7665,7 @@ "cannot 'yield from' a coroutine object " "in a non-coroutine generator"); stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } iter = iterable; @@ -4369,6 +7678,7 @@ PyObject *iter_o = PyObject_GetIter(iterable_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (iter_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } iter = PyStackRef_FromPyObjectSteal(iter_o); @@ -4379,13 +7689,19 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); } - stack_pointer[-1] = iter; + _tos_cache0 = iter; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } /* _FOR_ITER is not a viable micro-op for tier 2 because it is replaced */ - case _FOR_ITER_TIER_TWO: { + case _FOR_ITER_TIER_TWO_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef null_or_index; _PyStackRef iter; _PyStackRef next; @@ -4396,24 +7712,57 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (!PyStackRef_IsValid(item)) { if (PyStackRef_IsError(item)) { + stack_pointer[-1] = null_or_index; + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } if (true) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } } next = item; + _tos_cache0 = next; stack_pointer[-1] = null_or_index; - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } /* _INSTRUMENTED_FOR_ITER is not a viable micro-op for tier 2 because it is instrumented */ - case _ITER_CHECK_LIST: { + case _ITER_CHECK_LIST_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + null_or_index = _stack_item_1; + iter = _stack_item_0; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(iter_o) != &PyList_Type) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + assert(PyStackRef_IsTaggedInt(null_or_index)); + #ifdef Py_GIL_DISABLED + if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + #endif + _tos_cache1 = null_or_index; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _ITER_CHECK_LIST_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef null_or_index; _PyStackRef iter; null_or_index = stack_pointer[-1]; @@ -4421,21 +7770,104 @@ PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); if (Py_TYPE(iter_o) != &PyList_Type) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + assert(PyStackRef_IsTaggedInt(null_or_index)); + #ifdef Py_GIL_DISABLED + if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + #endif + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _ITER_CHECK_LIST_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + null_or_index = _stack_item_0; + iter = stack_pointer[-1]; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(iter_o) != &PyList_Type) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + assert(PyStackRef_IsTaggedInt(null_or_index)); + #ifdef Py_GIL_DISABLED + if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + #endif + _tos_cache0 = null_or_index; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _ITER_CHECK_LIST_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + null_or_index = _stack_item_2; + iter = _stack_item_1; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(iter_o) != &PyList_Type) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } assert(PyStackRef_IsTaggedInt(null_or_index)); #ifdef Py_GIL_DISABLED if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } #endif + _tos_cache2 = null_or_index; + _tos_cache1 = iter; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); break; } /* _ITER_JUMP_LIST is not a viable micro-op for tier 2 because it is replaced */ - case _GUARD_NOT_EXHAUSTED_LIST: { + case _GUARD_NOT_EXHAUSTED_LIST_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + null_or_index = _stack_item_1; + iter = _stack_item_0; + #ifndef Py_GIL_DISABLED + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(list_o) == &PyList_Type); + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + #endif + _tos_cache1 = null_or_index; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _GUARD_NOT_EXHAUSTED_LIST_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef null_or_index; _PyStackRef iter; null_or_index = stack_pointer[-1]; @@ -4445,15 +7877,64 @@ assert(Py_TYPE(list_o) == &PyList_Type); if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + #endif + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _GUARD_NOT_EXHAUSTED_LIST_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + null_or_index = _stack_item_0; + iter = stack_pointer[-1]; + #ifndef Py_GIL_DISABLED + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(list_o) == &PyList_Type); + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + #endif + _tos_cache0 = null_or_index; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _GUARD_NOT_EXHAUSTED_LIST_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + null_or_index = _stack_item_2; + iter = _stack_item_1; + #ifndef Py_GIL_DISABLED + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(list_o) == &PyList_Type); + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } #endif + _tos_cache2 = null_or_index; + _tos_cache1 = iter; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); break; } /* _ITER_NEXT_LIST is not a viable micro-op for tier 2 because it is replaced */ - case _ITER_NEXT_LIST_TIER_TWO: { + case _ITER_NEXT_LIST_TIER_TWO_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef null_or_index; _PyStackRef iter; _PyStackRef next; @@ -4470,6 +7951,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (result <= 0) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } #else @@ -4477,14 +7959,37 @@ next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, PyStackRef_UntagInt(null_or_index))); #endif null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); + _tos_cache0 = next; stack_pointer[-1] = null_or_index; - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _ITER_CHECK_TUPLE_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + null_or_index = _stack_item_1; + iter = _stack_item_0; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(iter_o) != &PyTuple_Type) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + assert(PyStackRef_IsTaggedInt(null_or_index)); + _tos_cache1 = null_or_index; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); break; } - case _ITER_CHECK_TUPLE: { + case _ITER_CHECK_TUPLE_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef null_or_index; _PyStackRef iter; null_or_index = stack_pointer[-1]; @@ -4492,15 +7997,81 @@ PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); if (Py_TYPE(iter_o) != &PyTuple_Type) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + assert(PyStackRef_IsTaggedInt(null_or_index)); + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _ITER_CHECK_TUPLE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + null_or_index = _stack_item_0; + iter = stack_pointer[-1]; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(iter_o) != &PyTuple_Type) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + assert(PyStackRef_IsTaggedInt(null_or_index)); + _tos_cache0 = null_or_index; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _ITER_CHECK_TUPLE_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + null_or_index = _stack_item_2; + iter = _stack_item_1; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(iter_o) != &PyTuple_Type) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } assert(PyStackRef_IsTaggedInt(null_or_index)); + _tos_cache2 = null_or_index; + _tos_cache1 = iter; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); break; } /* _ITER_JUMP_TUPLE is not a viable micro-op for tier 2 because it is replaced */ - case _GUARD_NOT_EXHAUSTED_TUPLE: { + case _GUARD_NOT_EXHAUSTED_TUPLE_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + null_or_index = _stack_item_1; + iter = _stack_item_0; + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(tuple_o) == &PyTuple_Type); + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = null_or_index; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _GUARD_NOT_EXHAUSTED_TUPLE_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef null_or_index; _PyStackRef iter; null_or_index = stack_pointer[-1]; @@ -4509,65 +8080,254 @@ assert(Py_TYPE(tuple_o) == &PyTuple_Type); if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _GUARD_NOT_EXHAUSTED_TUPLE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + null_or_index = _stack_item_0; + iter = stack_pointer[-1]; + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(tuple_o) == &PyTuple_Type); + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } + _tos_cache0 = null_or_index; + SET_CURRENT_CACHED_VALUES(1); break; } - case _ITER_NEXT_TUPLE: { + case _GUARD_NOT_EXHAUSTED_TUPLE_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + null_or_index = _stack_item_2; + iter = _stack_item_1; + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(tuple_o) == &PyTuple_Type); + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = null_or_index; + _tos_cache1 = iter; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _ITER_NEXT_TUPLE_r23: { + CHECK_CURRENT_CACHED_VALUES(2); _PyStackRef null_or_index; _PyStackRef iter; _PyStackRef next; - null_or_index = stack_pointer[-1]; - iter = stack_pointer[-2]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + null_or_index = _stack_item_1; + iter = _stack_item_0; PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(tuple_o) == &PyTuple_Type); uintptr_t i = PyStackRef_UntagInt(null_or_index); assert((size_t)i < (size_t)PyTuple_GET_SIZE(tuple_o)); next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(tuple_o, i)); null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); - stack_pointer[-1] = null_or_index; - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache2 = next; + _tos_cache1 = null_or_index; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(3); break; } - case _ITER_CHECK_RANGE: { + case _ITER_CHECK_RANGE_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + iter = _stack_item_0; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(r) != &PyRangeIter_Type) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + #ifdef Py_GIL_DISABLED + if (!_PyObject_IsUniquelyReferenced((PyObject *)r)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + #endif + _tos_cache1 = _stack_item_1; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _ITER_CHECK_RANGE_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef iter; iter = stack_pointer[-2]; _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); if (Py_TYPE(r) != &PyRangeIter_Type) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + #ifdef Py_GIL_DISABLED + if (!_PyObject_IsUniquelyReferenced((PyObject *)r)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + #endif + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _ITER_CHECK_RANGE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + iter = stack_pointer[-1]; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(r) != &PyRangeIter_Type) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + #ifdef Py_GIL_DISABLED + if (!_PyObject_IsUniquelyReferenced((PyObject *)r)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + #endif + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _ITER_CHECK_RANGE_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + iter = _stack_item_1; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(r) != &PyRangeIter_Type) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } #ifdef Py_GIL_DISABLED if (!_PyObject_IsUniquelyReferenced((PyObject *)r)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } #endif + _tos_cache2 = _stack_item_2; + _tos_cache1 = iter; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); break; } /* _ITER_JUMP_RANGE is not a viable micro-op for tier 2 because it is replaced */ - case _GUARD_NOT_EXHAUSTED_RANGE: { + case _GUARD_NOT_EXHAUSTED_RANGE_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + iter = _stack_item_0; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(r) == &PyRangeIter_Type); + if (r->len <= 0) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _GUARD_NOT_EXHAUSTED_RANGE_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef iter; iter = stack_pointer[-2]; _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); if (r->len <= 0) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _GUARD_NOT_EXHAUSTED_RANGE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + iter = stack_pointer[-1]; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(r) == &PyRangeIter_Type); + if (r->len <= 0) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _GUARD_NOT_EXHAUSTED_RANGE_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + iter = _stack_item_1; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(r) == &PyRangeIter_Type); + if (r->len <= 0) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } + _tos_cache2 = _stack_item_2; + _tos_cache1 = iter; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); break; } - case _ITER_NEXT_RANGE: { + case _ITER_NEXT_RANGE_r23: { + CHECK_CURRENT_CACHED_VALUES(2); _PyStackRef iter; _PyStackRef next; - iter = stack_pointer[-2]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + iter = _stack_item_0; _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); #ifdef Py_GIL_DISABLED @@ -4579,34 +8339,46 @@ r->len--; PyObject *res = PyLong_FromLong(value); if (res == NULL) { + stack_pointer[0] = iter; + stack_pointer[1] = _stack_item_1; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } next = PyStackRef_FromPyObjectSteal(res); - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache2 = next; + _tos_cache1 = _stack_item_1; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(3); break; } - case _FOR_ITER_GEN_FRAME: { + case _FOR_ITER_GEN_FRAME_r23: { + CHECK_CURRENT_CACHED_VALUES(2); _PyStackRef iter; _PyStackRef gen_frame; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; oparg = CURRENT_OPARG(); - iter = stack_pointer[-2]; + iter = _stack_item_0; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); if (Py_TYPE(gen) != &PyGen_Type) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } #ifdef Py_GIL_DISABLED if (!_PyObject_IsUniquelyReferenced((PyObject *)gen)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } #endif if (gen->gi_frame_state >= FRAME_EXECUTING) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } STAT_INC(FOR_ITER, hit); @@ -4618,25 +8390,30 @@ pushed_frame->previous = frame; frame->return_offset = (uint16_t)( 2 + oparg); gen_frame = PyStackRef_Wrap(pushed_frame); - stack_pointer[0] = gen_frame; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache2 = gen_frame; + _tos_cache1 = _stack_item_1; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(3); break; } - case _INSERT_NULL: { + case _INSERT_NULL_r10: { + CHECK_CURRENT_CACHED_VALUES(1); _PyStackRef self; _PyStackRef *method_and_self; - self = stack_pointer[-1]; - method_and_self = &stack_pointer[-1]; + _PyStackRef _stack_item_0 = _tos_cache0; + self = _stack_item_0; + method_and_self = &stack_pointer[0]; method_and_self[1] = self; method_and_self[0] = PyStackRef_NULL; - stack_pointer += 1; + stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); break; } - case _LOAD_SPECIAL: { + case _LOAD_SPECIAL_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef *method_and_self; oparg = CURRENT_OPARG(); method_and_self = &stack_pointer[-2]; @@ -4658,12 +8435,18 @@ _PyErr_Format(tstate, PyExc_TypeError, errfmt, owner); stack_pointer = _PyFrame_GetStackPointer(frame); } + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } - case _WITH_EXCEPT_START: { + case _WITH_EXCEPT_START_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef val; _PyStackRef lasti; _PyStackRef exit_self; @@ -4692,16 +8475,42 @@ Py_XDECREF(original_tb); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _PUSH_EXC_INFO_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef exc; + _PyStackRef prev_exc; + _PyStackRef new_exc; + _PyStackRef _stack_item_0 = _tos_cache0; + exc = _stack_item_0; + _PyErr_StackItem *exc_info = tstate->exc_info; + if (exc_info->exc_value != NULL) { + prev_exc = PyStackRef_FromPyObjectSteal(exc_info->exc_value); + } + else { + prev_exc = PyStackRef_None; + } + assert(PyStackRef_ExceptionInstanceCheck(exc)); + exc_info->exc_value = PyStackRef_AsPyObjectNew(exc); + new_exc = exc; + _tos_cache1 = new_exc; + _tos_cache0 = prev_exc; + SET_CURRENT_CACHED_VALUES(2); break; } - case _PUSH_EXC_INFO: { + case _PUSH_EXC_INFO_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef exc; _PyStackRef prev_exc; _PyStackRef new_exc; @@ -4716,14 +8525,57 @@ assert(PyStackRef_ExceptionInstanceCheck(exc)); exc_info->exc_value = PyStackRef_AsPyObjectNew(exc); new_exc = exc; + _tos_cache0 = new_exc; stack_pointer[-1] = prev_exc; - stack_pointer[0] = new_exc; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _PUSH_EXC_INFO_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef exc; + _PyStackRef prev_exc; + _PyStackRef new_exc; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + exc = _stack_item_1; + _PyErr_StackItem *exc_info = tstate->exc_info; + if (exc_info->exc_value != NULL) { + prev_exc = PyStackRef_FromPyObjectSteal(exc_info->exc_value); + } + else { + prev_exc = PyStackRef_None; + } + assert(PyStackRef_ExceptionInstanceCheck(exc)); + exc_info->exc_value = PyStackRef_AsPyObjectNew(exc); + new_exc = exc; + _tos_cache2 = new_exc; + _tos_cache1 = prev_exc; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + owner = _stack_item_0; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + PyDictValues *ivs = _PyObject_InlineValues(owner_o); + if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); break; } - case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT: { + case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef owner; owner = stack_pointer[-1]; PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); @@ -4731,12 +8583,76 @@ PyDictValues *ivs = _PyObject_InlineValues(owner_o); if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + owner = _stack_item_1; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + PyDictValues *ivs = _PyObject_InlineValues(owner_o); + if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); break; } - case _GUARD_KEYS_VERSION: { + case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + owner = _stack_item_2; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + PyDictValues *ivs = _PyObject_InlineValues(owner_o); + if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = owner; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _GUARD_KEYS_VERSION_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + owner = _stack_item_0; + uint32_t keys_version = (uint32_t)CURRENT_OPERAND0(); + PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _GUARD_KEYS_VERSION_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef owner; owner = stack_pointer[-1]; uint32_t keys_version = (uint32_t)CURRENT_OPERAND0(); @@ -4745,12 +8661,80 @@ PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _GUARD_KEYS_VERSION_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + owner = _stack_item_1; + uint32_t keys_version = (uint32_t)CURRENT_OPERAND0(); + PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _GUARD_KEYS_VERSION_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + owner = _stack_item_2; + uint32_t keys_version = (uint32_t)CURRENT_OPERAND0(); + PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } + _tos_cache2 = owner; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _LOAD_ATTR_METHOD_WITH_VALUES_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + owner = _stack_item_0; + PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + assert(oparg & 1); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + _tos_cache1 = self; + _tos_cache0 = attr; + SET_CURRENT_CACHED_VALUES(2); break; } - case _LOAD_ATTR_METHOD_WITH_VALUES: { + case _LOAD_ATTR_METHOD_WITH_VALUES_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef owner; _PyStackRef attr; _PyStackRef self; @@ -4763,96 +8747,273 @@ assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); attr = PyStackRef_FromPyObjectNew(descr); self = owner; + _tos_cache0 = self; stack_pointer[-1] = attr; - stack_pointer[0] = self; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _LOAD_ATTR_METHOD_WITH_VALUES_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + owner = _stack_item_1; + PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + assert(oparg & 1); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + _tos_cache2 = self; + _tos_cache1 = attr; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _LOAD_ATTR_METHOD_NO_DICT_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + owner = _stack_item_0; + PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + assert(oparg & 1); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + _tos_cache1 = self; + _tos_cache0 = attr; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _LOAD_ATTR_METHOD_NO_DICT_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; + oparg = CURRENT_OPARG(); + owner = stack_pointer[-1]; + PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + assert(oparg & 1); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + _tos_cache0 = self; + stack_pointer[-1] = attr; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _LOAD_ATTR_METHOD_NO_DICT_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + owner = _stack_item_1; + PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + assert(oparg & 1); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + _tos_cache2 = self; + _tos_cache1 = attr; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + owner = _stack_item_0; + PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + assert((oparg & 1) == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + attr = PyStackRef_FromPyObjectNew(descr); + _tos_cache0 = attr; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + owner = _stack_item_0; + PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + assert((oparg & 1) == 0); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + attr = PyStackRef_FromPyObjectNew(descr); + _tos_cache0 = attr; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _CHECK_ATTR_METHOD_LAZY_DICT_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + owner = _stack_item_0; + uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0(); + char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; + PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); + if (dict != NULL) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _CHECK_ATTR_METHOD_LAZY_DICT_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + _PyStackRef owner; + owner = stack_pointer[-1]; + uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0(); + char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; + PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); + if (dict != NULL) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _CHECK_ATTR_METHOD_LAZY_DICT_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + owner = _stack_item_1; + uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0(); + char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; + PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); + if (dict != NULL) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _CHECK_ATTR_METHOD_LAZY_DICT_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + owner = _stack_item_2; + uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0(); + char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; + PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); + if (dict != NULL) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = owner; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); break; } - case _LOAD_ATTR_METHOD_NO_DICT: { + case _LOAD_ATTR_METHOD_LAZY_DICT_r12: { + CHECK_CURRENT_CACHED_VALUES(1); _PyStackRef owner; _PyStackRef attr; _PyStackRef self; + _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); - owner = stack_pointer[-1]; + owner = _stack_item_0; PyObject *descr = (PyObject *)CURRENT_OPERAND0(); assert(oparg & 1); - assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); attr = PyStackRef_FromPyObjectNew(descr); self = owner; - stack_pointer[-1] = attr; - stack_pointer[0] = self; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - break; - } - - case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: { - _PyStackRef owner; - _PyStackRef attr; - oparg = CURRENT_OPARG(); - owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)CURRENT_OPERAND0(); - assert((oparg & 1) == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - attr = PyStackRef_FromPyObjectNew(descr); - stack_pointer[0] = attr; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache1 = self; + _tos_cache0 = attr; + SET_CURRENT_CACHED_VALUES(2); break; } - case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: { + case _LOAD_ATTR_METHOD_LAZY_DICT_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef owner; _PyStackRef attr; + _PyStackRef self; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; PyObject *descr = (PyObject *)CURRENT_OPERAND0(); - assert((oparg & 1) == 0); - assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); + assert(oparg & 1); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); attr = PyStackRef_FromPyObjectNew(descr); - stack_pointer[0] = attr; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - break; - } - - case _CHECK_ATTR_METHOD_LAZY_DICT: { - _PyStackRef owner; - owner = stack_pointer[-1]; - uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0(); - char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; - PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); - if (dict != NULL) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + self = owner; + _tos_cache0 = self; + stack_pointer[-1] = attr; + SET_CURRENT_CACHED_VALUES(1); break; } - case _LOAD_ATTR_METHOD_LAZY_DICT: { + case _LOAD_ATTR_METHOD_LAZY_DICT_r23: { + CHECK_CURRENT_CACHED_VALUES(2); _PyStackRef owner; _PyStackRef attr; _PyStackRef self; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; oparg = CURRENT_OPARG(); - owner = stack_pointer[-1]; + owner = _stack_item_1; PyObject *descr = (PyObject *)CURRENT_OPERAND0(); assert(oparg & 1); STAT_INC(LOAD_ATTR, hit); @@ -4860,14 +9021,15 @@ assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); attr = PyStackRef_FromPyObjectNew(descr); self = owner; - stack_pointer[-1] = attr; - stack_pointer[0] = self; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache2 = self; + _tos_cache1 = attr; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); break; } - case _MAYBE_EXPAND_METHOD: { + case _MAYBE_EXPAND_METHOD_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); @@ -4888,6 +9050,10 @@ } stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } @@ -4895,7 +9061,8 @@ /* _MONITOR_CALL is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ - case _PY_FRAME_GENERAL: { + case _PY_FRAME_GENERAL_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -4922,16 +9089,19 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (temp == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } new_frame = PyStackRef_Wrap(temp); - stack_pointer[0] = new_frame; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = new_frame; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _CHECK_FUNCTION_VERSION: { + case _CHECK_FUNCTION_VERSION_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef callable; oparg = CURRENT_OPARG(); callable = stack_pointer[-2 - oparg]; @@ -4939,29 +9109,93 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyFunction_Check(callable_o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } PyFunctionObject *func = (PyFunctionObject *)callable_o; if (func->func_version != func_version) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } + SET_CURRENT_CACHED_VALUES(0); break; } - case _CHECK_FUNCTION_VERSION_INLINE: { + case _CHECK_FUNCTION_VERSION_INLINE_r00: { + CHECK_CURRENT_CACHED_VALUES(0); uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); PyObject *callable_o = (PyObject *)CURRENT_OPERAND1(); assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; if (func->func_version != func_version) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } + SET_CURRENT_CACHED_VALUES(0); break; } - case _CHECK_METHOD_VERSION: { + case _CHECK_FUNCTION_VERSION_INLINE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef _stack_item_0 = _tos_cache0; + uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); + PyObject *callable_o = (PyObject *)CURRENT_OPERAND1(); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _CHECK_FUNCTION_VERSION_INLINE_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); + PyObject *callable_o = (PyObject *)CURRENT_OPERAND1(); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _CHECK_FUNCTION_VERSION_INLINE_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); + PyObject *callable_o = (PyObject *)CURRENT_OPERAND1(); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _CHECK_METHOD_VERSION_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef null; _PyStackRef callable; oparg = CURRENT_OPARG(); @@ -4971,25 +9205,31 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (Py_TYPE(callable_o) != &PyMethod_Type) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } PyObject *func = ((PyMethodObject *)callable_o)->im_func; if (!PyFunction_Check(func)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } if (((PyFunctionObject *)func)->func_version != func_version) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } if (!PyStackRef_IsNull(null)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } + SET_CURRENT_CACHED_VALUES(0); break; } - case _EXPAND_METHOD: { + case _EXPAND_METHOD_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); @@ -5007,26 +9247,35 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(temp); stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } - case _CHECK_IS_NOT_PY_CALLABLE: { + case _CHECK_IS_NOT_PY_CALLABLE_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef callable; oparg = CURRENT_OPARG(); callable = stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (PyFunction_Check(callable_o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } if (Py_TYPE(callable_o) == &PyMethod_Type) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } + SET_CURRENT_CACHED_VALUES(0); break; } - case _CALL_NON_PY_GENERAL: { + case _CALL_NON_PY_GENERAL_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -5065,6 +9314,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } _PyFrame_SetStackPointer(frame, stack_pointer); @@ -5094,16 +9344,19 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { + case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef null; _PyStackRef callable; oparg = CURRENT_OPARG(); @@ -5111,16 +9364,20 @@ callable = stack_pointer[-2 - oparg]; if (!PyStackRef_IsNull(null)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } if (Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } + SET_CURRENT_CACHED_VALUES(0); break; } - case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: { + case _INIT_CALL_BOUND_METHOD_EXACT_ARGS_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); @@ -5137,18 +9394,71 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(temp); stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _CHECK_PEP_523_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + if (tstate->interp->eval_frame) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _CHECK_PEP_523_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef _stack_item_0 = _tos_cache0; + if (tstate->interp->eval_frame) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _CHECK_PEP_523_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + if (tstate->interp->eval_frame) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); break; } - case _CHECK_PEP_523: { + case _CHECK_PEP_523_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; if (tstate->interp->eval_frame) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); break; } - case _CHECK_FUNCTION_EXACT_ARGS: { + case _CHECK_FUNCTION_EXACT_ARGS_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); @@ -5160,12 +9470,15 @@ PyCodeObject *code = (PyCodeObject *)func->func_code; if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } + SET_CURRENT_CACHED_VALUES(0); break; } - case _CHECK_STACK_SPACE: { + case _CHECK_STACK_SPACE_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef callable; oparg = CURRENT_OPARG(); callable = stack_pointer[-2 - oparg]; @@ -5174,20 +9487,71 @@ PyCodeObject *code = (PyCodeObject *)func->func_code; if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _CHECK_RECURSION_REMAINING_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + if (tstate->py_recursion_remaining <= 1) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } + SET_CURRENT_CACHED_VALUES(0); break; } - case _CHECK_RECURSION_REMAINING: { + case _CHECK_RECURSION_REMAINING_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef _stack_item_0 = _tos_cache0; if (tstate->py_recursion_remaining <= 1) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); break; } - case _INIT_CALL_PY_EXACT_ARGS_0: { + case _CHECK_RECURSION_REMAINING_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + if (tstate->py_recursion_remaining <= 1) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _CHECK_RECURSION_REMAINING_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + if (tstate->py_recursion_remaining <= 1) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _INIT_CALL_PY_EXACT_ARGS_0_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -5206,13 +9570,17 @@ first_non_self_local[i] = args[i]; } new_frame = PyStackRef_Wrap(pushed_frame); - stack_pointer[-2 - oparg] = new_frame; - stack_pointer += -1 - oparg; + _tos_cache0 = new_frame; + stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _INIT_CALL_PY_EXACT_ARGS_1: { + case _INIT_CALL_PY_EXACT_ARGS_1_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -5231,13 +9599,17 @@ first_non_self_local[i] = args[i]; } new_frame = PyStackRef_Wrap(pushed_frame); - stack_pointer[-2 - oparg] = new_frame; - stack_pointer += -1 - oparg; + _tos_cache0 = new_frame; + stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _INIT_CALL_PY_EXACT_ARGS_2: { + case _INIT_CALL_PY_EXACT_ARGS_2_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -5256,13 +9628,17 @@ first_non_self_local[i] = args[i]; } new_frame = PyStackRef_Wrap(pushed_frame); - stack_pointer[-2 - oparg] = new_frame; - stack_pointer += -1 - oparg; + _tos_cache0 = new_frame; + stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _INIT_CALL_PY_EXACT_ARGS_3: { + case _INIT_CALL_PY_EXACT_ARGS_3_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -5281,13 +9657,17 @@ first_non_self_local[i] = args[i]; } new_frame = PyStackRef_Wrap(pushed_frame); - stack_pointer[-2 - oparg] = new_frame; - stack_pointer += -1 - oparg; + _tos_cache0 = new_frame; + stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _INIT_CALL_PY_EXACT_ARGS_4: { + case _INIT_CALL_PY_EXACT_ARGS_4_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -5306,13 +9686,17 @@ first_non_self_local[i] = args[i]; } new_frame = PyStackRef_Wrap(pushed_frame); - stack_pointer[-2 - oparg] = new_frame; - stack_pointer += -1 - oparg; + _tos_cache0 = new_frame; + stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _INIT_CALL_PY_EXACT_ARGS: { + case _INIT_CALL_PY_EXACT_ARGS_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -5330,19 +9714,22 @@ first_non_self_local[i] = args[i]; } new_frame = PyStackRef_Wrap(pushed_frame); - stack_pointer[-2 - oparg] = new_frame; - stack_pointer += -1 - oparg; + _tos_cache0 = new_frame; + stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _PUSH_FRAME: { + case _PUSH_FRAME_r10: { + CHECK_CURRENT_CACHED_VALUES(1); _PyStackRef new_frame; - new_frame = stack_pointer[-1]; + _PyStackRef _stack_item_0 = _tos_cache0; + new_frame = _stack_item_0; assert(tstate->interp->eval_frame == NULL); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); @@ -5351,87 +9738,240 @@ LOAD_SP(); LOAD_IP(0); LLTRACE_RESUME_FRAME(); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _GUARD_NOS_NULL_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef null; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + null = _stack_item_0; + if (!PyStackRef_IsNull(null)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = null; + SET_CURRENT_CACHED_VALUES(2); break; } - case _GUARD_NOS_NULL: { + case _GUARD_NOS_NULL_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef null; null = stack_pointer[-2]; if (!PyStackRef_IsNull(null)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _GUARD_NOS_NULL_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef null; + _PyStackRef _stack_item_0 = _tos_cache0; + null = stack_pointer[-1]; + if (!PyStackRef_IsNull(null)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _GUARD_NOS_NULL_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef null; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + null = _stack_item_1; + if (!PyStackRef_IsNull(null)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = null; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _GUARD_NOS_NOT_NULL_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + nos = _stack_item_0; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (o == NULL) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } + _tos_cache1 = _stack_item_1; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); break; } - case _GUARD_NOS_NOT_NULL: { + case _GUARD_NOS_NOT_NULL_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef nos; nos = stack_pointer[-2]; PyObject *o = PyStackRef_AsPyObjectBorrow(nos); if (o == NULL) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _GUARD_NOS_NOT_NULL_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + nos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (o == NULL) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _GUARD_NOS_NOT_NULL_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + nos = _stack_item_1; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (o == NULL) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } + _tos_cache2 = _stack_item_2; + _tos_cache1 = nos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); break; } - case _GUARD_THIRD_NULL: { + case _GUARD_THIRD_NULL_r33: { + CHECK_CURRENT_CACHED_VALUES(3); _PyStackRef null; - null = stack_pointer[-3]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + null = _stack_item_0; if (!PyStackRef_IsNull(null)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = null; + SET_CURRENT_CACHED_VALUES(3); break; } - case _GUARD_CALLABLE_TYPE_1: { + case _GUARD_CALLABLE_TYPE_1_r33: { + CHECK_CURRENT_CACHED_VALUES(3); _PyStackRef callable; - callable = stack_pointer[-3]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + callable = _stack_item_0; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (callable_o != (PyObject *)&PyType_Type) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = callable; + SET_CURRENT_CACHED_VALUES(3); break; } - case _CALL_TYPE_1: { + case _CALL_TYPE_1_r31: { + CHECK_CURRENT_CACHED_VALUES(3); _PyStackRef arg; _PyStackRef null; _PyStackRef callable; _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; oparg = CURRENT_OPARG(); - arg = stack_pointer[-1]; - null = stack_pointer[-2]; - callable = stack_pointer[-3]; + arg = _stack_item_2; + null = _stack_item_1; + callable = _stack_item_0; PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); (void)callable; (void)null; STAT_INC(CALL, hit); res = PyStackRef_FromPyObjectNew(Py_TYPE(arg_o)); - stack_pointer[-3] = res; - stack_pointer += -2; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(arg); stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _GUARD_CALLABLE_STR_1: { + case _GUARD_CALLABLE_STR_1_r33: { + CHECK_CURRENT_CACHED_VALUES(3); _PyStackRef callable; - callable = stack_pointer[-3]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + callable = _stack_item_0; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (callable_o != (PyObject *)&PyUnicode_Type) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = callable; + SET_CURRENT_CACHED_VALUES(3); break; } - case _CALL_STR_1: { + case _CALL_STR_1_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef arg; _PyStackRef null; _PyStackRef callable; @@ -5454,27 +9994,39 @@ PyStackRef_CLOSE(arg); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _GUARD_CALLABLE_TUPLE_1: { + case _GUARD_CALLABLE_TUPLE_1_r33: { + CHECK_CURRENT_CACHED_VALUES(3); _PyStackRef callable; - callable = stack_pointer[-3]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + callable = _stack_item_0; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (callable_o != (PyObject *)&PyTuple_Type) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = callable; + SET_CURRENT_CACHED_VALUES(3); break; } - case _CALL_TUPLE_1: { + case _CALL_TUPLE_1_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef arg; _PyStackRef null; _PyStackRef callable; @@ -5497,16 +10049,19 @@ PyStackRef_CLOSE(arg); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _CHECK_AND_ALLOCATE_OBJECT: { + case _CHECK_AND_ALLOCATE_OBJECT_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); @@ -5516,15 +10071,18 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyStackRef_IsNull(self_or_null)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } if (!PyType_Check(callable_o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } PyTypeObject *tp = (PyTypeObject *)callable_o; if (FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } assert(tp->tp_new == PyBaseObject_Type.tp_new); @@ -5535,6 +10093,7 @@ PyCodeObject *code = (PyCodeObject *)init_func->func_code; if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); @@ -5542,6 +10101,7 @@ PyObject *self_o = PyType_GenericAlloc(tp, 0); stack_pointer = _PyFrame_GetStackPointer(frame); if (self_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } self_or_null = PyStackRef_FromPyObjectSteal(self_o); @@ -5552,10 +10112,15 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(temp); stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } - case _CREATE_INIT_FRAME: { + case _CREATE_INIT_FRAME_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef *args; _PyStackRef self; _PyStackRef init; @@ -5581,18 +10146,21 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FrameClearAndPop(tstate, shim); stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; tstate->py_recursion_remaining--; init_frame = PyStackRef_Wrap(temp); - stack_pointer[0] = init_frame; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = init_frame; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _EXIT_INIT_CHECK: { + case _EXIT_INIT_CHECK_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef should_be_none; should_be_none = stack_pointer[-1]; if (!PyStackRef_IsNone(should_be_none)) { @@ -5601,14 +10169,20 @@ "__init__() should return None, not '%.200s'", Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } - case _CALL_BUILTIN_CLASS: { + case _CALL_BUILTIN_CLASS_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -5620,6 +10194,7 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyType_Check(callable_o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } PyTypeObject *tp = (PyTypeObject *)callable_o; @@ -5631,6 +10206,7 @@ } if (tp->tp_vectorcall == NULL) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); @@ -5654,6 +10230,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } _PyFrame_SetStackPointer(frame, stack_pointer); @@ -5679,16 +10256,19 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _CALL_BUILTIN_O: { + case _CALL_BUILTIN_O_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -5705,18 +10285,22 @@ } if (total_args != 1) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } if (!PyCFunction_CheckExact(callable_o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } if (PyCFunction_GET_FLAGS(callable_o) != METH_O) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } if (_Py_ReachedRecursionLimit(tstate)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); @@ -5736,16 +10320,19 @@ PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _CALL_BUILTIN_FAST: { + case _CALL_BUILTIN_FAST_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -5763,10 +10350,12 @@ } if (!PyCFunction_CheckExact(callable_o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } if (PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); @@ -5791,6 +10380,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } _PyFrame_SetStackPointer(frame, stack_pointer); @@ -5820,16 +10410,19 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _CALL_BUILTIN_FAST_WITH_KEYWORDS: { + case _CALL_BUILTIN_FAST_WITH_KEYWORDS_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -5847,10 +10440,12 @@ } if (!PyCFunction_CheckExact(callable_o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } if (PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); @@ -5878,6 +10473,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } _PyFrame_SetStackPointer(frame, stack_pointer); @@ -5904,28 +10500,40 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _GUARD_CALLABLE_LEN: { + case _GUARD_CALLABLE_LEN_r33: { + CHECK_CURRENT_CACHED_VALUES(3); _PyStackRef callable; - callable = stack_pointer[-3]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + callable = _stack_item_0; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyInterpreterState *interp = tstate->interp; if (callable_o != interp->callable_cache.len) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = callable; + SET_CURRENT_CACHED_VALUES(3); break; } - case _CALL_LEN: { + case _CALL_LEN_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef arg; _PyStackRef null; _PyStackRef callable; @@ -5940,11 +10548,13 @@ Py_ssize_t len_i = PyObject_Length(arg_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (len_i < 0) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } PyObject *res_o = PyLong_FromSsize_t(len_i); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } stack_pointer += -1; @@ -5958,25 +10568,36 @@ PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _GUARD_CALLABLE_ISINSTANCE: { + case _GUARD_CALLABLE_ISINSTANCE_r33: { + CHECK_CURRENT_CACHED_VALUES(3); _PyStackRef callable; - callable = stack_pointer[-4]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + callable = stack_pointer[-1]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyInterpreterState *interp = tstate->interp; if (callable_o != interp->callable_cache.isinstance) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); break; } - case _CALL_ISINSTANCE: { + case _CALL_ISINSTANCE_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef cls; _PyStackRef instance; _PyStackRef null; @@ -5993,6 +10614,7 @@ int retval = PyObject_IsInstance(inst_o, cls_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (retval < 0) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } (void)null; @@ -6013,46 +10635,63 @@ stack_pointer = _PyFrame_GetStackPointer(frame); res = retval ? PyStackRef_True : PyStackRef_False; assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _GUARD_CALLABLE_LIST_APPEND: { + case _GUARD_CALLABLE_LIST_APPEND_r33: { + CHECK_CURRENT_CACHED_VALUES(3); _PyStackRef callable; - callable = stack_pointer[-3]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + callable = _stack_item_0; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyInterpreterState *interp = tstate->interp; if (callable_o != interp->callable_cache.list_append) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = callable; + SET_CURRENT_CACHED_VALUES(3); break; } - case _CALL_LIST_APPEND: { + case _CALL_LIST_APPEND_r30: { + CHECK_CURRENT_CACHED_VALUES(3); _PyStackRef arg; _PyStackRef self; _PyStackRef callable; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; oparg = CURRENT_OPARG(); - arg = stack_pointer[-1]; - self = stack_pointer[-2]; - callable = stack_pointer[-3]; + arg = _stack_item_2; + self = _stack_item_1; + callable = _stack_item_0; assert(oparg == 1); PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); if (!PyList_CheckExact(self_o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } if (!LOCK_OBJECT(self_o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); UNLOCK_OBJECT(self_o); - stack_pointer += -2; + stack_pointer[0] = callable; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(self); @@ -6063,6 +10702,7 @@ PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } #if TIER_ONE @@ -6070,10 +10710,15 @@ assert(next_instr->op.code == POP_TOP); SKIP_OVER(1); #endif + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } - case _CALL_METHOD_DESCRIPTOR_O: { + case _CALL_METHOD_DESCRIPTOR_O_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -6092,19 +10737,23 @@ PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (total_args != 2) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } PyMethodDef *meth = method->d_method; if (meth->ml_flags != METH_O) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } if (_Py_ReachedRecursionLimit(tstate)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } _PyStackRef arg_stackref = arguments[1]; @@ -6112,6 +10761,7 @@ if (!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), method->d_common.d_type)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); @@ -6142,16 +10792,19 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { + case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -6169,16 +10822,19 @@ } if (total_args == 0) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } PyMethodDef *meth = method->d_method; if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } PyTypeObject *d_type = method->d_common.d_type; @@ -6186,6 +10842,7 @@ assert(self != NULL); if (!Py_IS_TYPE(self, d_type)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); @@ -6210,6 +10867,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } _PyFrame_SetStackPointer(frame, stack_pointer); @@ -6238,16 +10896,19 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _CALL_METHOD_DESCRIPTOR_NOARGS: { + case _CALL_METHOD_DESCRIPTOR_NOARGS_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -6265,11 +10926,13 @@ } if (total_args != 1) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } PyMethodDef *meth = method->d_method; @@ -6277,14 +10940,17 @@ PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); if (!Py_IS_TYPE(self, method->d_common.d_type)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } if (meth->ml_flags != METH_NOARGS) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } if (_Py_ReachedRecursionLimit(tstate)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); @@ -6303,16 +10969,19 @@ PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _CALL_METHOD_DESCRIPTOR_FAST: { + case _CALL_METHOD_DESCRIPTOR_FAST_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -6330,22 +10999,26 @@ } if (total_args == 0) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } PyMethodDef *meth = method->d_method; if (meth->ml_flags != METH_FASTCALL) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); assert(self != NULL); if (!Py_IS_TYPE(self, method->d_common.d_type)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); @@ -6370,6 +11043,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } _PyFrame_SetStackPointer(frame, stack_pointer); @@ -6397,18 +11071,21 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } /* _MONITOR_CALL_KW is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ - case _MAYBE_EXPAND_METHOD_KW: { + case _MAYBE_EXPAND_METHOD_KW_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); @@ -6429,12 +11106,17 @@ } stack_pointer[-3 - oparg] = callable; stack_pointer[-2 - oparg] = self_or_null; + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } /* _DO_CALL_KW is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ - case _PY_FRAME_KW: { + case _PY_FRAME_KW_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef kwnames; _PyStackRef *args; _PyStackRef self_or_null; @@ -6471,62 +11153,79 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (temp == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } new_frame = PyStackRef_Wrap(temp); - stack_pointer[0] = new_frame; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = new_frame; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _CHECK_FUNCTION_VERSION_KW: { + case _CHECK_FUNCTION_VERSION_KW_r11: { + CHECK_CURRENT_CACHED_VALUES(1); _PyStackRef callable; + _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); - callable = stack_pointer[-3 - oparg]; + callable = stack_pointer[-2 - oparg]; uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyFunction_Check(callable_o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } PyFunctionObject *func = (PyFunctionObject *)callable_o; if (func->func_version != func_version) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); break; } - case _CHECK_METHOD_VERSION_KW: { + case _CHECK_METHOD_VERSION_KW_r11: { + CHECK_CURRENT_CACHED_VALUES(1); _PyStackRef null; _PyStackRef callable; + _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); - null = stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (Py_TYPE(callable_o) != &PyMethod_Type) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } PyObject *func = ((PyMethodObject *)callable_o)->im_func; if (!PyFunction_Check(func)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } if (((PyFunctionObject *)func)->func_version != func_version) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } if (!PyStackRef_IsNull(null)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); break; } - case _EXPAND_METHOD_KW: { + case _EXPAND_METHOD_KW_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); @@ -6544,26 +11243,37 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable_s); stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } - case _CHECK_IS_NOT_PY_CALLABLE_KW: { + case _CHECK_IS_NOT_PY_CALLABLE_KW_r11: { + CHECK_CURRENT_CACHED_VALUES(1); _PyStackRef callable; + _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); - callable = stack_pointer[-3 - oparg]; + callable = stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (PyFunction_Check(callable_o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } if (Py_TYPE(callable_o) == &PyMethod_Type) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); break; } - case _CALL_KW_NON_PY: { + case _CALL_KW_NON_PY_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef kwnames; _PyStackRef *args; _PyStackRef self_or_null; @@ -6607,6 +11317,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); @@ -6643,16 +11354,19 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _MAKE_CALLARGS_A_TUPLE: { + case _MAKE_CALLARGS_A_TUPLE_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef callargs; _PyStackRef func; callargs = stack_pointer[-2]; @@ -6663,12 +11377,14 @@ int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *tuple_o = PySequence_Tuple(callargs_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (tuple_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } _PyStackRef temp = callargs; @@ -6679,12 +11395,17 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } stack_pointer[-2] = callargs; + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } /* _DO_CALL_FUNCTION_EX is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ - case _MAKE_FUNCTION: { + case _MAKE_FUNCTION_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef codeobj_st; _PyStackRef func; codeobj_st = stack_pointer[-1]; @@ -6699,24 +11420,29 @@ PyStackRef_CLOSE(codeobj_st); stack_pointer = _PyFrame_GetStackPointer(frame); if (func_obj == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } _PyFunction_SetVersion( func_obj, ((PyCodeObject *)codeobj)->co_version); func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); - stack_pointer[0] = func; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = func; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _SET_FUNCTION_ATTRIBUTE: { + case _SET_FUNCTION_ATTRIBUTE_r21: { + CHECK_CURRENT_CACHED_VALUES(2); _PyStackRef func_in; _PyStackRef attr_st; _PyStackRef func_out; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; oparg = CURRENT_OPARG(); - func_in = stack_pointer[-1]; - attr_st = stack_pointer[-2]; + func_in = _stack_item_1; + attr_st = _stack_item_0; PyObject *func = PyStackRef_AsPyObjectBorrow(func_in); PyObject *attr = PyStackRef_AsPyObjectSteal(attr_st); func_out = func_in; @@ -6726,13 +11452,15 @@ PyObject **ptr = (PyObject **)(((char *)func) + offset); assert(*ptr == NULL); *ptr = attr; - stack_pointer[-2] = func_out; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = func_out; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _RETURN_GENERATOR: { + case _RETURN_GENERATOR_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef res; assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -6740,6 +11468,7 @@ PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); stack_pointer = _PyFrame_GetStackPointer(frame); if (gen == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } assert(STACK_LEVEL() == 0); @@ -6758,13 +11487,15 @@ stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); LLTRACE_RESUME_FRAME(); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _BUILD_SLICE: { + case _BUILD_SLICE_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef *args; _PyStackRef slice; oparg = CURRENT_OPARG(); @@ -6784,16 +11515,19 @@ stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); if (slice_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } slice = PyStackRef_FromPyObjectStealMortal(slice_o); - stack_pointer[0] = slice; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = slice; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _CONVERT_VALUE: { + case _CONVERT_VALUE_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef value; _PyStackRef result; oparg = CURRENT_OPARG(); @@ -6810,16 +11544,19 @@ PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); if (result_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } result = PyStackRef_FromPyObjectSteal(result_o); - stack_pointer[0] = result; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = result; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _FORMAT_SIMPLE: { + case _FORMAT_SIMPLE_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef value; _PyStackRef res; value = stack_pointer[-1]; @@ -6834,6 +11571,7 @@ PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); @@ -6842,13 +11580,15 @@ res = value; stack_pointer += -1; } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _FORMAT_WITH_SPEC: { + case _FORMAT_WITH_SPEC_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef fmt_spec; _PyStackRef value; _PyStackRef res; @@ -6868,61 +11608,104 @@ stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _COPY_1: { + case _COPY_1_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef bottom; + _PyStackRef top; + _PyStackRef _stack_item_0 = _tos_cache0; + bottom = _stack_item_0; + top = PyStackRef_DUP(bottom); + _tos_cache1 = top; + _tos_cache0 = bottom; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _COPY_1_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef bottom; _PyStackRef top; bottom = stack_pointer[-1]; top = PyStackRef_DUP(bottom); - stack_pointer[0] = top; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = top; + SET_CURRENT_CACHED_VALUES(1); break; } - case _COPY_2: { + case _COPY_1_r23: { + CHECK_CURRENT_CACHED_VALUES(2); _PyStackRef bottom; _PyStackRef top; - bottom = stack_pointer[-2]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + bottom = _stack_item_1; top = PyStackRef_DUP(bottom); - stack_pointer[0] = top; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache2 = top; + _tos_cache1 = bottom; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _COPY_2_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef bottom; + _PyStackRef top; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + bottom = _stack_item_0; + top = PyStackRef_DUP(bottom); + _tos_cache2 = top; + _tos_cache1 = _stack_item_1; + _tos_cache0 = bottom; + SET_CURRENT_CACHED_VALUES(3); break; } - case _COPY_3: { + case _COPY_3_r33: { + CHECK_CURRENT_CACHED_VALUES(3); _PyStackRef bottom; _PyStackRef top; - bottom = stack_pointer[-3]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + bottom = _stack_item_0; top = PyStackRef_DUP(bottom); - stack_pointer[0] = top; + _tos_cache2 = top; + _tos_cache1 = _stack_item_2; + _tos_cache0 = _stack_item_1; + stack_pointer[0] = bottom; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(3); break; } - case _COPY: { + case _COPY_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef bottom; _PyStackRef top; oparg = CURRENT_OPARG(); bottom = stack_pointer[-1 - (oparg-1)]; top = PyStackRef_DUP(bottom); - stack_pointer[0] = top; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = top; + SET_CURRENT_CACHED_VALUES(1); break; } - case _BINARY_OP: { + case _BINARY_OP_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef rhs; _PyStackRef lhs; _PyStackRef res; @@ -6936,6 +11719,7 @@ PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); @@ -6951,10 +11735,34 @@ stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _SWAP_2_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef top; + _PyStackRef bottom; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + top = _stack_item_1; + bottom = _stack_item_0; + _PyStackRef temp = bottom; + bottom = top; + top = temp; + _tos_cache1 = top; + _tos_cache0 = bottom; + SET_CURRENT_CACHED_VALUES(2); break; } - case _SWAP_2: { + case _SWAP_2_r00: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef top; _PyStackRef bottom; top = stack_pointer[-1]; @@ -6964,33 +11772,78 @@ top = temp; stack_pointer[-2] = bottom; stack_pointer[-1] = top; + SET_CURRENT_CACHED_VALUES(0); break; } - case _SWAP_3: { + case _SWAP_2_r11: { + CHECK_CURRENT_CACHED_VALUES(1); _PyStackRef top; _PyStackRef bottom; - top = stack_pointer[-1]; - bottom = stack_pointer[-3]; + _PyStackRef _stack_item_0 = _tos_cache0; + top = _stack_item_0; + bottom = stack_pointer[-1]; _PyStackRef temp = bottom; bottom = top; top = temp; - stack_pointer[-3] = bottom; - stack_pointer[-1] = top; + _tos_cache0 = top; + stack_pointer[-1] = bottom; + SET_CURRENT_CACHED_VALUES(1); break; } - case _SWAP: { + case _SWAP_2_r33: { + CHECK_CURRENT_CACHED_VALUES(3); _PyStackRef top; _PyStackRef bottom; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + top = _stack_item_2; + bottom = _stack_item_1; + _PyStackRef temp = bottom; + bottom = top; + top = temp; + _tos_cache2 = top; + _tos_cache1 = bottom; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _SWAP_3_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef top; + _PyStackRef bottom; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + top = _stack_item_2; + bottom = _stack_item_0; + _PyStackRef temp = bottom; + bottom = top; + top = temp; + _tos_cache2 = top; + _tos_cache1 = _stack_item_1; + _tos_cache0 = bottom; + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _SWAP_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef top; + _PyStackRef bottom; + _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); - top = stack_pointer[-1]; - bottom = stack_pointer[-2 - (oparg-2)]; + top = _stack_item_0; + bottom = stack_pointer[-1 - (oparg-2)]; _PyStackRef temp = bottom; bottom = top; top = temp; - stack_pointer[-2 - (oparg-2)] = bottom; - stack_pointer[-1] = top; + _tos_cache0 = top; + stack_pointer[-1 - (oparg-2)] = bottom; + SET_CURRENT_CACHED_VALUES(1); break; } @@ -7012,105 +11865,499 @@ /* _INSTRUMENTED_POP_JUMP_IF_NOT_NONE is not a viable micro-op for tier 2 because it is instrumented */ - case _GUARD_IS_TRUE_POP: { + case _GUARD_IS_TRUE_POP_r10: { + CHECK_CURRENT_CACHED_VALUES(1); _PyStackRef flag; - flag = stack_pointer[-1]; + _PyStackRef _stack_item_0 = _tos_cache0; + flag = _stack_item_0; int is_true = PyStackRef_IsTrue(flag); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); if (!is_true) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } + SET_CURRENT_CACHED_VALUES(0); break; } - case _GUARD_IS_FALSE_POP: { + case _GUARD_IS_FALSE_POP_r10: { + CHECK_CURRENT_CACHED_VALUES(1); _PyStackRef flag; - flag = stack_pointer[-1]; + _PyStackRef _stack_item_0 = _tos_cache0; + flag = _stack_item_0; int is_false = PyStackRef_IsFalse(flag); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); if (!is_false) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _GUARD_IS_NONE_POP_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef val; + _PyStackRef _stack_item_0 = _tos_cache0; + val = _stack_item_0; + int is_none = PyStackRef_IsNone(val); + if (!is_none) { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(val); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (1) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _GUARD_IS_NOT_NONE_POP_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef val; + _PyStackRef _stack_item_0 = _tos_cache0; + val = _stack_item_0; + int is_none = PyStackRef_IsNone(val); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(val); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (is_none) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _JUMP_TO_TOP_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _SET_IP_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0(); + frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _SET_IP_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef _stack_item_0 = _tos_cache0; + PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0(); + frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _SET_IP_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0(); + frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _SET_IP_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0(); + frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _CHECK_STACK_SPACE_OPERAND_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + uint32_t framesize = (uint32_t)CURRENT_OPERAND0(); + assert(framesize <= INT_MAX); + if (!_PyThreadState_HasStackSpace(tstate, framesize)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + if (tstate->py_recursion_remaining <= 1) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _CHECK_STACK_SPACE_OPERAND_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef _stack_item_0 = _tos_cache0; + uint32_t framesize = (uint32_t)CURRENT_OPERAND0(); + assert(framesize <= INT_MAX); + if (!_PyThreadState_HasStackSpace(tstate, framesize)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + if (tstate->py_recursion_remaining <= 1) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _CHECK_STACK_SPACE_OPERAND_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + uint32_t framesize = (uint32_t)CURRENT_OPERAND0(); + assert(framesize <= INT_MAX); + if (!_PyThreadState_HasStackSpace(tstate, framesize)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + if (tstate->py_recursion_remaining <= 1) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _CHECK_STACK_SPACE_OPERAND_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + uint32_t framesize = (uint32_t)CURRENT_OPERAND0(); + assert(framesize <= INT_MAX); + if (!_PyThreadState_HasStackSpace(tstate, framesize)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } + if (tstate->py_recursion_remaining <= 1) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _SAVE_RETURN_OFFSET_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + oparg = CURRENT_OPARG(); + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _SAVE_RETURN_OFFSET_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _SAVE_RETURN_OFFSET_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _SAVE_RETURN_OFFSET_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + oparg = CURRENT_OPARG(); + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _EXIT_TRACE_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); + _PyExitData *exit = (_PyExitData *)exit_p; + PyCodeObject *code = _PyFrame_GetCode(frame); + _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; + #if defined(Py_DEBUG) && !defined(_Py_JIT) + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + if (frame->lltrace >= 2) { + _PyFrame_SetStackPointer(frame, stack_pointer); + printf("SIDE EXIT: [UOp "); + _PyUOpPrint(&next_uop[-1]); + printf(", exit %lu, temp %d, target %d -> %s]\n", + exit - current_executor->exits, exit->temperature.value_and_backoff, + (int)(target - _PyFrame_GetBytecode(frame)), + _PyOpcode_OpName[target->op.code]); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + #endif + if (exit->executor && !exit->executor->vm_data.valid) { + exit->temperature = initial_temperature_backoff_counter(); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(exit->executor); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + if (exit->executor == NULL) { + _Py_BackoffCounter temperature = exit->temperature; + if (!backoff_counter_triggers(temperature)) { + exit->temperature = advance_backoff_counter(temperature); + GOTO_TIER_ONE(target); + Py_UNREACHABLE();} + _PyExecutorObject *executor; + if (target->op.code == ENTER_EXECUTOR) { + executor = code->co_executors->executors[target->op.arg]; + Py_INCREF(executor); + } + else { + int chain_depth = current_executor->vm_data.chain_depth + 1; + _PyFrame_SetStackPointer(frame, stack_pointer); + int optimized = _PyOptimizer_Optimize(frame, target, &executor, chain_depth); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (optimized <= 0) { + exit->temperature = restart_backoff_counter(temperature); + GOTO_TIER_ONE(optimized < 0 ? NULL : target); + Py_UNREACHABLE();} + exit->temperature = initial_temperature_backoff_counter(); + } + exit->executor = executor; + } + GOTO_TIER_TWO(exit->executor); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } - case _GUARD_IS_NONE_POP: { - _PyStackRef val; - val = stack_pointer[-1]; - int is_none = PyStackRef_IsNone(val); - if (!is_none) { + case _EXIT_TRACE_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef _stack_item_0 = _tos_cache0; + PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); + _PyExitData *exit = (_PyExitData *)exit_p; + PyCodeObject *code = _PyFrame_GetCode(frame); + _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; + #if defined(Py_DEBUG) && !defined(_Py_JIT) + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + if (frame->lltrace >= 2) { + stack_pointer[0] = _stack_item_0; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + printf("SIDE EXIT: [UOp "); + _PyUOpPrint(&next_uop[-1]); + printf(", exit %lu, temp %d, target %d -> %s]\n", + exit - current_executor->exits, exit->temperature.value_and_backoff, + (int)(target - _PyFrame_GetBytecode(frame)), + _PyOpcode_OpName[target->op.code]); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; + } + #endif + if (exit->executor && !exit->executor->vm_data.valid) { + exit->temperature = initial_temperature_backoff_counter(); + stack_pointer[0] = _stack_item_0; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(val); + Py_CLEAR(exit->executor); stack_pointer = _PyFrame_GetStackPointer(frame); - if (1) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); + stack_pointer += -1; + } + if (exit->executor == NULL) { + _Py_BackoffCounter temperature = exit->temperature; + if (!backoff_counter_triggers(temperature)) { + exit->temperature = advance_backoff_counter(temperature); + stack_pointer[0] = _stack_item_0; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + GOTO_TIER_ONE(target); + Py_UNREACHABLE();} + _PyExecutorObject *executor; + if (target->op.code == ENTER_EXECUTOR) { + executor = code->co_executors->executors[target->op.arg]; + Py_INCREF(executor); + } + else { + int chain_depth = current_executor->vm_data.chain_depth + 1; + stack_pointer[0] = _stack_item_0; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int optimized = _PyOptimizer_Optimize(frame, target, &executor, chain_depth); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (optimized <= 0) { + exit->temperature = restart_backoff_counter(temperature); + GOTO_TIER_ONE(optimized < 0 ? NULL : target); + Py_UNREACHABLE();} + exit->temperature = initial_temperature_backoff_counter(); + stack_pointer += -1; } + exit->executor = executor; } - stack_pointer += -1; + stack_pointer[0] = _stack_item_0; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); + GOTO_TIER_TWO(exit->executor); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } - case _GUARD_IS_NOT_NONE_POP: { - _PyStackRef val; - val = stack_pointer[-1]; - int is_none = PyStackRef_IsNone(val); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(val); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (is_none) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); + case _EXIT_TRACE_r20: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); + _PyExitData *exit = (_PyExitData *)exit_p; + PyCodeObject *code = _PyFrame_GetCode(frame); + _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; + #if defined(Py_DEBUG) && !defined(_Py_JIT) + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + if (frame->lltrace >= 2) { + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + printf("SIDE EXIT: [UOp "); + _PyUOpPrint(&next_uop[-1]); + printf(", exit %lu, temp %d, target %d -> %s]\n", + exit - current_executor->exits, exit->temperature.value_and_backoff, + (int)(target - _PyFrame_GetBytecode(frame)), + _PyOpcode_OpName[target->op.code]); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; } - break; - } - - case _JUMP_TO_TOP: { - JUMP_TO_JUMP_TARGET(); - break; - } - - case _SET_IP: { - PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0(); - frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; - break; - } - - case _CHECK_STACK_SPACE_OPERAND: { - uint32_t framesize = (uint32_t)CURRENT_OPERAND0(); - assert(framesize <= INT_MAX); - if (!_PyThreadState_HasStackSpace(tstate, framesize)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); + #endif + if (exit->executor && !exit->executor->vm_data.valid) { + exit->temperature = initial_temperature_backoff_counter(); + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(exit->executor); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; } - if (tstate->py_recursion_remaining <= 1) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); + if (exit->executor == NULL) { + _Py_BackoffCounter temperature = exit->temperature; + if (!backoff_counter_triggers(temperature)) { + exit->temperature = advance_backoff_counter(temperature); + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + GOTO_TIER_ONE(target); + Py_UNREACHABLE();} + _PyExecutorObject *executor; + if (target->op.code == ENTER_EXECUTOR) { + executor = code->co_executors->executors[target->op.arg]; + Py_INCREF(executor); + } + else { + int chain_depth = current_executor->vm_data.chain_depth + 1; + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int optimized = _PyOptimizer_Optimize(frame, target, &executor, chain_depth); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (optimized <= 0) { + exit->temperature = restart_backoff_counter(temperature); + GOTO_TIER_ONE(optimized < 0 ? NULL : target); + Py_UNREACHABLE();} + exit->temperature = initial_temperature_backoff_counter(); + stack_pointer += -2; + } + exit->executor = executor; } + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + GOTO_TIER_TWO(exit->executor); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } - case _SAVE_RETURN_OFFSET: { - oparg = CURRENT_OPARG(); - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - break; - } - - case _EXIT_TRACE: { + case _EXIT_TRACE_r30: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); _PyExitData *exit = (_PyExitData *)exit_p; PyCodeObject *code = _PyFrame_GetCode(frame); @@ -7118,6 +12365,11 @@ #if defined(Py_DEBUG) && !defined(_Py_JIT) OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); if (frame->lltrace >= 2) { + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer[2] = _stack_item_2; + stack_pointer += 3; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); printf("SIDE EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); @@ -7126,20 +12378,32 @@ (int)(target - _PyFrame_GetBytecode(frame)), _PyOpcode_OpName[target->op.code]); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; } #endif if (exit->executor && !exit->executor->vm_data.valid) { exit->temperature = initial_temperature_backoff_counter(); + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer[2] = _stack_item_2; + stack_pointer += 3; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); Py_CLEAR(exit->executor); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; } if (exit->executor == NULL) { _Py_BackoffCounter temperature = exit->temperature; if (!backoff_counter_triggers(temperature)) { exit->temperature = advance_backoff_counter(temperature); + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer[2] = _stack_item_2; + stack_pointer += 3; + assert(WITHIN_STACK_BOUNDS()); GOTO_TIER_ONE(target); - } + Py_UNREACHABLE();} _PyExecutorObject *executor; if (target->op.code == ENTER_EXECUTOR) { executor = code->co_executors->executors[target->op.arg]; @@ -7147,88 +12411,215 @@ } else { int chain_depth = current_executor->vm_data.chain_depth + 1; + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer[2] = _stack_item_2; + stack_pointer += 3; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int optimized = _PyOptimizer_Optimize(frame, target, &executor, chain_depth); stack_pointer = _PyFrame_GetStackPointer(frame); if (optimized <= 0) { exit->temperature = restart_backoff_counter(temperature); GOTO_TIER_ONE(optimized < 0 ? NULL : target); - } + Py_UNREACHABLE();} exit->temperature = initial_temperature_backoff_counter(); + stack_pointer += -3; } exit->executor = executor; } + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer[2] = _stack_item_2; + stack_pointer += 3; + assert(WITHIN_STACK_BOUNDS()); GOTO_TIER_TWO(exit->executor); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _CHECK_VALIDITY_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + if (!current_executor->vm_data.valid) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _CHECK_VALIDITY_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef _stack_item_0 = _tos_cache0; + if (!current_executor->vm_data.valid) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _CHECK_VALIDITY_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + if (!current_executor->vm_data.valid) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); break; } - case _CHECK_VALIDITY: { + case _CHECK_VALIDITY_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; if (!current_executor->vm_data.valid) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); break; } - case _LOAD_CONST_INLINE: { + case _LOAD_CONST_INLINE_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef value; PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); value = PyStackRef_FromPyObjectNew(ptr); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _LOAD_CONST_INLINE_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + value = PyStackRef_FromPyObjectNew(ptr); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _LOAD_CONST_INLINE_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + value = PyStackRef_FromPyObjectNew(ptr); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); break; } - case _POP_TOP_LOAD_CONST_INLINE: { + case _POP_TOP_LOAD_CONST_INLINE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); _PyStackRef pop; _PyStackRef value; - pop = stack_pointer[-1]; + _PyStackRef _stack_item_0 = _tos_cache0; + pop = _stack_item_0; PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(pop); stack_pointer = _PyFrame_GetStackPointer(frame); value = PyStackRef_FromPyObjectNew(ptr); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = value; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _LOAD_CONST_INLINE_BORROW: { + case _LOAD_CONST_INLINE_BORROW_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef value; PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); value = PyStackRef_FromPyObjectBorrow(ptr); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _LOAD_CONST_INLINE_BORROW_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + value = PyStackRef_FromPyObjectBorrow(ptr); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _LOAD_CONST_INLINE_BORROW_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + value = PyStackRef_FromPyObjectBorrow(ptr); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); break; } - case _POP_CALL: { + case _POP_CALL_r20: { + CHECK_CURRENT_CACHED_VALUES(2); _PyStackRef null; _PyStackRef callable; - null = stack_pointer[-1]; - callable = stack_pointer[-2]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + null = _stack_item_1; + callable = _stack_item_0; (void)null; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } - case _POP_CALL_ONE: { + case _POP_CALL_ONE_r30: { + CHECK_CURRENT_CACHED_VALUES(3); _PyStackRef pop; _PyStackRef null; _PyStackRef callable; - pop = stack_pointer[-1]; - null = stack_pointer[-2]; - callable = stack_pointer[-3]; - stack_pointer += -1; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + pop = _stack_item_2; + null = _stack_item_1; + callable = _stack_item_0; + stack_pointer[0] = callable; + stack_pointer[1] = null; + stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(pop); @@ -7239,19 +12630,29 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } - case _POP_CALL_TWO: { + case _POP_CALL_TWO_r30: { + CHECK_CURRENT_CACHED_VALUES(3); _PyStackRef pop2; _PyStackRef pop1; _PyStackRef null; _PyStackRef callable; - pop2 = stack_pointer[-1]; - pop1 = stack_pointer[-2]; - null = stack_pointer[-3]; - callable = stack_pointer[-4]; - stack_pointer += -1; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + pop2 = _stack_item_2; + pop1 = _stack_item_1; + null = _stack_item_0; + callable = stack_pointer[-1]; + stack_pointer[0] = null; + stack_pointer[1] = pop1; + stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(pop2); @@ -7267,34 +12668,43 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); break; } - case _POP_TOP_LOAD_CONST_INLINE_BORROW: { + case _POP_TOP_LOAD_CONST_INLINE_BORROW_r11: { + CHECK_CURRENT_CACHED_VALUES(1); _PyStackRef pop; _PyStackRef value; - pop = stack_pointer[-1]; + _PyStackRef _stack_item_0 = _tos_cache0; + pop = _stack_item_0; PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(pop); stack_pointer = _PyFrame_GetStackPointer(frame); - value = PyStackRef_FromPyObjectBorrow(ptr); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + value = PyStackRef_FromPyObjectBorrow(ptr); + _tos_cache0 = value; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _POP_TWO_LOAD_CONST_INLINE_BORROW: { + case _POP_TWO_LOAD_CONST_INLINE_BORROW_r21: { + CHECK_CURRENT_CACHED_VALUES(2); _PyStackRef pop2; _PyStackRef pop1; _PyStackRef value; - pop2 = stack_pointer[-1]; - pop1 = stack_pointer[-2]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + pop2 = _stack_item_1; + pop1 = _stack_item_0; PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); - stack_pointer += -1; + stack_pointer[0] = pop1; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(pop2); @@ -7305,42 +12715,51 @@ PyStackRef_CLOSE(pop1); stack_pointer = _PyFrame_GetStackPointer(frame); value = PyStackRef_FromPyObjectBorrow(ptr); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = value; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _POP_CALL_LOAD_CONST_INLINE_BORROW: { + case _POP_CALL_LOAD_CONST_INLINE_BORROW_r21: { + CHECK_CURRENT_CACHED_VALUES(2); _PyStackRef null; _PyStackRef callable; _PyStackRef value; - null = stack_pointer[-1]; - callable = stack_pointer[-2]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + null = _stack_item_1; + callable = _stack_item_0; PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); (void)null; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); value = PyStackRef_FromPyObjectBorrow(ptr); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = value; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW: { + case _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW_r31: { + CHECK_CURRENT_CACHED_VALUES(3); _PyStackRef pop; _PyStackRef null; _PyStackRef callable; _PyStackRef value; - pop = stack_pointer[-1]; - null = stack_pointer[-2]; - callable = stack_pointer[-3]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + pop = _stack_item_2; + null = _stack_item_1; + callable = _stack_item_0; PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); - stack_pointer += -1; + stack_pointer[0] = callable; + stack_pointer[1] = null; + stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(pop); @@ -7352,24 +12771,31 @@ PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); value = PyStackRef_FromPyObjectBorrow(ptr); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = value; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); break; } - case _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW: { + case _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW_r31: { + CHECK_CURRENT_CACHED_VALUES(3); _PyStackRef pop2; _PyStackRef pop1; _PyStackRef null; _PyStackRef callable; _PyStackRef value; - pop2 = stack_pointer[-1]; - pop1 = stack_pointer[-2]; - null = stack_pointer[-3]; - callable = stack_pointer[-4]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + pop2 = _stack_item_2; + pop1 = _stack_item_1; + null = _stack_item_0; + callable = stack_pointer[-1]; PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); - stack_pointer += -1; + stack_pointer[0] = null; + stack_pointer[1] = pop1; + stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(pop2); @@ -7386,13 +12812,31 @@ PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); value = PyStackRef_FromPyObjectBorrow(ptr); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = value; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _LOAD_CONST_UNDER_INLINE_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef old; + _PyStackRef value; + _PyStackRef new; + _PyStackRef _stack_item_0 = _tos_cache0; + old = _stack_item_0; + PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + new = old; + value = PyStackRef_FromPyObjectNew(ptr); + _tos_cache1 = new; + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(2); break; } - case _LOAD_CONST_UNDER_INLINE: { + case _LOAD_CONST_UNDER_INLINE_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef old; _PyStackRef value; _PyStackRef new; @@ -7400,14 +12844,48 @@ PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); new = old; value = PyStackRef_FromPyObjectNew(ptr); + _tos_cache0 = new; stack_pointer[-1] = value; - stack_pointer[0] = new; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _LOAD_CONST_UNDER_INLINE_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef old; + _PyStackRef value; + _PyStackRef new; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + old = _stack_item_1; + PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + new = old; + value = PyStackRef_FromPyObjectNew(ptr); + _tos_cache2 = new; + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _LOAD_CONST_UNDER_INLINE_BORROW_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef old; + _PyStackRef value; + _PyStackRef new; + _PyStackRef _stack_item_0 = _tos_cache0; + old = _stack_item_0; + PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + new = old; + value = PyStackRef_FromPyObjectBorrow(ptr); + _tos_cache1 = new; + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(2); break; } - case _LOAD_CONST_UNDER_INLINE_BORROW: { + case _LOAD_CONST_UNDER_INLINE_BORROW_r01: { + CHECK_CURRENT_CACHED_VALUES(0); _PyStackRef old; _PyStackRef value; _PyStackRef new; @@ -7415,65 +12893,340 @@ PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); new = old; value = PyStackRef_FromPyObjectBorrow(ptr); + _tos_cache0 = new; stack_pointer[-1] = value; - stack_pointer[0] = new; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _LOAD_CONST_UNDER_INLINE_BORROW_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef old; + _PyStackRef value; + _PyStackRef new; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + old = _stack_item_1; + PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + new = old; + value = PyStackRef_FromPyObjectBorrow(ptr); + _tos_cache2 = new; + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _CHECK_FUNCTION_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + if (func->func_version != func_version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _CHECK_FUNCTION_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef _stack_item_0 = _tos_cache0; + uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + if (func->func_version != func_version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _CHECK_FUNCTION_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + if (func->func_version != func_version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); break; } - case _CHECK_FUNCTION: { + case _CHECK_FUNCTION_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); if (func->func_version != func_version) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); break; } - case _START_EXECUTOR: { + case _START_EXECUTOR_r00: { + CHECK_CURRENT_CACHED_VALUES(0); PyObject *executor = (PyObject *)CURRENT_OPERAND0(); #ifndef _Py_JIT current_executor = (_PyExecutorObject*)executor; #endif assert(((_PyExecutorObject *)executor)->vm_data.valid); + SET_CURRENT_CACHED_VALUES(0); break; } - case _MAKE_WARM: { + case _MAKE_WARM_r00: { + CHECK_CURRENT_CACHED_VALUES(0); current_executor->vm_data.warm = true; if (--tstate->interp->trace_run_counter == 0) { _Py_set_eval_breaker_bit(tstate, _PY_EVAL_JIT_INVALIDATE_COLD_BIT); } + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _MAKE_WARM_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef _stack_item_0 = _tos_cache0; + current_executor->vm_data.warm = true; + if (--tstate->interp->trace_run_counter == 0) { + _Py_set_eval_breaker_bit(tstate, _PY_EVAL_JIT_INVALIDATE_COLD_BIT); + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _MAKE_WARM_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + current_executor->vm_data.warm = true; + if (--tstate->interp->trace_run_counter == 0) { + _Py_set_eval_breaker_bit(tstate, _PY_EVAL_JIT_INVALIDATE_COLD_BIT); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _MAKE_WARM_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + current_executor->vm_data.warm = true; + if (--tstate->interp->trace_run_counter == 0) { + _Py_set_eval_breaker_bit(tstate, _PY_EVAL_JIT_INVALIDATE_COLD_BIT); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _FATAL_ERROR_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(0); + Py_FatalError("Fatal error uop executed."); + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _FATAL_ERROR_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef _stack_item_0 = _tos_cache0; + assert(0); + Py_FatalError("Fatal error uop executed."); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _FATAL_ERROR_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + assert(0); + Py_FatalError("Fatal error uop executed."); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); break; } - case _FATAL_ERROR: { + case _FATAL_ERROR_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; assert(0); Py_FatalError("Fatal error uop executed."); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _DEOPT_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + GOTO_TIER_ONE(_PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _DEOPT_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef _stack_item_0 = _tos_cache0; + stack_pointer[0] = _stack_item_0; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + GOTO_TIER_ONE(_PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _DEOPT_r20: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + GOTO_TIER_ONE(_PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + SET_CURRENT_CACHED_VALUES(0); break; } - case _DEOPT: { + case _DEOPT_r30: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer[2] = _stack_item_2; + stack_pointer += 3; + assert(WITHIN_STACK_BOUNDS()); GOTO_TIER_ONE(_PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + SET_CURRENT_CACHED_VALUES(0); break; } - case _ERROR_POP_N: { + case _ERROR_POP_N_r00: { + CHECK_CURRENT_CACHED_VALUES(0); oparg = CURRENT_OPARG(); uint32_t target = (uint32_t)CURRENT_OPERAND0(); assert(oparg == 0); frame->instr_ptr = _PyFrame_GetBytecode(frame) + target; GOTO_TIER_ONE(NULL); + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _TIER2_RESUME_CHECK_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + #if defined(__EMSCRIPTEN__) + if (_Py_emscripten_signal_clock == 0) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + #endif + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + if (eval_breaker & _PY_EVAL_EVENTS_MASK) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + assert(tstate->tracing || eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version)); + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _TIER2_RESUME_CHECK_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef _stack_item_0 = _tos_cache0; + #if defined(__EMSCRIPTEN__) + if (_Py_emscripten_signal_clock == 0) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + #endif + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + if (eval_breaker & _PY_EVAL_EVENTS_MASK) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + assert(tstate->tracing || eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version)); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _TIER2_RESUME_CHECK_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + #if defined(__EMSCRIPTEN__) + if (_Py_emscripten_signal_clock == 0) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + #endif + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + if (eval_breaker & _PY_EVAL_EVENTS_MASK) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + assert(tstate->tracing || eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version)); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); break; } - case _TIER2_RESUME_CHECK: { + case _TIER2_RESUME_CHECK_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; #if defined(__EMSCRIPTEN__) if (_Py_emscripten_signal_clock == 0) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; @@ -7481,9 +13234,156 @@ uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); if (eval_breaker & _PY_EVAL_EVENTS_MASK) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } assert(tstate->tracing || eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version)); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _SPILL_OR_RELOAD_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + _tos_cache0 = stack_pointer[-1]; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _SPILL_OR_RELOAD_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + _tos_cache1 = stack_pointer[-1]; + _tos_cache0 = stack_pointer[-2]; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _SPILL_OR_RELOAD_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + _tos_cache2 = stack_pointer[-1]; + _tos_cache1 = stack_pointer[-2]; + _tos_cache0 = stack_pointer[-3]; + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _SPILL_OR_RELOAD_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef _stack_item_0 = _tos_cache0; + stack_pointer[0] = _stack_item_0; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _SPILL_OR_RELOAD_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef _stack_item_0 = _tos_cache0; + _tos_cache1 = _stack_item_0; + _tos_cache0 = stack_pointer[-1]; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(2); + break; + } + + case _SPILL_OR_RELOAD_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + _PyStackRef _stack_item_0 = _tos_cache0; + _tos_cache2 = _stack_item_0; + _tos_cache1 = stack_pointer[-1]; + _tos_cache0 = stack_pointer[-2]; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _SPILL_OR_RELOAD_r20: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _SPILL_OR_RELOAD_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _tos_cache0 = _stack_item_1; + stack_pointer[0] = _stack_item_0; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _SPILL_OR_RELOAD_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _tos_cache2 = _stack_item_1; + _tos_cache1 = _stack_item_0; + _tos_cache0 = stack_pointer[-1]; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(3); + break; + } + + case _SPILL_OR_RELOAD_r30: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer[2] = _stack_item_2; + stack_pointer += 3; + assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); + break; + } + + case _SPILL_OR_RELOAD_r31: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + _tos_cache0 = _stack_item_2; + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(1); + break; + } + + case _SPILL_OR_RELOAD_r32: { + CHECK_CURRENT_CACHED_VALUES(3); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + _tos_cache1 = _stack_item_2; + _tos_cache0 = _stack_item_1; + stack_pointer[0] = _stack_item_0; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(2); break; } diff --git a/Python/jit.c b/Python/jit.c index 01bc0076497c6d..990f2a6182bb2a 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -513,7 +513,7 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction trace[], siz data_size += group->data_size; combine_symbol_mask(group->trampoline_mask, state.trampolines.mask); } - group = &stencil_groups[_FATAL_ERROR]; + group = &stencil_groups[_FATAL_ERROR_r00]; code_size += group->code_size; data_size += group->data_size; combine_symbol_mask(group->trampoline_mask, state.trampolines.mask); @@ -556,7 +556,7 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction trace[], siz group->emit(code, data, executor, NULL, &state); code += group->code_size; data += group->data_size; - assert(trace[0].opcode == _START_EXECUTOR); + assert(trace[0].opcode == _START_EXECUTOR_r00); for (size_t i = 0; i < length; i++) { const _PyUOpInstruction *instruction = &trace[i]; group = &stencil_groups[instruction->opcode]; @@ -565,7 +565,7 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction trace[], siz data += group->data_size; } // Protect against accidental buffer overrun into data: - group = &stencil_groups[_FATAL_ERROR]; + group = &stencil_groups[_FATAL_ERROR_r00]; group->emit(code, data, executor, NULL, &state); code += group->code_size; data += group->data_size; diff --git a/Python/optimizer.c b/Python/optimizer.c index 8d01d605ef4a2a..823c5d943912dc 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -277,7 +277,7 @@ uop_dealloc(PyObject *op) { const char * _PyUOpName(int index) { - if (index < 0 || index > MAX_UOP_ID) { + if (index < 0 || index > MAX_UOP_REGS_ID) { return NULL; } return _PyOpcode_uop_name[index]; @@ -336,7 +336,9 @@ uop_item(PyObject *op, Py_ssize_t index) PyErr_SetNone(PyExc_IndexError); return NULL; } - const char *name = _PyUOpName(self->trace[index].opcode); + int opcode = self->trace[index].opcode; + int base_opcode = _PyUop_Uncached[opcode]; + const char *name = _PyUOpName(base_opcode); if (name == NULL) { name = ""; } @@ -995,16 +997,46 @@ count_exits(_PyUOpInstruction *buffer, int length) { int exit_count = 0; for (int i = 0; i < length; i++) { - int opcode = buffer[i].opcode; - if (opcode == _EXIT_TRACE) { + uint16_t base_opcode = _PyUop_Uncached[buffer[i].opcode]; + if (base_opcode == _EXIT_TRACE) { exit_count++; } } return exit_count; } +#define MAX_CACHED_REGISTER 3 + +/* The number of cached registers at any exit (`EXIT_IF` or `DEOPT_IF`) + * This is the number of cached at entries at start, unless the uop is + * marked as `exit_depth_is_output` in which case it is the number of + * cached entries at the end */ +static int +get_cached_entries_for_side_exit(_PyUOpInstruction *inst) +{ + // TO DO -- Add another generated table for this? + int base_opcode = _PyUop_Uncached[inst->opcode]; + assert(base_opcode != 0); + int input = -1; + /* Find number of cached entries at input. */ + for (int i = 0; i <= MAX_CACHED_REGISTER; i++) { + if (_PyUop_Caching[base_opcode].opcodes[i] == inst->opcode) { + input = i; + break; + } + } + if (input == -1) { + return -1; + } + if (_PyUop_Caching[base_opcode].exit_depth_is_output) { + return input + _PyUop_Caching[base_opcode].delta; + } + return input; +} + static void make_exit(_PyUOpInstruction *inst, int opcode, int target) { + assert(opcode > MAX_UOP_ID && opcode <= MAX_UOP_REGS_ID); inst->opcode = opcode; inst->oparg = 0; inst->operand0 = 0; @@ -1041,13 +1073,16 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) int next_spare = length; for (int i = 0; i < length; i++) { _PyUOpInstruction *inst = &buffer[i]; - int opcode = inst->opcode; + int base_opcode = _PyUop_Uncached[inst->opcode]; + assert(inst->opcode != _NOP); int32_t target = (int32_t)uop_get_target(inst); - if (_PyUop_Flags[opcode] & (HAS_EXIT_FLAG | HAS_DEOPT_FLAG)) { - uint16_t exit_op = (_PyUop_Flags[opcode] & HAS_EXIT_FLAG) ? + if (_PyUop_Flags[base_opcode] & (HAS_EXIT_FLAG | HAS_DEOPT_FLAG)) { + uint16_t base_exit_op = (_PyUop_Flags[base_opcode] & HAS_EXIT_FLAG) ? _EXIT_TRACE : _DEOPT; + int exit_depth = get_cached_entries_for_side_exit(inst); + uint16_t exit_op = _PyUop_Caching[base_exit_op].opcodes[exit_depth]; int32_t jump_target = target; - if (is_for_iter_test[opcode]) { + if (is_for_iter_test[base_opcode]) { /* Target the POP_TOP immediately after the END_FOR, * leaving only the iterator on the stack. */ int extended_arg = inst->oparg > 255; @@ -1064,14 +1099,14 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) buffer[i].jump_target = current_jump; buffer[i].format = UOP_FORMAT_JUMP; } - if (_PyUop_Flags[opcode] & HAS_ERROR_FLAG) { - int popped = (_PyUop_Flags[opcode] & HAS_ERROR_NO_POP_FLAG) ? - 0 : _PyUop_num_popped(opcode, inst->oparg); + if (_PyUop_Flags[base_opcode] & HAS_ERROR_FLAG) { + int popped = (_PyUop_Flags[base_opcode] & HAS_ERROR_NO_POP_FLAG) ? + 0 : _PyUop_num_popped(base_opcode, inst->oparg); if (target != current_error_target || popped != current_popped) { current_popped = popped; current_error = next_spare; current_error_target = target; - make_exit(&buffer[next_spare], _ERROR_POP_N, 0); + make_exit(&buffer[next_spare], _ERROR_POP_N_r00, 0); buffer[next_spare].operand0 = target; next_spare++; } @@ -1081,8 +1116,8 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) buffer[i].jump_target = 0; } } - if (opcode == _JUMP_TO_TOP) { - assert(buffer[0].opcode == _START_EXECUTOR); + if (base_opcode == _JUMP_TO_TOP) { + assert(buffer[0].opcode == _START_EXECUTOR_r00); buffer[i].format = UOP_FORMAT_JUMP; buffer[i].jump_target = 1; } @@ -1129,25 +1164,29 @@ sanity_check(_PyExecutorObject *executor) } bool ended = false; uint32_t i = 0; - CHECK(executor->trace[0].opcode == _START_EXECUTOR); + CHECK(executor->trace[0].opcode == _START_EXECUTOR_r00); for (; i < executor->code_size; i++) { const _PyUOpInstruction *inst = &executor->trace[i]; uint16_t opcode = inst->opcode; - CHECK(opcode <= MAX_UOP_ID); - CHECK(_PyOpcode_uop_name[opcode] != NULL); + uint16_t base_opcode = _PyUop_Uncached[opcode]; + CHECK(opcode > MAX_UOP_ID); + CHECK(opcode <= MAX_UOP_REGS_ID); + CHECK(base_opcode <= MAX_UOP_ID); + CHECK(base_opcode != 0); + CHECK(_PyOpcode_uop_name[base_opcode] != NULL); switch(inst->format) { case UOP_FORMAT_TARGET: - CHECK(target_unused(opcode)); + CHECK(target_unused(base_opcode)); break; case UOP_FORMAT_JUMP: CHECK(inst->jump_target < executor->code_size); break; } - if (_PyUop_Flags[opcode] & HAS_ERROR_FLAG) { + if (_PyUop_Flags[base_opcode] & HAS_ERROR_FLAG) { CHECK(inst->format == UOP_FORMAT_JUMP); CHECK(inst->error_target < executor->code_size); } - if (is_terminator(inst)) { + if (base_opcode == _EXIT_TRACE || base_opcode == _JUMP_TO_TOP) { ended = true; i++; break; @@ -1156,11 +1195,11 @@ sanity_check(_PyExecutorObject *executor) CHECK(ended); for (; i < executor->code_size; i++) { const _PyUOpInstruction *inst = &executor->trace[i]; - uint16_t opcode = inst->opcode; + uint16_t base_opcode = _PyUop_Uncached[inst->opcode]; CHECK( - opcode == _DEOPT || - opcode == _EXIT_TRACE || - opcode == _ERROR_POP_N); + base_opcode == _DEOPT || + base_opcode == _EXIT_TRACE || + base_opcode == _ERROR_POP_N); } } @@ -1188,14 +1227,14 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil } int next_exit = exit_count-1; _PyUOpInstruction *dest = (_PyUOpInstruction *)&executor->trace[length]; - assert(buffer[0].opcode == _START_EXECUTOR); + assert(buffer[0].opcode == _START_EXECUTOR_r00); buffer[0].operand0 = (uint64_t)executor; for (int i = length-1; i >= 0; i--) { - int opcode = buffer[i].opcode; + uint16_t base_opcode = _PyUop_Uncached[buffer[i].opcode]; dest--; *dest = buffer[i]; - assert(opcode != _POP_JUMP_IF_FALSE && opcode != _POP_JUMP_IF_TRUE); - if (opcode == _EXIT_TRACE) { + assert(base_opcode != _POP_JUMP_IF_FALSE && base_opcode != _POP_JUMP_IF_TRUE); + if (base_opcode == _EXIT_TRACE) { _PyExitData *exit = &executor->exits[next_exit]; exit->target = buffer[i].target; dest->operand0 = (uint64_t)exit; @@ -1204,7 +1243,7 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil } assert(next_exit == -1); assert(dest == executor->trace); - assert(dest->opcode == _START_EXECUTOR); + assert(dest->opcode == _START_EXECUTOR_r00); _Py_ExecutorInit(executor, dependencies); #ifdef Py_DEBUG char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); @@ -1258,6 +1297,48 @@ int effective_trace_length(_PyUOpInstruction *buffer, int length) } #endif +static int +stack_allocate(_PyUOpInstruction *buffer, int length) +{ + for (int i = length-1; i >= 0; i--) { + buffer[i*2].format = UOP_FORMAT_TARGET; + buffer[i*2].oparg = 0; + buffer[i*2].target = 0; + buffer[i*2+1] = buffer[i]; + } + int depth = 0; + for (int i = 0; i < length; i++) { + _PyUOpInstruction *spill_or_reload = &buffer[i*2]; + int uop = buffer[i*2+1].opcode; + if (uop == _NOP) { + // leave _NOPs to be cleaned up later + spill_or_reload->opcode = _NOP; + continue; + } + if (_PyUop_Caching[uop].opcodes[depth] > 0) { + spill_or_reload->opcode = _NOP; + } + else { + int min_depth = _PyUop_Caching[uop].min_input; + int max_depth = _PyUop_Caching[uop].max_input; + if (depth < min_depth) { + spill_or_reload->opcode = _PyUop_SpillsAndReloads[depth][min_depth]; + depth = min_depth; + } + else { + assert(depth != max_depth); + spill_or_reload->opcode = _PyUop_SpillsAndReloads[depth][max_depth]; + depth = max_depth; + } + } + assert(_PyUop_Caching[uop].opcodes[depth] != 0); + assert(spill_or_reload->opcode != 0); + buffer[i*2+1].opcode = _PyUop_Caching[uop].opcodes[depth]; + depth += _PyUop_Caching[uop].delta; + } + return length*2; +} + static int uop_optimize( _PyInterpreterFrame *frame, @@ -1270,12 +1351,12 @@ uop_optimize( _Py_BloomFilter_Init(&dependencies); _PyUOpInstruction buffer[UOP_MAX_TRACE_LENGTH]; OPT_STAT_INC(attempts); - int length = translate_bytecode_to_trace(frame, instr, buffer, UOP_MAX_TRACE_LENGTH, &dependencies, progress_needed); + int length = translate_bytecode_to_trace(frame, instr, buffer, UOP_MAX_TRACE_LENGTH/2, &dependencies, progress_needed); if (length <= 0) { // Error or nothing translated return length; } - assert(length < UOP_MAX_TRACE_LENGTH); + assert(length < UOP_MAX_TRACE_LENGTH/2); OPT_STAT_INC(traces_created); char *env_var = Py_GETENV("PYTHON_UOPS_OPTIMIZE"); if (env_var == NULL || *env_var == '\0' || *env_var > '0') { @@ -1286,7 +1367,7 @@ uop_optimize( return length; } } - assert(length < UOP_MAX_TRACE_LENGTH); + assert(length < UOP_MAX_TRACE_LENGTH/2); assert(length >= 1); /* Fix up */ for (int pc = 0; pc < length; pc++) { @@ -1302,6 +1383,7 @@ uop_optimize( assert(_PyOpcode_uop_name[buffer[pc].opcode]); } OPT_HIST(effective_trace_length(buffer, length), optimized_trace_length_hist); + length = stack_allocate(buffer, length); length = prepare_for_execution(buffer, length); assert(length <= UOP_MAX_TRACE_LENGTH); _PyExecutorObject *executor = make_executor_from_uops(buffer, length, &dependencies); @@ -1681,13 +1763,14 @@ executor_to_gv(_PyExecutorObject *executor, FILE *out) * https://graphviz.readthedocs.io/en/stable/manual.html#node-ports-compass */ _PyUOpInstruction const *inst = &executor->trace[i]; - const char *opname = _PyOpcode_uop_name[inst->opcode]; + uint16_t base_opcode = _PyUop_Uncached[inst->opcode]; + const char *opname = _PyOpcode_uop_name[base_opcode]; #ifdef Py_STATS fprintf(out, " %s -- %" PRIu64 "\n", i, opname, inst->execution_count); #else fprintf(out, " %s\n", i, opname); #endif - if (inst->opcode == _EXIT_TRACE || inst->opcode == _JUMP_TO_TOP) { + if (base_opcode == _EXIT_TRACE || base_opcode == _JUMP_TO_TOP) { break; } } @@ -1697,21 +1780,22 @@ executor_to_gv(_PyExecutorObject *executor, FILE *out) /* Write all the outgoing edges */ for (uint32_t i = 0; i < executor->code_size; i++) { _PyUOpInstruction const *inst = &executor->trace[i]; - uint16_t flags = _PyUop_Flags[inst->opcode]; + uint16_t base_opcode = _PyUop_Uncached[inst->opcode]; + uint16_t flags = _PyUop_Flags[base_opcode]; _PyExitData *exit = NULL; - if (inst->opcode == _EXIT_TRACE) { + if (base_opcode == _EXIT_TRACE) { exit = (_PyExitData *)inst->operand0; } else if (flags & HAS_EXIT_FLAG) { assert(inst->format == UOP_FORMAT_JUMP); _PyUOpInstruction const *exit_inst = &executor->trace[inst->jump_target]; - assert(exit_inst->opcode == _EXIT_TRACE); + assert(_PyUop_Uncached[exit_inst->opcode] == _EXIT_TRACE); exit = (_PyExitData *)exit_inst->operand0; } if (exit != NULL && exit->executor != NULL) { fprintf(out, "executor_%p:i%d -> executor_%p:start\n", executor, i, exit->executor); } - if (inst->opcode == _EXIT_TRACE || inst->opcode == _JUMP_TO_TOP) { + if (base_opcode == _EXIT_TRACE || base_opcode == _JUMP_TO_TOP) { break; } } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 41402200c1683e..da3230ef5642ce 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -3045,3 +3045,7 @@ break; } + case _SPILL_OR_RELOAD: { + break; + } + diff --git a/Python/specialize.c b/Python/specialize.c index fe8d04cf3442f1..e6c4aef07842af 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -14,7 +14,7 @@ #include "pycore_object.h" #include "pycore_opcode_metadata.h" // _PyOpcode_Caches #include "pycore_uop_metadata.h" // _PyOpcode_uop_name -#include "pycore_uop_ids.h" // MAX_UOP_ID +#include "pycore_uop_ids.h" // MAX_UOP_REGS_ID #include "pycore_opcode_utils.h" // RESUME_AT_FUNC_START #include "pycore_pylifecycle.h" // _PyOS_URandomNonblock() #include "pycore_runtime.h" // _Py_ID() @@ -33,8 +33,8 @@ GCStats _py_gc_stats[NUM_GENERATIONS] = { 0 }; static PyStats _Py_stats_struct = { .gc_stats = _py_gc_stats }; PyStats *_Py_stats = NULL; -#if PYSTATS_MAX_UOP_ID < MAX_UOP_ID -#error "Not enough space allocated for pystats. Increase PYSTATS_MAX_UOP_ID to at least MAX_UOP_ID" +#if PYSTATS_MAX_UOP_ID < MAX_UOP_REGS_ID +#error "Not enough space allocated for pystats. Increase PYSTATS_MAX_UOP_ID to at least MAX_UOP_REGS_ID" #endif #define ADD_STAT_TO_DICT(res, field) \ @@ -277,7 +277,7 @@ print_optimization_stats(FILE *out, OptimizationStats *stats) stats->optimizer_failure_reason_no_memory); fprintf(out, "Optimizer remove globals builtins changed: %" PRIu64 "\n", stats->remove_globals_builtins_changed); fprintf(out, "Optimizer remove globals incorrect keys: %" PRIu64 "\n", stats->remove_globals_incorrect_keys); - for (int i = 0; i <= MAX_UOP_ID; i++) { + for (int i = 0; i <= MAX_UOP_REGS_ID; i++) { if (stats->opcode[i].execution_count) { fprintf(out, "uops[%s].execution_count : %" PRIu64 "\n", _PyUOpName(i), stats->opcode[i].execution_count); } @@ -296,15 +296,15 @@ print_optimization_stats(FILE *out, OptimizationStats *stats) } } - for (int i = 1; i <= MAX_UOP_ID; i++){ - for (int j = 1; j <= MAX_UOP_ID; j++) { + for (int i = 1; i <= MAX_UOP_REGS_ID; i++){ + for (int j = 1; j <= MAX_UOP_REGS_ID; j++) { if (stats->opcode[i].pair_count[j]) { fprintf(out, "uop[%s].pair_count[%s] : %" PRIu64 "\n", _PyOpcode_uop_name[i], _PyOpcode_uop_name[j], stats->opcode[i].pair_count[j]); } } } - for (int i = 0; i < MAX_UOP_ID; i++) { + for (int i = 0; i < MAX_UOP_REGS_ID; i++) { if (stats->error_in_opcode[i]) { fprintf( out, diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 6466d2615cd14e..71f2dbd1e7d189 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -3,10 +3,12 @@ import lexer import parser import re -from typing import Optional, Callable +from typing import Optional, Callable, Iterator from parser import Stmt, SimpleStmt, BlockStmt, IfStmt, WhileStmt +MAX_CACHED_REGISTER = 3 + @dataclass class EscapingCall: stmt: SimpleStmt @@ -25,7 +27,7 @@ class Properties: eval_breaker: bool needs_this: bool always_exits: bool - stores_sp: bool + sync_sp: bool uses_co_consts: bool uses_co_names: bool uses_locals: bool @@ -63,7 +65,7 @@ def from_list(properties: list["Properties"]) -> "Properties": eval_breaker=any(p.eval_breaker for p in properties), needs_this=any(p.needs_this for p in properties), always_exits=any(p.always_exits for p in properties), - stores_sp=any(p.stores_sp for p in properties), + sync_sp=any(p.sync_sp for p in properties), uses_co_consts=any(p.uses_co_consts for p in properties), uses_co_names=any(p.uses_co_names for p in properties), uses_locals=any(p.uses_locals for p in properties), @@ -90,7 +92,7 @@ def infallible(self) -> bool: eval_breaker=False, needs_this=False, always_exits=False, - stores_sp=False, + sync_sp=False, uses_co_consts=False, uses_co_names=False, uses_locals=False, @@ -858,7 +860,7 @@ def compute_properties(op: parser.CodeDef) -> Properties: eval_breaker="CHECK_PERIODIC" in op.name, needs_this=variable_used(op, "this_instr"), always_exits=always_exits(op), - stores_sp=variable_used(op, "SYNC_SP"), + sync_sp=variable_used(op, "SYNC_SP"), uses_co_consts=variable_used(op, "FRAME_CO_CONSTS"), uses_co_names=variable_used(op, "FRAME_CO_NAMES"), uses_locals=variable_used(op, "GETLOCAL") and not has_free, @@ -1216,6 +1218,71 @@ def analyze_forest(forest: list[parser.AstNode]) -> Analysis: instructions, uops, families, pseudos, labels, opmap, first_arg, min_instrumented ) +#Simple heuristic for size to avoid too much stencil duplication +def is_large(uop: Uop) -> bool: + return len(list(uop.body.tokens())) > 80 + +def get_uop_cache_depths(uop: Uop) -> Iterator[tuple[int, int, int]]: + if uop.name == "_SPILL_OR_RELOAD": + for inputs in range(MAX_CACHED_REGISTER+1): + for outputs in range(MAX_CACHED_REGISTER+1): + if inputs != outputs: + yield inputs, outputs, inputs + return + if uop.name in ("_DEOPT", "_EXIT_TRACE"): + for i in range(MAX_CACHED_REGISTER+1): + yield i, 0, i + return + if uop.name in ("_START_EXECUTOR", "_JUMP_TO_TOP", "_ERROR_POP_N"): + yield 0, 0, 0 + return + non_decref_escape = False + for call in uop.properties.escaping_calls.values(): + if "DECREF" in call.call.text or "CLOSE" in call.call.text: + continue + non_decref_escape = True + has_exit = uop.properties.deopts or uop.properties.side_exit + ideal_inputs = 0 + has_array = False + for item in reversed(uop.stack.inputs): + if item.size: + has_array = True + break + if item.peek and uop.properties.escapes: + break + ideal_inputs += 1 + ideal_outputs = 0 + for item in reversed(uop.stack.outputs): + if item.size: + has_array = True + break + if item.peek and uop.properties.escapes: + break + ideal_outputs += 1 + if ideal_inputs > MAX_CACHED_REGISTER: + ideal_inputs = MAX_CACHED_REGISTER + if ideal_outputs > MAX_CACHED_REGISTER: + ideal_outputs = MAX_CACHED_REGISTER + if non_decref_escape: + yield 0, ideal_outputs, 0 + return + exit_depth = ideal_outputs if uop.properties.sync_sp else ideal_inputs + yield ideal_inputs, ideal_outputs, exit_depth + if uop.properties.escapes or uop.properties.sync_sp or has_array or is_large(uop): + return + if ideal_inputs == MAX_CACHED_REGISTER or ideal_outputs == MAX_CACHED_REGISTER: + return + inputs, outputs = ideal_inputs, ideal_outputs + if inputs < outputs: + inputs, outputs = 0, outputs-inputs + else: + inputs, outputs = inputs-outputs, 0 + while inputs <= MAX_CACHED_REGISTER and outputs <= MAX_CACHED_REGISTER: + if inputs != ideal_inputs: + yield inputs, outputs, inputs + inputs += 1 + outputs += 1 + def analyze_files(filenames: list[str]) -> Analysis: return analyze_forest(parser.parse_files(filenames)) diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 4c210fbf8d28e9..b13e5cd888c071 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -125,6 +125,7 @@ def __init__(self, out: CWriter, labels: dict[str, Label], cannot_escape: bool = "DISPATCH": self.dispatch, "INSTRUCTION_SIZE": self.instruction_size, "stack_pointer": self.stack_pointer, + "Py_UNREACHABLE": self.unreachable, } self.out = out self.labels = labels @@ -144,6 +145,19 @@ def dispatch( self.emit(tkn) return False + def unreachable( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ) -> bool: + self.emit(tkn) + emit_to(self.out, tkn_iter, "SEMI") + self.emit(";") + return False + def deopt_if( self, tkn: Token, @@ -533,10 +547,10 @@ def emit_MacroIfStmt( self.out.emit(stmt.condition) branch = stmt.else_ is not None reachable = True - if branch: - else_storage = storage.copy() + if_storage = storage + else_storage = storage.copy() for s in stmt.body: - r, tkn, storage = self._emit_stmt(s, uop, storage, inst) + r, tkn, if_storage = self._emit_stmt(s, uop, if_storage, inst) if tkn is not None: self.out.emit(tkn) if not r: @@ -551,7 +565,10 @@ def emit_MacroIfStmt( self.out.emit(tkn) if not r: reachable = False - else_storage.merge(storage, self.out) # type: ignore[possibly-undefined] + else_storage.merge(if_storage, self.out) + storage = if_storage + else: + if_storage.merge(else_storage, self.out) storage = else_storage self.out.emit(stmt.endif) return reachable, None, storage @@ -704,6 +721,8 @@ def cflags(p: Properties) -> str: flags.append("HAS_PURE_FLAG") if p.no_save_ip: flags.append("HAS_NO_SAVE_IP_FLAG") + if p.sync_sp: + flags.append("HAS_SYNC_SP_FLAG") if flags: return " | ".join(flags) else: diff --git a/Tools/cases_generator/opcode_metadata_generator.py b/Tools/cases_generator/opcode_metadata_generator.py index 0bcdc5395dcd8e..9dd60925cc9b84 100644 --- a/Tools/cases_generator/opcode_metadata_generator.py +++ b/Tools/cases_generator/opcode_metadata_generator.py @@ -54,7 +54,7 @@ "EXIT", "PURE", "PASSTHROUGH", - "OPARG_AND_1", + "SYNC_SP", "ERROR_NO_POP", "NO_SAVE_IP", ] diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py index 3a0e7e5d0d5636..c1f5e85e3ecd61 100644 --- a/Tools/cases_generator/stack.py +++ b/Tools/cases_generator/stack.py @@ -222,6 +222,13 @@ def __init__(self) -> None: self.logical_sp = PointerOffset.zero() self.variables: list[Local] = [] + + def push_cache(self, cached_items:list[str], out: CWriter) -> None: + for i, name in enumerate(cached_items): + out.start_line() + out.emit(f"_PyStackRef _stack_item_{i} = {name};\n") + self.push(Local.register(f"_stack_item_{i}")) + def drop(self, var: StackItem, check_liveness: bool) -> None: self.logical_sp = self.logical_sp.pop(var) if self.variables: @@ -499,6 +506,7 @@ def locals_cached(self) -> bool: return False def flush(self, out: CWriter) -> None: + self._print(out) self.clear_dead_inputs() self._push_defined_outputs() self.stack.flush(out) diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index fc3bc47286f7f6..5a50a80aa5cc2a 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -14,6 +14,9 @@ analyze_files, StackItem, analysis_error, + get_uop_cache_depths, + is_large, + MAX_CACHED_REGISTER, ) from generators_common import ( DEFAULT_INPUT, @@ -60,36 +63,19 @@ def declare_variables(uop: Uop, out: CWriter) -> None: class Tier2Emitter(Emitter): - def __init__(self, out: CWriter, labels: dict[str, Label]): + def __init__(self, out: CWriter, labels: dict[str, Label], exit_cache_depth: int): super().__init__(out, labels) self._replacers["oparg"] = self.oparg + self.exit_cache_depth = exit_cache_depth def goto_error(self, offset: int, storage: Storage) -> str: # To do: Add jump targets for popping values. if offset != 0: storage.copy().flush(self.out) - return f"JUMP_TO_ERROR();" - - def deopt_if( - self, - tkn: Token, - tkn_iter: TokenIterator, - uop: CodeSection, - storage: Storage, - inst: Instruction | None, - ) -> bool: - self.out.emit_at("if ", tkn) - lparen = next(tkn_iter) - self.emit(lparen) - assert lparen.kind == "LPAREN" - first_tkn = tkn_iter.peek() - emit_to(self.out, tkn_iter, "RPAREN") - next(tkn_iter) # Semi colon - self.emit(") {\n") - self.emit("UOP_STAT_INC(uopcode, miss);\n") - self.emit("JUMP_TO_JUMP_TARGET();\n") - self.emit("}\n") - return not always_true(first_tkn) + else: + storage.stack.copy().flush(self.out) + self.emit("SET_CURRENT_CACHED_VALUES(0);\n") + return "JUMP_TO_ERROR();" def exit_if( self, @@ -107,10 +93,13 @@ def exit_if( next(tkn_iter) # Semi colon self.emit(") {\n") self.emit("UOP_STAT_INC(uopcode, miss);\n") + self.emit(f"SET_CURRENT_CACHED_VALUES({self.exit_cache_depth});\n") self.emit("JUMP_TO_JUMP_TARGET();\n") self.emit("}\n") return not always_true(first_tkn) + deopt_if = exit_if + def oparg( self, tkn: Token, @@ -133,7 +122,7 @@ def oparg( return True -def write_uop(uop: Uop, emitter: Emitter, stack: Stack) -> Stack: +def write_uop(uop: Uop, emitter: Emitter, stack: Stack, cached_items: int = 0) -> Stack: locals: dict[str, Local] = {} try: emitter.out.start_line() @@ -155,6 +144,12 @@ def write_uop(uop: Uop, emitter: Emitter, stack: Stack) -> Stack: emitter.emit(f"{type}{cache.name} = ({cast})CURRENT_OPERAND{idx}();\n") idx += 1 _, storage = emitter.emit_tokens(uop, storage, None, False) + storage.stack._print(emitter.out) + while cached_items > 0: + emitter.out.start_line() + item = StackItem(f"_tos_cache{cached_items-1}", "", False, True) + storage.stack.pop(item, emitter.out) + cached_items -= 1 storage.flush(emitter.out) except StackError as ex: raise analysis_error(ex.args[0], uop.body.open) from None @@ -162,6 +157,11 @@ def write_uop(uop: Uop, emitter: Emitter, stack: Stack) -> Stack: SKIPS = ("_EXTENDED_ARG",) +def is_for_iter_test(uop: Uop) -> bool: + return uop.name in ( + "_GUARD_NOT_EXHAUSTED_RANGE", "_GUARD_NOT_EXHAUSTED_LIST", + "_GUARD_NOT_EXHAUSTED_TUPLE", "_FOR_ITER_TIER_TWO" + ) def generate_tier2( filenames: list[str], analysis: Analysis, outfile: TextIO, lines: bool @@ -176,7 +176,7 @@ def generate_tier2( """ ) out = CWriter(outfile, 2, lines) - emitter = Tier2Emitter(out, analysis.labels) + out.emit("\n") for name, uop in analysis.uops.items(): if uop.properties.tier == 1: @@ -189,16 +189,30 @@ def generate_tier2( f"/* {uop.name} is not a viable micro-op for tier 2 because it {why_not_viable} */\n\n" ) continue - out.emit(f"case {uop.name}: {{\n") - declare_variables(uop, out) - stack = Stack() - stack = write_uop(uop, emitter, stack) - out.start_line() - if not uop.properties.always_exits: - out.emit("break;\n") - out.start_line() - out.emit("}") - out.emit("\n\n") + zero_regs = is_large(uop) or uop.properties.escapes + for inputs, outputs, exit_depth in get_uop_cache_depths(uop): + emitter = Tier2Emitter(out, analysis.labels, exit_depth) + out.emit(f"case {uop.name}_r{inputs}{outputs}: {{\n") + out.emit(f"CHECK_CURRENT_CACHED_VALUES({inputs});\n") + declare_variables(uop, out) + stack = Stack() + stack.push_cache([f"_tos_cache{i}" for i in range(inputs)], out) + stack._print(out) + stack = write_uop(uop, emitter, stack, outputs) + out.start_line() + if zero_regs: + # TO DO -- For compilers that support it, + # replace this with a "clobber" to tell + # the compiler that these values are unused + # without having to emit any code. + for i in range(outputs, MAX_CACHED_REGISTER): + out.emit(f"_tos_cache{i} = PyStackRef_ZERO_BITS;\n") + out.emit(f"SET_CURRENT_CACHED_VALUES({outputs});\n") + if not uop.properties.always_exits: + out.emit("break;\n") + out.start_line() + out.emit("}") + out.emit("\n\n") outfile.write("#undef TIER_TWO\n") diff --git a/Tools/cases_generator/uop_id_generator.py b/Tools/cases_generator/uop_id_generator.py index aae89faaa928e1..485d431bc1bac5 100644 --- a/Tools/cases_generator/uop_id_generator.py +++ b/Tools/cases_generator/uop_id_generator.py @@ -8,6 +8,7 @@ from analyzer import ( Analysis, analyze_files, + get_uop_cache_depths, ) from generators_common import ( DEFAULT_INPUT, @@ -38,17 +39,21 @@ def generate_uop_ids( uops = [(uop.name, uop) for uop in analysis.uops.values()] # Sort so that _BASE comes immediately before _BASE_0, etc. for name, uop in sorted(uops): - if name in PRE_DEFINED: - continue - if uop.properties.tier == 1: + if name in PRE_DEFINED or uop.is_super() or uop.properties.tier == 1: continue if uop.implicitly_created and not distinct_namespace and not uop.replicated: out.emit(f"#define {name} {name[1:]}\n") else: out.emit(f"#define {name} {next_id}\n") next_id += 1 - out.emit(f"#define MAX_UOP_ID {next_id-1}\n") + for name, uop in sorted(uops): + if uop.properties.tier == 1: + continue + for inputs, outputs, _ in sorted(get_uop_cache_depths(uop)): + out.emit(f"#define {name}_r{inputs}{outputs} {next_id}\n") + next_id += 1 + out.emit(f"#define MAX_UOP_REGS_ID {next_id-1}\n") arg_parser = argparse.ArgumentParser( diff --git a/Tools/cases_generator/uop_metadata_generator.py b/Tools/cases_generator/uop_metadata_generator.py index 1cc23837a72dea..2bbc4df4edaf1c 100644 --- a/Tools/cases_generator/uop_metadata_generator.py +++ b/Tools/cases_generator/uop_metadata_generator.py @@ -8,6 +8,9 @@ from analyzer import ( Analysis, analyze_files, + get_uop_cache_depths, + Uop, + MAX_CACHED_REGISTER, ) from generators_common import ( DEFAULT_INPUT, @@ -21,17 +24,41 @@ DEFAULT_OUTPUT = ROOT / "Include/internal/pycore_uop_metadata.h" +def uop_cache_info(uop: Uop) -> str | None: + table_size = MAX_CACHED_REGISTER + 1 + if uop.name == "_SPILL_OR_RELOAD": + return None + min_inputs = 4 + uops = [ "0" ] * table_size + for inputs, outputs, exit_depth in get_uop_cache_depths(uop): + delta = outputs - inputs + uops[inputs] = f"{uop.name}_r{inputs}{outputs}" + for i in range(table_size): + if uops[i] != "0": + max_inputs = i + if i < min_inputs: + min_inputs = i + max_inputs, delta # type: ignore[possibly-undefined] + return f"{{ {min_inputs}, {max_inputs}, {delta}, {int(exit_depth == outputs)}, {{ {', '.join(uops)} }} }}" + def generate_names_and_flags(analysis: Analysis, out: CWriter) -> None: out.emit("extern const uint16_t _PyUop_Flags[MAX_UOP_ID+1];\n") out.emit("typedef struct _rep_range { uint8_t start; uint8_t stop; } ReplicationRange;\n") out.emit("extern const ReplicationRange _PyUop_Replication[MAX_UOP_ID+1];\n") - out.emit("extern const char * const _PyOpcode_uop_name[MAX_UOP_ID+1];\n\n") + out.emit("extern const char * const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1];\n\n") out.emit("extern int _PyUop_num_popped(int opcode, int oparg);\n\n") + out.emit("typedef struct _pyuop_info {\n") + out.emit("int8_t min_input; int8_t max_input; int8_t delta;\n") + out.emit(f"int8_t exit_depth_is_output; uint16_t opcodes[{MAX_CACHED_REGISTER+1}];\n") + out.emit("} _PyUopCachingInfo;\n") + out.emit("extern const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1];\n\n") + out.emit(f"extern const uint16_t _PyUop_SpillsAndReloads[{MAX_CACHED_REGISTER+1}][{MAX_CACHED_REGISTER+1}];\n") + out.emit("extern const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1];\n\n") out.emit("#ifdef NEED_OPCODE_METADATA\n") out.emit("const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {\n") for uop in analysis.uops.values(): - if uop.is_viable() and uop.properties.tier != 1: + if uop.is_viable() and uop.properties.tier != 1 and not uop.is_super(): out.emit(f"[{uop.name}] = {cflags(uop.properties)},\n") out.emit("};\n\n") @@ -42,16 +69,37 @@ def generate_names_and_flags(analysis: Analysis, out: CWriter) -> None: out.emit(f"[{uop.name}] = {{ {uop.replicated.start}, {uop.replicated.stop} }},\n") out.emit("};\n\n") - out.emit("const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = {\n") + out.emit("const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = {\n") + for uop in analysis.uops.values(): + if uop.is_viable() and uop.properties.tier != 1 and not uop.is_super(): + info = uop_cache_info(uop) + if info is not None: + out.emit(f"[{uop.name}] = {info},\n") + out.emit("};\n\n") + out.emit("const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = {\n"); + for uop in analysis.uops.values(): + if uop.is_viable() and uop.properties.tier != 1 and not uop.is_super(): + for inputs, outputs, _ in get_uop_cache_depths(uop): + out.emit(f"[{uop.name}_r{inputs}{outputs}] = {uop.name},\n") + out.emit("};\n\n") + out.emit(f"const uint16_t _PyUop_SpillsAndReloads[{MAX_CACHED_REGISTER+1}][{MAX_CACHED_REGISTER+1}] = {{\n") + for i in range(MAX_CACHED_REGISTER+1): + for j in range(MAX_CACHED_REGISTER+1): + if i != j: + out.emit(f"[{i}][{j}] = _SPILL_OR_RELOAD_r{i}{j},\n") + out.emit("};\n\n") + out.emit("const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = {\n") for uop in sorted(analysis.uops.values(), key=lambda t: t.name): - if uop.is_viable() and uop.properties.tier != 1: + if uop.is_viable() and uop.properties.tier != 1 and not uop.is_super(): out.emit(f'[{uop.name}] = "{uop.name}",\n') + for inputs, outputs, _ in get_uop_cache_depths(uop): + out.emit(f'[{uop.name}_r{inputs}{outputs}] = "{uop.name}_r{inputs}{outputs}",\n') out.emit("};\n") out.emit("int _PyUop_num_popped(int opcode, int oparg)\n{\n") out.emit("switch(opcode) {\n") null = CWriter.null() for uop in analysis.uops.values(): - if uop.is_viable() and uop.properties.tier != 1: + if uop.is_viable() and uop.properties.tier != 1 and not uop.is_super(): stack = Stack() for var in reversed(uop.stack.inputs): if var.peek: diff --git a/Tools/jit/_writer.py b/Tools/jit/_writer.py index 090b52660f009c..0c694f6ea92254 100644 --- a/Tools/jit/_writer.py +++ b/Tools/jit/_writer.py @@ -24,7 +24,7 @@ def _dump_footer( yield "" yield f"static const StencilGroup shim = {groups['shim'].as_c('shim')};" yield "" - yield "static const StencilGroup stencil_groups[MAX_UOP_ID + 1] = {" + yield "static const StencilGroup stencil_groups[MAX_UOP_REGS_ID + 1] = {" for opname, group in sorted(groups.items()): if opname == "shim": continue diff --git a/Tools/jit/jit.h b/Tools/jit/jit.h index 10829654eabb38..d5cf288c660f00 100644 --- a/Tools/jit/jit.h +++ b/Tools/jit/jit.h @@ -9,4 +9,5 @@ typedef jit_func __attribute__((preserve_none)) jit_func_preserve_none; #define DECLARE_TARGET(NAME) \ _Py_CODEUNIT *__attribute__((preserve_none, visibility("hidden"))) \ - NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate); + NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, \ + _PyStackRef _tos_cache0, _PyStackRef _tos_cache1, _PyStackRef _tos_cache2); diff --git a/Tools/jit/shim.c b/Tools/jit/shim.c index 0c7feb746c9679..a960990bc9752d 100644 --- a/Tools/jit/shim.c +++ b/Tools/jit/shim.c @@ -7,9 +7,11 @@ #include "jit.h" _Py_CODEUNIT * -_JIT_ENTRY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate) -{ +_JIT_ENTRY( + _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, + _PyStackRef _tos_cache0, _PyStackRef _tos_cache1, _PyStackRef _tos_cache2 +) { // Note that this is *not* a tail call: DECLARE_TARGET(_JIT_CONTINUE); - return _JIT_CONTINUE(frame, stack_pointer, tstate); + return _JIT_CONTINUE(frame, stack_pointer, tstate, _tos_cache0, _tos_cache1, _tos_cache2); } diff --git a/Tools/jit/template.c b/Tools/jit/template.c index d07f56e9ce6b42..a5e0492d3df5e1 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -53,7 +53,7 @@ do { \ _PyExecutorObject *_executor = (EXECUTOR); \ tstate->current_executor = (PyObject *)_executor; \ jit_func_preserve_none jitted = _executor->jit_side_entry; \ - __attribute__((musttail)) return jitted(frame, stack_pointer, tstate); \ + __attribute__((musttail)) return jitted(frame, stack_pointer, tstate, _tos_cache0, _tos_cache1, _tos_cache2); \ } while (0) #undef GOTO_TIER_ONE @@ -77,7 +77,7 @@ do { \ #define PATCH_JUMP(ALIAS) \ do { \ DECLARE_TARGET(ALIAS); \ - __attribute__((musttail)) return ALIAS(frame, stack_pointer, tstate); \ + __attribute__((musttail)) return ALIAS(frame, stack_pointer, tstate, _tos_cache0, _tos_cache1, _tos_cache2); \ } while (0) #undef JUMP_TO_JUMP_TARGET @@ -89,8 +89,10 @@ do { \ #define TIER_TWO 2 __attribute__((preserve_none)) _Py_CODEUNIT * -_JIT_ENTRY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate) -{ +_JIT_ENTRY( + _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, + _PyStackRef _tos_cache0, _PyStackRef _tos_cache1, _PyStackRef _tos_cache2 +) { // Locals that the instruction implementations expect to exist: PATCH_VALUE(_PyExecutorObject *, current_executor, _JIT_EXECUTOR) int oparg; pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy