diff --git a/Include/cpython/code.h b/Include/cpython/code.h index abcf1250603dfe..6bead361c79245 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -3,10 +3,22 @@ #ifndef Py_LIMITED_API #ifndef Py_CODE_H #define Py_CODE_H + #ifdef __cplusplus extern "C" { #endif + +/* Count of all "real" monitoring events (not derived from other events) */ +#define PY_MONITORING_UNGROUPED_EVENTS 14 +/* Count of all monitoring events */ +#define PY_MONITORING_EVENTS 16 + +/* Table of which tools are active for each monitored event. */ +typedef struct _Py_Monitors { + uint8_t tools[PY_MONITORING_UNGROUPED_EVENTS]; +} _Py_Monitors; + /* Each instruction in a code object is a fixed-width value, * currently 2 bytes: 1-byte opcode + 1-byte oparg. The EXTENDED_ARG * opcode allows for larger values but the current limit is 3 uses @@ -56,6 +68,35 @@ typedef struct { PyObject *_co_freevars; } _PyCoCached; +/* Ancilliary data structure used for instrumentation. + Line instrumentation creates an array of + these. One entry per code unit.*/ +typedef struct { + uint8_t original_opcode; + int8_t line_delta; +} _PyCoLineInstrumentationData; + +/* Main data structure used for instrumentation. + * This is allocated when needed for instrumentation + */ +typedef struct { + /* Monitoring specific to this code object */ + _Py_Monitors local_monitors; + /* Monitoring that is active on this code object */ + _Py_Monitors active_monitors; + /* The tools that are to be notified for events for the matching code unit */ + uint8_t *tools; + /* Information to support line events */ + _PyCoLineInstrumentationData *lines; + /* The tools that are to be notified for line events for the matching code unit */ + uint8_t *line_tools; + /* Information to support instruction events */ + /* The underlying instructions, which can themselves be instrumented */ + uint8_t *per_instruction_opcodes; + /* The tools that are to be notified for instruction events for the matching code unit */ + uint8_t *per_instruction_tools; +} _PyCoMonitoringData; + // To avoid repeating ourselves in deepfreeze.py, all PyCodeObject members are // defined in this macro: #define _PyCode_DEF(SIZE) { \ @@ -87,7 +128,6 @@ typedef struct { PyObject *co_exceptiontable; /* Byte string encoding exception handling \ table */ \ int co_flags; /* CO_..., see below */ \ - short _co_linearray_entry_size; /* Size of each entry in _co_linearray */ \ \ /* The rest are not so impactful on performance. */ \ int co_argcount; /* #arguments, except *args */ \ @@ -114,8 +154,9 @@ typedef struct { PyObject *co_linetable; /* bytes object that holds location info */ \ PyObject *co_weakreflist; /* to support weakrefs to code objects */ \ _PyCoCached *_co_cached; /* cached co_* attributes */ \ + uint64_t _co_instrumentation_version; /* current instrumentation version */ \ + _PyCoMonitoringData *_co_monitoring; /* Monitoring data */ \ int _co_firsttraceable; /* index of first traceable instruction */ \ - char *_co_linearray; /* array of line offsets */ \ /* Scratch space for extra data relating to the code object. \ Type is a void* to keep the format private in codeobject.c to force \ people to go through the proper APIs. */ \ diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 3efb241e8237e7..ea6ed8d2bc4a4c 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -58,12 +58,6 @@ typedef int (*Py_tracefunc)(PyObject *, PyFrameObject *, int, PyObject *); #define PyTrace_C_RETURN 6 #define PyTrace_OPCODE 7 - -typedef struct { - PyCodeObject *code; // The code object for the bounds. May be NULL. - PyCodeAddressRange bounds; // Only valid if code != NULL. -} PyTraceInfo; - // Internal structure: you should not use it directly, but use public functions // like PyThreadState_EnterTracing() and PyThreadState_LeaveTracing(). typedef struct _PyCFrame { @@ -77,7 +71,6 @@ typedef struct _PyCFrame { * discipline and make sure that instances of this struct cannot * accessed outside of their lifetime. */ - uint8_t use_tracing; // 0 or 255 (or'ed into opcode, hence 8-bit type) /* Pointer to the currently executing frame (it can be NULL) */ struct _PyInterpreterFrame *current_frame; struct _PyCFrame *previous; @@ -157,7 +150,7 @@ struct _ts { This is to prevent the actual trace/profile code from being recorded in the trace/profile. */ int tracing; - int tracing_what; /* The event currently being traced, if any. */ + int what_event; /* The event currently being monitored, if any. */ /* Pointer to current _PyCFrame in the C stack frame of the currently, * or most recently, executing _PyEval_EvalFrameDefault. */ @@ -228,8 +221,6 @@ struct _ts { /* Unique thread state id. */ uint64_t id; - PyTraceInfo trace_info; - _PyStackChunk *datastack_chunk; PyObject **datastack_top; PyObject **datastack_limit; diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index faf1be585e17ef..d32f37ac44d83c 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -441,32 +441,6 @@ adaptive_counter_backoff(uint16_t counter) { /* Line array cache for tracing */ -extern int _PyCode_CreateLineArray(PyCodeObject *co); - -static inline int -_PyCode_InitLineArray(PyCodeObject *co) -{ - if (co->_co_linearray) { - return 0; - } - return _PyCode_CreateLineArray(co); -} - -static inline int -_PyCode_LineNumberFromArray(PyCodeObject *co, int index) -{ - assert(co->_co_linearray != NULL); - assert(index >= 0); - assert(index < Py_SIZE(co)); - if (co->_co_linearray_entry_size == 2) { - return ((int16_t *)co->_co_linearray)[index]; - } - else { - assert(co->_co_linearray_entry_size == 4); - return ((int32_t *)co->_co_linearray)[index]; - } -} - typedef struct _PyShimCodeDef { const uint8_t *code; int codelen; @@ -500,6 +474,10 @@ extern uint32_t _Py_next_func_version; #define COMPARISON_NOT_EQUALS (COMPARISON_UNORDERED | COMPARISON_LESS_THAN | COMPARISON_GREATER_THAN) +extern int _Py_Instrument(PyCodeObject *co, PyInterpreterState *interp); + +extern int _Py_GetBaseOpcode(PyCodeObject *code, int offset); + #ifdef __cplusplus } diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index 5806cf05f174a9..856297aaea8ab4 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -19,6 +19,7 @@ struct _frame { struct _PyInterpreterFrame *f_frame; /* points to the frame data */ PyObject *f_trace; /* Trace function */ int f_lineno; /* Current line number. Only valid if non-zero */ + int f_last_traced_line; /* The last line traced for this frame */ char f_trace_lines; /* Emit per-line trace events? */ char f_trace_opcodes; /* Emit per-opcode trace events? */ char f_fast_as_locals; /* Have the fast locals of this frame been converted to a dict? */ @@ -137,10 +138,16 @@ _PyFrame_GetLocalsArray(_PyInterpreterFrame *frame) return frame->localsplus; } +/* Fetches the stack pointer, and sets stacktop to -1. + Having stacktop <= 0 ensures that invalid + values are not visible to the cycle GC. + We choose -1 rather than 0 to assist debugging. */ static inline PyObject** _PyFrame_GetStackPointer(_PyInterpreterFrame *frame) { - return frame->localsplus+frame->stacktop; + PyObject **sp = frame->localsplus + frame->stacktop; + frame->stacktop = -1; + return sp; } static inline void diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h new file mode 100644 index 00000000000000..e94d8755546efd --- /dev/null +++ b/Include/internal/pycore_instruments.h @@ -0,0 +1,107 @@ + +#ifndef Py_INTERNAL_INSTRUMENT_H +#define Py_INTERNAL_INSTRUMENT_H + + +#include "pycore_bitutils.h" // _Py_popcount32 +#include "pycore_frame.h" + +#include "cpython/code.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define PY_MONITORING_TOOL_IDS 8 + +/* Local events. + * These require bytecode instrumentation */ + +#define PY_MONITORING_EVENT_PY_START 0 +#define PY_MONITORING_EVENT_PY_RESUME 1 +#define PY_MONITORING_EVENT_PY_RETURN 2 +#define PY_MONITORING_EVENT_PY_YIELD 3 +#define PY_MONITORING_EVENT_CALL 4 +#define PY_MONITORING_EVENT_LINE 5 +#define PY_MONITORING_EVENT_INSTRUCTION 6 +#define PY_MONITORING_EVENT_JUMP 7 +#define PY_MONITORING_EVENT_BRANCH 8 +#define PY_MONITORING_EVENT_STOP_ITERATION 9 + +#define PY_MONITORING_INSTRUMENTED_EVENTS 10 + +/* Other events, mainly exceptions */ + +#define PY_MONITORING_EVENT_RAISE 10 +#define PY_MONITORING_EVENT_EXCEPTION_HANDLED 11 +#define PY_MONITORING_EVENT_PY_UNWIND 12 +#define PY_MONITORING_EVENT_PY_THROW 13 + + +/* Ancilliary events */ + +#define PY_MONITORING_EVENT_C_RETURN 14 +#define PY_MONITORING_EVENT_C_RAISE 15 + + +typedef uint32_t _PyMonitoringEventSet; + +/* Tool IDs */ + +/* These are defined in PEP 669 for convenience to avoid clashes */ +#define PY_MONITORING_DEBUGGER_ID 0 +#define PY_MONITORING_COVERAGE_ID 1 +#define PY_MONITORING_PROFILER_ID 2 +#define PY_MONITORING_OPTIMIZER_ID 5 + +/* Internal IDs used to suuport sys.setprofile() and sys.settrace() */ +#define PY_MONITORING_SYS_PROFILE_ID 6 +#define PY_MONITORING_SYS_TRACE_ID 7 + + +PyObject *_PyMonitoring_RegisterCallback(int tool_id, int event_id, PyObject *obj); + +int _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events); + +extern int +_Py_call_instrumentation(PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr); + +extern int +_Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, + _Py_CODEUNIT *instr); + +extern int +_Py_call_instrumentation_instruction( + PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr); + +int +_Py_call_instrumentation_jump( + PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *target); + +extern int +_Py_call_instrumentation_arg(PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg); + +extern int +_Py_call_instrumentation_2args(PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg0, PyObject *arg1); + +extern void +_Py_call_instrumentation_exc0(PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr); + +extern void +_Py_call_instrumentation_exc2(PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg0, PyObject *arg1); + +extern int +_Py_Instrumentation_GetLine(PyCodeObject *code, int index); + +extern PyObject _PyInstrumentation_MISSING; + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_INSTRUMENT_H */ diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index d64a68cd2da54e..86ae3d8dfc1860 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -24,6 +24,7 @@ extern "C" { #include "pycore_genobject.h" // struct _Py_async_gen_state #include "pycore_gc.h" // struct _gc_runtime_state #include "pycore_import.h" // struct _import_state +#include "pycore_instruments.h" // PY_MONITORING_EVENTS #include "pycore_list.h" // struct _Py_list_state #include "pycore_global_objects.h" // struct _Py_interp_static_objects #include "pycore_object_state.h" // struct _py_object_state @@ -37,7 +38,6 @@ struct _Py_long_state { int max_str_digits; }; - /* interpreter state */ /* PyInterpreterState holds the global state for one of the runtime's @@ -49,6 +49,9 @@ struct _is { PyInterpreterState *next; + uint64_t monitoring_version; + uint64_t last_restart_version; + struct pythreads { uint64_t next_unique_id; /* The linked list of threads, newest first. */ @@ -148,6 +151,15 @@ struct _is { struct callable_cache callable_cache; PyCodeObject *interpreter_trampoline; + _Py_Monitors monitors; + bool f_opcode_trace_set; + bool sys_profile_initialized; + bool sys_trace_initialized; + Py_ssize_t sys_profiling_threads; /* Count of threads with c_profilefunc set */ + Py_ssize_t sys_tracing_threads; /* Count of threads with c_tracefunc set */ + PyObject *monitoring_callables[PY_MONITORING_TOOL_IDS][PY_MONITORING_EVENTS]; + PyObject *monitoring_tool_names[PY_MONITORING_TOOL_IDS]; + struct _Py_interp_cached_objects cached_objects; struct _Py_interp_static_objects static_objects; diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index 81ca91465950b7..b2900c89406f57 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -125,6 +125,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [DICT_UPDATE] = DICT_UPDATE, [END_ASYNC_FOR] = END_ASYNC_FOR, [END_FOR] = END_FOR, + [END_SEND] = END_SEND, [EXTENDED_ARG] = EXTENDED_ARG, [FORMAT_VALUE] = FORMAT_VALUE, [FOR_ITER] = FOR_ITER, @@ -140,6 +141,23 @@ const uint8_t _PyOpcode_Deopt[256] = { [GET_YIELD_FROM_ITER] = GET_YIELD_FROM_ITER, [IMPORT_FROM] = IMPORT_FROM, [IMPORT_NAME] = IMPORT_NAME, + [INSTRUMENTED_CALL] = INSTRUMENTED_CALL, + [INSTRUMENTED_CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, + [INSTRUMENTED_END_FOR] = INSTRUMENTED_END_FOR, + [INSTRUMENTED_END_SEND] = INSTRUMENTED_END_SEND, + [INSTRUMENTED_FOR_ITER] = INSTRUMENTED_FOR_ITER, + [INSTRUMENTED_INSTRUCTION] = INSTRUMENTED_INSTRUCTION, + [INSTRUMENTED_JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD, + [INSTRUMENTED_JUMP_FORWARD] = INSTRUMENTED_JUMP_FORWARD, + [INSTRUMENTED_LINE] = INSTRUMENTED_LINE, + [INSTRUMENTED_POP_JUMP_IF_FALSE] = INSTRUMENTED_POP_JUMP_IF_FALSE, + [INSTRUMENTED_POP_JUMP_IF_NONE] = INSTRUMENTED_POP_JUMP_IF_NONE, + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE, + [INSTRUMENTED_POP_JUMP_IF_TRUE] = INSTRUMENTED_POP_JUMP_IF_TRUE, + [INSTRUMENTED_RESUME] = INSTRUMENTED_RESUME, + [INSTRUMENTED_RETURN_CONST] = INSTRUMENTED_RETURN_CONST, + [INSTRUMENTED_RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE, + [INSTRUMENTED_YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE, [INTERPRETER_EXIT] = INTERPRETER_EXIT, [IS_OP] = IS_OP, [JUMP_BACKWARD] = JUMP_BACKWARD, @@ -192,6 +210,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [PUSH_NULL] = PUSH_NULL, [RAISE_VARARGS] = RAISE_VARARGS, [RERAISE] = RERAISE, + [RESERVED] = RESERVED, [RESUME] = RESUME, [RETURN_CONST] = RETURN_CONST, [RETURN_GENERATOR] = RETURN_GENERATOR, @@ -236,17 +255,19 @@ static const char *const _PyOpcode_OpName[263] = { [PUSH_NULL] = "PUSH_NULL", [INTERPRETER_EXIT] = "INTERPRETER_EXIT", [END_FOR] = "END_FOR", + [END_SEND] = "END_SEND", [BINARY_OP_ADD_FLOAT] = "BINARY_OP_ADD_FLOAT", [BINARY_OP_ADD_INT] = "BINARY_OP_ADD_INT", [BINARY_OP_ADD_UNICODE] = "BINARY_OP_ADD_UNICODE", - [BINARY_OP_INPLACE_ADD_UNICODE] = "BINARY_OP_INPLACE_ADD_UNICODE", [NOP] = "NOP", - [BINARY_OP_MULTIPLY_FLOAT] = "BINARY_OP_MULTIPLY_FLOAT", + [BINARY_OP_INPLACE_ADD_UNICODE] = "BINARY_OP_INPLACE_ADD_UNICODE", [UNARY_NEGATIVE] = "UNARY_NEGATIVE", [UNARY_NOT] = "UNARY_NOT", + [BINARY_OP_MULTIPLY_FLOAT] = "BINARY_OP_MULTIPLY_FLOAT", [BINARY_OP_MULTIPLY_INT] = "BINARY_OP_MULTIPLY_INT", - [BINARY_OP_SUBTRACT_FLOAT] = "BINARY_OP_SUBTRACT_FLOAT", [UNARY_INVERT] = "UNARY_INVERT", + [BINARY_OP_SUBTRACT_FLOAT] = "BINARY_OP_SUBTRACT_FLOAT", + [RESERVED] = "RESERVED", [BINARY_OP_SUBTRACT_INT] = "BINARY_OP_SUBTRACT_INT", [BINARY_SUBSCR_DICT] = "BINARY_SUBSCR_DICT", [BINARY_SUBSCR_GETITEM] = "BINARY_SUBSCR_GETITEM", @@ -254,21 +275,21 @@ static const char *const _PyOpcode_OpName[263] = { [BINARY_SUBSCR_TUPLE_INT] = "BINARY_SUBSCR_TUPLE_INT", [CALL_PY_EXACT_ARGS] = "CALL_PY_EXACT_ARGS", [CALL_PY_WITH_DEFAULTS] = "CALL_PY_WITH_DEFAULTS", - [CALL_BOUND_METHOD_EXACT_ARGS] = "CALL_BOUND_METHOD_EXACT_ARGS", - [CALL_BUILTIN_CLASS] = "CALL_BUILTIN_CLASS", [BINARY_SUBSCR] = "BINARY_SUBSCR", [BINARY_SLICE] = "BINARY_SLICE", [STORE_SLICE] = "STORE_SLICE", - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", + [CALL_BOUND_METHOD_EXACT_ARGS] = "CALL_BOUND_METHOD_EXACT_ARGS", + [CALL_BUILTIN_CLASS] = "CALL_BUILTIN_CLASS", [GET_LEN] = "GET_LEN", [MATCH_MAPPING] = "MATCH_MAPPING", [MATCH_SEQUENCE] = "MATCH_SEQUENCE", [MATCH_KEYS] = "MATCH_KEYS", - [CALL_NO_KW_BUILTIN_FAST] = "CALL_NO_KW_BUILTIN_FAST", + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", [PUSH_EXC_INFO] = "PUSH_EXC_INFO", [CHECK_EXC_MATCH] = "CHECK_EXC_MATCH", [CHECK_EG_MATCH] = "CHECK_EG_MATCH", + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", + [CALL_NO_KW_BUILTIN_FAST] = "CALL_NO_KW_BUILTIN_FAST", [CALL_NO_KW_BUILTIN_O] = "CALL_NO_KW_BUILTIN_O", [CALL_NO_KW_ISINSTANCE] = "CALL_NO_KW_ISINSTANCE", [CALL_NO_KW_LEN] = "CALL_NO_KW_LEN", @@ -278,8 +299,6 @@ static const char *const _PyOpcode_OpName[263] = { [CALL_NO_KW_METHOD_DESCRIPTOR_O] = "CALL_NO_KW_METHOD_DESCRIPTOR_O", [CALL_NO_KW_STR_1] = "CALL_NO_KW_STR_1", [CALL_NO_KW_TUPLE_1] = "CALL_NO_KW_TUPLE_1", - [CALL_NO_KW_TYPE_1] = "CALL_NO_KW_TYPE_1", - [COMPARE_OP_FLOAT] = "COMPARE_OP_FLOAT", [WITH_EXCEPT_START] = "WITH_EXCEPT_START", [GET_AITER] = "GET_AITER", [GET_ANEXT] = "GET_ANEXT", @@ -287,39 +306,39 @@ static const char *const _PyOpcode_OpName[263] = { [BEFORE_WITH] = "BEFORE_WITH", [END_ASYNC_FOR] = "END_ASYNC_FOR", [CLEANUP_THROW] = "CLEANUP_THROW", + [CALL_NO_KW_TYPE_1] = "CALL_NO_KW_TYPE_1", + [COMPARE_OP_FLOAT] = "COMPARE_OP_FLOAT", [COMPARE_OP_INT] = "COMPARE_OP_INT", [COMPARE_OP_STR] = "COMPARE_OP_STR", - [FOR_ITER_LIST] = "FOR_ITER_LIST", - [FOR_ITER_TUPLE] = "FOR_ITER_TUPLE", [STORE_SUBSCR] = "STORE_SUBSCR", [DELETE_SUBSCR] = "DELETE_SUBSCR", + [FOR_ITER_LIST] = "FOR_ITER_LIST", + [FOR_ITER_TUPLE] = "FOR_ITER_TUPLE", [FOR_ITER_RANGE] = "FOR_ITER_RANGE", [FOR_ITER_GEN] = "FOR_ITER_GEN", [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", - [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", - [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", [GET_ITER] = "GET_ITER", [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", - [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", + [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS", - [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", - [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", + [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", + [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR", [RETURN_GENERATOR] = "RETURN_GENERATOR", + [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", + [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST", [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", + [RETURN_VALUE] = "RETURN_VALUE", [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST", + [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", - [RETURN_VALUE] = "RETURN_VALUE", [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", - [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", - [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", - [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", [POP_EXCEPT] = "POP_EXCEPT", [STORE_NAME] = "STORE_NAME", [DELETE_NAME] = "DELETE_NAME", @@ -342,9 +361,9 @@ static const char *const _PyOpcode_OpName[263] = { [IMPORT_NAME] = "IMPORT_NAME", [IMPORT_FROM] = "IMPORT_FROM", [JUMP_FORWARD] = "JUMP_FORWARD", + [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", + [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST", - [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", - [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE", [POP_JUMP_IF_TRUE] = "POP_JUMP_IF_TRUE", [LOAD_GLOBAL] = "LOAD_GLOBAL", @@ -372,9 +391,9 @@ static const char *const _PyOpcode_OpName[263] = { [STORE_DEREF] = "STORE_DEREF", [DELETE_DEREF] = "DELETE_DEREF", [JUMP_BACKWARD] = "JUMP_BACKWARD", - [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", + [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX", - [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", + [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", [EXTENDED_ARG] = "EXTENDED_ARG", [LIST_APPEND] = "LIST_APPEND", [SET_ADD] = "SET_ADD", @@ -384,14 +403,14 @@ static const char *const _PyOpcode_OpName[263] = { [YIELD_VALUE] = "YIELD_VALUE", [RESUME] = "RESUME", [MATCH_CLASS] = "MATCH_CLASS", - [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", - [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", + [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", + [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", [FORMAT_VALUE] = "FORMAT_VALUE", [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", [BUILD_STRING] = "BUILD_STRING", + [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", + [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", [SEND_GEN] = "SEND_GEN", - [159] = "<159>", - [160] = "<160>", [161] = "<161>", [LIST_EXTEND] = "LIST_EXTEND", [SET_UPDATE] = "SET_UPDATE", @@ -469,24 +488,24 @@ static const char *const _PyOpcode_OpName[263] = { [235] = "<235>", [236] = "<236>", [237] = "<237>", - [238] = "<238>", - [239] = "<239>", - [240] = "<240>", - [241] = "<241>", - [242] = "<242>", - [243] = "<243>", - [244] = "<244>", - [245] = "<245>", - [246] = "<246>", - [247] = "<247>", - [248] = "<248>", - [249] = "<249>", - [250] = "<250>", - [251] = "<251>", - [252] = "<252>", - [253] = "<253>", - [254] = "<254>", - [DO_TRACING] = "DO_TRACING", + [INSTRUMENTED_POP_JUMP_IF_NONE] = "INSTRUMENTED_POP_JUMP_IF_NONE", + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = "INSTRUMENTED_POP_JUMP_IF_NOT_NONE", + [INSTRUMENTED_RESUME] = "INSTRUMENTED_RESUME", + [INSTRUMENTED_CALL] = "INSTRUMENTED_CALL", + [INSTRUMENTED_RETURN_VALUE] = "INSTRUMENTED_RETURN_VALUE", + [INSTRUMENTED_YIELD_VALUE] = "INSTRUMENTED_YIELD_VALUE", + [INSTRUMENTED_CALL_FUNCTION_EX] = "INSTRUMENTED_CALL_FUNCTION_EX", + [INSTRUMENTED_JUMP_FORWARD] = "INSTRUMENTED_JUMP_FORWARD", + [INSTRUMENTED_JUMP_BACKWARD] = "INSTRUMENTED_JUMP_BACKWARD", + [INSTRUMENTED_RETURN_CONST] = "INSTRUMENTED_RETURN_CONST", + [INSTRUMENTED_FOR_ITER] = "INSTRUMENTED_FOR_ITER", + [INSTRUMENTED_POP_JUMP_IF_FALSE] = "INSTRUMENTED_POP_JUMP_IF_FALSE", + [INSTRUMENTED_POP_JUMP_IF_TRUE] = "INSTRUMENTED_POP_JUMP_IF_TRUE", + [INSTRUMENTED_END_FOR] = "INSTRUMENTED_END_FOR", + [INSTRUMENTED_END_SEND] = "INSTRUMENTED_END_SEND", + [INSTRUMENTED_INSTRUCTION] = "INSTRUMENTED_INSTRUCTION", + [INSTRUMENTED_LINE] = "INSTRUMENTED_LINE", + [255] = "<255>", [SETUP_FINALLY] = "SETUP_FINALLY", [SETUP_CLEANUP] = "SETUP_CLEANUP", [SETUP_WITH] = "SETUP_WITH", @@ -498,8 +517,6 @@ static const char *const _PyOpcode_OpName[263] = { #endif #define EXTRA_CASES \ - case 159: \ - case 160: \ case 161: \ case 166: \ case 167: \ @@ -569,23 +586,7 @@ static const char *const _PyOpcode_OpName[263] = { case 235: \ case 236: \ case 237: \ - case 238: \ - case 239: \ - case 240: \ - case 241: \ - case 242: \ - case 243: \ - case 244: \ - case 245: \ - case 246: \ - case 247: \ - case 248: \ - case 249: \ - case 250: \ - case 251: \ - case 252: \ - case 253: \ - case 254: \ + case 255: \ ; #ifdef __cplusplus diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index b5408622d9d4b2..6e5f2289cb6b95 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -133,16 +133,6 @@ extern void _PyThreadState_BindDetached(PyThreadState *); extern void _PyThreadState_UnbindDetached(PyThreadState *); -static inline void -_PyThreadState_UpdateTracingState(PyThreadState *tstate) -{ - bool use_tracing = - (tstate->tracing == 0) && - (tstate->c_tracefunc != NULL || tstate->c_profilefunc != NULL); - tstate->cframe->use_tracing = (use_tracing ? 255 : 0); -} - - /* Other */ PyAPI_FUNC(PyThreadState *) _PyThreadState_Swap( diff --git a/Include/opcode.h b/Include/opcode.h index 0ff84dc5a551a0..aa8716ef5b4030 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -13,10 +13,12 @@ extern "C" { #define PUSH_NULL 2 #define INTERPRETER_EXIT 3 #define END_FOR 4 +#define END_SEND 5 #define NOP 9 #define UNARY_NEGATIVE 11 #define UNARY_NOT 12 #define UNARY_INVERT 15 +#define RESERVED 17 #define BINARY_SUBSCR 25 #define BINARY_SLICE 26 #define STORE_SLICE 27 @@ -114,6 +116,24 @@ extern "C" { #define KW_NAMES 172 #define CALL_INTRINSIC_1 173 #define CALL_INTRINSIC_2 174 +#define MIN_INSTRUMENTED_OPCODE 238 +#define INSTRUMENTED_POP_JUMP_IF_NONE 238 +#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 239 +#define INSTRUMENTED_RESUME 240 +#define INSTRUMENTED_CALL 241 +#define INSTRUMENTED_RETURN_VALUE 242 +#define INSTRUMENTED_YIELD_VALUE 243 +#define INSTRUMENTED_CALL_FUNCTION_EX 244 +#define INSTRUMENTED_JUMP_FORWARD 245 +#define INSTRUMENTED_JUMP_BACKWARD 246 +#define INSTRUMENTED_RETURN_CONST 247 +#define INSTRUMENTED_FOR_ITER 248 +#define INSTRUMENTED_POP_JUMP_IF_FALSE 249 +#define INSTRUMENTED_POP_JUMP_IF_TRUE 250 +#define INSTRUMENTED_END_FOR 251 +#define INSTRUMENTED_END_SEND 252 +#define INSTRUMENTED_INSTRUCTION 253 +#define INSTRUMENTED_LINE 254 #define MIN_PSEUDO_OPCODE 256 #define SETUP_FINALLY 256 #define SETUP_CLEANUP 257 @@ -123,69 +143,68 @@ extern "C" { #define JUMP_NO_INTERRUPT 261 #define LOAD_METHOD 262 #define MAX_PSEUDO_OPCODE 262 -#define BINARY_OP_ADD_FLOAT 5 -#define BINARY_OP_ADD_INT 6 -#define BINARY_OP_ADD_UNICODE 7 -#define BINARY_OP_INPLACE_ADD_UNICODE 8 -#define BINARY_OP_MULTIPLY_FLOAT 10 -#define BINARY_OP_MULTIPLY_INT 13 -#define BINARY_OP_SUBTRACT_FLOAT 14 -#define BINARY_OP_SUBTRACT_INT 16 -#define BINARY_SUBSCR_DICT 17 -#define BINARY_SUBSCR_GETITEM 18 -#define BINARY_SUBSCR_LIST_INT 19 -#define BINARY_SUBSCR_TUPLE_INT 20 -#define CALL_PY_EXACT_ARGS 21 -#define CALL_PY_WITH_DEFAULTS 22 -#define CALL_BOUND_METHOD_EXACT_ARGS 23 -#define CALL_BUILTIN_CLASS 24 -#define CALL_BUILTIN_FAST_WITH_KEYWORDS 28 -#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 29 -#define CALL_NO_KW_BUILTIN_FAST 34 -#define CALL_NO_KW_BUILTIN_O 38 -#define CALL_NO_KW_ISINSTANCE 39 -#define CALL_NO_KW_LEN 40 -#define CALL_NO_KW_LIST_APPEND 41 -#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 42 -#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 43 -#define CALL_NO_KW_METHOD_DESCRIPTOR_O 44 -#define CALL_NO_KW_STR_1 45 -#define CALL_NO_KW_TUPLE_1 46 -#define CALL_NO_KW_TYPE_1 47 -#define COMPARE_OP_FLOAT 48 -#define COMPARE_OP_INT 56 -#define COMPARE_OP_STR 57 -#define FOR_ITER_LIST 58 -#define FOR_ITER_TUPLE 59 -#define FOR_ITER_RANGE 62 -#define FOR_ITER_GEN 63 -#define LOAD_ATTR_CLASS 64 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 65 -#define LOAD_ATTR_INSTANCE_VALUE 66 -#define LOAD_ATTR_MODULE 67 -#define LOAD_ATTR_PROPERTY 70 -#define LOAD_ATTR_SLOT 72 -#define LOAD_ATTR_WITH_HINT 73 -#define LOAD_ATTR_METHOD_LAZY_DICT 76 -#define LOAD_ATTR_METHOD_NO_DICT 77 -#define LOAD_ATTR_METHOD_WITH_VALUES 78 -#define LOAD_CONST__LOAD_FAST 79 -#define LOAD_FAST__LOAD_CONST 80 -#define LOAD_FAST__LOAD_FAST 81 -#define LOAD_GLOBAL_BUILTIN 82 -#define LOAD_GLOBAL_MODULE 84 -#define STORE_ATTR_INSTANCE_VALUE 86 -#define STORE_ATTR_SLOT 87 -#define STORE_ATTR_WITH_HINT 88 -#define STORE_FAST__LOAD_FAST 111 -#define STORE_FAST__STORE_FAST 112 -#define STORE_SUBSCR_DICT 113 -#define STORE_SUBSCR_LIST_INT 141 -#define UNPACK_SEQUENCE_LIST 143 -#define UNPACK_SEQUENCE_TUPLE 153 -#define UNPACK_SEQUENCE_TWO_TUPLE 154 -#define SEND_GEN 158 -#define DO_TRACING 255 +#define BINARY_OP_ADD_FLOAT 6 +#define BINARY_OP_ADD_INT 7 +#define BINARY_OP_ADD_UNICODE 8 +#define BINARY_OP_INPLACE_ADD_UNICODE 10 +#define BINARY_OP_MULTIPLY_FLOAT 13 +#define BINARY_OP_MULTIPLY_INT 14 +#define BINARY_OP_SUBTRACT_FLOAT 16 +#define BINARY_OP_SUBTRACT_INT 18 +#define BINARY_SUBSCR_DICT 19 +#define BINARY_SUBSCR_GETITEM 20 +#define BINARY_SUBSCR_LIST_INT 21 +#define BINARY_SUBSCR_TUPLE_INT 22 +#define CALL_PY_EXACT_ARGS 23 +#define CALL_PY_WITH_DEFAULTS 24 +#define CALL_BOUND_METHOD_EXACT_ARGS 28 +#define CALL_BUILTIN_CLASS 29 +#define CALL_BUILTIN_FAST_WITH_KEYWORDS 34 +#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 38 +#define CALL_NO_KW_BUILTIN_FAST 39 +#define CALL_NO_KW_BUILTIN_O 40 +#define CALL_NO_KW_ISINSTANCE 41 +#define CALL_NO_KW_LEN 42 +#define CALL_NO_KW_LIST_APPEND 43 +#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 44 +#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 45 +#define CALL_NO_KW_METHOD_DESCRIPTOR_O 46 +#define CALL_NO_KW_STR_1 47 +#define CALL_NO_KW_TUPLE_1 48 +#define CALL_NO_KW_TYPE_1 56 +#define COMPARE_OP_FLOAT 57 +#define COMPARE_OP_INT 58 +#define COMPARE_OP_STR 59 +#define FOR_ITER_LIST 62 +#define FOR_ITER_TUPLE 63 +#define FOR_ITER_RANGE 64 +#define FOR_ITER_GEN 65 +#define LOAD_ATTR_CLASS 66 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 67 +#define LOAD_ATTR_INSTANCE_VALUE 70 +#define LOAD_ATTR_MODULE 72 +#define LOAD_ATTR_PROPERTY 73 +#define LOAD_ATTR_SLOT 76 +#define LOAD_ATTR_WITH_HINT 77 +#define LOAD_ATTR_METHOD_LAZY_DICT 78 +#define LOAD_ATTR_METHOD_NO_DICT 79 +#define LOAD_ATTR_METHOD_WITH_VALUES 80 +#define LOAD_CONST__LOAD_FAST 81 +#define LOAD_FAST__LOAD_CONST 82 +#define LOAD_FAST__LOAD_FAST 84 +#define LOAD_GLOBAL_BUILTIN 86 +#define LOAD_GLOBAL_MODULE 87 +#define STORE_ATTR_INSTANCE_VALUE 88 +#define STORE_ATTR_SLOT 111 +#define STORE_ATTR_WITH_HINT 112 +#define STORE_FAST__LOAD_FAST 113 +#define STORE_FAST__STORE_FAST 141 +#define STORE_SUBSCR_DICT 143 +#define STORE_SUBSCR_LIST_INT 153 +#define UNPACK_SEQUENCE_LIST 154 +#define UNPACK_SEQUENCE_TUPLE 158 +#define UNPACK_SEQUENCE_TWO_TUPLE 159 +#define SEND_GEN 160 #define HAS_ARG(op) ((((op) >= HAVE_ARGUMENT) && (!IS_PSEUDO_OPCODE(op)))\ || ((op) == JUMP) \ diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index de6c434450fc82..c0c757d94d8781 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -439,6 +439,7 @@ def _write_atomic(path, data, mode=0o666): # Python 3.12a7 3523 (Convert COMPARE_AND_BRANCH back to COMPARE_OP) # Python 3.12a7 3524 (Shrink the BINARY_SUBSCR caches) # Python 3.12b1 3525 (Shrink the CALL caches) +# Python 3.12a7 3526 (Add instrumentation support) # Python 3.13 will start with 3550 @@ -455,7 +456,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3525).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3526).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c diff --git a/Lib/opcode.py b/Lib/opcode.py index b62dfa1bcb42c5..dd739e5dd3f6f8 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -83,6 +83,7 @@ def pseudo_op(name, op, real_ops): def_op('INTERPRETER_EXIT', 3) def_op('END_FOR', 4) +def_op('END_SEND', 5) def_op('NOP', 9) @@ -91,6 +92,10 @@ def pseudo_op(name, op, real_ops): def_op('UNARY_INVERT', 15) +# We reserve 17 as it is the initial value for the specializing counter +# This helps us catch cases where we attempt to execute a cache. +def_op('RESERVED', 17) + def_op('BINARY_SUBSCR', 25) def_op('BINARY_SLICE', 26) def_op('STORE_SLICE', 27) @@ -221,6 +226,28 @@ def pseudo_op(name, op, real_ops): def_op('CALL_INTRINSIC_1', 173) def_op('CALL_INTRINSIC_2', 174) +# Instrumented instructions +MIN_INSTRUMENTED_OPCODE = 238 + +def_op('INSTRUMENTED_POP_JUMP_IF_NONE', 238) +def_op('INSTRUMENTED_POP_JUMP_IF_NOT_NONE', 239) +def_op('INSTRUMENTED_RESUME', 240) +def_op('INSTRUMENTED_CALL', 241) +def_op('INSTRUMENTED_RETURN_VALUE', 242) +def_op('INSTRUMENTED_YIELD_VALUE', 243) +def_op('INSTRUMENTED_CALL_FUNCTION_EX', 244) +def_op('INSTRUMENTED_JUMP_FORWARD', 245) +def_op('INSTRUMENTED_JUMP_BACKWARD', 246) +def_op('INSTRUMENTED_RETURN_CONST', 247) +def_op('INSTRUMENTED_FOR_ITER', 248) +def_op('INSTRUMENTED_POP_JUMP_IF_FALSE', 249) +def_op('INSTRUMENTED_POP_JUMP_IF_TRUE', 250) +def_op('INSTRUMENTED_END_FOR', 251) +def_op('INSTRUMENTED_END_SEND', 252) +def_op('INSTRUMENTED_INSTRUCTION', 253) +def_op('INSTRUMENTED_LINE', 254) +# 255 is reserved + hasarg.extend([op for op in opmap.values() if op >= HAVE_ARGUMENT]) MIN_PSEUDO_OPCODE = 256 diff --git a/Lib/test/test__opcode.py b/Lib/test/test__opcode.py index 31f3c53992db13..7640c6fb57d4f3 100644 --- a/Lib/test/test__opcode.py +++ b/Lib/test/test__opcode.py @@ -20,6 +20,8 @@ def test_stack_effect(self): # All defined opcodes has_arg = dis.hasarg for name, code in filter(lambda item: item[0] not in dis.deoptmap, dis.opmap.items()): + if code >= opcode.MIN_INSTRUMENTED_OPCODE: + continue with self.subTest(opname=name): if code not in has_arg: stack_effect(code) @@ -47,6 +49,8 @@ def test_stack_effect_jump(self): has_exc = dis.hasexc has_jump = dis.hasjabs + dis.hasjrel for name, code in filter(lambda item: item[0] not in dis.deoptmap, dis.opmap.items()): + if code >= opcode.MIN_INSTRUMENTED_OPCODE: + continue with self.subTest(opname=name): if code not in has_arg: common = stack_effect(code) diff --git a/Lib/test/test_bdb.py b/Lib/test/test_bdb.py index 042c2daea7f797..fc4b8094316332 100644 --- a/Lib/test/test_bdb.py +++ b/Lib/test/test_bdb.py @@ -433,8 +433,9 @@ def __exit__(self, type_=None, value=None, traceback=None): not_empty = '' if self.tracer.set_list: not_empty += 'All paired tuples have not been processed, ' - not_empty += ('the last one was number %d' % + not_empty += ('the last one was number %d\n' % self.tracer.expect_set_no) + not_empty += repr(self.tracer.set_list) # Make a BdbNotExpectedError a unittest failure. if type_ is not None and issubclass(BdbNotExpectedError, type_): diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 7543c9ab342119..ecb3525a928468 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -349,14 +349,14 @@ def test_invalid_bytecode(self): def foo(): pass - # assert that opcode 238 is invalid - self.assertEqual(opname[238], '<238>') + # assert that opcode 229 is invalid + self.assertEqual(opname[229], '<229>') - # change first opcode to 0xee (=238) + # change first opcode to 0xeb (=229) foo.__code__ = foo.__code__.replace( - co_code=b'\xee' + foo.__code__.co_code[1:]) + co_code=b'\xe5' + foo.__code__.co_code[1:]) - msg = f"unknown opcode 238" + msg = f"unknown opcode 229" with self.assertRaisesRegex(SystemError, msg): foo() diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 4a2144743f6567..0a60a979614d52 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -479,8 +479,7 @@ async def _asyncwith(c): YIELD_VALUE 2 RESUME 3 JUMP_BACKWARD_NO_INTERRUPT 5 (to 14) - >> SWAP 2 - POP_TOP + >> END_SEND POP_TOP %3d LOAD_CONST 1 (1) @@ -492,11 +491,11 @@ async def _asyncwith(c): CALL 2 GET_AWAITABLE 2 LOAD_CONST 0 (None) - >> SEND 3 (to 62) + >> SEND 3 (to 60) YIELD_VALUE 2 RESUME 3 - JUMP_BACKWARD_NO_INTERRUPT 5 (to 52) - >> POP_TOP + JUMP_BACKWARD_NO_INTERRUPT 5 (to 50) + >> END_SEND POP_TOP %3d LOAD_CONST 2 (2) @@ -504,21 +503,20 @@ async def _asyncwith(c): RETURN_CONST 0 (None) %3d >> CLEANUP_THROW - JUMP_BACKWARD 26 (to 24) + JUMP_BACKWARD 25 (to 24) >> CLEANUP_THROW - JUMP_BACKWARD 9 (to 62) + JUMP_BACKWARD 9 (to 60) >> PUSH_EXC_INFO WITH_EXCEPT_START GET_AWAITABLE 2 LOAD_CONST 0 (None) - >> SEND 4 (to 100) + >> SEND 4 (to 98) YIELD_VALUE 3 RESUME 3 - JUMP_BACKWARD_NO_INTERRUPT 5 (to 88) + JUMP_BACKWARD_NO_INTERRUPT 5 (to 86) >> CLEANUP_THROW - >> SWAP 2 - POP_TOP - POP_JUMP_IF_TRUE 1 (to 108) + >> END_SEND + POP_JUMP_IF_TRUE 1 (to 104) RERAISE 2 >> POP_TOP POP_EXCEPT @@ -878,9 +876,9 @@ def test_boundaries(self): def test_widths(self): long_opcodes = set(['JUMP_BACKWARD_NO_INTERRUPT', - ]) + 'INSTRUMENTED_CALL_FUNCTION_EX']) for opcode, opname in enumerate(dis.opname): - if opname in long_opcodes: + if opname in long_opcodes or opname.startswith("INSTRUMENTED"): continue with self.subTest(opname=opname): width = dis._OPNAME_WIDTH diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py new file mode 100644 index 00000000000000..4aad3da61b6bee --- /dev/null +++ b/Lib/test/test_monitoring.py @@ -0,0 +1,1044 @@ +"""Test suite for the sys.monitoring.""" + +import collections +import functools +import operator +import sys +import types +import unittest + + +PAIR = (0,1) + +def f1(): + pass + +def f2(): + len([]) + sys.getsizeof(0) + +def floop(): + for item in PAIR: + pass + +def gen(): + yield + yield + +def g1(): + for _ in gen(): + pass + +TEST_TOOL = 2 +TEST_TOOL2 = 3 +TEST_TOOL3 = 4 + +class MonitoringBasicTest(unittest.TestCase): + + def test_has_objects(self): + m = sys.monitoring + m.events + m.use_tool_id + m.free_tool_id + m.get_tool + m.get_events + m.set_events + m.get_local_events + m.set_local_events + m.register_callback + m.restart_events + m.DISABLE + m.MISSING + m.events.NO_EVENTS + + def test_tool(self): + sys.monitoring.use_tool_id(TEST_TOOL, "MonitoringTest.Tool") + self.assertEqual(sys.monitoring.get_tool(TEST_TOOL), "MonitoringTest.Tool") + sys.monitoring.set_events(TEST_TOOL, 15) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL), 15) + sys.monitoring.set_events(TEST_TOOL, 0) + with self.assertRaises(ValueError): + sys.monitoring.set_events(TEST_TOOL, sys.monitoring.events.C_RETURN) + with self.assertRaises(ValueError): + sys.monitoring.set_events(TEST_TOOL, sys.monitoring.events.C_RAISE) + sys.monitoring.free_tool_id(TEST_TOOL) + self.assertEqual(sys.monitoring.get_tool(TEST_TOOL), None) + with self.assertRaises(ValueError): + sys.monitoring.set_events(TEST_TOOL, sys.monitoring.events.CALL) + + +class MonitoringTestBase: + + def setUp(self): + # Check that a previous test hasn't left monitoring on. + for tool in range(6): + self.assertEqual(sys.monitoring.get_events(tool), 0) + self.assertIs(sys.monitoring.get_tool(TEST_TOOL), None) + self.assertIs(sys.monitoring.get_tool(TEST_TOOL2), None) + self.assertIs(sys.monitoring.get_tool(TEST_TOOL3), None) + sys.monitoring.use_tool_id(TEST_TOOL, "test " + self.__class__.__name__) + sys.monitoring.use_tool_id(TEST_TOOL2, "test2 " + self.__class__.__name__) + sys.monitoring.use_tool_id(TEST_TOOL3, "test3 " + self.__class__.__name__) + + def tearDown(self): + # Check that test hasn't left monitoring on. + for tool in range(6): + self.assertEqual(sys.monitoring.get_events(tool), 0) + sys.monitoring.free_tool_id(TEST_TOOL) + sys.monitoring.free_tool_id(TEST_TOOL2) + sys.monitoring.free_tool_id(TEST_TOOL3) + + +class MonitoringCountTest(MonitoringTestBase, unittest.TestCase): + + def check_event_count(self, func, event, expected): + + class Counter: + def __init__(self): + self.count = 0 + def __call__(self, *args): + self.count += 1 + + counter = Counter() + sys.monitoring.register_callback(TEST_TOOL, event, counter) + if event == E.C_RETURN or event == E.C_RAISE: + sys.monitoring.set_events(TEST_TOOL, E.CALL) + else: + sys.monitoring.set_events(TEST_TOOL, event) + self.assertEqual(counter.count, 0) + counter.count = 0 + func() + self.assertEqual(counter.count, expected) + prev = sys.monitoring.register_callback(TEST_TOOL, event, None) + counter.count = 0 + func() + self.assertEqual(counter.count, 0) + self.assertEqual(prev, counter) + sys.monitoring.set_events(TEST_TOOL, 0) + + def test_start_count(self): + self.check_event_count(f1, E.PY_START, 1) + + def test_resume_count(self): + self.check_event_count(g1, E.PY_RESUME, 2) + + def test_return_count(self): + self.check_event_count(f1, E.PY_RETURN, 1) + + def test_call_count(self): + self.check_event_count(f2, E.CALL, 3) + + def test_c_return_count(self): + self.check_event_count(f2, E.C_RETURN, 2) + + +E = sys.monitoring.events + +SIMPLE_EVENTS = [ + (E.PY_START, "start"), + (E.PY_RESUME, "resume"), + (E.PY_RETURN, "return"), + (E.PY_YIELD, "yield"), + (E.JUMP, "jump"), + (E.BRANCH, "branch"), + (E.RAISE, "raise"), + (E.PY_UNWIND, "unwind"), + (E.EXCEPTION_HANDLED, "exception_handled"), + (E.C_RAISE, "c_raise"), + (E.C_RETURN, "c_return"), +] + +SIMPLE_EVENT_SET = functools.reduce(operator.or_, [ev for (ev, _) in SIMPLE_EVENTS], 0) | E.CALL + + +def just_pass(): + pass + +just_pass.events = [ + "py_call", + "start", + "return", +] + +def just_raise(): + raise Exception + +just_raise.events = [ + 'py_call', + "start", + "raise", + "unwind", +] + +def just_call(): + len([]) + +just_call.events = [ + 'py_call', + "start", + "c_call", + "c_return", + "return", +] + +def caught(): + try: + 1/0 + except Exception: + pass + +caught.events = [ + 'py_call', + "start", + "raise", + "exception_handled", + "branch", + "return", +] + +def nested_call(): + just_pass() + +nested_call.events = [ + "py_call", + "start", + "py_call", + "start", + "return", + "return", +] + +PY_CALLABLES = (types.FunctionType, types.MethodType) + +class MonitoringEventsBase(MonitoringTestBase): + + def gather_events(self, func): + events = [] + for event, event_name in SIMPLE_EVENTS: + def record(*args, event_name=event_name): + events.append(event_name) + sys.monitoring.register_callback(TEST_TOOL, event, record) + def record_call(code, offset, obj, arg): + if isinstance(obj, PY_CALLABLES): + events.append("py_call") + else: + events.append("c_call") + sys.monitoring.register_callback(TEST_TOOL, E.CALL, record_call) + sys.monitoring.set_events(TEST_TOOL, SIMPLE_EVENT_SET) + events = [] + try: + func() + except: + pass + sys.monitoring.set_events(TEST_TOOL, 0) + #Remove the final event, the call to `sys.monitoring.set_events` + events = events[:-1] + return events + + def check_events(self, func, expected=None): + events = self.gather_events(func) + if expected is None: + expected = func.events + self.assertEqual(events, expected) + + +class MonitoringEventsTest(MonitoringEventsBase, unittest.TestCase): + + def test_just_pass(self): + self.check_events(just_pass) + + def test_just_raise(self): + try: + self.check_events(just_raise) + except Exception: + pass + self.assertEqual(sys.monitoring.get_events(TEST_TOOL), 0) + + def test_just_call(self): + self.check_events(just_call) + + def test_caught(self): + self.check_events(caught) + + def test_nested_call(self): + self.check_events(nested_call) + +UP_EVENTS = (E.C_RETURN, E.C_RAISE, E.PY_RETURN, E.PY_UNWIND, E.PY_YIELD) +DOWN_EVENTS = (E.PY_START, E.PY_RESUME) + +from test.profilee import testfunc + +class SimulateProfileTest(MonitoringEventsBase, unittest.TestCase): + + def test_balanced(self): + events = self.gather_events(testfunc) + c = collections.Counter(events) + self.assertEqual(c["c_call"], c["c_return"]) + self.assertEqual(c["start"], c["return"] + c["unwind"]) + self.assertEqual(c["raise"], c["exception_handled"] + c["unwind"]) + + def test_frame_stack(self): + self.maxDiff = None + stack = [] + errors = [] + seen = set() + def up(*args): + frame = sys._getframe(1) + if not stack: + errors.append("empty") + else: + expected = stack.pop() + if frame != expected: + errors.append(f" Popping {frame} expected {expected}") + def down(*args): + frame = sys._getframe(1) + stack.append(frame) + seen.add(frame.f_code) + def call(code, offset, callable, arg): + if not isinstance(callable, PY_CALLABLES): + stack.append(sys._getframe(1)) + for event in UP_EVENTS: + sys.monitoring.register_callback(TEST_TOOL, event, up) + for event in DOWN_EVENTS: + sys.monitoring.register_callback(TEST_TOOL, event, down) + sys.monitoring.register_callback(TEST_TOOL, E.CALL, call) + sys.monitoring.set_events(TEST_TOOL, SIMPLE_EVENT_SET) + testfunc() + sys.monitoring.set_events(TEST_TOOL, 0) + self.assertEqual(errors, []) + self.assertEqual(stack, [sys._getframe()]) + self.assertEqual(len(seen), 9) + + +class CounterWithDisable: + + def __init__(self): + self.disable = False + self.count = 0 + + def __call__(self, *args): + self.count += 1 + if self.disable: + return sys.monitoring.DISABLE + + +class RecorderWithDisable: + + def __init__(self, events): + self.disable = False + self.events = events + + def __call__(self, code, event): + self.events.append(event) + if self.disable: + return sys.monitoring.DISABLE + + +class MontoringDisableAndRestartTest(MonitoringTestBase, unittest.TestCase): + + def test_disable(self): + try: + counter = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, counter) + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + self.assertEqual(counter.count, 0) + counter.count = 0 + f1() + self.assertEqual(counter.count, 1) + counter.disable = True + counter.count = 0 + f1() + self.assertEqual(counter.count, 1) + counter.count = 0 + f1() + self.assertEqual(counter.count, 0) + sys.monitoring.set_events(TEST_TOOL, 0) + finally: + sys.monitoring.restart_events() + + def test_restart(self): + try: + counter = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, counter) + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + counter.disable = True + f1() + counter.count = 0 + f1() + self.assertEqual(counter.count, 0) + sys.monitoring.restart_events() + counter.count = 0 + f1() + self.assertEqual(counter.count, 1) + sys.monitoring.set_events(TEST_TOOL, 0) + finally: + sys.monitoring.restart_events() + + +class MultipleMonitorsTest(MonitoringTestBase, unittest.TestCase): + + def test_two_same(self): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + counter1 = CounterWithDisable() + counter2 = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, counter1) + sys.monitoring.register_callback(TEST_TOOL2, E.PY_START, counter2) + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + sys.monitoring.set_events(TEST_TOOL2, E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL), E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL2), E.PY_START) + self.assertEqual(sys.monitoring._all_events(), {'PY_START': (1 << TEST_TOOL) | (1 << TEST_TOOL2)}) + counter1.count = 0 + counter2.count = 0 + f1() + count1 = counter1.count + count2 = counter2.count + self.assertEqual((count1, count2), (1, 1)) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.set_events(TEST_TOOL2, 0) + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, None) + sys.monitoring.register_callback(TEST_TOOL2, E.PY_START, None) + self.assertEqual(sys.monitoring._all_events(), {}) + + def test_three_same(self): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + counter1 = CounterWithDisable() + counter2 = CounterWithDisable() + counter3 = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, counter1) + sys.monitoring.register_callback(TEST_TOOL2, E.PY_START, counter2) + sys.monitoring.register_callback(TEST_TOOL3, E.PY_START, counter3) + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + sys.monitoring.set_events(TEST_TOOL2, E.PY_START) + sys.monitoring.set_events(TEST_TOOL3, E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL), E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL2), E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL3), E.PY_START) + self.assertEqual(sys.monitoring._all_events(), {'PY_START': (1 << TEST_TOOL) | (1 << TEST_TOOL2) | (1 << TEST_TOOL3)}) + counter1.count = 0 + counter2.count = 0 + counter3.count = 0 + f1() + count1 = counter1.count + count2 = counter2.count + count3 = counter3.count + self.assertEqual((count1, count2, count3), (1, 1, 1)) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.set_events(TEST_TOOL2, 0) + sys.monitoring.set_events(TEST_TOOL3, 0) + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, None) + sys.monitoring.register_callback(TEST_TOOL2, E.PY_START, None) + sys.monitoring.register_callback(TEST_TOOL3, E.PY_START, None) + self.assertEqual(sys.monitoring._all_events(), {}) + + def test_two_different(self): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + counter1 = CounterWithDisable() + counter2 = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, counter1) + sys.monitoring.register_callback(TEST_TOOL2, E.PY_RETURN, counter2) + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + sys.monitoring.set_events(TEST_TOOL2, E.PY_RETURN) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL), E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL2), E.PY_RETURN) + self.assertEqual(sys.monitoring._all_events(), {'PY_START': 1 << TEST_TOOL, 'PY_RETURN': 1 << TEST_TOOL2}) + counter1.count = 0 + counter2.count = 0 + f1() + count1 = counter1.count + count2 = counter2.count + self.assertEqual((count1, count2), (1, 1)) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.set_events(TEST_TOOL2, 0) + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, None) + sys.monitoring.register_callback(TEST_TOOL2, E.PY_RETURN, None) + self.assertEqual(sys.monitoring._all_events(), {}) + + def test_two_with_disable(self): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + counter1 = CounterWithDisable() + counter2 = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, counter1) + sys.monitoring.register_callback(TEST_TOOL2, E.PY_START, counter2) + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + sys.monitoring.set_events(TEST_TOOL2, E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL), E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL2), E.PY_START) + self.assertEqual(sys.monitoring._all_events(), {'PY_START': (1 << TEST_TOOL) | (1 << TEST_TOOL2)}) + counter1.count = 0 + counter2.count = 0 + counter1.disable = True + f1() + count1 = counter1.count + count2 = counter2.count + self.assertEqual((count1, count2), (1, 1)) + counter1.count = 0 + counter2.count = 0 + f1() + count1 = counter1.count + count2 = counter2.count + self.assertEqual((count1, count2), (0, 1)) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.set_events(TEST_TOOL2, 0) + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, None) + sys.monitoring.register_callback(TEST_TOOL2, E.PY_START, None) + self.assertEqual(sys.monitoring._all_events(), {}) + sys.monitoring.restart_events() + +class LineMonitoringTest(MonitoringTestBase, unittest.TestCase): + + def test_lines_single(self): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + events = [] + recorder = RecorderWithDisable(events) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, recorder) + sys.monitoring.set_events(TEST_TOOL, E.LINE) + f1() + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) + start = LineMonitoringTest.test_lines_single.__code__.co_firstlineno + self.assertEqual(events, [start+7, 14, start+8]) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) + self.assertEqual(sys.monitoring._all_events(), {}) + sys.monitoring.restart_events() + + def test_lines_loop(self): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + events = [] + recorder = RecorderWithDisable(events) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, recorder) + sys.monitoring.set_events(TEST_TOOL, E.LINE) + floop() + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) + start = LineMonitoringTest.test_lines_loop.__code__.co_firstlineno + self.assertEqual(events, [start+7, 21, 22, 22, 21, start+8]) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) + self.assertEqual(sys.monitoring._all_events(), {}) + sys.monitoring.restart_events() + + def test_lines_two(self): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + events = [] + recorder = RecorderWithDisable(events) + events2 = [] + recorder2 = RecorderWithDisable(events2) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, recorder) + sys.monitoring.register_callback(TEST_TOOL2, E.LINE, recorder2) + sys.monitoring.set_events(TEST_TOOL, E.LINE); sys.monitoring.set_events(TEST_TOOL2, E.LINE) + f1() + sys.monitoring.set_events(TEST_TOOL, 0); sys.monitoring.set_events(TEST_TOOL2, 0) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) + sys.monitoring.register_callback(TEST_TOOL2, E.LINE, None) + start = LineMonitoringTest.test_lines_two.__code__.co_firstlineno + expected = [start+10, 14, start+11] + self.assertEqual(events, expected) + self.assertEqual(events2, expected) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.set_events(TEST_TOOL2, 0) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) + sys.monitoring.register_callback(TEST_TOOL2, E.LINE, None) + self.assertEqual(sys.monitoring._all_events(), {}) + sys.monitoring.restart_events() + + def check_lines(self, func, expected, tool=TEST_TOOL): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + events = [] + recorder = RecorderWithDisable(events) + sys.monitoring.register_callback(tool, E.LINE, recorder) + sys.monitoring.set_events(tool, E.LINE) + func() + sys.monitoring.set_events(tool, 0) + sys.monitoring.register_callback(tool, E.LINE, None) + lines = [ line - func.__code__.co_firstlineno for line in events[1:-1] ] + self.assertEqual(lines, expected) + finally: + sys.monitoring.set_events(tool, 0) + + + def test_linear(self): + + def func(): + line = 1 + line = 2 + line = 3 + line = 4 + line = 5 + + self.check_lines(func, [1,2,3,4,5]) + + def test_branch(self): + def func(): + if "true".startswith("t"): + line = 2 + line = 3 + else: + line = 5 + line = 6 + + self.check_lines(func, [1,2,3,6]) + + def test_try_except(self): + + def func1(): + try: + line = 2 + line = 3 + except: + line = 5 + line = 6 + + self.check_lines(func1, [1,2,3,6]) + + def func2(): + try: + line = 2 + raise 3 + except: + line = 5 + line = 6 + + self.check_lines(func2, [1,2,3,4,5,6]) + + +class ExceptionRecorder: + + event_type = E.RAISE + + def __init__(self, events): + self.events = events + + def __call__(self, code, offset, exc): + self.events.append(("raise", type(exc))) + +class CheckEvents(MonitoringTestBase, unittest.TestCase): + + def check_events(self, func, expected, tool=TEST_TOOL, recorders=(ExceptionRecorder,)): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + event_list = [] + all_events = 0 + for recorder in recorders: + ev = recorder.event_type + sys.monitoring.register_callback(tool, ev, recorder(event_list)) + all_events |= ev + sys.monitoring.set_events(tool, all_events) + func() + sys.monitoring.set_events(tool, 0) + for recorder in recorders: + sys.monitoring.register_callback(tool, recorder.event_type, None) + self.assertEqual(event_list, expected) + finally: + sys.monitoring.set_events(tool, 0) + for recorder in recorders: + sys.monitoring.register_callback(tool, recorder.event_type, None) + +class StopiterationRecorder(ExceptionRecorder): + + event_type = E.STOP_ITERATION + +class ExceptionMontoringTest(CheckEvents): + + recorder = ExceptionRecorder + + def test_simple_try_except(self): + + def func1(): + try: + line = 2 + raise KeyError + except: + line = 5 + line = 6 + + self.check_events(func1, [("raise", KeyError)]) + + def gen(): + yield 1 + return 2 + + def implicit_stop_iteration(): + for _ in gen(): + pass + + self.check_events(implicit_stop_iteration, [("raise", StopIteration)], recorders=(StopiterationRecorder,)) + +class LineRecorder: + + event_type = E.LINE + + + def __init__(self, events): + self.events = events + + def __call__(self, code, line): + self.events.append(("line", code.co_name, line - code.co_firstlineno)) + +class CallRecorder: + + event_type = E.CALL + + def __init__(self, events): + self.events = events + + def __call__(self, code, offset, func, arg): + self.events.append(("call", func.__name__, arg)) + +class CEventRecorder: + + def __init__(self, events): + self.events = events + + def __call__(self, code, offset, func, arg): + self.events.append((self.event_name, func.__name__, arg)) + +class CReturnRecorder(CEventRecorder): + + event_type = E.C_RETURN + event_name = "C return" + +class CRaiseRecorder(CEventRecorder): + + event_type = E.C_RAISE + event_name = "C raise" + +MANY_RECORDERS = ExceptionRecorder, CallRecorder, LineRecorder, CReturnRecorder, CRaiseRecorder + +class TestManyEvents(CheckEvents): + + def test_simple(self): + + def func1(): + line1 = 1 + line2 = 2 + line3 = 3 + + self.check_events(func1, recorders = MANY_RECORDERS, expected = [ + ('line', 'check_events', 10), + ('call', 'func1', sys.monitoring.MISSING), + ('line', 'func1', 1), + ('line', 'func1', 2), + ('line', 'func1', 3), + ('line', 'check_events', 11), + ('call', 'set_events', 2)]) + + def test_c_call(self): + + def func2(): + line1 = 1 + [].append(2) + line3 = 3 + + self.check_events(func2, recorders = MANY_RECORDERS, expected = [ + ('line', 'check_events', 10), + ('call', 'func2', sys.monitoring.MISSING), + ('line', 'func2', 1), + ('line', 'func2', 2), + ('call', 'append', [2]), + ('C return', 'append', [2]), + ('line', 'func2', 3), + ('line', 'check_events', 11), + ('call', 'set_events', 2)]) + + def test_try_except(self): + + def func3(): + try: + line = 2 + raise KeyError + except: + line = 5 + line = 6 + + self.check_events(func3, recorders = MANY_RECORDERS, expected = [ + ('line', 'check_events', 10), + ('call', 'func3', sys.monitoring.MISSING), + ('line', 'func3', 1), + ('line', 'func3', 2), + ('line', 'func3', 3), + ('raise', KeyError), + ('line', 'func3', 4), + ('line', 'func3', 5), + ('line', 'func3', 6), + ('line', 'check_events', 11), + ('call', 'set_events', 2)]) + +class InstructionRecorder: + + event_type = E.INSTRUCTION + + def __init__(self, events): + self.events = events + + def __call__(self, code, offset): + # Filter out instructions in check_events to lower noise + if code.co_name != "check_events": + self.events.append(("instruction", code.co_name, offset)) + + +LINE_AND_INSTRUCTION_RECORDERS = InstructionRecorder, LineRecorder + +class TestLineAndInstructionEvents(CheckEvents): + maxDiff = None + + def test_simple(self): + + def func1(): + line1 = 1 + line2 = 2 + line3 = 3 + + self.check_events(func1, recorders = LINE_AND_INSTRUCTION_RECORDERS, expected = [ + ('line', 'check_events', 10), + ('line', 'func1', 1), + ('instruction', 'func1', 2), + ('instruction', 'func1', 4), + ('line', 'func1', 2), + ('instruction', 'func1', 6), + ('instruction', 'func1', 8), + ('line', 'func1', 3), + ('instruction', 'func1', 10), + ('instruction', 'func1', 12), + ('instruction', 'func1', 14), + ('line', 'check_events', 11)]) + + def test_c_call(self): + + def func2(): + line1 = 1 + [].append(2) + line3 = 3 + + self.check_events(func2, recorders = LINE_AND_INSTRUCTION_RECORDERS, expected = [ + ('line', 'check_events', 10), + ('line', 'func2', 1), + ('instruction', 'func2', 2), + ('instruction', 'func2', 4), + ('line', 'func2', 2), + ('instruction', 'func2', 6), + ('instruction', 'func2', 8), + ('instruction', 'func2', 28), + ('instruction', 'func2', 30), + ('instruction', 'func2', 38), + ('line', 'func2', 3), + ('instruction', 'func2', 40), + ('instruction', 'func2', 42), + ('instruction', 'func2', 44), + ('line', 'check_events', 11)]) + + def test_try_except(self): + + def func3(): + try: + line = 2 + raise KeyError + except: + line = 5 + line = 6 + + self.check_events(func3, recorders = LINE_AND_INSTRUCTION_RECORDERS, expected = [ + ('line', 'check_events', 10), + ('line', 'func3', 1), + ('instruction', 'func3', 2), + ('line', 'func3', 2), + ('instruction', 'func3', 4), + ('instruction', 'func3', 6), + ('line', 'func3', 3), + ('instruction', 'func3', 8), + ('instruction', 'func3', 18), + ('instruction', 'func3', 20), + ('line', 'func3', 4), + ('instruction', 'func3', 22), + ('line', 'func3', 5), + ('instruction', 'func3', 24), + ('instruction', 'func3', 26), + ('instruction', 'func3', 28), + ('line', 'func3', 6), + ('instruction', 'func3', 30), + ('instruction', 'func3', 32), + ('instruction', 'func3', 34), + ('line', 'check_events', 11)]) + +class TestInstallIncrementallly(MonitoringTestBase, unittest.TestCase): + + def check_events(self, func, must_include, tool=TEST_TOOL, recorders=(ExceptionRecorder,)): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + event_list = [] + all_events = 0 + for recorder in recorders: + all_events |= recorder.event_type + sys.monitoring.set_events(tool, all_events) + for recorder in recorders: + sys.monitoring.register_callback(tool, recorder.event_type, recorder(event_list)) + func() + sys.monitoring.set_events(tool, 0) + for recorder in recorders: + sys.monitoring.register_callback(tool, recorder.event_type, None) + for line in must_include: + self.assertIn(line, event_list) + finally: + sys.monitoring.set_events(tool, 0) + for recorder in recorders: + sys.monitoring.register_callback(tool, recorder.event_type, None) + + @staticmethod + def func1(): + line1 = 1 + + MUST_INCLUDE_LI = [ + ('instruction', 'func1', 2), + ('line', 'func1', 1), + ('instruction', 'func1', 4), + ('instruction', 'func1', 6)] + + def test_line_then_instruction(self): + recorders = [ LineRecorder, InstructionRecorder ] + self.check_events(self.func1, + recorders = recorders, must_include = self.EXPECTED_LI) + + def test_instruction_then_line(self): + recorders = [ InstructionRecorder, LineRecorderLowNoise ] + self.check_events(self.func1, + recorders = recorders, must_include = self.EXPECTED_LI) + + @staticmethod + def func2(): + len(()) + + MUST_INCLUDE_CI = [ + ('instruction', 'func2', 2), + ('call', 'func2', sys.monitoring.MISSING), + ('call', 'len', ()), + ('instruction', 'func2', 12), + ('instruction', 'func2', 14)] + + + + def test_line_then_instruction(self): + recorders = [ CallRecorder, InstructionRecorder ] + self.check_events(self.func2, + recorders = recorders, must_include = self.MUST_INCLUDE_CI) + + def test_instruction_then_line(self): + recorders = [ InstructionRecorder, CallRecorder ] + self.check_events(self.func2, + recorders = recorders, must_include = self.MUST_INCLUDE_CI) + +class TestLocalEvents(MonitoringTestBase, unittest.TestCase): + + def check_events(self, func, expected, tool=TEST_TOOL, recorders=(ExceptionRecorder,)): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + event_list = [] + all_events = 0 + for recorder in recorders: + ev = recorder.event_type + sys.monitoring.register_callback(tool, ev, recorder(event_list)) + all_events |= ev + sys.monitoring.set_local_events(tool, func.__code__, all_events) + func() + sys.monitoring.set_local_events(tool, func.__code__, 0) + for recorder in recorders: + sys.monitoring.register_callback(tool, recorder.event_type, None) + self.assertEqual(event_list, expected) + finally: + sys.monitoring.set_local_events(tool, func.__code__, 0) + for recorder in recorders: + sys.monitoring.register_callback(tool, recorder.event_type, None) + + + def test_simple(self): + + def func1(): + line1 = 1 + line2 = 2 + line3 = 3 + + self.check_events(func1, recorders = MANY_RECORDERS, expected = [ + ('line', 'func1', 1), + ('line', 'func1', 2), + ('line', 'func1', 3)]) + + def test_c_call(self): + + def func2(): + line1 = 1 + [].append(2) + line3 = 3 + + self.check_events(func2, recorders = MANY_RECORDERS, expected = [ + ('line', 'func2', 1), + ('line', 'func2', 2), + ('call', 'append', [2]), + ('C return', 'append', [2]), + ('line', 'func2', 3)]) + + def test_try_except(self): + + def func3(): + try: + line = 2 + raise KeyError + except: + line = 5 + line = 6 + + self.check_events(func3, recorders = MANY_RECORDERS, expected = [ + ('line', 'func3', 1), + ('line', 'func3', 2), + ('line', 'func3', 3), + ('raise', KeyError), + ('line', 'func3', 4), + ('line', 'func3', 5), + ('line', 'func3', 6)]) + + +class TestSetGetEvents(MonitoringTestBase, unittest.TestCase): + + def test_global(self): + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL), E.PY_START) + sys.monitoring.set_events(TEST_TOOL2, E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL2), E.PY_START) + sys.monitoring.set_events(TEST_TOOL, 0) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL), 0) + sys.monitoring.set_events(TEST_TOOL2,0) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL2), 0) + + def test_local(self): + code = f1.__code__ + sys.monitoring.set_local_events(TEST_TOOL, code, E.PY_START) + self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL, code), E.PY_START) + sys.monitoring.set_local_events(TEST_TOOL2, code, E.PY_START) + self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL2, code), E.PY_START) + sys.monitoring.set_local_events(TEST_TOOL, code, 0) + self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL, code), 0) + sys.monitoring.set_local_events(TEST_TOOL2, code, 0) + self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL2, code), 0) + +class TestUninitialized(unittest.TestCase, MonitoringTestBase): + + @staticmethod + def f(): + pass + + def test_get_local_events_uninitialized(self): + self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL, self.f.__code__), 0) diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index d7456d6d9480b2..89328b0bfa5abf 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1445,7 +1445,7 @@ class C(object): pass def func(): return sys._getframe() x = func() - check(x, size('3Pi3c7P2ic??2P')) + check(x, size('3Pii3c7P2ic??2P')) # function def func(): pass check(func, size('14Pi')) diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 4907c930e143d5..980321e169b9e5 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -2808,5 +2808,65 @@ def foo(*args): sys.settrace(sys.gettrace()) +class TestLinesAfterTraceStarted(TraceTestCase): + + def test_events(self): + tracer = Tracer() + sys._getframe().f_trace = tracer.trace + sys.settrace(tracer.trace) + line = 4 + line = 5 + sys.settrace(None) + self.compare_events( + TestLinesAfterTraceStarted.test_events.__code__.co_firstlineno, + tracer.events, [ + (4, 'line'), + (5, 'line'), + (6, 'line')]) + + +class TestSetLocalTrace(TraceTestCase): + + def test_with_branches(self): + + def tracefunc(frame, event, arg): + if frame.f_code.co_name == "func": + frame.f_trace = tracefunc + line = frame.f_lineno - frame.f_code.co_firstlineno + events.append((line, event)) + return tracefunc + + def func(arg = 1): + N = 1 + if arg >= 2: + not_reached = 3 + else: + reached = 5 + if arg >= 3: + not_reached = 7 + else: + reached = 9 + the_end = 10 + + EXPECTED_EVENTS = [ + (0, 'call'), + (1, 'line'), + (2, 'line'), + (5, 'line'), + (6, 'line'), + (9, 'line'), + (10, 'line'), + (10, 'return'), + ] + + events = [] + sys.settrace(tracefunc) + sys._getframe().f_trace = tracefunc + func() + self.assertEqual(events, EXPECTED_EVENTS) + sys.settrace(None) + + + if __name__ == "__main__": unittest.main() diff --git a/Makefile.pre.in b/Makefile.pre.in index 1c1bddcad82475..d3c60bd20fd4d6 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -394,7 +394,9 @@ PYTHON_OBJS= \ Python/import.o \ Python/importdl.o \ Python/initconfig.o \ + Python/instrumentation.o \ Python/intrinsics.o \ + Python/legacy_tracing.o \ Python/marshal.o \ Python/modsupport.o \ Python/mysnprintf.o \ diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-31-17-24-03.gh-issue-103082.isRUcV.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-31-17-24-03.gh-issue-103082.isRUcV.rst new file mode 100644 index 00000000000000..631ef4c7890450 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-03-31-17-24-03.gh-issue-103082.isRUcV.rst @@ -0,0 +1 @@ +Implement :pep:`669` Low Impact Monitoring for CPython. diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 755d0b85e7cf30..9b54c610581174 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -431,13 +431,13 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) if (_Py_next_func_version != 0) { _Py_next_func_version++; } + co->_co_monitoring = NULL; + co->_co_instrumentation_version = 0; /* not set */ co->co_weakreflist = NULL; co->co_extra = NULL; co->_co_cached = NULL; - co->_co_linearray_entry_size = 0; - co->_co_linearray = NULL; memcpy(_PyCode_CODE(co), PyBytes_AS_STRING(con->code), PyBytes_GET_SIZE(con->code)); int entry_point = 0; @@ -816,54 +816,6 @@ PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno) * source location tracking (co_lines/co_positions) ******************/ -/* Use co_linetable to compute the line number from a bytecode index, addrq. See - lnotab_notes.txt for the details of the lnotab representation. -*/ - -int -_PyCode_CreateLineArray(PyCodeObject *co) -{ - assert(co->_co_linearray == NULL); - PyCodeAddressRange bounds; - int size; - int max_line = 0; - _PyCode_InitAddressRange(co, &bounds); - while(_PyLineTable_NextAddressRange(&bounds)) { - if (bounds.ar_line > max_line) { - max_line = bounds.ar_line; - } - } - if (max_line < (1 << 15)) { - size = 2; - } - else { - size = 4; - } - co->_co_linearray = PyMem_Malloc(Py_SIZE(co)*size); - if (co->_co_linearray == NULL) { - PyErr_NoMemory(); - return -1; - } - co->_co_linearray_entry_size = size; - _PyCode_InitAddressRange(co, &bounds); - while(_PyLineTable_NextAddressRange(&bounds)) { - int start = bounds.ar_start / sizeof(_Py_CODEUNIT); - int end = bounds.ar_end / sizeof(_Py_CODEUNIT); - for (int index = start; index < end; index++) { - assert(index < (int)Py_SIZE(co)); - if (size == 2) { - assert(((int16_t)bounds.ar_line) == bounds.ar_line); - ((int16_t *)co->_co_linearray)[index] = bounds.ar_line; - } - else { - assert(size == 4); - ((int32_t *)co->_co_linearray)[index] = bounds.ar_line; - } - } - } - return 0; -} - int PyCode_Addr2Line(PyCodeObject *co, int addrq) { @@ -871,9 +823,6 @@ PyCode_Addr2Line(PyCodeObject *co, int addrq) return co->co_firstlineno; } assert(addrq >= 0 && addrq < _PyCode_NBYTES(co)); - if (co->_co_linearray) { - return _PyCode_LineNumberFromArray(co, addrq / sizeof(_Py_CODEUNIT)); - } PyCodeAddressRange bounds; _PyCode_InitAddressRange(co, &bounds); return _PyCode_CheckLineNumber(addrq, &bounds); @@ -1531,17 +1480,17 @@ PyCode_GetFreevars(PyCodeObject *code) } static void -deopt_code(_Py_CODEUNIT *instructions, Py_ssize_t len) +deopt_code(PyCodeObject *code, _Py_CODEUNIT *instructions) { + Py_ssize_t len = Py_SIZE(code); for (int i = 0; i < len; i++) { - _Py_CODEUNIT instruction = instructions[i]; - int opcode = _PyOpcode_Deopt[instruction.op.code]; + int opcode = _Py_GetBaseOpcode(code, i); int caches = _PyOpcode_Caches[opcode]; instructions[i].op.code = opcode; - while (caches--) { - instructions[++i].op.code = CACHE; - instructions[i].op.arg = 0; + for (int j = 1; j <= caches; j++) { + instructions[i+j].cache = 0; } + i += caches; } } @@ -1559,7 +1508,7 @@ _PyCode_GetCode(PyCodeObject *co) if (code == NULL) { return NULL; } - deopt_code((_Py_CODEUNIT *)PyBytes_AS_STRING(code), Py_SIZE(co)); + deopt_code(co, (_Py_CODEUNIT *)PyBytes_AS_STRING(code)); assert(co->_co_cached->_co_code == NULL); co->_co_cached->_co_code = Py_NewRef(code); return code; @@ -1693,6 +1642,30 @@ code_new_impl(PyTypeObject *type, int argcount, int posonlyargcount, return co; } +static void +free_monitoring_data(_PyCoMonitoringData *data) +{ + if (data == NULL) { + return; + } + if (data->tools) { + PyMem_Free(data->tools); + } + if (data->lines) { + PyMem_Free(data->lines); + } + if (data->line_tools) { + PyMem_Free(data->line_tools); + } + if (data->per_instruction_opcodes) { + PyMem_Free(data->per_instruction_opcodes); + } + if (data->per_instruction_tools) { + PyMem_Free(data->per_instruction_tools); + } + PyMem_Free(data); +} + static void code_dealloc(PyCodeObject *co) { @@ -1739,9 +1712,7 @@ code_dealloc(PyCodeObject *co) if (co->co_weakreflist != NULL) { PyObject_ClearWeakRefs((PyObject*)co); } - if (co->_co_linearray) { - PyMem_Free(co->_co_linearray); - } + free_monitoring_data(co->_co_monitoring); PyObject_Free(co); } @@ -1885,7 +1856,7 @@ code_hash(PyCodeObject *co) SCRAMBLE_IN(co->co_firstlineno); SCRAMBLE_IN(Py_SIZE(co)); for (int i = 0; i < Py_SIZE(co); i++) { - int deop = _PyOpcode_Deopt[_PyCode_CODE(co)[i].op.code]; + int deop = _Py_GetBaseOpcode(co, i); SCRAMBLE_IN(deop); SCRAMBLE_IN(_PyCode_CODE(co)[i].op.arg); i += _PyOpcode_Caches[deop]; @@ -2314,7 +2285,7 @@ _PyCode_ConstantKey(PyObject *op) void _PyStaticCode_Fini(PyCodeObject *co) { - deopt_code(_PyCode_CODE(co), Py_SIZE(co)); + deopt_code(co, _PyCode_CODE(co)); PyMem_Free(co->co_extra); if (co->_co_cached != NULL) { Py_CLEAR(co->_co_cached->_co_code); @@ -2329,10 +2300,8 @@ _PyStaticCode_Fini(PyCodeObject *co) PyObject_ClearWeakRefs((PyObject *)co); co->co_weakreflist = NULL; } - if (co->_co_linearray) { - PyMem_Free(co->_co_linearray); - co->_co_linearray = NULL; - } + free_monitoring_data(co->_co_monitoring); + co->_co_monitoring = NULL; } int diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 63590d58809e3e..ef0070199ab2c0 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -17,7 +17,6 @@ static PyMemberDef frame_memberlist[] = { {"f_trace_lines", T_BOOL, OFF(f_trace_lines), 0}, - {"f_trace_opcodes", T_BOOL, OFF(f_trace_opcodes), 0}, {NULL} /* Sentinel */ }; @@ -104,24 +103,29 @@ frame_getback(PyFrameObject *f, void *closure) return res; } -// Given the index of the effective opcode, scan back to construct the oparg -// with EXTENDED_ARG. This only works correctly with *unquickened* code, -// obtained via a call to _PyCode_GetCode! -static unsigned int -get_arg(const _Py_CODEUNIT *codestr, Py_ssize_t i) +static PyObject * +frame_gettrace_opcodes(PyFrameObject *f, void *closure) { - _Py_CODEUNIT word; - unsigned int oparg = codestr[i].op.arg; - if (i >= 1 && (word = codestr[i-1]).op.code == EXTENDED_ARG) { - oparg |= word.op.arg << 8; - if (i >= 2 && (word = codestr[i-2]).op.code == EXTENDED_ARG) { - oparg |= word.op.arg << 16; - if (i >= 3 && (word = codestr[i-3]).op.code == EXTENDED_ARG) { - oparg |= word.op.arg << 24; - } - } + PyObject *result = f->f_trace_opcodes ? Py_True : Py_False; + return Py_NewRef(result); +} + +static int +frame_settrace_opcodes(PyFrameObject *f, PyObject* value, void *Py_UNUSED(ignored)) +{ + if (!PyBool_Check(value)) { + PyErr_SetString(PyExc_TypeError, + "attribute value type must be bool"); + return -1; + } + if (value == Py_True) { + f->f_trace_opcodes = 1; + _PyInterpreterState_GET()->f_opcode_trace_set = true; } - return oparg; + else { + f->f_trace_opcodes = 0; + } + return 0; } /* Model the evaluation stack, to determine which jumps @@ -299,46 +303,52 @@ mark_stacks(PyCodeObject *code_obj, int len) while (todo) { todo = 0; /* Scan instructions */ - for (i = 0; i < len; i++) { + for (i = 0; i < len;) { int64_t next_stack = stacks[i]; + opcode = _Py_GetBaseOpcode(code_obj, i); + int oparg = 0; + while (opcode == EXTENDED_ARG) { + oparg = (oparg << 8) | code[i].op.arg; + i++; + opcode = _Py_GetBaseOpcode(code_obj, i); + stacks[i] = next_stack; + } + int next_i = i + _PyOpcode_Caches[opcode] + 1; if (next_stack == UNINITIALIZED) { + i = next_i; continue; } - opcode = code[i].op.code; + oparg = (oparg << 8) | code[i].op.arg; switch (opcode) { case POP_JUMP_IF_FALSE: case POP_JUMP_IF_TRUE: { int64_t target_stack; - int j = get_arg(code, i); - j += i + 1; + int j = next_i + oparg; assert(j < len); - if (stacks[j] == UNINITIALIZED && j < i) { - todo = 1; - } next_stack = pop_value(next_stack); target_stack = next_stack; assert(stacks[j] == UNINITIALIZED || stacks[j] == target_stack); stacks[j] = target_stack; - stacks[i+1] = next_stack; + stacks[next_i] = next_stack; break; } case SEND: - j = get_arg(code, i) + i + INLINE_CACHE_ENTRIES_SEND + 1; + j = oparg + i + INLINE_CACHE_ENTRIES_SEND + 1; assert(j < len); assert(stacks[j] == UNINITIALIZED || stacks[j] == next_stack); stacks[j] = next_stack; - stacks[i+1] = next_stack; + stacks[next_i] = next_stack; break; case JUMP_FORWARD: - j = get_arg(code, i) + i + 1; + j = oparg + i + 1; assert(j < len); assert(stacks[j] == UNINITIALIZED || stacks[j] == next_stack); stacks[j] = next_stack; break; case JUMP_BACKWARD: case JUMP_BACKWARD_NO_INTERRUPT: - j = i + 1 - get_arg(code, i); + j = i + 1 - oparg; assert(j >= 0); assert(j < len); if (stacks[j] == UNINITIALIZED && j < i) { @@ -350,13 +360,13 @@ mark_stacks(PyCodeObject *code_obj, int len) case GET_ITER: case GET_AITER: next_stack = push_value(pop_value(next_stack), Iterator); - stacks[i+1] = next_stack; + stacks[next_i] = next_stack; break; case FOR_ITER: { int64_t target_stack = push_value(next_stack, Object); - stacks[i+1] = target_stack; - j = get_arg(code, i) + 1 + INLINE_CACHE_ENTRIES_FOR_ITER + i; + stacks[next_i] = target_stack; + j = oparg + 1 + INLINE_CACHE_ENTRIES_FOR_ITER + i; assert(j < len); assert(stacks[j] == UNINITIALIZED || stacks[j] == target_stack); stacks[j] = target_stack; @@ -364,16 +374,16 @@ mark_stacks(PyCodeObject *code_obj, int len) } case END_ASYNC_FOR: next_stack = pop_value(pop_value(next_stack)); - stacks[i+1] = next_stack; + stacks[next_i] = next_stack; break; case PUSH_EXC_INFO: next_stack = push_value(next_stack, Except); - stacks[i+1] = next_stack; + stacks[next_i] = next_stack; break; case POP_EXCEPT: assert(top_of_stack(next_stack) == Except); next_stack = pop_value(next_stack); - stacks[i+1] = next_stack; + stacks[next_i] = next_stack; break; case RETURN_VALUE: assert(pop_value(next_stack) == EMPTY_STACK); @@ -389,57 +399,62 @@ mark_stacks(PyCodeObject *code_obj, int len) break; case PUSH_NULL: next_stack = push_value(next_stack, Null); - stacks[i+1] = next_stack; + stacks[next_i] = next_stack; break; case LOAD_GLOBAL: { - int j = get_arg(code, i); + int j = oparg; if (j & 1) { next_stack = push_value(next_stack, Null); } next_stack = push_value(next_stack, Object); - stacks[i+1] = next_stack; + stacks[next_i] = next_stack; break; } case LOAD_ATTR: { assert(top_of_stack(next_stack) == Object); - int j = get_arg(code, i); + int j = oparg; if (j & 1) { next_stack = pop_value(next_stack); next_stack = push_value(next_stack, Null); next_stack = push_value(next_stack, Object); } - stacks[i+1] = next_stack; + stacks[next_i] = next_stack; break; } case CALL: { - int args = get_arg(code, i); + int args = oparg; for (int j = 0; j < args+2; j++) { next_stack = pop_value(next_stack); } next_stack = push_value(next_stack, Object); - stacks[i+1] = next_stack; + stacks[next_i] = next_stack; break; } case SWAP: { - int n = get_arg(code, i); + int n = oparg; next_stack = stack_swap(next_stack, n); - stacks[i+1] = next_stack; + stacks[next_i] = next_stack; break; } case COPY: { - int n = get_arg(code, i); + int n = oparg; next_stack = push_value(next_stack, peek(next_stack, n)); - stacks[i+1] = next_stack; + stacks[next_i] = next_stack; break; } + case CACHE: + case RESERVED: + { + assert(0); + } default: { - int delta = PyCompile_OpcodeStackEffect(opcode, get_arg(code, i)); + int delta = PyCompile_OpcodeStackEffect(opcode, oparg); assert(delta != PY_INVALID_STACK_EFFECT); while (delta < 0) { next_stack = pop_value(next_stack); @@ -449,9 +464,10 @@ mark_stacks(PyCodeObject *code_obj, int len) next_stack = push_value(next_stack, Object); delta--; } - stacks[i+1] = next_stack; + stacks[next_i] = next_stack; } } + i = next_i; } /* Scan exception table */ unsigned char *start = (unsigned char *)PyBytes_AS_STRING(code_obj->co_exceptiontable); @@ -646,31 +662,43 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore * In addition, jumps are forbidden when not tracing, * as this is a debugging feature. */ - switch(PyThreadState_GET()->tracing_what) { - case PyTrace_EXCEPTION: - PyErr_SetString(PyExc_ValueError, - "can only jump from a 'line' trace event"); - return -1; - case PyTrace_CALL: + int what_event = PyThreadState_GET()->what_event; + if (what_event < 0) { + PyErr_Format(PyExc_ValueError, + "f_lineno can only be set in a trace function"); + return -1; + } + switch (what_event) { + case PY_MONITORING_EVENT_PY_RESUME: + case PY_MONITORING_EVENT_JUMP: + case PY_MONITORING_EVENT_BRANCH: + case PY_MONITORING_EVENT_LINE: + case PY_MONITORING_EVENT_PY_YIELD: + /* Setting f_lineno is allowed for the above events */ + break; + case PY_MONITORING_EVENT_PY_START: PyErr_Format(PyExc_ValueError, "can't jump from the 'call' trace event of a new frame"); return -1; - case PyTrace_LINE: - break; - case PyTrace_RETURN: - if (state == FRAME_SUSPENDED) { - break; - } - /* fall through */ - default: + case PY_MONITORING_EVENT_CALL: + case PY_MONITORING_EVENT_C_RETURN: PyErr_SetString(PyExc_ValueError, + "can't jump during a call"); + return -1; + case PY_MONITORING_EVENT_PY_RETURN: + case PY_MONITORING_EVENT_PY_UNWIND: + case PY_MONITORING_EVENT_PY_THROW: + case PY_MONITORING_EVENT_RAISE: + case PY_MONITORING_EVENT_C_RAISE: + case PY_MONITORING_EVENT_INSTRUCTION: + case PY_MONITORING_EVENT_EXCEPTION_HANDLED: + PyErr_Format(PyExc_ValueError, "can only jump from a 'line' trace event"); return -1; - } - if (!f->f_trace) { - PyErr_Format(PyExc_ValueError, - "f_lineno can only be set by a trace function"); - return -1; + default: + PyErr_SetString(PyExc_SystemError, + "unexpected event type"); + return -1; } int new_lineno; @@ -803,6 +831,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore start_stack = pop_value(start_stack); } /* Finally set the new lasti and return OK. */ + f->f_last_traced_line = new_lineno; f->f_lineno = 0; f->f_frame->prev_instr = _PyCode_CODE(f->f_frame->f_code) + best_addr; return 0; @@ -823,7 +852,10 @@ frame_settrace(PyFrameObject *f, PyObject* v, void *closure) if (v == Py_None) { v = NULL; } - Py_XSETREF(f->f_trace, Py_XNewRef(v)); + if (v != f->f_trace) { + Py_XSETREF(f->f_trace, Py_XNewRef(v)); + f->f_last_traced_line = -1; + } return 0; } @@ -838,6 +870,7 @@ static PyGetSetDef frame_getsetlist[] = { {"f_globals", (getter)frame_getglobals, NULL, NULL}, {"f_builtins", (getter)frame_getbuiltins, NULL, NULL}, {"f_code", (getter)frame_getcode, NULL, NULL}, + {"f_trace_opcodes", (getter)frame_gettrace_opcodes, (setter)frame_settrace_opcodes, NULL}, {0} }; @@ -1023,6 +1056,7 @@ _PyFrame_New_NoTrack(PyCodeObject *code) f->f_trace_opcodes = 0; f->f_fast_as_locals = 0; f->f_lineno = 0; + f->f_last_traced_line = -1; return f; } diff --git a/Objects/object.c b/Objects/object.c index 9dd5eb998217f6..fcd6fa59af8b65 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1978,6 +1978,7 @@ extern PyTypeObject _Py_GenericAliasIterType; extern PyTypeObject _PyMemoryIter_Type; extern PyTypeObject _PyLineIterator; extern PyTypeObject _PyPositionsIterator; +extern PyTypeObject _PyLegacyEventHandler_Type; static PyTypeObject* static_types[] = { // The two most important base types: must be initialized first and @@ -2075,6 +2076,7 @@ static PyTypeObject* static_types[] = { &_PyHamt_BitmapNode_Type, &_PyHamt_CollisionNode_Type, &_PyHamt_Type, + &_PyLegacyEventHandler_Type, &_PyInterpreterID_Type, &_PyLineIterator, &_PyManagedBuffer_Type, diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 28eced6d4b2862..e6ec239e3088b5 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -208,6 +208,8 @@ + + diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index e4faa89bb831d9..3f7cdbe370238d 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -211,6 +211,12 @@ Source Files + + Source Files + + + Source Files + Source Files diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 29f32db579fa40..5fdba3ace45dae 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -531,6 +531,8 @@ + + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 6a622fd93930ad..17fe4dce87da01 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -1175,6 +1175,12 @@ Source Files + + Source Files + + + Source Files + Python diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 72f85cc92b0c92..5c6398aba198f9 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -14,6 +14,7 @@ #include "pycore_function.h" #include "pycore_intrinsics.h" #include "pycore_long.h" // _PyLong_GetZero() +#include "pycore_instruments.h" #include "pycore_object.h" // _PyObject_GC_TRACK() #include "pycore_moduleobject.h" // PyModuleObject #include "pycore_opcode.h" // EXTRA_CASES @@ -134,11 +135,45 @@ dummy_func( inst(RESUME, (--)) { assert(tstate->cframe == &cframe); assert(frame == cframe.current_frame); - if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { + /* Possibly combine this with eval breaker */ + if (frame->f_code->_co_instrumentation_version != tstate->interp->monitoring_version) { + int err = _Py_Instrument(frame->f_code, tstate->interp); + ERROR_IF(err, error); + next_instr--; + } + else if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { goto handle_eval_breaker; } } + inst(INSTRUMENTED_RESUME, (--)) { + /* Possible performance enhancement: + * We need to check the eval breaker anyway, can we + * combine the instrument verison check and the eval breaker test? + */ + if (frame->f_code->_co_instrumentation_version != tstate->interp->monitoring_version) { + if (_Py_Instrument(frame->f_code, tstate->interp)) { + goto error; + } + next_instr--; + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation( + tstate, oparg > 0, frame, next_instr-1); + stack_pointer = _PyFrame_GetStackPointer(frame); + ERROR_IF(err, error); + if (frame->prev_instr != next_instr-1) { + /* Instrumentation has jumped */ + next_instr = frame->prev_instr; + DISPATCH(); + } + if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { + goto handle_eval_breaker; + } + } + } + inst(LOAD_CLOSURE, (-- value)) { /* We keep LOAD_CLOSURE so that the bytecode stays more readable. */ value = GETLOCAL(oparg); @@ -183,6 +218,34 @@ dummy_func( macro(END_FOR) = POP_TOP + POP_TOP; + inst(INSTRUMENTED_END_FOR, (receiver, value --)) { + /* Need to create a fake StopIteration error here, + * to conform to PEP 380 */ + if (PyGen_Check(receiver)) { + PyErr_SetObject(PyExc_StopIteration, value); + if (monitor_stop_iteration(tstate, frame, next_instr-1)) { + goto error; + } + PyErr_SetRaisedException(NULL); + } + DECREF_INPUTS(); + } + + inst(END_SEND, (receiver, value -- value)) { + Py_DECREF(receiver); + } + + inst(INSTRUMENTED_END_SEND, (receiver, value -- value)) { + if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { + PyErr_SetObject(PyExc_StopIteration, value); + if (monitor_stop_iteration(tstate, frame, next_instr-1)) { + goto error; + } + PyErr_SetRaisedException(NULL); + } + Py_DECREF(receiver); + } + inst(UNARY_NEGATIVE, (value -- res)) { res = PyNumber_Negative(value); DECREF_INPUTS(); @@ -222,7 +285,6 @@ dummy_func( inst(BINARY_OP_MULTIPLY_INT, (unused/1, left, right -- prod)) { - assert(cframe.use_tracing == 0); DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -233,7 +295,6 @@ dummy_func( } inst(BINARY_OP_MULTIPLY_FLOAT, (unused/1, left, right -- prod)) { - assert(cframe.use_tracing == 0); DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -243,7 +304,6 @@ dummy_func( } inst(BINARY_OP_SUBTRACT_INT, (unused/1, left, right -- sub)) { - assert(cframe.use_tracing == 0); DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -254,7 +314,6 @@ dummy_func( } inst(BINARY_OP_SUBTRACT_FLOAT, (unused/1, left, right -- sub)) { - assert(cframe.use_tracing == 0); DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -263,7 +322,6 @@ dummy_func( } inst(BINARY_OP_ADD_UNICODE, (unused/1, left, right -- res)) { - assert(cframe.use_tracing == 0); DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -280,7 +338,6 @@ dummy_func( // specializations, but there is no output. // At the end we just skip over the STORE_FAST. inst(BINARY_OP_INPLACE_ADD_UNICODE, (left, right --)) { - assert(cframe.use_tracing == 0); DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); _Py_CODEUNIT true_next = next_instr[INLINE_CACHE_ENTRIES_BINARY_OP]; @@ -310,7 +367,6 @@ dummy_func( } inst(BINARY_OP_ADD_FLOAT, (unused/1, left, right -- sum)) { - assert(cframe.use_tracing == 0); DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -320,7 +376,6 @@ dummy_func( } inst(BINARY_OP_ADD_INT, (unused/1, left, right -- sum)) { - assert(cframe.use_tracing == 0); DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -342,7 +397,6 @@ dummy_func( #if ENABLE_SPECIALIZATION _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_BinarySubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); @@ -386,7 +440,6 @@ dummy_func( } inst(BINARY_SUBSCR_LIST_INT, (unused/1, list, sub -- res)) { - assert(cframe.use_tracing == 0); DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); @@ -403,7 +456,6 @@ dummy_func( } inst(BINARY_SUBSCR_TUPLE_INT, (unused/1, tuple, sub -- res)) { - assert(cframe.use_tracing == 0); DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); @@ -420,7 +472,6 @@ dummy_func( } inst(BINARY_SUBSCR_DICT, (unused/1, dict, sub -- res)) { - assert(cframe.use_tracing == 0); DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); res = PyDict_GetItemWithError(dict, sub); @@ -479,7 +530,6 @@ dummy_func( inst(STORE_SUBSCR, (counter/1, v, container, sub -- )) { #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_StoreSubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); @@ -497,7 +547,6 @@ dummy_func( } inst(STORE_SUBSCR_LIST_INT, (unused/1, value, list, sub -- )) { - assert(cframe.use_tracing == 0); DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); @@ -517,7 +566,6 @@ dummy_func( } inst(STORE_SUBSCR_DICT, (unused/1, value, dict, sub -- )) { - assert(cframe.use_tracing == 0); DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); @@ -573,7 +621,6 @@ dummy_func( assert(EMPTY()); /* Restore previous cframe and return. */ tstate->cframe = cframe.previous; - tstate->cframe->use_tracing = cframe.use_tracing; assert(tstate->cframe->current_frame == frame->previous); assert(!_PyErr_Occurred(tstate)); _Py_LeaveRecursiveCallTstate(tstate); @@ -584,8 +631,24 @@ dummy_func( STACK_SHRINK(1); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); - TRACE_FUNCTION_EXIT(); - DTRACE_FUNCTION_EXIT(); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != &entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = cframe.current_frame = dying->previous; + _PyEvalFrameClearAndPop(tstate, dying); + _PyFrame_StackPush(frame, retval); + goto resume_frame; + } + + inst(INSTRUMENTED_RETURN_VALUE, (retval --)) { + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, next_instr-1, retval); + if (err) goto error; + STACK_SHRINK(1); + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_LeaveRecursiveCallPy(tstate); assert(frame != &entry_frame); // GH-99729: We need to unlink the frame *before* clearing it: @@ -601,8 +664,25 @@ dummy_func( Py_INCREF(retval); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); - TRACE_FUNCTION_EXIT(); - DTRACE_FUNCTION_EXIT(); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != &entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = cframe.current_frame = dying->previous; + _PyEvalFrameClearAndPop(tstate, dying); + _PyFrame_StackPush(frame, retval); + goto resume_frame; + } + + inst(INSTRUMENTED_RETURN_CONST, (--)) { + PyObject *retval = GETITEM(frame->f_code->co_consts, oparg); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, next_instr-1, retval); + if (err) goto error; + Py_INCREF(retval); + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); _Py_LeaveRecursiveCallPy(tstate); assert(frame != &entry_frame); // GH-99729: We need to unlink the frame *before* clearing it: @@ -730,7 +810,6 @@ dummy_func( #if ENABLE_SPECIALIZATION _PySendCache *cache = (_PySendCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_Send(receiver, next_instr); DISPATCH_SAME_OPARG(); @@ -739,6 +818,20 @@ dummy_func( DECREMENT_ADAPTIVE_COUNTER(cache->counter); #endif /* ENABLE_SPECIALIZATION */ assert(frame != &entry_frame); + if ((Py_TYPE(receiver) == &PyGen_Type || + Py_TYPE(receiver) == &PyCoro_Type) && ((PyGenObject *)receiver)->gi_frame_state < FRAME_EXECUTING) + { + PyGenObject *gen = (PyGenObject *)receiver; + _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; + frame->yield_offset = oparg; + STACK_SHRINK(1); + _PyFrame_StackPush(gen_frame, v); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + JUMPBY(INLINE_CACHE_ENTRIES_SEND + oparg); + DISPATCH_INLINED(gen_frame); + } if (Py_IsNone(v) && PyIter_Check(receiver)) { retval = Py_TYPE(receiver)->tp_iternext(receiver); } @@ -746,26 +839,22 @@ dummy_func( retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v); } if (retval == NULL) { - if (tstate->c_tracefunc != NULL - && _PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) - call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj, tstate, frame); + if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) + ) { + monitor_raise(tstate, frame, next_instr-1); + } if (_PyGen_FetchStopIterationValue(&retval) == 0) { assert(retval != NULL); JUMPBY(oparg); } else { - assert(retval == NULL); goto error; } } - else { - assert(retval != NULL); - } Py_DECREF(v); } inst(SEND_GEN, (unused/1, receiver, v -- receiver)) { - assert(cframe.use_tracing == 0); PyGenObject *gen = (PyGenObject *)receiver; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); @@ -782,6 +871,26 @@ dummy_func( DISPATCH_INLINED(gen_frame); } + inst(INSTRUMENTED_YIELD_VALUE, (retval -- unused)) { + assert(frame != &entry_frame); + PyGenObject *gen = _PyFrame_GetGenerator(frame); + gen->gi_frame_state = FRAME_SUSPENDED; + _PyFrame_SetStackPointer(frame, stack_pointer - 1); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_YIELD, + frame, next_instr-1, retval); + if (err) goto error; + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = cframe.current_frame = frame->previous; + gen_frame->previous = NULL; + frame->prev_instr -= frame->yield_offset; + _PyFrame_StackPush(frame, retval); + goto resume_frame; + } + inst(YIELD_VALUE, (retval -- unused)) { // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() @@ -790,8 +899,6 @@ dummy_func( PyGenObject *gen = _PyFrame_GetGenerator(frame); gen->gi_frame_state = FRAME_SUSPENDED; _PyFrame_SetStackPointer(frame, stack_pointer - 1); - TRACE_FUNCTION_EXIT(); - DTRACE_FUNCTION_EXIT(); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); @@ -930,7 +1037,6 @@ dummy_func( #if ENABLE_SPECIALIZATION _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_UnpackSequence(seq, next_instr, oparg); DISPATCH_SAME_OPARG(); @@ -994,7 +1100,6 @@ dummy_func( inst(STORE_ATTR, (counter/1, unused/3, v, owner --)) { #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - assert(cframe.use_tracing == 0); PyObject *name = GETITEM(frame->f_code->co_names, oparg); next_instr--; _Py_Specialize_StoreAttr(owner, next_instr, name); @@ -1111,7 +1216,6 @@ dummy_func( #if ENABLE_SPECIALIZATION _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); PyObject *name = GETITEM(frame->f_code->co_names, oparg>>1); next_instr--; _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); @@ -1163,7 +1267,6 @@ dummy_func( } inst(LOAD_GLOBAL_MODULE, (unused/1, index/1, version/1, unused/1 -- null if (oparg & 1), res)) { - assert(cframe.use_tracing == 0); DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); PyDictObject *dict = (PyDictObject *)GLOBALS(); DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); @@ -1177,11 +1280,11 @@ dummy_func( } inst(LOAD_GLOBAL_BUILTIN, (unused/1, index/1, mod_version/1, bltn_version/1 -- null if (oparg & 1), res)) { - assert(cframe.use_tracing == 0); DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); DEOPT_IF(!PyDict_CheckExact(BUILTINS()), LOAD_GLOBAL); PyDictObject *mdict = (PyDictObject *)GLOBALS(); PyDictObject *bdict = (PyDictObject *)BUILTINS(); + assert(opcode == LOAD_GLOBAL_BUILTIN); DEOPT_IF(mdict->ma_keys->dk_version != mod_version, LOAD_GLOBAL); DEOPT_IF(bdict->ma_keys->dk_version != bltn_version, LOAD_GLOBAL); assert(DK_IS_UNICODE(bdict->ma_keys)); @@ -1465,7 +1568,6 @@ dummy_func( #if ENABLE_SPECIALIZATION _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); PyObject *name = GETITEM(frame->f_code->co_names, oparg>>1); next_instr--; _Py_Specialize_LoadAttr(owner, next_instr, name); @@ -1511,7 +1613,6 @@ dummy_func( } inst(LOAD_ATTR_INSTANCE_VALUE, (unused/1, type_version/2, index/1, unused/5, owner -- res2 if (oparg & 1), res)) { - assert(cframe.use_tracing == 0); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -1528,7 +1629,6 @@ dummy_func( } inst(LOAD_ATTR_MODULE, (unused/1, type_version/2, index/1, unused/5, owner -- res2 if (oparg & 1), res)) { - assert(cframe.use_tracing == 0); DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict != NULL); @@ -1545,7 +1645,6 @@ dummy_func( } inst(LOAD_ATTR_WITH_HINT, (unused/1, type_version/2, index/1, unused/5, owner -- res2 if (oparg & 1), res)) { - assert(cframe.use_tracing == 0); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -1576,7 +1675,6 @@ dummy_func( } inst(LOAD_ATTR_SLOT, (unused/1, type_version/2, index/1, unused/5, owner -- res2 if (oparg & 1), res)) { - assert(cframe.use_tracing == 0); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -1590,7 +1688,6 @@ dummy_func( } inst(LOAD_ATTR_CLASS, (unused/1, type_version/2, unused/2, descr/4, cls -- res2 if (oparg & 1), res)) { - assert(cframe.use_tracing == 0); DEOPT_IF(!PyType_Check(cls), LOAD_ATTR); DEOPT_IF(((PyTypeObject *)cls)->tp_version_tag != type_version, @@ -1606,7 +1703,6 @@ dummy_func( } inst(LOAD_ATTR_PROPERTY, (unused/1, type_version/2, func_version/2, fget/4, owner -- unused if (oparg & 1), unused)) { - assert(cframe.use_tracing == 0); DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); @@ -1632,7 +1728,6 @@ dummy_func( } inst(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, (unused/1, type_version/2, func_version/2, getattribute/4, owner -- unused if (oparg & 1), unused)) { - assert(cframe.use_tracing == 0); DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); @@ -1660,7 +1755,6 @@ dummy_func( } inst(STORE_ATTR_INSTANCE_VALUE, (unused/1, type_version/2, index/1, value, owner --)) { - assert(cframe.use_tracing == 0); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -1681,7 +1775,6 @@ dummy_func( } inst(STORE_ATTR_WITH_HINT, (unused/1, type_version/2, hint/1, value, owner --)) { - assert(cframe.use_tracing == 0); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -1723,7 +1816,6 @@ dummy_func( } inst(STORE_ATTR_SLOT, (unused/1, type_version/2, index/1, value, owner --)) { - assert(cframe.use_tracing == 0); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -1746,7 +1838,6 @@ dummy_func( #if ENABLE_SPECIALIZATION _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_CompareOp(left, right, next_instr, oparg); DISPATCH_SAME_OPARG(); @@ -1761,7 +1852,6 @@ dummy_func( } inst(COMPARE_OP_FLOAT, (unused/1, left, right -- res)) { - assert(cframe.use_tracing == 0); DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); @@ -1777,7 +1867,6 @@ dummy_func( // Similar to COMPARE_OP_FLOAT inst(COMPARE_OP_INT, (unused/1, left, right -- res)) { - assert(cframe.use_tracing == 0); DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); @@ -1797,7 +1886,6 @@ dummy_func( // Similar to COMPARE_OP_FLOAT, but for ==, != only inst(COMPARE_OP_STR, (unused/1, left, right -- res)) { - assert(cframe.use_tracing == 0); DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); @@ -2044,7 +2132,6 @@ dummy_func( #if ENABLE_SPECIALIZATION _PyForIterCache *cache = (_PyForIterCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_ForIter(iter, next_instr, oparg); DISPATCH_SAME_OPARG(); @@ -2059,13 +2146,12 @@ dummy_func( if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { goto error; } - else if (tstate->c_tracefunc != NULL) { - call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj, tstate, frame); - } + monitor_raise(tstate, frame, next_instr-1); _PyErr_Clear(tstate); } /* iterator ended normally */ - assert(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == END_FOR); + assert(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == END_FOR || + next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == INSTRUMENTED_END_FOR); Py_DECREF(iter); STACK_SHRINK(1); /* Jump forward oparg, then skip following END_FOR instruction */ @@ -2075,8 +2161,35 @@ dummy_func( // Common case: no jump, leave it to the code generator } + inst(INSTRUMENTED_FOR_ITER, ( -- )) { + _Py_CODEUNIT *here = next_instr-1; + _Py_CODEUNIT *target; + PyObject *iter = TOP(); + PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); + if (next != NULL) { + PUSH(next); + target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER; + } + else { + if (_PyErr_Occurred(tstate)) { + if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + goto error; + } + monitor_raise(tstate, frame, here); + _PyErr_Clear(tstate); + } + /* iterator ended normally */ + assert(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == END_FOR || + next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == INSTRUMENTED_END_FOR); + STACK_SHRINK(1); + Py_DECREF(iter); + /* Skip END_FOR */ + target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1; + } + INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); + } + inst(FOR_ITER_LIST, (unused/1, iter -- iter, next)) { - assert(cframe.use_tracing == 0); DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); _PyListIterObject *it = (_PyListIterObject *)iter; STAT_INC(FOR_ITER, hit); @@ -2099,7 +2212,6 @@ dummy_func( } inst(FOR_ITER_TUPLE, (unused/1, iter -- iter, next)) { - assert(cframe.use_tracing == 0); _PyTupleIterObject *it = (_PyTupleIterObject *)iter; DEOPT_IF(Py_TYPE(it) != &PyTupleIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -2122,7 +2234,6 @@ dummy_func( } inst(FOR_ITER_RANGE, (unused/1, iter -- iter, next)) { - assert(cframe.use_tracing == 0); _PyRangeIterObject *r = (_PyRangeIterObject *)iter; DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -2143,7 +2254,6 @@ dummy_func( } inst(FOR_ITER_GEN, (unused/1, iter -- iter, unused)) { - assert(cframe.use_tracing == 0); PyGenObject *gen = (PyGenObject *)iter; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); @@ -2155,7 +2265,8 @@ dummy_func( gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER + oparg); - assert(next_instr->op.code == END_FOR); + assert(next_instr->op.code == END_FOR || + next_instr->op.code == INSTRUMENTED_END_FOR); DISPATCH_INLINED(gen_frame); } @@ -2264,7 +2375,6 @@ dummy_func( inst(LOAD_ATTR_METHOD_WITH_VALUES, (unused/1, type_version/2, keys_version/2, descr/4, self -- res2 if (oparg & 1), res)) { /* Cached method object */ - assert(cframe.use_tracing == 0); PyTypeObject *self_cls = Py_TYPE(self); assert(type_version != 0); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); @@ -2283,7 +2393,6 @@ dummy_func( } inst(LOAD_ATTR_METHOD_NO_DICT, (unused/1, type_version/2, unused/2, descr/4, self -- res2 if (oparg & 1), res)) { - assert(cframe.use_tracing == 0); PyTypeObject *self_cls = Py_TYPE(self); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); assert(self_cls->tp_dictoffset == 0); @@ -2296,7 +2405,6 @@ dummy_func( } inst(LOAD_ATTR_METHOD_LAZY_DICT, (unused/1, type_version/2, unused/2, descr/4, self -- res2 if (oparg & 1), res)) { - assert(cframe.use_tracing == 0); PyTypeObject *self_cls = Py_TYPE(self); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); Py_ssize_t dictoffset = self_cls->tp_dictoffset; @@ -2318,6 +2426,21 @@ dummy_func( kwnames = GETITEM(frame->f_code->co_consts, oparg); } + inst(INSTRUMENTED_CALL, ( -- )) { + int is_meth = PEEK(oparg+2) != NULL; + int total_args = oparg + is_meth; + PyObject *function = PEEK(total_args + 1); + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PEEK(total_args); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, next_instr-1, function, arg); + ERROR_IF(err, error); + _PyCallCache *cache = (_PyCallCache *)next_instr; + INCREMENT_ADAPTIVE_COUNTER(cache->counter); + GO_TO_INSTRUCTION(CALL); + } + // Cache layout: counter/1, func_version/2 // Neither CALL_INTRINSIC_1/2 nor CALL_FUNCTION_EX are members! family(call, INLINE_CACHE_ENTRIES_CALL) = { @@ -2359,7 +2482,6 @@ dummy_func( #if ENABLE_SPECIALIZATION _PyCallCache *cache = (_PyCallCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_Call(callable, next_instr, total_args, kwnames); DISPATCH_SAME_OPARG(); @@ -2402,16 +2524,26 @@ dummy_func( DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ - if (cframe.use_tracing) { - res = trace_call_function( - tstate, callable, args, - positional_args, kwnames); - } - else { - res = PyObject_Vectorcall( - callable, args, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames); + res = PyObject_Vectorcall( + callable, args, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames); + if (opcode == INSTRUMENTED_CALL) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PEEK(total_args); + if (res == NULL) { + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, next_instr-1, callable, arg); + } + else { + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, next_instr-1, callable, arg); + if (err < 0) { + Py_CLEAR(res); + } + } } kwnames = NULL; assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -2504,7 +2636,6 @@ dummy_func( inst(CALL_NO_KW_TYPE_1, (unused/1, unused/2, null, callable, args[oparg] -- res)) { assert(kwnames == NULL); - assert(cframe.use_tracing == 0); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); PyObject *obj = args[0]; @@ -2517,7 +2648,6 @@ dummy_func( inst(CALL_NO_KW_STR_1, (unused/1, unused/2, null, callable, args[oparg] -- res)) { assert(kwnames == NULL); - assert(cframe.use_tracing == 0); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL); @@ -2570,7 +2700,6 @@ dummy_func( } inst(CALL_NO_KW_BUILTIN_O, (unused/1, unused/2, method, callable, args[oparg] -- res)) { - assert(cframe.use_tracing == 0); /* Builtin METH_O functions */ assert(kwnames == NULL); int is_meth = method != NULL; @@ -2602,7 +2731,6 @@ dummy_func( } inst(CALL_NO_KW_BUILTIN_FAST, (unused/1, unused/2, method, callable, args[oparg] -- res)) { - assert(cframe.use_tracing == 0); /* Builtin METH_FASTCALL functions, without keywords */ assert(kwnames == NULL); int is_meth = method != NULL; @@ -2638,7 +2766,6 @@ dummy_func( } inst(CALL_BUILTIN_FAST_WITH_KEYWORDS, (unused/1, unused/2, method, callable, args[oparg] -- res)) { - assert(cframe.use_tracing == 0); /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ int is_meth = method != NULL; int total_args = oparg; @@ -2674,7 +2801,6 @@ dummy_func( } inst(CALL_NO_KW_LEN, (unused/1, unused/2, method, callable, args[oparg] -- res)) { - assert(cframe.use_tracing == 0); assert(kwnames == NULL); /* len(o) */ int is_meth = method != NULL; @@ -2702,7 +2828,6 @@ dummy_func( } inst(CALL_NO_KW_ISINSTANCE, (unused/1, unused/2, method, callable, args[oparg] -- res)) { - assert(cframe.use_tracing == 0); assert(kwnames == NULL); /* isinstance(o, o2) */ int is_meth = method != NULL; @@ -2733,7 +2858,6 @@ dummy_func( // This is secretly a super-instruction inst(CALL_NO_KW_LIST_APPEND, (unused/1, unused/2, method, self, args[oparg] -- unused)) { - assert(cframe.use_tracing == 0); assert(kwnames == NULL); assert(oparg == 1); assert(method != NULL); @@ -2882,12 +3006,14 @@ dummy_func( CHECK_EVAL_BREAKER(); } + inst(INSTRUMENTED_CALL_FUNCTION_EX, ( -- )) { + GO_TO_INSTRUCTION(CALL_FUNCTION_EX); + } + inst(CALL_FUNCTION_EX, (unused, func, callargs, kwargs if (oparg & 1) -- result)) { - if (oparg & 1) { - // DICT_MERGE is called before this opcode if there are kwargs. - // It converts all dict subtypes in kwargs into regular dicts. - assert(PyDict_CheckExact(kwargs)); - } + // DICT_MERGE is called before this opcode if there are kwargs. + // It converts all dict subtypes in kwargs into regular dicts. + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); if (!PyTuple_CheckExact(callargs)) { if (check_args_iterable(tstate, func, callargs) < 0) { goto error; @@ -2899,10 +3025,35 @@ dummy_func( Py_SETREF(callargs, tuple); } assert(PyTuple_CheckExact(callargs)); - - result = do_call_core(tstate, func, callargs, kwargs, cframe.use_tracing); + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); + if (opcode == INSTRUMENTED_CALL_FUNCTION_EX && + !PyFunction_Check(func) && !PyMethod_Check(func) + ) { + PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? + PyTuple_GET_ITEM(callargs, 0) : Py_None; + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, next_instr-1, func, arg); + if (err) goto error; + result = PyObject_Call(func, callargs, kwargs); + if (result == NULL) { + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, next_instr-1, func, arg); + } + else { + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, next_instr-1, func, arg); + if (err < 0) { + Py_CLEAR(result); + } + } + } + else { + result = PyObject_Call(func, callargs, kwargs); + } DECREF_INPUTS(); - assert(PEEK(3 + (oparg & 1)) == NULL); ERROR_IF(result == NULL, error); CHECK_EVAL_BREAKER(); @@ -3018,7 +3169,6 @@ dummy_func( #if ENABLE_SPECIALIZATION _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, &GETLOCAL(0)); DISPATCH_SAME_OPARG(); @@ -3039,9 +3189,105 @@ dummy_func( assert(oparg >= 2); } - inst(EXTENDED_ARG, (--)) { + inst(INSTRUMENTED_LINE, ( -- )) { + _Py_CODEUNIT *here = next_instr-1; + _PyFrame_SetStackPointer(frame, stack_pointer); + int original_opcode = _Py_call_instrumentation_line( + tstate, frame, here); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (original_opcode < 0) { + next_instr = here+1; + goto error; + } + next_instr = frame->prev_instr; + if (next_instr != here) { + DISPATCH(); + } + if (_PyOpcode_Caches[original_opcode]) { + _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); + INCREMENT_ADAPTIVE_COUNTER(cache->counter); + } + opcode = original_opcode; + DISPATCH_GOTO(); + } + + inst(INSTRUMENTED_INSTRUCTION, ( -- )) { + int next_opcode = _Py_call_instrumentation_instruction( + tstate, frame, next_instr-1); + ERROR_IF(next_opcode < 0, error); + next_instr--; + if (_PyOpcode_Caches[next_opcode]) { + _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); + INCREMENT_ADAPTIVE_COUNTER(cache->counter); + } + assert(next_opcode > 0 && next_opcode < 256); + opcode = next_opcode; + DISPATCH_GOTO(); + } + + inst(INSTRUMENTED_JUMP_FORWARD, ( -- )) { + INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); + } + + inst(INSTRUMENTED_JUMP_BACKWARD, ( -- )) { + INSTRUMENTED_JUMP(next_instr-1, next_instr-oparg, PY_MONITORING_EVENT_JUMP); + CHECK_EVAL_BREAKER(); + } + + inst(INSTRUMENTED_POP_JUMP_IF_TRUE, ( -- )) { + PyObject *cond = POP(); + int err = PyObject_IsTrue(cond); + Py_DECREF(cond); + ERROR_IF(err < 0, error); + _Py_CODEUNIT *here = next_instr-1; + assert(err == 0 || err == 1); + int offset = err*oparg; + INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + } + + inst(INSTRUMENTED_POP_JUMP_IF_FALSE, ( -- )) { + PyObject *cond = POP(); + int err = PyObject_IsTrue(cond); + Py_DECREF(cond); + ERROR_IF(err < 0, error); + _Py_CODEUNIT *here = next_instr-1; + assert(err == 0 || err == 1); + int offset = (1-err)*oparg; + INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + } + + inst(INSTRUMENTED_POP_JUMP_IF_NONE, ( -- )) { + PyObject *value = POP(); + _Py_CODEUNIT *here = next_instr-1; + int offset; + if (Py_IsNone(value)) { + _Py_DECREF_NO_DEALLOC(value); + offset = oparg; + } + else { + Py_DECREF(value); + offset = 0; + } + INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + } + + inst(INSTRUMENTED_POP_JUMP_IF_NOT_NONE, ( -- )) { + PyObject *value = POP(); + _Py_CODEUNIT *here = next_instr-1; + int offset; + if (Py_IsNone(value)) { + _Py_DECREF_NO_DEALLOC(value); + offset = 0; + } + else { + Py_DECREF(value); + offset = oparg; + } + INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + } + + inst(EXTENDED_ARG, ( -- )) { assert(oparg); - assert(cframe.use_tracing == 0); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; PRE_DISPATCH_GOTO(); @@ -3049,6 +3295,12 @@ dummy_func( } inst(CACHE, (--)) { + assert(0 && "Executing a cache."); + Py_UNREACHABLE(); + } + + inst(RESERVED, (--)) { + assert(0 && "Executing RESERVED instruction."); Py_UNREACHABLE(); } diff --git a/Python/ceval.c b/Python/ceval.c index 7d60cf987e9c47..a38c9ec9ad5f9c 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -10,6 +10,7 @@ #include "pycore_function.h" #include "pycore_intrinsics.h" #include "pycore_long.h" // _PyLong_GetZero() +#include "pycore_instruments.h" #include "pycore_object.h" // _PyObject_GC_TRACK() #include "pycore_moduleobject.h" // PyModuleObject #include "pycore_opcode.h" // EXTRA_CASES @@ -92,13 +93,6 @@ #define _Py_atomic_load_relaxed_int32(ATOMIC_VAL) _Py_atomic_load_relaxed(ATOMIC_VAL) #endif -/* Forward declarations */ -static PyObject *trace_call_function( - PyThreadState *tstate, PyObject *callable, PyObject **stack, - Py_ssize_t oparg, PyObject *kwnames); -static PyObject * do_call_core( - PyThreadState *tstate, PyObject *func, - PyObject *callargs, PyObject *kwdict, int use_tracing); #ifdef LLTRACE static void @@ -179,19 +173,22 @@ lltrace_resume_frame(_PyInterpreterFrame *frame) PyErr_SetRaisedException(exc); } #endif -static int call_trace(Py_tracefunc, PyObject *, - PyThreadState *, _PyInterpreterFrame *, - int, PyObject *); -static int call_trace_protected(Py_tracefunc, PyObject *, - PyThreadState *, _PyInterpreterFrame *, - int, PyObject *); -static void call_exc_trace(Py_tracefunc, PyObject *, - PyThreadState *, _PyInterpreterFrame *); -static int maybe_call_line_trace(Py_tracefunc, PyObject *, - PyThreadState *, _PyInterpreterFrame *, int); -static void maybe_dtrace_line(_PyInterpreterFrame *, PyTraceInfo *, int); -static void dtrace_function_entry(_PyInterpreterFrame *); -static void dtrace_function_return(_PyInterpreterFrame *); + +static void monitor_raise(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr); +static int monitor_stop_iteration(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr); +static void monitor_unwind(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr); +static void monitor_handled(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr, PyObject *exc); +static void monitor_throw(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr); static PyObject * import_name(PyThreadState *, _PyInterpreterFrame *, PyObject *, PyObject *, PyObject *); @@ -217,21 +214,6 @@ _PyEvalFrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame); "cannot access free variable '%s' where it is not associated with a" \ " value in enclosing scope" -#ifndef NDEBUG -/* Ensure that tstate is valid: sanity check for PyEval_AcquireThread() and - PyEval_RestoreThread(). Detect if tstate memory was freed. It can happen - when a thread continues to run after Python finalization, especially - daemon threads. */ -static int -is_tstate_valid(PyThreadState *tstate) -{ - assert(!_PyMem_IsPtrFreed(tstate)); - assert(!_PyMem_IsPtrFreed(tstate->interp)); - return 1; -} -#endif - - #ifdef HAVE_ERRNO_H #include #endif @@ -596,63 +578,6 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) #include "ceval_macros.h" -static int -trace_function_entry(PyThreadState *tstate, _PyInterpreterFrame *frame) -{ - if (tstate->c_tracefunc != NULL) { - /* tstate->c_tracefunc, if defined, is a - function that will be called on *every* entry - to a code block. Its return value, if not - None, is a function that will be called at - the start of each executed line of code. - (Actually, the function must return itself - in order to continue tracing.) The trace - functions are called with three arguments: - a pointer to the current frame, a string - indicating why the function is called, and - an argument which depends on the situation. - The global trace function is also called - whenever an exception is detected. */ - if (call_trace_protected(tstate->c_tracefunc, - tstate->c_traceobj, - tstate, frame, - PyTrace_CALL, Py_None)) { - /* Trace function raised an error */ - return -1; - } - } - if (tstate->c_profilefunc != NULL) { - /* Similar for c_profilefunc, except it needn't - return itself and isn't called for "line" events */ - if (call_trace_protected(tstate->c_profilefunc, - tstate->c_profileobj, - tstate, frame, - PyTrace_CALL, Py_None)) { - /* Profile function raised an error */ - return -1; - } - } - return 0; -} - -static int -trace_function_exit(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject *retval) -{ - if (tstate->c_tracefunc) { - if (call_trace_protected(tstate->c_tracefunc, tstate->c_traceobj, - tstate, frame, PyTrace_RETURN, retval)) { - return -1; - } - } - if (tstate->c_profilefunc) { - if (call_trace_protected(tstate->c_profilefunc, tstate->c_profileobj, - tstate, frame, PyTrace_RETURN, retval)) { - return -1; - } - } - return 0; -} - int _Py_CheckRecursiveCallPy( PyThreadState *tstate) @@ -730,7 +655,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int * strict stack discipline must be maintained. */ _PyCFrame *prev_cframe = tstate->cframe; - cframe.use_tracing = prev_cframe->use_tracing; cframe.previous = prev_cframe; tstate->cframe = &cframe; @@ -765,8 +689,11 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int if (_Py_EnterRecursivePy(tstate)) { goto exit_unwind; } - TRACE_FUNCTION_THROW_ENTRY(); - DTRACE_FUNCTION_ENTRY(); + /* Because this avoids the RESUME, + * we need to update instrumentation */ + _Py_Instrument(frame->f_code, tstate->interp); + monitor_throw(tstate, frame, frame->prev_instr); + /* TO DO -- Monitor throw entry. */ goto resume_with_error; } @@ -781,15 +708,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int assert(_PyInterpreterFrame_LASTI(frame) >= -1); \ /* Jump back to the last instruction executed... */ \ next_instr = frame->prev_instr + 1; \ - stack_pointer = _PyFrame_GetStackPointer(frame); \ - /* Set stackdepth to -1. \ - Update when returning or calling trace function. \ - Having stackdepth <= 0 ensures that invalid \ - values are not visible to the cycle GC. \ - We choose -1 rather than 0 to assist debugging. \ - */ \ - frame->stacktop = -1; - + stack_pointer = _PyFrame_GetStackPointer(frame); start_frame: if (_Py_EnterRecursivePy(tstate)) { @@ -845,91 +764,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #include "generated_cases.c.h" -#if USE_COMPUTED_GOTOS - TARGET_DO_TRACING: -#else - case DO_TRACING: -#endif - { - assert(cframe.use_tracing); - assert(tstate->tracing == 0); - if (INSTR_OFFSET() >= frame->f_code->_co_firsttraceable) { - int instr_prev = _PyInterpreterFrame_LASTI(frame); - frame->prev_instr = next_instr; - NEXTOPARG(); - // No _PyOpcode_Deopt here, since RESUME has no optimized forms: - if (opcode == RESUME) { - if (oparg < 2) { - CHECK_EVAL_BREAKER(); - } - /* Call tracing */ - TRACE_FUNCTION_ENTRY(); - DTRACE_FUNCTION_ENTRY(); - } - else { - /* line-by-line tracing support */ - if (PyDTrace_LINE_ENABLED()) { - maybe_dtrace_line(frame, &tstate->trace_info, instr_prev); - } - - if (cframe.use_tracing && - tstate->c_tracefunc != NULL && !tstate->tracing) { - int err; - /* see maybe_call_line_trace() - for expository comments */ - _PyFrame_SetStackPointer(frame, stack_pointer); - - err = maybe_call_line_trace(tstate->c_tracefunc, - tstate->c_traceobj, - tstate, frame, instr_prev); - // Reload possibly changed frame fields: - stack_pointer = _PyFrame_GetStackPointer(frame); - frame->stacktop = -1; - // next_instr is only reloaded if tracing *does not* raise. - // This is consistent with the behavior of older Python - // versions. If a trace function sets a new f_lineno and - // *then* raises, we use the *old* location when searching - // for an exception handler, displaying the traceback, and - // so on: - if (err) { - // next_instr wasn't incremented at the start of this - // instruction. Increment it before handling the error, - // so that it looks the same as a "normal" instruction: - next_instr++; - goto error; - } - // Reload next_instr. Don't increment it, though, since - // we're going to re-dispatch to the "true" instruction now: - next_instr = frame->prev_instr; - } - } - } - NEXTOPARG(); - PRE_DISPATCH_GOTO(); - // No _PyOpcode_Deopt here, since EXTENDED_ARG has no optimized forms: - while (opcode == EXTENDED_ARG) { - // CPython hasn't ever traced the instruction after an EXTENDED_ARG. - // Inline the EXTENDED_ARG here, so we can avoid branching there: - INSTRUCTION_START(EXTENDED_ARG); - opcode = next_instr->op.code; - oparg = oparg << 8 | next_instr->op.arg; - // Make sure the next instruction isn't a RESUME, since that needs - // to trace properly (and shouldn't have an EXTENDED_ARG, anyways): - assert(opcode != RESUME); - PRE_DISPATCH_GOTO(); - } - opcode = _PyOpcode_Deopt[opcode]; - if (_PyOpcode_Caches[opcode]) { - uint16_t *counter = &next_instr[1].cache; - // The instruction is going to decrement the counter, so we need to - // increment it here to make sure it doesn't try to specialize: - if (!ADAPTIVE_COUNTER_IS_MAX(*counter)) { - INCREMENT_ADAPTIVE_COUNTER(*counter); - } - } - DISPATCH_GOTO(); - } - #if USE_COMPUTED_GOTOS _unknown_opcode: #else @@ -988,12 +822,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int PyTraceBack_Here(f); } } - - if (tstate->c_tracefunc != NULL) { - /* Make sure state is set to FRAME_UNWINDING for tracing */ - call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj, - tstate, frame); - } + monitor_raise(tstate, frame, next_instr-1); exception_unwind: { @@ -1012,8 +841,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int } assert(STACK_LEVEL() == 0); _PyFrame_SetStackPointer(frame, stack_pointer); - TRACE_FUNCTION_UNWIND(); - DTRACE_FUNCTION_EXIT(); + monitor_unwind(tstate, frame, next_instr-1); goto exit_unwind; } @@ -1036,8 +864,10 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int available to the handler, so a program can emulate the Python main loop. */ - PUSH(_PyErr_GetRaisedException(tstate)); + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(exc); JUMPTO(handler); + monitor_handled(tstate, frame, next_instr, exc); /* Resume normal execution */ DISPATCH(); } @@ -1054,7 +884,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int if (frame == &entry_frame) { /* Restore previous cframe and exit */ tstate->cframe = cframe.previous; - tstate->cframe->use_tracing = cframe.use_tracing; assert(tstate->cframe->current_frame == frame->previous); _Py_LeaveRecursiveCallTstate(tstate); return NULL; @@ -2020,105 +1849,108 @@ unpack_iterable(PyThreadState *tstate, PyObject *v, return 0; } -static void -call_exc_trace(Py_tracefunc func, PyObject *self, - PyThreadState *tstate, - _PyInterpreterFrame *f) +static int +do_monitor_exc(PyThreadState *tstate, _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr, int event) { - PyObject *exc = _PyErr_GetRaisedException(tstate); - assert(exc && PyExceptionInstance_Check(exc)); - PyObject *type = PyExceptionInstance_Class(exc); - PyObject *traceback = PyException_GetTraceback(exc); - if (traceback == NULL) { - traceback = Py_NewRef(Py_None); + assert(event < PY_MONITORING_UNGROUPED_EVENTS); + PyObject *exc = PyErr_GetRaisedException(); + assert(exc != NULL); + int err = _Py_call_instrumentation_arg(tstate, event, frame, instr, exc); + if (err == 0) { + PyErr_SetRaisedException(exc); } - PyObject *arg = PyTuple_Pack(3, type, exc, traceback); - Py_XDECREF(traceback); - - if (arg == NULL) { - _PyErr_SetRaisedException(tstate, exc); - return; + else { + Py_DECREF(exc); } - int err = call_trace(func, self, tstate, f, PyTrace_EXCEPTION, arg); - Py_DECREF(arg); - if (err == 0) { - _PyErr_SetRaisedException(tstate, exc); + return err; +} + +static inline int +no_tools_for_event(PyThreadState *tstate, _PyInterpreterFrame *frame, int event) +{ + _PyCoMonitoringData *data = frame->f_code->_co_monitoring; + if (data) { + if (data->active_monitors.tools[event] == 0) { + return 1; + } } else { - Py_XDECREF(exc); + if (tstate->interp->monitors.tools[event] == 0) { + return 1; + } + } + return 0; +} + +static void +monitor_raise(PyThreadState *tstate, _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr) +{ + if (no_tools_for_event(tstate, frame, PY_MONITORING_EVENT_RAISE)) { + return; } + do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_RAISE); } static int -call_trace_protected(Py_tracefunc func, PyObject *obj, - PyThreadState *tstate, _PyInterpreterFrame *frame, - int what, PyObject *arg) +monitor_stop_iteration(PyThreadState *tstate, _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr) { - PyObject *exc = _PyErr_GetRaisedException(tstate); - int err = call_trace(func, obj, tstate, frame, what, arg); - if (err == 0) - { - _PyErr_SetRaisedException(tstate, exc); + if (no_tools_for_event(tstate, frame, PY_MONITORING_EVENT_STOP_ITERATION)) { return 0; } - else { - Py_XDECREF(exc); - return -1; + return do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_STOP_ITERATION); +} + +static void +monitor_unwind(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr) +{ + if (no_tools_for_event(tstate, frame, PY_MONITORING_EVENT_PY_UNWIND)) { + return; } + _Py_call_instrumentation_exc0(tstate, PY_MONITORING_EVENT_PY_UNWIND, frame, instr); } + static void -initialize_trace_info(PyTraceInfo *trace_info, _PyInterpreterFrame *frame) +monitor_handled(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr, PyObject *exc) { - PyCodeObject *code = frame->f_code; - if (trace_info->code != code) { - trace_info->code = code; - _PyCode_InitAddressRange(code, &trace_info->bounds); + if (no_tools_for_event(tstate, frame, PY_MONITORING_EVENT_EXCEPTION_HANDLED)) { + return; } + _Py_call_instrumentation_arg(tstate, PY_MONITORING_EVENT_EXCEPTION_HANDLED, frame, instr, exc); +} + +static void +monitor_throw(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr) +{ + if (no_tools_for_event(tstate, frame, PY_MONITORING_EVENT_PY_THROW)) { + return; + } + _Py_call_instrumentation_exc0(tstate, PY_MONITORING_EVENT_PY_THROW, frame, instr); } void PyThreadState_EnterTracing(PyThreadState *tstate) { + assert(tstate->tracing >= 0); tstate->tracing++; - tstate->cframe->use_tracing = 0; } void PyThreadState_LeaveTracing(PyThreadState *tstate) { - assert(tstate->tracing > 0 && tstate->cframe->use_tracing == 0); + assert(tstate->tracing > 0); tstate->tracing--; - _PyThreadState_UpdateTracingState(tstate); } -static int -call_trace(Py_tracefunc func, PyObject *obj, - PyThreadState *tstate, _PyInterpreterFrame *frame, - int what, PyObject *arg) -{ - int result; - if (tstate->tracing) { - return 0; - } - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f == NULL) { - return -1; - } - int old_what = tstate->tracing_what; - tstate->tracing_what = what; - PyThreadState_EnterTracing(tstate); - assert(_PyInterpreterFrame_LASTI(frame) >= 0); - if (_PyCode_InitLineArray(frame->f_code)) { - return -1; - } - f->f_lineno = _PyCode_LineNumberFromArray(frame->f_code, _PyInterpreterFrame_LASTI(frame)); - result = func(obj, f, what, arg); - f->f_lineno = 0; - PyThreadState_LeaveTracing(tstate); - tstate->tracing_what = old_what; - return result; -} PyObject* _PyEval_CallTracing(PyObject *func, PyObject *args) @@ -2126,7 +1958,6 @@ _PyEval_CallTracing(PyObject *func, PyObject *args) // Save and disable tracing PyThreadState *tstate = _PyThreadState_GET(); int save_tracing = tstate->tracing; - int save_use_tracing = tstate->cframe->use_tracing; tstate->tracing = 0; // Call the tracing function @@ -2134,81 +1965,9 @@ _PyEval_CallTracing(PyObject *func, PyObject *args) // Restore tracing tstate->tracing = save_tracing; - tstate->cframe->use_tracing = save_use_tracing; return result; } -/* See Objects/lnotab_notes.txt for a description of how tracing works. */ -static int -maybe_call_line_trace(Py_tracefunc func, PyObject *obj, - PyThreadState *tstate, _PyInterpreterFrame *frame, int instr_prev) -{ - int result = 0; - - /* If the last instruction falls at the start of a line or if it - represents a jump backwards, update the frame's line number and - then call the trace function if we're tracing source lines. - */ - if (_PyCode_InitLineArray(frame->f_code)) { - return -1; - } - int lastline; - if (instr_prev <= frame->f_code->_co_firsttraceable) { - lastline = -1; - } - else { - lastline = _PyCode_LineNumberFromArray(frame->f_code, instr_prev); - } - int line = _PyCode_LineNumberFromArray(frame->f_code, _PyInterpreterFrame_LASTI(frame)); - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f == NULL) { - return -1; - } - if (line != -1 && f->f_trace_lines) { - /* Trace backward edges (except in 'yield from') or if line number has changed */ - int trace = line != lastline || - (_PyInterpreterFrame_LASTI(frame) < instr_prev && - // SEND has no quickened forms, so no need to use _PyOpcode_Deopt - // here: - frame->prev_instr->op.code != SEND); - if (trace) { - result = call_trace(func, obj, tstate, frame, PyTrace_LINE, Py_None); - } - } - /* Always emit an opcode event if we're tracing all opcodes. */ - if (f->f_trace_opcodes && result == 0) { - result = call_trace(func, obj, tstate, frame, PyTrace_OPCODE, Py_None); - } - return result; -} - -int -_PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) -{ - assert(is_tstate_valid(tstate)); - /* The caller must hold the GIL */ - assert(PyGILState_Check()); - - /* Call _PySys_Audit() in the context of the current thread state, - even if tstate is not the current thread state. */ - PyThreadState *current_tstate = _PyThreadState_GET(); - if (_PySys_Audit(current_tstate, "sys.setprofile", NULL) < 0) { - return -1; - } - - tstate->c_profilefunc = func; - PyObject *old_profileobj = tstate->c_profileobj; - tstate->c_profileobj = Py_XNewRef(arg); - /* Flag that tracing or profiling is turned on */ - _PyThreadState_UpdateTracingState(tstate); - - // gh-98257: Only call Py_XDECREF() once the new profile function is fully - // set, so it's safe to call sys.setprofile() again (reentrant call). - Py_XDECREF(old_profileobj); - - return 0; -} - void PyEval_SetProfile(Py_tracefunc func, PyObject *arg) { @@ -2240,33 +1999,6 @@ PyEval_SetProfileAllThreads(Py_tracefunc func, PyObject *arg) } } -int -_PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) -{ - assert(is_tstate_valid(tstate)); - /* The caller must hold the GIL */ - assert(PyGILState_Check()); - - /* Call _PySys_Audit() in the context of the current thread state, - even if tstate is not the current thread state. */ - PyThreadState *current_tstate = _PyThreadState_GET(); - if (_PySys_Audit(current_tstate, "sys.settrace", NULL) < 0) { - return -1; - } - - tstate->c_tracefunc = func; - PyObject *old_traceobj = tstate->c_traceobj; - tstate->c_traceobj = Py_XNewRef(arg); - /* Flag that tracing or profiling is turned on */ - _PyThreadState_UpdateTracingState(tstate); - - // gh-98257: Only call Py_XDECREF() once the new trace function is fully - // set, so it's safe to call sys.settrace() again (reentrant call). - Py_XDECREF(old_traceobj); - - return 0; -} - void PyEval_SetTrace(Py_tracefunc func, PyObject *arg) { @@ -2492,114 +2224,6 @@ PyEval_GetFuncDesc(PyObject *func) return " object"; } -#define C_TRACE(x, call) \ -if (use_tracing && tstate->c_profilefunc) { \ - if (call_trace(tstate->c_profilefunc, tstate->c_profileobj, \ - tstate, tstate->cframe->current_frame, \ - PyTrace_C_CALL, func)) { \ - x = NULL; \ - } \ - else { \ - x = call; \ - if (tstate->c_profilefunc != NULL) { \ - if (x == NULL) { \ - call_trace_protected(tstate->c_profilefunc, \ - tstate->c_profileobj, \ - tstate, tstate->cframe->current_frame, \ - PyTrace_C_EXCEPTION, func); \ - /* XXX should pass (type, value, tb) */ \ - } else { \ - if (call_trace(tstate->c_profilefunc, \ - tstate->c_profileobj, \ - tstate, tstate->cframe->current_frame, \ - PyTrace_C_RETURN, func)) { \ - Py_DECREF(x); \ - x = NULL; \ - } \ - } \ - } \ - } \ -} else { \ - x = call; \ - } - - -static PyObject * -trace_call_function(PyThreadState *tstate, - PyObject *func, - PyObject **args, Py_ssize_t nargs, - PyObject *kwnames) -{ - int use_tracing = 1; - PyObject *x; - if (PyCFunction_CheckExact(func) || PyCMethod_CheckExact(func)) { - C_TRACE(x, PyObject_Vectorcall(func, args, nargs, kwnames)); - return x; - } - else if (Py_IS_TYPE(func, &PyMethodDescr_Type) && nargs > 0) { - /* We need to create a temporary bound method as argument - for profiling. - - If nargs == 0, then this cannot work because we have no - "self". In any case, the call itself would raise - TypeError (foo needs an argument), so we just skip - profiling. */ - PyObject *self = args[0]; - func = Py_TYPE(func)->tp_descr_get(func, self, (PyObject*)Py_TYPE(self)); - if (func == NULL) { - return NULL; - } - C_TRACE(x, PyObject_Vectorcall(func, - args+1, nargs-1, - kwnames)); - Py_DECREF(func); - return x; - } - return PyObject_Vectorcall(func, args, nargs | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames); -} - -static PyObject * -do_call_core(PyThreadState *tstate, - PyObject *func, - PyObject *callargs, - PyObject *kwdict, - int use_tracing - ) -{ - PyObject *result; - if (PyCFunction_CheckExact(func) || PyCMethod_CheckExact(func)) { - C_TRACE(result, PyObject_Call(func, callargs, kwdict)); - return result; - } - else if (Py_IS_TYPE(func, &PyMethodDescr_Type)) { - Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); - if (nargs > 0 && use_tracing) { - /* We need to create a temporary bound method as argument - for profiling. - - If nargs == 0, then this cannot work because we have no - "self". In any case, the call itself would raise - TypeError (foo needs an argument), so we just skip - profiling. */ - PyObject *self = PyTuple_GET_ITEM(callargs, 0); - func = Py_TYPE(func)->tp_descr_get(func, self, (PyObject*)Py_TYPE(self)); - if (func == NULL) { - return NULL; - } - - C_TRACE(result, _PyObject_FastCallDictTstate( - tstate, func, - &_PyTuple_ITEMS(callargs)[1], - nargs - 1, - kwdict)); - Py_DECREF(func); - return result; - } - } - EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); - return PyObject_Call(func, callargs, kwdict); -} - /* Extract a slice index from a PyLong or an object with the nb_index slot defined, and store in *pi. Silently reduce values larger than PY_SSIZE_T_MAX to PY_SSIZE_T_MAX, @@ -2973,69 +2597,6 @@ PyUnstable_Eval_RequestCodeExtraIndex(freefunc free) return new_index; } -static void -dtrace_function_entry(_PyInterpreterFrame *frame) -{ - const char *filename; - const char *funcname; - int lineno; - - PyCodeObject *code = frame->f_code; - filename = PyUnicode_AsUTF8(code->co_filename); - funcname = PyUnicode_AsUTF8(code->co_name); - lineno = _PyInterpreterFrame_GetLine(frame); - - PyDTrace_FUNCTION_ENTRY(filename, funcname, lineno); -} - -static void -dtrace_function_return(_PyInterpreterFrame *frame) -{ - const char *filename; - const char *funcname; - int lineno; - - PyCodeObject *code = frame->f_code; - filename = PyUnicode_AsUTF8(code->co_filename); - funcname = PyUnicode_AsUTF8(code->co_name); - lineno = _PyInterpreterFrame_GetLine(frame); - - PyDTrace_FUNCTION_RETURN(filename, funcname, lineno); -} - -/* DTrace equivalent of maybe_call_line_trace. */ -static void -maybe_dtrace_line(_PyInterpreterFrame *frame, - PyTraceInfo *trace_info, - int instr_prev) -{ - const char *co_filename, *co_name; - - /* If the last instruction executed isn't in the current - instruction window, reset the window. - */ - initialize_trace_info(trace_info, frame); - int lastline = _PyCode_CheckLineNumber(instr_prev*sizeof(_Py_CODEUNIT), &trace_info->bounds); - int addr = _PyInterpreterFrame_LASTI(frame) * sizeof(_Py_CODEUNIT); - int line = _PyCode_CheckLineNumber(addr, &trace_info->bounds); - if (line != -1) { - /* Trace backward edges or first instruction of a new line */ - if (_PyInterpreterFrame_LASTI(frame) < instr_prev || - (line != lastline && addr == trace_info->bounds.ar_start)) - { - co_filename = PyUnicode_AsUTF8(frame->f_code->co_filename); - if (!co_filename) { - co_filename = "?"; - } - co_name = PyUnicode_AsUTF8(frame->f_code->co_name); - if (!co_name) { - co_name = "?"; - } - PyDTrace_LINE(co_filename, co_name, line); - } - } -} - /* Implement Py_EnterRecursiveCall() and Py_LeaveRecursiveCall() as functions for the limited API. */ diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index c2257515a30599..485771ac65a767 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -93,8 +93,6 @@ { \ NEXTOPARG(); \ PRE_DISPATCH_GOTO(); \ - assert(cframe.use_tracing == 0 || cframe.use_tracing == 255); \ - opcode |= cframe.use_tracing OR_DTRACE_LINE; \ DISPATCH_GOTO(); \ } @@ -102,7 +100,6 @@ { \ opcode = next_instr->op.code; \ PRE_DISPATCH_GOTO(); \ - opcode |= cframe.use_tracing OR_DTRACE_LINE; \ DISPATCH_GOTO(); \ } @@ -183,7 +180,7 @@ GETITEM(PyObject *v, Py_ssize_t i) { #define PREDICT(next_op) \ do { \ _Py_CODEUNIT word = *next_instr; \ - opcode = word.op.code | cframe.use_tracing OR_DTRACE_LINE; \ + opcode = word.op.code; \ if (opcode == next_op) { \ oparg = word.op.arg; \ INSTRUCTION_START(next_op); \ @@ -283,47 +280,6 @@ GETITEM(PyObject *v, Py_ssize_t i) { #define BUILTINS() frame->f_builtins #define LOCALS() frame->f_locals -/* Shared opcode macros */ - -#define TRACE_FUNCTION_EXIT() \ - if (cframe.use_tracing) { \ - if (trace_function_exit(tstate, frame, retval)) { \ - Py_DECREF(retval); \ - goto exit_unwind; \ - } \ - } - -#define DTRACE_FUNCTION_EXIT() \ - if (PyDTrace_FUNCTION_RETURN_ENABLED()) { \ - dtrace_function_return(frame); \ - } - -#define TRACE_FUNCTION_UNWIND() \ - if (cframe.use_tracing) { \ - /* Since we are already unwinding, \ - * we don't care if this raises */ \ - trace_function_exit(tstate, frame, NULL); \ - } - -#define TRACE_FUNCTION_ENTRY() \ - if (cframe.use_tracing) { \ - _PyFrame_SetStackPointer(frame, stack_pointer); \ - int err = trace_function_entry(tstate, frame); \ - stack_pointer = _PyFrame_GetStackPointer(frame); \ - frame->stacktop = -1; \ - if (err) { \ - goto error; \ - } \ - } - -#define TRACE_FUNCTION_THROW_ENTRY() \ - if (cframe.use_tracing) { \ - assert(frame->stacktop >= 0); \ - if (trace_function_entry(tstate, frame)) { \ - goto exit_unwind; \ - } \ - } - #define DTRACE_FUNCTION_ENTRY() \ if (PyDTrace_FUNCTION_ENTRY_ENABLED()) { \ dtrace_function_entry(frame); \ @@ -371,3 +327,18 @@ do { \ _Py_DECREF_NO_DEALLOC(right); \ } \ } while (0) + +// If a trace function sets a new f_lineno and +// *then* raises, we use the destination when searching +// for an exception handler, displaying the traceback, and so on +#define INSTRUMENTED_JUMP(src, dest, event) \ +do { \ + _PyFrame_SetStackPointer(frame, stack_pointer); \ + int err = _Py_call_instrumentation_jump(tstate, event, frame, src, dest); \ + stack_pointer = _PyFrame_GetStackPointer(frame); \ + if (err) { \ + next_instr = (dest)+1; \ + goto error; \ + } \ + next_instr = frame->prev_instr; \ +} while (0); diff --git a/Python/clinic/instrumentation.c.h b/Python/clinic/instrumentation.c.h new file mode 100644 index 00000000000000..cf3984ca24bbfe --- /dev/null +++ b/Python/clinic/instrumentation.c.h @@ -0,0 +1,311 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif + + +PyDoc_STRVAR(monitoring_use_tool_id__doc__, +"use_tool_id($module, tool_id, name, /)\n" +"--\n" +"\n"); + +#define MONITORING_USE_TOOL_ID_METHODDEF \ + {"use_tool_id", _PyCFunction_CAST(monitoring_use_tool_id), METH_FASTCALL, monitoring_use_tool_id__doc__}, + +static PyObject * +monitoring_use_tool_id_impl(PyObject *module, int tool_id, PyObject *name); + +static PyObject * +monitoring_use_tool_id(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int tool_id; + PyObject *name; + + if (!_PyArg_CheckPositional("use_tool_id", nargs, 2, 2)) { + goto exit; + } + tool_id = _PyLong_AsInt(args[0]); + if (tool_id == -1 && PyErr_Occurred()) { + goto exit; + } + name = args[1]; + return_value = monitoring_use_tool_id_impl(module, tool_id, name); + +exit: + return return_value; +} + +PyDoc_STRVAR(monitoring_free_tool_id__doc__, +"free_tool_id($module, tool_id, /)\n" +"--\n" +"\n"); + +#define MONITORING_FREE_TOOL_ID_METHODDEF \ + {"free_tool_id", (PyCFunction)monitoring_free_tool_id, METH_O, monitoring_free_tool_id__doc__}, + +static PyObject * +monitoring_free_tool_id_impl(PyObject *module, int tool_id); + +static PyObject * +monitoring_free_tool_id(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + int tool_id; + + tool_id = _PyLong_AsInt(arg); + if (tool_id == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = monitoring_free_tool_id_impl(module, tool_id); + +exit: + return return_value; +} + +PyDoc_STRVAR(monitoring_get_tool__doc__, +"get_tool($module, tool_id, /)\n" +"--\n" +"\n"); + +#define MONITORING_GET_TOOL_METHODDEF \ + {"get_tool", (PyCFunction)monitoring_get_tool, METH_O, monitoring_get_tool__doc__}, + +static PyObject * +monitoring_get_tool_impl(PyObject *module, int tool_id); + +static PyObject * +monitoring_get_tool(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + int tool_id; + + tool_id = _PyLong_AsInt(arg); + if (tool_id == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = monitoring_get_tool_impl(module, tool_id); + +exit: + return return_value; +} + +PyDoc_STRVAR(monitoring_register_callback__doc__, +"register_callback($module, tool_id, event, func, /)\n" +"--\n" +"\n"); + +#define MONITORING_REGISTER_CALLBACK_METHODDEF \ + {"register_callback", _PyCFunction_CAST(monitoring_register_callback), METH_FASTCALL, monitoring_register_callback__doc__}, + +static PyObject * +monitoring_register_callback_impl(PyObject *module, int tool_id, int event, + PyObject *func); + +static PyObject * +monitoring_register_callback(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int tool_id; + int event; + PyObject *func; + + if (!_PyArg_CheckPositional("register_callback", nargs, 3, 3)) { + goto exit; + } + tool_id = _PyLong_AsInt(args[0]); + if (tool_id == -1 && PyErr_Occurred()) { + goto exit; + } + event = _PyLong_AsInt(args[1]); + if (event == -1 && PyErr_Occurred()) { + goto exit; + } + func = args[2]; + return_value = monitoring_register_callback_impl(module, tool_id, event, func); + +exit: + return return_value; +} + +PyDoc_STRVAR(monitoring_get_events__doc__, +"get_events($module, tool_id, /)\n" +"--\n" +"\n"); + +#define MONITORING_GET_EVENTS_METHODDEF \ + {"get_events", (PyCFunction)monitoring_get_events, METH_O, monitoring_get_events__doc__}, + +static int +monitoring_get_events_impl(PyObject *module, int tool_id); + +static PyObject * +monitoring_get_events(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + int tool_id; + int _return_value; + + tool_id = _PyLong_AsInt(arg); + if (tool_id == -1 && PyErr_Occurred()) { + goto exit; + } + _return_value = monitoring_get_events_impl(module, tool_id); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromLong((long)_return_value); + +exit: + return return_value; +} + +PyDoc_STRVAR(monitoring_set_events__doc__, +"set_events($module, tool_id, event_set, /)\n" +"--\n" +"\n"); + +#define MONITORING_SET_EVENTS_METHODDEF \ + {"set_events", _PyCFunction_CAST(monitoring_set_events), METH_FASTCALL, monitoring_set_events__doc__}, + +static PyObject * +monitoring_set_events_impl(PyObject *module, int tool_id, int event_set); + +static PyObject * +monitoring_set_events(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int tool_id; + int event_set; + + if (!_PyArg_CheckPositional("set_events", nargs, 2, 2)) { + goto exit; + } + tool_id = _PyLong_AsInt(args[0]); + if (tool_id == -1 && PyErr_Occurred()) { + goto exit; + } + event_set = _PyLong_AsInt(args[1]); + if (event_set == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = monitoring_set_events_impl(module, tool_id, event_set); + +exit: + return return_value; +} + +PyDoc_STRVAR(monitoring_get_local_events__doc__, +"get_local_events($module, tool_id, code, /)\n" +"--\n" +"\n"); + +#define MONITORING_GET_LOCAL_EVENTS_METHODDEF \ + {"get_local_events", _PyCFunction_CAST(monitoring_get_local_events), METH_FASTCALL, monitoring_get_local_events__doc__}, + +static int +monitoring_get_local_events_impl(PyObject *module, int tool_id, + PyObject *code); + +static PyObject * +monitoring_get_local_events(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int tool_id; + PyObject *code; + int _return_value; + + if (!_PyArg_CheckPositional("get_local_events", nargs, 2, 2)) { + goto exit; + } + tool_id = _PyLong_AsInt(args[0]); + if (tool_id == -1 && PyErr_Occurred()) { + goto exit; + } + code = args[1]; + _return_value = monitoring_get_local_events_impl(module, tool_id, code); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromLong((long)_return_value); + +exit: + return return_value; +} + +PyDoc_STRVAR(monitoring_set_local_events__doc__, +"set_local_events($module, tool_id, code, event_set, /)\n" +"--\n" +"\n"); + +#define MONITORING_SET_LOCAL_EVENTS_METHODDEF \ + {"set_local_events", _PyCFunction_CAST(monitoring_set_local_events), METH_FASTCALL, monitoring_set_local_events__doc__}, + +static PyObject * +monitoring_set_local_events_impl(PyObject *module, int tool_id, + PyObject *code, int event_set); + +static PyObject * +monitoring_set_local_events(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int tool_id; + PyObject *code; + int event_set; + + if (!_PyArg_CheckPositional("set_local_events", nargs, 3, 3)) { + goto exit; + } + tool_id = _PyLong_AsInt(args[0]); + if (tool_id == -1 && PyErr_Occurred()) { + goto exit; + } + code = args[1]; + event_set = _PyLong_AsInt(args[2]); + if (event_set == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = monitoring_set_local_events_impl(module, tool_id, code, event_set); + +exit: + return return_value; +} + +PyDoc_STRVAR(monitoring_restart_events__doc__, +"restart_events($module, /)\n" +"--\n" +"\n"); + +#define MONITORING_RESTART_EVENTS_METHODDEF \ + {"restart_events", (PyCFunction)monitoring_restart_events, METH_NOARGS, monitoring_restart_events__doc__}, + +static PyObject * +monitoring_restart_events_impl(PyObject *module); + +static PyObject * +monitoring_restart_events(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return monitoring_restart_events_impl(module); +} + +PyDoc_STRVAR(monitoring__all_events__doc__, +"_all_events($module, /)\n" +"--\n" +"\n"); + +#define MONITORING__ALL_EVENTS_METHODDEF \ + {"_all_events", (PyCFunction)monitoring__all_events, METH_NOARGS, monitoring__all_events__doc__}, + +static PyObject * +monitoring__all_events_impl(PyObject *module); + +static PyObject * +monitoring__all_events(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return monitoring__all_events_impl(module); +} +/*[clinic end generated code: output=11cc0803875b3ffa input=a9049054013a1b77]*/ diff --git a/Python/compile.c b/Python/compile.c index fed9ae7066e4f0..48c8cd454f0c25 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1502,8 +1502,7 @@ compiler_add_yield_from(struct compiler *c, location loc, int await) ADDOP(c, loc, CLEANUP_THROW); USE_LABEL(c, exit); - ADDOP_I(c, loc, SWAP, 2); - ADDOP(c, loc, POP_TOP); + ADDOP(c, loc, END_SEND); return SUCCESS; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 420d136bb98158..4d52e95dc04a27 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -8,24 +8,61 @@ } TARGET(RESUME) { - #line 135 "Python/bytecodes.c" + #line 136 "Python/bytecodes.c" assert(tstate->cframe == &cframe); assert(frame == cframe.current_frame); - if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { + /* Possibly combine this with eval breaker */ + if (frame->f_code->_co_instrumentation_version != tstate->interp->monitoring_version) { + int err = _Py_Instrument(frame->f_code, tstate->interp); + if (err) goto error; + next_instr--; + } + else if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { goto handle_eval_breaker; } - #line 18 "Python/generated_cases.c.h" + #line 24 "Python/generated_cases.c.h" + DISPATCH(); + } + + TARGET(INSTRUMENTED_RESUME) { + #line 150 "Python/bytecodes.c" + /* Possible performance enhancement: + * We need to check the eval breaker anyway, can we + * combine the instrument verison check and the eval breaker test? + */ + if (frame->f_code->_co_instrumentation_version != tstate->interp->monitoring_version) { + if (_Py_Instrument(frame->f_code, tstate->interp)) { + goto error; + } + next_instr--; + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation( + tstate, oparg > 0, frame, next_instr-1); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) goto error; + if (frame->prev_instr != next_instr-1) { + /* Instrumentation has jumped */ + next_instr = frame->prev_instr; + DISPATCH(); + } + if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { + goto handle_eval_breaker; + } + } + #line 55 "Python/generated_cases.c.h" DISPATCH(); } TARGET(LOAD_CLOSURE) { PyObject *value; - #line 143 "Python/bytecodes.c" + #line 178 "Python/bytecodes.c" /* We keep LOAD_CLOSURE so that the bytecode stays more readable. */ value = GETLOCAL(oparg); if (value == NULL) goto unbound_local_error; Py_INCREF(value); - #line 29 "Python/generated_cases.c.h" + #line 66 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -33,11 +70,11 @@ TARGET(LOAD_FAST_CHECK) { PyObject *value; - #line 150 "Python/bytecodes.c" + #line 185 "Python/bytecodes.c" value = GETLOCAL(oparg); if (value == NULL) goto unbound_local_error; Py_INCREF(value); - #line 41 "Python/generated_cases.c.h" + #line 78 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -45,11 +82,11 @@ TARGET(LOAD_FAST) { PyObject *value; - #line 156 "Python/bytecodes.c" + #line 191 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - #line 53 "Python/generated_cases.c.h" + #line 90 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -58,10 +95,10 @@ TARGET(LOAD_CONST) { PREDICTED(LOAD_CONST); PyObject *value; - #line 162 "Python/bytecodes.c" + #line 197 "Python/bytecodes.c" value = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(value); - #line 65 "Python/generated_cases.c.h" + #line 102 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -69,9 +106,9 @@ TARGET(STORE_FAST) { PyObject *value = stack_pointer[-1]; - #line 167 "Python/bytecodes.c" + #line 202 "Python/bytecodes.c" SETLOCAL(oparg, value); - #line 75 "Python/generated_cases.c.h" + #line 112 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -81,21 +118,21 @@ PyObject *_tmp_2; { PyObject *value; - #line 156 "Python/bytecodes.c" + #line 191 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - #line 89 "Python/generated_cases.c.h" + #line 126 "Python/generated_cases.c.h" _tmp_2 = value; } oparg = (next_instr++)->op.arg; { PyObject *value; - #line 156 "Python/bytecodes.c" + #line 191 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - #line 99 "Python/generated_cases.c.h" + #line 136 "Python/generated_cases.c.h" _tmp_1 = value; } STACK_GROW(2); @@ -109,20 +146,20 @@ PyObject *_tmp_2; { PyObject *value; - #line 156 "Python/bytecodes.c" + #line 191 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - #line 117 "Python/generated_cases.c.h" + #line 154 "Python/generated_cases.c.h" _tmp_2 = value; } oparg = (next_instr++)->op.arg; { PyObject *value; - #line 162 "Python/bytecodes.c" + #line 197 "Python/bytecodes.c" value = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(value); - #line 126 "Python/generated_cases.c.h" + #line 163 "Python/generated_cases.c.h" _tmp_1 = value; } STACK_GROW(2); @@ -135,18 +172,18 @@ PyObject *_tmp_1 = stack_pointer[-1]; { PyObject *value = _tmp_1; - #line 167 "Python/bytecodes.c" + #line 202 "Python/bytecodes.c" SETLOCAL(oparg, value); - #line 141 "Python/generated_cases.c.h" + #line 178 "Python/generated_cases.c.h" } oparg = (next_instr++)->op.arg; { PyObject *value; - #line 156 "Python/bytecodes.c" + #line 191 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - #line 150 "Python/generated_cases.c.h" + #line 187 "Python/generated_cases.c.h" _tmp_1 = value; } stack_pointer[-1] = _tmp_1; @@ -158,16 +195,16 @@ PyObject *_tmp_2 = stack_pointer[-2]; { PyObject *value = _tmp_1; - #line 167 "Python/bytecodes.c" + #line 202 "Python/bytecodes.c" SETLOCAL(oparg, value); - #line 164 "Python/generated_cases.c.h" + #line 201 "Python/generated_cases.c.h" } oparg = (next_instr++)->op.arg; { PyObject *value = _tmp_2; - #line 167 "Python/bytecodes.c" + #line 202 "Python/bytecodes.c" SETLOCAL(oparg, value); - #line 171 "Python/generated_cases.c.h" + #line 208 "Python/generated_cases.c.h" } STACK_SHRINK(2); DISPATCH(); @@ -178,20 +215,20 @@ PyObject *_tmp_2; { PyObject *value; - #line 162 "Python/bytecodes.c" + #line 197 "Python/bytecodes.c" value = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(value); - #line 185 "Python/generated_cases.c.h" + #line 222 "Python/generated_cases.c.h" _tmp_2 = value; } oparg = (next_instr++)->op.arg; { PyObject *value; - #line 156 "Python/bytecodes.c" + #line 191 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - #line 195 "Python/generated_cases.c.h" + #line 232 "Python/generated_cases.c.h" _tmp_1 = value; } STACK_GROW(2); @@ -202,8 +239,8 @@ TARGET(POP_TOP) { PyObject *value = stack_pointer[-1]; - #line 177 "Python/bytecodes.c" - #line 207 "Python/generated_cases.c.h" + #line 212 "Python/bytecodes.c" + #line 244 "Python/generated_cases.c.h" Py_DECREF(value); STACK_SHRINK(1); DISPATCH(); @@ -211,9 +248,9 @@ TARGET(PUSH_NULL) { PyObject *res; - #line 181 "Python/bytecodes.c" + #line 216 "Python/bytecodes.c" res = NULL; - #line 217 "Python/generated_cases.c.h" + #line 254 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); @@ -224,30 +261,79 @@ PyObject *_tmp_2 = stack_pointer[-2]; { PyObject *value = _tmp_1; - #line 177 "Python/bytecodes.c" - #line 229 "Python/generated_cases.c.h" + #line 212 "Python/bytecodes.c" + #line 266 "Python/generated_cases.c.h" Py_DECREF(value); } { PyObject *value = _tmp_2; - #line 177 "Python/bytecodes.c" - #line 235 "Python/generated_cases.c.h" + #line 212 "Python/bytecodes.c" + #line 272 "Python/generated_cases.c.h" Py_DECREF(value); } STACK_SHRINK(2); DISPATCH(); } + TARGET(INSTRUMENTED_END_FOR) { + PyObject *value = stack_pointer[-1]; + PyObject *receiver = stack_pointer[-2]; + #line 222 "Python/bytecodes.c" + /* Need to create a fake StopIteration error here, + * to conform to PEP 380 */ + if (PyGen_Check(receiver)) { + PyErr_SetObject(PyExc_StopIteration, value); + if (monitor_stop_iteration(tstate, frame, next_instr-1)) { + goto error; + } + PyErr_SetRaisedException(NULL); + } + #line 292 "Python/generated_cases.c.h" + Py_DECREF(receiver); + Py_DECREF(value); + STACK_SHRINK(2); + DISPATCH(); + } + + TARGET(END_SEND) { + PyObject *value = stack_pointer[-1]; + PyObject *receiver = stack_pointer[-2]; + #line 235 "Python/bytecodes.c" + Py_DECREF(receiver); + #line 304 "Python/generated_cases.c.h" + STACK_SHRINK(1); + stack_pointer[-1] = value; + DISPATCH(); + } + + TARGET(INSTRUMENTED_END_SEND) { + PyObject *value = stack_pointer[-1]; + PyObject *receiver = stack_pointer[-2]; + #line 239 "Python/bytecodes.c" + if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { + PyErr_SetObject(PyExc_StopIteration, value); + if (monitor_stop_iteration(tstate, frame, next_instr-1)) { + goto error; + } + PyErr_SetRaisedException(NULL); + } + Py_DECREF(receiver); + #line 322 "Python/generated_cases.c.h" + STACK_SHRINK(1); + stack_pointer[-1] = value; + DISPATCH(); + } + TARGET(UNARY_NEGATIVE) { PyObject *value = stack_pointer[-1]; PyObject *res; - #line 187 "Python/bytecodes.c" + #line 250 "Python/bytecodes.c" res = PyNumber_Negative(value); - #line 247 "Python/generated_cases.c.h" + #line 333 "Python/generated_cases.c.h" Py_DECREF(value); - #line 189 "Python/bytecodes.c" + #line 252 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; - #line 251 "Python/generated_cases.c.h" + #line 337 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -255,11 +341,11 @@ TARGET(UNARY_NOT) { PyObject *value = stack_pointer[-1]; PyObject *res; - #line 193 "Python/bytecodes.c" + #line 256 "Python/bytecodes.c" int err = PyObject_IsTrue(value); - #line 261 "Python/generated_cases.c.h" + #line 347 "Python/generated_cases.c.h" Py_DECREF(value); - #line 195 "Python/bytecodes.c" + #line 258 "Python/bytecodes.c" if (err < 0) goto pop_1_error; if (err == 0) { res = Py_True; @@ -268,7 +354,7 @@ res = Py_False; } Py_INCREF(res); - #line 272 "Python/generated_cases.c.h" + #line 358 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -276,13 +362,13 @@ TARGET(UNARY_INVERT) { PyObject *value = stack_pointer[-1]; PyObject *res; - #line 206 "Python/bytecodes.c" + #line 269 "Python/bytecodes.c" res = PyNumber_Invert(value); - #line 282 "Python/generated_cases.c.h" + #line 368 "Python/generated_cases.c.h" Py_DECREF(value); - #line 208 "Python/bytecodes.c" + #line 271 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; - #line 286 "Python/generated_cases.c.h" + #line 372 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -291,8 +377,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *prod; - #line 225 "Python/bytecodes.c" - assert(cframe.use_tracing == 0); + #line 288 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -300,7 +385,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (prod == NULL) goto pop_2_error; - #line 304 "Python/generated_cases.c.h" + #line 389 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = prod; next_instr += 1; @@ -311,15 +396,14 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *prod; - #line 236 "Python/bytecodes.c" - assert(cframe.use_tracing == 0); + #line 298 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); double dprod = ((PyFloatObject *)left)->ob_fval * ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dprod, prod); - #line 323 "Python/generated_cases.c.h" + #line 407 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = prod; next_instr += 1; @@ -330,8 +414,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *sub; - #line 246 "Python/bytecodes.c" - assert(cframe.use_tracing == 0); + #line 307 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -339,7 +422,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (sub == NULL) goto pop_2_error; - #line 343 "Python/generated_cases.c.h" + #line 426 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = sub; next_instr += 1; @@ -350,14 +433,13 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *sub; - #line 257 "Python/bytecodes.c" - assert(cframe.use_tracing == 0); + #line 317 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); double dsub = ((PyFloatObject *)left)->ob_fval - ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dsub, sub); - #line 361 "Python/generated_cases.c.h" + #line 443 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = sub; next_instr += 1; @@ -368,8 +450,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 266 "Python/bytecodes.c" - assert(cframe.use_tracing == 0); + #line 325 "Python/bytecodes.c" DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -377,7 +458,7 @@ _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); if (res == NULL) goto pop_2_error; - #line 381 "Python/generated_cases.c.h" + #line 462 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -387,8 +468,7 @@ TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; - #line 283 "Python/bytecodes.c" - assert(cframe.use_tracing == 0); + #line 341 "Python/bytecodes.c" DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); _Py_CODEUNIT true_next = next_instr[INLINE_CACHE_ENTRIES_BINARY_OP]; @@ -415,7 +495,7 @@ if (*target_local == NULL) goto pop_2_error; // The STORE_FAST is already done. JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP + 1); - #line 419 "Python/generated_cases.c.h" + #line 499 "Python/generated_cases.c.h" STACK_SHRINK(2); DISPATCH(); } @@ -424,15 +504,14 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *sum; - #line 313 "Python/bytecodes.c" - assert(cframe.use_tracing == 0); + #line 370 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); double dsum = ((PyFloatObject *)left)->ob_fval + ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dsum, sum); - #line 436 "Python/generated_cases.c.h" + #line 515 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = sum; next_instr += 1; @@ -443,8 +522,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *sum; - #line 323 "Python/bytecodes.c" - assert(cframe.use_tracing == 0); + #line 379 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -452,7 +530,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (sum == NULL) goto pop_2_error; - #line 456 "Python/generated_cases.c.h" + #line 534 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = sum; next_instr += 1; @@ -465,11 +543,10 @@ PyObject *sub = stack_pointer[-1]; PyObject *container = stack_pointer[-2]; PyObject *res; - #line 342 "Python/bytecodes.c" + #line 397 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_BinarySubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); @@ -478,12 +555,12 @@ DECREMENT_ADAPTIVE_COUNTER(cache->counter); #endif /* ENABLE_SPECIALIZATION */ res = PyObject_GetItem(container, sub); - #line 482 "Python/generated_cases.c.h" + #line 559 "Python/generated_cases.c.h" Py_DECREF(container); Py_DECREF(sub); - #line 355 "Python/bytecodes.c" + #line 409 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 487 "Python/generated_cases.c.h" + #line 564 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -495,7 +572,7 @@ PyObject *start = stack_pointer[-2]; PyObject *container = stack_pointer[-3]; PyObject *res; - #line 359 "Python/bytecodes.c" + #line 413 "Python/bytecodes.c" PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); // Can't use ERROR_IF() here, because we haven't // DECREF'ed container yet, and we still own slice. @@ -508,7 +585,7 @@ } Py_DECREF(container); if (res == NULL) goto pop_3_error; - #line 512 "Python/generated_cases.c.h" + #line 589 "Python/generated_cases.c.h" STACK_SHRINK(2); stack_pointer[-1] = res; DISPATCH(); @@ -519,7 +596,7 @@ PyObject *start = stack_pointer[-2]; PyObject *container = stack_pointer[-3]; PyObject *v = stack_pointer[-4]; - #line 374 "Python/bytecodes.c" + #line 428 "Python/bytecodes.c" PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); int err; if (slice == NULL) { @@ -532,7 +609,7 @@ Py_DECREF(v); Py_DECREF(container); if (err) goto pop_4_error; - #line 536 "Python/generated_cases.c.h" + #line 613 "Python/generated_cases.c.h" STACK_SHRINK(4); DISPATCH(); } @@ -541,8 +618,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *list = stack_pointer[-2]; PyObject *res; - #line 389 "Python/bytecodes.c" - assert(cframe.use_tracing == 0); + #line 443 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); @@ -556,7 +632,7 @@ Py_INCREF(res); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(list); - #line 560 "Python/generated_cases.c.h" + #line 636 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -567,8 +643,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *tuple = stack_pointer[-2]; PyObject *res; - #line 406 "Python/bytecodes.c" - assert(cframe.use_tracing == 0); + #line 459 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); @@ -582,7 +657,7 @@ Py_INCREF(res); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(tuple); - #line 586 "Python/generated_cases.c.h" + #line 661 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -593,8 +668,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *dict = stack_pointer[-2]; PyObject *res; - #line 423 "Python/bytecodes.c" - assert(cframe.use_tracing == 0); + #line 475 "Python/bytecodes.c" DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); res = PyDict_GetItemWithError(dict, sub); @@ -602,14 +676,14 @@ if (!_PyErr_Occurred(tstate)) { _PyErr_SetKeyError(sub); } - #line 606 "Python/generated_cases.c.h" + #line 680 "Python/generated_cases.c.h" Py_DECREF(dict); Py_DECREF(sub); - #line 432 "Python/bytecodes.c" + #line 483 "Python/bytecodes.c" if (true) goto pop_2_error; } Py_INCREF(res); // Do this before DECREF'ing dict, sub - #line 613 "Python/generated_cases.c.h" + #line 687 "Python/generated_cases.c.h" Py_DECREF(dict); Py_DECREF(sub); STACK_SHRINK(1); @@ -621,7 +695,7 @@ TARGET(BINARY_SUBSCR_GETITEM) { PyObject *sub = stack_pointer[-1]; PyObject *container = stack_pointer[-2]; - #line 439 "Python/bytecodes.c" + #line 490 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(container); DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; @@ -642,15 +716,15 @@ new_frame->localsplus[1] = sub; JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR); DISPATCH_INLINED(new_frame); - #line 646 "Python/generated_cases.c.h" + #line 720 "Python/generated_cases.c.h" } TARGET(LIST_APPEND) { PyObject *v = stack_pointer[-1]; PyObject *list = stack_pointer[-(2 + (oparg-1))]; - #line 462 "Python/bytecodes.c" + #line 513 "Python/bytecodes.c" if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; - #line 654 "Python/generated_cases.c.h" + #line 728 "Python/generated_cases.c.h" STACK_SHRINK(1); PREDICT(JUMP_BACKWARD); DISPATCH(); @@ -659,13 +733,13 @@ TARGET(SET_ADD) { PyObject *v = stack_pointer[-1]; PyObject *set = stack_pointer[-(2 + (oparg-1))]; - #line 467 "Python/bytecodes.c" + #line 518 "Python/bytecodes.c" int err = PySet_Add(set, v); - #line 665 "Python/generated_cases.c.h" + #line 739 "Python/generated_cases.c.h" Py_DECREF(v); - #line 469 "Python/bytecodes.c" + #line 520 "Python/bytecodes.c" if (err) goto pop_1_error; - #line 669 "Python/generated_cases.c.h" + #line 743 "Python/generated_cases.c.h" STACK_SHRINK(1); PREDICT(JUMP_BACKWARD); DISPATCH(); @@ -678,10 +752,9 @@ PyObject *container = stack_pointer[-2]; PyObject *v = stack_pointer[-3]; uint16_t counter = read_u16(&next_instr[0].cache); - #line 480 "Python/bytecodes.c" + #line 531 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_StoreSubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); @@ -694,13 +767,13 @@ #endif /* ENABLE_SPECIALIZATION */ /* container[sub] = v */ int err = PyObject_SetItem(container, sub, v); - #line 698 "Python/generated_cases.c.h" + #line 771 "Python/generated_cases.c.h" Py_DECREF(v); Py_DECREF(container); Py_DECREF(sub); - #line 496 "Python/bytecodes.c" + #line 546 "Python/bytecodes.c" if (err) goto pop_3_error; - #line 704 "Python/generated_cases.c.h" + #line 777 "Python/generated_cases.c.h" STACK_SHRINK(3); next_instr += 1; DISPATCH(); @@ -710,8 +783,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *list = stack_pointer[-2]; PyObject *value = stack_pointer[-3]; - #line 500 "Python/bytecodes.c" - assert(cframe.use_tracing == 0); + #line 550 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); @@ -728,7 +800,7 @@ Py_DECREF(old_value); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(list); - #line 732 "Python/generated_cases.c.h" + #line 804 "Python/generated_cases.c.h" STACK_SHRINK(3); next_instr += 1; DISPATCH(); @@ -738,14 +810,13 @@ PyObject *sub = stack_pointer[-1]; PyObject *dict = stack_pointer[-2]; PyObject *value = stack_pointer[-3]; - #line 520 "Python/bytecodes.c" - assert(cframe.use_tracing == 0); + #line 569 "Python/bytecodes.c" DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); Py_DECREF(dict); if (err) goto pop_3_error; - #line 749 "Python/generated_cases.c.h" + #line 820 "Python/generated_cases.c.h" STACK_SHRINK(3); next_instr += 1; DISPATCH(); @@ -754,15 +825,15 @@ TARGET(DELETE_SUBSCR) { PyObject *sub = stack_pointer[-1]; PyObject *container = stack_pointer[-2]; - #line 529 "Python/bytecodes.c" + #line 577 "Python/bytecodes.c" /* del container[sub] */ int err = PyObject_DelItem(container, sub); - #line 761 "Python/generated_cases.c.h" + #line 832 "Python/generated_cases.c.h" Py_DECREF(container); Py_DECREF(sub); - #line 532 "Python/bytecodes.c" + #line 580 "Python/bytecodes.c" if (err) goto pop_2_error; - #line 766 "Python/generated_cases.c.h" + #line 837 "Python/generated_cases.c.h" STACK_SHRINK(2); DISPATCH(); } @@ -770,14 +841,14 @@ TARGET(CALL_INTRINSIC_1) { PyObject *value = stack_pointer[-1]; PyObject *res; - #line 536 "Python/bytecodes.c" + #line 584 "Python/bytecodes.c" assert(oparg <= MAX_INTRINSIC_1); res = _PyIntrinsics_UnaryFunctions[oparg](tstate, value); - #line 777 "Python/generated_cases.c.h" + #line 848 "Python/generated_cases.c.h" Py_DECREF(value); - #line 539 "Python/bytecodes.c" + #line 587 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; - #line 781 "Python/generated_cases.c.h" + #line 852 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -786,15 +857,15 @@ PyObject *value1 = stack_pointer[-1]; PyObject *value2 = stack_pointer[-2]; PyObject *res; - #line 543 "Python/bytecodes.c" + #line 591 "Python/bytecodes.c" assert(oparg <= MAX_INTRINSIC_2); res = _PyIntrinsics_BinaryFunctions[oparg](tstate, value2, value1); - #line 793 "Python/generated_cases.c.h" + #line 864 "Python/generated_cases.c.h" Py_DECREF(value2); Py_DECREF(value1); - #line 546 "Python/bytecodes.c" + #line 594 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 798 "Python/generated_cases.c.h" + #line 869 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); @@ -802,7 +873,7 @@ TARGET(RAISE_VARARGS) { PyObject **args = (stack_pointer - oparg); - #line 550 "Python/bytecodes.c" + #line 598 "Python/bytecodes.c" PyObject *cause = NULL, *exc = NULL; switch (oparg) { case 2: @@ -820,34 +891,31 @@ break; } if (true) { STACK_SHRINK(oparg); goto error; } - #line 824 "Python/generated_cases.c.h" + #line 895 "Python/generated_cases.c.h" } TARGET(INTERPRETER_EXIT) { PyObject *retval = stack_pointer[-1]; - #line 570 "Python/bytecodes.c" + #line 618 "Python/bytecodes.c" assert(frame == &entry_frame); assert(_PyFrame_IsIncomplete(frame)); STACK_SHRINK(1); // Since we're not going to DISPATCH() assert(EMPTY()); /* Restore previous cframe and return. */ tstate->cframe = cframe.previous; - tstate->cframe->use_tracing = cframe.use_tracing; assert(tstate->cframe->current_frame == frame->previous); assert(!_PyErr_Occurred(tstate)); _Py_LeaveRecursiveCallTstate(tstate); return retval; - #line 841 "Python/generated_cases.c.h" + #line 911 "Python/generated_cases.c.h" } TARGET(RETURN_VALUE) { PyObject *retval = stack_pointer[-1]; - #line 584 "Python/bytecodes.c" + #line 631 "Python/bytecodes.c" STACK_SHRINK(1); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); - TRACE_FUNCTION_EXIT(); - DTRACE_FUNCTION_EXIT(); _Py_LeaveRecursiveCallPy(tstate); assert(frame != &entry_frame); // GH-99729: We need to unlink the frame *before* clearing it: @@ -856,17 +924,36 @@ _PyEvalFrameClearAndPop(tstate, dying); _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 860 "Python/generated_cases.c.h" + #line 928 "Python/generated_cases.c.h" + } + + TARGET(INSTRUMENTED_RETURN_VALUE) { + PyObject *retval = stack_pointer[-1]; + #line 645 "Python/bytecodes.c" + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, next_instr-1, retval); + if (err) goto error; + STACK_SHRINK(1); + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != &entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = cframe.current_frame = dying->previous; + _PyEvalFrameClearAndPop(tstate, dying); + _PyFrame_StackPush(frame, retval); + goto resume_frame; + #line 949 "Python/generated_cases.c.h" } TARGET(RETURN_CONST) { - #line 600 "Python/bytecodes.c" + #line 663 "Python/bytecodes.c" PyObject *retval = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(retval); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); - TRACE_FUNCTION_EXIT(); - DTRACE_FUNCTION_EXIT(); _Py_LeaveRecursiveCallPy(tstate); assert(frame != &entry_frame); // GH-99729: We need to unlink the frame *before* clearing it: @@ -875,13 +962,34 @@ _PyEvalFrameClearAndPop(tstate, dying); _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 879 "Python/generated_cases.c.h" + #line 966 "Python/generated_cases.c.h" + } + + TARGET(INSTRUMENTED_RETURN_CONST) { + #line 678 "Python/bytecodes.c" + PyObject *retval = GETITEM(frame->f_code->co_consts, oparg); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, next_instr-1, retval); + if (err) goto error; + Py_INCREF(retval); + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != &entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = cframe.current_frame = dying->previous; + _PyEvalFrameClearAndPop(tstate, dying); + _PyFrame_StackPush(frame, retval); + goto resume_frame; + #line 987 "Python/generated_cases.c.h" } TARGET(GET_AITER) { PyObject *obj = stack_pointer[-1]; PyObject *iter; - #line 617 "Python/bytecodes.c" + #line 697 "Python/bytecodes.c" unaryfunc getter = NULL; PyTypeObject *type = Py_TYPE(obj); @@ -894,16 +1002,16 @@ "'async for' requires an object with " "__aiter__ method, got %.100s", type->tp_name); - #line 898 "Python/generated_cases.c.h" + #line 1006 "Python/generated_cases.c.h" Py_DECREF(obj); - #line 630 "Python/bytecodes.c" + #line 710 "Python/bytecodes.c" if (true) goto pop_1_error; } iter = (*getter)(obj); - #line 905 "Python/generated_cases.c.h" + #line 1013 "Python/generated_cases.c.h" Py_DECREF(obj); - #line 635 "Python/bytecodes.c" + #line 715 "Python/bytecodes.c" if (iter == NULL) goto pop_1_error; if (Py_TYPE(iter)->tp_as_async == NULL || @@ -916,7 +1024,7 @@ Py_DECREF(iter); if (true) goto pop_1_error; } - #line 920 "Python/generated_cases.c.h" + #line 1028 "Python/generated_cases.c.h" stack_pointer[-1] = iter; DISPATCH(); } @@ -924,7 +1032,7 @@ TARGET(GET_ANEXT) { PyObject *aiter = stack_pointer[-1]; PyObject *awaitable; - #line 650 "Python/bytecodes.c" + #line 730 "Python/bytecodes.c" unaryfunc getter = NULL; PyObject *next_iter = NULL; PyTypeObject *type = Py_TYPE(aiter); @@ -968,7 +1076,7 @@ } } - #line 972 "Python/generated_cases.c.h" + #line 1080 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = awaitable; PREDICT(LOAD_CONST); @@ -979,16 +1087,16 @@ PREDICTED(GET_AWAITABLE); PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 697 "Python/bytecodes.c" + #line 777 "Python/bytecodes.c" iter = _PyCoro_GetAwaitableIter(iterable); if (iter == NULL) { format_awaitable_error(tstate, Py_TYPE(iterable), oparg); } - #line 990 "Python/generated_cases.c.h" + #line 1098 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 704 "Python/bytecodes.c" + #line 784 "Python/bytecodes.c" if (iter != NULL && PyCoro_CheckExact(iter)) { PyObject *yf = _PyGen_yf((PyGenObject*)iter); @@ -1006,7 +1114,7 @@ if (iter == NULL) goto pop_1_error; - #line 1010 "Python/generated_cases.c.h" + #line 1118 "Python/generated_cases.c.h" stack_pointer[-1] = iter; PREDICT(LOAD_CONST); DISPATCH(); @@ -1017,11 +1125,10 @@ PyObject *v = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; PyObject *retval; - #line 730 "Python/bytecodes.c" + #line 810 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PySendCache *cache = (_PySendCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_Send(receiver, next_instr); DISPATCH_SAME_OPARG(); @@ -1030,6 +1137,20 @@ DECREMENT_ADAPTIVE_COUNTER(cache->counter); #endif /* ENABLE_SPECIALIZATION */ assert(frame != &entry_frame); + if ((Py_TYPE(receiver) == &PyGen_Type || + Py_TYPE(receiver) == &PyCoro_Type) && ((PyGenObject *)receiver)->gi_frame_state < FRAME_EXECUTING) + { + PyGenObject *gen = (PyGenObject *)receiver; + _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; + frame->yield_offset = oparg; + STACK_SHRINK(1); + _PyFrame_StackPush(gen_frame, v); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + JUMPBY(INLINE_CACHE_ENTRIES_SEND + oparg); + DISPATCH_INLINED(gen_frame); + } if (Py_IsNone(v) && PyIter_Check(receiver)) { retval = Py_TYPE(receiver)->tp_iternext(receiver); } @@ -1037,23 +1158,20 @@ retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v); } if (retval == NULL) { - if (tstate->c_tracefunc != NULL - && _PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) - call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj, tstate, frame); + if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) + ) { + monitor_raise(tstate, frame, next_instr-1); + } if (_PyGen_FetchStopIterationValue(&retval) == 0) { assert(retval != NULL); JUMPBY(oparg); } else { - assert(retval == NULL); goto error; } } - else { - assert(retval != NULL); - } Py_DECREF(v); - #line 1057 "Python/generated_cases.c.h" + #line 1175 "Python/generated_cases.c.h" stack_pointer[-1] = retval; next_instr += 1; DISPATCH(); @@ -1062,8 +1180,7 @@ TARGET(SEND_GEN) { PyObject *v = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; - #line 768 "Python/bytecodes.c" - assert(cframe.use_tracing == 0); + #line 858 "Python/bytecodes.c" PyGenObject *gen = (PyGenObject *)receiver; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); @@ -1078,12 +1195,35 @@ tstate->exc_info = &gen->gi_exc_state; JUMPBY(INLINE_CACHE_ENTRIES_SEND + oparg); DISPATCH_INLINED(gen_frame); - #line 1082 "Python/generated_cases.c.h" + #line 1199 "Python/generated_cases.c.h" + } + + TARGET(INSTRUMENTED_YIELD_VALUE) { + PyObject *retval = stack_pointer[-1]; + #line 875 "Python/bytecodes.c" + assert(frame != &entry_frame); + PyGenObject *gen = _PyFrame_GetGenerator(frame); + gen->gi_frame_state = FRAME_SUSPENDED; + _PyFrame_SetStackPointer(frame, stack_pointer - 1); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_YIELD, + frame, next_instr-1, retval); + if (err) goto error; + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = cframe.current_frame = frame->previous; + gen_frame->previous = NULL; + frame->prev_instr -= frame->yield_offset; + _PyFrame_StackPush(frame, retval); + goto resume_frame; + #line 1222 "Python/generated_cases.c.h" } TARGET(YIELD_VALUE) { PyObject *retval = stack_pointer[-1]; - #line 786 "Python/bytecodes.c" + #line 895 "Python/bytecodes.c" // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. @@ -1091,8 +1231,6 @@ PyGenObject *gen = _PyFrame_GetGenerator(frame); gen->gi_frame_state = FRAME_SUSPENDED; _PyFrame_SetStackPointer(frame, stack_pointer - 1); - TRACE_FUNCTION_EXIT(); - DTRACE_FUNCTION_EXIT(); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); @@ -1102,15 +1240,15 @@ frame->prev_instr -= frame->yield_offset; _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 1106 "Python/generated_cases.c.h" + #line 1244 "Python/generated_cases.c.h" } TARGET(POP_EXCEPT) { PyObject *exc_value = stack_pointer[-1]; - #line 807 "Python/bytecodes.c" + #line 914 "Python/bytecodes.c" _PyErr_StackItem *exc_info = tstate->exc_info; Py_XSETREF(exc_info->exc_value, exc_value); - #line 1114 "Python/generated_cases.c.h" + #line 1252 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -1118,7 +1256,7 @@ TARGET(RERAISE) { PyObject *exc = stack_pointer[-1]; PyObject **values = (stack_pointer - (1 + oparg)); - #line 812 "Python/bytecodes.c" + #line 919 "Python/bytecodes.c" assert(oparg >= 0 && oparg <= 2); if (oparg) { PyObject *lasti = values[0]; @@ -1136,26 +1274,26 @@ Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); goto exception_unwind; - #line 1140 "Python/generated_cases.c.h" + #line 1278 "Python/generated_cases.c.h" } TARGET(END_ASYNC_FOR) { PyObject *exc = stack_pointer[-1]; PyObject *awaitable = stack_pointer[-2]; - #line 832 "Python/bytecodes.c" + #line 939 "Python/bytecodes.c" assert(exc && PyExceptionInstance_Check(exc)); if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { - #line 1149 "Python/generated_cases.c.h" + #line 1287 "Python/generated_cases.c.h" Py_DECREF(awaitable); Py_DECREF(exc); - #line 835 "Python/bytecodes.c" + #line 942 "Python/bytecodes.c" } else { Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); goto exception_unwind; } - #line 1159 "Python/generated_cases.c.h" + #line 1297 "Python/generated_cases.c.h" STACK_SHRINK(2); DISPATCH(); } @@ -1166,23 +1304,23 @@ PyObject *sub_iter = stack_pointer[-3]; PyObject *none; PyObject *value; - #line 844 "Python/bytecodes.c" + #line 951 "Python/bytecodes.c" assert(throwflag); assert(exc_value && PyExceptionInstance_Check(exc_value)); if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { value = Py_NewRef(((PyStopIterationObject *)exc_value)->value); - #line 1175 "Python/generated_cases.c.h" + #line 1313 "Python/generated_cases.c.h" Py_DECREF(sub_iter); Py_DECREF(last_sent_val); Py_DECREF(exc_value); - #line 849 "Python/bytecodes.c" + #line 956 "Python/bytecodes.c" none = Py_NewRef(Py_None); } else { _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); goto exception_unwind; } - #line 1186 "Python/generated_cases.c.h" + #line 1324 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = value; stack_pointer[-2] = none; @@ -1191,9 +1329,9 @@ TARGET(LOAD_ASSERTION_ERROR) { PyObject *value; - #line 858 "Python/bytecodes.c" + #line 965 "Python/bytecodes.c" value = Py_NewRef(PyExc_AssertionError); - #line 1197 "Python/generated_cases.c.h" + #line 1335 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -1201,7 +1339,7 @@ TARGET(LOAD_BUILD_CLASS) { PyObject *bc; - #line 862 "Python/bytecodes.c" + #line 969 "Python/bytecodes.c" if (PyDict_CheckExact(BUILTINS())) { bc = _PyDict_GetItemWithError(BUILTINS(), &_Py_ID(__build_class__)); @@ -1223,7 +1361,7 @@ if (true) goto error; } } - #line 1227 "Python/generated_cases.c.h" + #line 1365 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = bc; DISPATCH(); @@ -1231,33 +1369,33 @@ TARGET(STORE_NAME) { PyObject *v = stack_pointer[-1]; - #line 886 "Python/bytecodes.c" + #line 993 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); PyObject *ns = LOCALS(); int err; if (ns == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals found when storing %R", name); - #line 1242 "Python/generated_cases.c.h" + #line 1380 "Python/generated_cases.c.h" Py_DECREF(v); - #line 893 "Python/bytecodes.c" + #line 1000 "Python/bytecodes.c" if (true) goto pop_1_error; } if (PyDict_CheckExact(ns)) err = PyDict_SetItem(ns, name, v); else err = PyObject_SetItem(ns, name, v); - #line 1251 "Python/generated_cases.c.h" + #line 1389 "Python/generated_cases.c.h" Py_DECREF(v); - #line 900 "Python/bytecodes.c" + #line 1007 "Python/bytecodes.c" if (err) goto pop_1_error; - #line 1255 "Python/generated_cases.c.h" + #line 1393 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(DELETE_NAME) { - #line 904 "Python/bytecodes.c" + #line 1011 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); PyObject *ns = LOCALS(); int err; @@ -1274,7 +1412,7 @@ name); goto error; } - #line 1278 "Python/generated_cases.c.h" + #line 1416 "Python/generated_cases.c.h" DISPATCH(); } @@ -1282,11 +1420,10 @@ PREDICTED(UNPACK_SEQUENCE); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); PyObject *seq = stack_pointer[-1]; - #line 930 "Python/bytecodes.c" + #line 1037 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_UnpackSequence(seq, next_instr, oparg); DISPATCH_SAME_OPARG(); @@ -1296,11 +1433,11 @@ #endif /* ENABLE_SPECIALIZATION */ PyObject **top = stack_pointer + oparg - 1; int res = unpack_iterable(tstate, seq, oparg, -1, top); - #line 1300 "Python/generated_cases.c.h" + #line 1437 "Python/generated_cases.c.h" Py_DECREF(seq); - #line 944 "Python/bytecodes.c" + #line 1050 "Python/bytecodes.c" if (res == 0) goto pop_1_error; - #line 1304 "Python/generated_cases.c.h" + #line 1441 "Python/generated_cases.c.h" STACK_SHRINK(1); STACK_GROW(oparg); next_instr += 1; @@ -1310,14 +1447,14 @@ TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { PyObject *seq = stack_pointer[-1]; PyObject **values = stack_pointer - (1); - #line 948 "Python/bytecodes.c" + #line 1054 "Python/bytecodes.c" DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); assert(oparg == 2); STAT_INC(UNPACK_SEQUENCE, hit); values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); - #line 1321 "Python/generated_cases.c.h" + #line 1458 "Python/generated_cases.c.h" Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); @@ -1328,7 +1465,7 @@ TARGET(UNPACK_SEQUENCE_TUPLE) { PyObject *seq = stack_pointer[-1]; PyObject **values = stack_pointer - (1); - #line 958 "Python/bytecodes.c" + #line 1064 "Python/bytecodes.c" DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); @@ -1336,7 +1473,7 @@ for (int i = oparg; --i >= 0; ) { *values++ = Py_NewRef(items[i]); } - #line 1340 "Python/generated_cases.c.h" + #line 1477 "Python/generated_cases.c.h" Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); @@ -1347,7 +1484,7 @@ TARGET(UNPACK_SEQUENCE_LIST) { PyObject *seq = stack_pointer[-1]; PyObject **values = stack_pointer - (1); - #line 969 "Python/bytecodes.c" + #line 1075 "Python/bytecodes.c" DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); @@ -1355,7 +1492,7 @@ for (int i = oparg; --i >= 0; ) { *values++ = Py_NewRef(items[i]); } - #line 1359 "Python/generated_cases.c.h" + #line 1496 "Python/generated_cases.c.h" Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); @@ -1365,15 +1502,15 @@ TARGET(UNPACK_EX) { PyObject *seq = stack_pointer[-1]; - #line 980 "Python/bytecodes.c" + #line 1086 "Python/bytecodes.c" int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); PyObject **top = stack_pointer + totalargs - 1; int res = unpack_iterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); - #line 1373 "Python/generated_cases.c.h" + #line 1510 "Python/generated_cases.c.h" Py_DECREF(seq); - #line 984 "Python/bytecodes.c" + #line 1090 "Python/bytecodes.c" if (res == 0) goto pop_1_error; - #line 1377 "Python/generated_cases.c.h" + #line 1514 "Python/generated_cases.c.h" STACK_GROW((oparg & 0xFF) + (oparg >> 8)); DISPATCH(); } @@ -1384,10 +1521,9 @@ PyObject *owner = stack_pointer[-1]; PyObject *v = stack_pointer[-2]; uint16_t counter = read_u16(&next_instr[0].cache); - #line 995 "Python/bytecodes.c" + #line 1101 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - assert(cframe.use_tracing == 0); PyObject *name = GETITEM(frame->f_code->co_names, oparg); next_instr--; _Py_Specialize_StoreAttr(owner, next_instr, name); @@ -1401,12 +1537,12 @@ #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyObject_SetAttr(owner, name, v); - #line 1405 "Python/generated_cases.c.h" + #line 1541 "Python/generated_cases.c.h" Py_DECREF(v); Py_DECREF(owner); - #line 1012 "Python/bytecodes.c" + #line 1117 "Python/bytecodes.c" if (err) goto pop_2_error; - #line 1410 "Python/generated_cases.c.h" + #line 1546 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -1414,34 +1550,34 @@ TARGET(DELETE_ATTR) { PyObject *owner = stack_pointer[-1]; - #line 1016 "Python/bytecodes.c" + #line 1121 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyObject_SetAttr(owner, name, (PyObject *)NULL); - #line 1421 "Python/generated_cases.c.h" + #line 1557 "Python/generated_cases.c.h" Py_DECREF(owner); - #line 1019 "Python/bytecodes.c" + #line 1124 "Python/bytecodes.c" if (err) goto pop_1_error; - #line 1425 "Python/generated_cases.c.h" + #line 1561 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(STORE_GLOBAL) { PyObject *v = stack_pointer[-1]; - #line 1023 "Python/bytecodes.c" + #line 1128 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyDict_SetItem(GLOBALS(), name, v); - #line 1435 "Python/generated_cases.c.h" + #line 1571 "Python/generated_cases.c.h" Py_DECREF(v); - #line 1026 "Python/bytecodes.c" + #line 1131 "Python/bytecodes.c" if (err) goto pop_1_error; - #line 1439 "Python/generated_cases.c.h" + #line 1575 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(DELETE_GLOBAL) { - #line 1030 "Python/bytecodes.c" + #line 1135 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err; err = PyDict_DelItem(GLOBALS(), name); @@ -1453,13 +1589,13 @@ } goto error; } - #line 1457 "Python/generated_cases.c.h" + #line 1593 "Python/generated_cases.c.h" DISPATCH(); } TARGET(LOAD_NAME) { PyObject *v; - #line 1044 "Python/bytecodes.c" + #line 1149 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); PyObject *locals = LOCALS(); if (locals == NULL) { @@ -1518,7 +1654,7 @@ } } } - #line 1522 "Python/generated_cases.c.h" + #line 1658 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = v; DISPATCH(); @@ -1529,11 +1665,10 @@ static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyObject *null = NULL; PyObject *v; - #line 1111 "Python/bytecodes.c" + #line 1216 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); PyObject *name = GETITEM(frame->f_code->co_names, oparg>>1); next_instr--; _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); @@ -1582,7 +1717,7 @@ } } null = NULL; - #line 1586 "Python/generated_cases.c.h" + #line 1721 "Python/generated_cases.c.h" STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = v; @@ -1596,8 +1731,7 @@ PyObject *res; uint16_t index = read_u16(&next_instr[1].cache); uint16_t version = read_u16(&next_instr[2].cache); - #line 1166 "Python/bytecodes.c" - assert(cframe.use_tracing == 0); + #line 1270 "Python/bytecodes.c" DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); PyDictObject *dict = (PyDictObject *)GLOBALS(); DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); @@ -1608,7 +1742,7 @@ Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; - #line 1612 "Python/generated_cases.c.h" + #line 1746 "Python/generated_cases.c.h" STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -1623,12 +1757,12 @@ uint16_t index = read_u16(&next_instr[1].cache); uint16_t mod_version = read_u16(&next_instr[2].cache); uint16_t bltn_version = read_u16(&next_instr[3].cache); - #line 1180 "Python/bytecodes.c" - assert(cframe.use_tracing == 0); + #line 1283 "Python/bytecodes.c" DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); DEOPT_IF(!PyDict_CheckExact(BUILTINS()), LOAD_GLOBAL); PyDictObject *mdict = (PyDictObject *)GLOBALS(); PyDictObject *bdict = (PyDictObject *)BUILTINS(); + assert(opcode == LOAD_GLOBAL_BUILTIN); DEOPT_IF(mdict->ma_keys->dk_version != mod_version, LOAD_GLOBAL); DEOPT_IF(bdict->ma_keys->dk_version != bltn_version, LOAD_GLOBAL); assert(DK_IS_UNICODE(bdict->ma_keys)); @@ -1638,7 +1772,7 @@ Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; - #line 1642 "Python/generated_cases.c.h" + #line 1776 "Python/generated_cases.c.h" STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -1648,16 +1782,16 @@ } TARGET(DELETE_FAST) { - #line 1197 "Python/bytecodes.c" + #line 1300 "Python/bytecodes.c" PyObject *v = GETLOCAL(oparg); if (v == NULL) goto unbound_local_error; SETLOCAL(oparg, NULL); - #line 1656 "Python/generated_cases.c.h" + #line 1790 "Python/generated_cases.c.h" DISPATCH(); } TARGET(MAKE_CELL) { - #line 1203 "Python/bytecodes.c" + #line 1306 "Python/bytecodes.c" // "initial" is probably NULL but not if it's an arg (or set // via PyFrame_LocalsToFast() before MAKE_CELL has run). PyObject *initial = GETLOCAL(oparg); @@ -1666,12 +1800,12 @@ goto resume_with_error; } SETLOCAL(oparg, cell); - #line 1670 "Python/generated_cases.c.h" + #line 1804 "Python/generated_cases.c.h" DISPATCH(); } TARGET(DELETE_DEREF) { - #line 1214 "Python/bytecodes.c" + #line 1317 "Python/bytecodes.c" PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); // Can't use ERROR_IF here. @@ -1682,13 +1816,13 @@ } PyCell_SET(cell, NULL); Py_DECREF(oldobj); - #line 1686 "Python/generated_cases.c.h" + #line 1820 "Python/generated_cases.c.h" DISPATCH(); } TARGET(LOAD_CLASSDEREF) { PyObject *value; - #line 1227 "Python/bytecodes.c" + #line 1330 "Python/bytecodes.c" PyObject *name, *locals = LOCALS(); assert(locals); assert(oparg >= 0 && oparg < frame->f_code->co_nlocalsplus); @@ -1720,7 +1854,7 @@ } Py_INCREF(value); } - #line 1724 "Python/generated_cases.c.h" + #line 1858 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -1728,7 +1862,7 @@ TARGET(LOAD_DEREF) { PyObject *value; - #line 1261 "Python/bytecodes.c" + #line 1364 "Python/bytecodes.c" PyObject *cell = GETLOCAL(oparg); value = PyCell_GET(cell); if (value == NULL) { @@ -1736,7 +1870,7 @@ if (true) goto error; } Py_INCREF(value); - #line 1740 "Python/generated_cases.c.h" + #line 1874 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -1744,18 +1878,18 @@ TARGET(STORE_DEREF) { PyObject *v = stack_pointer[-1]; - #line 1271 "Python/bytecodes.c" + #line 1374 "Python/bytecodes.c" PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); PyCell_SET(cell, v); Py_XDECREF(oldobj); - #line 1753 "Python/generated_cases.c.h" + #line 1887 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(COPY_FREE_VARS) { - #line 1278 "Python/bytecodes.c" + #line 1381 "Python/bytecodes.c" /* Copy closure variables to free variables */ PyCodeObject *co = frame->f_code; assert(PyFunction_Check(frame->f_funcobj)); @@ -1766,22 +1900,22 @@ PyObject *o = PyTuple_GET_ITEM(closure, i); frame->localsplus[offset + i] = Py_NewRef(o); } - #line 1770 "Python/generated_cases.c.h" + #line 1904 "Python/generated_cases.c.h" DISPATCH(); } TARGET(BUILD_STRING) { PyObject **pieces = (stack_pointer - oparg); PyObject *str; - #line 1291 "Python/bytecodes.c" + #line 1394 "Python/bytecodes.c" str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); - #line 1779 "Python/generated_cases.c.h" + #line 1913 "Python/generated_cases.c.h" for (int _i = oparg; --_i >= 0;) { Py_DECREF(pieces[_i]); } - #line 1293 "Python/bytecodes.c" + #line 1396 "Python/bytecodes.c" if (str == NULL) { STACK_SHRINK(oparg); goto error; } - #line 1785 "Python/generated_cases.c.h" + #line 1919 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = str; @@ -1791,10 +1925,10 @@ TARGET(BUILD_TUPLE) { PyObject **values = (stack_pointer - oparg); PyObject *tup; - #line 1297 "Python/bytecodes.c" + #line 1400 "Python/bytecodes.c" tup = _PyTuple_FromArraySteal(values, oparg); if (tup == NULL) { STACK_SHRINK(oparg); goto error; } - #line 1798 "Python/generated_cases.c.h" + #line 1932 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = tup; @@ -1804,10 +1938,10 @@ TARGET(BUILD_LIST) { PyObject **values = (stack_pointer - oparg); PyObject *list; - #line 1302 "Python/bytecodes.c" + #line 1405 "Python/bytecodes.c" list = _PyList_FromArraySteal(values, oparg); if (list == NULL) { STACK_SHRINK(oparg); goto error; } - #line 1811 "Python/generated_cases.c.h" + #line 1945 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = list; @@ -1817,7 +1951,7 @@ TARGET(LIST_EXTEND) { PyObject *iterable = stack_pointer[-1]; PyObject *list = stack_pointer[-(2 + (oparg-1))]; - #line 1307 "Python/bytecodes.c" + #line 1410 "Python/bytecodes.c" PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); if (none_val == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && @@ -1828,13 +1962,13 @@ "Value after * must be an iterable, not %.200s", Py_TYPE(iterable)->tp_name); } - #line 1832 "Python/generated_cases.c.h" + #line 1966 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 1318 "Python/bytecodes.c" + #line 1421 "Python/bytecodes.c" if (true) goto pop_1_error; } Py_DECREF(none_val); - #line 1838 "Python/generated_cases.c.h" + #line 1972 "Python/generated_cases.c.h" Py_DECREF(iterable); STACK_SHRINK(1); DISPATCH(); @@ -1843,13 +1977,13 @@ TARGET(SET_UPDATE) { PyObject *iterable = stack_pointer[-1]; PyObject *set = stack_pointer[-(2 + (oparg-1))]; - #line 1325 "Python/bytecodes.c" + #line 1428 "Python/bytecodes.c" int err = _PySet_Update(set, iterable); - #line 1849 "Python/generated_cases.c.h" + #line 1983 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 1327 "Python/bytecodes.c" + #line 1430 "Python/bytecodes.c" if (err < 0) goto pop_1_error; - #line 1853 "Python/generated_cases.c.h" + #line 1987 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -1857,7 +1991,7 @@ TARGET(BUILD_SET) { PyObject **values = (stack_pointer - oparg); PyObject *set; - #line 1331 "Python/bytecodes.c" + #line 1434 "Python/bytecodes.c" set = PySet_New(NULL); if (set == NULL) goto error; @@ -1872,7 +2006,7 @@ Py_DECREF(set); if (true) { STACK_SHRINK(oparg); goto error; } } - #line 1876 "Python/generated_cases.c.h" + #line 2010 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = set; @@ -1882,7 +2016,7 @@ TARGET(BUILD_MAP) { PyObject **values = (stack_pointer - oparg*2); PyObject *map; - #line 1348 "Python/bytecodes.c" + #line 1451 "Python/bytecodes.c" map = _PyDict_FromItems( values, 2, values+1, 2, @@ -1890,13 +2024,13 @@ if (map == NULL) goto error; - #line 1894 "Python/generated_cases.c.h" + #line 2028 "Python/generated_cases.c.h" for (int _i = oparg*2; --_i >= 0;) { Py_DECREF(values[_i]); } - #line 1356 "Python/bytecodes.c" + #line 1459 "Python/bytecodes.c" if (map == NULL) { STACK_SHRINK(oparg*2); goto error; } - #line 1900 "Python/generated_cases.c.h" + #line 2034 "Python/generated_cases.c.h" STACK_SHRINK(oparg*2); STACK_GROW(1); stack_pointer[-1] = map; @@ -1904,7 +2038,7 @@ } TARGET(SETUP_ANNOTATIONS) { - #line 1360 "Python/bytecodes.c" + #line 1463 "Python/bytecodes.c" int err; PyObject *ann_dict; if (LOCALS() == NULL) { @@ -1944,7 +2078,7 @@ Py_DECREF(ann_dict); } } - #line 1948 "Python/generated_cases.c.h" + #line 2082 "Python/generated_cases.c.h" DISPATCH(); } @@ -1952,7 +2086,7 @@ PyObject *keys = stack_pointer[-1]; PyObject **values = (stack_pointer - (1 + oparg)); PyObject *map; - #line 1402 "Python/bytecodes.c" + #line 1505 "Python/bytecodes.c" if (!PyTuple_CheckExact(keys) || PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { _PyErr_SetString(tstate, PyExc_SystemError, @@ -1962,14 +2096,14 @@ map = _PyDict_FromItems( &PyTuple_GET_ITEM(keys, 0), 1, values, 1, oparg); - #line 1966 "Python/generated_cases.c.h" + #line 2100 "Python/generated_cases.c.h" for (int _i = oparg; --_i >= 0;) { Py_DECREF(values[_i]); } Py_DECREF(keys); - #line 1412 "Python/bytecodes.c" + #line 1515 "Python/bytecodes.c" if (map == NULL) { STACK_SHRINK(oparg); goto pop_1_error; } - #line 1973 "Python/generated_cases.c.h" + #line 2107 "Python/generated_cases.c.h" STACK_SHRINK(oparg); stack_pointer[-1] = map; DISPATCH(); @@ -1977,7 +2111,7 @@ TARGET(DICT_UPDATE) { PyObject *update = stack_pointer[-1]; - #line 1416 "Python/bytecodes.c" + #line 1519 "Python/bytecodes.c" PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (PyDict_Update(dict, update) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { @@ -1985,12 +2119,12 @@ "'%.200s' object is not a mapping", Py_TYPE(update)->tp_name); } - #line 1989 "Python/generated_cases.c.h" + #line 2123 "Python/generated_cases.c.h" Py_DECREF(update); - #line 1424 "Python/bytecodes.c" + #line 1527 "Python/bytecodes.c" if (true) goto pop_1_error; } - #line 1994 "Python/generated_cases.c.h" + #line 2128 "Python/generated_cases.c.h" Py_DECREF(update); STACK_SHRINK(1); DISPATCH(); @@ -1998,17 +2132,17 @@ TARGET(DICT_MERGE) { PyObject *update = stack_pointer[-1]; - #line 1430 "Python/bytecodes.c" + #line 1533 "Python/bytecodes.c" PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (_PyDict_MergeEx(dict, update, 2) < 0) { format_kwargs_error(tstate, PEEK(3 + oparg), update); - #line 2007 "Python/generated_cases.c.h" + #line 2141 "Python/generated_cases.c.h" Py_DECREF(update); - #line 1435 "Python/bytecodes.c" + #line 1538 "Python/bytecodes.c" if (true) goto pop_1_error; } - #line 2012 "Python/generated_cases.c.h" + #line 2146 "Python/generated_cases.c.h" Py_DECREF(update); STACK_SHRINK(1); PREDICT(CALL_FUNCTION_EX); @@ -2018,13 +2152,13 @@ TARGET(MAP_ADD) { PyObject *value = stack_pointer[-1]; PyObject *key = stack_pointer[-2]; - #line 1442 "Python/bytecodes.c" + #line 1545 "Python/bytecodes.c" PyObject *dict = PEEK(oparg + 2); // key, value are still on the stack assert(PyDict_CheckExact(dict)); /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; - #line 2028 "Python/generated_cases.c.h" + #line 2162 "Python/generated_cases.c.h" STACK_SHRINK(2); PREDICT(JUMP_BACKWARD); DISPATCH(); @@ -2036,11 +2170,10 @@ PyObject *owner = stack_pointer[-1]; PyObject *res2 = NULL; PyObject *res; - #line 1465 "Python/bytecodes.c" + #line 1568 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); PyObject *name = GETITEM(frame->f_code->co_names, oparg>>1); next_instr--; _Py_Specialize_LoadAttr(owner, next_instr, name); @@ -2071,9 +2204,9 @@ NULL | meth | arg1 | ... | argN */ - #line 2075 "Python/generated_cases.c.h" + #line 2208 "Python/generated_cases.c.h" Py_DECREF(owner); - #line 1500 "Python/bytecodes.c" + #line 1602 "Python/bytecodes.c" if (meth == NULL) goto pop_1_error; res2 = NULL; res = meth; @@ -2082,12 +2215,12 @@ else { /* Classic, pushes one value. */ res = PyObject_GetAttr(owner, name); - #line 2086 "Python/generated_cases.c.h" + #line 2219 "Python/generated_cases.c.h" Py_DECREF(owner); - #line 1509 "Python/bytecodes.c" + #line 1611 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; } - #line 2091 "Python/generated_cases.c.h" + #line 2224 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -2101,8 +2234,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1514 "Python/bytecodes.c" - assert(cframe.use_tracing == 0); + #line 1616 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2115,7 +2247,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2119 "Python/generated_cases.c.h" + #line 2251 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2130,8 +2262,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1531 "Python/bytecodes.c" - assert(cframe.use_tracing == 0); + #line 1632 "Python/bytecodes.c" DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict != NULL); @@ -2144,7 +2275,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2148 "Python/generated_cases.c.h" + #line 2279 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2159,8 +2290,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1548 "Python/bytecodes.c" - assert(cframe.use_tracing == 0); + #line 1648 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2187,7 +2317,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2191 "Python/generated_cases.c.h" + #line 2321 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2202,8 +2332,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1579 "Python/bytecodes.c" - assert(cframe.use_tracing == 0); + #line 1678 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2213,7 +2342,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2217 "Python/generated_cases.c.h" + #line 2346 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2228,8 +2357,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 1593 "Python/bytecodes.c" - assert(cframe.use_tracing == 0); + #line 1691 "Python/bytecodes.c" DEOPT_IF(!PyType_Check(cls), LOAD_ATTR); DEOPT_IF(((PyTypeObject *)cls)->tp_version_tag != type_version, @@ -2241,7 +2369,7 @@ res = descr; assert(res != NULL); Py_INCREF(res); - #line 2245 "Python/generated_cases.c.h" + #line 2373 "Python/generated_cases.c.h" Py_DECREF(cls); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2255,8 +2383,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t func_version = read_u32(&next_instr[3].cache); PyObject *fget = read_obj(&next_instr[5].cache); - #line 1609 "Python/bytecodes.c" - assert(cframe.use_tracing == 0); + #line 1706 "Python/bytecodes.c" DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); @@ -2279,7 +2406,7 @@ new_frame->localsplus[0] = owner; JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); DISPATCH_INLINED(new_frame); - #line 2283 "Python/generated_cases.c.h" + #line 2410 "Python/generated_cases.c.h" } TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { @@ -2287,8 +2414,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t func_version = read_u32(&next_instr[3].cache); PyObject *getattribute = read_obj(&next_instr[5].cache); - #line 1635 "Python/bytecodes.c" - assert(cframe.use_tracing == 0); + #line 1731 "Python/bytecodes.c" DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); @@ -2313,7 +2439,7 @@ new_frame->localsplus[1] = Py_NewRef(name); JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); DISPATCH_INLINED(new_frame); - #line 2317 "Python/generated_cases.c.h" + #line 2443 "Python/generated_cases.c.h" } TARGET(STORE_ATTR_INSTANCE_VALUE) { @@ -2321,8 +2447,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1663 "Python/bytecodes.c" - assert(cframe.use_tracing == 0); + #line 1758 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2340,7 +2465,7 @@ Py_DECREF(old_value); } Py_DECREF(owner); - #line 2344 "Python/generated_cases.c.h" + #line 2469 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2351,8 +2476,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t hint = read_u16(&next_instr[3].cache); - #line 1684 "Python/bytecodes.c" - assert(cframe.use_tracing == 0); + #line 1778 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2391,7 +2515,7 @@ /* PEP 509 */ dict->ma_version_tag = new_version; Py_DECREF(owner); - #line 2395 "Python/generated_cases.c.h" + #line 2519 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2402,8 +2526,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1726 "Python/bytecodes.c" - assert(cframe.use_tracing == 0); + #line 1819 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2413,7 +2536,7 @@ *(PyObject **)addr = value; Py_XDECREF(old_value); Py_DECREF(owner); - #line 2417 "Python/generated_cases.c.h" + #line 2540 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2425,11 +2548,10 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1746 "Python/bytecodes.c" + #line 1838 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_CompareOp(left, right, next_instr, oparg); DISPATCH_SAME_OPARG(); @@ -2439,12 +2561,12 @@ #endif /* ENABLE_SPECIALIZATION */ assert((oparg >> 4) <= Py_GE); res = PyObject_RichCompare(left, right, oparg>>4); - #line 2443 "Python/generated_cases.c.h" + #line 2565 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 1760 "Python/bytecodes.c" + #line 1851 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 2448 "Python/generated_cases.c.h" + #line 2570 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2455,8 +2577,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1764 "Python/bytecodes.c" - assert(cframe.use_tracing == 0); + #line 1855 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); @@ -2468,7 +2589,7 @@ _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); res = (sign_ish & oparg) ? Py_True : Py_False; Py_INCREF(res); - #line 2472 "Python/generated_cases.c.h" + #line 2593 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2479,8 +2600,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1780 "Python/bytecodes.c" - assert(cframe.use_tracing == 0); + #line 1870 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); @@ -2496,7 +2616,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); res = (sign_ish & oparg) ? Py_True : Py_False; Py_INCREF(res); - #line 2500 "Python/generated_cases.c.h" + #line 2620 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2507,8 +2627,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1800 "Python/bytecodes.c" - assert(cframe.use_tracing == 0); + #line 1889 "Python/bytecodes.c" DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); @@ -2521,7 +2640,7 @@ assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; Py_INCREF(res); - #line 2525 "Python/generated_cases.c.h" + #line 2644 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2532,14 +2651,14 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 1816 "Python/bytecodes.c" + #line 1904 "Python/bytecodes.c" int res = Py_Is(left, right) ^ oparg; - #line 2538 "Python/generated_cases.c.h" + #line 2657 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 1818 "Python/bytecodes.c" + #line 1906 "Python/bytecodes.c" b = Py_NewRef(res ? Py_True : Py_False); - #line 2543 "Python/generated_cases.c.h" + #line 2662 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = b; DISPATCH(); @@ -2549,15 +2668,15 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 1822 "Python/bytecodes.c" + #line 1910 "Python/bytecodes.c" int res = PySequence_Contains(right, left); - #line 2555 "Python/generated_cases.c.h" + #line 2674 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 1824 "Python/bytecodes.c" + #line 1912 "Python/bytecodes.c" if (res < 0) goto pop_2_error; b = Py_NewRef((res^oparg) ? Py_True : Py_False); - #line 2561 "Python/generated_cases.c.h" + #line 2680 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = b; DISPATCH(); @@ -2568,12 +2687,12 @@ PyObject *exc_value = stack_pointer[-2]; PyObject *rest; PyObject *match; - #line 1829 "Python/bytecodes.c" + #line 1917 "Python/bytecodes.c" if (check_except_star_type_valid(tstate, match_type) < 0) { - #line 2574 "Python/generated_cases.c.h" + #line 2693 "Python/generated_cases.c.h" Py_DECREF(exc_value); Py_DECREF(match_type); - #line 1831 "Python/bytecodes.c" + #line 1919 "Python/bytecodes.c" if (true) goto pop_2_error; } @@ -2581,10 +2700,10 @@ rest = NULL; int res = exception_group_match(exc_value, match_type, &match, &rest); - #line 2585 "Python/generated_cases.c.h" + #line 2704 "Python/generated_cases.c.h" Py_DECREF(exc_value); Py_DECREF(match_type); - #line 1839 "Python/bytecodes.c" + #line 1927 "Python/bytecodes.c" if (res < 0) goto pop_2_error; assert((match == NULL) == (rest == NULL)); @@ -2593,7 +2712,7 @@ if (!Py_IsNone(match)) { PyErr_SetHandledException(match); } - #line 2597 "Python/generated_cases.c.h" + #line 2716 "Python/generated_cases.c.h" stack_pointer[-1] = match; stack_pointer[-2] = rest; DISPATCH(); @@ -2603,21 +2722,21 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 1850 "Python/bytecodes.c" + #line 1938 "Python/bytecodes.c" assert(PyExceptionInstance_Check(left)); if (check_except_type_valid(tstate, right) < 0) { - #line 2610 "Python/generated_cases.c.h" + #line 2729 "Python/generated_cases.c.h" Py_DECREF(right); - #line 1853 "Python/bytecodes.c" + #line 1941 "Python/bytecodes.c" if (true) goto pop_1_error; } int res = PyErr_GivenExceptionMatches(left, right); - #line 2617 "Python/generated_cases.c.h" + #line 2736 "Python/generated_cases.c.h" Py_DECREF(right); - #line 1858 "Python/bytecodes.c" + #line 1946 "Python/bytecodes.c" b = Py_NewRef(res ? Py_True : Py_False); - #line 2621 "Python/generated_cases.c.h" + #line 2740 "Python/generated_cases.c.h" stack_pointer[-1] = b; DISPATCH(); } @@ -2626,15 +2745,15 @@ PyObject *fromlist = stack_pointer[-1]; PyObject *level = stack_pointer[-2]; PyObject *res; - #line 1862 "Python/bytecodes.c" + #line 1950 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_name(tstate, frame, name, fromlist, level); - #line 2633 "Python/generated_cases.c.h" + #line 2752 "Python/generated_cases.c.h" Py_DECREF(level); Py_DECREF(fromlist); - #line 1865 "Python/bytecodes.c" + #line 1953 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 2638 "Python/generated_cases.c.h" + #line 2757 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); @@ -2643,29 +2762,29 @@ TARGET(IMPORT_FROM) { PyObject *from = stack_pointer[-1]; PyObject *res; - #line 1869 "Python/bytecodes.c" + #line 1957 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_from(tstate, from, name); if (res == NULL) goto error; - #line 2651 "Python/generated_cases.c.h" + #line 2770 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); } TARGET(JUMP_FORWARD) { - #line 1875 "Python/bytecodes.c" + #line 1963 "Python/bytecodes.c" JUMPBY(oparg); - #line 2660 "Python/generated_cases.c.h" + #line 2779 "Python/generated_cases.c.h" DISPATCH(); } TARGET(JUMP_BACKWARD) { PREDICTED(JUMP_BACKWARD); - #line 1879 "Python/bytecodes.c" + #line 1967 "Python/bytecodes.c" assert(oparg < INSTR_OFFSET()); JUMPBY(-oparg); - #line 2669 "Python/generated_cases.c.h" + #line 2788 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -2673,7 +2792,7 @@ TARGET(POP_JUMP_IF_FALSE) { PREDICTED(POP_JUMP_IF_FALSE); PyObject *cond = stack_pointer[-1]; - #line 1885 "Python/bytecodes.c" + #line 1973 "Python/bytecodes.c" if (Py_IsTrue(cond)) { _Py_DECREF_NO_DEALLOC(cond); } @@ -2683,9 +2802,9 @@ } else { int err = PyObject_IsTrue(cond); - #line 2687 "Python/generated_cases.c.h" + #line 2806 "Python/generated_cases.c.h" Py_DECREF(cond); - #line 1895 "Python/bytecodes.c" + #line 1983 "Python/bytecodes.c" if (err == 0) { JUMPBY(oparg); } @@ -2693,14 +2812,14 @@ if (err < 0) goto pop_1_error; } } - #line 2697 "Python/generated_cases.c.h" + #line 2816 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_TRUE) { PyObject *cond = stack_pointer[-1]; - #line 1905 "Python/bytecodes.c" + #line 1993 "Python/bytecodes.c" if (Py_IsFalse(cond)) { _Py_DECREF_NO_DEALLOC(cond); } @@ -2710,9 +2829,9 @@ } else { int err = PyObject_IsTrue(cond); - #line 2714 "Python/generated_cases.c.h" + #line 2833 "Python/generated_cases.c.h" Py_DECREF(cond); - #line 1915 "Python/bytecodes.c" + #line 2003 "Python/bytecodes.c" if (err > 0) { JUMPBY(oparg); } @@ -2720,67 +2839,67 @@ if (err < 0) goto pop_1_error; } } - #line 2724 "Python/generated_cases.c.h" + #line 2843 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_NOT_NONE) { PyObject *value = stack_pointer[-1]; - #line 1925 "Python/bytecodes.c" + #line 2013 "Python/bytecodes.c" if (!Py_IsNone(value)) { - #line 2733 "Python/generated_cases.c.h" + #line 2852 "Python/generated_cases.c.h" Py_DECREF(value); - #line 1927 "Python/bytecodes.c" + #line 2015 "Python/bytecodes.c" JUMPBY(oparg); } else { _Py_DECREF_NO_DEALLOC(value); } - #line 2741 "Python/generated_cases.c.h" + #line 2860 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_NONE) { PyObject *value = stack_pointer[-1]; - #line 1935 "Python/bytecodes.c" + #line 2023 "Python/bytecodes.c" if (Py_IsNone(value)) { _Py_DECREF_NO_DEALLOC(value); JUMPBY(oparg); } else { - #line 2754 "Python/generated_cases.c.h" + #line 2873 "Python/generated_cases.c.h" Py_DECREF(value); - #line 1941 "Python/bytecodes.c" + #line 2029 "Python/bytecodes.c" } - #line 2758 "Python/generated_cases.c.h" + #line 2877 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(JUMP_BACKWARD_NO_INTERRUPT) { - #line 1945 "Python/bytecodes.c" + #line 2033 "Python/bytecodes.c" /* This bytecode is used in the `yield from` or `await` loop. * If there is an interrupt, we want it handled in the innermost * generator or coroutine, so we deliberately do not check it here. * (see bpo-30039). */ JUMPBY(-oparg); - #line 2771 "Python/generated_cases.c.h" + #line 2890 "Python/generated_cases.c.h" DISPATCH(); } TARGET(GET_LEN) { PyObject *obj = stack_pointer[-1]; PyObject *len_o; - #line 1954 "Python/bytecodes.c" + #line 2042 "Python/bytecodes.c" // PUSH(len(TOS)) Py_ssize_t len_i = PyObject_Length(obj); if (len_i < 0) goto error; len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) goto error; - #line 2784 "Python/generated_cases.c.h" + #line 2903 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = len_o; DISPATCH(); @@ -2791,16 +2910,16 @@ PyObject *type = stack_pointer[-2]; PyObject *subject = stack_pointer[-3]; PyObject *attrs; - #line 1962 "Python/bytecodes.c" + #line 2050 "Python/bytecodes.c" // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. assert(PyTuple_CheckExact(names)); attrs = match_class(tstate, subject, type, oparg, names); - #line 2800 "Python/generated_cases.c.h" + #line 2919 "Python/generated_cases.c.h" Py_DECREF(subject); Py_DECREF(type); Py_DECREF(names); - #line 1967 "Python/bytecodes.c" + #line 2055 "Python/bytecodes.c" if (attrs) { assert(PyTuple_CheckExact(attrs)); // Success! } @@ -2808,7 +2927,7 @@ if (_PyErr_Occurred(tstate)) goto pop_3_error; attrs = Py_NewRef(Py_None); // Failure! } - #line 2812 "Python/generated_cases.c.h" + #line 2931 "Python/generated_cases.c.h" STACK_SHRINK(2); stack_pointer[-1] = attrs; DISPATCH(); @@ -2817,10 +2936,10 @@ TARGET(MATCH_MAPPING) { PyObject *subject = stack_pointer[-1]; PyObject *res; - #line 1977 "Python/bytecodes.c" + #line 2065 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; res = Py_NewRef(match ? Py_True : Py_False); - #line 2824 "Python/generated_cases.c.h" + #line 2943 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; PREDICT(POP_JUMP_IF_FALSE); @@ -2830,10 +2949,10 @@ TARGET(MATCH_SEQUENCE) { PyObject *subject = stack_pointer[-1]; PyObject *res; - #line 1983 "Python/bytecodes.c" + #line 2071 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; res = Py_NewRef(match ? Py_True : Py_False); - #line 2837 "Python/generated_cases.c.h" + #line 2956 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; PREDICT(POP_JUMP_IF_FALSE); @@ -2844,11 +2963,11 @@ PyObject *keys = stack_pointer[-1]; PyObject *subject = stack_pointer[-2]; PyObject *values_or_none; - #line 1989 "Python/bytecodes.c" + #line 2077 "Python/bytecodes.c" // On successful match, PUSH(values). Otherwise, PUSH(None). values_or_none = match_keys(tstate, subject, keys); if (values_or_none == NULL) goto error; - #line 2852 "Python/generated_cases.c.h" + #line 2971 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = values_or_none; DISPATCH(); @@ -2857,14 +2976,14 @@ TARGET(GET_ITER) { PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 1995 "Python/bytecodes.c" + #line 2083 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ iter = PyObject_GetIter(iterable); - #line 2864 "Python/generated_cases.c.h" + #line 2983 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 1998 "Python/bytecodes.c" + #line 2086 "Python/bytecodes.c" if (iter == NULL) goto pop_1_error; - #line 2868 "Python/generated_cases.c.h" + #line 2987 "Python/generated_cases.c.h" stack_pointer[-1] = iter; DISPATCH(); } @@ -2872,7 +2991,7 @@ TARGET(GET_YIELD_FROM_ITER) { PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 2002 "Python/bytecodes.c" + #line 2090 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ if (PyCoro_CheckExact(iterable)) { /* `iterable` is a coroutine */ @@ -2895,11 +3014,11 @@ if (iter == NULL) { goto error; } - #line 2899 "Python/generated_cases.c.h" + #line 3018 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 2025 "Python/bytecodes.c" + #line 2113 "Python/bytecodes.c" } - #line 2903 "Python/generated_cases.c.h" + #line 3022 "Python/generated_cases.c.h" stack_pointer[-1] = iter; PREDICT(LOAD_CONST); DISPATCH(); @@ -2910,11 +3029,10 @@ static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2044 "Python/bytecodes.c" + #line 2132 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyForIterCache *cache = (_PyForIterCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_ForIter(iter, next_instr, oparg); DISPATCH_SAME_OPARG(); @@ -2929,13 +3047,12 @@ if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { goto error; } - else if (tstate->c_tracefunc != NULL) { - call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj, tstate, frame); - } + monitor_raise(tstate, frame, next_instr-1); _PyErr_Clear(tstate); } /* iterator ended normally */ - assert(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == END_FOR); + assert(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == END_FOR || + next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == INSTRUMENTED_END_FOR); Py_DECREF(iter); STACK_SHRINK(1); /* Jump forward oparg, then skip following END_FOR instruction */ @@ -2943,18 +3060,48 @@ DISPATCH(); } // Common case: no jump, leave it to the code generator - #line 2947 "Python/generated_cases.c.h" + #line 3064 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; DISPATCH(); } + TARGET(INSTRUMENTED_FOR_ITER) { + #line 2165 "Python/bytecodes.c" + _Py_CODEUNIT *here = next_instr-1; + _Py_CODEUNIT *target; + PyObject *iter = TOP(); + PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); + if (next != NULL) { + PUSH(next); + target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER; + } + else { + if (_PyErr_Occurred(tstate)) { + if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + goto error; + } + monitor_raise(tstate, frame, here); + _PyErr_Clear(tstate); + } + /* iterator ended normally */ + assert(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == END_FOR || + next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == INSTRUMENTED_END_FOR); + STACK_SHRINK(1); + Py_DECREF(iter); + /* Skip END_FOR */ + target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1; + } + INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); + #line 3098 "Python/generated_cases.c.h" + DISPATCH(); + } + TARGET(FOR_ITER_LIST) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2079 "Python/bytecodes.c" - assert(cframe.use_tracing == 0); + #line 2193 "Python/bytecodes.c" DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); _PyListIterObject *it = (_PyListIterObject *)iter; STAT_INC(FOR_ITER, hit); @@ -2974,7 +3121,7 @@ DISPATCH(); end_for_iter_list: // Common case: no jump, leave it to the code generator - #line 2978 "Python/generated_cases.c.h" + #line 3125 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -2984,8 +3131,7 @@ TARGET(FOR_ITER_TUPLE) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2102 "Python/bytecodes.c" - assert(cframe.use_tracing == 0); + #line 2215 "Python/bytecodes.c" _PyTupleIterObject *it = (_PyTupleIterObject *)iter; DEOPT_IF(Py_TYPE(it) != &PyTupleIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -3005,7 +3151,7 @@ DISPATCH(); end_for_iter_tuple: // Common case: no jump, leave it to the code generator - #line 3009 "Python/generated_cases.c.h" + #line 3155 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3015,8 +3161,7 @@ TARGET(FOR_ITER_RANGE) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2125 "Python/bytecodes.c" - assert(cframe.use_tracing == 0); + #line 2237 "Python/bytecodes.c" _PyRangeIterObject *r = (_PyRangeIterObject *)iter; DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -3034,7 +3179,7 @@ if (next == NULL) { goto error; } - #line 3038 "Python/generated_cases.c.h" + #line 3183 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3043,8 +3188,7 @@ TARGET(FOR_ITER_GEN) { PyObject *iter = stack_pointer[-1]; - #line 2146 "Python/bytecodes.c" - assert(cframe.use_tracing == 0); + #line 2257 "Python/bytecodes.c" PyGenObject *gen = (PyGenObject *)iter; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); @@ -3056,16 +3200,17 @@ gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER + oparg); - assert(next_instr->op.code == END_FOR); + assert(next_instr->op.code == END_FOR || + next_instr->op.code == INSTRUMENTED_END_FOR); DISPATCH_INLINED(gen_frame); - #line 3062 "Python/generated_cases.c.h" + #line 3207 "Python/generated_cases.c.h" } TARGET(BEFORE_ASYNC_WITH) { PyObject *mgr = stack_pointer[-1]; PyObject *exit; PyObject *res; - #line 2163 "Python/bytecodes.c" + #line 2274 "Python/bytecodes.c" PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); if (enter == NULL) { if (!_PyErr_Occurred(tstate)) { @@ -3088,16 +3233,16 @@ Py_DECREF(enter); goto error; } - #line 3092 "Python/generated_cases.c.h" + #line 3237 "Python/generated_cases.c.h" Py_DECREF(mgr); - #line 2186 "Python/bytecodes.c" + #line 2297 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); if (true) goto pop_1_error; } - #line 3101 "Python/generated_cases.c.h" + #line 3246 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -3109,7 +3254,7 @@ PyObject *mgr = stack_pointer[-1]; PyObject *exit; PyObject *res; - #line 2196 "Python/bytecodes.c" + #line 2307 "Python/bytecodes.c" /* pop the context manager, push its __exit__ and the * value returned from calling its __enter__ */ @@ -3135,16 +3280,16 @@ Py_DECREF(enter); goto error; } - #line 3139 "Python/generated_cases.c.h" + #line 3284 "Python/generated_cases.c.h" Py_DECREF(mgr); - #line 2222 "Python/bytecodes.c" + #line 2333 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); if (true) goto pop_1_error; } - #line 3148 "Python/generated_cases.c.h" + #line 3293 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -3156,7 +3301,7 @@ PyObject *lasti = stack_pointer[-3]; PyObject *exit_func = stack_pointer[-4]; PyObject *res; - #line 2231 "Python/bytecodes.c" + #line 2342 "Python/bytecodes.c" /* At the top of the stack are 4 values: - val: TOP = exc_info() - unused: SECOND = previous exception @@ -3177,7 +3322,7 @@ res = PyObject_Vectorcall(exit_func, stack + 1, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); if (res == NULL) goto error; - #line 3181 "Python/generated_cases.c.h" + #line 3326 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); @@ -3186,7 +3331,7 @@ TARGET(PUSH_EXC_INFO) { PyObject *new_exc = stack_pointer[-1]; PyObject *prev_exc; - #line 2254 "Python/bytecodes.c" + #line 2365 "Python/bytecodes.c" _PyErr_StackItem *exc_info = tstate->exc_info; if (exc_info->exc_value != NULL) { prev_exc = exc_info->exc_value; @@ -3196,7 +3341,7 @@ } assert(PyExceptionInstance_Check(new_exc)); exc_info->exc_value = Py_NewRef(new_exc); - #line 3200 "Python/generated_cases.c.h" + #line 3345 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = new_exc; stack_pointer[-2] = prev_exc; @@ -3210,9 +3355,8 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t keys_version = read_u32(&next_instr[3].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2266 "Python/bytecodes.c" + #line 2377 "Python/bytecodes.c" /* Cached method object */ - assert(cframe.use_tracing == 0); PyTypeObject *self_cls = Py_TYPE(self); assert(type_version != 0); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); @@ -3228,7 +3372,7 @@ assert(_PyType_HasFeature(Py_TYPE(res2), Py_TPFLAGS_METHOD_DESCRIPTOR)); res = self; assert(oparg & 1); - #line 3232 "Python/generated_cases.c.h" + #line 3376 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3242,8 +3386,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2286 "Python/bytecodes.c" - assert(cframe.use_tracing == 0); + #line 2396 "Python/bytecodes.c" PyTypeObject *self_cls = Py_TYPE(self); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); assert(self_cls->tp_dictoffset == 0); @@ -3253,7 +3396,7 @@ res2 = Py_NewRef(descr); res = self; assert(oparg & 1); - #line 3257 "Python/generated_cases.c.h" + #line 3400 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3267,8 +3410,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2299 "Python/bytecodes.c" - assert(cframe.use_tracing == 0); + #line 2408 "Python/bytecodes.c" PyTypeObject *self_cls = Py_TYPE(self); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); Py_ssize_t dictoffset = self_cls->tp_dictoffset; @@ -3282,7 +3424,7 @@ res2 = Py_NewRef(descr); res = self; assert(oparg & 1); - #line 3286 "Python/generated_cases.c.h" + #line 3428 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3291,14 +3433,31 @@ } TARGET(KW_NAMES) { - #line 2316 "Python/bytecodes.c" + #line 2424 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg < PyTuple_GET_SIZE(frame->f_code->co_consts)); kwnames = GETITEM(frame->f_code->co_consts, oparg); - #line 3299 "Python/generated_cases.c.h" + #line 3441 "Python/generated_cases.c.h" DISPATCH(); } + TARGET(INSTRUMENTED_CALL) { + #line 2430 "Python/bytecodes.c" + int is_meth = PEEK(oparg+2) != NULL; + int total_args = oparg + is_meth; + PyObject *function = PEEK(total_args + 1); + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PEEK(total_args); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, next_instr-1, function, arg); + if (err) goto error; + _PyCallCache *cache = (_PyCallCache *)next_instr; + INCREMENT_ADAPTIVE_COUNTER(cache->counter); + GO_TO_INSTRUCTION(CALL); + #line 3459 "Python/generated_cases.c.h" + } + TARGET(CALL) { PREDICTED(CALL); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -3306,7 +3465,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2352 "Python/bytecodes.c" + #line 2475 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3317,7 +3476,6 @@ #if ENABLE_SPECIALIZATION _PyCallCache *cache = (_PyCallCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_Call(callable, next_instr, total_args, kwnames); DISPATCH_SAME_OPARG(); @@ -3360,16 +3518,26 @@ DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ - if (cframe.use_tracing) { - res = trace_call_function( - tstate, callable, args, - positional_args, kwnames); - } - else { - res = PyObject_Vectorcall( - callable, args, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames); + res = PyObject_Vectorcall( + callable, args, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames); + if (opcode == INSTRUMENTED_CALL) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PEEK(total_args); + if (res == NULL) { + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, next_instr-1, callable, arg); + } + else { + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, next_instr-1, callable, arg); + if (err < 0) { + Py_CLEAR(res); + } + } } kwnames = NULL; assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -3378,7 +3546,7 @@ Py_DECREF(args[i]); } if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3382 "Python/generated_cases.c.h" + #line 3550 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3390,7 +3558,7 @@ TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; - #line 2430 "Python/bytecodes.c" + #line 2562 "Python/bytecodes.c" DEOPT_IF(method != NULL, CALL); DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); STAT_INC(CALL, hit); @@ -3400,7 +3568,7 @@ PEEK(oparg + 2) = Py_NewRef(meth); // method Py_DECREF(callable); GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); - #line 3404 "Python/generated_cases.c.h" + #line 3572 "Python/generated_cases.c.h" } TARGET(CALL_PY_EXACT_ARGS) { @@ -3409,7 +3577,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); - #line 2442 "Python/bytecodes.c" + #line 2574 "Python/bytecodes.c" assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; @@ -3434,7 +3602,7 @@ STACK_SHRINK(oparg + 2); JUMPBY(INLINE_CACHE_ENTRIES_CALL); DISPATCH_INLINED(new_frame); - #line 3438 "Python/generated_cases.c.h" + #line 3606 "Python/generated_cases.c.h" } TARGET(CALL_PY_WITH_DEFAULTS) { @@ -3442,7 +3610,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); - #line 2469 "Python/bytecodes.c" + #line 2601 "Python/bytecodes.c" assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; @@ -3477,7 +3645,7 @@ STACK_SHRINK(oparg + 2); JUMPBY(INLINE_CACHE_ENTRIES_CALL); DISPATCH_INLINED(new_frame); - #line 3481 "Python/generated_cases.c.h" + #line 3649 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_TYPE_1) { @@ -3485,9 +3653,8 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2506 "Python/bytecodes.c" + #line 2638 "Python/bytecodes.c" assert(kwnames == NULL); - assert(cframe.use_tracing == 0); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); PyObject *obj = args[0]; @@ -3496,7 +3663,7 @@ res = Py_NewRef(Py_TYPE(obj)); Py_DECREF(obj); Py_DECREF(&PyType_Type); // I.e., callable - #line 3500 "Python/generated_cases.c.h" + #line 3667 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3509,9 +3676,8 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2519 "Python/bytecodes.c" + #line 2650 "Python/bytecodes.c" assert(kwnames == NULL); - assert(cframe.use_tracing == 0); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL); @@ -3521,7 +3687,7 @@ Py_DECREF(arg); Py_DECREF(&PyUnicode_Type); // I.e., callable if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3525 "Python/generated_cases.c.h" + #line 3691 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3535,7 +3701,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2534 "Python/bytecodes.c" + #line 2664 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3546,7 +3712,7 @@ Py_DECREF(arg); Py_DECREF(&PyTuple_Type); // I.e., tuple if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3550 "Python/generated_cases.c.h" + #line 3716 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3560,7 +3726,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2548 "Python/bytecodes.c" + #line 2678 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3582,7 +3748,7 @@ } Py_DECREF(tp); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3586 "Python/generated_cases.c.h" + #line 3752 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3596,8 +3762,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2573 "Python/bytecodes.c" - assert(cframe.use_tracing == 0); + #line 2703 "Python/bytecodes.c" /* Builtin METH_O functions */ assert(kwnames == NULL); int is_meth = method != NULL; @@ -3625,7 +3790,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3629 "Python/generated_cases.c.h" + #line 3794 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3639,8 +3804,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2605 "Python/bytecodes.c" - assert(cframe.use_tracing == 0); + #line 2734 "Python/bytecodes.c" /* Builtin METH_FASTCALL functions, without keywords */ assert(kwnames == NULL); int is_meth = method != NULL; @@ -3672,7 +3836,7 @@ 'invalid'). In those cases an exception is set, so we must handle it. */ - #line 3676 "Python/generated_cases.c.h" + #line 3840 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3686,8 +3850,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2641 "Python/bytecodes.c" - assert(cframe.use_tracing == 0); + #line 2769 "Python/bytecodes.c" /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ int is_meth = method != NULL; int total_args = oparg; @@ -3719,7 +3882,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3723 "Python/generated_cases.c.h" + #line 3886 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3733,8 +3896,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2677 "Python/bytecodes.c" - assert(cframe.use_tracing == 0); + #line 2804 "Python/bytecodes.c" assert(kwnames == NULL); /* len(o) */ int is_meth = method != NULL; @@ -3759,7 +3921,7 @@ Py_DECREF(callable); Py_DECREF(arg); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3763 "Python/generated_cases.c.h" + #line 3925 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3772,8 +3934,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2705 "Python/bytecodes.c" - assert(cframe.use_tracing == 0); + #line 2831 "Python/bytecodes.c" assert(kwnames == NULL); /* isinstance(o, o2) */ int is_meth = method != NULL; @@ -3800,7 +3961,7 @@ Py_DECREF(cls); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3804 "Python/generated_cases.c.h" + #line 3965 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3812,8 +3973,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *self = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; - #line 2736 "Python/bytecodes.c" - assert(cframe.use_tracing == 0); + #line 2861 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); assert(method != NULL); @@ -3831,14 +3991,14 @@ JUMPBY(INLINE_CACHE_ENTRIES_CALL + 1); assert(next_instr[-1].op.code == POP_TOP); DISPATCH(); - #line 3835 "Python/generated_cases.c.h" + #line 3995 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) { PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2757 "Python/bytecodes.c" + #line 2881 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -3869,7 +4029,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3873 "Python/generated_cases.c.h" + #line 4033 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3882,7 +4042,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2791 "Python/bytecodes.c" + #line 2915 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3911,7 +4071,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3915 "Python/generated_cases.c.h" + #line 4075 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3924,7 +4084,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2823 "Python/bytecodes.c" + #line 2947 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 0 || oparg == 1); int is_meth = method != NULL; @@ -3953,7 +4113,7 @@ Py_DECREF(self); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3957 "Python/generated_cases.c.h" + #line 4117 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3966,7 +4126,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2855 "Python/bytecodes.c" + #line 2979 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -3994,7 +4154,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3998 "Python/generated_cases.c.h" + #line 4158 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4003,18 +4163,22 @@ DISPATCH(); } + TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { + #line 3010 "Python/bytecodes.c" + GO_TO_INSTRUCTION(CALL_FUNCTION_EX); + #line 4170 "Python/generated_cases.c.h" + } + TARGET(CALL_FUNCTION_EX) { PREDICTED(CALL_FUNCTION_EX); PyObject *kwargs = (oparg & 1) ? stack_pointer[-(((oparg & 1) ? 1 : 0))] : NULL; PyObject *callargs = stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))]; PyObject *func = stack_pointer[-(2 + ((oparg & 1) ? 1 : 0))]; PyObject *result; - #line 2886 "Python/bytecodes.c" - if (oparg & 1) { - // DICT_MERGE is called before this opcode if there are kwargs. - // It converts all dict subtypes in kwargs into regular dicts. - assert(PyDict_CheckExact(kwargs)); - } + #line 3014 "Python/bytecodes.c" + // DICT_MERGE is called before this opcode if there are kwargs. + // It converts all dict subtypes in kwargs into regular dicts. + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); if (!PyTuple_CheckExact(callargs)) { if (check_args_iterable(tstate, func, callargs) < 0) { goto error; @@ -4026,17 +4190,42 @@ Py_SETREF(callargs, tuple); } assert(PyTuple_CheckExact(callargs)); - - result = do_call_core(tstate, func, callargs, kwargs, cframe.use_tracing); - #line 4032 "Python/generated_cases.c.h" + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); + if (opcode == INSTRUMENTED_CALL_FUNCTION_EX && + !PyFunction_Check(func) && !PyMethod_Check(func) + ) { + PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? + PyTuple_GET_ITEM(callargs, 0) : Py_None; + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, next_instr-1, func, arg); + if (err) goto error; + result = PyObject_Call(func, callargs, kwargs); + if (result == NULL) { + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, next_instr-1, func, arg); + } + else { + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, next_instr-1, func, arg); + if (err < 0) { + Py_CLEAR(result); + } + } + } + else { + result = PyObject_Call(func, callargs, kwargs); + } + #line 4222 "Python/generated_cases.c.h" Py_DECREF(func); Py_DECREF(callargs); Py_XDECREF(kwargs); - #line 2905 "Python/bytecodes.c" - + #line 3057 "Python/bytecodes.c" assert(PEEK(3 + (oparg & 1)) == NULL); if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } - #line 4040 "Python/generated_cases.c.h" + #line 4229 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 1) ? 1 : 0)); STACK_SHRINK(2); stack_pointer[-1] = result; @@ -4051,7 +4240,7 @@ PyObject *kwdefaults = (oparg & 0x02) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0))] : NULL; PyObject *defaults = (oparg & 0x01) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x01) ? 1 : 0))] : NULL; PyObject *func; - #line 2916 "Python/bytecodes.c" + #line 3067 "Python/bytecodes.c" PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); @@ -4080,14 +4269,14 @@ func_obj->func_version = ((PyCodeObject *)codeobj)->co_version; func = (PyObject *)func_obj; - #line 4084 "Python/generated_cases.c.h" + #line 4273 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 0x01) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x08) ? 1 : 0)); stack_pointer[-1] = func; DISPATCH(); } TARGET(RETURN_GENERATOR) { - #line 2947 "Python/bytecodes.c" + #line 3098 "Python/bytecodes.c" assert(PyFunction_Check(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); @@ -4108,7 +4297,7 @@ frame = cframe.current_frame = prev; _PyFrame_StackPush(frame, (PyObject *)gen); goto resume_frame; - #line 4112 "Python/generated_cases.c.h" + #line 4301 "Python/generated_cases.c.h" } TARGET(BUILD_SLICE) { @@ -4116,15 +4305,15 @@ PyObject *stop = stack_pointer[-(1 + ((oparg == 3) ? 1 : 0))]; PyObject *start = stack_pointer[-(2 + ((oparg == 3) ? 1 : 0))]; PyObject *slice; - #line 2970 "Python/bytecodes.c" + #line 3121 "Python/bytecodes.c" slice = PySlice_New(start, stop, step); - #line 4122 "Python/generated_cases.c.h" + #line 4311 "Python/generated_cases.c.h" Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); - #line 2972 "Python/bytecodes.c" + #line 3123 "Python/bytecodes.c" if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } - #line 4128 "Python/generated_cases.c.h" + #line 4317 "Python/generated_cases.c.h" STACK_SHRINK(((oparg == 3) ? 1 : 0)); STACK_SHRINK(1); stack_pointer[-1] = slice; @@ -4135,7 +4324,7 @@ PyObject *fmt_spec = ((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? stack_pointer[-((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))] : NULL; PyObject *value = stack_pointer[-(1 + (((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))]; PyObject *result; - #line 2976 "Python/bytecodes.c" + #line 3127 "Python/bytecodes.c" /* Handles f-string value formatting. */ PyObject *(*conv_fn)(PyObject *); int which_conversion = oparg & FVC_MASK; @@ -4170,7 +4359,7 @@ Py_DECREF(value); Py_XDECREF(fmt_spec); if (result == NULL) { STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); goto pop_1_error; } - #line 4174 "Python/generated_cases.c.h" + #line 4363 "Python/generated_cases.c.h" STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); stack_pointer[-1] = result; DISPATCH(); @@ -4179,10 +4368,10 @@ TARGET(COPY) { PyObject *bottom = stack_pointer[-(1 + (oparg-1))]; PyObject *top; - #line 3013 "Python/bytecodes.c" + #line 3164 "Python/bytecodes.c" assert(oparg > 0); top = Py_NewRef(bottom); - #line 4186 "Python/generated_cases.c.h" + #line 4375 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = top; DISPATCH(); @@ -4194,11 +4383,10 @@ PyObject *rhs = stack_pointer[-1]; PyObject *lhs = stack_pointer[-2]; PyObject *res; - #line 3018 "Python/bytecodes.c" + #line 3169 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, &GETLOCAL(0)); DISPATCH_SAME_OPARG(); @@ -4210,12 +4398,12 @@ assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); assert(binary_ops[oparg]); res = binary_ops[oparg](lhs, rhs); - #line 4214 "Python/generated_cases.c.h" + #line 4402 "Python/generated_cases.c.h" Py_DECREF(lhs); Py_DECREF(rhs); - #line 3034 "Python/bytecodes.c" + #line 3184 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 4219 "Python/generated_cases.c.h" + #line 4407 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -4225,27 +4413,153 @@ TARGET(SWAP) { PyObject *top = stack_pointer[-1]; PyObject *bottom = stack_pointer[-(2 + (oparg-2))]; - #line 3039 "Python/bytecodes.c" + #line 3189 "Python/bytecodes.c" assert(oparg >= 2); - #line 4231 "Python/generated_cases.c.h" + #line 4419 "Python/generated_cases.c.h" stack_pointer[-1] = bottom; stack_pointer[-(2 + (oparg-2))] = top; DISPATCH(); } + TARGET(INSTRUMENTED_LINE) { + #line 3193 "Python/bytecodes.c" + _Py_CODEUNIT *here = next_instr-1; + _PyFrame_SetStackPointer(frame, stack_pointer); + int original_opcode = _Py_call_instrumentation_line( + tstate, frame, here); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (original_opcode < 0) { + next_instr = here+1; + goto error; + } + next_instr = frame->prev_instr; + if (next_instr != here) { + DISPATCH(); + } + if (_PyOpcode_Caches[original_opcode]) { + _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); + INCREMENT_ADAPTIVE_COUNTER(cache->counter); + } + opcode = original_opcode; + DISPATCH_GOTO(); + #line 4446 "Python/generated_cases.c.h" + } + + TARGET(INSTRUMENTED_INSTRUCTION) { + #line 3215 "Python/bytecodes.c" + int next_opcode = _Py_call_instrumentation_instruction( + tstate, frame, next_instr-1); + if (next_opcode < 0) goto error; + next_instr--; + if (_PyOpcode_Caches[next_opcode]) { + _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); + INCREMENT_ADAPTIVE_COUNTER(cache->counter); + } + assert(next_opcode > 0 && next_opcode < 256); + opcode = next_opcode; + DISPATCH_GOTO(); + #line 4462 "Python/generated_cases.c.h" + } + + TARGET(INSTRUMENTED_JUMP_FORWARD) { + #line 3229 "Python/bytecodes.c" + INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); + #line 4468 "Python/generated_cases.c.h" + DISPATCH(); + } + + TARGET(INSTRUMENTED_JUMP_BACKWARD) { + #line 3233 "Python/bytecodes.c" + INSTRUMENTED_JUMP(next_instr-1, next_instr-oparg, PY_MONITORING_EVENT_JUMP); + #line 4475 "Python/generated_cases.c.h" + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { + #line 3238 "Python/bytecodes.c" + PyObject *cond = POP(); + int err = PyObject_IsTrue(cond); + Py_DECREF(cond); + if (err < 0) goto error; + _Py_CODEUNIT *here = next_instr-1; + assert(err == 0 || err == 1); + int offset = err*oparg; + INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + #line 4490 "Python/generated_cases.c.h" + DISPATCH(); + } + + TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { + #line 3249 "Python/bytecodes.c" + PyObject *cond = POP(); + int err = PyObject_IsTrue(cond); + Py_DECREF(cond); + if (err < 0) goto error; + _Py_CODEUNIT *here = next_instr-1; + assert(err == 0 || err == 1); + int offset = (1-err)*oparg; + INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + #line 4504 "Python/generated_cases.c.h" + DISPATCH(); + } + + TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { + #line 3260 "Python/bytecodes.c" + PyObject *value = POP(); + _Py_CODEUNIT *here = next_instr-1; + int offset; + if (Py_IsNone(value)) { + _Py_DECREF_NO_DEALLOC(value); + offset = oparg; + } + else { + Py_DECREF(value); + offset = 0; + } + INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + #line 4522 "Python/generated_cases.c.h" + DISPATCH(); + } + + TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { + #line 3275 "Python/bytecodes.c" + PyObject *value = POP(); + _Py_CODEUNIT *here = next_instr-1; + int offset; + if (Py_IsNone(value)) { + _Py_DECREF_NO_DEALLOC(value); + offset = 0; + } + else { + Py_DECREF(value); + offset = oparg; + } + INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + #line 4540 "Python/generated_cases.c.h" + DISPATCH(); + } + TARGET(EXTENDED_ARG) { - #line 3043 "Python/bytecodes.c" + #line 3290 "Python/bytecodes.c" assert(oparg); - assert(cframe.use_tracing == 0); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); - #line 4245 "Python/generated_cases.c.h" + #line 4551 "Python/generated_cases.c.h" } TARGET(CACHE) { - #line 3052 "Python/bytecodes.c" + #line 3298 "Python/bytecodes.c" + assert(0 && "Executing a cache."); + Py_UNREACHABLE(); + #line 4558 "Python/generated_cases.c.h" + } + + TARGET(RESERVED) { + #line 3303 "Python/bytecodes.c" + assert(0 && "Executing RESERVED instruction."); Py_UNREACHABLE(); - #line 4251 "Python/generated_cases.c.h" + #line 4565 "Python/generated_cases.c.h" } diff --git a/Python/instrumentation.c b/Python/instrumentation.c new file mode 100644 index 00000000000000..39a7eaa3bd2d42 --- /dev/null +++ b/Python/instrumentation.c @@ -0,0 +1,2021 @@ + + +#include "Python.h" +#include "pycore_call.h" +#include "pycore_frame.h" +#include "pycore_interp.h" +#include "pycore_long.h" +#include "pycore_namespace.h" +#include "pycore_object.h" +#include "pycore_opcode.h" +#include "pycore_pyerrors.h" +#include "pycore_pystate.h" + +/* Uncomment this to dump debugging output when assertions fail */ +// #define INSTRUMENT_DEBUG 1 + +static PyObject DISABLE = +{ + _PyObject_IMMORTAL_REFCNT, + &PyBaseObject_Type +}; + +PyObject _PyInstrumentation_MISSING = +{ + _PyObject_IMMORTAL_REFCNT, + &PyBaseObject_Type +}; + +static const int8_t EVENT_FOR_OPCODE[256] = { + [RETURN_CONST] = PY_MONITORING_EVENT_PY_RETURN, + [INSTRUMENTED_RETURN_CONST] = PY_MONITORING_EVENT_PY_RETURN, + [RETURN_VALUE] = PY_MONITORING_EVENT_PY_RETURN, + [INSTRUMENTED_RETURN_VALUE] = PY_MONITORING_EVENT_PY_RETURN, + [CALL] = PY_MONITORING_EVENT_CALL, + [INSTRUMENTED_CALL] = PY_MONITORING_EVENT_CALL, + [CALL_FUNCTION_EX] = PY_MONITORING_EVENT_CALL, + [INSTRUMENTED_CALL_FUNCTION_EX] = PY_MONITORING_EVENT_CALL, + [RESUME] = -1, + [YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD, + [INSTRUMENTED_YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD, + [JUMP_FORWARD] = PY_MONITORING_EVENT_JUMP, + [JUMP_BACKWARD] = PY_MONITORING_EVENT_JUMP, + [POP_JUMP_IF_FALSE] = PY_MONITORING_EVENT_BRANCH, + [POP_JUMP_IF_TRUE] = PY_MONITORING_EVENT_BRANCH, + [POP_JUMP_IF_NONE] = PY_MONITORING_EVENT_BRANCH, + [POP_JUMP_IF_NOT_NONE] = PY_MONITORING_EVENT_BRANCH, + [INSTRUMENTED_JUMP_FORWARD] = PY_MONITORING_EVENT_JUMP, + [INSTRUMENTED_JUMP_BACKWARD] = PY_MONITORING_EVENT_JUMP, + [INSTRUMENTED_POP_JUMP_IF_FALSE] = PY_MONITORING_EVENT_BRANCH, + [INSTRUMENTED_POP_JUMP_IF_TRUE] = PY_MONITORING_EVENT_BRANCH, + [INSTRUMENTED_POP_JUMP_IF_NONE] = PY_MONITORING_EVENT_BRANCH, + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = PY_MONITORING_EVENT_BRANCH, + [FOR_ITER] = PY_MONITORING_EVENT_BRANCH, + [INSTRUMENTED_FOR_ITER] = PY_MONITORING_EVENT_BRANCH, + [END_FOR] = PY_MONITORING_EVENT_STOP_ITERATION, + [INSTRUMENTED_END_FOR] = PY_MONITORING_EVENT_STOP_ITERATION, + [END_SEND] = PY_MONITORING_EVENT_STOP_ITERATION, + [INSTRUMENTED_END_SEND] = PY_MONITORING_EVENT_STOP_ITERATION, +}; + +static const uint8_t DE_INSTRUMENT[256] = { + [INSTRUMENTED_RESUME] = RESUME, + [INSTRUMENTED_RETURN_VALUE] = RETURN_VALUE, + [INSTRUMENTED_RETURN_CONST] = RETURN_CONST, + [INSTRUMENTED_CALL] = CALL, + [INSTRUMENTED_CALL_FUNCTION_EX] = CALL_FUNCTION_EX, + [INSTRUMENTED_YIELD_VALUE] = YIELD_VALUE, + [INSTRUMENTED_JUMP_FORWARD] = JUMP_FORWARD, + [INSTRUMENTED_JUMP_BACKWARD] = JUMP_BACKWARD, + [INSTRUMENTED_POP_JUMP_IF_FALSE] = POP_JUMP_IF_FALSE, + [INSTRUMENTED_POP_JUMP_IF_TRUE] = POP_JUMP_IF_TRUE, + [INSTRUMENTED_POP_JUMP_IF_NONE] = POP_JUMP_IF_NONE, + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = POP_JUMP_IF_NOT_NONE, + [INSTRUMENTED_FOR_ITER] = FOR_ITER, + [INSTRUMENTED_END_FOR] = END_FOR, + [INSTRUMENTED_END_SEND] = END_SEND, +}; + +static const uint8_t INSTRUMENTED_OPCODES[256] = { + [RETURN_CONST] = INSTRUMENTED_RETURN_CONST, + [INSTRUMENTED_RETURN_CONST] = INSTRUMENTED_RETURN_CONST, + [RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE, + [INSTRUMENTED_RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE, + [CALL] = INSTRUMENTED_CALL, + [INSTRUMENTED_CALL] = INSTRUMENTED_CALL, + [CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, + [INSTRUMENTED_CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, + [YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE, + [INSTRUMENTED_YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE, + [RESUME] = INSTRUMENTED_RESUME, + [INSTRUMENTED_RESUME] = INSTRUMENTED_RESUME, + [JUMP_FORWARD] = INSTRUMENTED_JUMP_FORWARD, + [INSTRUMENTED_JUMP_FORWARD] = INSTRUMENTED_JUMP_FORWARD, + [JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD, + [INSTRUMENTED_JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD, + [POP_JUMP_IF_FALSE] = INSTRUMENTED_POP_JUMP_IF_FALSE, + [INSTRUMENTED_POP_JUMP_IF_FALSE] = INSTRUMENTED_POP_JUMP_IF_FALSE, + [POP_JUMP_IF_TRUE] = INSTRUMENTED_POP_JUMP_IF_TRUE, + [INSTRUMENTED_POP_JUMP_IF_TRUE] = INSTRUMENTED_POP_JUMP_IF_TRUE, + [POP_JUMP_IF_NONE] = INSTRUMENTED_POP_JUMP_IF_NONE, + [INSTRUMENTED_POP_JUMP_IF_NONE] = INSTRUMENTED_POP_JUMP_IF_NONE, + [POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE, + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE, + [END_FOR] = INSTRUMENTED_END_FOR, + [INSTRUMENTED_END_FOR] = INSTRUMENTED_END_FOR, + [END_SEND] = INSTRUMENTED_END_SEND, + [INSTRUMENTED_END_SEND] = INSTRUMENTED_END_SEND, + + [INSTRUMENTED_LINE] = INSTRUMENTED_LINE, + [INSTRUMENTED_INSTRUCTION] = INSTRUMENTED_INSTRUCTION, +}; + +static inline bool +opcode_has_event(int opcode) { + return opcode < INSTRUMENTED_LINE && + INSTRUMENTED_OPCODES[opcode] > 0; +} + +static inline bool +is_instrumented(int opcode) { + assert(opcode != 0); + assert(opcode != RESERVED); + return opcode >= MIN_INSTRUMENTED_OPCODE; +} + +static inline bool +monitors_equals(_Py_Monitors a, _Py_Monitors b) +{ + for (int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { + if (a.tools[i] != b.tools[i]) { + return false; + } + } + return true; +} + +static inline _Py_Monitors +monitors_sub(_Py_Monitors a, _Py_Monitors b) +{ + _Py_Monitors res; + for (int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { + res.tools[i] = a.tools[i] & ~b.tools[i]; + } + return res; +} + +static inline _Py_Monitors +monitors_and(_Py_Monitors a, _Py_Monitors b) +{ + _Py_Monitors res; + for (int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { + res.tools[i] = a.tools[i] & b.tools[i]; + } + return res; +} + +static inline _Py_Monitors +monitors_or(_Py_Monitors a, _Py_Monitors b) +{ + _Py_Monitors res; + for (int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { + res.tools[i] = a.tools[i] | b.tools[i]; + } + return res; +} + +static inline bool +monitors_are_empty(_Py_Monitors m) +{ + for (int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { + if (m.tools[i]) { + return false; + } + } + return true; +} + +static inline bool +multiple_tools(_Py_Monitors *m) +{ + for (int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { + if (_Py_popcount32(m->tools[i]) > 1) { + return true; + } + } + return false; +} + +static inline _PyMonitoringEventSet +get_events(_Py_Monitors *m, int tool_id) +{ + _PyMonitoringEventSet result = 0; + for (int e = 0; e < PY_MONITORING_UNGROUPED_EVENTS; e++) { + if ((m->tools[e] >> tool_id) & 1) { + result |= (1 << e); + } + } + return result; +} + +/* Line delta. + * 8 bit value. + * if line_delta == -128: + * line = None # represented as -1 + * elif line_delta == -127: + * line = PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT)); + * else: + * line = first_line + (offset >> OFFSET_SHIFT) + line_delta; + */ + +#define NO_LINE -128 +#define COMPUTED_LINE -127 + +#define OFFSET_SHIFT 4 + +static int8_t +compute_line_delta(PyCodeObject *code, int offset, int line) +{ + if (line < 0) { + return NO_LINE; + } + int delta = line - code->co_firstlineno - (offset >> OFFSET_SHIFT); + if (delta <= INT8_MAX && delta > COMPUTED_LINE) { + return delta; + } + return COMPUTED_LINE; +} + +static int +compute_line(PyCodeObject *code, int offset, int8_t line_delta) +{ + if (line_delta > COMPUTED_LINE) { + return code->co_firstlineno + (offset >> OFFSET_SHIFT) + line_delta; + } + if (line_delta == NO_LINE) { + + return -1; + } + assert(line_delta == COMPUTED_LINE); + /* Look it up */ + return PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT)); +} + +static int +instruction_length(PyCodeObject *code, int offset) +{ + int opcode = _PyCode_CODE(code)[offset].op.code; + assert(opcode != 0); + assert(opcode != RESERVED); + if (opcode == INSTRUMENTED_LINE) { + opcode = code->_co_monitoring->lines[offset].original_opcode; + } + if (opcode == INSTRUMENTED_INSTRUCTION) { + opcode = code->_co_monitoring->per_instruction_opcodes[offset]; + } + int deinstrumented = DE_INSTRUMENT[opcode]; + if (deinstrumented) { + opcode = deinstrumented; + } + else { + opcode = _PyOpcode_Deopt[opcode]; + } + assert(opcode != 0); + assert(!is_instrumented(opcode)); + assert(opcode == _PyOpcode_Deopt[opcode]); + return 1 + _PyOpcode_Caches[opcode]; +} + +#ifdef INSTRUMENT_DEBUG + +static void +dump_instrumentation_data_tools(PyCodeObject *code, uint8_t *tools, int i, FILE*out) +{ + if (tools == NULL) { + fprintf(out, "tools = NULL"); + } + else { + fprintf(out, "tools = %d", tools[i]); + } +} + +static void +dump_instrumentation_data_lines(PyCodeObject *code, _PyCoLineInstrumentationData *lines, int i, FILE*out) +{ + if (lines == NULL) { + fprintf(out, ", lines = NULL"); + } + else if (lines[i].original_opcode == 0) { + fprintf(out, ", lines = {original_opcode = No LINE (0), line_delta = %d)", lines[i].line_delta); + } + else { + fprintf(out, ", lines = {original_opcode = %s, line_delta = %d)", _PyOpcode_OpName[lines[i].original_opcode], lines[i].line_delta); + } +} + +static void +dump_instrumentation_data_line_tools(PyCodeObject *code, uint8_t *line_tools, int i, FILE*out) +{ + if (line_tools == NULL) { + fprintf(out, ", line_tools = NULL"); + } + else { + fprintf(out, ", line_tools = %d", line_tools[i]); + } +} + +static void +dump_instrumentation_data_per_instruction(PyCodeObject *code, _PyCoMonitoringData *data, int i, FILE*out) +{ + if (data->per_instruction_opcodes == NULL) { + fprintf(out, ", per-inst opcode = NULL"); + } + else { + fprintf(out, ", per-inst opcode = %s", _PyOpcode_OpName[data->per_instruction_opcodes[i]]); + } + if (data->per_instruction_tools == NULL) { + fprintf(out, ", per-inst tools = NULL"); + } + else { + fprintf(out, ", per-inst tools = %d", data->per_instruction_tools[i]); + } +} + +static void +dump_monitors(const char *prefix, _Py_Monitors monitors, FILE*out) +{ + fprintf(out, "%s monitors:\n", prefix); + for (int event = 0; event < PY_MONITORING_UNGROUPED_EVENTS; event++) { + fprintf(out, " Event %d: Tools %x\n", event, monitors.tools[event]); + } +} + +/* Like _Py_GetBaseOpcode but without asserts. + * Does its best to give the right answer, but won't abort + * if something is wrong */ +int get_base_opcode_best_attempt(PyCodeObject *code, int offset) +{ + int opcode = _Py_OPCODE(_PyCode_CODE(code)[offset]); + if (INSTRUMENTED_OPCODES[opcode] != opcode) { + /* Not instrumented */ + return _PyOpcode_Deopt[opcode] == 0 ? opcode : _PyOpcode_Deopt[opcode]; + } + if (opcode == INSTRUMENTED_INSTRUCTION) { + if (code->_co_monitoring->per_instruction_opcodes[offset] == 0) { + return opcode; + } + opcode = code->_co_monitoring->per_instruction_opcodes[offset]; + } + if (opcode == INSTRUMENTED_LINE) { + if (code->_co_monitoring->lines[offset].original_opcode == 0) { + return opcode; + } + opcode = code->_co_monitoring->lines[offset].original_opcode; + } + int deinstrumented = DE_INSTRUMENT[opcode]; + if (deinstrumented) { + return deinstrumented; + } + if (_PyOpcode_Deopt[opcode] == 0) { + return opcode; + } + return _PyOpcode_Deopt[opcode]; +} + +/* No error checking -- Don't use this for anything but experimental debugging */ +static void +dump_instrumentation_data(PyCodeObject *code, int star, FILE*out) +{ + _PyCoMonitoringData *data = code->_co_monitoring; + fprintf(out, "\n"); + PyObject_Print(code->co_name, out, Py_PRINT_RAW); + fprintf(out, "\n"); + if (data == NULL) { + fprintf(out, "NULL\n"); + return; + } + dump_monitors("Global", PyInterpreterState_Get()->monitors, out); + dump_monitors("Code", data->local_monitors, out); + dump_monitors("Active", data->active_monitors, out); + int code_len = (int)Py_SIZE(code); + bool starred = false; + for (int i = 0; i < code_len; i += instruction_length(code, i)) { + _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; + int opcode = instr->op.code; + if (i == star) { + fprintf(out, "** "); + starred = true; + } + fprintf(out, "Offset: %d, line: %d %s: ", i, PyCode_Addr2Line(code, i*2), _PyOpcode_OpName[opcode]); + dump_instrumentation_data_tools(code, data->tools, i, out); + dump_instrumentation_data_lines(code, data->lines, i, out); + dump_instrumentation_data_line_tools(code, data->line_tools, i, out); + dump_instrumentation_data_per_instruction(code, data, i, out); + fprintf(out, "\n"); + ; + } + if (!starred && star >= 0) { + fprintf(out, "Error offset not at valid instruction offset: %d\n", star); + fprintf(out, " "); + dump_instrumentation_data_tools(code, data->tools, star, out); + dump_instrumentation_data_lines(code, data->lines, star, out); + dump_instrumentation_data_line_tools(code, data->line_tools, star, out); + dump_instrumentation_data_per_instruction(code, data, star, out); + fprintf(out, "\n"); + } +} + +#define CHECK(test) do { \ + if (!(test)) { \ + dump_instrumentation_data(code, i, stderr); \ + } \ + assert(test); \ +} while (0) + +bool valid_opcode(int opcode) { + if (opcode > 0 && + opcode != RESERVED && + opcode < 255 && + _PyOpcode_OpName[opcode] && + _PyOpcode_OpName[opcode][0] != '<' + ) { + return true; + } + return false; +} + +static void +sanity_check_instrumentation(PyCodeObject *code) +{ + _PyCoMonitoringData *data = code->_co_monitoring; + if (data == NULL) { + return; + } + _Py_Monitors active_monitors = PyInterpreterState_Get()->monitors; + if (code->_co_monitoring) { + _Py_Monitors local_monitors = code->_co_monitoring->local_monitors; + active_monitors = monitors_or(active_monitors, local_monitors); + } + assert(monitors_equals( + code->_co_monitoring->active_monitors, + active_monitors) + ); + int code_len = (int)Py_SIZE(code); + for (int i = 0; i < code_len;) { + int opcode = _PyCode_CODE(code)[i].op.code; + int base_opcode = _Py_GetBaseOpcode(code, i); + CHECK(valid_opcode(opcode)); + CHECK(valid_opcode(base_opcode)); + if (opcode == INSTRUMENTED_INSTRUCTION) { + opcode = data->per_instruction_opcodes[i]; + if (!is_instrumented(opcode)) { + CHECK(_PyOpcode_Deopt[opcode] == opcode); + } + if (data->per_instruction_tools) { + uint8_t tools = active_monitors.tools[PY_MONITORING_EVENT_INSTRUCTION]; + CHECK((tools & data->per_instruction_tools[i]) == data->per_instruction_tools[i]); + } + } + if (opcode == INSTRUMENTED_LINE) { + CHECK(data->lines); + CHECK(valid_opcode(data->lines[i].original_opcode)); + opcode = data->lines[i].original_opcode; + CHECK(opcode != END_FOR); + CHECK(opcode != RESUME); + CHECK(opcode != INSTRUMENTED_RESUME); + if (!is_instrumented(opcode)) { + CHECK(_PyOpcode_Deopt[opcode] == opcode); + } + CHECK(opcode != INSTRUMENTED_LINE); + } + else if (data->lines && !is_instrumented(opcode)) { + CHECK(data->lines[i].original_opcode == 0 || + data->lines[i].original_opcode == base_opcode || + DE_INSTRUMENT[data->lines[i].original_opcode] == base_opcode); + } + if (is_instrumented(opcode)) { + CHECK(DE_INSTRUMENT[opcode] == base_opcode); + int event = EVENT_FOR_OPCODE[DE_INSTRUMENT[opcode]]; + if (event < 0) { + /* RESUME fixup */ + event = _PyCode_CODE(code)[i].op.arg; + } + CHECK(active_monitors.tools[event] != 0); + } + if (data->lines && base_opcode != END_FOR) { + int line1 = compute_line(code, i, data->lines[i].line_delta); + int line2 = PyCode_Addr2Line(code, i*sizeof(_Py_CODEUNIT)); + CHECK(line1 == line2); + } + CHECK(valid_opcode(opcode)); + if (data->tools) { + uint8_t local_tools = data->tools[i]; + if (opcode_has_event(base_opcode)) { + int event = EVENT_FOR_OPCODE[base_opcode]; + if (event == -1) { + /* RESUME fixup */ + event = _PyCode_CODE(code)[i].op.arg; + } + CHECK((active_monitors.tools[event] & local_tools) == local_tools); + } + else { + CHECK(local_tools == 0xff); + } + } + i += instruction_length(code, i); + assert(i <= code_len); + } +} +#else + +#define CHECK(test) assert(test) + +#endif + +/* Get the underlying opcode, stripping instrumentation */ +int _Py_GetBaseOpcode(PyCodeObject *code, int i) +{ + int opcode = _PyCode_CODE(code)[i].op.code; + if (opcode == INSTRUMENTED_LINE) { + opcode = code->_co_monitoring->lines[i].original_opcode; + } + if (opcode == INSTRUMENTED_INSTRUCTION) { + opcode = code->_co_monitoring->per_instruction_opcodes[i]; + } + CHECK(opcode != INSTRUMENTED_INSTRUCTION); + CHECK(opcode != INSTRUMENTED_LINE); + int deinstrumented = DE_INSTRUMENT[opcode]; + if (deinstrumented) { + return deinstrumented; + } + return _PyOpcode_Deopt[opcode]; +} + +static void +de_instrument(PyCodeObject *code, int i, int event) +{ + assert(event != PY_MONITORING_EVENT_INSTRUCTION); + assert(event != PY_MONITORING_EVENT_LINE); + + _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; + uint8_t *opcode_ptr = &instr->op.code; + int opcode = *opcode_ptr; + if (opcode == INSTRUMENTED_LINE) { + opcode_ptr = &code->_co_monitoring->lines[i].original_opcode; + opcode = *opcode_ptr; + } + if (opcode == INSTRUMENTED_INSTRUCTION) { + opcode_ptr = &code->_co_monitoring->per_instruction_opcodes[i]; + opcode = *opcode_ptr; + } + int deinstrumented = DE_INSTRUMENT[opcode]; + if (deinstrumented == 0) { + return; + } + CHECK(_PyOpcode_Deopt[deinstrumented] == deinstrumented); + *opcode_ptr = deinstrumented; + if (_PyOpcode_Caches[deinstrumented]) { + instr[1].cache = adaptive_counter_warmup(); + } +} + +static void +de_instrument_line(PyCodeObject *code, int i) +{ + _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; + uint8_t *opcode_ptr = &instr->op.code; + int opcode =*opcode_ptr; + if (opcode != INSTRUMENTED_LINE) { + return; + } + _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[i]; + int original_opcode = lines->original_opcode; + CHECK(original_opcode != 0); + CHECK(original_opcode == _PyOpcode_Deopt[original_opcode]); + *opcode_ptr = instr->op.code = original_opcode; + if (_PyOpcode_Caches[original_opcode]) { + instr[1].cache = adaptive_counter_warmup(); + } + assert(*opcode_ptr != INSTRUMENTED_LINE); + assert(instr->op.code != INSTRUMENTED_LINE); +} + + +static void +de_instrument_per_instruction(PyCodeObject *code, int i) +{ + _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; + uint8_t *opcode_ptr = &instr->op.code; + int opcode =*opcode_ptr; + if (opcode == INSTRUMENTED_LINE) { + opcode_ptr = &code->_co_monitoring->lines[i].original_opcode; + opcode = *opcode_ptr; + } + if (opcode != INSTRUMENTED_INSTRUCTION) { + return; + } + int original_opcode = code->_co_monitoring->per_instruction_opcodes[i]; + CHECK(original_opcode != 0); + CHECK(original_opcode == _PyOpcode_Deopt[original_opcode]); + instr->op.code = original_opcode; + if (_PyOpcode_Caches[original_opcode]) { + instr[1].cache = adaptive_counter_warmup(); + } + assert(instr->op.code != INSTRUMENTED_INSTRUCTION); + /* Keep things clean for sanity check */ + code->_co_monitoring->per_instruction_opcodes[i] = 0; +} + + +static void +instrument(PyCodeObject *code, int i) +{ + _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; + uint8_t *opcode_ptr = &instr->op.code; + int opcode =*opcode_ptr; + if (opcode == INSTRUMENTED_LINE) { + _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[i]; + opcode_ptr = &lines->original_opcode; + opcode = *opcode_ptr; + } + if (opcode == INSTRUMENTED_INSTRUCTION) { + opcode_ptr = &code->_co_monitoring->per_instruction_opcodes[i]; + opcode = *opcode_ptr; + CHECK(!is_instrumented(opcode)); + CHECK(opcode == _PyOpcode_Deopt[opcode]); + } + CHECK(opcode != 0); + if (!is_instrumented(opcode)) { + int deopt = _PyOpcode_Deopt[opcode]; + int instrumented = INSTRUMENTED_OPCODES[deopt]; + assert(instrumented); + *opcode_ptr = instrumented; + if (_PyOpcode_Caches[deopt]) { + instr[1].cache = adaptive_counter_warmup(); + } + } +} + +static void +instrument_line(PyCodeObject *code, int i) +{ + uint8_t *opcode_ptr = &_PyCode_CODE(code)[i].op.code; + int opcode =*opcode_ptr; + if (opcode == INSTRUMENTED_LINE) { + return; + } + _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[i]; + lines->original_opcode = _PyOpcode_Deopt[opcode]; + CHECK(lines->original_opcode > 0); + *opcode_ptr = INSTRUMENTED_LINE; +} + +static void +instrument_per_instruction(PyCodeObject *code, int i) +{ + _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; + uint8_t *opcode_ptr = &instr->op.code; + int opcode =*opcode_ptr; + if (opcode == INSTRUMENTED_LINE) { + _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[i]; + opcode_ptr = &lines->original_opcode; + opcode = *opcode_ptr; + } + if (opcode == INSTRUMENTED_INSTRUCTION) { + return; + } + CHECK(opcode != 0); + if (is_instrumented(opcode)) { + code->_co_monitoring->per_instruction_opcodes[i] = opcode; + } + else { + assert(opcode != 0); + assert(_PyOpcode_Deopt[opcode] != 0); + assert(_PyOpcode_Deopt[opcode] != RESUME); + code->_co_monitoring->per_instruction_opcodes[i] = _PyOpcode_Deopt[opcode]; + } + assert(code->_co_monitoring->per_instruction_opcodes[i] > 0); + *opcode_ptr = INSTRUMENTED_INSTRUCTION; +} + +#ifndef NDEBUG +static bool +instruction_has_event(PyCodeObject *code, int offset) +{ + _Py_CODEUNIT instr = _PyCode_CODE(code)[offset]; + int opcode = instr.op.code; + if (opcode == INSTRUMENTED_LINE) { + opcode = code->_co_monitoring->lines[offset].original_opcode; + } + if (opcode == INSTRUMENTED_INSTRUCTION) { + opcode = code->_co_monitoring->per_instruction_opcodes[offset]; + } + return opcode_has_event(opcode); +} +#endif + +static void +remove_tools(PyCodeObject * code, int offset, int event, int tools) +{ + assert(event != PY_MONITORING_EVENT_LINE); + assert(event != PY_MONITORING_EVENT_INSTRUCTION); + assert(event < PY_MONITORING_INSTRUMENTED_EVENTS); + assert(instruction_has_event(code, offset)); + _PyCoMonitoringData *monitoring = code->_co_monitoring; + if (monitoring && monitoring->tools) { + monitoring->tools[offset] &= ~tools; + if (monitoring->tools[offset] == 0) { + de_instrument(code, offset, event); + } + } + else { + /* Single tool */ + uint8_t single_tool = code->_co_monitoring->active_monitors.tools[event]; + assert(_Py_popcount32(single_tool) <= 1); + if (((single_tool & tools) == single_tool)) { + de_instrument(code, offset, event); + } + } +} + +#ifndef NDEBUG +static bool +tools_is_subset_for_event(PyCodeObject * code, int event, int tools) +{ + int global_tools = PyInterpreterState_Get()->monitors.tools[event]; + int local_tools = code->_co_monitoring->local_monitors.tools[event]; + return tools == ((global_tools | local_tools) & tools); +} +#endif + +static void +remove_line_tools(PyCodeObject * code, int offset, int tools) +{ + assert(code->_co_monitoring); + if (code->_co_monitoring->line_tools) + { + uint8_t *toolsptr = &code->_co_monitoring->line_tools[offset]; + *toolsptr &= ~tools; + if (*toolsptr == 0 ) { + de_instrument_line(code, offset); + } + } + else { + /* Single tool */ + uint8_t single_tool = code->_co_monitoring->active_monitors.tools[PY_MONITORING_EVENT_LINE]; + assert(_Py_popcount32(single_tool) <= 1); + if (((single_tool & tools) == single_tool)) { + de_instrument_line(code, offset); + } + } +} + +static void +add_tools(PyCodeObject * code, int offset, int event, int tools) +{ + assert(event != PY_MONITORING_EVENT_LINE); + assert(event != PY_MONITORING_EVENT_INSTRUCTION); + assert(event < PY_MONITORING_INSTRUMENTED_EVENTS); + assert(code->_co_monitoring); + if (code->_co_monitoring && + code->_co_monitoring->tools + ) { + code->_co_monitoring->tools[offset] |= tools; + } + else { + /* Single tool */ + assert(_Py_popcount32(tools) == 1); + assert(tools_is_subset_for_event(code, event, tools)); + } + instrument(code, offset); +} + +static void +add_line_tools(PyCodeObject * code, int offset, int tools) +{ + assert(tools_is_subset_for_event(code, PY_MONITORING_EVENT_LINE, tools)); + assert(code->_co_monitoring); + if (code->_co_monitoring->line_tools + ) { + code->_co_monitoring->line_tools[offset] |= tools; + } + else { + /* Single tool */ + assert(_Py_popcount32(tools) == 1); + } + instrument_line(code, offset); +} + + +static void +add_per_instruction_tools(PyCodeObject * code, int offset, int tools) +{ + assert(tools_is_subset_for_event(code, PY_MONITORING_EVENT_INSTRUCTION, tools)); + assert(code->_co_monitoring); + if (code->_co_monitoring->per_instruction_tools + ) { + code->_co_monitoring->per_instruction_tools[offset] |= tools; + } + else { + /* Single tool */ + assert(_Py_popcount32(tools) == 1); + } + instrument_per_instruction(code, offset); +} + + +static void +remove_per_instruction_tools(PyCodeObject * code, int offset, int tools) +{ + assert(code->_co_monitoring); + if (code->_co_monitoring->per_instruction_tools) + { + uint8_t *toolsptr = &code->_co_monitoring->per_instruction_tools[offset]; + *toolsptr &= ~tools; + if (*toolsptr == 0 ) { + de_instrument_per_instruction(code, offset); + } + } + else { + /* Single tool */ + uint8_t single_tool = code->_co_monitoring->active_monitors.tools[PY_MONITORING_EVENT_INSTRUCTION]; + assert(_Py_popcount32(single_tool) <= 1); + if (((single_tool & tools) == single_tool)) { + de_instrument_per_instruction(code, offset); + } + } +} + + +/* Return 1 if DISABLE returned, -1 if error, 0 otherwise */ +static int +call_one_instrument( + PyInterpreterState *interp, PyThreadState *tstate, PyObject **args, + Py_ssize_t nargsf, int8_t tool, int event) +{ + assert(0 <= tool && tool < 8); + assert(tstate->tracing == 0); + PyObject *instrument = interp->monitoring_callables[tool][event]; + if (instrument == NULL) { + return 0; + } + int old_what = tstate->what_event; + tstate->what_event = event; + tstate->tracing++; + PyObject *res = _PyObject_VectorcallTstate(tstate, instrument, args, nargsf, NULL); + tstate->tracing--; + tstate->what_event = old_what; + if (res == NULL) { + return -1; + } + Py_DECREF(res); + return (res == &DISABLE); +} + +static const int8_t MOST_SIGNIFICANT_BITS[16] = { + -1, 0, 1, 1, + 2, 2, 2, 2, + 3, 3, 3, 3, + 3, 3, 3, 3, +}; + +/* We could use _Py_bit_length here, but that is designed for larger (32/64) bit ints, + and can perform relatively poorly on platforms without the necessary intrinsics. */ +static inline int most_significant_bit(uint8_t bits) { + assert(bits != 0); + if (bits > 15) { + return MOST_SIGNIFICANT_BITS[bits>>4]+4; + } + else { + return MOST_SIGNIFICANT_BITS[bits]; + } +} + +static bool +is_version_up_to_date(PyCodeObject *code, PyInterpreterState *interp) +{ + return interp->monitoring_version == code->_co_instrumentation_version; +} + +#ifndef NDEBUG +static bool +instrumentation_cross_checks(PyInterpreterState *interp, PyCodeObject *code) +{ + _Py_Monitors expected = monitors_or( + interp->monitors, + code->_co_monitoring->local_monitors); + return monitors_equals(code->_co_monitoring->active_monitors, expected); +} +#endif + +static inline uint8_t +get_tools_for_instruction(PyCodeObject * code, int i, int event) +{ + uint8_t tools; + assert(event != PY_MONITORING_EVENT_LINE); + assert(event != PY_MONITORING_EVENT_INSTRUCTION); + assert(instrumentation_cross_checks(PyThreadState_GET()->interp, code)); + _PyCoMonitoringData *monitoring = code->_co_monitoring; + if (event >= PY_MONITORING_UNGROUPED_EVENTS) { + assert(event == PY_MONITORING_EVENT_C_RAISE || + event == PY_MONITORING_EVENT_C_RETURN); + event = PY_MONITORING_EVENT_CALL; + } + if (event < PY_MONITORING_INSTRUMENTED_EVENTS && monitoring->tools) { + tools = monitoring->tools[i]; + } + else { + tools = code->_co_monitoring->active_monitors.tools[event]; + } + CHECK(tools_is_subset_for_event(code, event, tools)); + CHECK((tools & code->_co_monitoring->active_monitors.tools[event]) == tools); + return tools; +} + +static int +call_instrumentation_vector( + PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, Py_ssize_t nargs, PyObject *args[]) +{ + if (tstate->tracing) { + return 0; + } + assert(!_PyErr_Occurred(tstate)); + assert(args[0] == NULL); + PyCodeObject *code = frame->f_code; + assert(code->_co_instrumentation_version == tstate->interp->monitoring_version); + assert(is_version_up_to_date(code, tstate->interp)); + assert(instrumentation_cross_checks(tstate->interp, code)); + assert(args[1] == NULL); + args[1] = (PyObject *)code; + int offset = (int)(instr - _PyCode_CODE(code)); + /* Offset visible to user should be the offset in bytes, as that is the + * convention for APIs involving code offsets. */ + int bytes_offset = offset * (int)sizeof(_Py_CODEUNIT); + PyObject *offset_obj = PyLong_FromSsize_t(bytes_offset); + if (offset_obj == NULL) { + return -1; + } + assert(args[2] == NULL); + args[2] = offset_obj; + uint8_t tools = get_tools_for_instruction(code, offset, event); + Py_ssize_t nargsf = nargs | PY_VECTORCALL_ARGUMENTS_OFFSET; + PyObject **callargs = &args[1]; + int err = 0; + PyInterpreterState *interp = tstate->interp; + while (tools) { + int tool = most_significant_bit(tools); + assert(tool >= 0 && tool < 8); + assert(tools & (1 << tool)); + tools ^= (1 << tool); + int res = call_one_instrument(interp, tstate, callargs, nargsf, tool, event); + if (res == 0) { + /* Nothing to do */ + } + else if (res < 0) { + /* error */ + err = -1; + break; + } + else { + /* DISABLE */ + remove_tools(code, offset, event, 1 << tool); + } + } + Py_DECREF(offset_obj); + return err; +} + +int +_Py_call_instrumentation( + PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) +{ + PyObject *args[3] = { NULL, NULL, NULL }; + return call_instrumentation_vector(tstate, event, frame, instr, 2, args); +} + +int +_Py_call_instrumentation_arg( + PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg) +{ + PyObject *args[4] = { NULL, NULL, NULL, arg }; + return call_instrumentation_vector(tstate, event, frame, instr, 3, args); +} + +int +_Py_call_instrumentation_2args( + PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg0, PyObject *arg1) +{ + PyObject *args[5] = { NULL, NULL, NULL, arg0, arg1 }; + return call_instrumentation_vector(tstate, event, frame, instr, 4, args); +} + +int +_Py_call_instrumentation_jump( + PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *target +) { + assert(event == PY_MONITORING_EVENT_JUMP || + event == PY_MONITORING_EVENT_BRANCH); + assert(frame->prev_instr == instr); + frame->prev_instr = target; + PyCodeObject *code = frame->f_code; + int to = (int)(target - _PyCode_CODE(code)); + PyObject *to_obj = PyLong_FromLong(to * (int)sizeof(_Py_CODEUNIT)); + if (to_obj == NULL) { + return -1; + } + PyObject *args[4] = { NULL, NULL, NULL, to_obj }; + int err = call_instrumentation_vector(tstate, event, frame, instr, 3, args); + Py_DECREF(to_obj); + return err; +} + +static void +call_instrumentation_vector_protected( + PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, Py_ssize_t nargs, PyObject *args[]) +{ + assert(_PyErr_Occurred(tstate)); + PyObject *exc = _PyErr_GetRaisedException(tstate); + int err = call_instrumentation_vector(tstate, event, frame, instr, nargs, args); + if (err) { + Py_XDECREF(exc); + } + else { + _PyErr_SetRaisedException(tstate, exc); + } + assert(_PyErr_Occurred(tstate)); +} + +void +_Py_call_instrumentation_exc0( + PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) +{ + assert(_PyErr_Occurred(tstate)); + PyObject *args[3] = { NULL, NULL, NULL }; + call_instrumentation_vector_protected(tstate, event, frame, instr, 2, args); +} + +void +_Py_call_instrumentation_exc2( + PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg0, PyObject *arg1) +{ + assert(_PyErr_Occurred(tstate)); + PyObject *args[5] = { NULL, NULL, NULL, arg0, arg1 }; + call_instrumentation_vector_protected(tstate, event, frame, instr, 4, args); +} + + +int +_Py_Instrumentation_GetLine(PyCodeObject *code, int index) +{ + _PyCoMonitoringData *monitoring = code->_co_monitoring; + assert(monitoring != NULL); + assert(monitoring->lines != NULL); + assert(index >= code->_co_firsttraceable); + assert(index < Py_SIZE(code)); + _PyCoLineInstrumentationData *line_data = &monitoring->lines[index]; + int8_t line_delta = line_data->line_delta; + int line = compute_line(code, index, line_delta); + return line; +} + +int +_Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr) +{ + frame->prev_instr = instr; + PyCodeObject *code = frame->f_code; + assert(is_version_up_to_date(code, tstate->interp)); + assert(instrumentation_cross_checks(tstate->interp, code)); + int i = (int)(instr - _PyCode_CODE(code)); + _PyCoMonitoringData *monitoring = code->_co_monitoring; + _PyCoLineInstrumentationData *line_data = &monitoring->lines[i]; + uint8_t original_opcode = line_data->original_opcode; + if (tstate->tracing) { + goto done; + } + PyInterpreterState *interp = tstate->interp; + int8_t line_delta = line_data->line_delta; + int line = compute_line(code, i, line_delta); + uint8_t tools = code->_co_monitoring->line_tools != NULL ? + code->_co_monitoring->line_tools[i] : + (interp->monitors.tools[PY_MONITORING_EVENT_LINE] | + code->_co_monitoring->local_monitors.tools[PY_MONITORING_EVENT_LINE] + ); + PyObject *line_obj = PyLong_FromSsize_t(line); + if (line_obj == NULL) { + return -1; + } + PyObject *args[3] = { NULL, (PyObject *)code, line_obj }; + while (tools) { + int tool = most_significant_bit(tools); + assert(tool >= 0 && tool < 8); + assert(tools & (1 << tool)); + tools &= ~(1 << tool); + int res = call_one_instrument(interp, tstate, &args[1], + 2 | PY_VECTORCALL_ARGUMENTS_OFFSET, + tool, PY_MONITORING_EVENT_LINE); + if (res == 0) { + /* Nothing to do */ + } + else if (res < 0) { + /* error */ + Py_DECREF(line_obj); + return -1; + } + else { + /* DISABLE */ + remove_line_tools(code, i, 1 << tool); + } + } + Py_DECREF(line_obj); +done: + assert(original_opcode != 0); + assert(original_opcode < INSTRUMENTED_LINE); + assert(_PyOpcode_Deopt[original_opcode] == original_opcode); + return original_opcode; +} + +int +_Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr) +{ + PyCodeObject *code = frame->f_code; + assert(is_version_up_to_date(code, tstate->interp)); + assert(instrumentation_cross_checks(tstate->interp, code)); + int offset = (int)(instr - _PyCode_CODE(code)); + _PyCoMonitoringData *instrumentation_data = code->_co_monitoring; + assert(instrumentation_data->per_instruction_opcodes); + int next_opcode = instrumentation_data->per_instruction_opcodes[offset]; + if (tstate->tracing) { + return next_opcode; + } + PyInterpreterState *interp = tstate->interp; + uint8_t tools = instrumentation_data->per_instruction_tools != NULL ? + instrumentation_data->per_instruction_tools[offset] : + (interp->monitors.tools[PY_MONITORING_EVENT_INSTRUCTION] | + code->_co_monitoring->local_monitors.tools[PY_MONITORING_EVENT_INSTRUCTION] + ); + int bytes_offset = offset * (int)sizeof(_Py_CODEUNIT); + PyObject *offset_obj = PyLong_FromSsize_t(bytes_offset); + if (offset_obj == NULL) { + return -1; + } + PyObject *args[3] = { NULL, (PyObject *)code, offset_obj }; + while (tools) { + int tool = most_significant_bit(tools); + assert(tool >= 0 && tool < 8); + assert(tools & (1 << tool)); + tools &= ~(1 << tool); + int res = call_one_instrument(interp, tstate, &args[1], + 2 | PY_VECTORCALL_ARGUMENTS_OFFSET, + tool, PY_MONITORING_EVENT_INSTRUCTION); + if (res == 0) { + /* Nothing to do */ + } + else if (res < 0) { + /* error */ + Py_DECREF(offset_obj); + return -1; + } + else { + /* DISABLE */ + remove_per_instruction_tools(code, offset, 1 << tool); + } + } + Py_DECREF(offset_obj); + assert(next_opcode != 0); + return next_opcode; +} + + +PyObject * +_PyMonitoring_RegisterCallback(int tool_id, int event_id, PyObject *obj) +{ + PyInterpreterState *is = _PyInterpreterState_Get(); + assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); + assert(0 <= event_id && event_id < PY_MONITORING_EVENTS); + PyObject *callback = is->monitoring_callables[tool_id][event_id]; + is->monitoring_callables[tool_id][event_id] = Py_XNewRef(obj); + return callback; +} + +static void +initialize_tools(PyCodeObject *code) +{ + uint8_t* tools = code->_co_monitoring->tools; + assert(tools != NULL); + int code_len = (int)Py_SIZE(code); + for (int i = 0; i < code_len; i++) { + _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; + int opcode = instr->op.code; + if (opcode == INSTRUMENTED_LINE) { + opcode = code->_co_monitoring->lines[i].original_opcode; + } + bool instrumented = is_instrumented(opcode); + if (instrumented) { + opcode = DE_INSTRUMENT[opcode]; + assert(opcode != 0); + } + opcode = _PyOpcode_Deopt[opcode]; + if (opcode_has_event(opcode)) { + if (instrumented) { + int8_t event; + if (opcode == RESUME) { + event = instr->op.arg != 0; + } + else { + event = EVENT_FOR_OPCODE[opcode]; + assert(event > 0); + } + assert(event >= 0); + assert(event < PY_MONITORING_INSTRUMENTED_EVENTS); + tools[i] = code->_co_monitoring->active_monitors.tools[event]; + CHECK(tools[i] != 0); + } + else { + tools[i] = 0; + } + } +#ifdef Py_DEBUG + /* Initialize tools for invalid locations to all ones to try to catch errors */ + else { + tools[i] = 0xff; + } + for (int j = 1; j <= _PyOpcode_Caches[opcode]; j++) { + tools[i+j] = 0xff; + } +#endif + i += _PyOpcode_Caches[opcode]; + } +} + +#define NO_LINE -128 + +static void +initialize_lines(PyCodeObject *code) +{ + _PyCoLineInstrumentationData *line_data = code->_co_monitoring->lines; + assert(line_data != NULL); + int code_len = (int)Py_SIZE(code); + PyCodeAddressRange range; + _PyCode_InitAddressRange(code, &range); + for (int i = 0; i < code->_co_firsttraceable && i < code_len; i++) { + line_data[i].original_opcode = 0; + line_data[i].line_delta = -127; + } + int current_line = -1; + for (int i = code->_co_firsttraceable; i < code_len; ) { + int opcode = _Py_GetBaseOpcode(code, i); + int line = _PyCode_CheckLineNumber(i*(int)sizeof(_Py_CODEUNIT), &range); + line_data[i].line_delta = compute_line_delta(code, i, line); + int length = instruction_length(code, i); + switch (opcode) { + case END_ASYNC_FOR: + case END_FOR: + case END_SEND: + case RESUME: + /* END_FOR cannot start a line, as it is skipped by FOR_ITER + * END_SEND cannot start a line, as it is skipped by SEND + * RESUME must not be instrumented with INSTRUMENT_LINE */ + line_data[i].original_opcode = 0; + break; + default: + if (line != current_line && line >= 0) { + line_data[i].original_opcode = opcode; + } + else { + line_data[i].original_opcode = 0; + } + if (line >= 0) { + current_line = line; + } + } + for (int j = 1; j < length; j++) { + line_data[i+j].original_opcode = 0; + line_data[i+j].line_delta = NO_LINE; + } + switch (opcode) { + case RETURN_VALUE: + case RAISE_VARARGS: + case RERAISE: + /* Blocks of code after these terminators + * should be treated as different lines */ + current_line = -1; + } + i += length; + } +} + +static void +initialize_line_tools(PyCodeObject *code, _Py_Monitors *all_events) +{ + uint8_t *line_tools = code->_co_monitoring->line_tools; + assert(line_tools != NULL); + int code_len = (int)Py_SIZE(code); + for (int i = 0; i < code_len; i++) { + line_tools[i] = all_events->tools[PY_MONITORING_EVENT_LINE]; + } +} + +static +int allocate_instrumentation_data(PyCodeObject *code) +{ + + if (code->_co_monitoring == NULL) { + code->_co_monitoring = PyMem_Malloc(sizeof(_PyCoMonitoringData)); + if (code->_co_monitoring == NULL) { + PyErr_NoMemory(); + return -1; + } + code->_co_monitoring->local_monitors = (_Py_Monitors){ 0 }; + code->_co_monitoring->active_monitors = (_Py_Monitors){ 0 }; + code->_co_monitoring->tools = NULL; + code->_co_monitoring->lines = NULL; + code->_co_monitoring->line_tools = NULL; + code->_co_monitoring->per_instruction_opcodes = NULL; + code->_co_monitoring->per_instruction_tools = NULL; + } + return 0; +} + +static int +update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) +{ + int code_len = (int)Py_SIZE(code); + if (allocate_instrumentation_data(code)) { + return -1; + } + _Py_Monitors all_events = monitors_or( + interp->monitors, + code->_co_monitoring->local_monitors); + bool multitools = multiple_tools(&all_events); + if (code->_co_monitoring->tools == NULL && multitools) { + code->_co_monitoring->tools = PyMem_Malloc(code_len); + if (code->_co_monitoring->tools == NULL) { + PyErr_NoMemory(); + return -1; + } + initialize_tools(code); + } + if (all_events.tools[PY_MONITORING_EVENT_LINE]) { + if (code->_co_monitoring->lines == NULL) { + code->_co_monitoring->lines = PyMem_Malloc(code_len * sizeof(_PyCoLineInstrumentationData)); + if (code->_co_monitoring->lines == NULL) { + PyErr_NoMemory(); + return -1; + } + initialize_lines(code); + } + if (multitools && code->_co_monitoring->line_tools == NULL) { + code->_co_monitoring->line_tools = PyMem_Malloc(code_len); + if (code->_co_monitoring->line_tools == NULL) { + PyErr_NoMemory(); + return -1; + } + initialize_line_tools(code, &all_events); + } + } + if (all_events.tools[PY_MONITORING_EVENT_INSTRUCTION]) { + if (code->_co_monitoring->per_instruction_opcodes == NULL) { + code->_co_monitoring->per_instruction_opcodes = PyMem_Malloc(code_len * sizeof(_PyCoLineInstrumentationData)); + if (code->_co_monitoring->per_instruction_opcodes == NULL) { + PyErr_NoMemory(); + return -1; + } + /* This may not be necessary, as we can initialize this memory lazily, but it helps catch errors. */ + for (int i = 0; i < code_len; i++) { + code->_co_monitoring->per_instruction_opcodes[i] = 0; + } + } + if (multitools && code->_co_monitoring->per_instruction_tools == NULL) { + code->_co_monitoring->per_instruction_tools = PyMem_Malloc(code_len); + if (code->_co_monitoring->per_instruction_tools == NULL) { + PyErr_NoMemory(); + return -1; + } + /* This may not be necessary, as we can initialize this memory lazily, but it helps catch errors. */ + for (int i = 0; i < code_len; i++) { + code->_co_monitoring->per_instruction_tools[i] = 0; + } + } + } + return 0; +} + +static const uint8_t super_instructions[256] = { + [LOAD_FAST__LOAD_FAST] = 1, + [LOAD_FAST__LOAD_CONST] = 1, + [STORE_FAST__LOAD_FAST] = 1, + [STORE_FAST__STORE_FAST] = 1, + [LOAD_CONST__LOAD_FAST] = 1, +}; + +/* Should use instruction metadata for this */ +static bool +is_super_instruction(int opcode) { + return super_instructions[opcode] != 0; +} + +int +_Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) +{ + + if (is_version_up_to_date(code, interp)) { + assert( + interp->monitoring_version == 0 || + instrumentation_cross_checks(interp, code) + ); + return 0; + } + int code_len = (int)Py_SIZE(code); + if (update_instrumentation_data(code, interp)) { + return -1; + } + _Py_Monitors active_events = monitors_or( + interp->monitors, + code->_co_monitoring->local_monitors); + _Py_Monitors new_events; + _Py_Monitors removed_events; + + bool restarted = interp->last_restart_version > code->_co_instrumentation_version; + if (restarted) { + removed_events = code->_co_monitoring->active_monitors; + new_events = active_events; + } + else { + removed_events = monitors_sub(code->_co_monitoring->active_monitors, active_events); + new_events = monitors_sub(active_events, code->_co_monitoring->active_monitors); + assert(monitors_are_empty(monitors_and(new_events, removed_events))); + } + code->_co_monitoring->active_monitors = active_events; + code->_co_instrumentation_version = interp->monitoring_version; + if (monitors_are_empty(new_events) && monitors_are_empty(removed_events)) { +#ifdef INSTRUMENT_DEBUG + sanity_check_instrumentation(code); +#endif + return 0; + } + /* Insert instrumentation */ + for (int i = 0; i < code_len; i+= instruction_length(code, i)) { + _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; + if (is_super_instruction(instr->op.code)) { + instr->op.code = _PyOpcode_Deopt[instr->op.code]; + } + CHECK(instr->op.code != 0); + int base_opcode = _Py_GetBaseOpcode(code, i); + if (opcode_has_event(base_opcode)) { + int8_t event; + if (base_opcode == RESUME) { + event = instr->op.arg > 0; + } + else { + event = EVENT_FOR_OPCODE[base_opcode]; + assert(event > 0); + } + uint8_t removed_tools = removed_events.tools[event]; + if (removed_tools) { + remove_tools(code, i, event, removed_tools); + } + uint8_t new_tools = new_events.tools[event]; + if (new_tools) { + add_tools(code, i, event, new_tools); + } + } + } + uint8_t new_line_tools = new_events.tools[PY_MONITORING_EVENT_LINE]; + uint8_t removed_line_tools = removed_events.tools[PY_MONITORING_EVENT_LINE]; + if (new_line_tools | removed_line_tools) { + _PyCoLineInstrumentationData *line_data = code->_co_monitoring->lines; + for (int i = code->_co_firsttraceable; i < code_len;) { + if (line_data[i].original_opcode) { + if (removed_line_tools) { + remove_line_tools(code, i, removed_line_tools); + } + if (new_line_tools) { + add_line_tools(code, i, new_line_tools); + } + } + i += instruction_length(code, i); + } + } + uint8_t new_per_instruction_tools = new_events.tools[PY_MONITORING_EVENT_INSTRUCTION]; + uint8_t removed_per_instruction_tools = removed_events.tools[PY_MONITORING_EVENT_INSTRUCTION]; + if (new_per_instruction_tools | removed_per_instruction_tools) { + for (int i = code->_co_firsttraceable; i < code_len;) { + int opcode = _Py_GetBaseOpcode(code, i); + if (opcode == RESUME || opcode == END_FOR) { + i += instruction_length(code, i); + continue; + } + if (removed_per_instruction_tools) { + remove_per_instruction_tools(code, i, removed_per_instruction_tools); + } + if (new_per_instruction_tools) { + add_per_instruction_tools(code, i, new_per_instruction_tools); + } + i += instruction_length(code, i); + } + } +#ifdef INSTRUMENT_DEBUG + sanity_check_instrumentation(code); +#endif + return 0; +} + +#define C_RETURN_EVENTS \ + ((1 << PY_MONITORING_EVENT_C_RETURN) | \ + (1 << PY_MONITORING_EVENT_C_RAISE)) + +#define C_CALL_EVENTS \ + (C_RETURN_EVENTS | (1 << PY_MONITORING_EVENT_CALL)) + + +static int +instrument_all_executing_code_objects(PyInterpreterState *interp) { + _PyRuntimeState *runtime = &_PyRuntime; + HEAD_LOCK(runtime); + PyThreadState* ts = PyInterpreterState_ThreadHead(interp); + HEAD_UNLOCK(runtime); + while (ts) { + _PyInterpreterFrame *frame = ts->cframe->current_frame; + while (frame) { + if (frame->owner != FRAME_OWNED_BY_CSTACK) { + if (_Py_Instrument(frame->f_code, interp)) { + return -1; + } + } + frame = frame->previous; + } + HEAD_LOCK(runtime); + ts = PyThreadState_Next(ts); + HEAD_UNLOCK(runtime); + } + return 0; +} + +static void +set_events(_Py_Monitors *m, int tool_id, _PyMonitoringEventSet events) +{ + assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); + for (int e = 0; e < PY_MONITORING_UNGROUPED_EVENTS; e++) { + uint8_t *tools = &m->tools[e]; + int val = (events >> e) & 1; + *tools &= ~(1 << tool_id); + *tools |= (val << tool_id); + } +} + +static int +check_tool(PyInterpreterState *interp, int tool_id) +{ + if (tool_id < PY_MONITORING_SYS_PROFILE_ID && + interp->monitoring_tool_names[tool_id] == NULL + ) { + PyErr_Format(PyExc_ValueError, "tool %d is not in use", tool_id); + return -1; + } + return 0; +} + +int +_PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) +{ + assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); + PyInterpreterState *interp = _PyInterpreterState_Get(); + assert(events < (1 << PY_MONITORING_UNGROUPED_EVENTS)); + if (check_tool(interp, tool_id)) { + return -1; + } + uint32_t existing_events = get_events(&interp->monitors, tool_id); + if (existing_events == events) { + return 0; + } + set_events(&interp->monitors, tool_id, events); + interp->monitoring_version++; + return instrument_all_executing_code_objects(interp); +} + +int +_PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEventSet events) +{ + assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); + PyInterpreterState *interp = _PyInterpreterState_Get(); + assert(events < (1 << PY_MONITORING_UNGROUPED_EVENTS)); + if (check_tool(interp, tool_id)) { + return -1; + } + if (allocate_instrumentation_data(code)) { + return -1; + } + _Py_Monitors *local = &code->_co_monitoring->local_monitors; + uint32_t existing_events = get_events(local, tool_id); + if (existing_events == events) { + return 0; + } + set_events(local, tool_id, events); + if (is_version_up_to_date(code, interp)) { + /* Force instrumentation update */ + code->_co_instrumentation_version = UINT64_MAX; + } + if (_Py_Instrument(code, interp)) { + return -1; + } + return 0; +} + +/*[clinic input] +module monitoring +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=37257f5987a360cf]*/ +/*[clinic end generated code]*/ + +#include "clinic/instrumentation.c.h" + +static int +check_valid_tool(int tool_id) +{ + if (tool_id < 0 || tool_id >= PY_MONITORING_SYS_PROFILE_ID) { + PyErr_Format(PyExc_ValueError, "invalid tool %d (must be between 0 and 5)", tool_id); + return -1; + } + return 0; +} + +/*[clinic input] +monitoring.use_tool_id + + tool_id: int + name: object + / + +[clinic start generated code]*/ + +static PyObject * +monitoring_use_tool_id_impl(PyObject *module, int tool_id, PyObject *name) +/*[clinic end generated code: output=30d76dc92b7cd653 input=ebc453761c621be1]*/ +{ + if (check_valid_tool(tool_id)) { + return NULL; + } + if (!PyUnicode_Check(name)) { + PyErr_SetString(PyExc_ValueError, "tool name must be a str"); + return NULL; + } + PyInterpreterState *interp = _PyInterpreterState_Get(); + if (interp->monitoring_tool_names[tool_id] != NULL) { + PyErr_Format(PyExc_ValueError, "tool %d is already in use", tool_id); + return NULL; + } + interp->monitoring_tool_names[tool_id] = Py_NewRef(name); + Py_RETURN_NONE; +} + +/*[clinic input] +monitoring.free_tool_id + + tool_id: int + / + +[clinic start generated code]*/ + +static PyObject * +monitoring_free_tool_id_impl(PyObject *module, int tool_id) +/*[clinic end generated code: output=86c2d2a1219a8591 input=a23fb6be3a8618e9]*/ +{ + if (check_valid_tool(tool_id)) { + return NULL; + } + PyInterpreterState *interp = _PyInterpreterState_Get(); + Py_CLEAR(interp->monitoring_tool_names[tool_id]); + Py_RETURN_NONE; +} + +/*[clinic input] +monitoring.get_tool + + tool_id: int + / + +[clinic start generated code]*/ + +static PyObject * +monitoring_get_tool_impl(PyObject *module, int tool_id) +/*[clinic end generated code: output=1c05a98b404a9a16 input=eeee9bebd0bcae9d]*/ + +/*[clinic end generated code]*/ +{ + if (check_valid_tool(tool_id)) { + return NULL; + } + PyInterpreterState *interp = _PyInterpreterState_Get(); + PyObject *name = interp->monitoring_tool_names[tool_id]; + if (name == NULL) { + Py_RETURN_NONE; + } + return Py_NewRef(name); +} + +/*[clinic input] +monitoring.register_callback + + + tool_id: int + event: int + func: object + / + +[clinic start generated code]*/ + +static PyObject * +monitoring_register_callback_impl(PyObject *module, int tool_id, int event, + PyObject *func) +/*[clinic end generated code: output=e64daa363004030c input=df6d70ea4cf81007]*/ +{ + if (check_valid_tool(tool_id)) { + return NULL; + } + if (_Py_popcount32(event) != 1) { + PyErr_SetString(PyExc_ValueError, "The callback can only be set for one event at a time"); + return NULL; + } + int event_id = _Py_bit_length(event)-1; + if (event_id < 0 || event_id >= PY_MONITORING_EVENTS) { + PyErr_Format(PyExc_ValueError, "invalid event %d", event); + return NULL; + } + if (func == Py_None) { + func = NULL; + } + func = _PyMonitoring_RegisterCallback(tool_id, event_id, func); + if (func == NULL) { + Py_RETURN_NONE; + } + return func; +} + +/*[clinic input] +monitoring.get_events -> int + + tool_id: int + / + +[clinic start generated code]*/ + +static int +monitoring_get_events_impl(PyObject *module, int tool_id) +/*[clinic end generated code: output=4450cc13f826c8c0 input=a64b238f76c4b2f7]*/ +{ + if (check_valid_tool(tool_id)) { + return -1; + } + _Py_Monitors *m = &_PyInterpreterState_Get()->monitors; + _PyMonitoringEventSet event_set = get_events(m, tool_id); + return event_set; +} + +/*[clinic input] +monitoring.set_events + + tool_id: int + event_set: int + / + +[clinic start generated code]*/ + +static PyObject * +monitoring_set_events_impl(PyObject *module, int tool_id, int event_set) +/*[clinic end generated code: output=1916c1e49cfb5bdb input=a77ba729a242142b]*/ +{ + if (check_valid_tool(tool_id)) { + return NULL; + } + if (event_set < 0 || event_set >= (1 << PY_MONITORING_EVENTS)) { + PyErr_Format(PyExc_ValueError, "invalid event set 0x%x", event_set); + return NULL; + } + if ((event_set & C_RETURN_EVENTS) && (event_set & C_CALL_EVENTS) != C_CALL_EVENTS) { + PyErr_Format(PyExc_ValueError, "cannot set C_RETURN or C_RAISE events independently"); + return NULL; + } + event_set &= ~C_RETURN_EVENTS; + if (_PyMonitoring_SetEvents(tool_id, event_set)) { + return NULL; + } + Py_RETURN_NONE; +} + +/*[clinic input] +monitoring.get_local_events -> int + + tool_id: int + code: object + / + +[clinic start generated code]*/ + +static int +monitoring_get_local_events_impl(PyObject *module, int tool_id, + PyObject *code) +/*[clinic end generated code: output=d3e92c1c9c1de8f9 input=bb0f927530386a94]*/ +{ + if (!PyCode_Check(code)) { + PyErr_Format( + PyExc_TypeError, + "code must be a code object" + ); + return -1; + } + if (check_valid_tool(tool_id)) { + return -1; + } + _PyMonitoringEventSet event_set = 0; + _PyCoMonitoringData *data = ((PyCodeObject *)code)->_co_monitoring; + if (data != NULL) { + for (int e = 0; e < PY_MONITORING_UNGROUPED_EVENTS; e++) { + if ((data->local_monitors.tools[e] >> tool_id) & 1) { + event_set |= (1 << e); + } + } + } + return event_set; +} + +/*[clinic input] +monitoring.set_local_events + + tool_id: int + code: object + event_set: int + / + +[clinic start generated code]*/ + +static PyObject * +monitoring_set_local_events_impl(PyObject *module, int tool_id, + PyObject *code, int event_set) +/*[clinic end generated code: output=68cc755a65dfea99 input=5655ecd78d937a29]*/ +{ + if (!PyCode_Check(code)) { + PyErr_Format( + PyExc_TypeError, + "code must be a code object" + ); + return NULL; + } + if (check_valid_tool(tool_id)) { + return NULL; + } + if (event_set < 0 || event_set >= (1 << PY_MONITORING_EVENTS)) { + PyErr_Format(PyExc_ValueError, "invalid event set 0x%x", event_set); + return NULL; + } + if ((event_set & C_RETURN_EVENTS) && (event_set & C_CALL_EVENTS) != C_CALL_EVENTS) { + PyErr_Format(PyExc_ValueError, "cannot set C_RETURN or C_RAISE events independently"); + return NULL; + } + event_set &= ~C_RETURN_EVENTS; + if (_PyMonitoring_SetLocalEvents((PyCodeObject*)code, tool_id, event_set)) { + return NULL; + } + Py_RETURN_NONE; +} + +/*[clinic input] +monitoring.restart_events + +[clinic start generated code]*/ + +static PyObject * +monitoring_restart_events_impl(PyObject *module) +/*[clinic end generated code: output=e025dd5ba33314c4 input=add8a855063c8008]*/ +{ + /* We want to ensure that: + * last restart version > instrumented version for all code objects + * last restart version < current version + */ + PyInterpreterState *interp = _PyInterpreterState_Get(); + interp->last_restart_version = interp->monitoring_version + 1; + interp->monitoring_version = interp->last_restart_version + 1; + if (instrument_all_executing_code_objects(interp)) { + return NULL; + } + Py_RETURN_NONE; +} + +static int +add_power2_constant(PyObject *obj, const char *name, int i) +{ + PyObject *val = PyLong_FromLong(1<monitors.tools[e]; + if (tools == 0) { + continue; + } + PyObject *tools_obj = PyLong_FromLong(tools); + assert(tools_obj != NULL); + int err = PyDict_SetItemString(res, event_names[e], tools_obj); + Py_DECREF(tools_obj); + if (err < 0) { + Py_DECREF(res); + return NULL; + } + } + return res; +} + +static PyMethodDef methods[] = { + MONITORING_USE_TOOL_ID_METHODDEF + MONITORING_FREE_TOOL_ID_METHODDEF + MONITORING_GET_TOOL_METHODDEF + MONITORING_REGISTER_CALLBACK_METHODDEF + MONITORING_GET_EVENTS_METHODDEF + MONITORING_SET_EVENTS_METHODDEF + MONITORING_GET_LOCAL_EVENTS_METHODDEF + MONITORING_SET_LOCAL_EVENTS_METHODDEF + MONITORING_RESTART_EVENTS_METHODDEF + MONITORING__ALL_EVENTS_METHODDEF + {NULL, NULL} // sentinel +}; + +static struct PyModuleDef monitoring_module = { + PyModuleDef_HEAD_INIT, + .m_name = "sys.monitoring", + .m_size = -1, /* multiple "initialization" just copies the module dict. */ + .m_methods = methods, +}; + +PyObject *_Py_CreateMonitoringObject(void) +{ + PyObject *mod = _PyModule_CreateInitialized(&monitoring_module, PYTHON_API_VERSION); + if (mod == NULL) { + return NULL; + } + if (PyObject_SetAttrString(mod, "DISABLE", &DISABLE)) { + goto error; + } + if (PyObject_SetAttrString(mod, "MISSING", &_PyInstrumentation_MISSING)) { + goto error; + } + PyObject *events = _PyNamespace_New(NULL); + if (events == NULL) { + goto error; + } + int err = PyObject_SetAttrString(mod, "events", events); + Py_DECREF(events); + if (err) { + goto error; + } + for (int i = 0; i < PY_MONITORING_EVENTS; i++) { + if (add_power2_constant(events, event_names[i], i)) { + goto error; + } + } + err = PyObject_SetAttrString(events, "NO_EVENTS", _PyLong_GetZero()); + if (err) goto error; + PyObject *val = PyLong_FromLong(PY_MONITORING_DEBUGGER_ID); + err = PyObject_SetAttrString(mod, "DEBUGGER_ID", val); + Py_DECREF(val); + if (err) goto error; + val = PyLong_FromLong(PY_MONITORING_COVERAGE_ID); + err = PyObject_SetAttrString(mod, "COVERAGE_ID", val); + Py_DECREF(val); + if (err) goto error; + val = PyLong_FromLong(PY_MONITORING_PROFILER_ID); + err = PyObject_SetAttrString(mod, "PROFILER_ID", val); + Py_DECREF(val); + if (err) goto error; + val = PyLong_FromLong(PY_MONITORING_OPTIMIZER_ID); + err = PyObject_SetAttrString(mod, "OPTIMIZER_ID", val); + Py_DECREF(val); + if (err) goto error; + return mod; +error: + Py_DECREF(mod); + return NULL; +} diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c new file mode 100644 index 00000000000000..cf345bddda79b0 --- /dev/null +++ b/Python/legacy_tracing.c @@ -0,0 +1,528 @@ +/* Support for legacy tracing on top of PEP 669 instrumentation + * Provides callables to forward PEP 669 events to legacy events. + */ + +#include +#include "Python.h" +#include "pycore_ceval.h" +#include "pycore_object.h" +#include "pycore_sysmodule.h" + +typedef struct _PyLegacyEventHandler { + PyObject_HEAD + vectorcallfunc vectorcall; + int event; +} _PyLegacyEventHandler; + +/* The Py_tracefunc function expects the following arguments: + * obj: the trace object (PyObject *) + * frame: the current frame (PyFrameObject *) + * kind: the kind of event, see PyTrace_XXX #defines (int) + * arg: The arg (a PyObject *) + */ + +static PyObject * +call_profile_func(_PyLegacyEventHandler *self, PyObject *arg) +{ + PyThreadState *tstate = _PyThreadState_GET(); + if (tstate->c_profilefunc == NULL) { + Py_RETURN_NONE; + } + PyFrameObject *frame = PyEval_GetFrame(); + if (frame == NULL) { + PyErr_SetString(PyExc_SystemError, + "Missing frame when calling profile function."); + return NULL; + } + Py_INCREF(frame); + int err = tstate->c_profilefunc(tstate->c_profileobj, frame, self->event, arg); + Py_DECREF(frame); + if (err) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +sys_profile_func2( + _PyLegacyEventHandler *self, PyObject *const *args, + size_t nargsf, PyObject *kwnames +) { + assert(kwnames == NULL); + assert(PyVectorcall_NARGS(nargsf) == 2); + return call_profile_func(self, Py_None); +} + +static PyObject * +sys_profile_func3( + _PyLegacyEventHandler *self, PyObject *const *args, + size_t nargsf, PyObject *kwnames +) { + assert(kwnames == NULL); + assert(PyVectorcall_NARGS(nargsf) == 3); + return call_profile_func(self, args[2]); +} + +static PyObject * +sys_profile_call_or_return( + _PyLegacyEventHandler *self, PyObject *const *args, + size_t nargsf, PyObject *kwnames +) { + assert(kwnames == NULL); + assert(PyVectorcall_NARGS(nargsf) == 4); + PyObject *callable = args[2]; + if (PyCFunction_Check(callable)) { + return call_profile_func(self, callable); + } + if (Py_TYPE(callable) == &PyMethodDescr_Type) { + PyObject *self_arg = args[3]; + /* For backwards compatibility need to + * convert to builtin method */ + + /* If no arg, skip */ + if (self_arg == &_PyInstrumentation_MISSING) { + Py_RETURN_NONE; + } + PyObject *meth = Py_TYPE(callable)->tp_descr_get( + callable, self_arg, (PyObject*)Py_TYPE(self_arg)); + if (meth == NULL) { + return NULL; + } + PyObject *res = call_profile_func(self, meth); + Py_DECREF(meth); + return res; + } + Py_RETURN_NONE; +} + +static PyObject * +call_trace_func(_PyLegacyEventHandler *self, PyObject *arg) +{ + PyThreadState *tstate = _PyThreadState_GET(); + if (tstate->c_tracefunc == NULL) { + Py_RETURN_NONE; + } + PyFrameObject *frame = PyEval_GetFrame(); + if (frame == NULL) { + PyErr_SetString(PyExc_SystemError, + "Missing frame when calling trace function."); + return NULL; + } + Py_INCREF(frame); + int err = tstate->c_tracefunc(tstate->c_traceobj, frame, self->event, arg); + Py_DECREF(frame); + if (err) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +sys_trace_exception_func( + _PyLegacyEventHandler *self, PyObject *const *args, + size_t nargsf, PyObject *kwnames +) { + assert(kwnames == NULL); + assert(PyVectorcall_NARGS(nargsf) == 3); + PyObject *exc = args[2]; + assert(PyExceptionInstance_Check(exc)); + PyObject *type = (PyObject *)Py_TYPE(exc); + PyObject *tb = PyException_GetTraceback(exc); + if (tb == NULL) { + tb = Py_NewRef(Py_None); + } + PyObject *tuple = PyTuple_Pack(3, type, exc, tb); + Py_DECREF(tb); + if (tuple == NULL) { + return NULL; + } + PyObject *res = call_trace_func(self, tuple); + Py_DECREF(tuple); + return res; +} + +static PyObject * +sys_trace_func2( + _PyLegacyEventHandler *self, PyObject *const *args, + size_t nargsf, PyObject *kwnames +) { + assert(kwnames == NULL); + assert(PyVectorcall_NARGS(nargsf) == 2); + return call_trace_func(self, Py_None); +} + +static PyObject * +sys_trace_return( + _PyLegacyEventHandler *self, PyObject *const *args, + size_t nargsf, PyObject *kwnames +) { + assert(!PyErr_Occurred()); + assert(kwnames == NULL); + assert(PyVectorcall_NARGS(nargsf) == 3); + assert(PyCode_Check(args[0])); + PyObject *val = args[2]; + PyObject *res = call_trace_func(self, val); + return res; +} + +static PyObject * +sys_trace_yield( + _PyLegacyEventHandler *self, PyObject *const *args, + size_t nargsf, PyObject *kwnames +) { + assert(kwnames == NULL); + assert(PyVectorcall_NARGS(nargsf) == 3); + return call_trace_func(self, args[2]); +} + +static PyObject * +sys_trace_instruction_func( + _PyLegacyEventHandler *self, PyObject *const *args, + size_t nargsf, PyObject *kwnames +) { + assert(kwnames == NULL); + assert(PyVectorcall_NARGS(nargsf) == 2); + PyFrameObject *frame = PyEval_GetFrame(); + if (frame == NULL) { + PyErr_SetString(PyExc_SystemError, + "Missing frame when calling trace function."); + return NULL; + } + if (!frame->f_trace_opcodes) { + Py_RETURN_NONE; + } + Py_INCREF(frame); + PyThreadState *tstate = _PyThreadState_GET(); + int err = tstate->c_tracefunc(tstate->c_traceobj, frame, self->event, Py_None); + frame->f_lineno = 0; + Py_DECREF(frame); + if (err) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +trace_line( + PyThreadState *tstate, _PyLegacyEventHandler *self, + PyFrameObject *frame, int line +) { + if (!frame->f_trace_lines) { + Py_RETURN_NONE; + } + if (line < 0) { + Py_RETURN_NONE; + } + frame ->f_last_traced_line = line; + Py_INCREF(frame); + frame->f_lineno = line; + int err = tstate->c_tracefunc(tstate->c_traceobj, frame, self->event, Py_None); + frame->f_lineno = 0; + Py_DECREF(frame); + if (err) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +sys_trace_line_func( + _PyLegacyEventHandler *self, PyObject *const *args, + size_t nargsf, PyObject *kwnames +) { + assert(kwnames == NULL); + PyThreadState *tstate = _PyThreadState_GET(); + if (tstate->c_tracefunc == NULL) { + Py_RETURN_NONE; + } + assert(PyVectorcall_NARGS(nargsf) == 2); + int line = _PyLong_AsInt(args[1]); + assert(line >= 0); + PyFrameObject *frame = PyEval_GetFrame(); + if (frame == NULL) { + PyErr_SetString(PyExc_SystemError, + "Missing frame when calling trace function."); + return NULL; + } + assert(args[0] == (PyObject *)frame->f_frame->f_code); + if (frame ->f_last_traced_line == line) { + /* Already traced this line */ + Py_RETURN_NONE; + } + return trace_line(tstate, self, frame, line); +} + + +static PyObject * +sys_trace_jump_func( + _PyLegacyEventHandler *self, PyObject *const *args, + size_t nargsf, PyObject *kwnames +) { + assert(kwnames == NULL); + PyThreadState *tstate = _PyThreadState_GET(); + if (tstate->c_tracefunc == NULL) { + Py_RETURN_NONE; + } + assert(PyVectorcall_NARGS(nargsf) == 3); + int from = _PyLong_AsInt(args[1])/sizeof(_Py_CODEUNIT); + assert(from >= 0); + int to = _PyLong_AsInt(args[2])/sizeof(_Py_CODEUNIT); + assert(to >= 0); + PyFrameObject *frame = PyEval_GetFrame(); + if (frame == NULL) { + PyErr_SetString(PyExc_SystemError, + "Missing frame when calling trace function."); + return NULL; + } + if (!frame->f_trace_lines) { + Py_RETURN_NONE; + } + PyCodeObject *code = (PyCodeObject *)args[0]; + assert(PyCode_Check(code)); + assert(code == frame->f_frame->f_code); + /* We can call _Py_Instrumentation_GetLine because we always set + * line events for tracing */ + int to_line = _Py_Instrumentation_GetLine(code, to); + /* Backward jump: Always generate event + * Forward jump: Only generate event if jumping to different line. */ + if (to > from && frame->f_last_traced_line == to_line) { + /* Already traced this line */ + Py_RETURN_NONE; + } + return trace_line(tstate, self, frame, to_line); +} + +/* We don't care about the exception here, + * we just treat it as a possible new line + */ +static PyObject * +sys_trace_exception_handled( + _PyLegacyEventHandler *self, PyObject *const *args, + size_t nargsf, PyObject *kwnames +) { + assert(kwnames == NULL); + PyThreadState *tstate = _PyThreadState_GET(); + if (tstate->c_tracefunc == NULL) { + Py_RETURN_NONE; + } + assert(PyVectorcall_NARGS(nargsf) == 3); + PyFrameObject *frame = PyEval_GetFrame(); + PyCodeObject *code = (PyCodeObject *)args[0]; + assert(PyCode_Check(code)); + assert(code == frame->f_frame->f_code); + assert(PyLong_Check(args[1])); + int offset = _PyLong_AsInt(args[1])/sizeof(_Py_CODEUNIT); + /* We can call _Py_Instrumentation_GetLine because we always set + * line events for tracing */ + int line = _Py_Instrumentation_GetLine(code, offset); + if (frame->f_last_traced_line == line) { + /* Already traced this line */ + Py_RETURN_NONE; + } + return trace_line(tstate, self, frame, line); +} + + +PyTypeObject _PyLegacyEventHandler_Type = { + _PyVarObject_IMMORTAL_INIT(&PyType_Type, 0), + "sys.legacy_event_handler", + sizeof(_PyLegacyEventHandler), + .tp_dealloc = (destructor)PyObject_Free, + .tp_vectorcall_offset = offsetof(_PyLegacyEventHandler, vectorcall), + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_HAVE_VECTORCALL | Py_TPFLAGS_DISALLOW_INSTANTIATION, + .tp_call = PyVectorcall_Call, +}; + +static int +set_callbacks(int tool, vectorcallfunc vectorcall, int legacy_event, int event1, int event2) +{ + _PyLegacyEventHandler *callback = + PyObject_NEW(_PyLegacyEventHandler, &_PyLegacyEventHandler_Type); + if (callback == NULL) { + return -1; + } + callback->vectorcall = vectorcall; + callback->event = legacy_event; + Py_XDECREF(_PyMonitoring_RegisterCallback(tool, event1, (PyObject *)callback)); + if (event2 >= 0) { + Py_XDECREF(_PyMonitoring_RegisterCallback(tool, event2, (PyObject *)callback)); + } + Py_DECREF(callback); + return 0; +} + +#ifndef NDEBUG +/* Ensure that tstate is valid: sanity check for PyEval_AcquireThread() and + PyEval_RestoreThread(). Detect if tstate memory was freed. It can happen + when a thread continues to run after Python finalization, especially + daemon threads. */ +static int +is_tstate_valid(PyThreadState *tstate) +{ + assert(!_PyMem_IsPtrFreed(tstate)); + assert(!_PyMem_IsPtrFreed(tstate->interp)); + return 1; +} +#endif + +int +_PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) +{ + assert(is_tstate_valid(tstate)); + /* The caller must hold the GIL */ + assert(PyGILState_Check()); + + /* Call _PySys_Audit() in the context of the current thread state, + even if tstate is not the current thread state. */ + PyThreadState *current_tstate = _PyThreadState_GET(); + if (_PySys_Audit(current_tstate, "sys.setprofile", NULL) < 0) { + return -1; + } + /* Setup PEP 669 monitoring callbacks and events. */ + if (!tstate->interp->sys_profile_initialized) { + tstate->interp->sys_profile_initialized = true; + if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, + (vectorcallfunc)sys_profile_func2, PyTrace_CALL, + PY_MONITORING_EVENT_PY_START, PY_MONITORING_EVENT_PY_RESUME)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, + (vectorcallfunc)sys_profile_func3, PyTrace_RETURN, + PY_MONITORING_EVENT_PY_RETURN, PY_MONITORING_EVENT_PY_YIELD)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, + (vectorcallfunc)sys_profile_func2, PyTrace_RETURN, + PY_MONITORING_EVENT_PY_UNWIND, -1)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, + (vectorcallfunc)sys_profile_call_or_return, PyTrace_C_CALL, + PY_MONITORING_EVENT_CALL, -1)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, + (vectorcallfunc)sys_profile_call_or_return, PyTrace_C_RETURN, + PY_MONITORING_EVENT_C_RETURN, -1)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, + (vectorcallfunc)sys_profile_call_or_return, PyTrace_C_EXCEPTION, + PY_MONITORING_EVENT_C_RAISE, -1)) { + return -1; + } + } + + int delta = (func != NULL) - (tstate->c_profilefunc != NULL); + tstate->c_profilefunc = func; + PyObject *old_profileobj = tstate->c_profileobj; + tstate->c_profileobj = Py_XNewRef(arg); + Py_XDECREF(old_profileobj); + tstate->interp->sys_profiling_threads += delta; + assert(tstate->interp->sys_profiling_threads >= 0); + + uint32_t events = 0; + if (tstate->interp->sys_profiling_threads) { + events = + (1 << PY_MONITORING_EVENT_PY_START) | (1 << PY_MONITORING_EVENT_PY_RESUME) | + (1 << PY_MONITORING_EVENT_PY_RETURN) | (1 << PY_MONITORING_EVENT_PY_YIELD) | + (1 << PY_MONITORING_EVENT_CALL) | (1 << PY_MONITORING_EVENT_PY_UNWIND); + } + return _PyMonitoring_SetEvents(PY_MONITORING_SYS_PROFILE_ID, events); +} + +int +_PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) +{ + assert(is_tstate_valid(tstate)); + /* The caller must hold the GIL */ + assert(PyGILState_Check()); + + /* Call _PySys_Audit() in the context of the current thread state, + even if tstate is not the current thread state. */ + PyThreadState *current_tstate = _PyThreadState_GET(); + if (_PySys_Audit(current_tstate, "sys.settrace", NULL) < 0) { + return -1; + } + + assert(tstate->interp->sys_tracing_threads >= 0); + /* Setup PEP 669 monitoring callbacks and events. */ + if (!tstate->interp->sys_trace_initialized) { + tstate->interp->sys_trace_initialized = true; + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, + (vectorcallfunc)sys_trace_func2, PyTrace_CALL, + PY_MONITORING_EVENT_PY_START, PY_MONITORING_EVENT_PY_RESUME)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, + (vectorcallfunc)sys_trace_func2, PyTrace_CALL, + PY_MONITORING_EVENT_PY_THROW, -1)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, + (vectorcallfunc)sys_trace_return, PyTrace_RETURN, + PY_MONITORING_EVENT_PY_RETURN, -1)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, + (vectorcallfunc)sys_trace_yield, PyTrace_RETURN, + PY_MONITORING_EVENT_PY_YIELD, -1)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, + (vectorcallfunc)sys_trace_exception_func, PyTrace_EXCEPTION, + PY_MONITORING_EVENT_RAISE, PY_MONITORING_EVENT_STOP_ITERATION)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, + (vectorcallfunc)sys_trace_line_func, PyTrace_LINE, + PY_MONITORING_EVENT_LINE, -1)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, + (vectorcallfunc)sys_trace_func2, PyTrace_RETURN, + PY_MONITORING_EVENT_PY_UNWIND, -1)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, + (vectorcallfunc)sys_trace_jump_func, PyTrace_LINE, + PY_MONITORING_EVENT_JUMP, PY_MONITORING_EVENT_BRANCH)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, + (vectorcallfunc)sys_trace_instruction_func, PyTrace_OPCODE, + PY_MONITORING_EVENT_INSTRUCTION, -1)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, + (vectorcallfunc)sys_trace_exception_handled, PyTrace_LINE, + PY_MONITORING_EVENT_EXCEPTION_HANDLED, -1)) { + return -1; + } + } + + int delta = (func != NULL) - (tstate->c_tracefunc != NULL); + tstate->c_tracefunc = func; + PyObject *old_traceobj = tstate->c_traceobj; + tstate->c_traceobj = Py_XNewRef(arg); + Py_XDECREF(old_traceobj); + tstate->interp->sys_tracing_threads += delta; + assert(tstate->interp->sys_tracing_threads >= 0); + + uint32_t events = 0; + if (tstate->interp->sys_tracing_threads) { + events = + (1 << PY_MONITORING_EVENT_PY_START) | (1 << PY_MONITORING_EVENT_PY_RESUME) | + (1 << PY_MONITORING_EVENT_PY_RETURN) | (1 << PY_MONITORING_EVENT_PY_YIELD) | + (1 << PY_MONITORING_EVENT_RAISE) | (1 << PY_MONITORING_EVENT_LINE) | + (1 << PY_MONITORING_EVENT_JUMP) | (1 << PY_MONITORING_EVENT_BRANCH) | + (1 << PY_MONITORING_EVENT_PY_UNWIND) | (1 << PY_MONITORING_EVENT_PY_THROW) | + (1 << PY_MONITORING_EVENT_STOP_ITERATION) | + (1 << PY_MONITORING_EVENT_EXCEPTION_HANDLED); + if (tstate->interp->f_opcode_trace_set) { + events |= (1 << PY_MONITORING_EVENT_INSTRUCTION); + } + } + return _PyMonitoring_SetEvents(PY_MONITORING_SYS_TRACE_ID, events); +} diff --git a/Python/makeopcodetargets.py b/Python/makeopcodetargets.py index 33a4b4a76a1253..5aa31803397ce4 100755 --- a/Python/makeopcodetargets.py +++ b/Python/makeopcodetargets.py @@ -32,7 +32,6 @@ def write_contents(f): """ opcode = find_module('opcode') targets = ['_unknown_opcode'] * 256 - targets[255] = "TARGET_DO_TRACING" for opname, op in opcode.opmap.items(): if not opcode.is_pseudo(op): targets[op] = "TARGET_%s" % opname diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index f57b76aeb31050..4681ed03aff582 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -13,6 +13,8 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 0; case RESUME: return 0; + case INSTRUMENTED_RESUME: + return 0; case LOAD_CLOSURE: return 0; case LOAD_FAST_CHECK: @@ -39,6 +41,12 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 0; case END_FOR: return 1+1; + case INSTRUMENTED_END_FOR: + return 2; + case END_SEND: + return 2; + case INSTRUMENTED_END_SEND: + return 2; case UNARY_NEGATIVE: return 1; case UNARY_NOT: @@ -97,8 +105,12 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 1; case RETURN_VALUE: return 1; + case INSTRUMENTED_RETURN_VALUE: + return 1; case RETURN_CONST: return 0; + case INSTRUMENTED_RETURN_CONST: + return 0; case GET_AITER: return 1; case GET_ANEXT: @@ -109,6 +121,8 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 2; case SEND_GEN: return 2; + case INSTRUMENTED_YIELD_VALUE: + return 1; case YIELD_VALUE: return 1; case POP_EXCEPT: @@ -263,6 +277,8 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 1; case FOR_ITER: return 1; + case INSTRUMENTED_FOR_ITER: + return 0; case FOR_ITER_LIST: return 1; case FOR_ITER_TUPLE: @@ -287,6 +303,8 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 1; case KW_NAMES: return 0; + case INSTRUMENTED_CALL: + return 0; case CALL: return oparg + 2; case CALL_BOUND_METHOD_EXACT_ARGS: @@ -323,6 +341,8 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return oparg + 2; case CALL_NO_KW_METHOD_DESCRIPTOR_FAST: return oparg + 2; + case INSTRUMENTED_CALL_FUNCTION_EX: + return 0; case CALL_FUNCTION_EX: return ((oparg & 1) ? 1 : 0) + 3; case MAKE_FUNCTION: @@ -339,10 +359,28 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 2; case SWAP: return (oparg-2) + 2; + case INSTRUMENTED_LINE: + return 0; + case INSTRUMENTED_INSTRUCTION: + return 0; + case INSTRUMENTED_JUMP_FORWARD: + return 0; + case INSTRUMENTED_JUMP_BACKWARD: + return 0; + case INSTRUMENTED_POP_JUMP_IF_TRUE: + return 0; + case INSTRUMENTED_POP_JUMP_IF_FALSE: + return 0; + case INSTRUMENTED_POP_JUMP_IF_NONE: + return 0; + case INSTRUMENTED_POP_JUMP_IF_NOT_NONE: + return 0; case EXTENDED_ARG: return 0; case CACHE: return 0; + case RESERVED: + return 0; default: return -1; } @@ -359,6 +397,8 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 0; case RESUME: return 0; + case INSTRUMENTED_RESUME: + return 0; case LOAD_CLOSURE: return 1; case LOAD_FAST_CHECK: @@ -385,6 +425,12 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 1; case END_FOR: return 0+0; + case INSTRUMENTED_END_FOR: + return 0; + case END_SEND: + return 1; + case INSTRUMENTED_END_SEND: + return 1; case UNARY_NEGATIVE: return 1; case UNARY_NOT: @@ -443,8 +489,12 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 0; case RETURN_VALUE: return 0; + case INSTRUMENTED_RETURN_VALUE: + return 0; case RETURN_CONST: return 0; + case INSTRUMENTED_RETURN_CONST: + return 0; case GET_AITER: return 1; case GET_ANEXT: @@ -455,6 +505,8 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 2; case SEND_GEN: return 1; + case INSTRUMENTED_YIELD_VALUE: + return 1; case YIELD_VALUE: return 1; case POP_EXCEPT: @@ -609,6 +661,8 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 1; case FOR_ITER: return 2; + case INSTRUMENTED_FOR_ITER: + return 0; case FOR_ITER_LIST: return 2; case FOR_ITER_TUPLE: @@ -633,6 +687,8 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return ((oparg & 1) ? 1 : 0) + 1; case KW_NAMES: return 0; + case INSTRUMENTED_CALL: + return 0; case CALL: return 1; case CALL_BOUND_METHOD_EXACT_ARGS: @@ -669,6 +725,8 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 1; case CALL_NO_KW_METHOD_DESCRIPTOR_FAST: return 1; + case INSTRUMENTED_CALL_FUNCTION_EX: + return 0; case CALL_FUNCTION_EX: return 1; case MAKE_FUNCTION: @@ -685,10 +743,28 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 1; case SWAP: return (oparg-2) + 2; + case INSTRUMENTED_LINE: + return 0; + case INSTRUMENTED_INSTRUCTION: + return 0; + case INSTRUMENTED_JUMP_FORWARD: + return 0; + case INSTRUMENTED_JUMP_BACKWARD: + return 0; + case INSTRUMENTED_POP_JUMP_IF_TRUE: + return 0; + case INSTRUMENTED_POP_JUMP_IF_FALSE: + return 0; + case INSTRUMENTED_POP_JUMP_IF_NONE: + return 0; + case INSTRUMENTED_POP_JUMP_IF_NOT_NONE: + return 0; case EXTENDED_ARG: return 0; case CACHE: return 0; + case RESERVED: + return 0; default: return -1; } @@ -707,6 +783,7 @@ extern const struct opcode_metadata _PyOpcode_opcode_metadata[256]; const struct opcode_metadata _PyOpcode_opcode_metadata[256] = { [NOP] = { true, INSTR_FMT_IX }, [RESUME] = { true, INSTR_FMT_IB }, + [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB }, [LOAD_CLOSURE] = { true, INSTR_FMT_IB }, [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB }, [LOAD_FAST] = { true, INSTR_FMT_IB }, @@ -720,6 +797,9 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[256] = { [POP_TOP] = { true, INSTR_FMT_IX }, [PUSH_NULL] = { true, INSTR_FMT_IX }, [END_FOR] = { true, INSTR_FMT_IB }, + [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX }, + [END_SEND] = { true, INSTR_FMT_IX }, + [INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX }, [UNARY_NEGATIVE] = { true, INSTR_FMT_IX }, [UNARY_NOT] = { true, INSTR_FMT_IX }, [UNARY_INVERT] = { true, INSTR_FMT_IX }, @@ -749,12 +829,15 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[256] = { [RAISE_VARARGS] = { true, INSTR_FMT_IB }, [INTERPRETER_EXIT] = { true, INSTR_FMT_IX }, [RETURN_VALUE] = { true, INSTR_FMT_IX }, + [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX }, [RETURN_CONST] = { true, INSTR_FMT_IB }, + [INSTRUMENTED_RETURN_CONST] = { true, INSTR_FMT_IB }, [GET_AITER] = { true, INSTR_FMT_IX }, [GET_ANEXT] = { true, INSTR_FMT_IX }, [GET_AWAITABLE] = { true, INSTR_FMT_IB }, [SEND] = { true, INSTR_FMT_IBC }, [SEND_GEN] = { true, INSTR_FMT_IBC }, + [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IX }, [YIELD_VALUE] = { true, INSTR_FMT_IX }, [POP_EXCEPT] = { true, INSTR_FMT_IX }, [RERAISE] = { true, INSTR_FMT_IB }, @@ -832,6 +915,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[256] = { [GET_ITER] = { true, INSTR_FMT_IX }, [GET_YIELD_FROM_ITER] = { true, INSTR_FMT_IX }, [FOR_ITER] = { true, INSTR_FMT_IBC }, + [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IB }, [FOR_ITER_LIST] = { true, INSTR_FMT_IBC }, [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC }, [FOR_ITER_RANGE] = { true, INSTR_FMT_IBC }, @@ -844,6 +928,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[256] = { [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000 }, [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000 }, [KW_NAMES] = { true, INSTR_FMT_IB }, + [INSTRUMENTED_CALL] = { true, INSTR_FMT_IB }, [CALL] = { true, INSTR_FMT_IBC00 }, [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00 }, [CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00 }, @@ -862,6 +947,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[256] = { [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00 }, [CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = { true, INSTR_FMT_IBC00 }, [CALL_NO_KW_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00 }, + [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IX }, [CALL_FUNCTION_EX] = { true, INSTR_FMT_IB }, [MAKE_FUNCTION] = { true, INSTR_FMT_IB }, [RETURN_GENERATOR] = { true, INSTR_FMT_IX }, @@ -870,7 +956,16 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[256] = { [COPY] = { true, INSTR_FMT_IB }, [BINARY_OP] = { true, INSTR_FMT_IBC }, [SWAP] = { true, INSTR_FMT_IB }, + [INSTRUMENTED_LINE] = { true, INSTR_FMT_IX }, + [INSTRUMENTED_INSTRUCTION] = { true, INSTR_FMT_IX }, + [INSTRUMENTED_JUMP_FORWARD] = { true, INSTR_FMT_IB }, + [INSTRUMENTED_JUMP_BACKWARD] = { true, INSTR_FMT_IB }, + [INSTRUMENTED_POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IB }, + [INSTRUMENTED_POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IB }, + [INSTRUMENTED_POP_JUMP_IF_NONE] = { true, INSTR_FMT_IB }, + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IB }, [EXTENDED_ARG] = { true, INSTR_FMT_IB }, [CACHE] = { true, INSTR_FMT_IX }, + [RESERVED] = { true, INSTR_FMT_IX }, }; #endif diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index c502471bcd17b6..9d6616666f7ac1 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -4,17 +4,19 @@ static void *opcode_targets[256] = { &&TARGET_PUSH_NULL, &&TARGET_INTERPRETER_EXIT, &&TARGET_END_FOR, + &&TARGET_END_SEND, &&TARGET_BINARY_OP_ADD_FLOAT, &&TARGET_BINARY_OP_ADD_INT, &&TARGET_BINARY_OP_ADD_UNICODE, - &&TARGET_BINARY_OP_INPLACE_ADD_UNICODE, &&TARGET_NOP, - &&TARGET_BINARY_OP_MULTIPLY_FLOAT, + &&TARGET_BINARY_OP_INPLACE_ADD_UNICODE, &&TARGET_UNARY_NEGATIVE, &&TARGET_UNARY_NOT, + &&TARGET_BINARY_OP_MULTIPLY_FLOAT, &&TARGET_BINARY_OP_MULTIPLY_INT, - &&TARGET_BINARY_OP_SUBTRACT_FLOAT, &&TARGET_UNARY_INVERT, + &&TARGET_BINARY_OP_SUBTRACT_FLOAT, + &&TARGET_RESERVED, &&TARGET_BINARY_OP_SUBTRACT_INT, &&TARGET_BINARY_SUBSCR_DICT, &&TARGET_BINARY_SUBSCR_GETITEM, @@ -22,21 +24,21 @@ static void *opcode_targets[256] = { &&TARGET_BINARY_SUBSCR_TUPLE_INT, &&TARGET_CALL_PY_EXACT_ARGS, &&TARGET_CALL_PY_WITH_DEFAULTS, - &&TARGET_CALL_BOUND_METHOD_EXACT_ARGS, - &&TARGET_CALL_BUILTIN_CLASS, &&TARGET_BINARY_SUBSCR, &&TARGET_BINARY_SLICE, &&TARGET_STORE_SLICE, - &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS, - &&TARGET_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, + &&TARGET_CALL_BOUND_METHOD_EXACT_ARGS, + &&TARGET_CALL_BUILTIN_CLASS, &&TARGET_GET_LEN, &&TARGET_MATCH_MAPPING, &&TARGET_MATCH_SEQUENCE, &&TARGET_MATCH_KEYS, - &&TARGET_CALL_NO_KW_BUILTIN_FAST, + &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS, &&TARGET_PUSH_EXC_INFO, &&TARGET_CHECK_EXC_MATCH, &&TARGET_CHECK_EG_MATCH, + &&TARGET_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, + &&TARGET_CALL_NO_KW_BUILTIN_FAST, &&TARGET_CALL_NO_KW_BUILTIN_O, &&TARGET_CALL_NO_KW_ISINSTANCE, &&TARGET_CALL_NO_KW_LEN, @@ -46,8 +48,6 @@ static void *opcode_targets[256] = { &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_O, &&TARGET_CALL_NO_KW_STR_1, &&TARGET_CALL_NO_KW_TUPLE_1, - &&TARGET_CALL_NO_KW_TYPE_1, - &&TARGET_COMPARE_OP_FLOAT, &&TARGET_WITH_EXCEPT_START, &&TARGET_GET_AITER, &&TARGET_GET_ANEXT, @@ -55,39 +55,39 @@ static void *opcode_targets[256] = { &&TARGET_BEFORE_WITH, &&TARGET_END_ASYNC_FOR, &&TARGET_CLEANUP_THROW, + &&TARGET_CALL_NO_KW_TYPE_1, + &&TARGET_COMPARE_OP_FLOAT, &&TARGET_COMPARE_OP_INT, &&TARGET_COMPARE_OP_STR, - &&TARGET_FOR_ITER_LIST, - &&TARGET_FOR_ITER_TUPLE, &&TARGET_STORE_SUBSCR, &&TARGET_DELETE_SUBSCR, + &&TARGET_FOR_ITER_LIST, + &&TARGET_FOR_ITER_TUPLE, &&TARGET_FOR_ITER_RANGE, &&TARGET_FOR_ITER_GEN, &&TARGET_LOAD_ATTR_CLASS, &&TARGET_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, - &&TARGET_LOAD_ATTR_INSTANCE_VALUE, - &&TARGET_LOAD_ATTR_MODULE, &&TARGET_GET_ITER, &&TARGET_GET_YIELD_FROM_ITER, - &&TARGET_LOAD_ATTR_PROPERTY, + &&TARGET_LOAD_ATTR_INSTANCE_VALUE, &&TARGET_LOAD_BUILD_CLASS, - &&TARGET_LOAD_ATTR_SLOT, - &&TARGET_LOAD_ATTR_WITH_HINT, + &&TARGET_LOAD_ATTR_MODULE, + &&TARGET_LOAD_ATTR_PROPERTY, &&TARGET_LOAD_ASSERTION_ERROR, &&TARGET_RETURN_GENERATOR, + &&TARGET_LOAD_ATTR_SLOT, + &&TARGET_LOAD_ATTR_WITH_HINT, &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, &&TARGET_LOAD_ATTR_METHOD_NO_DICT, &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, &&TARGET_LOAD_CONST__LOAD_FAST, &&TARGET_LOAD_FAST__LOAD_CONST, + &&TARGET_RETURN_VALUE, &&TARGET_LOAD_FAST__LOAD_FAST, + &&TARGET_SETUP_ANNOTATIONS, &&TARGET_LOAD_GLOBAL_BUILTIN, - &&TARGET_RETURN_VALUE, &&TARGET_LOAD_GLOBAL_MODULE, - &&TARGET_SETUP_ANNOTATIONS, &&TARGET_STORE_ATTR_INSTANCE_VALUE, - &&TARGET_STORE_ATTR_SLOT, - &&TARGET_STORE_ATTR_WITH_HINT, &&TARGET_POP_EXCEPT, &&TARGET_STORE_NAME, &&TARGET_DELETE_NAME, @@ -110,9 +110,9 @@ static void *opcode_targets[256] = { &&TARGET_IMPORT_NAME, &&TARGET_IMPORT_FROM, &&TARGET_JUMP_FORWARD, + &&TARGET_STORE_ATTR_SLOT, + &&TARGET_STORE_ATTR_WITH_HINT, &&TARGET_STORE_FAST__LOAD_FAST, - &&TARGET_STORE_FAST__STORE_FAST, - &&TARGET_STORE_SUBSCR_DICT, &&TARGET_POP_JUMP_IF_FALSE, &&TARGET_POP_JUMP_IF_TRUE, &&TARGET_LOAD_GLOBAL, @@ -140,9 +140,9 @@ static void *opcode_targets[256] = { &&TARGET_STORE_DEREF, &&TARGET_DELETE_DEREF, &&TARGET_JUMP_BACKWARD, - &&TARGET_STORE_SUBSCR_LIST_INT, + &&TARGET_STORE_FAST__STORE_FAST, &&TARGET_CALL_FUNCTION_EX, - &&TARGET_UNPACK_SEQUENCE_LIST, + &&TARGET_STORE_SUBSCR_DICT, &&TARGET_EXTENDED_ARG, &&TARGET_LIST_APPEND, &&TARGET_SET_ADD, @@ -152,15 +152,15 @@ static void *opcode_targets[256] = { &&TARGET_YIELD_VALUE, &&TARGET_RESUME, &&TARGET_MATCH_CLASS, - &&TARGET_UNPACK_SEQUENCE_TUPLE, - &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, + &&TARGET_STORE_SUBSCR_LIST_INT, + &&TARGET_UNPACK_SEQUENCE_LIST, &&TARGET_FORMAT_VALUE, &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_STRING, + &&TARGET_UNPACK_SEQUENCE_TUPLE, + &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, &&TARGET_SEND_GEN, &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_LIST_EXTEND, &&TARGET_SET_UPDATE, &&TARGET_DICT_MERGE, @@ -237,22 +237,22 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&TARGET_DO_TRACING + &&TARGET_INSTRUMENTED_POP_JUMP_IF_NONE, + &&TARGET_INSTRUMENTED_POP_JUMP_IF_NOT_NONE, + &&TARGET_INSTRUMENTED_RESUME, + &&TARGET_INSTRUMENTED_CALL, + &&TARGET_INSTRUMENTED_RETURN_VALUE, + &&TARGET_INSTRUMENTED_YIELD_VALUE, + &&TARGET_INSTRUMENTED_CALL_FUNCTION_EX, + &&TARGET_INSTRUMENTED_JUMP_FORWARD, + &&TARGET_INSTRUMENTED_JUMP_BACKWARD, + &&TARGET_INSTRUMENTED_RETURN_CONST, + &&TARGET_INSTRUMENTED_FOR_ITER, + &&TARGET_INSTRUMENTED_POP_JUMP_IF_FALSE, + &&TARGET_INSTRUMENTED_POP_JUMP_IF_TRUE, + &&TARGET_INSTRUMENTED_END_FOR, + &&TARGET_INSTRUMENTED_END_SEND, + &&TARGET_INSTRUMENTED_INSTRUCTION, + &&TARGET_INSTRUMENTED_LINE, + &&_unknown_opcode }; diff --git a/Python/pystate.c b/Python/pystate.c index 1e59a8c5f89717..3818434c485194 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -637,7 +637,6 @@ free_interpreter(PyInterpreterState *interp) main interpreter. We fix those fields here, in addition to the other dynamically initialized fields. */ - static void init_interpreter(PyInterpreterState *interp, _PyRuntimeState *runtime, int64_t id, @@ -662,12 +661,22 @@ init_interpreter(PyInterpreterState *interp, _PyGC_InitState(&interp->gc); PyConfig_InitPythonConfig(&interp->config); _PyType_InitCache(interp); + for(int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { + interp->monitors.tools[i] = 0; + } + for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { + for(int e = 0; e < PY_MONITORING_EVENTS; e++) { + interp->monitoring_callables[t][e] = NULL; + } + } + interp->sys_profile_initialized = false; + interp->sys_trace_initialized = false; if (interp != &runtime->_main_interpreter) { /* Fix the self-referential, statically initialized fields. */ interp->dtoa = (struct _dtoa_state)_dtoa_state_INIT(interp); } - + interp->f_opcode_trace_set = false; interp->_initialized = 1; } @@ -800,6 +809,20 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) Py_CLEAR(interp->audit_hooks); + for(int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { + interp->monitors.tools[i] = 0; + } + for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { + for(int e = 0; e < PY_MONITORING_EVENTS; e++) { + Py_CLEAR(interp->monitoring_callables[t][e]); + } + } + interp->sys_profile_initialized = false; + interp->sys_trace_initialized = false; + for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { + Py_CLEAR(interp->monitoring_tool_names[t]); + } + PyConfig_Clear(&interp->config); Py_CLEAR(interp->codec_search_path); Py_CLEAR(interp->codec_search_cache); @@ -857,7 +880,7 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) interp->code_watchers[i] = NULL; } interp->active_code_watchers = 0; - + interp->f_opcode_trace_set = false; // XXX Once we have one allocator per interpreter (i.e. // per-interpreter GC) we must ensure that all of the interpreter's // objects have been cleaned up at the point. @@ -1249,6 +1272,7 @@ init_threadstate(PyThreadState *tstate, tstate->datastack_chunk = NULL; tstate->datastack_top = NULL; tstate->datastack_limit = NULL; + tstate->what_event = -1; tstate->_status.initialized = 1; } @@ -1424,8 +1448,14 @@ PyThreadState_Clear(PyThreadState *tstate) "PyThreadState_Clear: warning: thread still has a generator\n"); } - tstate->c_profilefunc = NULL; - tstate->c_tracefunc = NULL; + if (tstate->c_profilefunc != NULL) { + tstate->interp->sys_profiling_threads--; + tstate->c_profilefunc = NULL; + } + if (tstate->c_tracefunc != NULL) { + tstate->interp->sys_tracing_threads--; + tstate->c_tracefunc = NULL; + } Py_CLEAR(tstate->c_profileobj); Py_CLEAR(tstate->c_traceobj); diff --git a/Python/specialize.c b/Python/specialize.c index a9d3226ee39f5f..3fa28f409892dc 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -273,7 +273,8 @@ _PyCode_Quicken(PyCodeObject *code) _Py_CODEUNIT *instructions = _PyCode_CODE(code); for (int i = 0; i < Py_SIZE(code); i++) { int previous_opcode = opcode; - opcode = _PyOpcode_Deopt[instructions[i].op.code]; + opcode = _Py_GetBaseOpcode(code, i); + assert(opcode < MIN_INSTRUMENTED_OPCODE); int caches = _PyOpcode_Caches[opcode]; if (caches) { instructions[i + 1].cache = adaptive_counter_warmup(); @@ -1737,6 +1738,7 @@ _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, { assert(ENABLE_SPECIALIZATION); assert(_PyOpcode_Caches[CALL] == INLINE_CACHE_ENTRIES_CALL); + assert(_Py_OPCODE(*instr) != INSTRUMENTED_CALL); _PyCallCache *cache = (_PyCallCache *)(instr + 1); int fail; if (PyCFunction_CheckExact(callable)) { @@ -2149,7 +2151,9 @@ _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr, int oparg) goto success; } else if (tp == &PyGen_Type && oparg <= SHRT_MAX) { - assert(instr[oparg + INLINE_CACHE_ENTRIES_FOR_ITER + 1].op.code == END_FOR); + assert(instr[oparg + INLINE_CACHE_ENTRIES_FOR_ITER + 1].op.code == END_FOR || + instr[oparg + INLINE_CACHE_ENTRIES_FOR_ITER + 1].op.code == INSTRUMENTED_END_FOR + ); instr->op.code = FOR_ITER_GEN; goto success; } diff --git a/Python/sysmodule.c b/Python/sysmodule.c index f1a294de598420..4d693a1be1f89e 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -3409,6 +3409,7 @@ _PySys_SetPreliminaryStderr(PyObject *sysdict) return _PyStatus_ERR("can't set preliminary stderr"); } +PyObject *_Py_CreateMonitoringObject(void); /* Create sys module without all attributes. _PySys_UpdateConfig() should be called later to add remaining attributes. */ @@ -3458,6 +3459,16 @@ _PySys_Create(PyThreadState *tstate, PyObject **sysmod_p) goto error; } + PyObject *monitoring = _Py_CreateMonitoringObject(); + if (monitoring == NULL) { + goto error; + } + int err = PyDict_SetItemString(sysdict, "monitoring", monitoring); + Py_DECREF(monitoring); + if (err < 0) { + goto error; + } + assert(!_PyErr_Occurred(tstate)); *sysmod_p = sysmod; diff --git a/Tools/build/deepfreeze.py b/Tools/build/deepfreeze.py index ec808526b7bbb7..aba5fecd8b1a99 100644 --- a/Tools/build/deepfreeze.py +++ b/Tools/build/deepfreeze.py @@ -255,7 +255,6 @@ def generate_code(self, name: str, code: types.CodeType) -> str: self.write(f".co_names = {co_names},") self.write(f".co_exceptiontable = {co_exceptiontable},") self.field(code, "co_flags") - self.write("._co_linearray_entry_size = 0,") self.field(code, "co_argcount") self.field(code, "co_posonlyargcount") self.field(code, "co_kwonlyargcount") @@ -276,7 +275,6 @@ def generate_code(self, name: str, code: types.CodeType) -> str: self.write(f".co_qualname = {co_qualname},") self.write(f".co_linetable = {co_linetable},") self.write(f"._co_cached = NULL,") - self.write("._co_linearray = NULL,") self.write(f".co_code_adaptive = {co_code_adaptive},") for i, op in enumerate(code.co_code[::2]): if op == RESUME: diff --git a/Tools/build/generate_opcode_h.py b/Tools/build/generate_opcode_h.py index 614b17df740929..ccf3c5673c4e18 100644 --- a/Tools/build/generate_opcode_h.py +++ b/Tools/build/generate_opcode_h.py @@ -89,6 +89,7 @@ def main(opcode_py, outfile='Include/opcode.h', internaloutfile='Include/interna HAVE_ARGUMENT = opcode["HAVE_ARGUMENT"] MIN_PSEUDO_OPCODE = opcode["MIN_PSEUDO_OPCODE"] MAX_PSEUDO_OPCODE = opcode["MAX_PSEUDO_OPCODE"] + MIN_INSTRUMENTED_OPCODE = opcode["MIN_INSTRUMENTED_OPCODE"] NUM_OPCODES = len(opname) used = [ False ] * len(opname) @@ -105,9 +106,6 @@ def main(opcode_py, outfile='Include/opcode.h', internaloutfile='Include/interna specialized_opmap[name] = next_op opname_including_specialized[next_op] = name used[next_op] = True - specialized_opmap['DO_TRACING'] = 255 - opname_including_specialized[255] = 'DO_TRACING' - used[255] = True with open(outfile, 'w') as fobj, open(internaloutfile, 'w') as iobj: fobj.write(header) @@ -120,6 +118,8 @@ def main(opcode_py, outfile='Include/opcode.h', internaloutfile='Include/interna fobj.write(DEFINE.format("HAVE_ARGUMENT", HAVE_ARGUMENT)) if op == MIN_PSEUDO_OPCODE: fobj.write(DEFINE.format("MIN_PSEUDO_OPCODE", MIN_PSEUDO_OPCODE)) + if op == MIN_INSTRUMENTED_OPCODE: + fobj.write(DEFINE.format("MIN_INSTRUMENTED_OPCODE", MIN_INSTRUMENTED_OPCODE)) fobj.write(DEFINE.format(name, op)) diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 0620c7e13925b5..8745f8abc224fe 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -135,6 +135,9 @@ Objects/stringlib/unicode_format.h - PyFieldNameIter_Type - Objects/unicodeobject.c - EncodingMapType - #Objects/unicodeobject.c - PyFieldNameIter_Type - #Objects/unicodeobject.c - PyFormatterIter_Type - +Python/legacy_tracing.c - _PyLegacyEventHandler_Type - +Objects/object.c - _PyLegacyEventHandler_Type - + ##----------------------- ## static builtin structseq @@ -297,6 +300,8 @@ Objects/object.c - _Py_NotImplementedStruct - Objects/setobject.c - _dummy_struct - Objects/setobject.c - _PySet_Dummy - Objects/sliceobject.c - _Py_EllipsisObject - +Python/instrumentation.c - DISABLE - +Python/instrumentation.c - _PyInstrumentation_MISSING - ################################## pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy