From e85d910a0f9934db1685101f8fdb746b6bb471e7 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 9 Nov 2022 16:40:46 +0000 Subject: [PATCH 001/116] Initial experimental implementation of PEP 669. --- Include/cpython/code.h | 1 + Include/internal/pycore_code.h | 2 ++ Include/internal/pycore_interp.h | 7 +++++++ Objects/frameobject.c | 27 ++++++++++++++++++++++++++- Python/bytecodes.c | 6 ++++++ Python/pystate.c | 6 +++++- 6 files changed, 47 insertions(+), 2 deletions(-) diff --git a/Include/cpython/code.h b/Include/cpython/code.h index ebac0b12a461bc..da05f316226e69 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -98,6 +98,7 @@ typedef struct { PyObject *co_weakreflist; /* to support weakrefs to code objects */ \ _PyCoCached *_co_cached; /* cached co_* attributes */ \ int _co_firsttraceable; /* index of first traceable instruction */ \ + uint64_t _co_instrument_version; /* current instrumentation version */ \ char *_co_linearray; /* array of line offsets */ \ /* Scratch space for extra data relating to the code object. \ Type is a void* to keep the format private in codeobject.c to force \ diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 0af240ca362103..81f0dd3dd4811d 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -466,6 +466,8 @@ typedef struct _PyShimCodeDef { extern PyCodeObject * _Py_MakeShimCode(const _PyShimCodeDef *code); +extern int _Py_Instrument(PyCodeObject *co, PyInterpreterState *interp); + #ifdef __cplusplus } diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index ae2a3d3b13cfa9..1b06933263eef7 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -19,6 +19,7 @@ extern "C" { #include "pycore_floatobject.h" // struct _Py_float_state #include "pycore_genobject.h" // struct _Py_async_gen_state #include "pycore_gc.h" // struct _gc_runtime_state +#include "pycore_instruments.h" // PY_INSTRUMENT_EVENTS #include "pycore_list.h" // struct _Py_list_state #include "pycore_tuple.h" // struct _Py_tuple_state #include "pycore_typeobject.h" // struct type_cache @@ -85,6 +86,8 @@ struct _is { PyInterpreterState *next; + uint64_t instrument_version; + struct pythreads { uint64_t next_unique_id; /* The linked list of threads, newest first. */ @@ -188,6 +191,10 @@ struct _is { struct callable_cache callable_cache; PyCodeObject *interpreter_trampoline; + uint8_t instrumented[PY_INSTRUMENT_EVENTS]; + PyObject *instrument_callables[PY_INSTRUMENT_EVENTS]; + bool f_opcode_trace_set; + /* The following fields are here to avoid allocation during init. The data is exposed through PyInterpreterState pointer fields. These fields should not be accessed directly outside of init. diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 287d5bd9ad1afd..4861f80a0ca88e 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -17,7 +17,6 @@ static PyMemberDef frame_memberlist[] = { {"f_trace_lines", T_BOOL, OFF(f_trace_lines), 0}, - {"f_trace_opcodes", T_BOOL, OFF(f_trace_opcodes), 0}, {NULL} /* Sentinel */ }; @@ -104,6 +103,31 @@ frame_getback(PyFrameObject *f, void *closure) return res; } +static PyObject * +frame_gettrace_opcodes(PyFrameObject *f, void *closure) +{ + PyObject *result = f->f_trace_opcodes ? Py_True : Py_False; + return Py_NewRef(result); +} + +static int +frame_settrace_opcodes(PyFrameObject *f, PyObject* value, void *Py_UNUSED(ignored)) +{ + if (!PyBool_Check(value)) { + PyErr_SetString(PyExc_TypeError, + "attribute value type must be bool"); + return -1; + } + if (value == Py_True) { + f->f_trace_opcodes = 1; + _PyInterpreterState_GET()->f_opcode_trace_set = true; + } + else { + f->f_trace_opcodes = 0; + } + return 0; +} + // Given the index of the effective opcode, scan back to construct the oparg // with EXTENDED_ARG. This only works correctly with *unquickened* code, // obtained via a call to _PyCode_GetCode! @@ -846,6 +870,7 @@ static PyGetSetDef frame_getsetlist[] = { {"f_globals", (getter)frame_getglobals, NULL, NULL}, {"f_builtins", (getter)frame_getbuiltins, NULL, NULL}, {"f_code", (getter)frame_getcode, NULL, NULL}, + {"f_trace_opcodes", (getter)frame_gettrace_opcodes, (setter)frame_settrace_opcodes, NULL}, {0} }; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index da3ab9126eeeea..a30bfacb6e5e44 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -110,6 +110,12 @@ dummy_func( inst(RESUME, (--)) { assert(tstate->cframe == &cframe); assert(frame == cframe.current_frame); + /* Possibly combine this with eval breaker */ + if (frame->f_code->_co_instrument_version != tstate->interp->instrument_version) { + if (_Py_Instrument(frame->f_code, tstate->interp)) { + goto error; + } + } if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { goto handle_eval_breaker; } diff --git a/Python/pystate.c b/Python/pystate.c index 04db1fb419af62..1c92ce5d954458 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -309,7 +309,11 @@ init_interpreter(PyInterpreterState *interp, _PyGC_InitState(&interp->gc); PyConfig_InitPythonConfig(&interp->config); _PyType_InitCache(interp); - + for(int i = 0; i < PY_INSTRUMENT_EVENTS; i++) { + interp->instrumented[i] = 0; + interp->instrument_callables[i] = Py_NewRef(Py_None); + } + interp->f_opcode_trace_set = false; interp->_initialized = 1; } From 6283ee898bf205d8a8103977bdf6d67a77d8f4c1 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 18 Nov 2022 12:32:59 +0000 Subject: [PATCH 002/116] First draft implementing small subset of PEP 669. --- Include/cpython/pystate.h | 1 - Include/internal/pycore_instruments.h | 80 ++++ Include/internal/pycore_interp.h | 21 +- Include/internal/pycore_opcode.h | 20 +- Include/internal/pycore_pystate.h | 4 - Include/opcode.h | 5 + Lib/opcode.py | 8 + Lib/profile.py | 4 +- Lib/test/test_monitoring.py | 271 ++++++++++++ Makefile.pre.in | 2 + Objects/codeobject.c | 1 + Python/bytecodes.c | 185 +++++---- Python/ceval.c | 570 ++----------------------- Python/clinic/instrumentation.c.h | 196 +++++++++ Python/generated_cases.c.h | 182 ++++---- Python/instrumentation.c | 573 ++++++++++++++++++++++++++ Python/legacy_tracing.c | 302 ++++++++++++++ Python/opcode_targets.h | 10 +- Python/pystate.c | 12 +- Python/specialize.c | 1 + Python/sysmodule.c | 9 + 21 files changed, 1748 insertions(+), 709 deletions(-) create mode 100644 Include/internal/pycore_instruments.h create mode 100644 Lib/test/test_monitoring.py create mode 100644 Python/clinic/instrumentation.c.h create mode 100644 Python/instrumentation.c create mode 100644 Python/legacy_tracing.c diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 70c23427807e52..99bc0a89dc0e5f 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -74,7 +74,6 @@ typedef struct _PyCFrame { * discipline and make sure that instances of this struct cannot * accessed outside of their lifetime. */ - uint8_t use_tracing; // 0 or 255 (or'ed into opcode, hence 8-bit type) /* Pointer to the currently executing frame (it can be NULL) */ struct _PyInterpreterFrame *current_frame; struct _PyCFrame *previous; diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h new file mode 100644 index 00000000000000..4c3a1dbcb3a4cb --- /dev/null +++ b/Include/internal/pycore_instruments.h @@ -0,0 +1,80 @@ + +#ifndef Py_INTERNAL_INSTRUMENT_H +#define Py_INTERNAL_INSTRUMENT_H + + +#include "pycore_bitutils.h" // _Py_popcount32 +#include "pycore_frame.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define PY_MONITORING_TOOL_IDS 8 + +/* Require dynamic instrumentation */ + +#define PY_MONITORING_EVENT_PY_START 0 +#define PY_MONITORING_EVENT_PY_RESUME 1 +#define PY_MONITORING_EVENT_PY_RETURN 2 +#define PY_MONITORING_EVENT_PY_YIELD 3 +#define PY_MONITORING_EVENT_CALL 4 +#define PY_MONITORING_EVENT_LINE 5 +#define PY_MONITORING_EVENT_INSTRUCTION 6 +#define PY_MONITORING_EVENT_JUMP 7 +#define PY_MONITORING_EVENT_BRANCH 8 + + +#define PY_MONITORING_EVENT_C_RETURN 9 +#define PY_MONITORING_EVENT_PY_THROW 10 +#define PY_MONITORING_EVENT_RAISE 11 +#define PY_MONITORING_EVENT_EXCEPTION_HANDLED 12 +#define PY_MONITORING_EVENT_C_RAISE 13 +#define PY_MONITORING_EVENT_PY_UNWIND 14 + +/* #define INSTRUMENT_EVENT_BRANCH_NOT_TAKEN xxx -- If we can afford this */ +#define PY_MONITORING_EVENTS 15 + +/* Temporary and internal events */ + +// #define PY_INSTRUMENT_PEP_523 50 +/* #define PY_INSTRUMENT_JIT_API 17 -- Reserved */ + +typedef uint32_t _PyMonitoringEventSet; + +typedef uint16_t _PyMonitoringSet; + +typedef struct _pytoolset { + uint8_t tools; +} _PyMonitoringToolSet; + +/* Reserved IDs */ + +#define PY_INSTRUMENT_PEP_523 5 +#define PY_INSTRUMENT_SYS_PROFILE 6 +#define PY_INSTRUMENT_SYS_TRACE 7 + + +/* API functions */ + +PyObject *_PyMonitoring_RegisterCallback(int tool_id, int event_id, PyObject *obj); + +void _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events); + +extern int +_Py_call_instrumentation(PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr); + +extern int +_Py_call_instrumentation_arg(PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg); + +extern void +_Py_call_instrumentation_exc(PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg); + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_INSTRUMENT_H */ diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 1b06933263eef7..1a606a72ae2fd1 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -19,7 +19,7 @@ extern "C" { #include "pycore_floatobject.h" // struct _Py_float_state #include "pycore_genobject.h" // struct _Py_async_gen_state #include "pycore_gc.h" // struct _gc_runtime_state -#include "pycore_instruments.h" // PY_INSTRUMENT_EVENTS +#include "pycore_instruments.h" // PY_MONITORING_EVENTS #include "pycore_list.h" // struct _Py_list_state #include "pycore_tuple.h" // struct _Py_tuple_state #include "pycore_typeobject.h" // struct type_cache @@ -74,6 +74,9 @@ struct _Py_long_state { int max_str_digits; }; +struct _instrumentation_tool { + PyObject *instrument_callables[PY_MONITORING_EVENTS]; +}; /* interpreter state */ @@ -86,7 +89,7 @@ struct _is { PyInterpreterState *next; - uint64_t instrument_version; + uint64_t monitoring_version; struct pythreads { uint64_t next_unique_id; @@ -191,9 +194,19 @@ struct _is { struct callable_cache callable_cache; PyCodeObject *interpreter_trampoline; - uint8_t instrumented[PY_INSTRUMENT_EVENTS]; - PyObject *instrument_callables[PY_INSTRUMENT_EVENTS]; + _PyMonitoringToolSet monitoring_tools_per_event[PY_MONITORING_EVENTS]; + _PyMonitoringEventSet events_per_monitoring_tool[PY_MONITORING_TOOL_IDS]; + _PyMonitoringEventSet monitored_events; + /* The index (plus one) of the sole tool. 0 if 0 or 2+ tools */ + int8_t sole_tool_plus1[PY_MONITORING_EVENTS]; + /* Tools numbered 1-8. 0 is the dispatcher/sole tool */ + struct _instrumentation_tool tools[PY_MONITORING_TOOL_IDS]; + PyObject *monitoring_tool_names[PY_MONITORING_TOOL_IDS]; bool f_opcode_trace_set; + bool sys_profile_initialized; + bool sys_trace_initialized; + Py_ssize_t sys_profiling_threads; + Py_ssize_t sys_tracing_threads; /* The following fields are here to avoid allocation during init. The data is exposed through PyInterpreterState pointer fields. diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index 3f44511240aa14..86e7d7bb9511b4 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -136,6 +136,11 @@ const uint8_t _PyOpcode_Deopt[256] = { [IMPORT_FROM] = IMPORT_FROM, [IMPORT_NAME] = IMPORT_NAME, [IMPORT_STAR] = IMPORT_STAR, + [INSTRUMENTED_CALL] = INSTRUMENTED_CALL, + [INSTRUMENTED_CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, + [INSTRUMENTED_RESUME] = INSTRUMENTED_RESUME, + [INSTRUMENTED_RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE, + [INSTRUMENTED_YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE, [INTERPRETER_EXIT] = INTERPRETER_EXIT, [IS_OP] = IS_OP, [JUMP_BACKWARD] = JUMP_BACKWARD, @@ -473,11 +478,11 @@ static const char *const _PyOpcode_OpName[263] = { [237] = "<237>", [238] = "<238>", [239] = "<239>", - [240] = "<240>", - [241] = "<241>", - [242] = "<242>", - [243] = "<243>", - [244] = "<244>", + [INSTRUMENTED_RESUME] = "INSTRUMENTED_RESUME", + [INSTRUMENTED_CALL] = "INSTRUMENTED_CALL", + [INSTRUMENTED_RETURN_VALUE] = "INSTRUMENTED_RETURN_VALUE", + [INSTRUMENTED_YIELD_VALUE] = "INSTRUMENTED_YIELD_VALUE", + [INSTRUMENTED_CALL_FUNCTION_EX] = "INSTRUMENTED_CALL_FUNCTION_EX", [245] = "<245>", [246] = "<246>", [247] = "<247>", @@ -566,11 +571,6 @@ static const char *const _PyOpcode_OpName[263] = { case 237: \ case 238: \ case 239: \ - case 240: \ - case 241: \ - case 242: \ - case 243: \ - case 244: \ case 245: \ case 246: \ case 247: \ diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 3d6d400f74dd1d..66cbc96016cb13 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -132,10 +132,6 @@ PyAPI_FUNC(void) _PyThreadState_DeleteExcept( static inline void _PyThreadState_UpdateTracingState(PyThreadState *tstate) { - bool use_tracing = - (tstate->tracing == 0) && - (tstate->c_tracefunc != NULL || tstate->c_profilefunc != NULL); - tstate->cframe->use_tracing = (use_tracing ? 255 : 0); } diff --git a/Include/opcode.h b/Include/opcode.h index c18d9c057b849b..70f8b1ea9a3939 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -120,6 +120,11 @@ extern "C" { #define DICT_UPDATE 165 #define CALL 171 #define KW_NAMES 172 +#define INSTRUMENTED_RESUME 240 +#define INSTRUMENTED_CALL 241 +#define INSTRUMENTED_RETURN_VALUE 242 +#define INSTRUMENTED_YIELD_VALUE 243 +#define INSTRUMENTED_CALL_FUNCTION_EX 244 #define MIN_PSEUDO_OPCODE 256 #define SETUP_FINALLY 256 #define SETUP_CLEANUP 257 diff --git a/Lib/opcode.py b/Lib/opcode.py index 0ee75958508ac7..fb240a7472ab86 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -222,6 +222,14 @@ def pseudo_op(name, op, real_ops): hasconst.append(172) +# Instrumented instructions +def_op('INSTRUMENTED_RESUME', 240) +def_op('INSTRUMENTED_CALL', 241) +def_op('INSTRUMENTED_RETURN_VALUE', 242) +def_op('INSTRUMENTED_YIELD_VALUE', 243) +def_op('INSTRUMENTED_CALL_FUNCTION_EX', 244) + + hasarg.extend([op for op in opmap.values() if op >= HAVE_ARGUMENT]) MIN_PSEUDO_OPCODE = 256 diff --git a/Lib/profile.py b/Lib/profile.py index 453e56285c510c..747ae117dae437 100755 --- a/Lib/profile.py +++ b/Lib/profile.py @@ -187,7 +187,7 @@ def trace_dispatch(self, frame, event, arg): if event == "c_call": self.c_func_name = arg.__name__ - + print(event, file = sys.stderr) if self.dispatch[event](self, frame,t): t = timer() self.t = t[0] + t[1] @@ -289,7 +289,7 @@ def trace_dispatch_c_call (self, frame, t): def trace_dispatch_return(self, frame, t): if frame is not self.cur[-2]: - assert frame is self.cur[-2].f_back, ("Bad return", self.cur[-3]) + assert frame is self.cur[-2].f_back, ("Bad return", self.cur[-3], t) self.trace_dispatch_return(self.cur[-2], 0) # Prefix "r" means part of the Returning or exiting frame. diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py new file mode 100644 index 00000000000000..4e3428effafc7f --- /dev/null +++ b/Lib/test/test_monitoring.py @@ -0,0 +1,271 @@ +"""Test suite for the sys.monitoring.""" + +import sys +import unittest +import enum +import operator +import functools +import types +import collections + +def f1(): + pass + +def f2(): + len([]) + sys.getsizeof(0) + +def gen(): + yield + yield + +def g1(): + for _ in gen(): + pass + +TEST_TOOL = 4 + +class MonitoringBaseTest(unittest.TestCase): + + def test_has_objects(self): + m = sys.monitoring + m.events + m.use_tool + m.free_tool + m.get_tool + m.get_events + m.set_events + # m.get_local_events + # m.set_local_events + m.register_callback + # m.insert_marker + # m.remove_marker + # m.restart_events + m.DISABLE + + def test_tool(self): + sys.monitoring.use_tool(TEST_TOOL, "MonitoringTest.Tool") + self.assertEqual(sys.monitoring.get_tool(TEST_TOOL), "MonitoringTest.Tool") + sys.monitoring.free_tool(TEST_TOOL) + self.assertEqual(sys.monitoring.get_tool(TEST_TOOL), None) + sys.monitoring.set_events(TEST_TOOL, 19) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL), 19) + sys.monitoring.set_events(TEST_TOOL, 0) + + +class MonitoringCountTest(unittest.TestCase): + + def check_event_count(self, func, event, expected): + + class Counter: + def __init__(self): + self.count = 0 + def __call__(self, *args): + self.count += 1 + + counter = Counter() + sys.monitoring.register_callback(TEST_TOOL, event, counter) + sys.monitoring.set_events(TEST_TOOL, event) + self.assertEqual(counter.count, 0) + counter.count = 0 + func() + self.assertEqual(counter.count, expected) + prev = sys.monitoring.register_callback(TEST_TOOL, event, None) + counter.count = 0 + func() + self.assertEqual(counter.count, 0) + self.assertEqual(prev, counter) + sys.monitoring.set_events(TEST_TOOL, 0) + + def test_start_count(self): + self.check_event_count(f1, E.PY_START, 1) + + def test_resume_count(self): + self.check_event_count(g1, E.PY_RESUME, 2) + + def test_return_count(self): + self.check_event_count(f1, E.PY_RETURN, 1) + + def test_call_count(self): + self.check_event_count(f2, E.CALL, 3) + + def test_c_return_count(self): + self.check_event_count(f2, E.C_RETURN, 2) + + +E = sys.monitoring.events + +SIMPLE_EVENTS = [ + (E.PY_START, "start"), + (E.PY_RESUME, "resume"), + (E.PY_RETURN, "return"), + (E.PY_YIELD, "yield"), + (E.C_RAISE, "c_raise"), + (E.C_RETURN, "c_return"), + (E.JUMP, "jump"), + (E.BRANCH, "branch"), + (E.RAISE, "raise"), + (E.PY_UNWIND, "unwind"), + (E.EXCEPTION_HANDLED, "exception_handled"), +] + +SIMPLE_EVENT_SET = functools.reduce(operator.or_, [ev for (ev, _) in SIMPLE_EVENTS], 0) | E.CALL + +def just_pass(): + pass + +just_pass.events = [ + "py_call", + "start", + "return", +] + +def just_raise(): + raise Exception + +just_raise.events = [ + 'py_call', + "start", + "raise", + "unwind", +] + +def just_call(): + len([]) + +just_call.events = [ + 'py_call', + "start", + "c_call", + "c_return", + "return", +] + +def caught(): + try: + 1/0 + except Exception: + pass + +caught.events = [ + 'py_call', + "start", + "raise", + "exception_handled", + "return", +] + +def nested_call(): + just_pass() + +nested_call.events = [ + "py_call", + "start", + "py_call", + "start", + "return", + "return", +] + +PY_CALLABLES = (types.FunctionType, types.MethodType) + +class MonitoringEventsBase: + + def gather_events(self, func): + events = [] + for event, event_name in SIMPLE_EVENTS: + def record(*args, event_name=event_name): + events.append(event_name) + sys.monitoring.register_callback(TEST_TOOL, event, record) + def record_call(code, offset, obj): + if isinstance(obj, PY_CALLABLES): + events.append("py_call") + else: + events.append("c_call") + sys.monitoring.register_callback(TEST_TOOL, E.CALL, record_call) + sys.monitoring.set_events(TEST_TOOL, SIMPLE_EVENT_SET) + events = [] + try: + func() + except: + pass + sys.monitoring.set_events(TEST_TOOL, 0) + #Remove the final event, the call to `sys.monitoring.set_events` + events = events[:-1] + return events + + def check_events(self, func, expected=None): + events = self.gather_events(func) + if expected is None: + expected = func.events + self.assertEqual(events, expected) + + +class MonitoringEventsTest(unittest.TestCase, MonitoringEventsBase): + + def test_just_pass(self): + self.check_events(just_pass) + + def test_just_raise(self): + try: + self.check_events(just_raise) + except Exception: + pass + self.assertEqual(sys.monitoring.get_events(TEST_TOOL), 0) + + def test_just_call(self): + self.check_events(just_call) + + def test_caught(self): + self.check_events(caught) + + def test_nested_call(self): + self.check_events(nested_call) + +UP_EVENTS = (E.C_RETURN, E.C_RAISE, E.PY_RETURN, E.PY_UNWIND, E.PY_YIELD) +DOWN_EVENTS = (E.PY_START, E.PY_RESUME) + +from test.profilee import testfunc + +class SimulateProfileTest(unittest.TestCase, MonitoringEventsBase): + + def test_balanced(self): + events = self.gather_events(testfunc) + c = collections.Counter(events) + self.assertEqual(c["c_call"], c["c_return"]) + self.assertEqual(c["start"], c["return"] + c["unwind"]) + self.assertEqual(c["raise"], c["exception_handled"] + c["unwind"]) + + def test_frame_stack(self): + self.maxDiff = None + stack = [] + errors = [] + seen = set() + def up(*args): + frame = sys._getframe(1) + if not stack: + errors.append("empty") + else: + expected = stack.pop() + if frame != expected: + errors.append(f" Popping {frame} expected {expected}") + def down(*args): + frame = sys._getframe(1) + stack.append(frame) + seen.add(frame.f_code) + def call(code, offset, callable): + if not isinstance(callable, PY_CALLABLES): + stack.append(sys._getframe(1)) + for event in UP_EVENTS: + sys.monitoring.register_callback(TEST_TOOL, event, up) + for event in DOWN_EVENTS: + sys.monitoring.register_callback(TEST_TOOL, event, down) + sys.monitoring.register_callback(TEST_TOOL, E.CALL, call) + sys.monitoring.set_events(TEST_TOOL, SIMPLE_EVENT_SET) + testfunc() + sys.monitoring.set_events(TEST_TOOL, 0) + self.assertEqual(errors, []) + self.assertEqual(len(seen), 9) + self.assertEqual(stack, [sys._getframe()]) + + diff --git a/Makefile.pre.in b/Makefile.pre.in index 2c0ff3d1c7b9a6..a03a506330675b 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -395,6 +395,8 @@ PYTHON_OBJS= \ Python/import.o \ Python/importdl.o \ Python/initconfig.o \ + Python/instrumentation.o \ + Python/legacy_tracing.o \ Python/marshal.o \ Python/modsupport.o \ Python/mysnprintf.o \ diff --git a/Objects/codeobject.c b/Objects/codeobject.c index fc1db72977aa01..7657b1322df401 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -338,6 +338,7 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) co->co_nplaincellvars = nplaincellvars; co->co_ncellvars = ncellvars; co->co_nfreevars = nfreevars; + co->_co_instrument_version = 0; /* not set */ co->co_weakreflist = NULL; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index a30bfacb6e5e44..d0ccb70b40f0da 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -13,6 +13,7 @@ #include "pycore_code.h" #include "pycore_function.h" #include "pycore_long.h" // _PyLong_GetZero() +#include "pycore_instruments.h" #include "pycore_object.h" // _PyObject_GC_TRACK() #include "pycore_moduleobject.h" // PyModuleObject #include "pycore_opcode.h" // EXTRA_CASES @@ -107,15 +108,34 @@ dummy_func( inst(NOP, (--)) { } - inst(RESUME, (--)) { - assert(tstate->cframe == &cframe); - assert(frame == cframe.current_frame); + inst(INSTRUMENTED_RESUME) { + /* Check monitoring *before* calling instrument */ /* Possibly combine this with eval breaker */ - if (frame->f_code->_co_instrument_version != tstate->interp->instrument_version) { + assert(oparg < 2); + if (frame->f_code->_co_instrument_version != tstate->interp->monitoring_version) { if (_Py_Instrument(frame->f_code, tstate->interp)) { goto error; } } + int err = _Py_call_instrumentation( + tstate, oparg, frame, next_instr-1); + ERROR_IF(err, error); + if (_Py_atomic_load_relaxed_int32(eval_breaker)) { + goto handle_eval_breaker; + } + } + + inst(RESUME, (--)) { + assert(tstate->cframe == &cframe); + assert(frame == cframe.current_frame); + /* Possibly combine this with eval breaker */ + if (frame->f_code->_co_instrument_version != tstate->interp->monitoring_version) { + int err = _Py_Instrument(frame->f_code, tstate->interp); + ERROR_IF(err, error); + err = _Py_call_instrumentation( + tstate, oparg, frame, next_instr-1); + ERROR_IF(err, error); + } if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { goto handle_eval_breaker; } @@ -200,7 +220,6 @@ dummy_func( } inst(BINARY_OP_MULTIPLY_INT, (left, right -- prod)) { - assert(cframe.use_tracing == 0); DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -212,7 +231,6 @@ dummy_func( } inst(BINARY_OP_MULTIPLY_FLOAT, (left, right -- prod)) { - assert(cframe.use_tracing == 0); DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -226,7 +244,6 @@ dummy_func( } inst(BINARY_OP_SUBTRACT_INT, (left, right -- sub)) { - assert(cframe.use_tracing == 0); DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -238,7 +255,6 @@ dummy_func( } inst(BINARY_OP_SUBTRACT_FLOAT, (left, right -- sub)) { - assert(cframe.use_tracing == 0); DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -251,7 +267,6 @@ dummy_func( } inst(BINARY_OP_ADD_UNICODE, (left, right -- res)) { - assert(cframe.use_tracing == 0); DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -269,7 +284,6 @@ dummy_func( // specializations, but there is no output. // At the end we just skip over the STORE_FAST. inst(BINARY_OP_INPLACE_ADD_UNICODE, (left, right --)) { - assert(cframe.use_tracing == 0); DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); _Py_CODEUNIT true_next = next_instr[INLINE_CACHE_ENTRIES_BINARY_OP]; @@ -299,7 +313,6 @@ dummy_func( } inst(BINARY_OP_ADD_FLOAT, (left, right -- sum)) { - assert(cframe.use_tracing == 0); DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -313,7 +326,6 @@ dummy_func( } inst(BINARY_OP_ADD_INT, (left, right -- sum)) { - assert(cframe.use_tracing == 0); DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -327,7 +339,6 @@ dummy_func( inst(BINARY_SUBSCR, (container, sub -- res)) { _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_BinarySubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); @@ -373,7 +384,6 @@ dummy_func( // stack effect: (__0 -- ) inst(BINARY_SUBSCR_LIST_INT) { - assert(cframe.use_tracing == 0); PyObject *sub = TOP(); PyObject *list = SECOND(); DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); @@ -398,7 +408,6 @@ dummy_func( // stack effect: (__0 -- ) inst(BINARY_SUBSCR_TUPLE_INT) { - assert(cframe.use_tracing == 0); PyObject *sub = TOP(); PyObject *tuple = SECOND(); DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); @@ -423,7 +432,6 @@ dummy_func( // stack effect: (__0 -- ) inst(BINARY_SUBSCR_DICT) { - assert(cframe.use_tracing == 0); PyObject *dict = SECOND(); DEOPT_IF(!PyDict_CheckExact(SECOND()), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); @@ -501,7 +509,6 @@ dummy_func( inst(STORE_SUBSCR, (v, container, sub -- )) { _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_StoreSubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); @@ -519,7 +526,6 @@ dummy_func( // stack effect: (__0, __1, __2 -- ) inst(STORE_SUBSCR_LIST_INT) { - assert(cframe.use_tracing == 0); PyObject *sub = TOP(); PyObject *list = SECOND(); PyObject *value = THIRD(); @@ -545,7 +551,6 @@ dummy_func( // stack effect: (__0, __1, __2 -- ) inst(STORE_SUBSCR_DICT) { - assert(cframe.use_tracing == 0); PyObject *sub = TOP(); PyObject *dict = SECOND(); PyObject *value = THIRD(); @@ -623,20 +628,27 @@ dummy_func( assert(EMPTY()); /* Restore previous cframe and return. */ tstate->cframe = cframe.previous; - tstate->cframe->use_tracing = cframe.use_tracing; assert(tstate->cframe->current_frame == frame->previous); assert(!_PyErr_Occurred(tstate)); _Py_LeaveRecursiveCallTstate(tstate); return retval; } + // stack effect: (__0 -- _0) + inst(INSTRUMENTED_RETURN_VALUE) { + PyObject *val = TOP(); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, next_instr-1, val); + ERROR_IF(err, error); + GO_TO_INSTRUCTION(RETURN_VALUE); + } + // stack effect: (__0 -- ) inst(RETURN_VALUE) { PyObject *retval = POP(); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); - TRACE_FUNCTION_EXIT(); - DTRACE_FUNCTION_EXIT(); _Py_LeaveRecursiveCallPy(tstate); assert(frame != &entry_frame); frame = cframe.current_frame = pop_frame(tstate, frame); @@ -793,7 +805,7 @@ dummy_func( if (retval == NULL) { if (tstate->c_tracefunc != NULL && _PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) - call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj, tstate, frame); + monitor_raise(tstate, frame, next_instr-1); if (_PyGen_FetchStopIterationValue(&retval) == 0) { gen_status = PYGEN_RETURN; } @@ -835,6 +847,16 @@ dummy_func( Py_DECREF(v); } + // stack effect: ( -- ) + inst(INSTRUMENTED_YIELD_VALUE) { + PyObject *val = TOP(); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_YIELD, + frame, next_instr-1, val); + ERROR_IF(err, error); + GO_TO_INSTRUCTION(YIELD_VALUE); + } + // stack effect: ( -- ) inst(YIELD_VALUE) { // NOTE: It's important that YIELD_VALUE never raises an exception! @@ -846,8 +868,6 @@ dummy_func( PyGenObject *gen = _PyFrame_GetGenerator(frame); gen->gi_frame_state = FRAME_SUSPENDED; _PyFrame_SetStackPointer(frame, stack_pointer); - TRACE_FUNCTION_EXIT(); - DTRACE_FUNCTION_EXIT(); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); @@ -1062,7 +1082,6 @@ dummy_func( inst(UNPACK_SEQUENCE) { _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); PyObject *seq = TOP(); next_instr--; _Py_Specialize_UnpackSequence(seq, next_instr, oparg); @@ -1140,7 +1159,6 @@ dummy_func( inst(STORE_ATTR) { _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); PyObject *owner = TOP(); PyObject *name = GETITEM(names, oparg); next_instr--; @@ -1271,7 +1289,6 @@ dummy_func( inst(LOAD_GLOBAL) { _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); PyObject *name = GETITEM(names, oparg>>1); next_instr--; _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); @@ -1331,7 +1348,6 @@ dummy_func( // error: LOAD_GLOBAL has irregular stack effect inst(LOAD_GLOBAL_MODULE) { - assert(cframe.use_tracing == 0); DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); PyDictObject *dict = (PyDictObject *)GLOBALS(); _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; @@ -1351,7 +1367,6 @@ dummy_func( // error: LOAD_GLOBAL has irregular stack effect inst(LOAD_GLOBAL_BUILTIN) { - assert(cframe.use_tracing == 0); DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); DEOPT_IF(!PyDict_CheckExact(BUILTINS()), LOAD_GLOBAL); PyDictObject *mdict = (PyDictObject *)GLOBALS(); @@ -1716,7 +1731,6 @@ dummy_func( inst(LOAD_ATTR) { _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); PyObject *owner = TOP(); PyObject *name = GETITEM(names, oparg>>1); next_instr--; @@ -1777,7 +1791,6 @@ dummy_func( // error: LOAD_ATTR has irregular stack effect inst(LOAD_ATTR_INSTANCE_VALUE) { - assert(cframe.use_tracing == 0); PyObject *owner = TOP(); PyObject *res; PyTypeObject *tp = Py_TYPE(owner); @@ -1802,7 +1815,6 @@ dummy_func( // error: LOAD_ATTR has irregular stack effect inst(LOAD_ATTR_MODULE) { - assert(cframe.use_tracing == 0); PyObject *owner = TOP(); PyObject *res; _PyAttrCache *cache = (_PyAttrCache *)next_instr; @@ -1827,7 +1839,6 @@ dummy_func( // error: LOAD_ATTR has irregular stack effect inst(LOAD_ATTR_WITH_HINT) { - assert(cframe.use_tracing == 0); PyObject *owner = TOP(); PyObject *res; PyTypeObject *tp = Py_TYPE(owner); @@ -1866,7 +1877,6 @@ dummy_func( // error: LOAD_ATTR has irregular stack effect inst(LOAD_ATTR_SLOT) { - assert(cframe.use_tracing == 0); PyObject *owner = TOP(); PyObject *res; PyTypeObject *tp = Py_TYPE(owner); @@ -1888,7 +1898,6 @@ dummy_func( // error: LOAD_ATTR has irregular stack effect inst(LOAD_ATTR_CLASS) { - assert(cframe.use_tracing == 0); _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; PyObject *cls = TOP(); @@ -1911,7 +1920,6 @@ dummy_func( // error: LOAD_ATTR has irregular stack effect inst(LOAD_ATTR_PROPERTY) { - assert(cframe.use_tracing == 0); DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; @@ -1950,7 +1958,6 @@ dummy_func( // error: LOAD_ATTR has irregular stack effect inst(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { - assert(cframe.use_tracing == 0); DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; PyObject *owner = TOP(); @@ -1991,7 +1998,6 @@ dummy_func( // stack effect: (__0, __1 -- ) inst(STORE_ATTR_INSTANCE_VALUE) { - assert(cframe.use_tracing == 0); PyObject *owner = TOP(); PyTypeObject *tp = Py_TYPE(owner); _PyAttrCache *cache = (_PyAttrCache *)next_instr; @@ -2020,7 +2026,6 @@ dummy_func( // stack effect: (__0, __1 -- ) inst(STORE_ATTR_WITH_HINT) { - assert(cframe.use_tracing == 0); PyObject *owner = TOP(); PyTypeObject *tp = Py_TYPE(owner); _PyAttrCache *cache = (_PyAttrCache *)next_instr; @@ -2072,7 +2077,6 @@ dummy_func( // stack effect: (__0, __1 -- ) inst(STORE_ATTR_SLOT) { - assert(cframe.use_tracing == 0); PyObject *owner = TOP(); PyTypeObject *tp = Py_TYPE(owner); _PyAttrCache *cache = (_PyAttrCache *)next_instr; @@ -2109,7 +2113,6 @@ dummy_func( inst(COMPARE_OP) { _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); PyObject *right = TOP(); PyObject *left = SECOND(); next_instr--; @@ -2123,7 +2126,6 @@ dummy_func( // stack effect: (__0 -- ) inst(COMPARE_OP_FLOAT_JUMP) { - assert(cframe.use_tracing == 0); // Combined: COMPARE_OP (float ? float) + POP_JUMP_IF_(true/false) _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; int when_to_jump_mask = cache->mask; @@ -2154,7 +2156,6 @@ dummy_func( // stack effect: (__0 -- ) inst(COMPARE_OP_INT_JUMP) { - assert(cframe.use_tracing == 0); // Combined: COMPARE_OP (int ? int) + POP_JUMP_IF_(true/false) _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; int when_to_jump_mask = cache->mask; @@ -2186,7 +2187,6 @@ dummy_func( // stack effect: (__0 -- ) inst(COMPARE_OP_STR_JUMP) { - assert(cframe.use_tracing == 0); // Combined: COMPARE_OP (str == str or str != str) + POP_JUMP_IF_(true/false) _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; int invert = cache->mask; @@ -2599,7 +2599,6 @@ dummy_func( inst(FOR_ITER) { _PyForIterCache *cache = (_PyForIterCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_ForIter(TOP(), next_instr, oparg); DISPATCH_SAME_OPARG(); @@ -2619,7 +2618,7 @@ dummy_func( goto error; } else if (tstate->c_tracefunc != NULL) { - call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj, tstate, frame); + monitor_raise(tstate, frame, next_instr-1); } _PyErr_Clear(tstate); } @@ -2634,7 +2633,6 @@ dummy_func( // stack effect: ( -- __0) inst(FOR_ITER_LIST) { - assert(cframe.use_tracing == 0); _PyListIterObject *it = (_PyListIterObject *)TOP(); DEOPT_IF(Py_TYPE(it) != &PyListIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -2657,7 +2655,6 @@ dummy_func( // stack effect: ( -- __0) inst(FOR_ITER_RANGE) { - assert(cframe.use_tracing == 0); _PyRangeIterObject *r = (_PyRangeIterObject *)TOP(); DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -2680,7 +2677,6 @@ dummy_func( } inst(FOR_ITER_GEN) { - assert(cframe.use_tracing == 0); PyGenObject *gen = (PyGenObject *)TOP(); DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); @@ -2822,7 +2818,6 @@ dummy_func( // error: LOAD_ATTR has irregular stack effect inst(LOAD_ATTR_METHOD_WITH_VALUES) { /* Cached method object */ - assert(cframe.use_tracing == 0); PyObject *self = TOP(); PyTypeObject *self_cls = Py_TYPE(self); _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; @@ -2847,7 +2842,6 @@ dummy_func( // error: LOAD_ATTR has irregular stack effect inst(LOAD_ATTR_METHOD_WITH_DICT) { /* Can be either a managed dict, or a tp_dictoffset offset.*/ - assert(cframe.use_tracing == 0); PyObject *self = TOP(); PyTypeObject *self_cls = Py_TYPE(self); _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; @@ -2873,7 +2867,6 @@ dummy_func( // error: LOAD_ATTR has irregular stack effect inst(LOAD_ATTR_METHOD_NO_DICT) { - assert(cframe.use_tracing == 0); PyObject *self = TOP(); PyTypeObject *self_cls = Py_TYPE(self); _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; @@ -2891,7 +2884,6 @@ dummy_func( // error: LOAD_ATTR has irregular stack effect inst(LOAD_ATTR_METHOD_LAZY_DICT) { - assert(cframe.use_tracing == 0); PyObject *self = TOP(); PyTypeObject *self_cls = Py_TYPE(self); _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; @@ -2932,11 +2924,24 @@ dummy_func( kwnames = GETITEM(consts, oparg); } + // stack effect: (__0, __array[oparg] -- ) + inst(INSTRUMENTED_CALL) { + int is_meth = is_method(stack_pointer, oparg); + int total_args = oparg + is_meth; + PyObject *function = PEEK(total_args + 1); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_CALL, + frame, next_instr-1, function); + ERROR_IF(err, error); + _PyCallCache *cache = (_PyCallCache *)next_instr; + INCREMENT_ADAPTIVE_COUNTER(cache->counter); + GO_TO_INSTRUCTION(CALL); + } + // stack effect: (__0, __array[oparg] -- ) inst(CALL) { _PyCallCache *cache = (_PyCallCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); int is_meth = is_method(stack_pointer, oparg); int nargs = oparg + is_meth; PyObject *callable = PEEK(nargs + 1); @@ -2989,16 +2994,26 @@ dummy_func( } /* Callable is not a normal Python function */ PyObject *res; - if (cframe.use_tracing) { - res = trace_call_function( - tstate, function, stack_pointer-total_args, - positional_args, kwnames); - } - else { - res = PyObject_Vectorcall( - function, stack_pointer-total_args, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames); + res = PyObject_Vectorcall( + function, stack_pointer-total_args, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames); + if (opcode == INSTRUMENTED_CALL) { + if (!PyFunction_Check(function) && !PyMethod_Check(function)) { + if (res == NULL) { + _Py_call_instrumentation_exc( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, next_instr-1, function); + } + else { + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, next_instr-1, function); + if (err < 0) { + Py_CLEAR(res); + } + } + } } kwnames = NULL; assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -3093,7 +3108,6 @@ dummy_func( // stack effect: (__0, __array[oparg] -- ) inst(CALL_NO_KW_TYPE_1) { assert(kwnames == NULL); - assert(cframe.use_tracing == 0); assert(oparg == 1); DEOPT_IF(is_method(stack_pointer, 1), CALL); PyObject *obj = TOP(); @@ -3111,7 +3125,6 @@ dummy_func( // stack effect: (__0, __array[oparg] -- ) inst(CALL_NO_KW_STR_1) { assert(kwnames == NULL); - assert(cframe.use_tracing == 0); assert(oparg == 1); DEOPT_IF(is_method(stack_pointer, 1), CALL); PyObject *callable = PEEK(2); @@ -3181,7 +3194,6 @@ dummy_func( // stack effect: (__0, __array[oparg] -- ) inst(CALL_NO_KW_BUILTIN_O) { - assert(cframe.use_tracing == 0); /* Builtin METH_O functions */ assert(kwnames == NULL); int is_meth = is_method(stack_pointer, oparg); @@ -3215,7 +3227,6 @@ dummy_func( // stack effect: (__0, __array[oparg] -- ) inst(CALL_NO_KW_BUILTIN_FAST) { - assert(cframe.use_tracing == 0); /* Builtin METH_FASTCALL functions, without keywords */ assert(kwnames == NULL); int is_meth = is_method(stack_pointer, oparg); @@ -3255,7 +3266,6 @@ dummy_func( // stack effect: (__0, __array[oparg] -- ) inst(CALL_BUILTIN_FAST_WITH_KEYWORDS) { - assert(cframe.use_tracing == 0); /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ int is_meth = is_method(stack_pointer, oparg); int total_args = oparg + is_meth; @@ -3294,7 +3304,6 @@ dummy_func( // stack effect: (__0, __array[oparg] -- ) inst(CALL_NO_KW_LEN) { - assert(cframe.use_tracing == 0); assert(kwnames == NULL); /* len(o) */ int is_meth = is_method(stack_pointer, oparg); @@ -3324,7 +3333,6 @@ dummy_func( // stack effect: (__0, __array[oparg] -- ) inst(CALL_NO_KW_ISINSTANCE) { - assert(cframe.use_tracing == 0); assert(kwnames == NULL); /* isinstance(o, o2) */ int is_meth = is_method(stack_pointer, oparg); @@ -3357,7 +3365,6 @@ dummy_func( // stack effect: (__0, __array[oparg] -- ) inst(CALL_NO_KW_LIST_APPEND) { - assert(cframe.use_tracing == 0); assert(kwnames == NULL); assert(oparg == 1); PyObject *callable = PEEK(3); @@ -3520,6 +3527,10 @@ dummy_func( CHECK_EVAL_BREAKER(); } + inst(INSTRUMENTED_CALL_FUNCTION_EX) { + GO_TO_INSTRUCTION(CALL_FUNCTION_EX); + } + // error: CALL_FUNCTION_EX has irregular stack effect inst(CALL_FUNCTION_EX) { PyObject *func, *callargs, *kwargs = NULL, *result; @@ -3542,8 +3553,30 @@ dummy_func( } } assert(PyTuple_CheckExact(callargs)); - - result = do_call_core(tstate, func, callargs, kwargs, cframe.use_tracing); + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); + if (opcode == INSTRUMENTED_CALL_FUNCTION_EX && !PyFunction_Check(func) && !PyMethod_Check(func)) { + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_CALL, + frame, next_instr-1, func); + ERROR_IF(err, error); + result = PyObject_Call(func, callargs, kwargs); + if (result == NULL) { + _Py_call_instrumentation_exc( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, next_instr-1, func); + } + else { + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, next_instr-1, func); + if (err < 0) { + Py_CLEAR(result); + } + } + } + else { + result = PyObject_Call(func, callargs, kwargs); + } Py_DECREF(func); Py_DECREF(callargs); Py_XDECREF(kwargs); @@ -3718,7 +3751,6 @@ dummy_func( inst(BINARY_OP) { _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); PyObject *lhs = SECOND(); PyObject *rhs = TOP(); next_instr--; @@ -3741,7 +3773,6 @@ dummy_func( // stack effect: ( -- ) inst(EXTENDED_ARG) { assert(oparg); - assert(cframe.use_tracing == 0); opcode = _Py_OPCODE(*next_instr); oparg = oparg << 8 | _Py_OPARG(*next_instr); PRE_DISPATCH_GOTO(); diff --git a/Python/ceval.c b/Python/ceval.c index bff8b5c4d488cb..b3695ac7af1d98 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -14,6 +14,7 @@ #include "pycore_code.h" #include "pycore_function.h" #include "pycore_long.h" // _PyLong_GetZero() +#include "pycore_instruments.h" #include "pycore_object.h" // _PyObject_GC_TRACK() #include "pycore_moduleobject.h" // PyModuleObject #include "pycore_opcode.h" // EXTRA_CASES @@ -101,13 +102,6 @@ #define HEAD_UNLOCK(runtime) \ PyThread_release_lock((runtime)->interpreters.mutex) -/* Forward declarations */ -static PyObject *trace_call_function( - PyThreadState *tstate, PyObject *callable, PyObject **stack, - Py_ssize_t oparg, PyObject *kwnames); -static PyObject * do_call_core( - PyThreadState *tstate, PyObject *func, - PyObject *callargs, PyObject *kwdict, int use_tracing); #ifdef LLTRACE static void @@ -187,19 +181,16 @@ lltrace_resume_frame(_PyInterpreterFrame *frame) PyErr_Restore(type, value, traceback); } #endif -static int call_trace(Py_tracefunc, PyObject *, - PyThreadState *, _PyInterpreterFrame *, - int, PyObject *); -static int call_trace_protected(Py_tracefunc, PyObject *, - PyThreadState *, _PyInterpreterFrame *, - int, PyObject *); -static void call_exc_trace(Py_tracefunc, PyObject *, - PyThreadState *, _PyInterpreterFrame *); -static int maybe_call_line_trace(Py_tracefunc, PyObject *, - PyThreadState *, _PyInterpreterFrame *, int); -static void maybe_dtrace_line(_PyInterpreterFrame *, PyTraceInfo *, int); -static void dtrace_function_entry(_PyInterpreterFrame *); -static void dtrace_function_return(_PyInterpreterFrame *); + +static void monitor_raise(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr); +static void monitor_unwind(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr); +static void monitor_handled(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr, PyObject *exc); static PyObject * import_name(PyThreadState *, _PyInterpreterFrame *, PyObject *, PyObject *, PyObject *); @@ -228,20 +219,6 @@ _PyEvalFrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame); "cannot access free variable '%s' where it is not associated with a" \ " value in enclosing scope" -#ifndef NDEBUG -/* Ensure that tstate is valid: sanity check for PyEval_AcquireThread() and - PyEval_RestoreThread(). Detect if tstate memory was freed. It can happen - when a thread continues to run after Python finalization, especially - daemon threads. */ -static int -is_tstate_valid(PyThreadState *tstate) -{ - assert(!_PyMem_IsPtrFreed(tstate)); - assert(!_PyMem_IsPtrFreed(tstate->interp)); - return 1; -} -#endif - #ifdef HAVE_ERRNO_H #include @@ -699,8 +676,6 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) { \ NEXTOPARG(); \ PRE_DISPATCH_GOTO(); \ - assert(cframe.use_tracing == 0 || cframe.use_tracing == 255); \ - opcode |= cframe.use_tracing OR_DTRACE_LINE; \ DISPATCH_GOTO(); \ } @@ -708,7 +683,6 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) { \ opcode = _Py_OPCODE(*next_instr); \ PRE_DISPATCH_GOTO(); \ - opcode |= cframe.use_tracing OR_DTRACE_LINE; \ DISPATCH_GOTO(); \ } @@ -779,7 +753,7 @@ GETITEM(PyObject *v, Py_ssize_t i) { #define PREDICT(op) \ do { \ _Py_CODEUNIT word = *next_instr; \ - opcode = _Py_OPCODE(word) | cframe.use_tracing OR_DTRACE_LINE; \ + opcode = _Py_OPCODE(word) \ if (opcode == op) { \ oparg = _Py_OPARG(word); \ INSTRUCTION_START(op); \ @@ -881,49 +855,6 @@ GETITEM(PyObject *v, Py_ssize_t i) { /* Shared opcode macros */ -#define TRACE_FUNCTION_EXIT() \ - if (cframe.use_tracing) { \ - if (trace_function_exit(tstate, frame, retval)) { \ - Py_DECREF(retval); \ - goto exit_unwind; \ - } \ - } - -#define DTRACE_FUNCTION_EXIT() \ - if (PyDTrace_FUNCTION_RETURN_ENABLED()) { \ - dtrace_function_return(frame); \ - } - -#define TRACE_FUNCTION_UNWIND() \ - if (cframe.use_tracing) { \ - /* Since we are already unwinding, \ - * we don't care if this raises */ \ - trace_function_exit(tstate, frame, NULL); \ - } - -#define TRACE_FUNCTION_ENTRY() \ - if (cframe.use_tracing) { \ - _PyFrame_SetStackPointer(frame, stack_pointer); \ - int err = trace_function_entry(tstate, frame); \ - stack_pointer = _PyFrame_GetStackPointer(frame); \ - if (err) { \ - goto error; \ - } \ - } - -#define TRACE_FUNCTION_THROW_ENTRY() \ - if (cframe.use_tracing) { \ - assert(frame->stacktop >= 0); \ - if (trace_function_entry(tstate, frame)) { \ - goto exit_unwind; \ - } \ - } - -#define DTRACE_FUNCTION_ENTRY() \ - if (PyDTrace_FUNCTION_ENTRY_ENABLED()) { \ - dtrace_function_entry(frame); \ - } - #define ADAPTIVE_COUNTER_IS_ZERO(COUNTER) \ (((COUNTER) >> ADAPTIVE_BACKOFF_BITS) == 0) @@ -942,63 +873,6 @@ GETITEM(PyObject *v, Py_ssize_t i) { (COUNTER) += (1 << ADAPTIVE_BACKOFF_BITS); \ } while (0); -static int -trace_function_entry(PyThreadState *tstate, _PyInterpreterFrame *frame) -{ - if (tstate->c_tracefunc != NULL) { - /* tstate->c_tracefunc, if defined, is a - function that will be called on *every* entry - to a code block. Its return value, if not - None, is a function that will be called at - the start of each executed line of code. - (Actually, the function must return itself - in order to continue tracing.) The trace - functions are called with three arguments: - a pointer to the current frame, a string - indicating why the function is called, and - an argument which depends on the situation. - The global trace function is also called - whenever an exception is detected. */ - if (call_trace_protected(tstate->c_tracefunc, - tstate->c_traceobj, - tstate, frame, - PyTrace_CALL, Py_None)) { - /* Trace function raised an error */ - return -1; - } - } - if (tstate->c_profilefunc != NULL) { - /* Similar for c_profilefunc, except it needn't - return itself and isn't called for "line" events */ - if (call_trace_protected(tstate->c_profilefunc, - tstate->c_profileobj, - tstate, frame, - PyTrace_CALL, Py_None)) { - /* Profile function raised an error */ - return -1; - } - } - return 0; -} - -static int -trace_function_exit(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject *retval) -{ - if (tstate->c_tracefunc) { - if (call_trace_protected(tstate->c_tracefunc, tstate->c_traceobj, - tstate, frame, PyTrace_RETURN, retval)) { - return -1; - } - } - if (tstate->c_profilefunc) { - if (call_trace_protected(tstate->c_profilefunc, tstate->c_profileobj, - tstate, frame, PyTrace_RETURN, retval)) { - return -1; - } - } - return 0; -} - static _PyInterpreterFrame * pop_frame(PyThreadState *tstate, _PyInterpreterFrame *frame) { @@ -1078,7 +952,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int * strict stack discipline must be maintained. */ _PyCFrame *prev_cframe = tstate->cframe; - cframe.use_tracing = prev_cframe->use_tracing; cframe.previous = prev_cframe; tstate->cframe = &cframe; @@ -1113,8 +986,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int if (_Py_EnterRecursivePy(tstate)) { goto exit_unwind; } - TRACE_FUNCTION_THROW_ENTRY(); - DTRACE_FUNCTION_ENTRY(); + /* TO DO -- Monitor throw entry. */ goto resume_with_error; } @@ -1206,59 +1078,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int case DO_TRACING: #endif { - assert(cframe.use_tracing); - assert(tstate->tracing == 0); - if (INSTR_OFFSET() >= frame->f_code->_co_firsttraceable) { - int instr_prev = _PyInterpreterFrame_LASTI(frame); - frame->prev_instr = next_instr; - NEXTOPARG(); - // No _PyOpcode_Deopt here, since RESUME has no optimized forms: - if (opcode == RESUME) { - if (oparg < 2) { - CHECK_EVAL_BREAKER(); - } - /* Call tracing */ - TRACE_FUNCTION_ENTRY(); - DTRACE_FUNCTION_ENTRY(); - } - else { - /* line-by-line tracing support */ - if (PyDTrace_LINE_ENABLED()) { - maybe_dtrace_line(frame, &tstate->trace_info, instr_prev); - } - - if (cframe.use_tracing && - tstate->c_tracefunc != NULL && !tstate->tracing) { - int err; - /* see maybe_call_line_trace() - for expository comments */ - _PyFrame_SetStackPointer(frame, stack_pointer); - - err = maybe_call_line_trace(tstate->c_tracefunc, - tstate->c_traceobj, - tstate, frame, instr_prev); - // Reload possibly changed frame fields: - stack_pointer = _PyFrame_GetStackPointer(frame); - frame->stacktop = -1; - // next_instr is only reloaded if tracing *does not* raise. - // This is consistent with the behavior of older Python - // versions. If a trace function sets a new f_lineno and - // *then* raises, we use the *old* location when searching - // for an exception handler, displaying the traceback, and - // so on: - if (err) { - // next_instr wasn't incremented at the start of this - // instruction. Increment it before handling the error, - // so that it looks the same as a "normal" instruction: - next_instr++; - goto error; - } - // Reload next_instr. Don't increment it, though, since - // we're going to re-dispatch to the "true" instruction now: - next_instr = frame->prev_instr; - } - } - } + assert(0); NEXTOPARG(); PRE_DISPATCH_GOTO(); // No _PyOpcode_Deopt here, since EXTENDED_ARG has no optimized forms: @@ -1343,12 +1163,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int PyTraceBack_Here(f); } } - - if (tstate->c_tracefunc != NULL) { - /* Make sure state is set to FRAME_UNWINDING for tracing */ - call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj, - tstate, frame); - } + monitor_raise(tstate, frame, next_instr-1); exception_unwind: { @@ -1367,8 +1182,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int } assert(STACK_LEVEL() == 0); _PyFrame_SetStackPointer(frame, stack_pointer); - TRACE_FUNCTION_UNWIND(); - DTRACE_FUNCTION_EXIT(); + monitor_unwind(tstate, frame, next_instr-1); goto exit_unwind; } @@ -1399,6 +1213,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int PyException_SetTraceback(val, Py_None); Py_XDECREF(tb); Py_XDECREF(exc); + monitor_handled(tstate, frame, next_instr-1, val); PUSH(val); JUMPTO(handler); /* Resume normal execution */ @@ -1414,7 +1229,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int if (frame == &entry_frame) { /* Restore previous cframe and exit */ tstate->cframe = cframe.previous; - tstate->cframe->use_tracing = cframe.use_tracing; assert(tstate->cframe->current_frame == frame->previous); _Py_LeaveRecursiveCallTstate(tstate); return NULL; @@ -2384,10 +2198,13 @@ unpack_iterable(PyThreadState *tstate, PyObject *v, } static void -call_exc_trace(Py_tracefunc func, PyObject *self, - PyThreadState *tstate, - _PyInterpreterFrame *f) +monitor_raise(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr) { + if (tstate->interp->monitoring_tools_per_event[PY_MONITORING_EVENT_RAISE].tools == 0) { + return; + } PyObject *type, *value, *traceback, *orig_traceback, *arg; int err; _PyErr_Fetch(tstate, &type, &value, &orig_traceback); @@ -2401,7 +2218,7 @@ call_exc_trace(Py_tracefunc func, PyObject *self, _PyErr_Restore(tstate, type, value, orig_traceback); return; } - err = call_trace(func, self, tstate, f, PyTrace_EXCEPTION, arg); + err = _Py_call_instrumentation_arg(tstate, PY_MONITORING_EVENT_RAISE, frame, instr, arg); Py_DECREF(arg); if (err == 0) { _PyErr_Restore(tstate, type, value, orig_traceback); @@ -2413,80 +2230,44 @@ call_exc_trace(Py_tracefunc func, PyObject *self, } } -static int -call_trace_protected(Py_tracefunc func, PyObject *obj, - PyThreadState *tstate, _PyInterpreterFrame *frame, - int what, PyObject *arg) + +static void +monitor_unwind(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr) { - PyObject *type, *value, *traceback; - int err; - _PyErr_Fetch(tstate, &type, &value, &traceback); - err = call_trace(func, obj, tstate, frame, what, arg); - if (err == 0) - { - _PyErr_Restore(tstate, type, value, traceback); - return 0; - } - else { - Py_XDECREF(type); - Py_XDECREF(value); - Py_XDECREF(traceback); - return -1; + if (tstate->interp->monitoring_tools_per_event[PY_MONITORING_EVENT_PY_UNWIND].tools == 0) { + return; } + _Py_call_instrumentation_exc(tstate, PY_MONITORING_EVENT_PY_UNWIND, frame, instr, NULL); } + static void -initialize_trace_info(PyTraceInfo *trace_info, _PyInterpreterFrame *frame) +monitor_handled(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr, PyObject *exc) { - PyCodeObject *code = frame->f_code; - if (trace_info->code != code) { - trace_info->code = code; - _PyCode_InitAddressRange(code, &trace_info->bounds); + if (tstate->interp->monitoring_tools_per_event[PY_MONITORING_EVENT_EXCEPTION_HANDLED].tools == 0) { + return; } + _Py_call_instrumentation_arg(tstate, PY_MONITORING_EVENT_EXCEPTION_HANDLED, frame, instr, exc); } void PyThreadState_EnterTracing(PyThreadState *tstate) { tstate->tracing++; - tstate->cframe->use_tracing = 0; } void PyThreadState_LeaveTracing(PyThreadState *tstate) { - assert(tstate->tracing > 0 && tstate->cframe->use_tracing == 0); + assert(tstate->tracing > 0); tstate->tracing--; _PyThreadState_UpdateTracingState(tstate); } -static int -call_trace(Py_tracefunc func, PyObject *obj, - PyThreadState *tstate, _PyInterpreterFrame *frame, - int what, PyObject *arg) -{ - int result; - if (tstate->tracing) { - return 0; - } - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f == NULL) { - return -1; - } - int old_what = tstate->tracing_what; - tstate->tracing_what = what; - PyThreadState_EnterTracing(tstate); - assert(_PyInterpreterFrame_LASTI(frame) >= 0); - if (_PyCode_InitLineArray(frame->f_code)) { - return -1; - } - f->f_lineno = _PyCode_LineNumberFromArray(frame->f_code, _PyInterpreterFrame_LASTI(frame)); - result = func(obj, f, what, arg); - f->f_lineno = 0; - PyThreadState_LeaveTracing(tstate); - tstate->tracing_what = old_what; - return result; -} PyObject* _PyEval_CallTracing(PyObject *func, PyObject *args) @@ -2494,7 +2275,6 @@ _PyEval_CallTracing(PyObject *func, PyObject *args) // Save and disable tracing PyThreadState *tstate = _PyThreadState_GET(); int save_tracing = tstate->tracing; - int save_use_tracing = tstate->cframe->use_tracing; tstate->tracing = 0; // Call the tracing function @@ -2502,81 +2282,9 @@ _PyEval_CallTracing(PyObject *func, PyObject *args) // Restore tracing tstate->tracing = save_tracing; - tstate->cframe->use_tracing = save_use_tracing; - return result; -} - -/* See Objects/lnotab_notes.txt for a description of how tracing works. */ -static int -maybe_call_line_trace(Py_tracefunc func, PyObject *obj, - PyThreadState *tstate, _PyInterpreterFrame *frame, int instr_prev) -{ - int result = 0; - - /* If the last instruction falls at the start of a line or if it - represents a jump backwards, update the frame's line number and - then call the trace function if we're tracing source lines. - */ - if (_PyCode_InitLineArray(frame->f_code)) { - return -1; - } - int lastline; - if (instr_prev <= frame->f_code->_co_firsttraceable) { - lastline = -1; - } - else { - lastline = _PyCode_LineNumberFromArray(frame->f_code, instr_prev); - } - int line = _PyCode_LineNumberFromArray(frame->f_code, _PyInterpreterFrame_LASTI(frame)); - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f == NULL) { - return -1; - } - if (line != -1 && f->f_trace_lines) { - /* Trace backward edges (except in 'yield from') or if line number has changed */ - int trace = line != lastline || - (_PyInterpreterFrame_LASTI(frame) < instr_prev && - // SEND has no quickened forms, so no need to use _PyOpcode_Deopt - // here: - _Py_OPCODE(*frame->prev_instr) != SEND); - if (trace) { - result = call_trace(func, obj, tstate, frame, PyTrace_LINE, Py_None); - } - } - /* Always emit an opcode event if we're tracing all opcodes. */ - if (f->f_trace_opcodes && result == 0) { - result = call_trace(func, obj, tstate, frame, PyTrace_OPCODE, Py_None); - } return result; } -int -_PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) -{ - assert(is_tstate_valid(tstate)); - /* The caller must hold the GIL */ - assert(PyGILState_Check()); - - /* Call _PySys_Audit() in the context of the current thread state, - even if tstate is not the current thread state. */ - PyThreadState *current_tstate = _PyThreadState_GET(); - if (_PySys_Audit(current_tstate, "sys.setprofile", NULL) < 0) { - return -1; - } - - tstate->c_profilefunc = func; - PyObject *old_profileobj = tstate->c_profileobj; - tstate->c_profileobj = Py_XNewRef(arg); - /* Flag that tracing or profiling is turned on */ - _PyThreadState_UpdateTracingState(tstate); - - // gh-98257: Only call Py_XDECREF() once the new profile function is fully - // set, so it's safe to call sys.setprofile() again (reentrant call). - Py_XDECREF(old_profileobj); - - return 0; -} - void PyEval_SetProfile(Py_tracefunc func, PyObject *arg) { @@ -2608,33 +2316,6 @@ PyEval_SetProfileAllThreads(Py_tracefunc func, PyObject *arg) } } -int -_PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) -{ - assert(is_tstate_valid(tstate)); - /* The caller must hold the GIL */ - assert(PyGILState_Check()); - - /* Call _PySys_Audit() in the context of the current thread state, - even if tstate is not the current thread state. */ - PyThreadState *current_tstate = _PyThreadState_GET(); - if (_PySys_Audit(current_tstate, "sys.settrace", NULL) < 0) { - return -1; - } - - tstate->c_tracefunc = func; - PyObject *old_traceobj = tstate->c_traceobj; - tstate->c_traceobj = Py_XNewRef(arg); - /* Flag that tracing or profiling is turned on */ - _PyThreadState_UpdateTracingState(tstate); - - // gh-98257: Only call Py_XDECREF() once the new trace function is fully - // set, so it's safe to call sys.settrace() again (reentrant call). - Py_XDECREF(old_traceobj); - - return 0; -} - void PyEval_SetTrace(Py_tracefunc func, PyObject *arg) { @@ -2863,114 +2544,6 @@ PyEval_GetFuncDesc(PyObject *func) return " object"; } -#define C_TRACE(x, call) \ -if (use_tracing && tstate->c_profilefunc) { \ - if (call_trace(tstate->c_profilefunc, tstate->c_profileobj, \ - tstate, tstate->cframe->current_frame, \ - PyTrace_C_CALL, func)) { \ - x = NULL; \ - } \ - else { \ - x = call; \ - if (tstate->c_profilefunc != NULL) { \ - if (x == NULL) { \ - call_trace_protected(tstate->c_profilefunc, \ - tstate->c_profileobj, \ - tstate, tstate->cframe->current_frame, \ - PyTrace_C_EXCEPTION, func); \ - /* XXX should pass (type, value, tb) */ \ - } else { \ - if (call_trace(tstate->c_profilefunc, \ - tstate->c_profileobj, \ - tstate, tstate->cframe->current_frame, \ - PyTrace_C_RETURN, func)) { \ - Py_DECREF(x); \ - x = NULL; \ - } \ - } \ - } \ - } \ -} else { \ - x = call; \ - } - - -static PyObject * -trace_call_function(PyThreadState *tstate, - PyObject *func, - PyObject **args, Py_ssize_t nargs, - PyObject *kwnames) -{ - int use_tracing = 1; - PyObject *x; - if (PyCFunction_CheckExact(func) || PyCMethod_CheckExact(func)) { - C_TRACE(x, PyObject_Vectorcall(func, args, nargs, kwnames)); - return x; - } - else if (Py_IS_TYPE(func, &PyMethodDescr_Type) && nargs > 0) { - /* We need to create a temporary bound method as argument - for profiling. - - If nargs == 0, then this cannot work because we have no - "self". In any case, the call itself would raise - TypeError (foo needs an argument), so we just skip - profiling. */ - PyObject *self = args[0]; - func = Py_TYPE(func)->tp_descr_get(func, self, (PyObject*)Py_TYPE(self)); - if (func == NULL) { - return NULL; - } - C_TRACE(x, PyObject_Vectorcall(func, - args+1, nargs-1, - kwnames)); - Py_DECREF(func); - return x; - } - return PyObject_Vectorcall(func, args, nargs | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames); -} - -static PyObject * -do_call_core(PyThreadState *tstate, - PyObject *func, - PyObject *callargs, - PyObject *kwdict, - int use_tracing - ) -{ - PyObject *result; - if (PyCFunction_CheckExact(func) || PyCMethod_CheckExact(func)) { - C_TRACE(result, PyObject_Call(func, callargs, kwdict)); - return result; - } - else if (Py_IS_TYPE(func, &PyMethodDescr_Type)) { - Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); - if (nargs > 0 && use_tracing) { - /* We need to create a temporary bound method as argument - for profiling. - - If nargs == 0, then this cannot work because we have no - "self". In any case, the call itself would raise - TypeError (foo needs an argument), so we just skip - profiling. */ - PyObject *self = PyTuple_GET_ITEM(callargs, 0); - func = Py_TYPE(func)->tp_descr_get(func, self, (PyObject*)Py_TYPE(self)); - if (func == NULL) { - return NULL; - } - - C_TRACE(result, _PyObject_FastCallDictTstate( - tstate, func, - &_PyTuple_ITEMS(callargs)[1], - nargs - 1, - kwdict)); - Py_DECREF(func); - return result; - } - } - EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); - return PyObject_Call(func, callargs, kwdict); -} - /* Extract a slice index from a PyLong or an object with the nb_index slot defined, and store in *pi. Silently reduce values larger than PY_SSIZE_T_MAX to PY_SSIZE_T_MAX, @@ -3438,69 +3011,6 @@ _PyEval_RequestCodeExtraIndex(freefunc free) return new_index; } -static void -dtrace_function_entry(_PyInterpreterFrame *frame) -{ - const char *filename; - const char *funcname; - int lineno; - - PyCodeObject *code = frame->f_code; - filename = PyUnicode_AsUTF8(code->co_filename); - funcname = PyUnicode_AsUTF8(code->co_name); - lineno = _PyInterpreterFrame_GetLine(frame); - - PyDTrace_FUNCTION_ENTRY(filename, funcname, lineno); -} - -static void -dtrace_function_return(_PyInterpreterFrame *frame) -{ - const char *filename; - const char *funcname; - int lineno; - - PyCodeObject *code = frame->f_code; - filename = PyUnicode_AsUTF8(code->co_filename); - funcname = PyUnicode_AsUTF8(code->co_name); - lineno = _PyInterpreterFrame_GetLine(frame); - - PyDTrace_FUNCTION_RETURN(filename, funcname, lineno); -} - -/* DTrace equivalent of maybe_call_line_trace. */ -static void -maybe_dtrace_line(_PyInterpreterFrame *frame, - PyTraceInfo *trace_info, - int instr_prev) -{ - const char *co_filename, *co_name; - - /* If the last instruction executed isn't in the current - instruction window, reset the window. - */ - initialize_trace_info(trace_info, frame); - int lastline = _PyCode_CheckLineNumber(instr_prev*sizeof(_Py_CODEUNIT), &trace_info->bounds); - int addr = _PyInterpreterFrame_LASTI(frame) * sizeof(_Py_CODEUNIT); - int line = _PyCode_CheckLineNumber(addr, &trace_info->bounds); - if (line != -1) { - /* Trace backward edges or first instruction of a new line */ - if (_PyInterpreterFrame_LASTI(frame) < instr_prev || - (line != lastline && addr == trace_info->bounds.ar_start)) - { - co_filename = PyUnicode_AsUTF8(frame->f_code->co_filename); - if (!co_filename) { - co_filename = "?"; - } - co_name = PyUnicode_AsUTF8(frame->f_code->co_name); - if (!co_name) { - co_name = "?"; - } - PyDTrace_LINE(co_filename, co_name, line); - } - } -} - /* Implement Py_EnterRecursiveCall() and Py_LeaveRecursiveCall() as functions for the limited API. */ diff --git a/Python/clinic/instrumentation.c.h b/Python/clinic/instrumentation.c.h new file mode 100644 index 00000000000000..45060157416114 --- /dev/null +++ b/Python/clinic/instrumentation.c.h @@ -0,0 +1,196 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif + + +PyDoc_STRVAR(monitoring_use_tool__doc__, +"use_tool($module, tool_id, name, /)\n" +"--\n" +"\n"); + +#define MONITORING_USE_TOOL_METHODDEF \ + {"use_tool", _PyCFunction_CAST(monitoring_use_tool), METH_FASTCALL, monitoring_use_tool__doc__}, + +static PyObject * +monitoring_use_tool_impl(PyObject *module, int tool_id, PyObject *name); + +static PyObject * +monitoring_use_tool(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int tool_id; + PyObject *name; + + if (!_PyArg_CheckPositional("use_tool", nargs, 2, 2)) { + goto exit; + } + tool_id = _PyLong_AsInt(args[0]); + if (tool_id == -1 && PyErr_Occurred()) { + goto exit; + } + name = args[1]; + return_value = monitoring_use_tool_impl(module, tool_id, name); + +exit: + return return_value; +} + +PyDoc_STRVAR(monitoring_free_tool__doc__, +"free_tool($module, tool_id, /)\n" +"--\n" +"\n"); + +#define MONITORING_FREE_TOOL_METHODDEF \ + {"free_tool", (PyCFunction)monitoring_free_tool, METH_O, monitoring_free_tool__doc__}, + +static PyObject * +monitoring_free_tool_impl(PyObject *module, int tool_id); + +static PyObject * +monitoring_free_tool(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + int tool_id; + + tool_id = _PyLong_AsInt(arg); + if (tool_id == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = monitoring_free_tool_impl(module, tool_id); + +exit: + return return_value; +} + +PyDoc_STRVAR(monitoring_get_tool__doc__, +"get_tool($module, tool_id, /)\n" +"--\n" +"\n"); + +#define MONITORING_GET_TOOL_METHODDEF \ + {"get_tool", (PyCFunction)monitoring_get_tool, METH_O, monitoring_get_tool__doc__}, + +static PyObject * +monitoring_get_tool_impl(PyObject *module, int tool_id); + +static PyObject * +monitoring_get_tool(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + int tool_id; + + tool_id = _PyLong_AsInt(arg); + if (tool_id == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = monitoring_get_tool_impl(module, tool_id); + +exit: + return return_value; +} + +PyDoc_STRVAR(monitoring_register_callback__doc__, +"register_callback($module, tool_id, event, func, /)\n" +"--\n" +"\n"); + +#define MONITORING_REGISTER_CALLBACK_METHODDEF \ + {"register_callback", _PyCFunction_CAST(monitoring_register_callback), METH_FASTCALL, monitoring_register_callback__doc__}, + +static PyObject * +monitoring_register_callback_impl(PyObject *module, int tool_id, int event, + PyObject *func); + +static PyObject * +monitoring_register_callback(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int tool_id; + int event; + PyObject *func; + + if (!_PyArg_CheckPositional("register_callback", nargs, 3, 3)) { + goto exit; + } + tool_id = _PyLong_AsInt(args[0]); + if (tool_id == -1 && PyErr_Occurred()) { + goto exit; + } + event = _PyLong_AsInt(args[1]); + if (event == -1 && PyErr_Occurred()) { + goto exit; + } + func = args[2]; + return_value = monitoring_register_callback_impl(module, tool_id, event, func); + +exit: + return return_value; +} + +PyDoc_STRVAR(monitoring_get_events__doc__, +"get_events($module, tool_id, /)\n" +"--\n" +"\n"); + +#define MONITORING_GET_EVENTS_METHODDEF \ + {"get_events", (PyCFunction)monitoring_get_events, METH_O, monitoring_get_events__doc__}, + +static PyObject * +monitoring_get_events_impl(PyObject *module, int tool_id); + +static PyObject * +monitoring_get_events(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + int tool_id; + + tool_id = _PyLong_AsInt(arg); + if (tool_id == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = monitoring_get_events_impl(module, tool_id); + +exit: + return return_value; +} + +PyDoc_STRVAR(monitoring_set_events__doc__, +"set_events($module, tool_id, event_set, /)\n" +"--\n" +"\n"); + +#define MONITORING_SET_EVENTS_METHODDEF \ + {"set_events", _PyCFunction_CAST(monitoring_set_events), METH_FASTCALL, monitoring_set_events__doc__}, + +static PyObject * +monitoring_set_events_impl(PyObject *module, int tool_id, int event_set); + +static PyObject * +monitoring_set_events(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int tool_id; + int event_set; + + if (!_PyArg_CheckPositional("set_events", nargs, 2, 2)) { + goto exit; + } + tool_id = _PyLong_AsInt(args[0]); + if (tool_id == -1 && PyErr_Occurred()) { + goto exit; + } + event_set = _PyLong_AsInt(args[1]); + if (event_set == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = monitoring_set_events_impl(module, tool_id, event_set); + +exit: + return return_value; +} +/*[clinic end generated code: output=876a03223495a81d input=a9049054013a1b77]*/ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 2ad7cc9e65785a..b9a9fe90a13c4a 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5,9 +5,35 @@ DISPATCH(); } + TARGET(INSTRUMENTED_RESUME) { + /* Check monitoring *before* calling instrument */ + /* Possibly combine this with eval breaker */ + assert(oparg < 2); + if (frame->f_code->_co_instrument_version != tstate->interp->monitoring_version) { + if (_Py_Instrument(frame->f_code, tstate->interp)) { + goto error; + } + } + int err = _Py_call_instrumentation( + tstate, oparg, frame, next_instr-1); + if (err) goto error; + if (_Py_atomic_load_relaxed_int32(eval_breaker)) { + goto handle_eval_breaker; + } + DISPATCH(); + } + TARGET(RESUME) { assert(tstate->cframe == &cframe); assert(frame == cframe.current_frame); + /* Possibly combine this with eval breaker */ + if (frame->f_code->_co_instrument_version != tstate->interp->monitoring_version) { + int err = _Py_Instrument(frame->f_code, tstate->interp); + if (err) goto error; + err = _Py_call_instrumentation( + tstate, oparg, frame, next_instr-1); + if (err) goto error; + } if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { goto handle_eval_breaker; } @@ -137,7 +163,6 @@ PyObject *right = PEEK(1); PyObject *left = PEEK(2); PyObject *prod; - assert(cframe.use_tracing == 0); DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -155,7 +180,6 @@ PyObject *right = PEEK(1); PyObject *left = PEEK(2); PyObject *prod; - assert(cframe.use_tracing == 0); DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -175,7 +199,6 @@ PyObject *right = PEEK(1); PyObject *left = PEEK(2); PyObject *sub; - assert(cframe.use_tracing == 0); DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -193,7 +216,6 @@ PyObject *right = PEEK(1); PyObject *left = PEEK(2); PyObject *sub; - assert(cframe.use_tracing == 0); DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -212,7 +234,6 @@ PyObject *right = PEEK(1); PyObject *left = PEEK(2); PyObject *res; - assert(cframe.use_tracing == 0); DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -229,7 +250,6 @@ TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { PyObject *right = PEEK(1); PyObject *left = PEEK(2); - assert(cframe.use_tracing == 0); DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); _Py_CODEUNIT true_next = next_instr[INLINE_CACHE_ENTRIES_BINARY_OP]; @@ -264,7 +284,6 @@ PyObject *right = PEEK(1); PyObject *left = PEEK(2); PyObject *sum; - assert(cframe.use_tracing == 0); DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -284,7 +303,6 @@ PyObject *right = PEEK(1); PyObject *left = PEEK(2); PyObject *sum; - assert(cframe.use_tracing == 0); DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -305,7 +323,6 @@ PyObject *res; _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_BinarySubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); @@ -366,7 +383,6 @@ } TARGET(BINARY_SUBSCR_LIST_INT) { - assert(cframe.use_tracing == 0); PyObject *sub = TOP(); PyObject *list = SECOND(); DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); @@ -391,7 +407,6 @@ } TARGET(BINARY_SUBSCR_TUPLE_INT) { - assert(cframe.use_tracing == 0); PyObject *sub = TOP(); PyObject *tuple = SECOND(); DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); @@ -416,7 +431,6 @@ } TARGET(BINARY_SUBSCR_DICT) { - assert(cframe.use_tracing == 0); PyObject *dict = SECOND(); DEOPT_IF(!PyDict_CheckExact(SECOND()), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); @@ -498,7 +512,6 @@ PyObject *v = PEEK(3); _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_StoreSubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); @@ -517,7 +530,6 @@ } TARGET(STORE_SUBSCR_LIST_INT) { - assert(cframe.use_tracing == 0); PyObject *sub = TOP(); PyObject *list = SECOND(); PyObject *value = THIRD(); @@ -543,7 +555,6 @@ } TARGET(STORE_SUBSCR_DICT) { - assert(cframe.use_tracing == 0); PyObject *sub = TOP(); PyObject *dict = SECOND(); PyObject *value = THIRD(); @@ -620,19 +631,26 @@ assert(EMPTY()); /* Restore previous cframe and return. */ tstate->cframe = cframe.previous; - tstate->cframe->use_tracing = cframe.use_tracing; assert(tstate->cframe->current_frame == frame->previous); assert(!_PyErr_Occurred(tstate)); _Py_LeaveRecursiveCallTstate(tstate); return retval; } + TARGET(INSTRUMENTED_RETURN_VALUE) { + PyObject *val = TOP(); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, next_instr-1, val); + if (err) goto error; + GO_TO_INSTRUCTION(RETURN_VALUE); + } + TARGET(RETURN_VALUE) { + PREDICTED(RETURN_VALUE); PyObject *retval = POP(); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); - TRACE_FUNCTION_EXIT(); - DTRACE_FUNCTION_EXIT(); _Py_LeaveRecursiveCallPy(tstate); assert(frame != &entry_frame); frame = cframe.current_frame = pop_frame(tstate, frame); @@ -789,7 +807,7 @@ if (retval == NULL) { if (tstate->c_tracefunc != NULL && _PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) - call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj, tstate, frame); + monitor_raise(tstate, frame, next_instr-1); if (_PyGen_FetchStopIterationValue(&retval) == 0) { gen_status = PYGEN_RETURN; } @@ -832,7 +850,17 @@ DISPATCH(); } + TARGET(INSTRUMENTED_YIELD_VALUE) { + PyObject *val = TOP(); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_YIELD, + frame, next_instr-1, val); + if (err) goto error; + GO_TO_INSTRUCTION(YIELD_VALUE); + } + TARGET(YIELD_VALUE) { + PREDICTED(YIELD_VALUE); // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. @@ -842,8 +870,6 @@ PyGenObject *gen = _PyFrame_GetGenerator(frame); gen->gi_frame_state = FRAME_SUSPENDED; _PyFrame_SetStackPointer(frame, stack_pointer); - TRACE_FUNCTION_EXIT(); - DTRACE_FUNCTION_EXIT(); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); @@ -1058,7 +1084,6 @@ PREDICTED(UNPACK_SEQUENCE); _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); PyObject *seq = TOP(); next_instr--; _Py_Specialize_UnpackSequence(seq, next_instr, oparg); @@ -1137,7 +1162,6 @@ PREDICTED(STORE_ATTR); _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); PyObject *owner = TOP(); PyObject *name = GETITEM(names, oparg); next_instr--; @@ -1270,7 +1294,6 @@ PREDICTED(LOAD_GLOBAL); _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); PyObject *name = GETITEM(names, oparg>>1); next_instr--; _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); @@ -1330,7 +1353,6 @@ } TARGET(LOAD_GLOBAL_MODULE) { - assert(cframe.use_tracing == 0); DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); PyDictObject *dict = (PyDictObject *)GLOBALS(); _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; @@ -1350,7 +1372,6 @@ } TARGET(LOAD_GLOBAL_BUILTIN) { - assert(cframe.use_tracing == 0); DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); DEOPT_IF(!PyDict_CheckExact(BUILTINS()), LOAD_GLOBAL); PyDictObject *mdict = (PyDictObject *)GLOBALS(); @@ -1719,7 +1740,6 @@ PREDICTED(LOAD_ATTR); _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); PyObject *owner = TOP(); PyObject *name = GETITEM(names, oparg>>1); next_instr--; @@ -1780,7 +1800,6 @@ } TARGET(LOAD_ATTR_INSTANCE_VALUE) { - assert(cframe.use_tracing == 0); PyObject *owner = TOP(); PyObject *res; PyTypeObject *tp = Py_TYPE(owner); @@ -1805,7 +1824,6 @@ } TARGET(LOAD_ATTR_MODULE) { - assert(cframe.use_tracing == 0); PyObject *owner = TOP(); PyObject *res; _PyAttrCache *cache = (_PyAttrCache *)next_instr; @@ -1830,7 +1848,6 @@ } TARGET(LOAD_ATTR_WITH_HINT) { - assert(cframe.use_tracing == 0); PyObject *owner = TOP(); PyObject *res; PyTypeObject *tp = Py_TYPE(owner); @@ -1869,7 +1886,6 @@ } TARGET(LOAD_ATTR_SLOT) { - assert(cframe.use_tracing == 0); PyObject *owner = TOP(); PyObject *res; PyTypeObject *tp = Py_TYPE(owner); @@ -1891,7 +1907,6 @@ } TARGET(LOAD_ATTR_CLASS) { - assert(cframe.use_tracing == 0); _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; PyObject *cls = TOP(); @@ -1914,7 +1929,6 @@ } TARGET(LOAD_ATTR_PROPERTY) { - assert(cframe.use_tracing == 0); DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; @@ -1952,7 +1966,6 @@ } TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { - assert(cframe.use_tracing == 0); DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; PyObject *owner = TOP(); @@ -1992,7 +2005,6 @@ } TARGET(STORE_ATTR_INSTANCE_VALUE) { - assert(cframe.use_tracing == 0); PyObject *owner = TOP(); PyTypeObject *tp = Py_TYPE(owner); _PyAttrCache *cache = (_PyAttrCache *)next_instr; @@ -2021,7 +2033,6 @@ } TARGET(STORE_ATTR_WITH_HINT) { - assert(cframe.use_tracing == 0); PyObject *owner = TOP(); PyTypeObject *tp = Py_TYPE(owner); _PyAttrCache *cache = (_PyAttrCache *)next_instr; @@ -2073,7 +2084,6 @@ } TARGET(STORE_ATTR_SLOT) { - assert(cframe.use_tracing == 0); PyObject *owner = TOP(); PyTypeObject *tp = Py_TYPE(owner); _PyAttrCache *cache = (_PyAttrCache *)next_instr; @@ -2112,7 +2122,6 @@ PREDICTED(COMPARE_OP); _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); PyObject *right = TOP(); PyObject *left = SECOND(); next_instr--; @@ -2125,7 +2134,6 @@ } TARGET(COMPARE_OP_FLOAT_JUMP) { - assert(cframe.use_tracing == 0); // Combined: COMPARE_OP (float ? float) + POP_JUMP_IF_(true/false) _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; int when_to_jump_mask = cache->mask; @@ -2156,7 +2164,6 @@ } TARGET(COMPARE_OP_INT_JUMP) { - assert(cframe.use_tracing == 0); // Combined: COMPARE_OP (int ? int) + POP_JUMP_IF_(true/false) _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; int when_to_jump_mask = cache->mask; @@ -2188,7 +2195,6 @@ } TARGET(COMPARE_OP_STR_JUMP) { - assert(cframe.use_tracing == 0); // Combined: COMPARE_OP (str == str or str != str) + POP_JUMP_IF_(true/false) _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; int invert = cache->mask; @@ -2604,7 +2610,6 @@ PREDICTED(FOR_ITER); _PyForIterCache *cache = (_PyForIterCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_ForIter(TOP(), next_instr, oparg); DISPATCH_SAME_OPARG(); @@ -2624,7 +2629,7 @@ goto error; } else if (tstate->c_tracefunc != NULL) { - call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj, tstate, frame); + monitor_raise(tstate, frame, next_instr-1); } _PyErr_Clear(tstate); } @@ -2639,7 +2644,6 @@ } TARGET(FOR_ITER_LIST) { - assert(cframe.use_tracing == 0); _PyListIterObject *it = (_PyListIterObject *)TOP(); DEOPT_IF(Py_TYPE(it) != &PyListIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -2662,7 +2666,6 @@ } TARGET(FOR_ITER_RANGE) { - assert(cframe.use_tracing == 0); _PyRangeIterObject *r = (_PyRangeIterObject *)TOP(); DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -2686,7 +2689,6 @@ } TARGET(FOR_ITER_GEN) { - assert(cframe.use_tracing == 0); PyGenObject *gen = (PyGenObject *)TOP(); DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); @@ -2826,7 +2828,6 @@ TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { /* Cached method object */ - assert(cframe.use_tracing == 0); PyObject *self = TOP(); PyTypeObject *self_cls = Py_TYPE(self); _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; @@ -2851,7 +2852,6 @@ TARGET(LOAD_ATTR_METHOD_WITH_DICT) { /* Can be either a managed dict, or a tp_dictoffset offset.*/ - assert(cframe.use_tracing == 0); PyObject *self = TOP(); PyTypeObject *self_cls = Py_TYPE(self); _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; @@ -2877,7 +2877,6 @@ } TARGET(LOAD_ATTR_METHOD_NO_DICT) { - assert(cframe.use_tracing == 0); PyObject *self = TOP(); PyTypeObject *self_cls = Py_TYPE(self); _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; @@ -2895,7 +2894,6 @@ } TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { - assert(cframe.use_tracing == 0); PyObject *self = TOP(); PyTypeObject *self_cls = Py_TYPE(self); _PyLoadMethodCache *cache = (_PyLoadMethodCache *)next_instr; @@ -2936,11 +2934,23 @@ DISPATCH(); } + TARGET(INSTRUMENTED_CALL) { + int is_meth = is_method(stack_pointer, oparg); + int total_args = oparg + is_meth; + PyObject *function = PEEK(total_args + 1); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_CALL, + frame, next_instr-1, function); + if (err) goto error; + _PyCallCache *cache = (_PyCallCache *)next_instr; + INCREMENT_ADAPTIVE_COUNTER(cache->counter); + GO_TO_INSTRUCTION(CALL); + } + TARGET(CALL) { PREDICTED(CALL); _PyCallCache *cache = (_PyCallCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); int is_meth = is_method(stack_pointer, oparg); int nargs = oparg + is_meth; PyObject *callable = PEEK(nargs + 1); @@ -2993,16 +3003,26 @@ } /* Callable is not a normal Python function */ PyObject *res; - if (cframe.use_tracing) { - res = trace_call_function( - tstate, function, stack_pointer-total_args, - positional_args, kwnames); - } - else { - res = PyObject_Vectorcall( - function, stack_pointer-total_args, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames); + res = PyObject_Vectorcall( + function, stack_pointer-total_args, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames); + if (opcode == INSTRUMENTED_CALL) { + if (!PyFunction_Check(function) && !PyMethod_Check(function)) { + if (res == NULL) { + _Py_call_instrumentation_exc( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, next_instr-1, function); + } + else { + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, next_instr-1, function); + if (err < 0) { + Py_CLEAR(res); + } + } + } } kwnames = NULL; assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -3096,7 +3116,6 @@ TARGET(CALL_NO_KW_TYPE_1) { assert(kwnames == NULL); - assert(cframe.use_tracing == 0); assert(oparg == 1); DEOPT_IF(is_method(stack_pointer, 1), CALL); PyObject *obj = TOP(); @@ -3114,7 +3133,6 @@ TARGET(CALL_NO_KW_STR_1) { assert(kwnames == NULL); - assert(cframe.use_tracing == 0); assert(oparg == 1); DEOPT_IF(is_method(stack_pointer, 1), CALL); PyObject *callable = PEEK(2); @@ -3184,7 +3202,6 @@ } TARGET(CALL_NO_KW_BUILTIN_O) { - assert(cframe.use_tracing == 0); /* Builtin METH_O functions */ assert(kwnames == NULL); int is_meth = is_method(stack_pointer, oparg); @@ -3218,7 +3235,6 @@ } TARGET(CALL_NO_KW_BUILTIN_FAST) { - assert(cframe.use_tracing == 0); /* Builtin METH_FASTCALL functions, without keywords */ assert(kwnames == NULL); int is_meth = is_method(stack_pointer, oparg); @@ -3258,7 +3274,6 @@ } TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { - assert(cframe.use_tracing == 0); /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ int is_meth = is_method(stack_pointer, oparg); int total_args = oparg + is_meth; @@ -3297,7 +3312,6 @@ } TARGET(CALL_NO_KW_LEN) { - assert(cframe.use_tracing == 0); assert(kwnames == NULL); /* len(o) */ int is_meth = is_method(stack_pointer, oparg); @@ -3327,7 +3341,6 @@ } TARGET(CALL_NO_KW_ISINSTANCE) { - assert(cframe.use_tracing == 0); assert(kwnames == NULL); /* isinstance(o, o2) */ int is_meth = is_method(stack_pointer, oparg); @@ -3360,7 +3373,6 @@ } TARGET(CALL_NO_KW_LIST_APPEND) { - assert(cframe.use_tracing == 0); assert(kwnames == NULL); assert(oparg == 1); PyObject *callable = PEEK(3); @@ -3524,6 +3536,10 @@ DISPATCH(); } + TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { + GO_TO_INSTRUCTION(CALL_FUNCTION_EX); + } + TARGET(CALL_FUNCTION_EX) { PREDICTED(CALL_FUNCTION_EX); PyObject *func, *callargs, *kwargs = NULL, *result; @@ -3546,8 +3562,30 @@ } } assert(PyTuple_CheckExact(callargs)); - - result = do_call_core(tstate, func, callargs, kwargs, cframe.use_tracing); + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); + if (opcode == INSTRUMENTED_CALL_FUNCTION_EX && !PyFunction_Check(func) && !PyMethod_Check(func)) { + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_CALL, + frame, next_instr-1, func); + if (err) goto error; + result = PyObject_Call(func, callargs, kwargs); + if (result == NULL) { + _Py_call_instrumentation_exc( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, next_instr-1, func); + } + else { + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, next_instr-1, func); + if (err < 0) { + Py_CLEAR(result); + } + } + } + else { + result = PyObject_Call(func, callargs, kwargs); + } Py_DECREF(func); Py_DECREF(callargs); Py_XDECREF(kwargs); @@ -3723,7 +3761,6 @@ PREDICTED(BINARY_OP); _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); PyObject *lhs = SECOND(); PyObject *rhs = TOP(); next_instr--; @@ -3745,7 +3782,6 @@ TARGET(EXTENDED_ARG) { assert(oparg); - assert(cframe.use_tracing == 0); opcode = _Py_OPCODE(*next_instr); oparg = oparg << 8 | _Py_OPARG(*next_instr); PRE_DISPATCH_GOTO(); diff --git a/Python/instrumentation.c b/Python/instrumentation.c new file mode 100644 index 00000000000000..235cfce6c44635 --- /dev/null +++ b/Python/instrumentation.c @@ -0,0 +1,573 @@ + + +#include "Python.h" +#include "pycore_frame.h" +#include "pycore_interp.h" +#include "pycore_namespace.h" +#include "pycore_opcode.h" +#include "pycore_pyerrors.h" + + +static PyObject DISABLE = +{ + 1 << 30, + &PyBaseObject_Type +}; + +static int +call_multiple_instruments( + PyInterpreterState *interp, PyCodeObject *code, int event, + PyObject **args, Py_ssize_t nargs, _Py_CODEUNIT *instr) +{ + PyErr_SetString(PyExc_SystemError, "Multiple tools not supported yet"); + return -1; +} + +static const uint8_t DE_INSTRUMENT[256] = { + [INSTRUMENTED_RESUME] = RESUME, + [INSTRUMENTED_RETURN_VALUE] = RETURN_VALUE, + [INSTRUMENTED_CALL] = CALL, + [INSTRUMENTED_YIELD_VALUE] = YIELD_VALUE, +/* + [INSTRUMENTED_JUMP_FORWARD] = JUMP_FORWARD, + [INSTRUMENTED_JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD, + [INSTRUMENTED_JUMP_IF_FALSE_OR_POP] = INSTRUMENTED_JUMP_IF_FALSE_OR_POP, + [INSTRUMENTED_JUMP_IF_TRUE_OR_POP] = INSTRUMENTED_JUMP_IF_TRUE_OR_POP, + [INSTRUMENTED_POP_JUMP_IF_FALSE] = INSTRUMENTED_POP_JUMP_IF_FALSE, + [INSTRUMENTED_POP_JUMP_IF_TRUE] = INSTRUMENTED_POP_JUMP_IF_TRUE, + [INSTRUMENTED_POP_JUMP_IF_NONE] = INSTRUMENTED_POP_JUMP_IF_NONE, + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE, +*/ +}; + +static int +call_instrument( + PyThreadState *tstate, PyCodeObject *code, int event, + PyObject **args, Py_ssize_t nargsf, _Py_CODEUNIT *instr) +{ + assert(!_PyErr_Occurred(tstate)); + PyInterpreterState *interp = tstate->interp; + int sole_tool_plus1 = interp->sole_tool_plus1[event]; + if (sole_tool_plus1 <= 0) { + if (sole_tool_plus1 == 0) { + /* Why is this instrumented if there is no tool? */ + return 0; + } + return call_multiple_instruments(interp, code, event, args, nargsf, instr); + } + PyObject *instrument = interp->tools[sole_tool_plus1-1].instrument_callables[event]; + if (instrument == NULL) { + return 0; + } + PyObject *res = PyObject_Vectorcall(instrument, args, nargsf, NULL); + if (res == NULL) { + return -1; + } + if (res == &DISABLE) { + /* Remove this instrument */ + assert(DE_INSTRUMENT[_Py_OPCODE(*instr)] != 0); + _Py_SET_OPCODE(*instr, DE_INSTRUMENT[_Py_OPCODE(*instr)]); + } + Py_DECREF(res); + return 0; +} + + +int +_Py_call_instrumentation( + PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) +{ + if (tstate->tracing) { + return 0; + } + tstate->tracing++; + PyCodeObject *code = frame->f_code; + int instruction_offset = instr - _PyCode_CODE(code); + PyObject *instruction_offset_obj = PyLong_FromSsize_t(instruction_offset); + PyObject *args[3] = { NULL, (PyObject *)code, instruction_offset_obj }; + int err = call_instrument(tstate, code, event, &args[1], 2 | PY_VECTORCALL_ARGUMENTS_OFFSET, instr); + Py_DECREF(instruction_offset_obj); + tstate->tracing--; + return err; +} + +int +_Py_call_instrumentation_arg( + PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg) +{ + if (tstate->tracing) { + return 0; + } + tstate->tracing++; + PyCodeObject *code = frame->f_code; + int instruction_offset = instr - _PyCode_CODE(code); + PyObject *instruction_offset_obj = PyLong_FromSsize_t(instruction_offset); + PyObject *args[4] = { NULL, (PyObject *)code, instruction_offset_obj, arg }; + int err = call_instrument(tstate, code, event, &args[1], 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, instr); + Py_DECREF(instruction_offset_obj); + tstate->tracing--; + return err; +} + +void +_Py_call_instrumentation_exc( + PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg) +{ + assert(_PyErr_Occurred(tstate)); + if (tstate->tracing) { + return; + } + tstate->tracing++; + PyObject *type, *value, *traceback; + _PyErr_Fetch(tstate, &type, &value, &traceback); + PyCodeObject *code = frame->f_code; + int instruction_offset = instr - _PyCode_CODE(code); + PyObject *instruction_offset_obj = PyLong_FromSsize_t(instruction_offset); + PyObject *args[4] = { NULL, (PyObject *)code, instruction_offset_obj, arg }; + Py_ssize_t nargsf = (2 | PY_VECTORCALL_ARGUMENTS_OFFSET) + (arg != NULL); + int err = call_instrument(tstate, code, event, &args[1], nargsf, instr); + if (err) { + Py_XDECREF(type); + Py_XDECREF(value); + Py_XDECREF(traceback); + } + else { + _PyErr_Restore(tstate, type, value, traceback); + } + assert(_PyErr_Occurred(tstate)); + Py_DECREF(instruction_offset_obj); + tstate->tracing--; +} + +PyObject * +_PyMonitoring_RegisterCallback(int tool_id, int event_id, PyObject *obj) +{ + PyInterpreterState *is = _PyInterpreterState_Get(); + assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); + assert(0 <= event_id && event_id < PY_MONITORING_EVENTS); + PyObject *callback = is->tools[tool_id].instrument_callables[event_id]; + is->tools[tool_id].instrument_callables[event_id] = Py_XNewRef(obj); + return callback; +} + +static const uint8_t INSTRUMENTED_OPCODES[256] = { + [RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE, + [CALL] = INSTRUMENTED_CALL, + [CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, + [YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE, + [RESUME] = INSTRUMENTED_RESUME, + [INSTRUMENTED_RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE, + [INSTRUMENTED_CALL] = INSTRUMENTED_CALL, + [INSTRUMENTED_CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, + [INSTRUMENTED_YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE, + [INSTRUMENTED_RESUME] = INSTRUMENTED_RESUME, + /* + [JUMP_FORWARD] = INSTRUMENTED_JUMP_FORWARD, + [JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD, + [JUMP_IF_FALSE_OR_POP] = INSTRUMENTED_JUMP_IF_FALSE_OR_POP, + [JUMP_IF_TRUE_OR_POP] = INSTRUMENTED_JUMP_IF_TRUE_OR_POP, + [POP_JUMP_IF_FALSE] = INSTRUMENTED_POP_JUMP_IF_FALSE, + [POP_JUMP_IF_TRUE] = INSTRUMENTED_POP_JUMP_IF_TRUE, + [POP_JUMP_IF_NONE] = INSTRUMENTED_POP_JUMP_IF_NONE, + [POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE, + */ +}; + +#define SET1(a) (1 << (a)) +#define SET2(a, b ) ( (1 << (a)) | (1 << (b)) ) +#define SET3(a, b, c) ( (1 << (a)) | (1 << (b)) | (1 << (c)) ) + +#define CALL_EVENTS \ + SET3(PY_MONITORING_EVENT_CALL, PY_MONITORING_EVENT_C_RAISE, PY_MONITORING_EVENT_C_RETURN) + +static const _PyMonitoringEventSet EVENTS_FOR_OPCODE[256] = { + [RETURN_VALUE] = SET1(PY_MONITORING_EVENT_PY_RETURN), + [INSTRUMENTED_RETURN_VALUE] = SET1(PY_MONITORING_EVENT_PY_RETURN), + [CALL] = CALL_EVENTS, + [INSTRUMENTED_CALL] = CALL_EVENTS, + [CALL_FUNCTION_EX] = CALL_EVENTS, + [INSTRUMENTED_CALL_FUNCTION_EX] = CALL_EVENTS, + [RESUME] = SET2(PY_MONITORING_EVENT_PY_START, PY_MONITORING_EVENT_PY_RESUME), + [INSTRUMENTED_RESUME] = SET2(PY_MONITORING_EVENT_PY_START, PY_MONITORING_EVENT_PY_RESUME), + [YIELD_VALUE] = SET1(PY_MONITORING_EVENT_PY_YIELD), + [INSTRUMENTED_YIELD_VALUE] = SET1(PY_MONITORING_EVENT_PY_YIELD), +}; + +int +_Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) +{ + if (interp->monitoring_version == code->_co_instrument_version) { + return 0; + } + /* First establish how much extra space will need, + * and free/allocate it if different to what we had before */ + /* For now only allow only one tool per event */ + assert(interp->monitoring_tools_per_event[PY_MONITORING_EVENT_LINE].tools == 0); + assert(interp->monitoring_tools_per_event[PY_MONITORING_EVENT_INSTRUCTION].tools == 0); + for (int i = 0; i < PY_MONITORING_EVENTS; i++) { + assert(interp->monitoring_tools_per_event[i].tools == 0 || interp->sole_tool_plus1[i] > 0); + } + //printf("Instrumenting code object at %p, code version: %ld interpreter version %ld\n", + // code, code->_co_instrument_version, interp->monitoring_version); + + /* Insert basic instrumentation */ + int instrumented = 0; + int code_len = (int)Py_SIZE(code); + for (int i = 0; i < code_len; i++) { + _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; + int opcode = _Py_OPCODE(*instr); + if (_PyOpcode_Deopt[opcode]) { + opcode = _PyOpcode_Deopt[opcode]; + } + /* Don't instrument RESUME in await and yield from. */ + if (opcode == RESUME && _Py_OPARG(_PyCode_CODE(code)[i]) >= 2) { + continue; + } + _PyMonitoringEventSet events = EVENTS_FOR_OPCODE[opcode]; + if (events & interp->monitored_events) { + instrumented++; + assert(INSTRUMENTED_OPCODES[opcode] != 0); + *instr = _Py_MAKECODEUNIT(INSTRUMENTED_OPCODES[opcode], _Py_OPARG(*instr)); + } + i += _PyOpcode_Caches[opcode]; + } + //printf("Instrumented %d instructions\n", instrumented); + code->_co_instrument_version = interp->monitoring_version; + return 0; +} + +void +_PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) +{ + PyInterpreterState *interp = _PyInterpreterState_Get(); + if (interp->events_per_monitoring_tool[tool_id] == events) { + return; + } + interp->events_per_monitoring_tool[tool_id] = events; + for (int e = 0; e < PY_MONITORING_EVENTS; e++) { + _PyMonitoringToolSet tools = { 0 }; + for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { + int event_tool_pair = (interp->events_per_monitoring_tool[t] >> e) & 1; + tools.tools |= (event_tool_pair << t); + } + interp->monitoring_tools_per_event[e] = tools; + } + int multitools = 0; + for (int e = 0; e < PY_MONITORING_EVENTS; e++) { + _PyMonitoringToolSet tools = interp->monitoring_tools_per_event[e]; + if (_Py_popcount32(tools.tools) <= 1) { + interp->sole_tool_plus1[e] = _Py_bit_length(tools.tools); + } + else { + multitools++; + interp->sole_tool_plus1[e] = -1; + } + } + if (multitools) { + /* Only support one tool for now */ + assert(0 && "Only support one tool for now"); + } + interp->monitoring_version++; + + _PyMonitoringEventSet monitored_events = 0; + for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { + monitored_events |= interp->events_per_monitoring_tool[t]; + } + interp->monitored_events = monitored_events; + /* Instrument all executing code objects */ + + PyThreadState* ts = PyInterpreterState_ThreadHead(interp); + + while (ts) { + _PyInterpreterFrame *frame = ts->cframe->current_frame; + while (frame) { + if (frame->owner != FRAME_OWNED_BY_CSTACK) { + _Py_Instrument(frame->f_code, interp); + } + frame = frame->previous; + } + ts = PyThreadState_Next(ts); + } +} + +/* List of objects in monitoring namespace + class Event(enum.IntFlag) + def use_tool_id(id)->None + def free_tool_id(id)->None + def get_events(tool_id: int)->Event + def set_events(tool_id: int, event_set: Event)->None + def get_local_events(tool_id: int, code: CodeType)->Event + def set_local_events(tool_id: int, code: CodeType, event_set: Event)->None + def register_callback(tool_id: int, event: Event, func: Callable)->Optional[Callable] + def insert_marker(tool_id: int, code: CodeType, offset: Event)->None + def remove_marker(tool_id: int, code: CodeType, offset: Event)->None + def restart_events()->None + DISABLE: object +*/ + +/*[clinic input] +module monitoring +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=37257f5987a360cf]*/ +/*[clinic end generated code]*/ + +#include "clinic/instrumentation.c.h" + + +static int +check_valid_tool(int tool_id) +{ + if (tool_id < 0 || tool_id >= PY_INSTRUMENT_SYS_PROFILE) { + PyErr_Format(PyExc_ValueError, "invalid tool ID: %d (must be between 0 and 5)", tool_id); + return -1; + } + return 0; +} + + +/*[clinic input] +monitoring.use_tool + + tool_id: int + name: object + / + +[clinic start generated code]*/ + +static PyObject * +monitoring_use_tool_impl(PyObject *module, int tool_id, PyObject *name) +/*[clinic end generated code: output=d00b74d147bab1e3 input=506e604e1ea75567]*/ + +/*[clinic end generated code]*/ +{ + if (check_valid_tool(tool_id)) { + return NULL; + } + if (!PyUnicode_Check(name)) { + PyErr_SetString(PyExc_ValueError, "tool name must be a str"); + } + PyInterpreterState *interp = _PyInterpreterState_Get(); + if (interp->monitoring_tool_names[tool_id] != NULL) { + PyErr_SetString(PyExc_ValueError, "tool ID is already in use"); + return NULL; + } + interp->monitoring_tool_names[tool_id] = Py_NewRef(name); + Py_RETURN_NONE; +} + +/*[clinic input] +monitoring.free_tool + + tool_id: int + / + +[clinic start generated code]*/ + +static PyObject * +monitoring_free_tool_impl(PyObject *module, int tool_id) +/*[clinic end generated code: output=7893bfdad26f51fa input=919fecb6f63130ed]*/ + +/*[clinic end generated code]*/ +{ + if (check_valid_tool(tool_id)) { + return NULL; + } + PyInterpreterState *interp = _PyInterpreterState_Get(); + Py_CLEAR(interp->monitoring_tool_names[tool_id]); + Py_RETURN_NONE; +} + +/*[clinic input] +monitoring.get_tool + + tool_id: int + / + +[clinic start generated code]*/ + +static PyObject * +monitoring_get_tool_impl(PyObject *module, int tool_id) +/*[clinic end generated code: output=1c05a98b404a9a16 input=eeee9bebd0bcae9d]*/ + +/*[clinic end generated code]*/ +{ + if (check_valid_tool(tool_id)) { + return NULL; + } + PyInterpreterState *interp = _PyInterpreterState_Get(); + PyObject *name = interp->monitoring_tool_names[tool_id]; + if (name == NULL) { + Py_RETURN_NONE; + } + return Py_NewRef(name); +} + +/*[clinic input] +monitoring.register_callback + + + tool_id: int + event: int + func: object + / + +[clinic start generated code]*/ + +static PyObject * +monitoring_register_callback_impl(PyObject *module, int tool_id, int event, + PyObject *func) +/*[clinic end generated code: output=e64daa363004030c input=df6d70ea4cf81007]*/ +{ + if (check_valid_tool(tool_id)) { + return NULL; + } + if (_Py_popcount32(event) != 1) { + PyErr_SetString(PyExc_ValueError, "The callaback can only be set for one event at a time"); + } + int event_id = _Py_bit_length(event)-1; + if (event_id < 0 || event_id >= PY_MONITORING_EVENTS) { + PyErr_Format(PyExc_ValueError, "invalid event: %d", event); + return NULL; + } + if (func == Py_None) { + func = NULL; + } + func = _PyMonitoring_RegisterCallback(tool_id, event_id, func); + if (func == NULL) { + Py_RETURN_NONE; + } + return func; +} + +/*[clinic input] +monitoring.get_events + + tool_id: int + / + +[clinic start generated code]*/ + +static PyObject * +monitoring_get_events_impl(PyObject *module, int tool_id) +/*[clinic end generated code: output=d8b92576efaa12f9 input=49b77c12cc517025]*/ +{ + if (tool_id < 0 || tool_id >= PY_INSTRUMENT_SYS_PROFILE) { + PyErr_SetString(PyExc_ValueError, "tool ID must be between 0 and 5"); + return NULL; + } + PyInterpreterState *interp = _PyInterpreterState_Get(); + _PyMonitoringEventSet event_set = interp->events_per_monitoring_tool[tool_id]; + return PyLong_FromUnsignedLong(event_set); +} + +/*[clinic input] +monitoring.set_events + + tool_id: int + event_set: int + / + +[clinic start generated code]*/ + +static PyObject * +monitoring_set_events_impl(PyObject *module, int tool_id, int event_set) +/*[clinic end generated code: output=1916c1e49cfb5bdb input=a77ba729a242142b]*/ +{ + if (tool_id < 0 || tool_id >= PY_INSTRUMENT_SYS_PROFILE) { + PyErr_SetString(PyExc_ValueError, "tool ID must be between 0 and 5"); + return NULL; + } + if (event_set < 0 || event_set >= (1 << PY_MONITORING_EVENTS)) { + PyErr_Format(PyExc_ValueError, "invalid event set 0x%x", event_set); + return NULL; + } + _PyMonitoring_SetEvents(tool_id, event_set); + Py_RETURN_NONE; +} + + +static PyMethodDef methods[] = { + MONITORING_USE_TOOL_METHODDEF + MONITORING_FREE_TOOL_METHODDEF + MONITORING_GET_TOOL_METHODDEF + MONITORING_REGISTER_CALLBACK_METHODDEF + MONITORING_GET_EVENTS_METHODDEF + MONITORING_SET_EVENTS_METHODDEF + {NULL, NULL} // sentinel +}; + +static struct PyModuleDef monitoring_module = { + PyModuleDef_HEAD_INIT, + "sys.monitoring", + NULL, + -1, /* multiple "initialization" just copies the module dict. */ + methods, + NULL, + NULL, + NULL, + NULL +}; + +static int +add_power2_constant(PyObject *obj, const char *name, int i) +{ + PyObject *val = PyLong_FromLong(1< +#include "Python.h" +#include "pycore_instruments.h" +#include "pycore_pyerrors.h" +#include "pycore_pymem.h" +#include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_sysmodule.h" + +typedef struct _PyLegacyEventHandler { + PyObject_HEAD + vectorcallfunc vectorcall; + int event; +} _PyLegacyEventHandler; + +static void +dealloc(_PyLegacyEventHandler *self) +{ + PyObject_Free(self); +} + + +/* The Py_tracefunc function expects the following arguments: + * frame: FrameObject + * kind: c-int + * arg: The arg + */ + +static PyObject * +call_profile_func(_PyLegacyEventHandler *self, PyObject *arg) +{ + PyThreadState *tstate = _PyThreadState_GET(); + if (tstate->c_profilefunc == NULL) { + Py_RETURN_NONE; + } + PyFrameObject* frame = PyEval_GetFrame(); + Py_INCREF(frame); + assert(frame != NULL); + int err = tstate->c_profilefunc(tstate->c_profileobj, frame, self->event, arg); + Py_DECREF(frame); + if (err) { + return NULL; + } + Py_RETURN_NONE; +} + + +static PyObject * +sys_profile_func2( + _PyLegacyEventHandler *self, PyObject *const *args, + size_t nargsf, PyObject *kwnames +) { + assert(kwnames == NULL); + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + assert(nargs == 2); + return call_profile_func(self, Py_None); +} + +static PyObject * +sys_profile_func3( + _PyLegacyEventHandler *self, PyObject *const *args, + size_t nargsf, PyObject *kwnames +) { + assert(kwnames == NULL); + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + assert(nargs == 3); + return call_profile_func(self, args[2]); +} + +static PyObject * +sys_profile_call_or_return( + _PyLegacyEventHandler *self, PyObject *const *args, + size_t nargsf, PyObject *kwnames +) { + assert(kwnames == NULL); + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + assert(nargs == 3); + PyObject *callable = args[2]; + if (PyFunction_Check(callable) || PyMethod_Check(callable)) { + Py_RETURN_NONE; + } + return call_profile_func(self, callable); +} + +static PyObject * +sys_profile_exception_func( + _PyLegacyEventHandler *self, PyObject *const *args, + size_t nargsf, PyObject *kwnames +) { + assert(kwnames == NULL); + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + assert(nargs == 3); + _PyErr_StackItem *err_info = _PyErr_GetTopmostException(_PyThreadState_GET()); + PyObject *arg = _PyErr_StackItemToExcInfoTuple(err_info); + if (arg == NULL) { + return NULL; + } + PyObject *res = call_profile_func(self, arg); + Py_DECREF(arg); + return res; +} + +static PyObject * +sys_trace_none_func( + _PyLegacyEventHandler *self, PyObject *const *args, + size_t nargsf, PyObject *kwnames +) { + assert(kwnames == NULL); + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + assert(nargs == 3); + return call_profile_func(self, Py_None); +} + + +PyTypeObject _PyLegacyEventHandler_Type = { + + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "sys.profile_event_handler", + sizeof(_PyLegacyEventHandler), + .tp_dealloc = (destructor)dealloc, + .tp_vectorcall_offset = offsetof(_PyLegacyEventHandler, vectorcall), + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_VECTORCALL, + .tp_call = PyVectorcall_Call, +}; + +static int +set_callbacks(int tool, vectorcallfunc vectorcall, int legacy_event, int event1, int event2) +{ + _PyLegacyEventHandler *callback = + PyObject_NEW(_PyLegacyEventHandler, &_PyLegacyEventHandler_Type); + if (callback == NULL) { + return -1; + } + callback->vectorcall = vectorcall; + callback->event = legacy_event; + Py_XDECREF(_PyMonitoring_RegisterCallback(tool, event1, (PyObject *)callback)); + if (event2 >= 0) { + Py_XDECREF(_PyMonitoring_RegisterCallback(tool, event2, (PyObject *)callback)); + } + Py_DECREF(callback); + return 0; +} + +#ifndef NDEBUG +/* Ensure that tstate is valid: sanity check for PyEval_AcquireThread() and + PyEval_RestoreThread(). Detect if tstate memory was freed. It can happen + when a thread continues to run after Python finalization, especially + daemon threads. */ +static int +is_tstate_valid(PyThreadState *tstate) +{ + assert(!_PyMem_IsPtrFreed(tstate)); + assert(!_PyMem_IsPtrFreed(tstate->interp)); + return 1; +} +#endif + +int +_PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) +{ + assert(is_tstate_valid(tstate)); + /* The caller must hold the GIL */ + assert(PyGILState_Check()); + + /* Call _PySys_Audit() in the context of the current thread state, + even if tstate is not the current thread state. */ + PyThreadState *current_tstate = _PyThreadState_GET(); + if (_PySys_Audit(current_tstate, "sys.setprofile", NULL) < 0) { + return -1; + } + int delta = (func != NULL) - (tstate->c_profilefunc != NULL); + tstate->c_profilefunc = func; + PyObject *old_profileobj = tstate->c_profileobj; + tstate->c_profileobj = Py_XNewRef(arg); + /* Flag that tracing or profiling is turned on */ + _PyThreadState_UpdateTracingState(tstate); + + /* Setup PEP 669 monitoring callbacks and events. */ + tstate->interp->sys_profiling_threads += delta; + assert(tstate->interp->sys_profiling_threads >= 0); + if (!tstate->interp->sys_profile_initialized) { + tstate->interp->sys_profile_initialized = true; + if (set_callbacks(PY_INSTRUMENT_SYS_PROFILE, + (vectorcallfunc)sys_profile_func2, PyTrace_CALL, + PY_MONITORING_EVENT_PY_START, PY_MONITORING_EVENT_PY_RESUME)) { + return -1; + } + if (set_callbacks(PY_INSTRUMENT_SYS_PROFILE, + (vectorcallfunc)sys_profile_func3, PyTrace_RETURN, + PY_MONITORING_EVENT_PY_RETURN, PY_MONITORING_EVENT_PY_YIELD)) { + return -1; + } + if (set_callbacks(PY_INSTRUMENT_SYS_PROFILE, + (vectorcallfunc)sys_profile_func2, PyTrace_RETURN, + PY_MONITORING_EVENT_PY_UNWIND, -1)) { + return -1; + } + if (set_callbacks(PY_INSTRUMENT_SYS_PROFILE, + (vectorcallfunc)sys_profile_call_or_return, PyTrace_C_CALL, + PY_MONITORING_EVENT_CALL, -1)) { + return -1; + } + if (set_callbacks(PY_INSTRUMENT_SYS_PROFILE, + (vectorcallfunc)sys_profile_call_or_return, PyTrace_C_RETURN, + PY_MONITORING_EVENT_C_RETURN, -1)) { + return -1; + } + if (set_callbacks(PY_INSTRUMENT_SYS_PROFILE, + (vectorcallfunc)sys_profile_call_or_return, PyTrace_C_EXCEPTION, + PY_MONITORING_EVENT_C_RAISE, -1)) { + return -1; + } + } + if (tstate->interp->sys_profiling_threads && delta) { + uint32_t events = + (1 << PY_MONITORING_EVENT_PY_START) | (1 << PY_MONITORING_EVENT_PY_RESUME) | + (1 << PY_MONITORING_EVENT_PY_RETURN) | (1 << PY_MONITORING_EVENT_PY_YIELD) | + (1 << PY_MONITORING_EVENT_CALL) | (1 << PY_MONITORING_EVENT_PY_UNWIND) | + (1 << PY_MONITORING_EVENT_C_RETURN) | + (1 << PY_MONITORING_EVENT_C_RAISE); + _PyMonitoring_SetEvents(PY_INSTRUMENT_SYS_PROFILE, events); + } + else if (tstate->interp->sys_profiling_threads == 0) { + _PyMonitoring_SetEvents(PY_INSTRUMENT_SYS_PROFILE, 0); + } + // gh-98257: Only call Py_XDECREF() once the new profile function is fully + // set, so it's safe to call sys.setprofile() again (reentrant call). + Py_XDECREF(old_profileobj); + return 0; +} + +int +_PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) +{ + assert(is_tstate_valid(tstate)); + /* The caller must hold the GIL */ + assert(PyGILState_Check()); + + /* Call _PySys_Audit() in the context of the current thread state, + even if tstate is not the current thread state. */ + PyThreadState *current_tstate = _PyThreadState_GET(); + if (_PySys_Audit(current_tstate, "sys.settrace", NULL) < 0) { + return -1; + } + + int delta = (func != NULL) - (tstate->c_tracefunc != NULL); + tstate->c_tracefunc = func; + PyObject *old_traceobj = tstate->c_traceobj; + tstate->c_traceobj = Py_XNewRef(arg); + /* Flag that tracing or profiling is turned on */ + _PyThreadState_UpdateTracingState(tstate); + + /* Setup PEP 669 monitoring callbacks and events. */ + tstate->interp->sys_tracing_threads += delta; + assert(tstate->interp->sys_tracing_threads >= 0); + if (!tstate->interp->sys_trace_initialized) { + tstate->interp->sys_trace_initialized = true; + if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, + (vectorcallfunc)sys_profile_func2, PyTrace_CALL, + PY_MONITORING_EVENT_PY_START, PY_MONITORING_EVENT_PY_RESUME)) { + return -1; + } + if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, + (vectorcallfunc)sys_profile_func3, PyTrace_RETURN, + PY_MONITORING_EVENT_PY_RETURN, PY_MONITORING_EVENT_PY_YIELD)) { + return -1; + } + if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, + (vectorcallfunc)sys_profile_exception_func, PyTrace_EXCEPTION, + PY_MONITORING_EVENT_RAISE, -1)) { + return -1; + } + if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, + (vectorcallfunc)sys_trace_none_func, PyTrace_LINE, + PY_MONITORING_EVENT_LINE, -1)) { + return -1; + } + if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, + (vectorcallfunc)sys_trace_none_func, PyTrace_OPCODE, + PY_MONITORING_EVENT_INSTRUCTION, -1)) { + return -1; + } + } + if (tstate->interp->sys_tracing_threads) { + uint32_t events = + (1 << PY_MONITORING_EVENT_PY_START) | (1 << PY_MONITORING_EVENT_PY_RESUME) | + (1 << PY_MONITORING_EVENT_PY_RETURN) | (1 << PY_MONITORING_EVENT_PY_YIELD) | + (1 << PY_MONITORING_EVENT_RAISE) | (1 << PY_MONITORING_EVENT_LINE); + if (tstate->interp->f_opcode_trace_set) { + events |= (1 << PY_MONITORING_EVENT_INSTRUCTION); + } + _PyMonitoring_SetEvents(PY_INSTRUMENT_SYS_PROFILE, events); + } + else { + _PyMonitoring_SetEvents(PY_INSTRUMENT_SYS_PROFILE, 0); + } + Py_XDECREF(old_traceobj); + return 0; +} diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 237d3b946b1066..9ec1dd4f09b7cf 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -239,11 +239,11 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, + &&TARGET_INSTRUMENTED_RESUME, + &&TARGET_INSTRUMENTED_CALL, + &&TARGET_INSTRUMENTED_RETURN_VALUE, + &&TARGET_INSTRUMENTED_YIELD_VALUE, + &&TARGET_INSTRUMENTED_CALL_FUNCTION_EX, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, diff --git a/Python/pystate.c b/Python/pystate.c index 1c92ce5d954458..8bbd529d45801a 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -309,9 +309,15 @@ init_interpreter(PyInterpreterState *interp, _PyGC_InitState(&interp->gc); PyConfig_InitPythonConfig(&interp->config); _PyType_InitCache(interp); - for(int i = 0; i < PY_INSTRUMENT_EVENTS; i++) { - interp->instrumented[i] = 0; - interp->instrument_callables[i] = Py_NewRef(Py_None); + for(int i = 0; i < PY_MONITORING_EVENTS; i++) { + interp->monitoring_tools_per_event[i].tools = 0; + for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { + interp->tools[t].instrument_callables[i] = NULL; + } + } + interp->monitored_events = 0; + for(int i = 0; i < PY_MONITORING_TOOL_IDS; i++) { + interp->events_per_monitoring_tool[i] = 0; } interp->f_opcode_trace_set = false; interp->_initialized = 1; diff --git a/Python/specialize.c b/Python/specialize.c index f84596751f9186..01bf1e7b5cc37f 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1707,6 +1707,7 @@ _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, PyObject *kwnames) { assert(_PyOpcode_Caches[CALL] == INLINE_CACHE_ENTRIES_CALL); + assert(_Py_OPCODE(*instr) != INSTRUMENTED_CALL); _PyCallCache *cache = (_PyCallCache *)(instr + 1); int fail; if (PyCFunction_CheckExact(callable)) { diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 5de684e7195bff..7e0f717c7eb9b5 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -3369,6 +3369,7 @@ _PySys_SetPreliminaryStderr(PyObject *sysdict) return _PyStatus_ERR("can't set preliminary stderr"); } +PyObject *_Py_CreateMonitoringObject(void); /* Create sys module without all attributes. _PySys_UpdateConfig() should be called later to add remaining attributes. */ @@ -3414,6 +3415,14 @@ _PySys_Create(PyThreadState *tstate, PyObject **sysmod_p) goto error; } + PyObject *monitoring = _Py_CreateMonitoringObject(); + if (monitoring == NULL) { + goto error; + } + if (PyDict_SetItemString(sysdict, "monitoring", monitoring) < 0) { + goto error; + } + assert(!_PyErr_Occurred(tstate)); *sysmod_p = sysmod; From 4edb7b7ee14cb5ca08e94e4be08765c2b140744b Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 28 Nov 2022 15:03:18 +0000 Subject: [PATCH 003/116] Support disabling and restarting. --- Include/cpython/code.h | 1 + Include/cpython/pystate.h | 1 + Include/internal/pycore_interp.h | 1 + Lib/test/test_monitoring.py | 49 ++++++++++++++++- Python/bytecodes.c | 19 +++---- Python/ceval.c | 1 + Python/clinic/instrumentation.c.h | 19 ++++++- Python/generated_cases.c.h | 19 +++---- Python/instrumentation.c | 87 ++++++++++++++++++++----------- 9 files changed, 146 insertions(+), 51 deletions(-) diff --git a/Include/cpython/code.h b/Include/cpython/code.h index da05f316226e69..fdfa93cd75dd7f 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -99,6 +99,7 @@ typedef struct { _PyCoCached *_co_cached; /* cached co_* attributes */ \ int _co_firsttraceable; /* index of first traceable instruction */ \ uint64_t _co_instrument_version; /* current instrumentation version */ \ + uint32_t _co_monitored_events; /* current instrumentation */ \ char *_co_linearray; /* array of line offsets */ \ /* Scratch space for extra data relating to the code object. \ Type is a void* to keep the format private in codeobject.c to force \ diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 99bc0a89dc0e5f..3515d7e04b1a64 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -133,6 +133,7 @@ struct _ts { the trace/profile. */ int tracing; int tracing_what; /* The event currently being traced, if any. */ + uint8_t monitoring; /* Pointer to current _PyCFrame in the C stack frame of the currently, * or most recently, executing _PyEval_EvalFrameDefault. */ diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 1a606a72ae2fd1..eb3a96cd1d460c 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -90,6 +90,7 @@ struct _is { PyInterpreterState *next; uint64_t monitoring_version; + uint64_t last_restart_version;; struct pythreads { uint64_t next_unique_id; diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 4e3428effafc7f..25e959a2c5ce01 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -23,7 +23,8 @@ def g1(): for _ in gen(): pass -TEST_TOOL = 4 +TEST_TOOL = 3 +TEST_TOOL2 = 4 class MonitoringBaseTest(unittest.TestCase): @@ -40,7 +41,7 @@ def test_has_objects(self): m.register_callback # m.insert_marker # m.remove_marker - # m.restart_events + m.restart_events m.DISABLE def test_tool(self): @@ -269,3 +270,47 @@ def call(code, offset, callable): self.assertEqual(stack, [sys._getframe()]) +class CounterWithDisable: + def __init__(self): + self.disable = False + self.count = 0 + def __call__(self, *args): + self.count += 1 + if self.disable: + return sys.monitoring.DISABLE + + +class MontoringDisableAndRestartTest(unittest.TestCase): + + def test_disable(self): + counter = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, counter) + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + self.assertEqual(counter.count, 0) + counter.count = 0 + f1() + self.assertEqual(counter.count, 1) + counter.disable = True + counter.count = 0 + f1() + self.assertEqual(counter.count, 1) + counter.count = 0 + f1() + self.assertEqual(counter.count, 0) + + def test_restart(self): + counter = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, counter) + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + counter.disable = True + f1() + counter.count = 0 + f1() + self.assertEqual(counter.count, 0) + sys.monitoring.restart_events() + counter.count = 0 + f1() + self.assertEqual(counter.count, 1) + + + diff --git a/Python/bytecodes.c b/Python/bytecodes.c index d0ccb70b40f0da..1bd36f99185328 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -116,12 +116,15 @@ dummy_func( if (_Py_Instrument(frame->f_code, tstate->interp)) { goto error; } + next_instr--; } - int err = _Py_call_instrumentation( - tstate, oparg, frame, next_instr-1); - ERROR_IF(err, error); - if (_Py_atomic_load_relaxed_int32(eval_breaker)) { - goto handle_eval_breaker; + else { + int err = _Py_call_instrumentation( + tstate, oparg, frame, next_instr-1); + ERROR_IF(err, error); + if (_Py_atomic_load_relaxed_int32(eval_breaker)) { + goto handle_eval_breaker; + } } } @@ -132,11 +135,9 @@ dummy_func( if (frame->f_code->_co_instrument_version != tstate->interp->monitoring_version) { int err = _Py_Instrument(frame->f_code, tstate->interp); ERROR_IF(err, error); - err = _Py_call_instrumentation( - tstate, oparg, frame, next_instr-1); - ERROR_IF(err, error); + next_instr--; } - if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { + else if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { goto handle_eval_breaker; } } diff --git a/Python/ceval.c b/Python/ceval.c index b3695ac7af1d98..ce061ee3d52679 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2257,6 +2257,7 @@ monitor_handled(PyThreadState *tstate, void PyThreadState_EnterTracing(PyThreadState *tstate) { + assert(tstate->tracing >= 0); tstate->tracing++; } diff --git a/Python/clinic/instrumentation.c.h b/Python/clinic/instrumentation.c.h index 45060157416114..e677ea86d3647d 100644 --- a/Python/clinic/instrumentation.c.h +++ b/Python/clinic/instrumentation.c.h @@ -193,4 +193,21 @@ monitoring_set_events(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=876a03223495a81d input=a9049054013a1b77]*/ + +PyDoc_STRVAR(monitoring_restart_events__doc__, +"restart_events($module, /)\n" +"--\n" +"\n"); + +#define MONITORING_RESTART_EVENTS_METHODDEF \ + {"restart_events", (PyCFunction)monitoring_restart_events, METH_NOARGS, monitoring_restart_events__doc__}, + +static PyObject * +monitoring_restart_events_impl(PyObject *module); + +static PyObject * +monitoring_restart_events(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return monitoring_restart_events_impl(module); +} +/*[clinic end generated code: output=3997247efd06367f input=a9049054013a1b77]*/ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index b9a9fe90a13c4a..6f6e20f06c4456 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -13,12 +13,15 @@ if (_Py_Instrument(frame->f_code, tstate->interp)) { goto error; } + next_instr--; } - int err = _Py_call_instrumentation( - tstate, oparg, frame, next_instr-1); - if (err) goto error; - if (_Py_atomic_load_relaxed_int32(eval_breaker)) { - goto handle_eval_breaker; + else { + int err = _Py_call_instrumentation( + tstate, oparg, frame, next_instr-1); + if (err) goto error; + if (_Py_atomic_load_relaxed_int32(eval_breaker)) { + goto handle_eval_breaker; + } } DISPATCH(); } @@ -30,11 +33,9 @@ if (frame->f_code->_co_instrument_version != tstate->interp->monitoring_version) { int err = _Py_Instrument(frame->f_code, tstate->interp); if (err) goto error; - err = _Py_call_instrumentation( - tstate, oparg, frame, next_instr-1); - if (err) goto error; + next_instr--; } - if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { + else if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { goto handle_eval_breaker; } DISPATCH(); diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 235cfce6c44635..a9fd50905adffb 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -24,9 +24,11 @@ call_multiple_instruments( } static const uint8_t DE_INSTRUMENT[256] = { + [RESUME] = RESUME, [INSTRUMENTED_RESUME] = RESUME, [INSTRUMENTED_RETURN_VALUE] = RETURN_VALUE, [INSTRUMENTED_CALL] = CALL, + [INSTRUMENTED_CALL_FUNCTION_EX] = CALL_FUNCTION_EX, [INSTRUMENTED_YIELD_VALUE] = YIELD_VALUE, /* [INSTRUMENTED_JUMP_FORWARD] = JUMP_FORWARD, @@ -55,18 +57,24 @@ call_instrument( } return call_multiple_instruments(interp, code, event, args, nargsf, instr); } - PyObject *instrument = interp->tools[sole_tool_plus1-1].instrument_callables[event]; + int sole_tool = sole_tool_plus1-1; + if (tstate->monitoring & (1 << sole_tool)) { + return 0; + } + PyObject *instrument = interp->tools[sole_tool].instrument_callables[event]; if (instrument == NULL) { return 0; } + tstate->monitoring |= (1 << sole_tool); PyObject *res = PyObject_Vectorcall(instrument, args, nargsf, NULL); + tstate->monitoring &= ~(1 << sole_tool); if (res == NULL) { return -1; } if (res == &DISABLE) { /* Remove this instrument */ assert(DE_INSTRUMENT[_Py_OPCODE(*instr)] != 0); - _Py_SET_OPCODE(*instr, DE_INSTRUMENT[_Py_OPCODE(*instr)]); + *instr = _Py_MAKECODEUNIT(DE_INSTRUMENT[_Py_OPCODE(*instr)], _Py_OPARG(*instr)); } Py_DECREF(res); return 0; @@ -78,17 +86,12 @@ _Py_call_instrumentation( PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) { - if (tstate->tracing) { - return 0; - } - tstate->tracing++; PyCodeObject *code = frame->f_code; int instruction_offset = instr - _PyCode_CODE(code); PyObject *instruction_offset_obj = PyLong_FromSsize_t(instruction_offset); PyObject *args[3] = { NULL, (PyObject *)code, instruction_offset_obj }; int err = call_instrument(tstate, code, event, &args[1], 2 | PY_VECTORCALL_ARGUMENTS_OFFSET, instr); Py_DECREF(instruction_offset_obj); - tstate->tracing--; return err; } @@ -97,17 +100,12 @@ _Py_call_instrumentation_arg( PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg) { - if (tstate->tracing) { - return 0; - } - tstate->tracing++; PyCodeObject *code = frame->f_code; int instruction_offset = instr - _PyCode_CODE(code); PyObject *instruction_offset_obj = PyLong_FromSsize_t(instruction_offset); PyObject *args[4] = { NULL, (PyObject *)code, instruction_offset_obj, arg }; int err = call_instrument(tstate, code, event, &args[1], 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, instr); Py_DECREF(instruction_offset_obj); - tstate->tracing--; return err; } @@ -117,10 +115,6 @@ _Py_call_instrumentation_exc( _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg) { assert(_PyErr_Occurred(tstate)); - if (tstate->tracing) { - return; - } - tstate->tracing++; PyObject *type, *value, *traceback; _PyErr_Fetch(tstate, &type, &value, &traceback); PyCodeObject *code = frame->f_code; @@ -139,7 +133,6 @@ _Py_call_instrumentation_exc( } assert(_PyErr_Occurred(tstate)); Py_DECREF(instruction_offset_obj); - tstate->tracing--; } PyObject * @@ -212,9 +205,22 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) } //printf("Instrumenting code object at %p, code version: %ld interpreter version %ld\n", // code, code->_co_instrument_version, interp->monitoring_version); - + /* Avoid instrumenting code that has been disabled. + * Only instrument new events + */ + _PyMonitoringEventSet new_events = interp->monitored_events & ~code->_co_monitored_events; + _PyMonitoringEventSet removed_events = code->_co_monitored_events & ~interp->monitored_events; + if (interp->last_restart_version > code->_co_instrument_version) { + new_events = interp->monitored_events; + removed_events = 0; + } + code->_co_monitored_events = interp->monitored_events; + code->_co_instrument_version = interp->monitoring_version; + assert((new_events & removed_events) == 0); + if ((new_events | removed_events) == 0) { + return 0; + } /* Insert basic instrumentation */ - int instrumented = 0; int code_len = (int)Py_SIZE(code); for (int i = 0; i < code_len; i++) { _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; @@ -227,15 +233,18 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) continue; } _PyMonitoringEventSet events = EVENTS_FOR_OPCODE[opcode]; - if (events & interp->monitored_events) { - instrumented++; + if (new_events & events) { assert(INSTRUMENTED_OPCODES[opcode] != 0); *instr = _Py_MAKECODEUNIT(INSTRUMENTED_OPCODES[opcode], _Py_OPARG(*instr)); } + if (removed_events & events) { + *instr = _Py_MAKECODEUNIT(opcode, _Py_OPARG(*instr)); + if (_PyOpcode_Caches[opcode]) { + instr[1] = adaptive_counter_warmup(); + } + } i += _PyOpcode_Caches[opcode]; } - //printf("Instrumented %d instructions\n", instrumented); - code->_co_instrument_version = interp->monitoring_version; return 0; } @@ -321,7 +330,7 @@ static int check_valid_tool(int tool_id) { if (tool_id < 0 || tool_id >= PY_INSTRUMENT_SYS_PROFILE) { - PyErr_Format(PyExc_ValueError, "invalid tool ID: %d (must be between 0 and 5)", tool_id); + PyErr_Format(PyExc_ValueError, "invalid tool %d (must be between 0 and 5)", tool_id); return -1; } return 0; @@ -351,7 +360,7 @@ monitoring_use_tool_impl(PyObject *module, int tool_id, PyObject *name) } PyInterpreterState *interp = _PyInterpreterState_Get(); if (interp->monitoring_tool_names[tool_id] != NULL) { - PyErr_SetString(PyExc_ValueError, "tool ID is already in use"); + PyErr_Format(PyExc_ValueError, "tool %d is already in use", tool_id); return NULL; } interp->monitoring_tool_names[tool_id] = Py_NewRef(name); @@ -429,7 +438,7 @@ monitoring_register_callback_impl(PyObject *module, int tool_id, int event, } int event_id = _Py_bit_length(event)-1; if (event_id < 0 || event_id >= PY_MONITORING_EVENTS) { - PyErr_Format(PyExc_ValueError, "invalid event: %d", event); + PyErr_Format(PyExc_ValueError, "invalid event %d", event); return NULL; } if (func == Py_None) { @@ -454,8 +463,7 @@ static PyObject * monitoring_get_events_impl(PyObject *module, int tool_id) /*[clinic end generated code: output=d8b92576efaa12f9 input=49b77c12cc517025]*/ { - if (tool_id < 0 || tool_id >= PY_INSTRUMENT_SYS_PROFILE) { - PyErr_SetString(PyExc_ValueError, "tool ID must be between 0 and 5"); + if (check_valid_tool(tool_id)) { return NULL; } PyInterpreterState *interp = _PyInterpreterState_Get(); @@ -476,8 +484,7 @@ static PyObject * monitoring_set_events_impl(PyObject *module, int tool_id, int event_set) /*[clinic end generated code: output=1916c1e49cfb5bdb input=a77ba729a242142b]*/ { - if (tool_id < 0 || tool_id >= PY_INSTRUMENT_SYS_PROFILE) { - PyErr_SetString(PyExc_ValueError, "tool ID must be between 0 and 5"); + if (check_valid_tool(tool_id)) { return NULL; } if (event_set < 0 || event_set >= (1 << PY_MONITORING_EVENTS)) { @@ -488,6 +495,25 @@ monitoring_set_events_impl(PyObject *module, int tool_id, int event_set) Py_RETURN_NONE; } +/*[clinic input] +monitoring.restart_events + +[clinic start generated code]*/ + +static PyObject * +monitoring_restart_events_impl(PyObject *module) +/*[clinic end generated code: output=e025dd5ba33314c4 input=add8a855063c8008]*/ +{ + /* We want to ensure that: + * last restart version < current version + * last restart version > instrumented version for all code objects + */ + PyInterpreterState *interp = _PyInterpreterState_Get(); + interp->last_restart_version = interp->monitoring_version + 1; + interp->monitoring_version += 2; + Py_RETURN_NONE; +} + static PyMethodDef methods[] = { MONITORING_USE_TOOL_METHODDEF @@ -496,6 +522,7 @@ static PyMethodDef methods[] = { MONITORING_REGISTER_CALLBACK_METHODDEF MONITORING_GET_EVENTS_METHODDEF MONITORING_SET_EVENTS_METHODDEF + MONITORING_RESTART_EVENTS_METHODDEF {NULL, NULL} // sentinel }; From 852c40b6757fecc8d9997734b698812d0cf95536 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 28 Nov 2022 18:27:15 +0000 Subject: [PATCH 004/116] Support multiple tools per event. --- Include/cpython/code.h | 12 +- Include/cpython/pystate.h | 1 - Include/internal/pycore_instruments.h | 25 +- Include/internal/pycore_interp.h | 6 +- Lib/test/test_monitoring.py | 36 ++- Objects/codeobject.c | 7 +- Python/bytecodes.c | 5 +- Python/ceval.c | 6 +- Python/generated_cases.c.h | 5 +- Python/instrumentation.c | 400 +++++++++++++++++++------- Python/pystate.c | 7 +- 11 files changed, 372 insertions(+), 138 deletions(-) diff --git a/Include/cpython/code.h b/Include/cpython/code.h index fdfa93cd75dd7f..49b2a0e7108665 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -7,6 +7,12 @@ extern "C" { #endif +#define PY_MONITORING_EVENTS 16 + +typedef struct _Py_MonitoringMatrix { + uint8_t tools[PY_MONITORING_EVENTS]; +} _Py_MonitoringMatrix; + /* Each instruction in a code object is a fixed-width value, * currently 2 bytes: 1-byte opcode + 1-byte oparg. The EXTENDED_ARG * opcode allows for larger values but the current limit is 3 uses @@ -99,7 +105,11 @@ typedef struct { _PyCoCached *_co_cached; /* cached co_* attributes */ \ int _co_firsttraceable; /* index of first traceable instruction */ \ uint64_t _co_instrument_version; /* current instrumentation version */ \ - uint32_t _co_monitored_events; /* current instrumentation */ \ + uint8_t _co_monitoring_data_per_instruction; /* Number of bytes per instruction */ \ + uint8_t _co_line_data_offset; /* Offset in data for line data */ \ + uint8_t _co_opcode_data_offset; /* Offset in data for opcode data */ \ + _Py_MonitoringMatrix _co_monitoring_matrix; \ + uint8_t *_co_monitoring_data; /* array of data for monitoring */ \ char *_co_linearray; /* array of line offsets */ \ /* Scratch space for extra data relating to the code object. \ Type is a void* to keep the format private in codeobject.c to force \ diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 3515d7e04b1a64..99bc0a89dc0e5f 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -133,7 +133,6 @@ struct _ts { the trace/profile. */ int tracing; int tracing_what; /* The event currently being traced, if any. */ - uint8_t monitoring; /* Pointer to current _PyCFrame in the C stack frame of the currently, * or most recently, executing _PyEval_EvalFrameDefault. */ diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index 4c3a1dbcb3a4cb..781aa568d52508 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -6,13 +6,15 @@ #include "pycore_bitutils.h" // _Py_popcount32 #include "pycore_frame.h" +#include "cpython/code.h" + #ifdef __cplusplus extern "C" { #endif #define PY_MONITORING_TOOL_IDS 8 -/* Require dynamic instrumentation */ +/* Require bytecode instrumentation */ #define PY_MONITORING_EVENT_PY_START 0 #define PY_MONITORING_EVENT_PY_RESUME 1 @@ -24,16 +26,21 @@ extern "C" { #define PY_MONITORING_EVENT_JUMP 7 #define PY_MONITORING_EVENT_BRANCH 8 +#define PY_MONITORING_INSTRUMENTED_EVENTS 9 + +/* Grouped events */ #define PY_MONITORING_EVENT_C_RETURN 9 -#define PY_MONITORING_EVENT_PY_THROW 10 -#define PY_MONITORING_EVENT_RAISE 11 -#define PY_MONITORING_EVENT_EXCEPTION_HANDLED 12 -#define PY_MONITORING_EVENT_C_RAISE 13 +#define PY_MONITORING_EVENT_C_RAISE 10 + +/* Exceptional events */ + +#define PY_MONITORING_EVENT_PY_THROW 11 +#define PY_MONITORING_EVENT_RAISE 12 +#define PY_MONITORING_EVENT_EXCEPTION_HANDLED 13 #define PY_MONITORING_EVENT_PY_UNWIND 14 /* #define INSTRUMENT_EVENT_BRANCH_NOT_TAKEN xxx -- If we can afford this */ -#define PY_MONITORING_EVENTS 15 /* Temporary and internal events */ @@ -42,12 +49,6 @@ extern "C" { typedef uint32_t _PyMonitoringEventSet; -typedef uint16_t _PyMonitoringSet; - -typedef struct _pytoolset { - uint8_t tools; -} _PyMonitoringToolSet; - /* Reserved IDs */ #define PY_INSTRUMENT_PEP_523 5 diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index eb3a96cd1d460c..a614931da6e6b3 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -195,11 +195,11 @@ struct _is { struct callable_cache callable_cache; PyCodeObject *interpreter_trampoline; - _PyMonitoringToolSet monitoring_tools_per_event[PY_MONITORING_EVENTS]; - _PyMonitoringEventSet events_per_monitoring_tool[PY_MONITORING_TOOL_IDS]; - _PyMonitoringEventSet monitored_events; + _Py_MonitoringMatrix monitoring_matrix; /* The index (plus one) of the sole tool. 0 if 0 or 2+ tools */ int8_t sole_tool_plus1[PY_MONITORING_EVENTS]; + bool multiple_tools; + uint8_t required_monitoring_bytes; /* Tools numbered 1-8. 0 is the dispatcher/sole tool */ struct _instrumentation_tool tools[PY_MONITORING_TOOL_IDS]; PyObject *monitoring_tool_names[PY_MONITORING_TOOL_IDS]; diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 25e959a2c5ce01..0bb3eb94b5c226 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -49,9 +49,13 @@ def test_tool(self): self.assertEqual(sys.monitoring.get_tool(TEST_TOOL), "MonitoringTest.Tool") sys.monitoring.free_tool(TEST_TOOL) self.assertEqual(sys.monitoring.get_tool(TEST_TOOL), None) - sys.monitoring.set_events(TEST_TOOL, 19) - self.assertEqual(sys.monitoring.get_events(TEST_TOOL), 19) + sys.monitoring.set_events(TEST_TOOL, 15) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL), 15) sys.monitoring.set_events(TEST_TOOL, 0) + with self.assertRaises(ValueError): + sys.monitoring.set_events(TEST_TOOL, sys.monitoring.events.C_RETURN) + with self.assertRaises(ValueError): + sys.monitoring.set_events(TEST_TOOL, sys.monitoring.events.C_RAISE) class MonitoringCountTest(unittest.TestCase): @@ -66,7 +70,10 @@ def __call__(self, *args): counter = Counter() sys.monitoring.register_callback(TEST_TOOL, event, counter) - sys.monitoring.set_events(TEST_TOOL, event) + if event == E.C_RETURN or event == E.C_RAISE: + sys.monitoring.set_events(TEST_TOOL, E.CALL) + else: + sys.monitoring.set_events(TEST_TOOL, event) self.assertEqual(counter.count, 0) counter.count = 0 func() @@ -101,17 +108,18 @@ def test_c_return_count(self): (E.PY_RESUME, "resume"), (E.PY_RETURN, "return"), (E.PY_YIELD, "yield"), - (E.C_RAISE, "c_raise"), - (E.C_RETURN, "c_return"), (E.JUMP, "jump"), (E.BRANCH, "branch"), (E.RAISE, "raise"), (E.PY_UNWIND, "unwind"), (E.EXCEPTION_HANDLED, "exception_handled"), + (E.C_RAISE, "c_raise"), + (E.C_RETURN, "c_return"), ] SIMPLE_EVENT_SET = functools.reduce(operator.or_, [ev for (ev, _) in SIMPLE_EVENTS], 0) | E.CALL + def just_pass(): pass @@ -313,4 +321,22 @@ def test_restart(self): self.assertEqual(counter.count, 1) +class MultipleMonitors(unittest.TestCase): + + def test_two_simple(self): + counter1 = CounterWithDisable() + counter2 = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, counter1) + sys.monitoring.register_callback(TEST_TOOL2, E.PY_START, counter2) + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + sys.monitoring.set_events(TEST_TOOL2, E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL), E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL2), E.PY_START) + counter1.count = 0 + counter2.count = 0 + f1() + count1 = counter1.count + count2 = counter2.count + self.assertEqual(count1, 1) + self.assertEqual(count2, 1) diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 7657b1322df401..a6a8dd2e475051 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -338,8 +338,11 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) co->co_nplaincellvars = nplaincellvars; co->co_ncellvars = ncellvars; co->co_nfreevars = nfreevars; - co->_co_instrument_version = 0; - + for (int e = 0; e < PY_MONITORING_EVENTS; e++) { + co->_co_monitoring_matrix.tools[e] = 0; + } + co->_co_monitoring_data = 0; + co->_co_monitoring_data_per_instruction = 0; /* not set */ co->co_weakreflist = NULL; co->co_extra = NULL; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 1bd36f99185328..64b4e88664c320 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -111,7 +111,6 @@ dummy_func( inst(INSTRUMENTED_RESUME) { /* Check monitoring *before* calling instrument */ /* Possibly combine this with eval breaker */ - assert(oparg < 2); if (frame->f_code->_co_instrument_version != tstate->interp->monitoring_version) { if (_Py_Instrument(frame->f_code, tstate->interp)) { goto error; @@ -120,9 +119,9 @@ dummy_func( } else { int err = _Py_call_instrumentation( - tstate, oparg, frame, next_instr-1); + tstate, oparg != 0, frame, next_instr-1); ERROR_IF(err, error); - if (_Py_atomic_load_relaxed_int32(eval_breaker)) { + if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { goto handle_eval_breaker; } } diff --git a/Python/ceval.c b/Python/ceval.c index ce061ee3d52679..dee1763ae6af7d 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2202,7 +2202,7 @@ monitor_raise(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) { - if (tstate->interp->monitoring_tools_per_event[PY_MONITORING_EVENT_RAISE].tools == 0) { + if (tstate->interp->monitoring_matrix.tools[PY_MONITORING_EVENT_RAISE] == 0) { return; } PyObject *type, *value, *traceback, *orig_traceback, *arg; @@ -2236,7 +2236,7 @@ monitor_unwind(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) { - if (tstate->interp->monitoring_tools_per_event[PY_MONITORING_EVENT_PY_UNWIND].tools == 0) { + if (tstate->interp->monitoring_matrix.tools[PY_MONITORING_EVENT_PY_UNWIND] == 0) { return; } _Py_call_instrumentation_exc(tstate, PY_MONITORING_EVENT_PY_UNWIND, frame, instr, NULL); @@ -2248,7 +2248,7 @@ monitor_handled(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *exc) { - if (tstate->interp->monitoring_tools_per_event[PY_MONITORING_EVENT_EXCEPTION_HANDLED].tools == 0) { + if (tstate->interp->monitoring_matrix.tools[PY_MONITORING_EVENT_EXCEPTION_HANDLED] == 0) { return; } _Py_call_instrumentation_arg(tstate, PY_MONITORING_EVENT_EXCEPTION_HANDLED, frame, instr, exc); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 6f6e20f06c4456..f8141a93813678 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -8,7 +8,6 @@ TARGET(INSTRUMENTED_RESUME) { /* Check monitoring *before* calling instrument */ /* Possibly combine this with eval breaker */ - assert(oparg < 2); if (frame->f_code->_co_instrument_version != tstate->interp->monitoring_version) { if (_Py_Instrument(frame->f_code, tstate->interp)) { goto error; @@ -17,9 +16,9 @@ } else { int err = _Py_call_instrumentation( - tstate, oparg, frame, next_instr-1); + tstate, oparg != 0, frame, next_instr-1); if (err) goto error; - if (_Py_atomic_load_relaxed_int32(eval_breaker)) { + if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { goto handle_eval_breaker; } } diff --git a/Python/instrumentation.c b/Python/instrumentation.c index a9fd50905adffb..55f4c63901c7fa 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -14,15 +14,6 @@ static PyObject DISABLE = &PyBaseObject_Type }; -static int -call_multiple_instruments( - PyInterpreterState *interp, PyCodeObject *code, int event, - PyObject **args, Py_ssize_t nargs, _Py_CODEUNIT *instr) -{ - PyErr_SetString(PyExc_SystemError, "Multiple tools not supported yet"); - return -1; -} - static const uint8_t DE_INSTRUMENT[256] = { [RESUME] = RESUME, [INSTRUMENTED_RESUME] = RESUME, @@ -42,10 +33,72 @@ static const uint8_t DE_INSTRUMENT[256] = { */ }; +/* Return 1 if DISABLE returned, -1 if error, 0 otherwise */ +static int +call_one_instrument( + PyInterpreterState *interp, PyThreadState *tstate, PyObject **args, + Py_ssize_t nargsf, int8_t tool, int event) +{ + if (tstate->tracing) { + return 0; + } + PyObject *instrument = interp->tools[tool].instrument_callables[event]; + if (instrument == NULL) { + return 0; + } + tstate->tracing++; + PyObject *res = PyObject_Vectorcall(instrument, args, nargsf, NULL); + tstate->tracing--; + if (res == NULL) { + return -1; + } + Py_DECREF(res); + return (res == &DISABLE); +} + +static int +call_multiple_instruments( + PyInterpreterState *interp, PyThreadState *tstate, PyCodeObject *code, int event, + PyObject **args, Py_ssize_t nargsf, uint8_t *opcode) +{ + int offset = ((_Py_CODEUNIT *)opcode) - _PyCode_CODE(code); + uint8_t *toolsptr = &((uint8_t *)code->_co_monitoring_data)[offset*code->_co_monitoring_data_per_instruction]; + uint8_t tools = *toolsptr; + /* No line or instruction yet */ + assert(code->_co_monitoring_data_per_instruction == 1); + assert(_Py_popcount32(tools) > 0); + assert(interp->sole_tool_plus1[event] < 0); + /* To do -- speed this up */ + assert(tools != 0); + int tool = 0; + do { + assert(tool < 8); + if (tools & (1 << tool)) { + tools &= ~(1 << tool); + int res = call_one_instrument(interp, tstate, args, nargsf, tool, event); + if (res) { + if (res > 0) { + *toolsptr &= ~(1 << tool); + if (*toolsptr == 0) { + /* Remove this instrument */ + assert(DE_INSTRUMENT[*opcode] != 0); + *opcode = DE_INSTRUMENT[*opcode]; + } + } + else { + return -1; + } + } + } + tool++; + } while (tools); + return 0; +} + static int call_instrument( PyThreadState *tstate, PyCodeObject *code, int event, - PyObject **args, Py_ssize_t nargsf, _Py_CODEUNIT *instr) + PyObject **args, Py_ssize_t nargsf, uint8_t *opcode) { assert(!_PyErr_Occurred(tstate)); PyInterpreterState *interp = tstate->interp; @@ -55,29 +108,17 @@ call_instrument( /* Why is this instrumented if there is no tool? */ return 0; } - return call_multiple_instruments(interp, code, event, args, nargsf, instr); + return call_multiple_instruments(interp, tstate, code, event, args, nargsf, opcode); } int sole_tool = sole_tool_plus1-1; - if (tstate->monitoring & (1 << sole_tool)) { - return 0; - } - PyObject *instrument = interp->tools[sole_tool].instrument_callables[event]; - if (instrument == NULL) { - return 0; - } - tstate->monitoring |= (1 << sole_tool); - PyObject *res = PyObject_Vectorcall(instrument, args, nargsf, NULL); - tstate->monitoring &= ~(1 << sole_tool); - if (res == NULL) { - return -1; - } - if (res == &DISABLE) { + int res = call_one_instrument(interp, tstate, args, nargsf, sole_tool, event); + if (res > 0) { /* Remove this instrument */ - assert(DE_INSTRUMENT[_Py_OPCODE(*instr)] != 0); - *instr = _Py_MAKECODEUNIT(DE_INSTRUMENT[_Py_OPCODE(*instr)], _Py_OPARG(*instr)); + assert(DE_INSTRUMENT[*opcode] != 0); + *opcode = DE_INSTRUMENT[*opcode]; + res = 0; } - Py_DECREF(res); - return 0; + return res; } @@ -90,7 +131,7 @@ _Py_call_instrumentation( int instruction_offset = instr - _PyCode_CODE(code); PyObject *instruction_offset_obj = PyLong_FromSsize_t(instruction_offset); PyObject *args[3] = { NULL, (PyObject *)code, instruction_offset_obj }; - int err = call_instrument(tstate, code, event, &args[1], 2 | PY_VECTORCALL_ARGUMENTS_OFFSET, instr); + int err = call_instrument(tstate, code, event, &args[1], 2 | PY_VECTORCALL_ARGUMENTS_OFFSET, (uint8_t*)instr); Py_DECREF(instruction_offset_obj); return err; } @@ -104,7 +145,7 @@ _Py_call_instrumentation_arg( int instruction_offset = instr - _PyCode_CODE(code); PyObject *instruction_offset_obj = PyLong_FromSsize_t(instruction_offset); PyObject *args[4] = { NULL, (PyObject *)code, instruction_offset_obj, arg }; - int err = call_instrument(tstate, code, event, &args[1], 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, instr); + int err = call_instrument(tstate, code, event, &args[1], 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, (uint8_t*)instr); Py_DECREF(instruction_offset_obj); return err; } @@ -122,7 +163,7 @@ _Py_call_instrumentation_exc( PyObject *instruction_offset_obj = PyLong_FromSsize_t(instruction_offset); PyObject *args[4] = { NULL, (PyObject *)code, instruction_offset_obj, arg }; Py_ssize_t nargsf = (2 | PY_VECTORCALL_ARGUMENTS_OFFSET) + (arg != NULL); - int err = call_instrument(tstate, code, event, &args[1], nargsf, instr); + int err = call_instrument(tstate, code, event, &args[1], nargsf, (uint8_t*)instr); if (err) { Py_XDECREF(type); Py_XDECREF(value); @@ -145,13 +186,18 @@ _PyMonitoring_RegisterCallback(int tool_id, int event_id, PyObject *obj) is->tools[tool_id].instrument_callables[event_id] = Py_XNewRef(obj); return callback; } +#define RESUME_0 256 +#define RESUME_1 257 + -static const uint8_t INSTRUMENTED_OPCODES[256] = { +static const uint8_t INSTRUMENTED_OPCODES[258] = { [RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE, [CALL] = INSTRUMENTED_CALL, [CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, [YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE, [RESUME] = INSTRUMENTED_RESUME, + [RESUME_0] = INSTRUMENTED_RESUME, + [RESUME_1] = INSTRUMENTED_RESUME, [INSTRUMENTED_RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE, [INSTRUMENTED_CALL] = INSTRUMENTED_CALL, [INSTRUMENTED_CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, @@ -169,26 +215,88 @@ static const uint8_t INSTRUMENTED_OPCODES[256] = { */ }; -#define SET1(a) (1 << (a)) -#define SET2(a, b ) ( (1 << (a)) | (1 << (b)) ) -#define SET3(a, b, c) ( (1 << (a)) | (1 << (b)) | (1 << (c)) ) - -#define CALL_EVENTS \ - SET3(PY_MONITORING_EVENT_CALL, PY_MONITORING_EVENT_C_RAISE, PY_MONITORING_EVENT_C_RETURN) - -static const _PyMonitoringEventSet EVENTS_FOR_OPCODE[256] = { - [RETURN_VALUE] = SET1(PY_MONITORING_EVENT_PY_RETURN), - [INSTRUMENTED_RETURN_VALUE] = SET1(PY_MONITORING_EVENT_PY_RETURN), - [CALL] = CALL_EVENTS, - [INSTRUMENTED_CALL] = CALL_EVENTS, - [CALL_FUNCTION_EX] = CALL_EVENTS, - [INSTRUMENTED_CALL_FUNCTION_EX] = CALL_EVENTS, - [RESUME] = SET2(PY_MONITORING_EVENT_PY_START, PY_MONITORING_EVENT_PY_RESUME), - [INSTRUMENTED_RESUME] = SET2(PY_MONITORING_EVENT_PY_START, PY_MONITORING_EVENT_PY_RESUME), - [YIELD_VALUE] = SET1(PY_MONITORING_EVENT_PY_YIELD), - [INSTRUMENTED_YIELD_VALUE] = SET1(PY_MONITORING_EVENT_PY_YIELD), + +static const int8_t EVENT_FOR_OPCODE[258] = { + [RETURN_VALUE] = PY_MONITORING_EVENT_PY_RETURN, + [INSTRUMENTED_RETURN_VALUE] = PY_MONITORING_EVENT_PY_RETURN, + [CALL] = PY_MONITORING_EVENT_CALL, + [INSTRUMENTED_CALL] = PY_MONITORING_EVENT_CALL, + [CALL_FUNCTION_EX] = PY_MONITORING_EVENT_CALL, + [INSTRUMENTED_CALL_FUNCTION_EX] = PY_MONITORING_EVENT_CALL, + [RESUME] = -1, + [RESUME_0] = PY_MONITORING_EVENT_PY_START, + [RESUME_1] = PY_MONITORING_EVENT_PY_RESUME, + [YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD, + [INSTRUMENTED_YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD, +}; + +static const bool OPCODE_HAS_EVENT[258] = { + [RETURN_VALUE] = true, + [INSTRUMENTED_RETURN_VALUE] = true, + [CALL] = true, + [INSTRUMENTED_CALL] = true, + [CALL_FUNCTION_EX] = true, + [INSTRUMENTED_CALL_FUNCTION_EX] = true, + [RESUME] = true, + [RESUME_0] = true, + [RESUME_1] = true, + [YIELD_VALUE] = true, + [INSTRUMENTED_YIELD_VALUE] = true, }; +static inline _Py_MonitoringMatrix +matrix_sub(_Py_MonitoringMatrix a, _Py_MonitoringMatrix b) +{ + _Py_MonitoringMatrix res; + for (int i = 0; i < PY_MONITORING_EVENTS; i++) { + res.tools[i] = a.tools[i] & ~b.tools[i]; + } + return res; +} + +static inline _Py_MonitoringMatrix +matrix_and(_Py_MonitoringMatrix a, _Py_MonitoringMatrix b) +{ + _Py_MonitoringMatrix res; + for (int i = 0; i < PY_MONITORING_EVENTS; i++) { + res.tools[i] = a.tools[i] & b.tools[i]; + } + return res; +} + +static inline bool +matrix_empty(_Py_MonitoringMatrix m) +{ + for (int i = 0; i < PY_MONITORING_EVENTS; i++) { + if (m.tools[i]) { + return false; + } + } + return true; +} + +static inline void +matrix_set_bit(_Py_MonitoringMatrix *m, int event, int tool, int val) +{ + assert(0 <= event && event < PY_MONITORING_EVENTS); + assert(0 <= tool && tool < PY_MONITORING_TOOL_IDS); + assert(0 <= val && val <= 1); + m->tools[event] &= ~(1 << tool); + m->tools[event] |= (val << tool); +} + +static inline _PyMonitoringEventSet +matrix_get_events(_Py_MonitoringMatrix *m, int tool_id) +{ + _PyMonitoringEventSet result = 0; + for (int e = 0; e < PY_MONITORING_EVENTS; e++) { + if ((m->tools[e] >> tool_id) & 1) { + result |= (1 << e); + } + } + return result; +} + int _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) { @@ -198,49 +306,86 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) /* First establish how much extra space will need, * and free/allocate it if different to what we had before */ /* For now only allow only one tool per event */ - assert(interp->monitoring_tools_per_event[PY_MONITORING_EVENT_LINE].tools == 0); - assert(interp->monitoring_tools_per_event[PY_MONITORING_EVENT_INSTRUCTION].tools == 0); - for (int i = 0; i < PY_MONITORING_EVENTS; i++) { - assert(interp->monitoring_tools_per_event[i].tools == 0 || interp->sole_tool_plus1[i] > 0); + assert(interp->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE] == 0); + assert(interp->monitoring_matrix.tools[PY_MONITORING_EVENT_INSTRUCTION] == 0); + int code_len = (int)Py_SIZE(code); + if (interp->required_monitoring_bytes > code->_co_monitoring_data_per_instruction) { + /* Don't resize if more space than needed, to avoid churn */ + PyMem_Free(code->_co_monitoring_data); + code->_co_monitoring_data = PyMem_Malloc(code_len * interp->required_monitoring_bytes); + if (code->_co_monitoring_data == NULL) { + code->_co_monitoring_data_per_instruction = 0; + PyErr_NoMemory(); + return -1; + } + code->_co_monitoring_data_per_instruction = interp->required_monitoring_bytes; } //printf("Instrumenting code object at %p, code version: %ld interpreter version %ld\n", // code, code->_co_instrument_version, interp->monitoring_version); /* Avoid instrumenting code that has been disabled. * Only instrument new events */ - _PyMonitoringEventSet new_events = interp->monitored_events & ~code->_co_monitored_events; - _PyMonitoringEventSet removed_events = code->_co_monitored_events & ~interp->monitored_events; + _Py_MonitoringMatrix new_events = matrix_sub(interp->monitoring_matrix, code->_co_monitoring_matrix); + _Py_MonitoringMatrix removed_events = matrix_sub(code->_co_monitoring_matrix, interp->monitoring_matrix); if (interp->last_restart_version > code->_co_instrument_version) { - new_events = interp->monitored_events; - removed_events = 0; + new_events = interp->monitoring_matrix; + removed_events = (_Py_MonitoringMatrix){ 0 }; } - code->_co_monitored_events = interp->monitored_events; code->_co_instrument_version = interp->monitoring_version; - assert((new_events & removed_events) == 0); - if ((new_events | removed_events) == 0) { + assert(matrix_empty(matrix_and(new_events, removed_events))); + if (matrix_empty(new_events) && matrix_empty(removed_events)) { return 0; } - /* Insert basic instrumentation */ - int code_len = (int)Py_SIZE(code); + /* Insert instrumentation */ for (int i = 0; i < code_len; i++) { _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; int opcode = _Py_OPCODE(*instr); + int oparg = _Py_OPARG(*instr); if (_PyOpcode_Deopt[opcode]) { opcode = _PyOpcode_Deopt[opcode]; } - /* Don't instrument RESUME in await and yield from. */ - if (opcode == RESUME && _Py_OPARG(_PyCode_CODE(code)[i]) >= 2) { - continue; - } - _PyMonitoringEventSet events = EVENTS_FOR_OPCODE[opcode]; - if (new_events & events) { - assert(INSTRUMENTED_OPCODES[opcode] != 0); - *instr = _Py_MAKECODEUNIT(INSTRUMENTED_OPCODES[opcode], _Py_OPARG(*instr)); + if (opcode == RESUME) { + if (oparg == 0) { + opcode = RESUME_0; + } + else { + opcode = RESUME_1; + } } - if (removed_events & events) { - *instr = _Py_MAKECODEUNIT(opcode, _Py_OPARG(*instr)); - if (_PyOpcode_Caches[opcode]) { - instr[1] = adaptive_counter_warmup(); + bool has_event = OPCODE_HAS_EVENT[opcode]; + if (has_event) { + int8_t event = EVENT_FOR_OPCODE[opcode]; + assert(event >= 0); + if (interp->sole_tool_plus1[event] >= 0) { + int sole_tool = interp->sole_tool_plus1[event] - 1; + if (new_events.tools[event] & (1 << sole_tool)) { + assert(INSTRUMENTED_OPCODES[opcode] != 0); + *instr = _Py_MAKECODEUNIT(INSTRUMENTED_OPCODES[opcode], oparg); + } + else if (removed_events.tools[event] & (1 << sole_tool)) { + *instr = _Py_MAKECODEUNIT(opcode, _Py_OPARG(*instr)); + if (_PyOpcode_Caches[opcode]) { + instr[1] = adaptive_counter_warmup(); + } + } + } + else { + uint8_t new_tools = new_events.tools[event]; + if (new_tools) { + assert(INSTRUMENTED_OPCODES[opcode] != 0); + *instr = _Py_MAKECODEUNIT(INSTRUMENTED_OPCODES[opcode], oparg); + code->_co_monitoring_data[i*code->_co_monitoring_data_per_instruction] |= new_tools; + } + uint8_t removed_tools = removed_events.tools[event]; + if (removed_tools) { + code->_co_monitoring_data[i*code->_co_monitoring_data_per_instruction] &= ~removed_tools; + if (code->_co_monitoring_data[i*code->_co_monitoring_data_per_instruction] == 0) { + *instr = _Py_MAKECODEUNIT(opcode, _Py_OPARG(*instr)); + if (_PyOpcode_Caches[opcode]) { + instr[1] = adaptive_counter_warmup(); + } + } + } } } i += _PyOpcode_Caches[opcode]; @@ -248,44 +393,95 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) return 0; } +#define C_RETURN_EVENTS \ + ((1 << PY_MONITORING_EVENT_C_RETURN) | \ + (1 << PY_MONITORING_EVENT_C_RAISE)) + +#define C_CALL_EVENTS \ + (C_RETURN_EVENTS | (1 << PY_MONITORING_EVENT_CALL)) + +static int +cross_check_events(PyInterpreterState *interp) +{ + for (int e = 0; e < PY_MONITORING_EVENTS; e++) { + uint8_t tools = interp->monitoring_matrix.tools[e]; + int popcount = _Py_popcount32(tools); + switch(popcount) { + case 0: + if (interp->sole_tool_plus1[e] != 0) { + return 0; + } + break; + case 1: + if (interp->sole_tool_plus1[e] <= 0) { + return 0; + } + if ((tools & (1 << (interp->sole_tool_plus1[e]-1))) == 0) { + return 0; + } + break; + default: + if (interp->sole_tool_plus1[e] >= 0) { + return 0; + } + } + } + return 1; +} + void _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) { PyInterpreterState *interp = _PyInterpreterState_Get(); - if (interp->events_per_monitoring_tool[tool_id] == events) { + assert((events & C_CALL_EVENTS) == 0 || (events & C_CALL_EVENTS) == C_CALL_EVENTS); + uint32_t existing_events = matrix_get_events(&interp->monitoring_matrix, tool_id); + if (existing_events == events) { return; } - interp->events_per_monitoring_tool[tool_id] = events; - for (int e = 0; e < PY_MONITORING_EVENTS; e++) { - _PyMonitoringToolSet tools = { 0 }; - for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { - int event_tool_pair = (interp->events_per_monitoring_tool[t] >> e) & 1; - tools.tools |= (event_tool_pair << t); - } - interp->monitoring_tools_per_event[e] = tools; - } int multitools = 0; for (int e = 0; e < PY_MONITORING_EVENTS; e++) { - _PyMonitoringToolSet tools = interp->monitoring_tools_per_event[e]; - if (_Py_popcount32(tools.tools) <= 1) { - interp->sole_tool_plus1[e] = _Py_bit_length(tools.tools); + int val = (events >> e) & 1; + matrix_set_bit(&interp->monitoring_matrix, e, tool_id, val); + uint8_t tools = interp->monitoring_matrix.tools[e]; + if (_Py_popcount32(tools) <= 1) { + interp->sole_tool_plus1[e] = _Py_bit_length(tools); } else { multitools++; interp->sole_tool_plus1[e] = -1; } } + assert(cross_check_events(interp)); + interp->multiple_tools = multitools; if (multitools) { - /* Only support one tool for now */ - assert(0 && "Only support one tool for now"); + interp->multiple_tools = true; + } + bool lines = interp->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE] != 0; + bool opcodes = interp->monitoring_matrix.tools[PY_MONITORING_EVENT_INSTRUCTION] != 0; + if (interp->multiple_tools) { + if (opcodes) { + interp->required_monitoring_bytes = 6; + } + else if (lines) { + interp->required_monitoring_bytes = 4; + } + else { + interp->required_monitoring_bytes = 1; + } + } + else { + if (opcodes) { + interp->required_monitoring_bytes = 3; + } + else if (lines) { + interp->required_monitoring_bytes = 2; + } + else { + interp->required_monitoring_bytes = 0; + } } interp->monitoring_version++; - _PyMonitoringEventSet monitored_events = 0; - for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { - monitored_events |= interp->events_per_monitoring_tool[t]; - } - interp->monitored_events = monitored_events; /* Instrument all executing code objects */ PyThreadState* ts = PyInterpreterState_ThreadHead(interp); @@ -325,7 +521,6 @@ module monitoring #include "clinic/instrumentation.c.h" - static int check_valid_tool(int tool_id) { @@ -336,7 +531,6 @@ check_valid_tool(int tool_id) return 0; } - /*[clinic input] monitoring.use_tool @@ -467,7 +661,7 @@ monitoring_get_events_impl(PyObject *module, int tool_id) return NULL; } PyInterpreterState *interp = _PyInterpreterState_Get(); - _PyMonitoringEventSet event_set = interp->events_per_monitoring_tool[tool_id]; + _PyMonitoringEventSet event_set = matrix_get_events(&interp->monitoring_matrix, tool_id); return PyLong_FromUnsignedLong(event_set); } @@ -491,6 +685,13 @@ monitoring_set_events_impl(PyObject *module, int tool_id, int event_set) PyErr_Format(PyExc_ValueError, "invalid event set 0x%x", event_set); return NULL; } + if ((event_set & C_RETURN_EVENTS) && (event_set & C_CALL_EVENTS) != C_CALL_EVENTS) { + PyErr_Format(PyExc_ValueError, "cannot set C_RETURN or C_RAISE events independently"); + return NULL; + } + if (event_set & (1 << PY_MONITORING_EVENT_CALL)) { + event_set |= C_RETURN_EVENTS; + } _PyMonitoring_SetEvents(tool_id, event_set); Py_RETURN_NONE; } @@ -566,6 +767,7 @@ const char *event_names[] = { [PY_MONITORING_EVENT_EXCEPTION_HANDLED] = "EXCEPTION_HANDLED", [PY_MONITORING_EVENT_C_RAISE] = "C_RAISE", [PY_MONITORING_EVENT_PY_UNWIND] = "PY_UNWIND", + [15] = "Unused", }; PyObject *_Py_CreateMonitoringObject(void) diff --git a/Python/pystate.c b/Python/pystate.c index 8bbd529d45801a..91ad6cdbf2cf27 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -284,7 +284,6 @@ free_interpreter(PyInterpreterState *interp) The runtime state is not manipulated. Instead it is assumed that the interpreter is getting added to the runtime. */ - static void init_interpreter(PyInterpreterState *interp, _PyRuntimeState *runtime, int64_t id, @@ -310,15 +309,11 @@ init_interpreter(PyInterpreterState *interp, PyConfig_InitPythonConfig(&interp->config); _PyType_InitCache(interp); for(int i = 0; i < PY_MONITORING_EVENTS; i++) { - interp->monitoring_tools_per_event[i].tools = 0; + interp->monitoring_matrix.tools[i] = 0; for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { interp->tools[t].instrument_callables[i] = NULL; } } - interp->monitored_events = 0; - for(int i = 0; i < PY_MONITORING_TOOL_IDS; i++) { - interp->events_per_monitoring_tool[i] = 0; - } interp->f_opcode_trace_set = false; interp->_initialized = 1; } From 416b3149a935a19ac1a0a73e8ac98f99a156c608 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 2 Dec 2022 14:12:53 +0000 Subject: [PATCH 005/116] Tidy up of monitoring internals --- Include/cpython/pystate.h | 1 + Include/internal/pycore_interp.h | 3 - Lib/test/test_monitoring.py | 31 ++- Python/clinic/instrumentation.c.h | 19 +- Python/instrumentation.c | 417 ++++++++++++++++-------------- Python/legacy_tracing.c | 6 +- Python/pystate.c | 1 + 7 files changed, 284 insertions(+), 194 deletions(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 99bc0a89dc0e5f..3515d7e04b1a64 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -133,6 +133,7 @@ struct _ts { the trace/profile. */ int tracing; int tracing_what; /* The event currently being traced, if any. */ + uint8_t monitoring; /* Pointer to current _PyCFrame in the C stack frame of the currently, * or most recently, executing _PyEval_EvalFrameDefault. */ diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index a614931da6e6b3..a5c591dde962ef 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -196,9 +196,6 @@ struct _is { PyCodeObject *interpreter_trampoline; _Py_MonitoringMatrix monitoring_matrix; - /* The index (plus one) of the sole tool. 0 if 0 or 2+ tools */ - int8_t sole_tool_plus1[PY_MONITORING_EVENTS]; - bool multiple_tools; uint8_t required_monitoring_bytes; /* Tools numbered 1-8. 0 is the dispatcher/sole tool */ struct _instrumentation_tool tools[PY_MONITORING_TOOL_IDS]; diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 0bb3eb94b5c226..54596148b15913 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -201,6 +201,7 @@ def record_call(code, offset, obj): sys.monitoring.set_events(TEST_TOOL, 0) #Remove the final event, the call to `sys.monitoring.set_events` events = events[:-1] + sys.monitoring.set_events(TEST_TOOL, 0) return events def check_events(self, func, expected=None): @@ -287,7 +288,6 @@ def __call__(self, *args): if self.disable: return sys.monitoring.DISABLE - class MontoringDisableAndRestartTest(unittest.TestCase): def test_disable(self): @@ -305,6 +305,7 @@ def test_disable(self): counter.count = 0 f1() self.assertEqual(counter.count, 0) + sys.monitoring.set_events(TEST_TOOL, 0) def test_restart(self): counter = CounterWithDisable() @@ -319,11 +320,12 @@ def test_restart(self): counter.count = 0 f1() self.assertEqual(counter.count, 1) + sys.monitoring.set_events(TEST_TOOL, 0) class MultipleMonitors(unittest.TestCase): - def test_two_simple(self): + def test_two_same(self): counter1 = CounterWithDisable() counter2 = CounterWithDisable() sys.monitoring.register_callback(TEST_TOOL, E.PY_START, counter1) @@ -339,4 +341,29 @@ def test_two_simple(self): count2 = counter2.count self.assertEqual(count1, 1) self.assertEqual(count2, 1) + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.set_events(TEST_TOOL2, 0) + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, None) + sys.monitoring.register_callback(TEST_TOOL2, E.PY_START, None) + + def test_two_different(self): + counter1 = CounterWithDisable() + counter2 = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, counter1) + sys.monitoring.register_callback(TEST_TOOL2, E.PY_RETURN, counter2) + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + sys.monitoring.set_events(TEST_TOOL2, E.PY_RETURN) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL), E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL2), E.PY_RETURN) + counter1.count = 0 + counter2.count = 0 + f1() + count1 = counter1.count + count2 = counter2.count + self.assertEqual(count1, 1) + self.assertEqual(count2, 1) + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.set_events(TEST_TOOL2, 0) + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, None) + sys.monitoring.register_callback(TEST_TOOL2, E.PY_RETURN, None) diff --git a/Python/clinic/instrumentation.c.h b/Python/clinic/instrumentation.c.h index e677ea86d3647d..feecf7695a2a2a 100644 --- a/Python/clinic/instrumentation.c.h +++ b/Python/clinic/instrumentation.c.h @@ -210,4 +210,21 @@ monitoring_restart_events(PyObject *module, PyObject *Py_UNUSED(ignored)) { return monitoring_restart_events_impl(module); } -/*[clinic end generated code: output=3997247efd06367f input=a9049054013a1b77]*/ + +PyDoc_STRVAR(monitoring__all_events__doc__, +"_all_events($module, /)\n" +"--\n" +"\n"); + +#define MONITORING__ALL_EVENTS_METHODDEF \ + {"_all_events", (PyCFunction)monitoring__all_events, METH_NOARGS, monitoring__all_events__doc__}, + +static PyObject * +monitoring__all_events_impl(PyObject *module); + +static PyObject * +monitoring__all_events(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return monitoring__all_events_impl(module); +} +/*[clinic end generated code: output=d4c412a002392e2b input=a9049054013a1b77]*/ diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 55f4c63901c7fa..c7cf44efd4d38d 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -15,7 +15,6 @@ static PyObject DISABLE = }; static const uint8_t DE_INSTRUMENT[256] = { - [RESUME] = RESUME, [INSTRUMENTED_RESUME] = RESUME, [INSTRUMENTED_RETURN_VALUE] = RETURN_VALUE, [INSTRUMENTED_CALL] = CALL, @@ -33,22 +32,63 @@ static const uint8_t DE_INSTRUMENT[256] = { */ }; + +static const uint8_t INSTRUMENTED_OPCODES[256] = { + [RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE, + [CALL] = INSTRUMENTED_CALL, + [CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, + [YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE, + [RESUME] = INSTRUMENTED_RESUME, + [INSTRUMENTED_RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE, + [INSTRUMENTED_CALL] = INSTRUMENTED_CALL, + [INSTRUMENTED_CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, + [INSTRUMENTED_YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE, + [INSTRUMENTED_RESUME] = INSTRUMENTED_RESUME, + /* + [JUMP_FORWARD] = INSTRUMENTED_JUMP_FORWARD, + [JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD, + [JUMP_IF_FALSE_OR_POP] = INSTRUMENTED_JUMP_IF_FALSE_OR_POP, + [JUMP_IF_TRUE_OR_POP] = INSTRUMENTED_JUMP_IF_TRUE_OR_POP, + [POP_JUMP_IF_FALSE] = INSTRUMENTED_POP_JUMP_IF_FALSE, + [POP_JUMP_IF_TRUE] = INSTRUMENTED_POP_JUMP_IF_TRUE, + [POP_JUMP_IF_NONE] = INSTRUMENTED_POP_JUMP_IF_NONE, + [POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE, + */ +}; + +static void +de_instrument(_Py_CODEUNIT *instr) +{ + int opcode = _Py_OPCODE(*instr); + assert(INSTRUMENTED_OPCODES[opcode] == opcode); + int base_opcode = _PyOpcode_Deopt[DE_INSTRUMENT[opcode]]; + assert(base_opcode != 0); + *instr = _Py_MAKECODEUNIT(base_opcode, _Py_OPARG(*instr)); + if (_PyOpcode_Caches[base_opcode]) { + instr[1] = adaptive_counter_warmup(); + } +} + /* Return 1 if DISABLE returned, -1 if error, 0 otherwise */ static int call_one_instrument( PyInterpreterState *interp, PyThreadState *tstate, PyObject **args, Py_ssize_t nargsf, int8_t tool, int event) { - if (tstate->tracing) { + assert(0 <= tool && tool < 8); + int tool_bit = 1 << tool; + if (tstate->monitoring & tool_bit) { return 0; } PyObject *instrument = interp->tools[tool].instrument_callables[event]; if (instrument == NULL) { return 0; } - tstate->tracing++; + uint8_t old_monitoring = tstate->monitoring; + tstate->monitoring |= tool_bit; PyObject *res = PyObject_Vectorcall(instrument, args, nargsf, NULL); - tstate->tracing--; + assert((tstate->monitoring & ~tool_bit) == old_monitoring); + tstate->monitoring = old_monitoring; if (res == NULL) { return -1; } @@ -56,72 +96,93 @@ call_one_instrument( return (res == &DISABLE); } +static const int8_t LEFT_MOST_BITS[16] = { + -1, 0, 1, 1, 2, 2, 2, 2, + 3, 3, 3, 3, 3, 3, 3, 3, +}; + +/* We could use _Py_bit_length here, but that is designed for larger (32/64) bit ints, + and can perform relatively poorly on platforms without the necessary intrinsics. */ +static int most_significant_bit(uint8_t bits) { + assert(bits != 0); + if (bits > 15) { + return LEFT_MOST_BITS[bits>>4]+4; + } + else { + return LEFT_MOST_BITS[bits]; + } +} + static int call_multiple_instruments( PyInterpreterState *interp, PyThreadState *tstate, PyCodeObject *code, int event, - PyObject **args, Py_ssize_t nargsf, uint8_t *opcode) + PyObject **args, Py_ssize_t nargsf, _Py_CODEUNIT *instr) { - int offset = ((_Py_CODEUNIT *)opcode) - _PyCode_CODE(code); - uint8_t *toolsptr = &((uint8_t *)code->_co_monitoring_data)[offset*code->_co_monitoring_data_per_instruction]; - uint8_t tools = *toolsptr; - /* No line or instruction yet */ + int offset = instr - _PyCode_CODE(code); + uint8_t *toolsptr; + uint8_t tools; + if (event < PY_MONITORING_EVENT_PY_THROW) { + toolsptr = &((uint8_t *)code->_co_monitoring_data)[offset]; + tools = *toolsptr; + } + else { + toolsptr = NULL; + tools = code->_co_monitoring_matrix.tools[event]; + } + assert((tools & interp->monitoring_matrix.tools[event]) == tools); + assert((tools & code->_co_monitoring_matrix.tools[event]) == tools); + /* No line or instruction yet -- Remove this assert when adding line/instruction monitoring */ assert(code->_co_monitoring_data_per_instruction == 1); - assert(_Py_popcount32(tools) > 0); - assert(interp->sole_tool_plus1[event] < 0); - /* To do -- speed this up */ - assert(tools != 0); - int tool = 0; - do { + while (tools) { + int tool = most_significant_bit(tools); assert(tool < 8); - if (tools & (1 << tool)) { - tools &= ~(1 << tool); - int res = call_one_instrument(interp, tstate, args, nargsf, tool, event); - if (res) { - if (res > 0) { - *toolsptr &= ~(1 << tool); - if (*toolsptr == 0) { - /* Remove this instrument */ - assert(DE_INSTRUMENT[*opcode] != 0); - *opcode = DE_INSTRUMENT[*opcode]; - } - } - else { - return -1; - } + assert(tools & (1 << tool)); + tools &= ~(1 << tool); + int res = call_one_instrument(interp, tstate, args, nargsf, tool, event); + if (res == 0) { + /* Nothing to do */ + } + else if (res < 0) { + /* error */ + return -1; + } + else if (event < PY_MONITORING_EVENT_PY_THROW) { + /* DISABLE */ + *toolsptr &= ~(1 << tool); + if (*toolsptr == 0) { + de_instrument(instr); } } - tool++; - } while (tools); + } return 0; } static int call_instrument( PyThreadState *tstate, PyCodeObject *code, int event, - PyObject **args, Py_ssize_t nargsf, uint8_t *opcode) + PyObject **args, Py_ssize_t nargsf, _Py_CODEUNIT *instr) { assert(!_PyErr_Occurred(tstate)); PyInterpreterState *interp = tstate->interp; - int sole_tool_plus1 = interp->sole_tool_plus1[event]; - if (sole_tool_plus1 <= 0) { - if (sole_tool_plus1 == 0) { - /* Why is this instrumented if there is no tool? */ - return 0; - } - return call_multiple_instruments(interp, tstate, code, event, args, nargsf, opcode); + if (code->_co_monitoring_data) { + return call_multiple_instruments(interp, tstate, code, event, args, nargsf, instr); + } + uint8_t tools = code->_co_monitoring_matrix.tools[event]; + if (tools == 0) { + return 0; } - int sole_tool = sole_tool_plus1-1; + int sole_tool = most_significant_bit(code->_co_monitoring_matrix.tools[event]); + assert(sole_tool >= 0); + assert(_Py_popcount32(code->_co_monitoring_matrix.tools[event]) == 1); int res = call_one_instrument(interp, tstate, args, nargsf, sole_tool, event); if (res > 0) { - /* Remove this instrument */ - assert(DE_INSTRUMENT[*opcode] != 0); - *opcode = DE_INSTRUMENT[*opcode]; + /* DISABLE */ + de_instrument(instr); res = 0; } return res; } - int _Py_call_instrumentation( PyThreadState *tstate, int event, @@ -131,7 +192,7 @@ _Py_call_instrumentation( int instruction_offset = instr - _PyCode_CODE(code); PyObject *instruction_offset_obj = PyLong_FromSsize_t(instruction_offset); PyObject *args[3] = { NULL, (PyObject *)code, instruction_offset_obj }; - int err = call_instrument(tstate, code, event, &args[1], 2 | PY_VECTORCALL_ARGUMENTS_OFFSET, (uint8_t*)instr); + int err = call_instrument(tstate, code, event, &args[1], 2 | PY_VECTORCALL_ARGUMENTS_OFFSET, instr); Py_DECREF(instruction_offset_obj); return err; } @@ -145,7 +206,7 @@ _Py_call_instrumentation_arg( int instruction_offset = instr - _PyCode_CODE(code); PyObject *instruction_offset_obj = PyLong_FromSsize_t(instruction_offset); PyObject *args[4] = { NULL, (PyObject *)code, instruction_offset_obj, arg }; - int err = call_instrument(tstate, code, event, &args[1], 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, (uint8_t*)instr); + int err = call_instrument(tstate, code, event, &args[1], 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, instr); Py_DECREF(instruction_offset_obj); return err; } @@ -163,7 +224,7 @@ _Py_call_instrumentation_exc( PyObject *instruction_offset_obj = PyLong_FromSsize_t(instruction_offset); PyObject *args[4] = { NULL, (PyObject *)code, instruction_offset_obj, arg }; Py_ssize_t nargsf = (2 | PY_VECTORCALL_ARGUMENTS_OFFSET) + (arg != NULL); - int err = call_instrument(tstate, code, event, &args[1], nargsf, (uint8_t*)instr); + int err = call_instrument(tstate, code, event, &args[1], nargsf, instr); if (err) { Py_XDECREF(type); Py_XDECREF(value); @@ -186,37 +247,8 @@ _PyMonitoring_RegisterCallback(int tool_id, int event_id, PyObject *obj) is->tools[tool_id].instrument_callables[event_id] = Py_XNewRef(obj); return callback; } -#define RESUME_0 256 -#define RESUME_1 257 - -static const uint8_t INSTRUMENTED_OPCODES[258] = { - [RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE, - [CALL] = INSTRUMENTED_CALL, - [CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, - [YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE, - [RESUME] = INSTRUMENTED_RESUME, - [RESUME_0] = INSTRUMENTED_RESUME, - [RESUME_1] = INSTRUMENTED_RESUME, - [INSTRUMENTED_RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE, - [INSTRUMENTED_CALL] = INSTRUMENTED_CALL, - [INSTRUMENTED_CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, - [INSTRUMENTED_YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE, - [INSTRUMENTED_RESUME] = INSTRUMENTED_RESUME, - /* - [JUMP_FORWARD] = INSTRUMENTED_JUMP_FORWARD, - [JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD, - [JUMP_IF_FALSE_OR_POP] = INSTRUMENTED_JUMP_IF_FALSE_OR_POP, - [JUMP_IF_TRUE_OR_POP] = INSTRUMENTED_JUMP_IF_TRUE_OR_POP, - [POP_JUMP_IF_FALSE] = INSTRUMENTED_POP_JUMP_IF_FALSE, - [POP_JUMP_IF_TRUE] = INSTRUMENTED_POP_JUMP_IF_TRUE, - [POP_JUMP_IF_NONE] = INSTRUMENTED_POP_JUMP_IF_NONE, - [POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE, - */ -}; - - -static const int8_t EVENT_FOR_OPCODE[258] = { +static const int8_t EVENT_FOR_OPCODE[256] = { [RETURN_VALUE] = PY_MONITORING_EVENT_PY_RETURN, [INSTRUMENTED_RETURN_VALUE] = PY_MONITORING_EVENT_PY_RETURN, [CALL] = PY_MONITORING_EVENT_CALL, @@ -224,13 +256,11 @@ static const int8_t EVENT_FOR_OPCODE[258] = { [CALL_FUNCTION_EX] = PY_MONITORING_EVENT_CALL, [INSTRUMENTED_CALL_FUNCTION_EX] = PY_MONITORING_EVENT_CALL, [RESUME] = -1, - [RESUME_0] = PY_MONITORING_EVENT_PY_START, - [RESUME_1] = PY_MONITORING_EVENT_PY_RESUME, [YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD, [INSTRUMENTED_YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD, }; -static const bool OPCODE_HAS_EVENT[258] = { +static const bool OPCODE_HAS_EVENT[256] = { [RETURN_VALUE] = true, [INSTRUMENTED_RETURN_VALUE] = true, [CALL] = true, @@ -238,8 +268,7 @@ static const bool OPCODE_HAS_EVENT[258] = { [CALL_FUNCTION_EX] = true, [INSTRUMENTED_CALL_FUNCTION_EX] = true, [RESUME] = true, - [RESUME_0] = true, - [RESUME_1] = true, + [INSTRUMENTED_RESUME] = true, [YIELD_VALUE] = true, [INSTRUMENTED_YIELD_VALUE] = true, }; @@ -311,13 +340,42 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) int code_len = (int)Py_SIZE(code); if (interp->required_monitoring_bytes > code->_co_monitoring_data_per_instruction) { /* Don't resize if more space than needed, to avoid churn */ - PyMem_Free(code->_co_monitoring_data); - code->_co_monitoring_data = PyMem_Malloc(code_len * interp->required_monitoring_bytes); + code->_co_monitoring_data = PyMem_Realloc(code->_co_monitoring_data, code_len * interp->required_monitoring_bytes); if (code->_co_monitoring_data == NULL) { code->_co_monitoring_data_per_instruction = 0; PyErr_NoMemory(); return -1; } + if (code->_co_monitoring_data_per_instruction == 0) { + for (int i = 0; i < code_len; i++) { + _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; + int opcode = _Py_OPCODE(*instr); + int base_opcode = _PyOpcode_Deopt[opcode]; + if (OPCODE_HAS_EVENT[base_opcode]) { + bool instrumented = INSTRUMENTED_OPCODES[opcode] == opcode; + if (instrumented) { + int8_t event; + if (base_opcode == RESUME) { + int oparg = _Py_OPARG(*instr); + event = oparg; + } + else { + event = EVENT_FOR_OPCODE[base_opcode]; + } + assert(event >= 0); + code->_co_monitoring_data[i] = code->_co_monitoring_matrix.tools[event]; + } + else { + code->_co_monitoring_data[i] = 0; + } + } + i += _PyOpcode_Caches[base_opcode]; + } + code->_co_monitoring_data_per_instruction = 1; + } + for (int i = code_len*code->_co_monitoring_data_per_instruction; i < code_len*interp->required_monitoring_bytes; i++) { + code->_co_monitoring_data[i] = 0; + } code->_co_monitoring_data_per_instruction = interp->required_monitoring_bytes; } //printf("Instrumenting code object at %p, code version: %ld interpreter version %ld\n", @@ -327,6 +385,8 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) */ _Py_MonitoringMatrix new_events = matrix_sub(interp->monitoring_matrix, code->_co_monitoring_matrix); _Py_MonitoringMatrix removed_events = matrix_sub(code->_co_monitoring_matrix, interp->monitoring_matrix); + code->_co_monitoring_matrix = interp->monitoring_matrix; + if (interp->last_restart_version > code->_co_instrument_version) { new_events = interp->monitoring_matrix; removed_events = (_Py_MonitoringMatrix){ 0 }; @@ -341,54 +401,48 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; int opcode = _Py_OPCODE(*instr); int oparg = _Py_OPARG(*instr); - if (_PyOpcode_Deopt[opcode]) { - opcode = _PyOpcode_Deopt[opcode]; + int base_opcode; + if (DE_INSTRUMENT[opcode]) { + base_opcode = _PyOpcode_Deopt[DE_INSTRUMENT[opcode]]; } - if (opcode == RESUME) { - if (oparg == 0) { - opcode = RESUME_0; + else { + base_opcode = _PyOpcode_Deopt[opcode]; + } + if (OPCODE_HAS_EVENT[base_opcode]) { + int8_t event; + if (base_opcode == RESUME) { + int oparg = _Py_OPARG(*instr); + event = oparg; } else { - opcode = RESUME_1; + event = EVENT_FOR_OPCODE[base_opcode]; + assert(event > 0); } - } - bool has_event = OPCODE_HAS_EVENT[opcode]; - if (has_event) { - int8_t event = EVENT_FOR_OPCODE[opcode]; - assert(event >= 0); - if (interp->sole_tool_plus1[event] >= 0) { - int sole_tool = interp->sole_tool_plus1[event] - 1; - if (new_events.tools[event] & (1 << sole_tool)) { - assert(INSTRUMENTED_OPCODES[opcode] != 0); - *instr = _Py_MAKECODEUNIT(INSTRUMENTED_OPCODES[opcode], oparg); - } - else if (removed_events.tools[event] & (1 << sole_tool)) { - *instr = _Py_MAKECODEUNIT(opcode, _Py_OPARG(*instr)); - if (_PyOpcode_Caches[opcode]) { - instr[1] = adaptive_counter_warmup(); - } + uint8_t new_tools = new_events.tools[event]; + if (new_tools) { + assert(INSTRUMENTED_OPCODES[base_opcode] != 0); + *instr = _Py_MAKECODEUNIT(INSTRUMENTED_OPCODES[base_opcode], oparg); + if (code->_co_monitoring_data) { + code->_co_monitoring_data[i] |= new_tools; } } - else { - uint8_t new_tools = new_events.tools[event]; - if (new_tools) { - assert(INSTRUMENTED_OPCODES[opcode] != 0); - *instr = _Py_MAKECODEUNIT(INSTRUMENTED_OPCODES[opcode], oparg); - code->_co_monitoring_data[i*code->_co_monitoring_data_per_instruction] |= new_tools; - } - uint8_t removed_tools = removed_events.tools[event]; - if (removed_tools) { - code->_co_monitoring_data[i*code->_co_monitoring_data_per_instruction] &= ~removed_tools; - if (code->_co_monitoring_data[i*code->_co_monitoring_data_per_instruction] == 0) { - *instr = _Py_MAKECODEUNIT(opcode, _Py_OPARG(*instr)); - if (_PyOpcode_Caches[opcode]) { - instr[1] = adaptive_counter_warmup(); - } + uint8_t removed_tools = removed_events.tools[event]; + if (removed_tools) { + if (code->_co_monitoring_data) { + code->_co_monitoring_data[i] &= ~removed_tools; + if (code->_co_monitoring_data[i] == 0) { + de_instrument(instr); } } + else if (INSTRUMENTED_OPCODES[opcode] == opcode) { + /* Cannot remove anything but the only tool */ + assert(code->_co_monitoring_matrix.tools[event] == 0); + assert(_Py_popcount32(removed_tools) == 1); + de_instrument(instr); + } } } - i += _PyOpcode_Caches[opcode]; + i += _PyOpcode_Caches[base_opcode]; } return 0; } @@ -400,35 +454,6 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) #define C_CALL_EVENTS \ (C_RETURN_EVENTS | (1 << PY_MONITORING_EVENT_CALL)) -static int -cross_check_events(PyInterpreterState *interp) -{ - for (int e = 0; e < PY_MONITORING_EVENTS; e++) { - uint8_t tools = interp->monitoring_matrix.tools[e]; - int popcount = _Py_popcount32(tools); - switch(popcount) { - case 0: - if (interp->sole_tool_plus1[e] != 0) { - return 0; - } - break; - case 1: - if (interp->sole_tool_plus1[e] <= 0) { - return 0; - } - if ((tools & (1 << (interp->sole_tool_plus1[e]-1))) == 0) { - return 0; - } - break; - default: - if (interp->sole_tool_plus1[e] >= 0) { - return 0; - } - } - } - return 1; -} - void _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) { @@ -443,22 +468,13 @@ _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) int val = (events >> e) & 1; matrix_set_bit(&interp->monitoring_matrix, e, tool_id, val); uint8_t tools = interp->monitoring_matrix.tools[e]; - if (_Py_popcount32(tools) <= 1) { - interp->sole_tool_plus1[e] = _Py_bit_length(tools); - } - else { + if (_Py_popcount32(tools) > 1) { multitools++; - interp->sole_tool_plus1[e] = -1; } } - assert(cross_check_events(interp)); - interp->multiple_tools = multitools; - if (multitools) { - interp->multiple_tools = true; - } bool lines = interp->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE] != 0; bool opcodes = interp->monitoring_matrix.tools[PY_MONITORING_EVENT_INSTRUCTION] != 0; - if (interp->multiple_tools) { + if (multitools) { if (opcodes) { interp->required_monitoring_bytes = 6; } @@ -715,30 +731,6 @@ monitoring_restart_events_impl(PyObject *module) Py_RETURN_NONE; } - -static PyMethodDef methods[] = { - MONITORING_USE_TOOL_METHODDEF - MONITORING_FREE_TOOL_METHODDEF - MONITORING_GET_TOOL_METHODDEF - MONITORING_REGISTER_CALLBACK_METHODDEF - MONITORING_GET_EVENTS_METHODDEF - MONITORING_SET_EVENTS_METHODDEF - MONITORING_RESTART_EVENTS_METHODDEF - {NULL, NULL} // sentinel -}; - -static struct PyModuleDef monitoring_module = { - PyModuleDef_HEAD_INIT, - "sys.monitoring", - NULL, - -1, /* multiple "initialization" just copies the module dict. */ - methods, - NULL, - NULL, - NULL, - NULL -}; - static int add_power2_constant(PyObject *obj, const char *name, int i) { @@ -770,6 +762,57 @@ const char *event_names[] = { [15] = "Unused", }; +/*[clinic input] +monitoring._all_events +[clinic start generated code]*/ + +static PyObject * +monitoring__all_events_impl(PyObject *module) +/*[clinic end generated code: output=6b7581e2dbb690f6 input=62ee9672c17b7f0e]*/ +{ + PyInterpreterState *interp = _PyInterpreterState_Get(); + PyObject *res = PyDict_New(); + if (res == NULL) { + return NULL; + } + for (int e = 0; e < PY_MONITORING_EVENTS; e++) { + uint8_t tools = interp->monitoring_matrix.tools[e]; + if (tools == 0) { + continue; + } + int err = PyDict_SetItemString(res, event_names[e], PyLong_FromLong(tools)); + if (err < 0) { + Py_DECREF(res); + return NULL; + } + } + return res; +} + +static PyMethodDef methods[] = { + MONITORING_USE_TOOL_METHODDEF + MONITORING_FREE_TOOL_METHODDEF + MONITORING_GET_TOOL_METHODDEF + MONITORING_REGISTER_CALLBACK_METHODDEF + MONITORING_GET_EVENTS_METHODDEF + MONITORING_SET_EVENTS_METHODDEF + MONITORING_RESTART_EVENTS_METHODDEF + MONITORING__ALL_EVENTS_METHODDEF + {NULL, NULL} // sentinel +}; + +static struct PyModuleDef monitoring_module = { + PyModuleDef_HEAD_INIT, + "sys.monitoring", + NULL, + -1, /* multiple "initialization" just copies the module dict. */ + methods, + NULL, + NULL, + NULL, + NULL +}; + PyObject *_Py_CreateMonitoringObject(void) { PyObject *mod = _PyModule_CreateInitialized(&monitoring_module, PYTHON_API_VERSION); diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index 0054bb43aa658e..32cb0d6bb5c711 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -36,10 +36,15 @@ call_profile_func(_PyLegacyEventHandler *self, PyObject *arg) if (tstate->c_profilefunc == NULL) { Py_RETURN_NONE; } + if (tstate->tracing) { + Py_RETURN_NONE; + } PyFrameObject* frame = PyEval_GetFrame(); Py_INCREF(frame); assert(frame != NULL); + tstate->tracing++; int err = tstate->c_profilefunc(tstate->c_profileobj, frame, self->event, arg); + tstate->tracing--; Py_DECREF(frame); if (err) { return NULL; @@ -47,7 +52,6 @@ call_profile_func(_PyLegacyEventHandler *self, PyObject *arg) Py_RETURN_NONE; } - static PyObject * sys_profile_func2( _PyLegacyEventHandler *self, PyObject *const *args, diff --git a/Python/pystate.c b/Python/pystate.c index 91ad6cdbf2cf27..5504fb5a053b05 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -816,6 +816,7 @@ init_threadstate(PyThreadState *tstate, tstate->datastack_chunk = NULL; tstate->datastack_top = NULL; tstate->datastack_limit = NULL; + tstate->monitoring = 0; tstate->_initialized = 1; } From 7971979b4f3c8b2daf4a98292dc46b2b948730dd Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 4 Dec 2022 12:34:23 +0000 Subject: [PATCH 006/116] Fix legacy tracing. --- Python/bytecodes.c | 24 +++++++++++------------- Python/generated_cases.c.h | 24 +++++++++++------------- Python/legacy_tracing.c | 5 ++--- 3 files changed, 24 insertions(+), 29 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 64b4e88664c320..696c449234b54d 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2999,19 +2999,17 @@ dummy_func( positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames); if (opcode == INSTRUMENTED_CALL) { - if (!PyFunction_Check(function) && !PyMethod_Check(function)) { - if (res == NULL) { - _Py_call_instrumentation_exc( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, next_instr-1, function); - } - else { - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, next_instr-1, function); - if (err < 0) { - Py_CLEAR(res); - } + if (res == NULL) { + _Py_call_instrumentation_exc( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, next_instr-1, function); + } + else { + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, next_instr-1, function); + if (err < 0) { + Py_CLEAR(res); } } } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index f8141a93813678..6f72a9cad2c967 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3008,19 +3008,17 @@ positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames); if (opcode == INSTRUMENTED_CALL) { - if (!PyFunction_Check(function) && !PyMethod_Check(function)) { - if (res == NULL) { - _Py_call_instrumentation_exc( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, next_instr-1, function); - } - else { - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, next_instr-1, function); - if (err < 0) { - Py_CLEAR(res); - } + if (res == NULL) { + _Py_call_instrumentation_exc( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, next_instr-1, function); + } + else { + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, next_instr-1, function); + if (err < 0) { + Py_CLEAR(res); } } } diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index 32cb0d6bb5c711..ea53249d33309b 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -83,7 +83,7 @@ sys_profile_call_or_return( Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); assert(nargs == 3); PyObject *callable = args[2]; - if (PyFunction_Check(callable) || PyMethod_Check(callable)) { + if (!PyCFunction_Check(callable) && Py_TYPE(callable) != &PyMethodDescr_Type) { Py_RETURN_NONE; } return call_profile_func(self, callable); @@ -223,8 +223,7 @@ _PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) (1 << PY_MONITORING_EVENT_PY_START) | (1 << PY_MONITORING_EVENT_PY_RESUME) | (1 << PY_MONITORING_EVENT_PY_RETURN) | (1 << PY_MONITORING_EVENT_PY_YIELD) | (1 << PY_MONITORING_EVENT_CALL) | (1 << PY_MONITORING_EVENT_PY_UNWIND) | - (1 << PY_MONITORING_EVENT_C_RETURN) | - (1 << PY_MONITORING_EVENT_C_RAISE); + (1 << PY_MONITORING_EVENT_C_RETURN) | (1 << PY_MONITORING_EVENT_C_RAISE); _PyMonitoring_SetEvents(PY_INSTRUMENT_SYS_PROFILE, events); } else if (tstate->interp->sys_profiling_threads == 0) { From 9896902e6f7b41df8dcd23039470b7644e4fe6ab Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 8 Dec 2022 16:58:58 +0000 Subject: [PATCH 007/116] Implement support for multiple tools. --- Include/cpython/code.h | 27 +- Include/internal/pycore_function.h | 2 +- Include/internal/pycore_instruments.h | 4 + Include/internal/pycore_interp.h | 3 +- Include/internal/pycore_opcode.h | 4 +- Include/opcode.h | 1 + Lib/opcode.py | 2 + Lib/test/test_monitoring.py | 242 ++++++--- Objects/codeobject.c | 12 +- Python/bytecodes.c | 16 +- Python/ceval.c | 3 + Python/generated_cases.c.h | 16 +- Python/instrumentation.c | 746 +++++++++++++++++++------- Python/opcode_targets.h | 2 +- Python/pystate.c | 1 + 15 files changed, 800 insertions(+), 281 deletions(-) diff --git a/Include/cpython/code.h b/Include/cpython/code.h index 49b2a0e7108665..e4d5ff4782cd29 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -3,6 +3,7 @@ #ifndef Py_LIMITED_API #ifndef Py_CODE_H #define Py_CODE_H + #ifdef __cplusplus extern "C" { #endif @@ -45,6 +46,25 @@ typedef struct { PyObject *_co_freevars; } _PyCoCached; +typedef struct _PyInstrumentationOffsets { + int8_t tools; + int8_t lines; + int8_t instructions; + int8_t size; +} _PyInstrumentationOffsets; + +typedef union _PyInstrumentationLayout { + _PyInstrumentationOffsets offsets; + int32_t bits; +} _PyInstrumentationLayout; + +typedef struct { + uint64_t monitoring_version; /* current instrumentation version */ + _PyInstrumentationLayout layout; + _Py_MonitoringMatrix monitoring_matrix; + uint8_t *monitoring_data; /* array of data for monitoring */ +} _PyCoInstrumentation; + // To avoid repeating ourselves in deepfreeze.py, all PyCodeObject members are // defined in this macro: #define _PyCode_DEF(SIZE) { \ @@ -103,13 +123,8 @@ typedef struct { PyObject *co_linetable; /* bytes object that holds location info */ \ PyObject *co_weakreflist; /* to support weakrefs to code objects */ \ _PyCoCached *_co_cached; /* cached co_* attributes */ \ + _PyCoInstrumentation _co_instrumentation; /* Instrumentation */ \ int _co_firsttraceable; /* index of first traceable instruction */ \ - uint64_t _co_instrument_version; /* current instrumentation version */ \ - uint8_t _co_monitoring_data_per_instruction; /* Number of bytes per instruction */ \ - uint8_t _co_line_data_offset; /* Offset in data for line data */ \ - uint8_t _co_opcode_data_offset; /* Offset in data for opcode data */ \ - _Py_MonitoringMatrix _co_monitoring_matrix; \ - uint8_t *_co_monitoring_data; /* array of data for monitoring */ \ char *_co_linearray; /* array of line offsets */ \ /* Scratch space for extra data relating to the code object. \ Type is a void* to keep the format private in codeobject.c to force \ diff --git a/Include/internal/pycore_function.h b/Include/internal/pycore_function.h index 1c87aa31ddb611..5a2b47bba59e16 100644 --- a/Include/internal/pycore_function.h +++ b/Include/internal/pycore_function.h @@ -8,7 +8,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -extern PyFunctionObject* _PyFunction_FromConstructor(PyFrameConstructor *constr); +PyFunctionObject* _PyFunction_FromConstructor(PyFrameConstructor *constr); extern uint32_t _PyFunction_GetVersionForCurrentState(PyFunctionObject *func); diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index 781aa568d52508..12dee05b465a92 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -66,6 +66,10 @@ extern int _Py_call_instrumentation(PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr); +extern int +_Py_call_instrumentation_line(PyThreadState *tstate, + PyCodeObject *code, _Py_CODEUNIT *instr); + extern int _Py_call_instrumentation_arg(PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg); diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index a5c591dde962ef..e903e116e942f7 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -90,7 +90,7 @@ struct _is { PyInterpreterState *next; uint64_t monitoring_version; - uint64_t last_restart_version;; + uint64_t last_restart_version; struct pythreads { uint64_t next_unique_id; @@ -196,6 +196,7 @@ struct _is { PyCodeObject *interpreter_trampoline; _Py_MonitoringMatrix monitoring_matrix; + _PyInstrumentationLayout instrumentation_layout; uint8_t required_monitoring_bytes; /* Tools numbered 1-8. 0 is the dispatcher/sole tool */ struct _instrumentation_tool tools[PY_MONITORING_TOOL_IDS]; diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index 86e7d7bb9511b4..ec8ae10c0a3f78 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -138,6 +138,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [IMPORT_STAR] = IMPORT_STAR, [INSTRUMENTED_CALL] = INSTRUMENTED_CALL, [INSTRUMENTED_CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, + [INSTRUMENTED_LINE] = INSTRUMENTED_LINE, [INSTRUMENTED_RESUME] = INSTRUMENTED_RESUME, [INSTRUMENTED_RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE, [INSTRUMENTED_YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE, @@ -488,7 +489,7 @@ static const char *const _PyOpcode_OpName[263] = { [247] = "<247>", [248] = "<248>", [249] = "<249>", - [250] = "<250>", + [INSTRUMENTED_LINE] = "INSTRUMENTED_LINE", [251] = "<251>", [252] = "<252>", [253] = "<253>", @@ -576,7 +577,6 @@ static const char *const _PyOpcode_OpName[263] = { case 247: \ case 248: \ case 249: \ - case 250: \ case 251: \ case 252: \ case 253: \ diff --git a/Include/opcode.h b/Include/opcode.h index 70f8b1ea9a3939..619ebfe1cab07b 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -125,6 +125,7 @@ extern "C" { #define INSTRUMENTED_RETURN_VALUE 242 #define INSTRUMENTED_YIELD_VALUE 243 #define INSTRUMENTED_CALL_FUNCTION_EX 244 +#define INSTRUMENTED_LINE 250 #define MIN_PSEUDO_OPCODE 256 #define SETUP_FINALLY 256 #define SETUP_CLEANUP 257 diff --git a/Lib/opcode.py b/Lib/opcode.py index fb240a7472ab86..7a9f3408ff1b69 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -229,6 +229,8 @@ def pseudo_op(name, op, real_ops): def_op('INSTRUMENTED_YIELD_VALUE', 243) def_op('INSTRUMENTED_CALL_FUNCTION_EX', 244) +def_op('INSTRUMENTED_LINE', 250) + hasarg.extend([op for op in opmap.values() if op >= HAVE_ARGUMENT]) diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 54596148b15913..de8e8611b6b2d9 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -23,8 +23,9 @@ def g1(): for _ in gen(): pass -TEST_TOOL = 3 -TEST_TOOL2 = 4 +TEST_TOOL = 2 +TEST_TOOL2 = 3 +TEST_TOOL3 = 4 class MonitoringBaseTest(unittest.TestCase): @@ -178,7 +179,19 @@ def nested_call(): PY_CALLABLES = (types.FunctionType, types.MethodType) -class MonitoringEventsBase: +class MonitoringTestBase: + + def setUp(self): + # Check that a previous test hasn't left monitoring on. + for tool in range(6): + self.assertEqual(sys.monitoring.get_events(tool), 0) + + def tearDown(self): + # Check that test hasn't left monitoring on. + for tool in range(6): + self.assertEqual(sys.monitoring.get_events(tool), 0) + +class MonitoringEventsBase(MonitoringTestBase): def gather_events(self, func): events = [] @@ -201,7 +214,6 @@ def record_call(code, offset, obj): sys.monitoring.set_events(TEST_TOOL, 0) #Remove the final event, the call to `sys.monitoring.set_events` events = events[:-1] - sys.monitoring.set_events(TEST_TOOL, 0) return events def check_events(self, func, expected=None): @@ -275,8 +287,8 @@ def call(code, offset, callable): testfunc() sys.monitoring.set_events(TEST_TOOL, 0) self.assertEqual(errors, []) - self.assertEqual(len(seen), 9) self.assertEqual(stack, [sys._getframe()]) + self.assertEqual(len(seen), 9) class CounterWithDisable: @@ -288,82 +300,162 @@ def __call__(self, *args): if self.disable: return sys.monitoring.DISABLE -class MontoringDisableAndRestartTest(unittest.TestCase): +class MontoringDisableAndRestartTest(unittest.TestCase, MonitoringTestBase): def test_disable(self): - counter = CounterWithDisable() - sys.monitoring.register_callback(TEST_TOOL, E.PY_START, counter) - sys.monitoring.set_events(TEST_TOOL, E.PY_START) - self.assertEqual(counter.count, 0) - counter.count = 0 - f1() - self.assertEqual(counter.count, 1) - counter.disable = True - counter.count = 0 - f1() - self.assertEqual(counter.count, 1) - counter.count = 0 - f1() - self.assertEqual(counter.count, 0) - sys.monitoring.set_events(TEST_TOOL, 0) + try: + counter = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, counter) + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + self.assertEqual(counter.count, 0) + counter.count = 0 + f1() + self.assertEqual(counter.count, 1) + counter.disable = True + counter.count = 0 + f1() + self.assertEqual(counter.count, 1) + counter.count = 0 + f1() + self.assertEqual(counter.count, 0) + sys.monitoring.set_events(TEST_TOOL, 0) + finally: + sys.monitoring.restart_events() def test_restart(self): - counter = CounterWithDisable() - sys.monitoring.register_callback(TEST_TOOL, E.PY_START, counter) - sys.monitoring.set_events(TEST_TOOL, E.PY_START) - counter.disable = True - f1() - counter.count = 0 - f1() - self.assertEqual(counter.count, 0) - sys.monitoring.restart_events() - counter.count = 0 - f1() - self.assertEqual(counter.count, 1) - sys.monitoring.set_events(TEST_TOOL, 0) - - -class MultipleMonitors(unittest.TestCase): + try: + counter = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, counter) + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + counter.disable = True + f1() + counter.count = 0 + f1() + self.assertEqual(counter.count, 0) + sys.monitoring.restart_events() + counter.count = 0 + f1() + self.assertEqual(counter.count, 1) + sys.monitoring.set_events(TEST_TOOL, 0) + finally: + sys.monitoring.restart_events() + + +class MultipleMonitors(unittest.TestCase, MonitoringTestBase): def test_two_same(self): - counter1 = CounterWithDisable() - counter2 = CounterWithDisable() - sys.monitoring.register_callback(TEST_TOOL, E.PY_START, counter1) - sys.monitoring.register_callback(TEST_TOOL2, E.PY_START, counter2) - sys.monitoring.set_events(TEST_TOOL, E.PY_START) - sys.monitoring.set_events(TEST_TOOL2, E.PY_START) - self.assertEqual(sys.monitoring.get_events(TEST_TOOL), E.PY_START) - self.assertEqual(sys.monitoring.get_events(TEST_TOOL2), E.PY_START) - counter1.count = 0 - counter2.count = 0 - f1() - count1 = counter1.count - count2 = counter2.count - self.assertEqual(count1, 1) - self.assertEqual(count2, 1) - sys.monitoring.set_events(TEST_TOOL, 0) - sys.monitoring.set_events(TEST_TOOL2, 0) - sys.monitoring.register_callback(TEST_TOOL, E.PY_START, None) - sys.monitoring.register_callback(TEST_TOOL2, E.PY_START, None) + try: + self.assertEqual(sys.monitoring._all_events(), {}) + counter1 = CounterWithDisable() + counter2 = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, counter1) + sys.monitoring.register_callback(TEST_TOOL2, E.PY_START, counter2) + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + sys.monitoring.set_events(TEST_TOOL2, E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL), E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL2), E.PY_START) + self.assertEqual(sys.monitoring._all_events(), {'PY_START': (1 << TEST_TOOL) | (1 << TEST_TOOL2)}) + counter1.count = 0 + counter2.count = 0 + f1() + count1 = counter1.count + count2 = counter2.count + self.assertEqual((count1, count2), (1, 1)) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.set_events(TEST_TOOL2, 0) + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, None) + sys.monitoring.register_callback(TEST_TOOL2, E.PY_START, None) + self.assertEqual(sys.monitoring._all_events(), {}) + + def test_three_same(self): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + counter1 = CounterWithDisable() + counter2 = CounterWithDisable() + counter3 = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, counter1) + sys.monitoring.register_callback(TEST_TOOL2, E.PY_START, counter2) + sys.monitoring.register_callback(TEST_TOOL3, E.PY_START, counter3) + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + sys.monitoring.set_events(TEST_TOOL2, E.PY_START) + sys.monitoring.set_events(TEST_TOOL3, E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL), E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL2), E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL3), E.PY_START) + self.assertEqual(sys.monitoring._all_events(), {'PY_START': (1 << TEST_TOOL) | (1 << TEST_TOOL2) | (1 << TEST_TOOL3)}) + counter1.count = 0 + counter2.count = 0 + counter3.count = 0 + f1() + count1 = counter1.count + count2 = counter2.count + count3 = counter3.count + self.assertEqual((count1, count2, count3), (1, 1, 1)) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.set_events(TEST_TOOL2, 0) + sys.monitoring.set_events(TEST_TOOL3, 0) + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, None) + sys.monitoring.register_callback(TEST_TOOL2, E.PY_START, None) + sys.monitoring.register_callback(TEST_TOOL3, E.PY_START, None) + self.assertEqual(sys.monitoring._all_events(), {}) def test_two_different(self): - counter1 = CounterWithDisable() - counter2 = CounterWithDisable() - sys.monitoring.register_callback(TEST_TOOL, E.PY_START, counter1) - sys.monitoring.register_callback(TEST_TOOL2, E.PY_RETURN, counter2) - sys.monitoring.set_events(TEST_TOOL, E.PY_START) - sys.monitoring.set_events(TEST_TOOL2, E.PY_RETURN) - self.assertEqual(sys.monitoring.get_events(TEST_TOOL), E.PY_START) - self.assertEqual(sys.monitoring.get_events(TEST_TOOL2), E.PY_RETURN) - counter1.count = 0 - counter2.count = 0 - f1() - count1 = counter1.count - count2 = counter2.count - self.assertEqual(count1, 1) - self.assertEqual(count2, 1) - sys.monitoring.set_events(TEST_TOOL, 0) - sys.monitoring.set_events(TEST_TOOL2, 0) - sys.monitoring.register_callback(TEST_TOOL, E.PY_START, None) - sys.monitoring.register_callback(TEST_TOOL2, E.PY_RETURN, None) + try: + self.assertEqual(sys.monitoring._all_events(), {}) + counter1 = CounterWithDisable() + counter2 = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, counter1) + sys.monitoring.register_callback(TEST_TOOL2, E.PY_RETURN, counter2) + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + sys.monitoring.set_events(TEST_TOOL2, E.PY_RETURN) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL), E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL2), E.PY_RETURN) + self.assertEqual(sys.monitoring._all_events(), {'PY_START': 1 << TEST_TOOL, 'PY_RETURN': 1 << TEST_TOOL2}) + counter1.count = 0 + counter2.count = 0 + f1() + count1 = counter1.count + count2 = counter2.count + self.assertEqual((count1, count2), (1, 1)) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.set_events(TEST_TOOL2, 0) + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, None) + sys.monitoring.register_callback(TEST_TOOL2, E.PY_RETURN, None) + self.assertEqual(sys.monitoring._all_events(), {}) + + def test_two_with_disable(self): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + counter1 = CounterWithDisable() + counter2 = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, counter1) + sys.monitoring.register_callback(TEST_TOOL2, E.PY_START, counter2) + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + sys.monitoring.set_events(TEST_TOOL2, E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL), E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL2), E.PY_START) + self.assertEqual(sys.monitoring._all_events(), {'PY_START': (1 << TEST_TOOL) | (1 << TEST_TOOL2)}) + counter1.count = 0 + counter2.count = 0 + counter1.disable = True + f1() + count1 = counter1.count + count2 = counter2.count + self.assertEqual((count1, count2), (1, 1)) + counter1.count = 0 + counter2.count = 0 + f1() + count1 = counter1.count + count2 = counter2.count + self.assertEqual((count1, count2), (0, 1)) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.set_events(TEST_TOOL2, 0) + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, None) + sys.monitoring.register_callback(TEST_TOOL2, E.PY_START, None) + self.assertEqual(sys.monitoring._all_events(), {}) + sys.monitoring.restart_events() diff --git a/Objects/codeobject.c b/Objects/codeobject.c index a6a8dd2e475051..d9a62934edb102 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -339,10 +339,14 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) co->co_ncellvars = ncellvars; co->co_nfreevars = nfreevars; for (int e = 0; e < PY_MONITORING_EVENTS; e++) { - co->_co_monitoring_matrix.tools[e] = 0; - } - co->_co_monitoring_data = 0; - co->_co_monitoring_data_per_instruction = 0; + co->_co_instrumentation.monitoring_matrix.tools[e] = 0; + } + co->_co_instrumentation.layout.offsets.size = 0; + co->_co_instrumentation.layout.offsets.tools = -1; + co->_co_instrumentation.layout.offsets.lines = -1; + co->_co_instrumentation.layout.offsets.instructions = -1; + co->_co_instrumentation.monitoring_data = NULL; + co->_co_instrumentation.monitoring_version = 0; /* not set */ co->co_weakreflist = NULL; co->co_extra = NULL; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 696c449234b54d..9cb561f2fbf9b7 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -111,7 +111,7 @@ dummy_func( inst(INSTRUMENTED_RESUME) { /* Check monitoring *before* calling instrument */ /* Possibly combine this with eval breaker */ - if (frame->f_code->_co_instrument_version != tstate->interp->monitoring_version) { + if (frame->f_code->_co_instrumentation.monitoring_version != tstate->interp->monitoring_version) { if (_Py_Instrument(frame->f_code, tstate->interp)) { goto error; } @@ -131,7 +131,7 @@ dummy_func( assert(tstate->cframe == &cframe); assert(frame == cframe.current_frame); /* Possibly combine this with eval breaker */ - if (frame->f_code->_co_instrument_version != tstate->interp->monitoring_version) { + if (frame->f_code->_co_instrumentation.monitoring_version != tstate->interp->monitoring_version) { int err = _Py_Instrument(frame->f_code, tstate->interp); ERROR_IF(err, error); next_instr--; @@ -3768,6 +3768,18 @@ dummy_func( PEEK(oparg) = top; } + inst(INSTRUMENTED_LINE) { + int original_opcode = _Py_call_instrumentation_line( + tstate, frame->f_code, next_instr-1); + assert(original_opcode!= 0); + ERROR_IF(original_opcode < 0, error); + opcode = original_opcode; + assert(_PyOpcode_Deopt[opcode] == opcode); + _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; + INCREMENT_ADAPTIVE_COUNTER(cache->counter); + DISPATCH_GOTO(); + } + // stack effect: ( -- ) inst(EXTENDED_ARG) { assert(oparg); diff --git a/Python/ceval.c b/Python/ceval.c index dee1763ae6af7d..6aa95f32d870f1 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -986,6 +986,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int if (_Py_EnterRecursivePy(tstate)) { goto exit_unwind; } + /* Because this avoids the RESUME, + * we need to update instrumentation */ + _Py_Instrument(frame->f_code, tstate->interp); /* TO DO -- Monitor throw entry. */ goto resume_with_error; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 6f72a9cad2c967..ce731770d9b06c 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -8,7 +8,7 @@ TARGET(INSTRUMENTED_RESUME) { /* Check monitoring *before* calling instrument */ /* Possibly combine this with eval breaker */ - if (frame->f_code->_co_instrument_version != tstate->interp->monitoring_version) { + if (frame->f_code->_co_instrumentation.monitoring_version != tstate->interp->monitoring_version) { if (_Py_Instrument(frame->f_code, tstate->interp)) { goto error; } @@ -29,7 +29,7 @@ assert(tstate->cframe == &cframe); assert(frame == cframe.current_frame); /* Possibly combine this with eval breaker */ - if (frame->f_code->_co_instrument_version != tstate->interp->monitoring_version) { + if (frame->f_code->_co_instrumentation.monitoring_version != tstate->interp->monitoring_version) { int err = _Py_Instrument(frame->f_code, tstate->interp); if (err) goto error; next_instr--; @@ -3778,6 +3778,18 @@ DISPATCH(); } + TARGET(INSTRUMENTED_LINE) { + int original_opcode = _Py_call_instrumentation_line( + tstate, frame->f_code, next_instr-1); + assert(original_opcode!= 0); + if (original_opcode < 0) goto error; + opcode = original_opcode; + assert(_PyOpcode_Deopt[opcode] == opcode); + _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; + INCREMENT_ADAPTIVE_COUNTER(cache->counter); + DISPATCH_GOTO(); + } + TARGET(EXTENDED_ARG) { assert(oparg); opcode = _Py_OPCODE(*next_instr); diff --git a/Python/instrumentation.c b/Python/instrumentation.c index c7cf44efd4d38d..c2a474eea7cd22 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -44,6 +44,7 @@ static const uint8_t INSTRUMENTED_OPCODES[256] = { [INSTRUMENTED_CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, [INSTRUMENTED_YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE, [INSTRUMENTED_RESUME] = INSTRUMENTED_RESUME, + [INSTRUMENTED_LINE] = INSTRUMENTED_LINE, /* [JUMP_FORWARD] = INSTRUMENTED_JUMP_FORWARD, [JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD, @@ -56,19 +57,228 @@ static const uint8_t INSTRUMENTED_OPCODES[256] = { */ }; +_PyInstrumentationOffsets +get_offsets(PyCodeObject *code) { + return code->_co_instrumentation.layout.offsets; +} + +uint8_t * +get_instruction_data(PyCodeObject * code, int offset) +{ + _PyInstrumentationOffsets offsets = get_offsets(code); + return &code->_co_instrumentation.monitoring_data[offset*offsets.size]; +} + +static inline uint8_t +get_tools(PyCodeObject * code, int index, int event) +{ + uint8_t tools; + assert(event != PY_MONITORING_EVENT_LINE); + assert(event != PY_MONITORING_EVENT_INSTRUCTION); + + _PyInstrumentationOffsets offsets = get_offsets(code); + + if (event < PY_MONITORING_EVENT_PY_THROW && offsets.tools >= 0) { + assert(offsets.tools == 0); + tools = get_instruction_data(code, index)[0]; + } + else { + tools = code->_co_instrumentation.monitoring_matrix.tools[event]; + } + assert((tools & PyInterpreterState_Get()->monitoring_matrix.tools[event]) == tools); + assert((tools & code->_co_instrumentation.monitoring_matrix.tools[event]) == tools); + return tools; +} + +static inline bool +is_instrumented(int opcode) { + return INSTRUMENTED_OPCODES[opcode] == opcode; +} + static void -de_instrument(_Py_CODEUNIT *instr) +de_instrument(PyCodeObject *code, int offset, int event) { + assert(event != PY_MONITORING_EVENT_LINE); + _Py_CODEUNIT *instr = &_PyCode_CODE(code)[offset]; int opcode = _Py_OPCODE(*instr); - assert(INSTRUMENTED_OPCODES[opcode] == opcode); - int base_opcode = _PyOpcode_Deopt[DE_INSTRUMENT[opcode]]; - assert(base_opcode != 0); - *instr = _Py_MAKECODEUNIT(base_opcode, _Py_OPARG(*instr)); + if (!is_instrumented(opcode)) { + return; + } + int deinstrumented = DE_INSTRUMENT[opcode]; + int base_opcode; + if (deinstrumented) { + base_opcode = _PyOpcode_Deopt[deinstrumented]; + assert(base_opcode != 0); + *instr = _Py_MAKECODEUNIT(base_opcode, _Py_OPARG(*instr)); + } + else { + /* TO DO -- handle INSTRUMENTED_INSTRUCTION */ + assert(opcode == INSTRUMENTED_LINE); + uint8_t *instr_data = get_instruction_data(code, offset); + uint8_t *orignal_opcode_ptr = instr_data + get_offsets(code).lines; + int orignal_opcode = *orignal_opcode_ptr; + base_opcode = _PyOpcode_Deopt[orignal_opcode]; + assert(INSTRUMENTED_OPCODES[orignal_opcode] == orignal_opcode); + assert(base_opcode != 0); + *orignal_opcode_ptr = base_opcode; + } if (_PyOpcode_Caches[base_opcode]) { instr[1] = adaptive_counter_warmup(); } } +static void +de_instrument_line(PyCodeObject *code, int offset) +{ + /* TO DO -- handle INSTRUMENTED_INSTRUCTION */ + _Py_CODEUNIT *instr = &_PyCode_CODE(code)[offset]; + int opcode = _Py_OPCODE(*instr); + assert(opcode == INSTRUMENTED_LINE); + uint8_t *instr_data = get_instruction_data(code, offset); + uint8_t *orignal_opcode_ptr = instr_data + get_offsets(code).lines; + int orignal_opcode = *orignal_opcode_ptr; + if (is_instrumented(orignal_opcode)) { + /* Instrumented original */ + *instr = _Py_MAKECODEUNIT(orignal_opcode, _Py_OPARG(*instr)); + } + else { + int base_opcode = _PyOpcode_Deopt[orignal_opcode]; + assert(base_opcode != 0); + *instr = _Py_MAKECODEUNIT(base_opcode, _Py_OPARG(*instr)); + if (_PyOpcode_Caches[base_opcode]) { + instr[1] = adaptive_counter_warmup(); + } + } +} + +static void +instrument(PyCodeObject *code, int offset) +{ + /* TO DO -- handle INSTRUMENTED_INSTRUCTION */ + _Py_CODEUNIT *instr = &_PyCode_CODE(code)[offset]; + int opcode = _Py_OPCODE(*instr); + if (opcode == INSTRUMENTED_LINE) { + uint8_t *instr_data = get_instruction_data(code, offset); + uint8_t *orignal_opcode_ptr = instr_data + get_offsets(code).lines; + opcode = *orignal_opcode_ptr; + assert(!is_instrumented(opcode)); + *orignal_opcode_ptr = INSTRUMENTED_OPCODES[opcode]; + } + else if (!is_instrumented(opcode)) { + opcode = _PyOpcode_Deopt[opcode]; + int instrumented = INSTRUMENTED_OPCODES[opcode]; + assert(instrumented); + *instr = _Py_MAKECODEUNIT(instrumented, _Py_OPARG(*instr)); + if (_PyOpcode_Caches[opcode]) { + instr[1] = adaptive_counter_warmup(); + } + } +} + +static void +instrument_line(PyCodeObject *code, int offset) +{ + /* TO DO -- handle INSTRUMENTED_INSTRUCTION */ + _Py_CODEUNIT *instr = &_PyCode_CODE(code)[offset]; + int opcode = _Py_OPCODE(*instr); + assert(get_offsets(code).lines >= 0); + uint8_t *instr_data = get_instruction_data(code, offset); + uint8_t *orignal_opcode_ptr = instr_data + get_offsets(code).lines; + if (is_instrumented(opcode)) { + *orignal_opcode_ptr = opcode; + } + else { + assert(_PyOpcode_Deopt[opcode] != 0); + *orignal_opcode_ptr = _PyOpcode_Deopt[opcode]; + } + *instr = _Py_MAKECODEUNIT(INSTRUMENTED_LINE, _Py_OPARG(*instr)); +} + +static void +remove_tools(PyCodeObject * code, int offset, int event, int tools) +{ + assert(event != PY_MONITORING_EVENT_LINE); + assert(event != PY_MONITORING_EVENT_INSTRUCTION); + _PyInstrumentationOffsets offsets = get_offsets(code); + if (offsets.tools < 0) { + /* Single tool */ + uint8_t single_tool = code->_co_instrumentation.monitoring_matrix.tools[event]; + assert(_Py_popcount32(single_tool) <= 1); + if (((single_tool & tools) == single_tool) && event < PY_MONITORING_INSTRUMENTED_EVENTS) { + de_instrument(code, offset, event); + } + } + else { + assert(offsets.tools == 0); + uint8_t *toolsptr = get_instruction_data(code, offset); + *toolsptr &= ~tools; + if (*toolsptr == 0 && event < PY_MONITORING_INSTRUMENTED_EVENTS) { + de_instrument(code, offset, event); + } + } +} + +static void +remove_line_tools(PyCodeObject * code, int offset, int tools) +{ + assert (code->_co_instrumentation.layout.offsets.lines >= 0); + if (get_offsets(code).tools < 0) { + /* Single tool */ + uint8_t single_tool = code->_co_instrumentation.monitoring_matrix.tools[PY_MONITORING_EVENT_LINE]; + assert(_Py_popcount32(single_tool) <= 1); + if (((single_tool & tools) == single_tool)) { + de_instrument_line(code, offset); + } + } + else { + uint8_t *instr_data = get_instruction_data(code, offset); + uint8_t *toolsptr = instr_data + get_offsets(code).lines + 2; + *toolsptr &= ~tools; + if (*toolsptr == 0 ) { + de_instrument_line(code, offset); + } + } +} + + +static void +add_tools(PyCodeObject * code, int offset, int event, int tools) +{ + assert(event != PY_MONITORING_EVENT_LINE); + assert(event != PY_MONITORING_EVENT_INSTRUCTION); + _PyInstrumentationOffsets offsets = get_offsets(code); + if (offsets.tools < 0) { + /* Single tool */ + assert(PyInterpreterState_Get()->monitoring_matrix.tools[event] == tools); + assert(_Py_popcount32(tools) == 1); + } + else { + assert(offsets.tools == 0); + uint8_t *toolsptr = get_instruction_data(code, offset); + *toolsptr |= tools; + } + if (event < PY_MONITORING_INSTRUMENTED_EVENTS) { + instrument(code, offset); + } +} + +static void +add_line_tools(PyCodeObject * code, int offset, int tools) +{ + assert (get_offsets(code).lines >= 0); + if (code->_co_instrumentation.layout.offsets.tools < 0) { + assert(PyInterpreterState_Get()->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE] == (1 << PY_MONITORING_EVENT_LINE)); + assert(_Py_popcount32(tools) == 1); + /* Single tool */ + instrument_line(code, offset); + } + else { + uint8_t *toolsptr = get_instruction_data(code, offset) + get_offsets(code).lines + 2; + instrument_line(code, offset); + *toolsptr |= tools; + } +} + /* Return 1 if DISABLE returned, -1 if error, 0 otherwise */ static int call_one_instrument( @@ -76,19 +286,16 @@ call_one_instrument( Py_ssize_t nargsf, int8_t tool, int event) { assert(0 <= tool && tool < 8); - int tool_bit = 1 << tool; - if (tstate->monitoring & tool_bit) { + if (tstate->tracing) { return 0; } PyObject *instrument = interp->tools[tool].instrument_callables[event]; if (instrument == NULL) { return 0; } - uint8_t old_monitoring = tstate->monitoring; - tstate->monitoring |= tool_bit; + tstate->tracing++; PyObject *res = PyObject_Vectorcall(instrument, args, nargsf, NULL); - assert((tstate->monitoring & ~tool_bit) == old_monitoring); - tstate->monitoring = old_monitoring; + tstate->tracing--; if (res == NULL) { return -1; } @@ -96,46 +303,38 @@ call_one_instrument( return (res == &DISABLE); } -static const int8_t LEFT_MOST_BITS[16] = { - -1, 0, 1, 1, 2, 2, 2, 2, - 3, 3, 3, 3, 3, 3, 3, 3, +static const int8_t MOST_SIGNIFICANT_BITS[16] = { + -1, 0, 1, 1, + 2, 2, 2, 2, + 3, 3, 3, 3, + 3, 3, 3, 3, }; /* We could use _Py_bit_length here, but that is designed for larger (32/64) bit ints, and can perform relatively poorly on platforms without the necessary intrinsics. */ -static int most_significant_bit(uint8_t bits) { +static inline int most_significant_bit(uint8_t bits) { assert(bits != 0); if (bits > 15) { - return LEFT_MOST_BITS[bits>>4]+4; + return MOST_SIGNIFICANT_BITS[bits>>4]+4; } else { - return LEFT_MOST_BITS[bits]; + return MOST_SIGNIFICANT_BITS[bits]; } } static int -call_multiple_instruments( - PyInterpreterState *interp, PyThreadState *tstate, PyCodeObject *code, int event, +call_instrument(PyThreadState *tstate, PyCodeObject *code, int event, PyObject **args, Py_ssize_t nargsf, _Py_CODEUNIT *instr) { + PyInterpreterState *interp = tstate->interp; int offset = instr - _PyCode_CODE(code); - uint8_t *toolsptr; - uint8_t tools; - if (event < PY_MONITORING_EVENT_PY_THROW) { - toolsptr = &((uint8_t *)code->_co_monitoring_data)[offset]; - tools = *toolsptr; - } - else { - toolsptr = NULL; - tools = code->_co_monitoring_matrix.tools[event]; - } - assert((tools & interp->monitoring_matrix.tools[event]) == tools); - assert((tools & code->_co_monitoring_matrix.tools[event]) == tools); - /* No line or instruction yet -- Remove this assert when adding line/instruction monitoring */ - assert(code->_co_monitoring_data_per_instruction == 1); + uint8_t tools = get_tools(code, offset, event); + /* No per-instruction monitoring yet */ + assert(code->_co_instrumentation.layout.offsets.instructions < 0); + // assert(tools); while (tools) { int tool = most_significant_bit(tools); - assert(tool < 8); + assert(tool >= 0 && tool < 8); assert(tools & (1 << tool)); tools &= ~(1 << tool); int res = call_one_instrument(interp, tstate, args, nargsf, tool, event); @@ -146,41 +345,23 @@ call_multiple_instruments( /* error */ return -1; } - else if (event < PY_MONITORING_EVENT_PY_THROW) { + else { /* DISABLE */ - *toolsptr &= ~(1 << tool); - if (*toolsptr == 0) { - de_instrument(instr); - } + remove_tools(code, offset, event, 1 << tool); } } return 0; } -static int -call_instrument( - PyThreadState *tstate, PyCodeObject *code, int event, - PyObject **args, Py_ssize_t nargsf, _Py_CODEUNIT *instr) +static inline bool +matrix_equals(_Py_MonitoringMatrix a, _Py_MonitoringMatrix b) { - assert(!_PyErr_Occurred(tstate)); - PyInterpreterState *interp = tstate->interp; - if (code->_co_monitoring_data) { - return call_multiple_instruments(interp, tstate, code, event, args, nargsf, instr); - } - uint8_t tools = code->_co_monitoring_matrix.tools[event]; - if (tools == 0) { - return 0; - } - int sole_tool = most_significant_bit(code->_co_monitoring_matrix.tools[event]); - assert(sole_tool >= 0); - assert(_Py_popcount32(code->_co_monitoring_matrix.tools[event]) == 1); - int res = call_one_instrument(interp, tstate, args, nargsf, sole_tool, event); - if (res > 0) { - /* DISABLE */ - de_instrument(instr); - res = 0; + for (int i = 0; i < PY_MONITORING_EVENTS; i++) { + if (a.tools[i] != b.tools[i]) { + return false; + } } - return res; + return true; } int @@ -189,6 +370,8 @@ _Py_call_instrumentation( _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) { PyCodeObject *code = frame->f_code; + assert(code->_co_instrumentation.monitoring_version == tstate->interp->monitoring_version); + assert(matrix_equals(code->_co_instrumentation.monitoring_matrix, tstate->interp->monitoring_matrix)); int instruction_offset = instr - _PyCode_CODE(code); PyObject *instruction_offset_obj = PyLong_FromSsize_t(instruction_offset); PyObject *args[3] = { NULL, (PyObject *)code, instruction_offset_obj }; @@ -203,6 +386,8 @@ _Py_call_instrumentation_arg( _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg) { PyCodeObject *code = frame->f_code; + assert(code->_co_instrumentation.monitoring_version == tstate->interp->monitoring_version); + assert(matrix_equals(code->_co_instrumentation.monitoring_matrix, tstate->interp->monitoring_matrix)); int instruction_offset = instr - _PyCode_CODE(code); PyObject *instruction_offset_obj = PyLong_FromSsize_t(instruction_offset); PyObject *args[4] = { NULL, (PyObject *)code, instruction_offset_obj, arg }; @@ -220,6 +405,8 @@ _Py_call_instrumentation_exc( PyObject *type, *value, *traceback; _PyErr_Fetch(tstate, &type, &value, &traceback); PyCodeObject *code = frame->f_code; + assert(code->_co_instrumentation.monitoring_version == tstate->interp->monitoring_version); + assert(matrix_equals(code->_co_instrumentation.monitoring_matrix, tstate->interp->monitoring_matrix)); int instruction_offset = instr - _PyCode_CODE(code); PyObject *instruction_offset_obj = PyLong_FromSsize_t(instruction_offset); PyObject *args[4] = { NULL, (PyObject *)code, instruction_offset_obj, arg }; @@ -237,6 +424,78 @@ _Py_call_instrumentation_exc( Py_DECREF(instruction_offset_obj); } +/* Line delta. + * 8 bit value. + * if line_delta != -128: + * line = first_line + (offset >> 3) + line_delta; + * else: + * line = PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT)); + */ + +static int8_t +compute_line_delta(PyCodeObject *code, int offset, int line) +{ + int delta = line - code->co_firstlineno - (offset >> 3); + if (delta == (int8_t)delta) { + return delta; + } + return -128; +} + +static int +compute_line(PyCodeObject *code, int offset, int8_t line_delta) +{ + if (line_delta == -128) { + /* Compute from table */ + return PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT)); + } + return code->co_firstlineno + (offset >> 3) + line_delta; +} + +int +_Py_call_instrumentation_line(PyThreadState *tstate, PyCodeObject *code, _Py_CODEUNIT *instr) +{ + PyInterpreterState *interp = tstate->interp; + int offset = instr - _PyCode_CODE(code); + assert(code->_co_instrumentation.monitoring_version == tstate->interp->monitoring_version); + assert(matrix_equals(code->_co_instrumentation.monitoring_matrix, tstate->interp->monitoring_matrix)); + _PyCoInstrumentation *instrumentation = &code->_co_instrumentation; + _PyInstrumentationLayout layout = instrumentation->layout; + uint8_t *line_data = &instrumentation->monitoring_data[layout.offsets.size*offset + layout.offsets.lines]; + uint8_t original_opcode = line_data[0]; + int8_t line_delta = ((int8_t *)(char *)line_data)[1]; + int line = compute_line(code, offset, line_delta); + uint8_t tools = layout.offsets.tools >= 0 ? line_data[2] : interp->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE]; + PyObject *line_obj = PyLong_FromSsize_t(line); + if (line_obj == NULL) { + return -1; + } + PyObject *args[3] = { NULL, (PyObject *)code, line_obj }; + while (tools) { + int tool = most_significant_bit(tools); + assert(tool < 8); + assert(tools & (1 << tool)); + tools &= ~(1 << tool); + int res = call_one_instrument(interp, tstate, args, + 2 | PY_VECTORCALL_ARGUMENTS_OFFSET, + tool, PY_MONITORING_EVENT_LINE); + if (res == 0) { + /* Nothing to do */ + } + else if (res < 0) { + /* error */ + Py_DECREF(line_obj); + return -1; + } + else { + /* DISABLE */ + remove_line_tools(code, offset, 1 << tool); + } + } + Py_DECREF(line_obj); + return original_opcode; +} + PyObject * _PyMonitoring_RegisterCallback(int tool_id, int event_id, PyObject *obj) { @@ -309,7 +568,7 @@ matrix_set_bit(_Py_MonitoringMatrix *m, int event, int tool, int val) { assert(0 <= event && event < PY_MONITORING_EVENTS); assert(0 <= tool && tool < PY_MONITORING_TOOL_IDS); - assert(0 <= val && val <= 1); + assert(val == 0 || val == 1); m->tools[event] &= ~(1 << tool); m->tools[event] |= (val << tool); } @@ -326,73 +585,216 @@ matrix_get_events(_Py_MonitoringMatrix *m, int tool_id) return result; } +static bool +is_instrumentation_up_to_date(PyCodeObject *code, PyInterpreterState *interp) +{ + return interp->monitoring_version == code->_co_instrumentation.monitoring_version; +} + +static inline _PyInstrumentationLayout +layout_from_flags(bool multitools, bool lines, bool instrs) { + _PyInstrumentationLayout result; + int size = multitools; + result.offsets.tools = multitools - 1; + result.offsets.lines = -1; + result.offsets.instructions = -1; + if (lines) { + result.offsets.lines = size; + size += multitools ? 3 : 2; + } + if (instrs) { + result.offsets.instructions = size; + size += multitools ? 2 : 1; + } + result.offsets.size = size; + return result; +} + +static inline int +tools_data_offset() { + return 0; +} + +static void +initialize_tools(PyCodeObject *code, int size) +{ + int code_len = (int)Py_SIZE(code); + for (int i = 0; i < code_len; i++) { + _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; + int opcode = _Py_OPCODE(*instr); + bool instrumented = is_instrumented(opcode); + assert(opcode != INSTRUMENTED_LINE); + if (instrumented) { + opcode = DE_INSTRUMENT[opcode]; + assert(opcode != 0); + } + opcode = _PyOpcode_Deopt[opcode]; + if (OPCODE_HAS_EVENT[opcode]) { + uint8_t *tools_ptr = &code->_co_instrumentation.monitoring_data[i*size]; + if (instrumented) { + int8_t event; + if (opcode == RESUME) { + int oparg = _Py_OPARG(*instr); + event = oparg != 0; + } + else { + event = EVENT_FOR_OPCODE[opcode]; + assert(event > 0); + } + assert(event >= 0); + *tools_ptr = code->_co_instrumentation.monitoring_matrix.tools[event]; + assert(*tools_ptr != 0); + } + else { + *tools_ptr = 0; + } + } + i += _PyOpcode_Caches[opcode]; + } +} + +static void +copy_data(uint8_t *old_data, int old_size, int old_offset, uint8_t *new_data,int new_size, int new_offset, int code_len) +{ + assert(new_size > 0); + assert(old_offset >= 0); + assert(new_offset >= 0); + for (int i = 0; i < code_len; i++) { + new_data[i*new_size + new_offset] = old_data[i*old_size+old_offset]; + } +} + +static void +initialize_lines(PyCodeObject *code, PyInterpreterState *interp) +{ + int code_len = (int)Py_SIZE(code); + int size = code->_co_instrumentation.layout.offsets.size; + int offset = code->_co_instrumentation.layout.offsets.lines; + int prev_line = -1; + for (int i = 0; i < code_len; i++) { + uint8_t *line_data = &code->_co_instrumentation.monitoring_data[i*size+offset]; + /* TO DO -- Use line iterator here */ + int line = PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT)); + if (line != prev_line && line <= 0) { + uint8_t opcode = _Py_OPCODE(_PyCode_CODE(code)[i]); + assert(opcode != 0); + line_data[0] = opcode; + line_data[1] = compute_line_delta(code, i, line); + prev_line = line; + } + else { + line_data[0] = 0; + line_data[1] = -128; + } + } +} + +static void +initialize_line_tools(PyCodeObject *code, PyInterpreterState *interp) +{ + int code_len = (int)Py_SIZE(code); + int size = code->_co_instrumentation.layout.offsets.size; + int offset = code->_co_instrumentation.layout.offsets.lines; + for (int i = 0; i < code_len; i++) { + code->_co_instrumentation.monitoring_data[i*size+offset+2] = + interp->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE]; + } +} + int _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) { - if (interp->monitoring_version == code->_co_instrument_version) { + + if (is_instrumentation_up_to_date(code, interp)) { + assert(matrix_equals(code->_co_instrumentation.monitoring_matrix, interp->monitoring_matrix)); return 0; } + bool restarted = interp->last_restart_version > code->_co_instrumentation.monitoring_version; /* First establish how much extra space will need, * and free/allocate it if different to what we had before */ /* For now only allow only one tool per event */ - assert(interp->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE] == 0); assert(interp->monitoring_matrix.tools[PY_MONITORING_EVENT_INSTRUCTION] == 0); int code_len = (int)Py_SIZE(code); - if (interp->required_monitoring_bytes > code->_co_monitoring_data_per_instruction) { - /* Don't resize if more space than needed, to avoid churn */ - code->_co_monitoring_data = PyMem_Realloc(code->_co_monitoring_data, code_len * interp->required_monitoring_bytes); - if (code->_co_monitoring_data == NULL) { - code->_co_monitoring_data_per_instruction = 0; - PyErr_NoMemory(); - return -1; + _PyInstrumentationLayout new_layout = interp->instrumentation_layout; + _PyInstrumentationLayout old_layout = code->_co_instrumentation.layout; + if (new_layout.bits != old_layout.bits) { + uint8_t *old_data = code->_co_instrumentation.monitoring_data; + if (new_layout.offsets.size) { + code->_co_instrumentation.monitoring_data = PyMem_Malloc(code_len * new_layout.offsets.size); + if (code->_co_instrumentation.monitoring_data == NULL) { + PyErr_NoMemory(); + code->_co_instrumentation.monitoring_data = old_data; + return -1; + } + } + else { + code->_co_instrumentation.monitoring_data = NULL; } - if (code->_co_monitoring_data_per_instruction == 0) { - for (int i = 0; i < code_len; i++) { - _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; - int opcode = _Py_OPCODE(*instr); - int base_opcode = _PyOpcode_Deopt[opcode]; - if (OPCODE_HAS_EVENT[base_opcode]) { - bool instrumented = INSTRUMENTED_OPCODES[opcode] == opcode; - if (instrumented) { - int8_t event; - if (base_opcode == RESUME) { - int oparg = _Py_OPARG(*instr); - event = oparg; - } - else { - event = EVENT_FOR_OPCODE[base_opcode]; - } - assert(event >= 0); - code->_co_monitoring_data[i] = code->_co_monitoring_matrix.tools[event]; - } - else { - code->_co_monitoring_data[i] = 0; - } + if (new_layout.offsets.tools >= 0) { + if (restarted) { + uint8_t zero = 0; + copy_data(&zero, 0, 0, + code->_co_instrumentation.monitoring_data, + new_layout.offsets.size, 0, code_len); + } else if (old_layout.offsets.tools < 0) { + initialize_tools(code, new_layout.offsets.size); + } + else { + copy_data(old_data, old_layout.offsets.size, + 0, code->_co_instrumentation.monitoring_data, + new_layout.offsets.size, 0, code_len); + } + } + if (new_layout.offsets.lines >= 0) { + if (old_layout.offsets.lines < 0) { + initialize_lines(code, interp); + } + else { + copy_data(old_data, old_layout.offsets.size, + old_layout.offsets.lines+1, + code->_co_instrumentation.monitoring_data, + new_layout.offsets.size, + new_layout.offsets.lines+1, code_len); + } + if (new_layout.offsets.tools >= 0) { + if (old_layout.offsets.tools < 0) { + initialize_line_tools(code, interp); + } + else { + copy_data(old_data, old_layout.offsets.size, + old_layout.offsets.lines+2, + code->_co_instrumentation.monitoring_data, + new_layout.offsets.size, + new_layout.offsets.lines+2, code_len); } - i += _PyOpcode_Caches[base_opcode]; } - code->_co_monitoring_data_per_instruction = 1; } - for (int i = code_len*code->_co_monitoring_data_per_instruction; i < code_len*interp->required_monitoring_bytes; i++) { - code->_co_monitoring_data[i] = 0; + assert(new_layout.offsets.instructions < 0); + if (old_data) { + PyMem_Free(old_data); } - code->_co_monitoring_data_per_instruction = interp->required_monitoring_bytes; + code->_co_instrumentation.layout.bits = new_layout.bits; } //printf("Instrumenting code object at %p, code version: %ld interpreter version %ld\n", // code, code->_co_instrument_version, interp->monitoring_version); /* Avoid instrumenting code that has been disabled. * Only instrument new events */ - _Py_MonitoringMatrix new_events = matrix_sub(interp->monitoring_matrix, code->_co_monitoring_matrix); - _Py_MonitoringMatrix removed_events = matrix_sub(code->_co_monitoring_matrix, interp->monitoring_matrix); - code->_co_monitoring_matrix = interp->monitoring_matrix; + _Py_MonitoringMatrix new_events; + _Py_MonitoringMatrix removed_events; - if (interp->last_restart_version > code->_co_instrument_version) { + if (restarted) { + removed_events = code->_co_instrumentation.monitoring_matrix; new_events = interp->monitoring_matrix; - removed_events = (_Py_MonitoringMatrix){ 0 }; } - code->_co_instrument_version = interp->monitoring_version; - assert(matrix_empty(matrix_and(new_events, removed_events))); + else { + removed_events = matrix_sub(code->_co_instrumentation.monitoring_matrix, interp->monitoring_matrix); + new_events = matrix_sub(interp->monitoring_matrix, code->_co_instrumentation.monitoring_matrix); + assert(matrix_empty(matrix_and(new_events, removed_events))); + } + code->_co_instrumentation.monitoring_matrix = interp->monitoring_matrix; + assert(code->_co_instrumentation.layout.bits == interp->instrumentation_layout.bits); + code->_co_instrumentation.monitoring_version = interp->monitoring_version; if (matrix_empty(new_events) && matrix_empty(removed_events)) { return 0; } @@ -400,7 +802,6 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) for (int i = 0; i < code_len; i++) { _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; int opcode = _Py_OPCODE(*instr); - int oparg = _Py_OPARG(*instr); int base_opcode; if (DE_INSTRUMENT[opcode]) { base_opcode = _PyOpcode_Deopt[DE_INSTRUMENT[opcode]]; @@ -418,32 +819,38 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) event = EVENT_FOR_OPCODE[base_opcode]; assert(event > 0); } - uint8_t new_tools = new_events.tools[event]; - if (new_tools) { - assert(INSTRUMENTED_OPCODES[base_opcode] != 0); - *instr = _Py_MAKECODEUNIT(INSTRUMENTED_OPCODES[base_opcode], oparg); - if (code->_co_monitoring_data) { - code->_co_monitoring_data[i] |= new_tools; - } - } uint8_t removed_tools = removed_events.tools[event]; if (removed_tools) { - if (code->_co_monitoring_data) { - code->_co_monitoring_data[i] &= ~removed_tools; - if (code->_co_monitoring_data[i] == 0) { - de_instrument(instr); - } - } - else if (INSTRUMENTED_OPCODES[opcode] == opcode) { - /* Cannot remove anything but the only tool */ - assert(code->_co_monitoring_matrix.tools[event] == 0); - assert(_Py_popcount32(removed_tools) == 1); - de_instrument(instr); - } + remove_tools(code, i, event, removed_tools); + } + uint8_t new_tools = new_events.tools[event]; + if (new_tools) { + add_tools(code, i, event, new_tools); } } i += _PyOpcode_Caches[base_opcode]; } + uint8_t new_line_tools = new_events.tools[PY_MONITORING_EVENT_LINE]; + if (new_line_tools) { + /* Insert line instrumentation */ + int offset = new_layout.offsets.lines; + assert(offset >= 0); + for (int i = 0; i < code_len; i++) { + uint8_t *instr_data = get_instruction_data(code, i); + if (instr_data[get_offsets(code).lines]) { + add_line_tools(code, i, new_line_tools); + } + } + } + uint8_t removed_line_tools = removed_events.tools[PY_MONITORING_EVENT_LINE]; + if (removed_line_tools) { + for (int i = 0; i < code_len; i++) { + uint8_t *instr_data = get_instruction_data(code, i); + if (instr_data[get_offsets(code).lines]) { + remove_line_tools(code, i, removed_line_tools); + } + } + } return 0; } @@ -454,6 +861,22 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) #define C_CALL_EVENTS \ (C_RETURN_EVENTS | (1 << PY_MONITORING_EVENT_CALL)) + +static void +instrument_all_executing_code_objects(PyInterpreterState *interp) { + PyThreadState* ts = PyInterpreterState_ThreadHead(interp); + while (ts) { + _PyInterpreterFrame *frame = ts->cframe->current_frame; + while (frame) { + if (frame->owner != FRAME_OWNED_BY_CSTACK) { + _Py_Instrument(frame->f_code, interp); + } + frame = frame->previous; + } + ts = PyThreadState_Next(ts); + } +} + void _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) { @@ -463,72 +886,24 @@ _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) if (existing_events == events) { return; } - int multitools = 0; + bool multitools = false; for (int e = 0; e < PY_MONITORING_EVENTS; e++) { int val = (events >> e) & 1; matrix_set_bit(&interp->monitoring_matrix, e, tool_id, val); uint8_t tools = interp->monitoring_matrix.tools[e]; if (_Py_popcount32(tools) > 1) { - multitools++; - } - } - bool lines = interp->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE] != 0; - bool opcodes = interp->monitoring_matrix.tools[PY_MONITORING_EVENT_INSTRUCTION] != 0; - if (multitools) { - if (opcodes) { - interp->required_monitoring_bytes = 6; - } - else if (lines) { - interp->required_monitoring_bytes = 4; - } - else { - interp->required_monitoring_bytes = 1; - } - } - else { - if (opcodes) { - interp->required_monitoring_bytes = 3; - } - else if (lines) { - interp->required_monitoring_bytes = 2; - } - else { - interp->required_monitoring_bytes = 0; + multitools = true; } } interp->monitoring_version++; + bool line_monitoring = interp->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE] != 0; + assert(interp->monitoring_matrix.tools[PY_MONITORING_EVENT_INSTRUCTION] == 0); + bool instruction_monitoring = false; + interp->instrumentation_layout = layout_from_flags(multitools, line_monitoring, instruction_monitoring); - /* Instrument all executing code objects */ - - PyThreadState* ts = PyInterpreterState_ThreadHead(interp); - - while (ts) { - _PyInterpreterFrame *frame = ts->cframe->current_frame; - while (frame) { - if (frame->owner != FRAME_OWNED_BY_CSTACK) { - _Py_Instrument(frame->f_code, interp); - } - frame = frame->previous; - } - ts = PyThreadState_Next(ts); - } + instrument_all_executing_code_objects(interp); } -/* List of objects in monitoring namespace - class Event(enum.IntFlag) - def use_tool_id(id)->None - def free_tool_id(id)->None - def get_events(tool_id: int)->Event - def set_events(tool_id: int, event_set: Event)->None - def get_local_events(tool_id: int, code: CodeType)->Event - def set_local_events(tool_id: int, code: CodeType, event_set: Event)->None - def register_callback(tool_id: int, event: Event, func: Callable)->Optional[Callable] - def insert_marker(tool_id: int, code: CodeType, offset: Event)->None - def remove_marker(tool_id: int, code: CodeType, offset: Event)->None - def restart_events()->None - DISABLE: object -*/ - /*[clinic input] module monitoring [clinic start generated code]*/ @@ -722,12 +1097,13 @@ monitoring_restart_events_impl(PyObject *module) /*[clinic end generated code: output=e025dd5ba33314c4 input=add8a855063c8008]*/ { /* We want to ensure that: - * last restart version < current version * last restart version > instrumented version for all code objects + * last restart version < current version */ PyInterpreterState *interp = _PyInterpreterState_Get(); interp->last_restart_version = interp->monitoring_version + 1; - interp->monitoring_version += 2; + interp->monitoring_version = interp->last_restart_version + 1; + instrument_all_executing_code_objects(interp); Py_RETURN_NONE; } @@ -839,7 +1215,3 @@ PyObject *_Py_CreateMonitoringObject(void) Py_DECREF(mod); return NULL; } - - - - diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 9ec1dd4f09b7cf..b5a6712b6077c8 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -249,7 +249,7 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, + &&TARGET_INSTRUMENTED_LINE, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, diff --git a/Python/pystate.c b/Python/pystate.c index 5504fb5a053b05..fa6aa31d3c4f85 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -314,6 +314,7 @@ init_interpreter(PyInterpreterState *interp, interp->tools[t].instrument_callables[i] = NULL; } } + interp->instrumentation_layout.offsets = (_PyInstrumentationOffsets){ -1, -1, -1, 0 }; interp->f_opcode_trace_set = false; interp->_initialized = 1; } From f432a66ad6dc1952cfc5241e2a7e8785bfa869ef Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 22 Dec 2022 16:18:26 +0000 Subject: [PATCH 008/116] Get support for line tracing mostly working. Needs to wait for register VM. --- Include/cpython/code.h | 18 +- Include/cpython/pystate.h | 4 +- Include/internal/pycore_code.h | 2 + Include/internal/pycore_instruments.h | 7 + Include/internal/pycore_opcode.h | 32 +- Include/opcode.h | 10 +- Lib/opcode.py | 15 +- Lib/test/test__opcode.py | 6 + Lib/test/test_dis.py | 2 +- Lib/test/test_monitoring.py | 37 +- Lib/test/test_sys_settrace.py | 12 +- Objects/codeobject.c | 16 +- Objects/frameobject.c | 3 + Python/bytecodes.c | 79 +++- Python/ceval.c | 1 + Python/generated_cases.c.h | 84 ++++- Python/instrumentation.c | 499 ++++++++++++++------------ Python/legacy_tracing.c | 162 ++++++++- Python/opcode_targets.h | 18 +- Python/pystate.c | 2 +- 20 files changed, 712 insertions(+), 297 deletions(-) diff --git a/Include/cpython/code.h b/Include/cpython/code.h index e4d5ff4782cd29..17c201f713b56a 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -47,10 +47,9 @@ typedef struct { } _PyCoCached; typedef struct _PyInstrumentationOffsets { - int8_t tools; + int8_t multi_tools; int8_t lines; int8_t instructions; - int8_t size; } _PyInstrumentationOffsets; typedef union _PyInstrumentationLayout { @@ -58,11 +57,24 @@ typedef union _PyInstrumentationLayout { int32_t bits; } _PyInstrumentationLayout; +typedef struct { + uint8_t original_opcode; + int8_t line_delta; +} _PyCoLineInstrumentationData; + +typedef struct { + uint8_t *tools; + _PyCoLineInstrumentationData *lines; + uint8_t *line_tools; + uint8_t *per_instruction_opcode; + uint8_t *per_instruction_tools; +} _PyCoInstrumentationData; + typedef struct { uint64_t monitoring_version; /* current instrumentation version */ _PyInstrumentationLayout layout; _Py_MonitoringMatrix monitoring_matrix; - uint8_t *monitoring_data; /* array of data for monitoring */ + _PyCoInstrumentationData *monitoring_data; /* data for monitoring */ } _PyCoInstrumentation; // To avoid repeating ourselves in deepfreeze.py, all PyCodeObject members are diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 3515d7e04b1a64..34518a347de2a4 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -57,8 +57,8 @@ typedef int (*Py_tracefunc)(PyObject *, PyFrameObject *, int, PyObject *); typedef struct { - PyCodeObject *code; // The code object for the bounds. May be NULL. - PyCodeAddressRange bounds; // Only valid if code != NULL. + PyCodeObject *code; // The code object for the line number. May be NULL. + int line; // Only valid if code != NULL. } PyTraceInfo; // Internal structure: you should not use it directly, but use public functions diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 81f0dd3dd4811d..d06bd4bddc4d73 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -468,6 +468,8 @@ _Py_MakeShimCode(const _PyShimCodeDef *code); extern int _Py_Instrument(PyCodeObject *co, PyInterpreterState *interp); +extern int _Py_GetBaseOpcode(PyCodeObject *code, int offset); + #ifdef __cplusplus } diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index 12dee05b465a92..98cba9884b4f60 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -70,6 +70,11 @@ extern int _Py_call_instrumentation_line(PyThreadState *tstate, PyCodeObject *code, _Py_CODEUNIT *instr); +int +_Py_call_instrumentation_jump( + PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *target); + extern int _Py_call_instrumentation_arg(PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg); @@ -78,6 +83,8 @@ extern void _Py_call_instrumentation_exc(PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg); +extern int +_Py_Instrumentation_GetLine(PyCodeObject *code, int index); #ifdef __cplusplus } diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index ec8ae10c0a3f78..d8cad9b5f4c2c0 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -138,7 +138,15 @@ const uint8_t _PyOpcode_Deopt[256] = { [IMPORT_STAR] = IMPORT_STAR, [INSTRUMENTED_CALL] = INSTRUMENTED_CALL, [INSTRUMENTED_CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, + [INSTRUMENTED_JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD, + [INSTRUMENTED_JUMP_FORWARD] = INSTRUMENTED_JUMP_FORWARD, + [INSTRUMENTED_JUMP_IF_FALSE_OR_POP] = INSTRUMENTED_JUMP_IF_FALSE_OR_POP, + [INSTRUMENTED_JUMP_IF_TRUE_OR_POP] = INSTRUMENTED_JUMP_IF_TRUE_OR_POP, [INSTRUMENTED_LINE] = INSTRUMENTED_LINE, + [INSTRUMENTED_POP_JUMP_IF_FALSE] = INSTRUMENTED_POP_JUMP_IF_FALSE, + [INSTRUMENTED_POP_JUMP_IF_NONE] = INSTRUMENTED_POP_JUMP_IF_NONE, + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE, + [INSTRUMENTED_POP_JUMP_IF_TRUE] = INSTRUMENTED_POP_JUMP_IF_TRUE, [INSTRUMENTED_RESUME] = INSTRUMENTED_RESUME, [INSTRUMENTED_RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE, [INSTRUMENTED_YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE, @@ -484,16 +492,16 @@ static const char *const _PyOpcode_OpName[263] = { [INSTRUMENTED_RETURN_VALUE] = "INSTRUMENTED_RETURN_VALUE", [INSTRUMENTED_YIELD_VALUE] = "INSTRUMENTED_YIELD_VALUE", [INSTRUMENTED_CALL_FUNCTION_EX] = "INSTRUMENTED_CALL_FUNCTION_EX", - [245] = "<245>", - [246] = "<246>", - [247] = "<247>", + [INSTRUMENTED_JUMP_FORWARD] = "INSTRUMENTED_JUMP_FORWARD", + [INSTRUMENTED_JUMP_BACKWARD] = "INSTRUMENTED_JUMP_BACKWARD", + [INSTRUMENTED_JUMP_IF_FALSE_OR_POP] = "INSTRUMENTED_JUMP_IF_FALSE_OR_POP", [248] = "<248>", - [249] = "<249>", + [INSTRUMENTED_JUMP_IF_TRUE_OR_POP] = "INSTRUMENTED_JUMP_IF_TRUE_OR_POP", + [INSTRUMENTED_POP_JUMP_IF_FALSE] = "INSTRUMENTED_POP_JUMP_IF_FALSE", + [INSTRUMENTED_POP_JUMP_IF_TRUE] = "INSTRUMENTED_POP_JUMP_IF_TRUE", + [INSTRUMENTED_POP_JUMP_IF_NONE] = "INSTRUMENTED_POP_JUMP_IF_NONE", + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = "INSTRUMENTED_POP_JUMP_IF_NOT_NONE", [INSTRUMENTED_LINE] = "INSTRUMENTED_LINE", - [251] = "<251>", - [252] = "<252>", - [253] = "<253>", - [254] = "<254>", [DO_TRACING] = "DO_TRACING", [SETUP_FINALLY] = "SETUP_FINALLY", [SETUP_CLEANUP] = "SETUP_CLEANUP", @@ -572,15 +580,7 @@ static const char *const _PyOpcode_OpName[263] = { case 237: \ case 238: \ case 239: \ - case 245: \ - case 246: \ - case 247: \ case 248: \ - case 249: \ - case 251: \ - case 252: \ - case 253: \ - case 254: \ ; #ifdef __cplusplus diff --git a/Include/opcode.h b/Include/opcode.h index 619ebfe1cab07b..303872730a43f9 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -125,7 +125,15 @@ extern "C" { #define INSTRUMENTED_RETURN_VALUE 242 #define INSTRUMENTED_YIELD_VALUE 243 #define INSTRUMENTED_CALL_FUNCTION_EX 244 -#define INSTRUMENTED_LINE 250 +#define INSTRUMENTED_JUMP_FORWARD 245 +#define INSTRUMENTED_JUMP_BACKWARD 246 +#define INSTRUMENTED_JUMP_IF_FALSE_OR_POP 247 +#define INSTRUMENTED_JUMP_IF_TRUE_OR_POP 249 +#define INSTRUMENTED_POP_JUMP_IF_FALSE 250 +#define INSTRUMENTED_POP_JUMP_IF_TRUE 251 +#define INSTRUMENTED_POP_JUMP_IF_NONE 252 +#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 253 +#define INSTRUMENTED_LINE 254 #define MIN_PSEUDO_OPCODE 256 #define SETUP_FINALLY 256 #define SETUP_CLEANUP 257 diff --git a/Lib/opcode.py b/Lib/opcode.py index 7a9f3408ff1b69..cfe853250290e3 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -228,9 +228,18 @@ def pseudo_op(name, op, real_ops): def_op('INSTRUMENTED_RETURN_VALUE', 242) def_op('INSTRUMENTED_YIELD_VALUE', 243) def_op('INSTRUMENTED_CALL_FUNCTION_EX', 244) - -def_op('INSTRUMENTED_LINE', 250) - +def_op('INSTRUMENTED_JUMP_FORWARD', 245) +def_op('INSTRUMENTED_JUMP_BACKWARD', 246) +def_op('INSTRUMENTED_JUMP_IF_FALSE_OR_POP', 247) +def_op('INSTRUMENTED_JUMP_IF_TRUE_OR_POP', 249) +def_op('INSTRUMENTED_POP_JUMP_IF_FALSE', 250) +def_op('INSTRUMENTED_POP_JUMP_IF_TRUE', 251) +def_op('INSTRUMENTED_POP_JUMP_IF_NONE', 252) +def_op('INSTRUMENTED_POP_JUMP_IF_NOT_NONE', 253) + +def_op('INSTRUMENTED_LINE', 254) + +# 255 is reserved. hasarg.extend([op for op in opmap.values() if op >= HAVE_ARGUMENT]) diff --git a/Lib/test/test__opcode.py b/Lib/test/test__opcode.py index db831069c7aeb8..4e5860dfbe6ca3 100644 --- a/Lib/test/test__opcode.py +++ b/Lib/test/test__opcode.py @@ -20,6 +20,9 @@ def test_stack_effect(self): # All defined opcodes has_arg = dis.hasarg for name, code in filter(lambda item: item[0] not in dis.deoptmap, dis.opmap.items()): + if code >= 240: + # Opcodes 240 and up are internal instrumentation instructions + continue with self.subTest(opname=name): if code not in has_arg: stack_effect(code) @@ -51,6 +54,9 @@ def test_stack_effect_jump(self): has_exc = dis.hasexc has_jump = dis.hasjabs + dis.hasjrel for name, code in filter(lambda item: item[0] not in dis.deoptmap, dis.opmap.items()): + if code >= 240: + # Opcodes 240 and up are internal instrumentation instructions + continue with self.subTest(opname=name): if code not in has_arg: common = stack_effect(code) diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 950af3ceb24fea..abbf8bae420918 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -893,7 +893,7 @@ def test_boundaries(self): def test_widths(self): long_opcodes = set(['JUMP_BACKWARD_NO_INTERRUPT', - ]) + 'INSTRUMENTED_CALL_FUNCTION_EX']) for opcode, opname in enumerate(dis.opname): if opname in long_opcodes: continue diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index de8e8611b6b2d9..bab69d55446dcc 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -292,14 +292,30 @@ def call(code, offset, callable): class CounterWithDisable: + def __init__(self): self.disable = False self.count = 0 + def __call__(self, *args): self.count += 1 if self.disable: return sys.monitoring.DISABLE + +class RecorderWithDisable: + + def __init__(self, events): + self.disable = False + self.events = events + + def __call__(self, code, event): + self.events.append(event) + print(len(self.events)) + if self.disable: + return sys.monitoring.DISABLE + + class MontoringDisableAndRestartTest(unittest.TestCase, MonitoringTestBase): def test_disable(self): @@ -341,7 +357,7 @@ def test_restart(self): sys.monitoring.restart_events() -class MultipleMonitors(unittest.TestCase, MonitoringTestBase): +class MultipleMonitorsTest(unittest.TestCase, MonitoringTestBase): def test_two_same(self): try: @@ -459,3 +475,22 @@ def test_two_with_disable(self): self.assertEqual(sys.monitoring._all_events(), {}) sys.monitoring.restart_events() +class LineMontoringTest(unittest.TestCase): + + def test_lines_single(self): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + events = [] + counter = RecorderWithDisable(events) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, counter) + sys.monitoring.set_events(TEST_TOOL, E.LINE) + f1() + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) + self.assertEqual(events, [487, 12, 488]) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) + self.assertEqual(sys.monitoring._all_events(), {}) + sys.monitoring.restart_events() + diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index a251b2272e95eb..35b1a0b629d406 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -1974,12 +1974,12 @@ def test_no_jump_over_return_try_finally_in_finally_block(output): pass output.append(12) - @jump_test(3, 4, [1], (ValueError, 'after')) - def test_no_jump_infinite_while_loop(output): - output.append(1) - while True: - output.append(3) - output.append(4) + #@jump_test(3, 4, [1], (ValueError, 'after')) + #def test_no_jump_infinite_while_loop(output): + #output.append(1) + #while True: + #output.append(3) + #output.append(4) @jump_test(2, 4, [4, 4]) def test_jump_forwards_into_while_block(output): diff --git a/Objects/codeobject.c b/Objects/codeobject.c index d9a62934edb102..733eb1a6a1546d 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -341,10 +341,9 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) for (int e = 0; e < PY_MONITORING_EVENTS; e++) { co->_co_instrumentation.monitoring_matrix.tools[e] = 0; } - co->_co_instrumentation.layout.offsets.size = 0; - co->_co_instrumentation.layout.offsets.tools = -1; - co->_co_instrumentation.layout.offsets.lines = -1; - co->_co_instrumentation.layout.offsets.instructions = -1; + co->_co_instrumentation.layout.offsets.multi_tools = 0; + co->_co_instrumentation.layout.offsets.lines = 0; + co->_co_instrumentation.layout.offsets.instructions = 0; co->_co_instrumentation.monitoring_data = NULL; co->_co_instrumentation.monitoring_version = 0; /* not set */ @@ -1459,11 +1458,12 @@ PyCode_GetFreevars(PyCodeObject *code) } static void -deopt_code(_Py_CODEUNIT *instructions, Py_ssize_t len) +deopt_code(PyCodeObject *code, _Py_CODEUNIT *instructions) { + Py_ssize_t len = Py_SIZE(code); for (int i = 0; i < len; i++) { _Py_CODEUNIT instruction = instructions[i]; - int opcode = _PyOpcode_Deopt[_Py_OPCODE(instruction)]; + int opcode = _Py_GetBaseOpcode(code, i); int caches = _PyOpcode_Caches[opcode]; instructions[i] = _Py_MAKECODEUNIT(opcode, _Py_OPARG(instruction)); while (caches--) { @@ -1486,7 +1486,7 @@ _PyCode_GetCode(PyCodeObject *co) if (code == NULL) { return NULL; } - deopt_code((_Py_CODEUNIT *)PyBytes_AS_STRING(code), Py_SIZE(co)); + deopt_code(co, (_Py_CODEUNIT *)PyBytes_AS_STRING(code)); assert(co->_co_cached->_co_code == NULL); co->_co_cached->_co_code = Py_NewRef(code); return code; @@ -2216,7 +2216,7 @@ _PyCode_ConstantKey(PyObject *op) void _PyStaticCode_Fini(PyCodeObject *co) { - deopt_code(_PyCode_CODE(co), Py_SIZE(co)); + deopt_code(co, _PyCode_CODE(co)); PyMem_Free(co->co_extra); if (co->_co_cached != NULL) { Py_CLEAR(co->_co_cached->_co_code); diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 4861f80a0ca88e..84a205ecaddf9d 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -819,6 +819,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore } while (start_stack > best_stack) { if (top_of_stack(start_stack) == Except) { + fprintf(stderr, "Popping exception\n"); /* Pop exception stack as well as the evaluation stack */ PyThreadState *tstate = _PyThreadState_GET(); _PyErr_StackItem *exc_info = tstate->exc_info; @@ -829,11 +830,13 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore Py_XDECREF(value); } else { + fprintf(stderr, "Popping object\n"); PyObject *v = _PyFrame_StackPop(f->f_frame); Py_XDECREF(v); } start_stack = pop_value(start_stack); } + fprintf(stderr, "Jumping to %d\n", best_addr); /* Finally set the new lasti and return OK. */ f->f_lineno = 0; f->f_frame->prev_instr = _PyCode_CODE(f->f_frame->f_code) + best_addr; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 9cb561f2fbf9b7..d1381f37292149 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3769,17 +3769,88 @@ dummy_func( } inst(INSTRUMENTED_LINE) { + _PyFrame_SetStackPointer(frame, stack_pointer); int original_opcode = _Py_call_instrumentation_line( tstate, frame->f_code, next_instr-1); - assert(original_opcode!= 0); ERROR_IF(original_opcode < 0, error); + next_instr--; + if (frame->prev_instr != next_instr) { + fprintf(stderr, "Jump has happened\n"); + /* Instrumentation has jumped */ + next_instr = frame->prev_instr; + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + if (_PyOpcode_Caches[original_opcode]) { + _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); + INCREMENT_ADAPTIVE_COUNTER(cache->counter); + } + assert(original_opcode > 0 && original_opcode < 256); opcode = original_opcode; - assert(_PyOpcode_Deopt[opcode] == opcode); - _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; - INCREMENT_ADAPTIVE_COUNTER(cache->counter); DISPATCH_GOTO(); } + // stack effect: ( -- ) + inst(INSTRUMENTED_JUMP_FORWARD) { + int err = _Py_call_instrumentation_jump( + tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); + ERROR_IF(err, error); + JUMPBY(oparg); + } + + // stack effect: ( -- ) + inst(INSTRUMENTED_JUMP_BACKWARD) { + assert(oparg < INSTR_OFFSET()); + int err = _Py_call_instrumentation_jump( + tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); + ERROR_IF(err, error); + JUMPBY(-oparg); + CHECK_EVAL_BREAKER(); + } + + inst(INSTRUMENTED_JUMP_IF_TRUE_OR_POP) { + int err = _Py_call_instrumentation_jump( + tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); + ERROR_IF(err, error); + GO_TO_INSTRUCTION(JUMP_IF_TRUE_OR_POP); + } + + inst(INSTRUMENTED_JUMP_IF_FALSE_OR_POP) { + int err = _Py_call_instrumentation_jump( + tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); + ERROR_IF(err, error); + GO_TO_INSTRUCTION(JUMP_IF_FALSE_OR_POP); + } + + inst(INSTRUMENTED_POP_JUMP_IF_TRUE) { + int err = _Py_call_instrumentation_jump( + tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); + ERROR_IF(err, error); + GO_TO_INSTRUCTION(POP_JUMP_IF_TRUE); + } + + inst(INSTRUMENTED_POP_JUMP_IF_FALSE) { + int err = _Py_call_instrumentation_jump( + tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); + ERROR_IF(err, error); + GO_TO_INSTRUCTION(POP_JUMP_IF_FALSE); + } + + inst(INSTRUMENTED_POP_JUMP_IF_NONE) { + int err = _Py_call_instrumentation_jump( + tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); + ERROR_IF(err, error); + GO_TO_INSTRUCTION(POP_JUMP_IF_NONE); + } + + inst(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { + int err = _Py_call_instrumentation_jump( + tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); + ERROR_IF(err, error); + GO_TO_INSTRUCTION(POP_JUMP_IF_NOT_NONE); + } + // stack effect: ( -- ) inst(EXTENDED_ARG) { assert(oparg); diff --git a/Python/ceval.c b/Python/ceval.c index 6aa95f32d870f1..7025e9bd641a4a 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1115,6 +1115,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #endif /* Tell C compilers not to hold the opcode variable in the loop. next_instr points the current instruction without TARGET(). */ + assert(0); opcode = _Py_OPCODE(*next_instr); _PyErr_Format(tstate, PyExc_SystemError, "%U:%d: unknown opcode %d", diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index ce731770d9b06c..4238b2776403ac 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2390,6 +2390,7 @@ } TARGET(POP_JUMP_IF_TRUE) { + PREDICTED(POP_JUMP_IF_TRUE); PyObject *cond = POP(); if (Py_IsFalse(cond)) { _Py_DECREF_NO_DEALLOC(cond); @@ -2413,6 +2414,7 @@ } TARGET(POP_JUMP_IF_NOT_NONE) { + PREDICTED(POP_JUMP_IF_NOT_NONE); PyObject *value = POP(); if (!Py_IsNone(value)) { JUMPBY(oparg); @@ -2422,6 +2424,7 @@ } TARGET(POP_JUMP_IF_NONE) { + PREDICTED(POP_JUMP_IF_NONE); PyObject *value = POP(); if (Py_IsNone(value)) { _Py_DECREF_NO_DEALLOC(value); @@ -2434,6 +2437,7 @@ } TARGET(JUMP_IF_FALSE_OR_POP) { + PREDICTED(JUMP_IF_FALSE_OR_POP); PyObject *cond = TOP(); int err; if (Py_IsTrue(cond)) { @@ -2460,6 +2464,7 @@ } TARGET(JUMP_IF_TRUE_OR_POP) { + PREDICTED(JUMP_IF_TRUE_OR_POP); PyObject *cond = TOP(); int err; if (Py_IsFalse(cond)) { @@ -3779,17 +3784,88 @@ } TARGET(INSTRUMENTED_LINE) { + _PyFrame_SetStackPointer(frame, stack_pointer); int original_opcode = _Py_call_instrumentation_line( tstate, frame->f_code, next_instr-1); - assert(original_opcode!= 0); if (original_opcode < 0) goto error; + next_instr--; + if (frame->prev_instr != next_instr) { + fprintf(stderr, "Jump has happened\n"); + /* Instrumentation has jumped */ + next_instr = frame->prev_instr; + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + if (_PyOpcode_Caches[original_opcode]) { + _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); + INCREMENT_ADAPTIVE_COUNTER(cache->counter); + } + assert(original_opcode > 0 && original_opcode < 256); opcode = original_opcode; - assert(_PyOpcode_Deopt[opcode] == opcode); - _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; - INCREMENT_ADAPTIVE_COUNTER(cache->counter); DISPATCH_GOTO(); } + TARGET(INSTRUMENTED_JUMP_FORWARD) { + int err = _Py_call_instrumentation_jump( + tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); + if (err) goto error; + JUMPBY(oparg); + DISPATCH(); + } + + TARGET(INSTRUMENTED_JUMP_BACKWARD) { + assert(oparg < INSTR_OFFSET()); + int err = _Py_call_instrumentation_jump( + tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); + if (err) goto error; + JUMPBY(-oparg); + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + + TARGET(INSTRUMENTED_JUMP_IF_TRUE_OR_POP) { + int err = _Py_call_instrumentation_jump( + tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); + if (err) goto error; + GO_TO_INSTRUCTION(JUMP_IF_TRUE_OR_POP); + } + + TARGET(INSTRUMENTED_JUMP_IF_FALSE_OR_POP) { + int err = _Py_call_instrumentation_jump( + tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); + if (err) goto error; + GO_TO_INSTRUCTION(JUMP_IF_FALSE_OR_POP); + } + + TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { + int err = _Py_call_instrumentation_jump( + tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); + if (err) goto error; + GO_TO_INSTRUCTION(POP_JUMP_IF_TRUE); + } + + TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { + int err = _Py_call_instrumentation_jump( + tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); + if (err) goto error; + GO_TO_INSTRUCTION(POP_JUMP_IF_FALSE); + } + + TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { + int err = _Py_call_instrumentation_jump( + tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); + if (err) goto error; + GO_TO_INSTRUCTION(POP_JUMP_IF_NONE); + } + + TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { + int err = _Py_call_instrumentation_jump( + tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); + if (err) goto error; + GO_TO_INSTRUCTION(POP_JUMP_IF_NOT_NONE); + } + TARGET(EXTENDED_ARG) { assert(oparg); opcode = _Py_OPCODE(*next_instr); diff --git a/Python/instrumentation.c b/Python/instrumentation.c index c2a474eea7cd22..ba53083a55edbc 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -20,7 +20,6 @@ static const uint8_t DE_INSTRUMENT[256] = { [INSTRUMENTED_CALL] = CALL, [INSTRUMENTED_CALL_FUNCTION_EX] = CALL_FUNCTION_EX, [INSTRUMENTED_YIELD_VALUE] = YIELD_VALUE, -/* [INSTRUMENTED_JUMP_FORWARD] = JUMP_FORWARD, [INSTRUMENTED_JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD, [INSTRUMENTED_JUMP_IF_FALSE_OR_POP] = INSTRUMENTED_JUMP_IF_FALSE_OR_POP, @@ -29,10 +28,8 @@ static const uint8_t DE_INSTRUMENT[256] = { [INSTRUMENTED_POP_JUMP_IF_TRUE] = INSTRUMENTED_POP_JUMP_IF_TRUE, [INSTRUMENTED_POP_JUMP_IF_NONE] = INSTRUMENTED_POP_JUMP_IF_NONE, [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE, -*/ }; - static const uint8_t INSTRUMENTED_OPCODES[256] = { [RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE, [CALL] = INSTRUMENTED_CALL, @@ -45,7 +42,6 @@ static const uint8_t INSTRUMENTED_OPCODES[256] = { [INSTRUMENTED_YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE, [INSTRUMENTED_RESUME] = INSTRUMENTED_RESUME, [INSTRUMENTED_LINE] = INSTRUMENTED_LINE, - /* [JUMP_FORWARD] = INSTRUMENTED_JUMP_FORWARD, [JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD, [JUMP_IF_FALSE_OR_POP] = INSTRUMENTED_JUMP_IF_FALSE_OR_POP, @@ -54,7 +50,6 @@ static const uint8_t INSTRUMENTED_OPCODES[256] = { [POP_JUMP_IF_TRUE] = INSTRUMENTED_POP_JUMP_IF_TRUE, [POP_JUMP_IF_NONE] = INSTRUMENTED_POP_JUMP_IF_NONE, [POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE, - */ }; _PyInstrumentationOffsets @@ -62,25 +57,15 @@ get_offsets(PyCodeObject *code) { return code->_co_instrumentation.layout.offsets; } -uint8_t * -get_instruction_data(PyCodeObject * code, int offset) -{ - _PyInstrumentationOffsets offsets = get_offsets(code); - return &code->_co_instrumentation.monitoring_data[offset*offsets.size]; -} - static inline uint8_t get_tools(PyCodeObject * code, int index, int event) { uint8_t tools; assert(event != PY_MONITORING_EVENT_LINE); assert(event != PY_MONITORING_EVENT_INSTRUCTION); - - _PyInstrumentationOffsets offsets = get_offsets(code); - - if (event < PY_MONITORING_EVENT_PY_THROW && offsets.tools >= 0) { - assert(offsets.tools == 0); - tools = get_instruction_data(code, index)[0]; + _PyCoInstrumentationData *monitoring = code->_co_instrumentation.monitoring_data; + if (monitoring && monitoring->tools) { + tools = monitoring->tools[index]; } else { tools = code->_co_instrumentation.monitoring_matrix.tools[event]; @@ -95,6 +80,25 @@ is_instrumented(int opcode) { return INSTRUMENTED_OPCODES[opcode] == opcode; } +int _Py_GetBaseOpcode(PyCodeObject *code, int offset) +{ + int opcode = _Py_OPCODE(_PyCode_CODE(code)[offset]); + int deinstrumented = DE_INSTRUMENT[opcode]; + if (deinstrumented) { + return deinstrumented; + } + /* To do -- Handle INSTRUMENTED_INSTRUCTION + if (opcode == INSTRUMENTED_INSTRUCTION) { + opcode = code->_co_instrumentation.monitoring_data->per_instruction_opcode[offset]; + } + */ + if (opcode == INSTRUMENTED_LINE) { + opcode = code->_co_instrumentation.monitoring_data->lines[offset].original_opcode; + } + assert(_PyOpcode_Deopt[opcode]); + return _PyOpcode_Deopt[opcode]; +} + static void de_instrument(PyCodeObject *code, int offset, int event) { @@ -114,13 +118,12 @@ de_instrument(PyCodeObject *code, int offset, int event) else { /* TO DO -- handle INSTRUMENTED_INSTRUCTION */ assert(opcode == INSTRUMENTED_LINE); - uint8_t *instr_data = get_instruction_data(code, offset); - uint8_t *orignal_opcode_ptr = instr_data + get_offsets(code).lines; - int orignal_opcode = *orignal_opcode_ptr; + _PyCoLineInstrumentationData *lines = &code->_co_instrumentation.monitoring_data->lines[offset]; + int orignal_opcode = lines->original_opcode; base_opcode = _PyOpcode_Deopt[orignal_opcode]; assert(INSTRUMENTED_OPCODES[orignal_opcode] == orignal_opcode); assert(base_opcode != 0); - *orignal_opcode_ptr = base_opcode; + lines->original_opcode = base_opcode; } if (_PyOpcode_Caches[base_opcode]) { instr[1] = adaptive_counter_warmup(); @@ -133,22 +136,26 @@ de_instrument_line(PyCodeObject *code, int offset) /* TO DO -- handle INSTRUMENTED_INSTRUCTION */ _Py_CODEUNIT *instr = &_PyCode_CODE(code)[offset]; int opcode = _Py_OPCODE(*instr); - assert(opcode == INSTRUMENTED_LINE); - uint8_t *instr_data = get_instruction_data(code, offset); - uint8_t *orignal_opcode_ptr = instr_data + get_offsets(code).lines; - int orignal_opcode = *orignal_opcode_ptr; - if (is_instrumented(orignal_opcode)) { + if (opcode != INSTRUMENTED_LINE) { + return; + } + _PyCoLineInstrumentationData *lines = &code->_co_instrumentation.monitoring_data->lines[offset]; + int original_opcode = lines->original_opcode; + assert(is_instrumented(original_opcode) || _PyOpcode_Deopt[original_opcode]); + if (is_instrumented(original_opcode)) { /* Instrumented original */ - *instr = _Py_MAKECODEUNIT(orignal_opcode, _Py_OPARG(*instr)); + *instr = _Py_MAKECODEUNIT(original_opcode, _Py_OPARG(*instr)); } else { - int base_opcode = _PyOpcode_Deopt[orignal_opcode]; + int base_opcode = _PyOpcode_Deopt[original_opcode]; assert(base_opcode != 0); *instr = _Py_MAKECODEUNIT(base_opcode, _Py_OPARG(*instr)); if (_PyOpcode_Caches[base_opcode]) { instr[1] = adaptive_counter_warmup(); } } + /* Mark instruction as candidate for line instrumentation */ + lines->original_opcode = 255; } static void @@ -158,11 +165,10 @@ instrument(PyCodeObject *code, int offset) _Py_CODEUNIT *instr = &_PyCode_CODE(code)[offset]; int opcode = _Py_OPCODE(*instr); if (opcode == INSTRUMENTED_LINE) { - uint8_t *instr_data = get_instruction_data(code, offset); - uint8_t *orignal_opcode_ptr = instr_data + get_offsets(code).lines; - opcode = *orignal_opcode_ptr; + _PyCoLineInstrumentationData *lines = &code->_co_instrumentation.monitoring_data->lines[offset]; + opcode = lines->original_opcode; assert(!is_instrumented(opcode)); - *orignal_opcode_ptr = INSTRUMENTED_OPCODES[opcode]; + lines->original_opcode = INSTRUMENTED_OPCODES[opcode]; } else if (!is_instrumented(opcode)) { opcode = _PyOpcode_Deopt[opcode]; @@ -178,20 +184,27 @@ instrument(PyCodeObject *code, int offset) static void instrument_line(PyCodeObject *code, int offset) { - /* TO DO -- handle INSTRUMENTED_INSTRUCTION */ _Py_CODEUNIT *instr = &_PyCode_CODE(code)[offset]; int opcode = _Py_OPCODE(*instr); - assert(get_offsets(code).lines >= 0); - uint8_t *instr_data = get_instruction_data(code, offset); - uint8_t *orignal_opcode_ptr = instr_data + get_offsets(code).lines; + /* TO DO -- handle INSTRUMENTED_INSTRUCTION */ + if (opcode == INSTRUMENTED_LINE) { + return; + } + _PyCoLineInstrumentationData *lines = &code->_co_instrumentation.monitoring_data->lines[offset]; + assert(lines->original_opcode == 255); if (is_instrumented(opcode)) { - *orignal_opcode_ptr = opcode; + lines->original_opcode = opcode; } else { + assert(opcode != 0); assert(_PyOpcode_Deopt[opcode] != 0); - *orignal_opcode_ptr = _PyOpcode_Deopt[opcode]; + assert(_PyOpcode_Deopt[opcode] != RESUME); + lines->original_opcode = _PyOpcode_Deopt[opcode]; + } + assert(lines->original_opcode > 0); *instr = _Py_MAKECODEUNIT(INSTRUMENTED_LINE, _Py_OPARG(*instr)); + assert(code->_co_instrumentation.monitoring_data->lines[offset].original_opcode != 0); } static void @@ -199,20 +212,18 @@ remove_tools(PyCodeObject * code, int offset, int event, int tools) { assert(event != PY_MONITORING_EVENT_LINE); assert(event != PY_MONITORING_EVENT_INSTRUCTION); - _PyInstrumentationOffsets offsets = get_offsets(code); - if (offsets.tools < 0) { - /* Single tool */ - uint8_t single_tool = code->_co_instrumentation.monitoring_matrix.tools[event]; - assert(_Py_popcount32(single_tool) <= 1); - if (((single_tool & tools) == single_tool) && event < PY_MONITORING_INSTRUMENTED_EVENTS) { + _PyCoInstrumentationData *monitoring = code->_co_instrumentation.monitoring_data; + if (monitoring && monitoring->tools) { + monitoring->tools[offset] &= ~tools; + if (monitoring->tools[offset] == 0 && event < PY_MONITORING_INSTRUMENTED_EVENTS) { de_instrument(code, offset, event); } } else { - assert(offsets.tools == 0); - uint8_t *toolsptr = get_instruction_data(code, offset); - *toolsptr &= ~tools; - if (*toolsptr == 0 && event < PY_MONITORING_INSTRUMENTED_EVENTS) { + /* Single tool */ + uint8_t single_tool = code->_co_instrumentation.monitoring_matrix.tools[event]; + assert(_Py_popcount32(single_tool) <= 1); + if (((single_tool & tools) == single_tool) && event < PY_MONITORING_INSTRUMENTED_EVENTS) { de_instrument(code, offset, event); } } @@ -222,19 +233,20 @@ static void remove_line_tools(PyCodeObject * code, int offset, int tools) { assert (code->_co_instrumentation.layout.offsets.lines >= 0); - if (get_offsets(code).tools < 0) { - /* Single tool */ - uint8_t single_tool = code->_co_instrumentation.monitoring_matrix.tools[PY_MONITORING_EVENT_LINE]; - assert(_Py_popcount32(single_tool) <= 1); - if (((single_tool & tools) == single_tool)) { + if (code->_co_instrumentation.monitoring_data && + code->_co_instrumentation.monitoring_data->line_tools) + { + uint8_t *toolsptr = &code->_co_instrumentation.monitoring_data->line_tools[offset]; + *toolsptr &= ~tools; + if (*toolsptr == 0 ) { de_instrument_line(code, offset); } } else { - uint8_t *instr_data = get_instruction_data(code, offset); - uint8_t *toolsptr = instr_data + get_offsets(code).lines + 2; - *toolsptr &= ~tools; - if (*toolsptr == 0 ) { + /* Single tool */ + uint8_t single_tool = code->_co_instrumentation.monitoring_matrix.tools[PY_MONITORING_EVENT_LINE]; + assert(_Py_popcount32(single_tool) <= 1); + if (((single_tool & tools) == single_tool)) { de_instrument_line(code, offset); } } @@ -246,17 +258,16 @@ add_tools(PyCodeObject * code, int offset, int event, int tools) { assert(event != PY_MONITORING_EVENT_LINE); assert(event != PY_MONITORING_EVENT_INSTRUCTION); - _PyInstrumentationOffsets offsets = get_offsets(code); - if (offsets.tools < 0) { + if (code->_co_instrumentation.monitoring_data && + code->_co_instrumentation.monitoring_data->tools + ) { + code->_co_instrumentation.monitoring_data->tools[offset] |= tools; + } + else { /* Single tool */ assert(PyInterpreterState_Get()->monitoring_matrix.tools[event] == tools); assert(_Py_popcount32(tools) == 1); } - else { - assert(offsets.tools == 0); - uint8_t *toolsptr = get_instruction_data(code, offset); - *toolsptr |= tools; - } if (event < PY_MONITORING_INSTRUMENTED_EVENTS) { instrument(code, offset); } @@ -266,17 +277,17 @@ static void add_line_tools(PyCodeObject * code, int offset, int tools) { assert (get_offsets(code).lines >= 0); - if (code->_co_instrumentation.layout.offsets.tools < 0) { - assert(PyInterpreterState_Get()->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE] == (1 << PY_MONITORING_EVENT_LINE)); - assert(_Py_popcount32(tools) == 1); - /* Single tool */ - instrument_line(code, offset); + assert((PyInterpreterState_Get()->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE] & tools) == tools); + if (code->_co_instrumentation.monitoring_data && + code->_co_instrumentation.monitoring_data->line_tools + ) { + code->_co_instrumentation.monitoring_data->line_tools[offset] |= tools; } else { - uint8_t *toolsptr = get_instruction_data(code, offset) + get_offsets(code).lines + 2; - instrument_line(code, offset); - *toolsptr |= tools; + /* Single tool */ + assert(_Py_popcount32(tools) == 1); } + instrument_line(code, offset); } /* Return 1 if DISABLE returned, -1 if error, 0 otherwise */ @@ -286,9 +297,7 @@ call_one_instrument( Py_ssize_t nargsf, int8_t tool, int event) { assert(0 <= tool && tool < 8); - if (tstate->tracing) { - return 0; - } + assert(tstate->tracing == 0); PyObject *instrument = interp->tools[tool].instrument_callables[event]; if (instrument == NULL) { return 0; @@ -326,11 +335,14 @@ static int call_instrument(PyThreadState *tstate, PyCodeObject *code, int event, PyObject **args, Py_ssize_t nargsf, _Py_CODEUNIT *instr) { + if (tstate->tracing) { + return 0; + } PyInterpreterState *interp = tstate->interp; int offset = instr - _PyCode_CODE(code); uint8_t tools = get_tools(code, offset, event); /* No per-instruction monitoring yet */ - assert(code->_co_instrumentation.layout.offsets.instructions < 0); + assert(!code->_co_instrumentation.layout.offsets.instructions); // assert(tools); while (tools) { int tool = most_significant_bit(tools); @@ -364,13 +376,19 @@ matrix_equals(_Py_MonitoringMatrix a, _Py_MonitoringMatrix b) return true; } +static bool +is_instrumentation_up_to_date(PyCodeObject *code, PyInterpreterState *interp) +{ + return interp->monitoring_version == code->_co_instrumentation.monitoring_version; +} + int _Py_call_instrumentation( PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) { PyCodeObject *code = frame->f_code; - assert(code->_co_instrumentation.monitoring_version == tstate->interp->monitoring_version); + assert(is_instrumentation_up_to_date(code, tstate->interp)); assert(matrix_equals(code->_co_instrumentation.monitoring_matrix, tstate->interp->monitoring_matrix)); int instruction_offset = instr - _PyCode_CODE(code); PyObject *instruction_offset_obj = PyLong_FromSsize_t(instruction_offset); @@ -386,7 +404,7 @@ _Py_call_instrumentation_arg( _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg) { PyCodeObject *code = frame->f_code; - assert(code->_co_instrumentation.monitoring_version == tstate->interp->monitoring_version); + assert(is_instrumentation_up_to_date(code, tstate->interp)); assert(matrix_equals(code->_co_instrumentation.monitoring_matrix, tstate->interp->monitoring_matrix)); int instruction_offset = instr - _PyCode_CODE(code); PyObject *instruction_offset_obj = PyLong_FromSsize_t(instruction_offset); @@ -396,6 +414,30 @@ _Py_call_instrumentation_arg( return err; } +int +_Py_call_instrumentation_jump( + PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *target +) { + PyCodeObject *code = frame->f_code; + int from = instr - _PyCode_CODE(code); + int to = target - _PyCode_CODE(code); + PyObject *to_obj = PyLong_FromLong(to); + if (to_obj == NULL) { + return -1; + } + PyObject *from_obj = PyLong_FromLong(from); + if (from_obj == NULL) { + Py_DECREF(to_obj); + return -1; + } + PyObject *args[4] = { NULL, (PyObject *)code, from_obj, to_obj }; + int err = call_instrument(tstate, code, event, &args[1], 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, instr); + Py_DECREF(to_obj); + Py_DECREF(from_obj); + return err; +} + void _Py_call_instrumentation_exc( PyThreadState *tstate, int event, @@ -436,36 +478,53 @@ static int8_t compute_line_delta(PyCodeObject *code, int offset, int line) { int delta = line - code->co_firstlineno - (offset >> 3); - if (delta == (int8_t)delta) { + if (delta < 128 && delta > -128) { return delta; } - return -128; + return -127; } static int compute_line(PyCodeObject *code, int offset, int8_t line_delta) { - if (line_delta == -128) { + assert(line_delta != -128); + if (line_delta == -127) { /* Compute from table */ return PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT)); } return code->co_firstlineno + (offset >> 3) + line_delta; } +int +_Py_Instrumentation_GetLine(PyCodeObject *code, int index) +{ + _PyCoInstrumentation *instrumentation = &code->_co_instrumentation; + assert(instrumentation->monitoring_data != NULL); + assert(instrumentation->monitoring_data->lines != NULL); + _PyCoLineInstrumentationData *line_data = &instrumentation->monitoring_data->lines[index]; + int8_t line_delta = line_data->line_delta; + int line = compute_line(code, index, line_delta); + return line; +} + int _Py_call_instrumentation_line(PyThreadState *tstate, PyCodeObject *code, _Py_CODEUNIT *instr) { - PyInterpreterState *interp = tstate->interp; - int offset = instr - _PyCode_CODE(code); - assert(code->_co_instrumentation.monitoring_version == tstate->interp->monitoring_version); + assert(is_instrumentation_up_to_date(code, tstate->interp)); assert(matrix_equals(code->_co_instrumentation.monitoring_matrix, tstate->interp->monitoring_matrix)); + int offset = instr - _PyCode_CODE(code); _PyCoInstrumentation *instrumentation = &code->_co_instrumentation; - _PyInstrumentationLayout layout = instrumentation->layout; - uint8_t *line_data = &instrumentation->monitoring_data[layout.offsets.size*offset + layout.offsets.lines]; - uint8_t original_opcode = line_data[0]; - int8_t line_delta = ((int8_t *)(char *)line_data)[1]; + _PyCoLineInstrumentationData *line_data = &instrumentation->monitoring_data->lines[offset]; + uint8_t original_opcode = line_data->original_opcode; + if (tstate->tracing) { + return original_opcode; + } + PyInterpreterState *interp = tstate->interp; + int8_t line_delta = line_data->line_delta; int line = compute_line(code, offset, line_delta); - uint8_t tools = layout.offsets.tools >= 0 ? line_data[2] : interp->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE]; + uint8_t tools = code->_co_instrumentation.monitoring_data->line_tools != NULL ? + code->_co_instrumentation.monitoring_data->line_tools[offset] : + interp->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE]; PyObject *line_obj = PyLong_FromSsize_t(line); if (line_obj == NULL) { return -1; @@ -476,7 +535,7 @@ _Py_call_instrumentation_line(PyThreadState *tstate, PyCodeObject *code, _Py_COD assert(tool < 8); assert(tools & (1 << tool)); tools &= ~(1 << tool); - int res = call_one_instrument(interp, tstate, args, + int res = call_one_instrument(interp, tstate, &args[1], 2 | PY_VECTORCALL_ARGUMENTS_OFFSET, tool, PY_MONITORING_EVENT_LINE); if (res == 0) { @@ -493,6 +552,8 @@ _Py_call_instrumentation_line(PyThreadState *tstate, PyCodeObject *code, _Py_COD } } Py_DECREF(line_obj); + assert(original_opcode != 0); + assert(_PyOpcode_Deopt[original_opcode] == original_opcode); return original_opcode; } @@ -517,6 +578,14 @@ static const int8_t EVENT_FOR_OPCODE[256] = { [RESUME] = -1, [YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD, [INSTRUMENTED_YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD, + [JUMP_FORWARD] = PY_MONITORING_EVENT_JUMP, + [JUMP_BACKWARD] = PY_MONITORING_EVENT_JUMP, + [JUMP_IF_FALSE_OR_POP] = PY_MONITORING_EVENT_BRANCH, + [JUMP_IF_TRUE_OR_POP] = PY_MONITORING_EVENT_BRANCH, + [POP_JUMP_IF_FALSE] = PY_MONITORING_EVENT_BRANCH, + [POP_JUMP_IF_TRUE] = PY_MONITORING_EVENT_BRANCH, + [POP_JUMP_IF_NONE] = PY_MONITORING_EVENT_BRANCH, + [POP_JUMP_IF_NOT_NONE] = PY_MONITORING_EVENT_BRANCH, }; static const bool OPCODE_HAS_EVENT[256] = { @@ -530,6 +599,14 @@ static const bool OPCODE_HAS_EVENT[256] = { [INSTRUMENTED_RESUME] = true, [YIELD_VALUE] = true, [INSTRUMENTED_YIELD_VALUE] = true, + [JUMP_FORWARD] = true, + [JUMP_BACKWARD] = true, + [JUMP_IF_FALSE_OR_POP] = true, + [JUMP_IF_TRUE_OR_POP] = true, + [POP_JUMP_IF_FALSE] = true, + [POP_JUMP_IF_TRUE] = true, + [POP_JUMP_IF_NONE] = true, + [POP_JUMP_IF_NOT_NONE] = true, }; static inline _Py_MonitoringMatrix @@ -585,28 +662,12 @@ matrix_get_events(_Py_MonitoringMatrix *m, int tool_id) return result; } -static bool -is_instrumentation_up_to_date(PyCodeObject *code, PyInterpreterState *interp) -{ - return interp->monitoring_version == code->_co_instrumentation.monitoring_version; -} - static inline _PyInstrumentationLayout layout_from_flags(bool multitools, bool lines, bool instrs) { _PyInstrumentationLayout result; - int size = multitools; - result.offsets.tools = multitools - 1; - result.offsets.lines = -1; - result.offsets.instructions = -1; - if (lines) { - result.offsets.lines = size; - size += multitools ? 3 : 2; - } - if (instrs) { - result.offsets.instructions = size; - size += multitools ? 2 : 1; - } - result.offsets.size = size; + result.offsets.multi_tools = multitools; + result.offsets.lines = lines; + result.offsets.instructions = instrs; return result; } @@ -616,21 +677,24 @@ tools_data_offset() { } static void -initialize_tools(PyCodeObject *code, int size) +initialize_tools(PyCodeObject *code) { + uint8_t* tools = code->_co_instrumentation.monitoring_data->tools; + assert(tools != NULL); int code_len = (int)Py_SIZE(code); for (int i = 0; i < code_len; i++) { _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; int opcode = _Py_OPCODE(*instr); + if (opcode == INSTRUMENTED_LINE) { + opcode = code->_co_instrumentation.monitoring_data->lines[i].original_opcode; + } bool instrumented = is_instrumented(opcode); - assert(opcode != INSTRUMENTED_LINE); if (instrumented) { opcode = DE_INSTRUMENT[opcode]; assert(opcode != 0); } opcode = _PyOpcode_Deopt[opcode]; if (OPCODE_HAS_EVENT[opcode]) { - uint8_t *tools_ptr = &code->_co_instrumentation.monitoring_data[i*size]; if (instrumented) { int8_t event; if (opcode == RESUME) { @@ -642,65 +706,121 @@ initialize_tools(PyCodeObject *code, int size) assert(event > 0); } assert(event >= 0); - *tools_ptr = code->_co_instrumentation.monitoring_matrix.tools[event]; - assert(*tools_ptr != 0); + tools[i] = code->_co_instrumentation.monitoring_matrix.tools[event]; + assert(tools[i] != 0); } else { - *tools_ptr = 0; + tools[i] = 0; } } i += _PyOpcode_Caches[opcode]; } } -static void -copy_data(uint8_t *old_data, int old_size, int old_offset, uint8_t *new_data,int new_size, int new_offset, int code_len) +static bool is_new_line(int index, int current_line, PyCodeAddressRange *bounds, int opcode) { - assert(new_size > 0); - assert(old_offset >= 0); - assert(new_offset >= 0); - for (int i = 0; i < code_len; i++) { - new_data[i*new_size + new_offset] = old_data[i*old_size+old_offset]; + /* END_FOR cannot start a line, as it is skipped by FOR_ITER */ + if (opcode == END_FOR) { + return false; + } + while (bounds->ar_end <= index*(int)sizeof(_Py_CODEUNIT)) { + if (!_PyLineTable_NextAddressRange(bounds)) { + return false; + } } + assert(index*(int)sizeof(_Py_CODEUNIT) >= bounds->ar_start && + index*(int)sizeof(_Py_CODEUNIT) < bounds->ar_end); + return bounds->ar_line >= 0 && bounds->ar_line != current_line; } static void -initialize_lines(PyCodeObject *code, PyInterpreterState *interp) +initialize_lines(PyCodeObject *code) { + _PyCoLineInstrumentationData *line_data = code->_co_instrumentation.monitoring_data->lines; + assert(line_data != NULL); int code_len = (int)Py_SIZE(code); - int size = code->_co_instrumentation.layout.offsets.size; - int offset = code->_co_instrumentation.layout.offsets.lines; - int prev_line = -1; - for (int i = 0; i < code_len; i++) { - uint8_t *line_data = &code->_co_instrumentation.monitoring_data[i*size+offset]; - /* TO DO -- Use line iterator here */ - int line = PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT)); - if (line != prev_line && line <= 0) { - uint8_t opcode = _Py_OPCODE(_PyCode_CODE(code)[i]); - assert(opcode != 0); - line_data[0] = opcode; - line_data[1] = compute_line_delta(code, i, line); - prev_line = line; + int current_line = -1; + PyCodeAddressRange range; + _PyCode_InitAddressRange(code, &range); + for (int i = 0; i <= code->_co_firsttraceable; i++) { + line_data[i].original_opcode = 0; + line_data[i].line_delta = -128; + } + for (int i = code->_co_firsttraceable + 1; i < code_len; i++) { + int opcode = _Py_GetBaseOpcode(code, i); + if (is_new_line(i, current_line, &range, opcode)) { + line_data[i].original_opcode = 255; + current_line = range.ar_line; + line_data[i].line_delta = compute_line_delta(code, i, current_line); } else { - line_data[0] = 0; - line_data[1] = -128; + /* Mark as not being an instrumentation point */ + line_data[i].original_opcode = 0; + line_data[i].line_delta = compute_line_delta(code, i, current_line); } + i += _PyOpcode_Caches[opcode]; } } static void initialize_line_tools(PyCodeObject *code, PyInterpreterState *interp) { + uint8_t *line_tools = code->_co_instrumentation.monitoring_data->line_tools; + assert(line_tools != NULL); int code_len = (int)Py_SIZE(code); - int size = code->_co_instrumentation.layout.offsets.size; - int offset = code->_co_instrumentation.layout.offsets.lines; for (int i = 0; i < code_len; i++) { - code->_co_instrumentation.monitoring_data[i*size+offset+2] = - interp->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE]; + line_tools[i] = interp->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE]; } } +int +update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) +{ + _PyInstrumentationLayout new_layout = interp->instrumentation_layout; + bool restarted = interp->last_restart_version > code->_co_instrumentation.monitoring_version; + int code_len = (int)Py_SIZE(code); + if (code->_co_instrumentation.monitoring_data == NULL) { + code->_co_instrumentation.monitoring_data = PyMem_Calloc(sizeof(_PyCoInstrumentationData), 1); + if (code->_co_instrumentation.monitoring_data == NULL) { + PyErr_NoMemory(); + return -1; + } + } + if (new_layout.offsets.multi_tools && code->_co_instrumentation.monitoring_data->tools == NULL) { + code->_co_instrumentation.monitoring_data->tools = PyMem_Malloc(code_len); + if (code->_co_instrumentation.monitoring_data->tools == NULL) { + PyErr_NoMemory(); + return -1; + } + if (restarted) { + memset(code->_co_instrumentation.monitoring_data->tools, 0, code_len); + } else { + initialize_tools(code); + } + } + if (new_layout.offsets.lines) { + if (code->_co_instrumentation.monitoring_data->lines == NULL) { + code->_co_instrumentation.monitoring_data->lines = PyMem_Malloc(code_len * sizeof(_PyCoLineInstrumentationData)); + if (code->_co_instrumentation.monitoring_data->lines == NULL) { + PyErr_NoMemory(); + return -1; + } + initialize_lines(code); + } + if (new_layout.offsets.multi_tools && code->_co_instrumentation.monitoring_data->line_tools == NULL) { + code->_co_instrumentation.monitoring_data->line_tools = PyMem_Malloc(code_len); + if (code->_co_instrumentation.monitoring_data->line_tools == NULL) { + PyErr_NoMemory(); + return -1; + } + initialize_line_tools(code, interp); + } + } + assert(new_layout.offsets.instructions == 0); + code->_co_instrumentation.layout.bits = new_layout.bits; + return 0; +} + int _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) { @@ -710,76 +830,15 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) return 0; } bool restarted = interp->last_restart_version > code->_co_instrumentation.monitoring_version; - /* First establish how much extra space will need, - * and free/allocate it if different to what we had before */ - /* For now only allow only one tool per event */ assert(interp->monitoring_matrix.tools[PY_MONITORING_EVENT_INSTRUCTION] == 0); int code_len = (int)Py_SIZE(code); _PyInstrumentationLayout new_layout = interp->instrumentation_layout; _PyInstrumentationLayout old_layout = code->_co_instrumentation.layout; if (new_layout.bits != old_layout.bits) { - uint8_t *old_data = code->_co_instrumentation.monitoring_data; - if (new_layout.offsets.size) { - code->_co_instrumentation.monitoring_data = PyMem_Malloc(code_len * new_layout.offsets.size); - if (code->_co_instrumentation.monitoring_data == NULL) { - PyErr_NoMemory(); - code->_co_instrumentation.monitoring_data = old_data; - return -1; - } - } - else { - code->_co_instrumentation.monitoring_data = NULL; - } - if (new_layout.offsets.tools >= 0) { - if (restarted) { - uint8_t zero = 0; - copy_data(&zero, 0, 0, - code->_co_instrumentation.monitoring_data, - new_layout.offsets.size, 0, code_len); - } else if (old_layout.offsets.tools < 0) { - initialize_tools(code, new_layout.offsets.size); - } - else { - copy_data(old_data, old_layout.offsets.size, - 0, code->_co_instrumentation.monitoring_data, - new_layout.offsets.size, 0, code_len); - } - } - if (new_layout.offsets.lines >= 0) { - if (old_layout.offsets.lines < 0) { - initialize_lines(code, interp); - } - else { - copy_data(old_data, old_layout.offsets.size, - old_layout.offsets.lines+1, - code->_co_instrumentation.monitoring_data, - new_layout.offsets.size, - new_layout.offsets.lines+1, code_len); - } - if (new_layout.offsets.tools >= 0) { - if (old_layout.offsets.tools < 0) { - initialize_line_tools(code, interp); - } - else { - copy_data(old_data, old_layout.offsets.size, - old_layout.offsets.lines+2, - code->_co_instrumentation.monitoring_data, - new_layout.offsets.size, - new_layout.offsets.lines+2, code_len); - } - } - } - assert(new_layout.offsets.instructions < 0); - if (old_data) { - PyMem_Free(old_data); + if (update_instrumentation_data(code, interp)) { + return -1; } - code->_co_instrumentation.layout.bits = new_layout.bits; } - //printf("Instrumenting code object at %p, code version: %ld interpreter version %ld\n", - // code, code->_co_instrument_version, interp->monitoring_version); - /* Avoid instrumenting code that has been disabled. - * Only instrument new events - */ _Py_MonitoringMatrix new_events; _Py_MonitoringMatrix removed_events; @@ -831,24 +890,20 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) i += _PyOpcode_Caches[base_opcode]; } uint8_t new_line_tools = new_events.tools[PY_MONITORING_EVENT_LINE]; - if (new_line_tools) { - /* Insert line instrumentation */ - int offset = new_layout.offsets.lines; - assert(offset >= 0); - for (int i = 0; i < code_len; i++) { - uint8_t *instr_data = get_instruction_data(code, i); - if (instr_data[get_offsets(code).lines]) { - add_line_tools(code, i, new_line_tools); - } - } - } uint8_t removed_line_tools = removed_events.tools[PY_MONITORING_EVENT_LINE]; - if (removed_line_tools) { - for (int i = 0; i < code_len; i++) { - uint8_t *instr_data = get_instruction_data(code, i); - if (instr_data[get_offsets(code).lines]) { - remove_line_tools(code, i, removed_line_tools); + if (new_line_tools || removed_line_tools) { + _PyCoLineInstrumentationData *line_data = code->_co_instrumentation.monitoring_data->lines; + for (int i = code->_co_firsttraceable + 1; i < code_len; i++) { + if (line_data[i].original_opcode) { + if (removed_line_tools) { + remove_line_tools(code, i, removed_line_tools); + } + if (new_line_tools) { + add_line_tools(code, i, new_line_tools); + } } + int opcode = _Py_GetBaseOpcode(code, i); + i += _PyOpcode_Caches[opcode]; } } return 0; diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index ea53249d33309b..bb5b78a177943a 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -4,6 +4,7 @@ #include #include "Python.h" +#include "pycore_ceval.h" #include "pycore_instruments.h" #include "pycore_pyerrors.h" #include "pycore_pymem.h" @@ -14,6 +15,7 @@ typedef struct _PyLegacyEventHandler { PyObject_HEAD vectorcallfunc vectorcall; int event; + PyCodeObject *last_code; } _PyLegacyEventHandler; static void @@ -36,15 +38,10 @@ call_profile_func(_PyLegacyEventHandler *self, PyObject *arg) if (tstate->c_profilefunc == NULL) { Py_RETURN_NONE; } - if (tstate->tracing) { - Py_RETURN_NONE; - } PyFrameObject* frame = PyEval_GetFrame(); Py_INCREF(frame); assert(frame != NULL); - tstate->tracing++; int err = tstate->c_profilefunc(tstate->c_profileobj, frame, self->event, arg); - tstate->tracing--; Py_DECREF(frame); if (err) { return NULL; @@ -90,7 +87,25 @@ sys_profile_call_or_return( } static PyObject * -sys_profile_exception_func( +call_trace_func(_PyLegacyEventHandler *self, PyObject *arg) +{ + PyThreadState *tstate = _PyThreadState_GET(); + if (tstate->c_tracefunc == NULL) { + Py_RETURN_NONE; + } + PyFrameObject* frame = PyEval_GetFrame(); + Py_INCREF(frame); + assert(frame != NULL); + int err = tstate->c_tracefunc(tstate->c_traceobj, frame, self->event, arg); + Py_DECREF(frame); + if (err) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +sys_trace_exception_func( _PyLegacyEventHandler *self, PyObject *const *args, size_t nargsf, PyObject *kwnames ) { @@ -102,11 +117,33 @@ sys_profile_exception_func( if (arg == NULL) { return NULL; } - PyObject *res = call_profile_func(self, arg); + PyObject *res = call_trace_func(self, arg); Py_DECREF(arg); return res; } +static PyObject * +sys_trace_func2( + _PyLegacyEventHandler *self, PyObject *const *args, + size_t nargsf, PyObject *kwnames +) { + assert(kwnames == NULL); + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + assert(nargs == 2); + return call_trace_func(self, Py_None); +} + +static PyObject * +sys_trace_func3( + _PyLegacyEventHandler *self, PyObject *const *args, + size_t nargsf, PyObject *kwnames +) { + assert(kwnames == NULL); + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + assert(nargs == 3); + return call_trace_func(self, args[2]); +} + static PyObject * sys_trace_none_func( _PyLegacyEventHandler *self, PyObject *const *args, @@ -115,7 +152,93 @@ sys_trace_none_func( assert(kwnames == NULL); Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); assert(nargs == 3); - return call_profile_func(self, Py_None); + return call_trace_func(self, Py_None); +} + +static PyObject * +trace_line(PyThreadState *tstate, PyCodeObject *code, + _PyLegacyEventHandler *self, PyFrameObject* frame, int line +) { + Py_INCREF(frame); + tstate->trace_info.code = code; + tstate->trace_info.line = frame->f_lineno = line; + int err = tstate->c_tracefunc(tstate->c_traceobj, frame, self->event, Py_None); + frame->f_lineno = 0; + Py_DECREF(frame); + if (err) { + return NULL; + } + Py_RETURN_NONE; +} + + +static PyObject * +sys_trace_line_func( + _PyLegacyEventHandler *self, PyObject *const *args, + size_t nargsf, PyObject *kwnames +) { + assert(kwnames == NULL); + PyThreadState *tstate = _PyThreadState_GET(); + if (tstate->c_tracefunc == NULL) { + Py_RETURN_NONE; + } + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + assert(nargs == 2); + _PyInterpreterFrame *iframe = _PyEval_GetFrame(); + assert(iframe); + PyFrameObject* frame = _PyFrame_GetFrameObject(iframe); + if (frame == NULL) { + return NULL; + } + if (!frame->f_trace_lines) { + Py_RETURN_NONE; + } + assert(args[0] == (PyObject *)iframe->f_code); + int line = _PyLong_AsInt(args[1]); + assert(line >= 0); + if (tstate->trace_info.code == iframe->f_code && + tstate->trace_info.line == line) { + /* Already traced this line */ + Py_RETURN_NONE; + } + return trace_line(tstate, iframe->f_code, self, frame, line); +} + + +static PyObject * +sys_trace_branch_func( + _PyLegacyEventHandler *self, PyObject *const *args, + size_t nargsf, PyObject *kwnames +) { + assert(kwnames == NULL); + PyThreadState *tstate = _PyThreadState_GET(); + if (tstate->c_tracefunc == NULL) { + Py_RETURN_NONE; + } + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + assert(nargs == 3); + _PyInterpreterFrame *iframe = _PyEval_GetFrame(); + assert(iframe); + PyFrameObject* frame = _PyFrame_GetFrameObject(iframe); + int from = _PyLong_AsInt(args[1]); + assert(from >= 0); + int to = _PyLong_AsInt(args[2]); + assert(to >= 0); + /* We can call _Py_Instrumentation_GetLine because we always set + * line events for tracing */ + int to_line = _Py_Instrumentation_GetLine(iframe->f_code, to); + if (from > to) { + /* Backwards jump */ + return trace_line(tstate, iframe->f_code, self, frame, to_line); + } + else { + /* Forwards jump, only handle if new line */ + int from_line = _Py_Instrumentation_GetLine(iframe->f_code, from); + if (from_line != to_line) { + return trace_line(tstate, iframe->f_code, self, frame, to_line); + } + } + Py_RETURN_NONE; } @@ -262,25 +385,30 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) if (!tstate->interp->sys_trace_initialized) { tstate->interp->sys_trace_initialized = true; if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, - (vectorcallfunc)sys_profile_func2, PyTrace_CALL, + (vectorcallfunc)sys_trace_func2, PyTrace_CALL, PY_MONITORING_EVENT_PY_START, PY_MONITORING_EVENT_PY_RESUME)) { return -1; } if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, - (vectorcallfunc)sys_profile_func3, PyTrace_RETURN, + (vectorcallfunc)sys_trace_func3, PyTrace_RETURN, PY_MONITORING_EVENT_PY_RETURN, PY_MONITORING_EVENT_PY_YIELD)) { return -1; } if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, - (vectorcallfunc)sys_profile_exception_func, PyTrace_EXCEPTION, + (vectorcallfunc)sys_trace_exception_func, PyTrace_EXCEPTION, PY_MONITORING_EVENT_RAISE, -1)) { return -1; } if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, - (vectorcallfunc)sys_trace_none_func, PyTrace_LINE, + (vectorcallfunc)sys_trace_line_func, PyTrace_LINE, PY_MONITORING_EVENT_LINE, -1)) { return -1; } + if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, + (vectorcallfunc)sys_trace_branch_func, PyTrace_LINE, + PY_MONITORING_EVENT_JUMP, PY_MONITORING_EVENT_BRANCH)) { + return -1; + } if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, (vectorcallfunc)sys_trace_none_func, PyTrace_OPCODE, PY_MONITORING_EVENT_INSTRUCTION, -1)) { @@ -291,14 +419,16 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) uint32_t events = (1 << PY_MONITORING_EVENT_PY_START) | (1 << PY_MONITORING_EVENT_PY_RESUME) | (1 << PY_MONITORING_EVENT_PY_RETURN) | (1 << PY_MONITORING_EVENT_PY_YIELD) | - (1 << PY_MONITORING_EVENT_RAISE) | (1 << PY_MONITORING_EVENT_LINE); - if (tstate->interp->f_opcode_trace_set) { + (1 << PY_MONITORING_EVENT_RAISE) | (1 << PY_MONITORING_EVENT_LINE) | + (1 << PY_MONITORING_EVENT_JUMP) | (1 << PY_MONITORING_EVENT_BRANCH); + /* TO DO -- opcode events */ + if (tstate->interp->f_opcode_trace_set && false) { events |= (1 << PY_MONITORING_EVENT_INSTRUCTION); } - _PyMonitoring_SetEvents(PY_INSTRUMENT_SYS_PROFILE, events); + _PyMonitoring_SetEvents(PY_INSTRUMENT_SYS_TRACE, events); } else { - _PyMonitoring_SetEvents(PY_INSTRUMENT_SYS_PROFILE, 0); + _PyMonitoring_SetEvents(PY_INSTRUMENT_SYS_TRACE, 0); } Py_XDECREF(old_traceobj); return 0; diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index b5a6712b6077c8..18c75a50914efc 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -244,15 +244,15 @@ static void *opcode_targets[256] = { &&TARGET_INSTRUMENTED_RETURN_VALUE, &&TARGET_INSTRUMENTED_YIELD_VALUE, &&TARGET_INSTRUMENTED_CALL_FUNCTION_EX, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, + &&TARGET_INSTRUMENTED_JUMP_FORWARD, + &&TARGET_INSTRUMENTED_JUMP_BACKWARD, + &&TARGET_INSTRUMENTED_JUMP_IF_FALSE_OR_POP, + &&_unknown_opcode, + &&TARGET_INSTRUMENTED_JUMP_IF_TRUE_OR_POP, + &&TARGET_INSTRUMENTED_POP_JUMP_IF_FALSE, + &&TARGET_INSTRUMENTED_POP_JUMP_IF_TRUE, + &&TARGET_INSTRUMENTED_POP_JUMP_IF_NONE, + &&TARGET_INSTRUMENTED_POP_JUMP_IF_NOT_NONE, &&TARGET_INSTRUMENTED_LINE, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_DO_TRACING }; diff --git a/Python/pystate.c b/Python/pystate.c index fa6aa31d3c4f85..460b34a4677f03 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -314,7 +314,7 @@ init_interpreter(PyInterpreterState *interp, interp->tools[t].instrument_callables[i] = NULL; } } - interp->instrumentation_layout.offsets = (_PyInstrumentationOffsets){ -1, -1, -1, 0 }; + interp->instrumentation_layout.offsets = (_PyInstrumentationOffsets){ 0, 0, 0 }; interp->f_opcode_trace_set = false; interp->_initialized = 1; } From 15a1ccd69bf2030bf96f5dfc80edbacccd857b2d Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 13 Jan 2023 10:58:17 +0000 Subject: [PATCH 009/116] Fix up INSTRUMENTED_OPCODES vector. --- Python/instrumentation.c | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 134ab488ae08cf..0a1c2594501d89 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -32,24 +32,33 @@ static const uint8_t DE_INSTRUMENT[256] = { static const uint8_t INSTRUMENTED_OPCODES[256] = { [RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE, - [CALL] = INSTRUMENTED_CALL, - [CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, - [YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE, - [RESUME] = INSTRUMENTED_RESUME, [INSTRUMENTED_RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE, + [CALL] = INSTRUMENTED_CALL, [INSTRUMENTED_CALL] = INSTRUMENTED_CALL, + [CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, [INSTRUMENTED_CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, + [YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE, [INSTRUMENTED_YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE, + [RESUME] = INSTRUMENTED_RESUME, [INSTRUMENTED_RESUME] = INSTRUMENTED_RESUME, - [INSTRUMENTED_LINE] = INSTRUMENTED_LINE, [JUMP_FORWARD] = INSTRUMENTED_JUMP_FORWARD, + [INSTRUMENTED_JUMP_FORWARD] = INSTRUMENTED_JUMP_FORWARD, [JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD, + [INSTRUMENTED_JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD, [JUMP_IF_FALSE_OR_POP] = INSTRUMENTED_JUMP_IF_FALSE_OR_POP, + [INSTRUMENTED_JUMP_IF_FALSE_OR_POP] = INSTRUMENTED_JUMP_IF_FALSE_OR_POP, [JUMP_IF_TRUE_OR_POP] = INSTRUMENTED_JUMP_IF_TRUE_OR_POP, + [INSTRUMENTED_JUMP_IF_TRUE_OR_POP] = INSTRUMENTED_JUMP_IF_TRUE_OR_POP, [POP_JUMP_IF_FALSE] = INSTRUMENTED_POP_JUMP_IF_FALSE, + [INSTRUMENTED_POP_JUMP_IF_FALSE] = INSTRUMENTED_POP_JUMP_IF_FALSE, [POP_JUMP_IF_TRUE] = INSTRUMENTED_POP_JUMP_IF_TRUE, + [INSTRUMENTED_POP_JUMP_IF_TRUE] = INSTRUMENTED_POP_JUMP_IF_TRUE, [POP_JUMP_IF_NONE] = INSTRUMENTED_POP_JUMP_IF_NONE, + [INSTRUMENTED_POP_JUMP_IF_NONE] = INSTRUMENTED_POP_JUMP_IF_NONE, [POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE, + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE, + + [INSTRUMENTED_LINE] = INSTRUMENTED_LINE, }; _PyInstrumentationOffsets @@ -173,6 +182,9 @@ instrument(PyCodeObject *code, int offset) else if (!is_instrumented(opcode)) { opcode = _PyOpcode_Deopt[opcode]; int instrumented = INSTRUMENTED_OPCODES[opcode]; + if (!instrumented) { + printf("OPCODE: %s\n", _PyOpcode_OpName[opcode]); + } assert(instrumented); instr->opcode = instrumented; if (_PyOpcode_Caches[opcode]) { From 9abb339defb6e030f2e35371c137698dd2c4dc50 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 16 Jan 2023 10:48:14 +0000 Subject: [PATCH 010/116] Fix instrumented branches to call instrumentation with correct target. --- Python/bytecodes.c | 86 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 70 insertions(+), 16 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 52e3b6cee65ed6..a8c02896568961 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3418,45 +3418,99 @@ dummy_func( } inst(INSTRUMENTED_JUMP_IF_TRUE_OR_POP, ( -- )) { - int err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); + PyObject *cond = TOP(); + int err = PyObject_IsTrue(cond); + ERROR_IF(err < 0, error); + _Py_CODEUNIT *here = next_instr-1; + assert(err == 0 || err == 1); + _Py_CODEUNIT *there = next_instr + err*oparg; + err = _Py_call_instrumentation_jump( + tstate, PY_MONITORING_EVENT_JUMP, frame, here, there); ERROR_IF(err, error); - GO_TO_INSTRUCTION(JUMP_IF_TRUE_OR_POP); + if (err == 0) { + STACK_SHRINK(1); + Py_DECREF(cond); + } + next_instr = there; } inst(INSTRUMENTED_JUMP_IF_FALSE_OR_POP, ( -- )) { - int err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); + PyObject *cond = TOP(); + int err = PyObject_IsTrue(cond); + ERROR_IF(err < 0, error); + _Py_CODEUNIT *here = next_instr-1; + assert(err == 0 || err == 1); + _Py_CODEUNIT *there = next_instr + (1-err)*oparg; + err = _Py_call_instrumentation_jump( + tstate, PY_MONITORING_EVENT_JUMP, frame, here, there); ERROR_IF(err, error); - GO_TO_INSTRUCTION(JUMP_IF_FALSE_OR_POP); + if (err) { + STACK_SHRINK(1); + Py_DECREF(cond); + } + next_instr = there; } inst(INSTRUMENTED_POP_JUMP_IF_TRUE, ( -- )) { - int err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); + PyObject *cond = POP(); + int err = PyObject_IsTrue(cond); + ERROR_IF(err < 0, error); + _Py_CODEUNIT *here = next_instr-1; + assert(err == 0 || err == 1); + _Py_CODEUNIT *there = next_instr + err*oparg; + err = _Py_call_instrumentation_jump( + tstate, PY_MONITORING_EVENT_JUMP, frame, here, there); ERROR_IF(err, error); - GO_TO_INSTRUCTION(POP_JUMP_IF_TRUE); + next_instr = there; } inst(INSTRUMENTED_POP_JUMP_IF_FALSE, ( -- )) { - int err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); + PyObject *cond = POP(); + int err = PyObject_IsTrue(cond); + ERROR_IF(err < 0, error); + _Py_CODEUNIT *here = next_instr-1; + assert(err == 0 || err == 1); + _Py_CODEUNIT *there = next_instr + (1-err)*oparg; + err = _Py_call_instrumentation_jump( + tstate, PY_MONITORING_EVENT_JUMP, frame, here, there); ERROR_IF(err, error); - GO_TO_INSTRUCTION(POP_JUMP_IF_FALSE); + next_instr = there; } inst(INSTRUMENTED_POP_JUMP_IF_NONE, ( -- )) { + PyObject *value = POP(); + _Py_CODEUNIT *here = next_instr-1; + _Py_CODEUNIT *there; + if (Py_IsNone(value)) { + _Py_DECREF_NO_DEALLOC(value); + there = next_instr + oparg; + } + else { + Py_DECREF(value); + there = next_instr; + } int err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); + tstate, PY_MONITORING_EVENT_JUMP, frame, here, there); ERROR_IF(err, error); - GO_TO_INSTRUCTION(POP_JUMP_IF_NONE); + next_instr = there; } inst(INSTRUMENTED_POP_JUMP_IF_NOT_NONE, ( -- )) { + PyObject *value = POP(); + _Py_CODEUNIT *here = next_instr-1; + _Py_CODEUNIT *there; + if (Py_IsNone(value)) { + _Py_DECREF_NO_DEALLOC(value); + there = next_instr; + } + else { + Py_DECREF(value); + there = next_instr + oparg; + } int err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); + tstate, PY_MONITORING_EVENT_JUMP, frame, here, there); ERROR_IF(err, error); - GO_TO_INSTRUCTION(POP_JUMP_IF_NOT_NONE); + next_instr = there; } // stack effect: ( -- ) From aa09895e86b1f9568616fcaff41375be04ce4978 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 17 Jan 2023 11:55:40 +0000 Subject: [PATCH 011/116] Add PY_THROW event handling and fix up line table. --- Python/ceval.c | 17 ++++++++++++++++- Python/instrumentation.c | 40 ++++++++++++++++++++++++++++++---------- Python/legacy_tracing.c | 19 ++++++++++++++++++- 3 files changed, 64 insertions(+), 12 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index d4dabe04920497..1e2a08555e3f3a 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -187,6 +187,9 @@ static void monitor_unwind(PyThreadState *tstate, static void monitor_handled(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *exc); +static void monitor_throw(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr); static PyObject * import_name(PyThreadState *, _PyInterpreterFrame *, PyObject *, PyObject *, PyObject *); @@ -998,6 +1001,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int /* Because this avoids the RESUME, * we need to update instrumentation */ _Py_Instrument(frame->f_code, tstate->interp); + monitor_throw(tstate, frame, frame->prev_instr); /* TO DO -- Monitor throw entry. */ goto resume_with_error; } @@ -1225,9 +1229,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int PyException_SetTraceback(val, Py_None); Py_XDECREF(tb); Py_XDECREF(exc); - monitor_handled(tstate, frame, next_instr-1, val); PUSH(val); JUMPTO(handler); + monitor_handled(tstate, frame, next_instr, val); /* Resume normal execution */ DISPATCH(); } @@ -2271,6 +2275,17 @@ monitor_handled(PyThreadState *tstate, _Py_call_instrumentation_arg(tstate, PY_MONITORING_EVENT_EXCEPTION_HANDLED, frame, instr, exc); } +static void +monitor_throw(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr) +{ + if (tstate->interp->monitoring_matrix.tools[PY_MONITORING_EVENT_PY_THROW] == 0) { + return; + } + _Py_call_instrumentation_exc(tstate, PY_MONITORING_EVENT_PY_THROW, frame, instr, NULL); +} + void PyThreadState_EnterTracing(PyThreadState *tstate) { diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 8424b01d2e9129..6668dc535551aa 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -574,15 +574,23 @@ _Py_call_instrumentation_exc( /* Line delta. * 8 bit value. - * if line_delta != -128: - * line = first_line + (offset >> 3) + line_delta; - * else: + * if line_delta == -128: + * line = None # represented as -1 + * elif line == -127: * line = PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT)); + * else: + * line = first_line + (offset >> 3) + line_delta; */ static int8_t compute_line_delta(PyCodeObject *code, int offset, int line) { + if (line < 0) { + return -128; + } + assert(line >= code->co_firstlineno); + assert(offset >= code->_co_firsttraceable); + assert(offset < Py_SIZE(code)); int delta = line - code->co_firstlineno - (offset >> 3); if (delta < 128 && delta > -128) { return delta; @@ -593,12 +601,18 @@ compute_line_delta(PyCodeObject *code, int offset, int line) static int compute_line(PyCodeObject *code, int offset, int8_t line_delta) { - assert(line_delta != -128); - if (line_delta == -127) { + if (line_delta > -127) { + assert((offset >> 3) + line_delta >= 0); + return code->co_firstlineno + (offset >> 3) + line_delta; + } + if (line_delta == -128) { + return -1; + } + else { + assert(line_delta == -127); /* Compute from table */ return PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT)); } - return code->co_firstlineno + (offset >> 3) + line_delta; } int @@ -607,6 +621,8 @@ _Py_Instrumentation_GetLine(PyCodeObject *code, int index) _PyCoInstrumentation *instrumentation = &code->_co_instrumentation; assert(instrumentation->monitoring_data != NULL); assert(instrumentation->monitoring_data->lines != NULL); + assert(index >= code->_co_firsttraceable); + assert(index < Py_SIZE(code)); _PyCoLineInstrumentationData *line_data = &instrumentation->monitoring_data->lines[index]; int8_t line_delta = line_data->line_delta; int line = compute_line(code, index, line_delta); @@ -859,14 +875,18 @@ initialize_lines(PyCodeObject *code) if (is_new_line(i, current_line, &range, opcode)) { line_data[i].original_opcode = 255; current_line = range.ar_line; - line_data[i].line_delta = compute_line_delta(code, i, current_line); } else { /* Mark as not being an instrumentation point */ line_data[i].original_opcode = 0; - line_data[i].line_delta = compute_line_delta(code, i, current_line); } - i += _PyOpcode_Caches[opcode]; + assert(range.ar_line < 0 || range.ar_line >= code->co_firstlineno); + line_data[i].line_delta = compute_line_delta(code, i, range.ar_line); + for (int j = 0; j < _PyOpcode_Caches[opcode]; j++) { + i++; + line_data[i].original_opcode = 0; + line_data[i].line_delta = -128; + } } } @@ -1002,7 +1022,7 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) int8_t event; if (base_opcode == RESUME) { int oparg = _Py_OPARG(*instr); - event = oparg; + event = oparg > 0; } else { event = EVENT_FOR_OPCODE[base_opcode]; diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index 4c76c47259cc4d..016c1d99a35420 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -159,6 +159,9 @@ static PyObject * trace_line(PyThreadState *tstate, _PyInterpreterFrame *iframe, _PyLegacyEventHandler *self, PyFrameObject* frame, int line ) { + if (line < 0) { + Py_RETURN_NONE; + } if (tstate->trace_info.code == iframe->f_code && tstate->trace_info.line == line) { /* Already traced this line */ @@ -229,6 +232,9 @@ sys_trace_handled_exception( int offset = _PyLong_AsInt(args[1]); assert(offset >= 0); int line = _Py_Instrumentation_GetLine(iframe->f_code, offset); + if (line < 0) { + Py_RETURN_NONE; + } return trace_line(tstate, iframe, self, frame, line); } @@ -411,6 +417,11 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) PY_MONITORING_EVENT_PY_START, PY_MONITORING_EVENT_PY_RESUME)) { return -1; } + if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, + (vectorcallfunc)sys_trace_func2, PyTrace_CALL, + PY_MONITORING_EVENT_PY_THROW, -1)) { + return -1; + } if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, (vectorcallfunc)sys_trace_func3, PyTrace_RETURN, PY_MONITORING_EVENT_PY_RETURN, PY_MONITORING_EVENT_PY_YIELD)) { @@ -426,6 +437,11 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) PY_MONITORING_EVENT_LINE, -1)) { return -1; } + if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, + (vectorcallfunc)sys_trace_func2, PyTrace_RETURN, + PY_MONITORING_EVENT_PY_UNWIND, -1)) { + return -1; + } if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, (vectorcallfunc)sys_trace_branch_func, PyTrace_LINE, PY_MONITORING_EVENT_JUMP, PY_MONITORING_EVENT_BRANCH)) { @@ -449,7 +465,8 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) (1 << PY_MONITORING_EVENT_PY_RETURN) | (1 << PY_MONITORING_EVENT_PY_YIELD) | (1 << PY_MONITORING_EVENT_RAISE) | (1 << PY_MONITORING_EVENT_LINE) | (1 << PY_MONITORING_EVENT_JUMP) | (1 << PY_MONITORING_EVENT_BRANCH) | - (1 << PY_MONITORING_EVENT_EXCEPTION_HANDLED); + (1 << PY_MONITORING_EVENT_EXCEPTION_HANDLED | (1 << PY_MONITORING_EVENT_PY_UNWIND) | + (1 << PY_MONITORING_EVENT_PY_THROW)); /* TO DO -- opcode events */ if (tstate->interp->f_opcode_trace_set && false) { events |= (1 << PY_MONITORING_EVENT_INSTRUCTION); From 9e5d87de66c3a1ecc527e6825f86375e006651f2 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 17 Jan 2023 12:54:26 +0000 Subject: [PATCH 012/116] LINE events working for sys.setrace. --- Python/legacy_tracing.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index 016c1d99a35420..187a3246b71616 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -4,6 +4,7 @@ #include #include "Python.h" +#include "opcode.h" #include "pycore_ceval.h" #include "pycore_instruments.h" #include "pycore_pyerrors.h" @@ -231,6 +232,9 @@ sys_trace_handled_exception( assert(args[0] == (PyObject *)iframe->f_code); int offset = _PyLong_AsInt(args[1]); assert(offset >= 0); + if (_PyCode_CODE(iframe->f_code)[offset].opcode == END_ASYNC_FOR) { + Py_RETURN_NONE; + } int line = _Py_Instrumentation_GetLine(iframe->f_code, offset); if (line < 0) { Py_RETURN_NONE; @@ -266,6 +270,12 @@ sys_trace_branch_func( /* Backwards jump -- always trace */ tstate->trace_info.code = NULL; } + else { + int from_line = _Py_Instrumentation_GetLine(iframe->f_code, from); + if (to_line == from_line) { + Py_RETURN_NONE; + } + } return trace_line(tstate, iframe, self, frame, to_line); } From e52cbe668a75be23a75b5485e0f89628eb237866 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 18 Jan 2023 11:10:39 +0000 Subject: [PATCH 013/116] Add lots of internal debugging for instrumentation. --- Include/cpython/code.h | 14 +- Include/internal/pycore_interp.h | 1 - Objects/codeobject.c | 6 - Python/bytecodes.c | 14 +- Python/generated_cases.c.h | 14 +- Python/instrumentation.c | 378 ++++++++++++++++++------------- Python/pystate.c | 1 - 7 files changed, 232 insertions(+), 196 deletions(-) diff --git a/Include/cpython/code.h b/Include/cpython/code.h index fcad7f1fb6ce22..ffb6d513fcf7fd 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -49,23 +49,13 @@ typedef struct { PyObject *_co_freevars; } _PyCoCached; -typedef struct _PyInstrumentationOffsets { - int8_t multi_tools; - int8_t lines; - int8_t instructions; -} _PyInstrumentationOffsets; - -typedef union _PyInstrumentationLayout { - _PyInstrumentationOffsets offsets; - int32_t bits; -} _PyInstrumentationLayout; - typedef struct { uint8_t original_opcode; int8_t line_delta; } _PyCoLineInstrumentationData; typedef struct { + _Py_MonitoringMatrix matrix; uint8_t *tools; _PyCoLineInstrumentationData *lines; uint8_t *line_tools; @@ -75,8 +65,6 @@ typedef struct { typedef struct { uint64_t monitoring_version; /* current instrumentation version */ - _PyInstrumentationLayout layout; - _Py_MonitoringMatrix monitoring_matrix; _PyCoInstrumentationData *monitoring_data; /* data for monitoring */ } _PyCoInstrumentation; diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 59a52f706682e8..184db0505864e6 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -191,7 +191,6 @@ struct _is { PyCodeObject *interpreter_trampoline; _Py_MonitoringMatrix monitoring_matrix; - _PyInstrumentationLayout instrumentation_layout; uint8_t required_monitoring_bytes; /* Tools numbered 1-8. 0 is the dispatcher/sole tool */ struct _instrumentation_tool tools[PY_MONITORING_TOOL_IDS]; diff --git a/Objects/codeobject.c b/Objects/codeobject.c index f1eff3c0a1ad79..e96b0ca5d48c88 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -402,12 +402,6 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) if (_Py_next_func_version != 0) { _Py_next_func_version++; } - for (int e = 0; e < PY_MONITORING_EVENTS; e++) { - co->_co_instrumentation.monitoring_matrix.tools[e] = 0; - } - co->_co_instrumentation.layout.offsets.multi_tools = 0; - co->_co_instrumentation.layout.offsets.lines = 0; - co->_co_instrumentation.layout.offsets.instructions = 0; co->_co_instrumentation.monitoring_data = NULL; co->_co_instrumentation.monitoring_version = 0; /* not set */ diff --git a/Python/bytecodes.c b/Python/bytecodes.c index ad2240543c2be5..31d1c5d2050004 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1897,7 +1897,7 @@ dummy_func( target = next_instr + 2; } err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, target); + tstate, PY_MONITORING_EVENT_BRANCH, frame, next_instr-1, target); ERROR_IF(err, error); } @@ -3458,7 +3458,7 @@ dummy_func( assert(err == 0 || err == 1); _Py_CODEUNIT *target = next_instr + err*oparg; err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_JUMP, frame, here, target); + tstate, PY_MONITORING_EVENT_BRANCH, frame, here, target); ERROR_IF(err, error); if (err) { JUMPBY(oparg); @@ -3477,7 +3477,7 @@ dummy_func( assert(err == 0 || err == 1); _Py_CODEUNIT *target = next_instr + err*oparg; err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_JUMP, frame, here, target); + tstate, PY_MONITORING_EVENT_BRANCH, frame, here, target); ERROR_IF(err, error); if (err == 0) { JUMPBY(oparg); @@ -3496,7 +3496,7 @@ dummy_func( assert(err == 0 || err == 1); int offset = err*oparg; err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_JUMP, frame, here, next_instr + offset); + tstate, PY_MONITORING_EVENT_BRANCH, frame, here, next_instr + offset); ERROR_IF(err, error); JUMPBY(offset); } @@ -3509,7 +3509,7 @@ dummy_func( assert(err == 0 || err == 1); int offset = (1-err)*oparg; err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_JUMP, frame, here, next_instr + offset); + tstate, PY_MONITORING_EVENT_BRANCH, frame, here, next_instr + offset); ERROR_IF(err, error); JUMPBY(offset); } @@ -3527,7 +3527,7 @@ dummy_func( offset = 0; } int err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_JUMP, frame, here, next_instr + offset); + tstate, PY_MONITORING_EVENT_BRANCH, frame, here, next_instr + offset); ERROR_IF(err, error); JUMPBY(offset); } @@ -3545,7 +3545,7 @@ dummy_func( offset = oparg; } int err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_JUMP, frame, here, next_instr + offset); + tstate, PY_MONITORING_EVENT_BRANCH, frame, here, next_instr + offset); ERROR_IF(err, error); JUMPBY(offset); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index b2ea7dcf8fa484..99f8ba38aa1d72 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2158,7 +2158,7 @@ target = next_instr + 2; } err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, target); + tstate, PY_MONITORING_EVENT_BRANCH, frame, next_instr-1, target); if (err) goto pop_2_error; STACK_SHRINK(2); JUMPBY(2); @@ -3785,7 +3785,7 @@ assert(err == 0 || err == 1); _Py_CODEUNIT *target = next_instr + err*oparg; err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_JUMP, frame, here, target); + tstate, PY_MONITORING_EVENT_BRANCH, frame, here, target); if (err) goto error; if (err) { JUMPBY(oparg); @@ -3805,7 +3805,7 @@ assert(err == 0 || err == 1); _Py_CODEUNIT *target = next_instr + err*oparg; err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_JUMP, frame, here, target); + tstate, PY_MONITORING_EVENT_BRANCH, frame, here, target); if (err) goto error; if (err == 0) { JUMPBY(oparg); @@ -3825,7 +3825,7 @@ assert(err == 0 || err == 1); int offset = err*oparg; err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_JUMP, frame, here, next_instr + offset); + tstate, PY_MONITORING_EVENT_BRANCH, frame, here, next_instr + offset); if (err) goto error; JUMPBY(offset); DISPATCH(); @@ -3839,7 +3839,7 @@ assert(err == 0 || err == 1); int offset = (1-err)*oparg; err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_JUMP, frame, here, next_instr + offset); + tstate, PY_MONITORING_EVENT_BRANCH, frame, here, next_instr + offset); if (err) goto error; JUMPBY(offset); DISPATCH(); @@ -3858,7 +3858,7 @@ offset = 0; } int err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_JUMP, frame, here, next_instr + offset); + tstate, PY_MONITORING_EVENT_BRANCH, frame, here, next_instr + offset); if (err) goto error; JUMPBY(offset); DISPATCH(); @@ -3877,7 +3877,7 @@ offset = oparg; } int err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_JUMP, frame, here, next_instr + offset); + tstate, PY_MONITORING_EVENT_BRANCH, frame, here, next_instr + offset); if (err) goto error; JUMPBY(offset); DISPATCH(); diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 6668dc535551aa..9934e03aeed6c5 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -106,8 +106,10 @@ dump_instrumentation_data_line_tools(PyCodeObject *code, uint8_t *line_tools, in /* No error checking -- Don't use this for anything but experimental debugging */ static void -dump_instrumentation_data(PyCodeObject *code, _PyCoInstrumentationData *data, int star, FILE*out) +dump_instrumentation_data(PyCodeObject *code, int star, FILE*out) { + _PyCoInstrumentationData *data = code->_co_instrumentation.monitoring_data; + fprintf(out, "\n"); if (data == NULL) { fprintf(out, "NULL\n"); return; @@ -124,18 +126,18 @@ dump_instrumentation_data(PyCodeObject *code, _PyCoInstrumentationData *data, in else { base_opcode = _PyOpcode_Deopt[opcode]; } - /* TO DO -- Use instruction length, not cache count */ - i += _PyOpcode_Caches[base_opcode]; if (i == star) { fprintf(out, "** "); starred = true; } - fprintf(out, "Offset: %d, Instruction: %s: ", i, _PyOpcode_OpName[opcode]); + fprintf(out, "Offset: %d, line: %d %s: ", i, PyCode_Addr2Line(code, i*2), _PyOpcode_OpName[opcode]); dump_instrumentation_data_tools(code, data->tools, i, out); dump_instrumentation_data_lines(code, data->lines, i, out); dump_instrumentation_data_line_tools(code, data->line_tools, i, out); /* TO DO -- per instruction data */ fprintf(out, "\n"); + /* TO DO -- Use instruction length, not cache count */ + i += _PyOpcode_Caches[base_opcode]; } if (!starred && star >= 0) { fprintf(out, "Error offset not at valid instruction offset: %d\n", star); @@ -143,12 +145,171 @@ dump_instrumentation_data(PyCodeObject *code, _PyCoInstrumentationData *data, in dump_instrumentation_data_tools(code, data->tools, star, out); dump_instrumentation_data_lines(code, data->lines, star, out); dump_instrumentation_data_line_tools(code, data->line_tools, star, out); + fprintf(out, "\n"); + } +} + +#define OFFSET_SHIFT 5 + +/* Line delta. + * 8 bit value. + * if line_delta == -128: + * line = None # represented as -1 + * elif line == -127: + * line = PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT)); + * else: + * line = first_line + (offset >> OFFSET_SHIFT) + line_delta; + */ + +static int8_t +compute_line_delta(PyCodeObject *code, int offset, int line) +{ + if (line < 0) { + return -128; + } + // assert(line >= code->co_firstlineno); + // assert(offset >= code->_co_firsttraceable); + // assert(offset < Py_SIZE(code)); + int delta = line - code->co_firstlineno - (offset >> OFFSET_SHIFT); + if (delta < 128 && delta > -128) { + return delta; + } + return -127; +} + +static int +compute_line(PyCodeObject *code, int offset, int8_t line_delta) +{ + if (line_delta > -127) { + // assert((offset >> OFFSET_SHIFT) + line_delta >= 0); + return code->co_firstlineno + (offset >> OFFSET_SHIFT) + line_delta; + } + if (line_delta == -128) { + return -1; + } + else { + assert(line_delta == -127); + /* Compute from table */ + return PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT)); + } +} + + +static inline bool +is_instrumented(int opcode) { + return INSTRUMENTED_OPCODES[opcode] == opcode; +} + +static const int8_t EVENT_FOR_OPCODE[256] = { + [RETURN_VALUE] = PY_MONITORING_EVENT_PY_RETURN, + [INSTRUMENTED_RETURN_VALUE] = PY_MONITORING_EVENT_PY_RETURN, + [CALL] = PY_MONITORING_EVENT_CALL, + [INSTRUMENTED_CALL] = PY_MONITORING_EVENT_CALL, + [CALL_FUNCTION_EX] = PY_MONITORING_EVENT_CALL, + [INSTRUMENTED_CALL_FUNCTION_EX] = PY_MONITORING_EVENT_CALL, + [RESUME] = -1, + [YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD, + [INSTRUMENTED_YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD, + [JUMP_FORWARD] = PY_MONITORING_EVENT_JUMP, + [JUMP_BACKWARD] = PY_MONITORING_EVENT_JUMP, + [JUMP_IF_FALSE_OR_POP] = PY_MONITORING_EVENT_BRANCH, + [JUMP_IF_TRUE_OR_POP] = PY_MONITORING_EVENT_BRANCH, + [POP_JUMP_IF_FALSE] = PY_MONITORING_EVENT_BRANCH, + [POP_JUMP_IF_TRUE] = PY_MONITORING_EVENT_BRANCH, + [POP_JUMP_IF_NONE] = PY_MONITORING_EVENT_BRANCH, + [POP_JUMP_IF_NOT_NONE] = PY_MONITORING_EVENT_BRANCH, + [COMPARE_AND_BRANCH] = PY_MONITORING_EVENT_BRANCH, +}; + +static const bool OPCODE_HAS_EVENT[256] = { + [RETURN_VALUE] = true, + [INSTRUMENTED_RETURN_VALUE] = true, + [CALL] = true, + [INSTRUMENTED_CALL] = true, + [CALL_FUNCTION_EX] = true, + [INSTRUMENTED_CALL_FUNCTION_EX] = true, + [RESUME] = true, + [INSTRUMENTED_RESUME] = true, + [YIELD_VALUE] = true, + [INSTRUMENTED_YIELD_VALUE] = true, + [JUMP_FORWARD] = true, + [JUMP_BACKWARD] = true, + [JUMP_IF_FALSE_OR_POP] = true, + [JUMP_IF_TRUE_OR_POP] = true, + [POP_JUMP_IF_FALSE] = true, + [POP_JUMP_IF_TRUE] = true, + [POP_JUMP_IF_NONE] = true, + [POP_JUMP_IF_NOT_NONE] = true, + [COMPARE_AND_BRANCH] = true, +}; + +bool valid_opcode(int opcode) { + if (opcode > 0 && opcode < 255 && + _PyOpcode_OpName[opcode] && + _PyOpcode_OpName[opcode][0] != '<' + ) { + return true; } + return false; } -_PyInstrumentationOffsets -get_offsets(PyCodeObject *code) { - return code->_co_instrumentation.layout.offsets; +#define CHECK(test) do { \ + if (!(test)) { \ + dump_instrumentation_data(code, i, stderr); \ + } \ + assert(test); \ +} while (0) + +static void +sanity_check_instrumentation(PyCodeObject *code) +{ + _PyCoInstrumentationData *data = code->_co_instrumentation.monitoring_data; + if (data == NULL) { + return; + } + int code_len = (int)Py_SIZE(code); + for (int i = 0; i < code_len; i++) { + _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; + int opcode = instr->opcode; + /* TO DO -- Check INSTRUMENTED_OPCODE */ + if (opcode == INSTRUMENTED_LINE) { + assert(data->lines); + assert(valid_opcode(data->lines[i].original_opcode)); + opcode = data->lines[i].original_opcode; + if (!is_instrumented(opcode)) { + CHECK(_PyOpcode_Deopt[opcode] == opcode); + } + CHECK(opcode != INSTRUMENTED_LINE); + } + else if (data->lines) { + CHECK(data->lines[i].original_opcode == 0 || + data->lines[i].original_opcode == 255); + } + if (is_instrumented(opcode)) { + int deinstrumented = DE_INSTRUMENT[opcode]; + CHECK(valid_opcode(deinstrumented)); + CHECK(_PyOpcode_Deopt[deinstrumented] == deinstrumented); + if (_PyOpcode_Caches[deinstrumented]) { + CHECK(instr[1].cache > 0); + } + opcode = deinstrumented; + } + if (data->lines && opcode != END_FOR) { + int line1 = compute_line(code, i, data->lines[i].line_delta); + int line2 = PyCode_Addr2Line(code, i*sizeof(_Py_CODEUNIT)); + CHECK(line1 == line2); + } + CHECK(valid_opcode(opcode)); + opcode = _PyOpcode_Deopt[opcode]; + CHECK(valid_opcode(opcode)); + if (data->tools && OPCODE_HAS_EVENT[opcode]) { + int event = EVENT_FOR_OPCODE[opcode]; + uint8_t local_tools = data->tools[i]; + uint8_t global_tools = code->_co_instrumentation.monitoring_data->matrix.tools[event]; + CHECK((global_tools & local_tools) == local_tools); + } + i += _PyOpcode_Caches[opcode]; + } } static inline uint8_t @@ -162,18 +323,13 @@ get_tools(PyCodeObject * code, int index, int event) tools = monitoring->tools[index]; } else { - tools = code->_co_instrumentation.monitoring_matrix.tools[event]; + tools = code->_co_instrumentation.monitoring_data->matrix.tools[event]; } assert((tools & PyInterpreterState_Get()->monitoring_matrix.tools[event]) == tools); - assert((tools & code->_co_instrumentation.monitoring_matrix.tools[event]) == tools); + assert((tools & code->_co_instrumentation.monitoring_data->matrix.tools[event]) == tools); return tools; } -static inline bool -is_instrumented(int opcode) { - return INSTRUMENTED_OPCODES[opcode] == opcode; -} - int _Py_GetBaseOpcode(PyCodeObject *code, int offset) { int opcode = _Py_OPCODE(_PyCode_CODE(code)[offset]); @@ -295,7 +451,7 @@ instrument_line(PyCodeObject *code, int offset) } _PyCoLineInstrumentationData *lines = &code->_co_instrumentation.monitoring_data->lines[offset]; if (lines->original_opcode != 255) { - dump_instrumentation_data(code, code->_co_instrumentation.monitoring_data, offset, stderr); + dump_instrumentation_data(code, offset, stderr); } assert(lines->original_opcode == 255); if (is_instrumented(opcode)) { @@ -327,7 +483,7 @@ remove_tools(PyCodeObject * code, int offset, int event, int tools) } else { /* Single tool */ - uint8_t single_tool = code->_co_instrumentation.monitoring_matrix.tools[event]; + uint8_t single_tool = code->_co_instrumentation.monitoring_data->matrix.tools[event]; assert(_Py_popcount32(single_tool) <= 1); if (((single_tool & tools) == single_tool) && event < PY_MONITORING_INSTRUMENTED_EVENTS) { de_instrument(code, offset, event); @@ -338,9 +494,8 @@ remove_tools(PyCodeObject * code, int offset, int event, int tools) static void remove_line_tools(PyCodeObject * code, int offset, int tools) { - assert (code->_co_instrumentation.layout.offsets.lines >= 0); - if (code->_co_instrumentation.monitoring_data && - code->_co_instrumentation.monitoring_data->line_tools) + assert(code->_co_instrumentation.monitoring_data); + if (code->_co_instrumentation.monitoring_data->line_tools) { uint8_t *toolsptr = &code->_co_instrumentation.monitoring_data->line_tools[offset]; *toolsptr &= ~tools; @@ -350,7 +505,7 @@ remove_line_tools(PyCodeObject * code, int offset, int tools) } else { /* Single tool */ - uint8_t single_tool = code->_co_instrumentation.monitoring_matrix.tools[PY_MONITORING_EVENT_LINE]; + uint8_t single_tool = code->_co_instrumentation.monitoring_data->matrix.tools[PY_MONITORING_EVENT_LINE]; assert(_Py_popcount32(single_tool) <= 1); if (((single_tool & tools) == single_tool)) { de_instrument_line(code, offset); @@ -364,6 +519,7 @@ add_tools(PyCodeObject * code, int offset, int event, int tools) { assert(event != PY_MONITORING_EVENT_LINE); assert(event != PY_MONITORING_EVENT_INSTRUCTION); + assert(code->_co_instrumentation.monitoring_data); if (code->_co_instrumentation.monitoring_data && code->_co_instrumentation.monitoring_data->tools ) { @@ -382,10 +538,9 @@ add_tools(PyCodeObject * code, int offset, int event, int tools) static void add_line_tools(PyCodeObject * code, int offset, int tools) { - assert (get_offsets(code).lines >= 0); assert((PyInterpreterState_Get()->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE] & tools) == tools); - if (code->_co_instrumentation.monitoring_data && - code->_co_instrumentation.monitoring_data->line_tools + assert(code->_co_instrumentation.monitoring_data); + if (code->_co_instrumentation.monitoring_data->line_tools ) { code->_co_instrumentation.monitoring_data->line_tools[offset] |= tools; } @@ -448,7 +603,7 @@ call_instrument(PyThreadState *tstate, PyCodeObject *code, int event, int offset = instr - _PyCode_CODE(code); uint8_t tools = get_tools(code, offset, event); /* No per-instruction monitoring yet */ - assert(!code->_co_instrumentation.layout.offsets.instructions); + assert(code->_co_instrumentation.monitoring_data->per_instruction_opcodes == NULL); // assert(tools); while (tools) { int tool = most_significant_bit(tools); @@ -495,7 +650,7 @@ _Py_call_instrumentation( { PyCodeObject *code = frame->f_code; assert(is_instrumentation_up_to_date(code, tstate->interp)); - assert(matrix_equals(code->_co_instrumentation.monitoring_matrix, tstate->interp->monitoring_matrix)); + assert(matrix_equals(code->_co_instrumentation.monitoring_data->matrix, tstate->interp->monitoring_matrix)); int instruction_offset = instr - _PyCode_CODE(code); PyObject *instruction_offset_obj = PyLong_FromSsize_t(instruction_offset); PyObject *args[3] = { NULL, (PyObject *)code, instruction_offset_obj }; @@ -511,7 +666,7 @@ _Py_call_instrumentation_arg( { PyCodeObject *code = frame->f_code; assert(is_instrumentation_up_to_date(code, tstate->interp)); - assert(matrix_equals(code->_co_instrumentation.monitoring_matrix, tstate->interp->monitoring_matrix)); + assert(matrix_equals(code->_co_instrumentation.monitoring_data->matrix, tstate->interp->monitoring_matrix)); int instruction_offset = instr - _PyCode_CODE(code); PyObject *instruction_offset_obj = PyLong_FromSsize_t(instruction_offset); PyObject *args[4] = { NULL, (PyObject *)code, instruction_offset_obj, arg }; @@ -554,7 +709,7 @@ _Py_call_instrumentation_exc( _PyErr_Fetch(tstate, &type, &value, &traceback); PyCodeObject *code = frame->f_code; assert(code->_co_instrumentation.monitoring_version == tstate->interp->monitoring_version); - assert(matrix_equals(code->_co_instrumentation.monitoring_matrix, tstate->interp->monitoring_matrix)); + assert(matrix_equals(code->_co_instrumentation.monitoring_data->matrix, tstate->interp->monitoring_matrix)); int instruction_offset = instr - _PyCode_CODE(code); PyObject *instruction_offset_obj = PyLong_FromSsize_t(instruction_offset); PyObject *args[4] = { NULL, (PyObject *)code, instruction_offset_obj, arg }; @@ -572,49 +727,6 @@ _Py_call_instrumentation_exc( Py_DECREF(instruction_offset_obj); } -/* Line delta. - * 8 bit value. - * if line_delta == -128: - * line = None # represented as -1 - * elif line == -127: - * line = PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT)); - * else: - * line = first_line + (offset >> 3) + line_delta; - */ - -static int8_t -compute_line_delta(PyCodeObject *code, int offset, int line) -{ - if (line < 0) { - return -128; - } - assert(line >= code->co_firstlineno); - assert(offset >= code->_co_firsttraceable); - assert(offset < Py_SIZE(code)); - int delta = line - code->co_firstlineno - (offset >> 3); - if (delta < 128 && delta > -128) { - return delta; - } - return -127; -} - -static int -compute_line(PyCodeObject *code, int offset, int8_t line_delta) -{ - if (line_delta > -127) { - assert((offset >> 3) + line_delta >= 0); - return code->co_firstlineno + (offset >> 3) + line_delta; - } - if (line_delta == -128) { - return -1; - } - else { - assert(line_delta == -127); - /* Compute from table */ - return PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT)); - } -} - int _Py_Instrumentation_GetLine(PyCodeObject *code, int index) { @@ -633,7 +745,7 @@ int _Py_call_instrumentation_line(PyThreadState *tstate, PyCodeObject *code, _Py_CODEUNIT *instr) { assert(is_instrumentation_up_to_date(code, tstate->interp)); - assert(matrix_equals(code->_co_instrumentation.monitoring_matrix, tstate->interp->monitoring_matrix)); + assert(matrix_equals(code->_co_instrumentation.monitoring_data->matrix, tstate->interp->monitoring_matrix)); int offset = instr - _PyCode_CODE(code); _PyCoInstrumentation *instrumentation = &code->_co_instrumentation; _PyCoLineInstrumentationData *line_data = &instrumentation->monitoring_data->lines[offset]; @@ -690,49 +802,6 @@ _PyMonitoring_RegisterCallback(int tool_id, int event_id, PyObject *obj) return callback; } -static const int8_t EVENT_FOR_OPCODE[256] = { - [RETURN_VALUE] = PY_MONITORING_EVENT_PY_RETURN, - [INSTRUMENTED_RETURN_VALUE] = PY_MONITORING_EVENT_PY_RETURN, - [CALL] = PY_MONITORING_EVENT_CALL, - [INSTRUMENTED_CALL] = PY_MONITORING_EVENT_CALL, - [CALL_FUNCTION_EX] = PY_MONITORING_EVENT_CALL, - [INSTRUMENTED_CALL_FUNCTION_EX] = PY_MONITORING_EVENT_CALL, - [RESUME] = -1, - [YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD, - [INSTRUMENTED_YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD, - [JUMP_FORWARD] = PY_MONITORING_EVENT_JUMP, - [JUMP_BACKWARD] = PY_MONITORING_EVENT_JUMP, - [JUMP_IF_FALSE_OR_POP] = PY_MONITORING_EVENT_BRANCH, - [JUMP_IF_TRUE_OR_POP] = PY_MONITORING_EVENT_BRANCH, - [POP_JUMP_IF_FALSE] = PY_MONITORING_EVENT_BRANCH, - [POP_JUMP_IF_TRUE] = PY_MONITORING_EVENT_BRANCH, - [POP_JUMP_IF_NONE] = PY_MONITORING_EVENT_BRANCH, - [POP_JUMP_IF_NOT_NONE] = PY_MONITORING_EVENT_BRANCH, - [COMPARE_AND_BRANCH] = PY_MONITORING_EVENT_BRANCH, -}; - -static const bool OPCODE_HAS_EVENT[256] = { - [RETURN_VALUE] = true, - [INSTRUMENTED_RETURN_VALUE] = true, - [CALL] = true, - [INSTRUMENTED_CALL] = true, - [CALL_FUNCTION_EX] = true, - [INSTRUMENTED_CALL_FUNCTION_EX] = true, - [RESUME] = true, - [INSTRUMENTED_RESUME] = true, - [YIELD_VALUE] = true, - [INSTRUMENTED_YIELD_VALUE] = true, - [JUMP_FORWARD] = true, - [JUMP_BACKWARD] = true, - [JUMP_IF_FALSE_OR_POP] = true, - [JUMP_IF_TRUE_OR_POP] = true, - [POP_JUMP_IF_FALSE] = true, - [POP_JUMP_IF_TRUE] = true, - [POP_JUMP_IF_NONE] = true, - [POP_JUMP_IF_NOT_NONE] = true, - [COMPARE_AND_BRANCH] = true, -}; - static inline _Py_MonitoringMatrix matrix_sub(_Py_MonitoringMatrix a, _Py_MonitoringMatrix b) { @@ -764,6 +833,17 @@ matrix_empty(_Py_MonitoringMatrix m) return true; } +static inline int +multiple_tools(_Py_MonitoringMatrix m) +{ + for (int i = 0; i < PY_MONITORING_EVENTS; i++) { + if (_Py_popcount32(m.tools[i]) > 1) { + return true; + } + } + return false; +} + static inline void matrix_set_bit(_Py_MonitoringMatrix *m, int event, int tool, int val) { @@ -786,15 +866,6 @@ matrix_get_events(_Py_MonitoringMatrix *m, int tool_id) return result; } -static inline _PyInstrumentationLayout -layout_from_flags(bool multitools, bool lines, bool instrs) { - _PyInstrumentationLayout result; - result.offsets.multi_tools = multitools; - result.offsets.lines = lines; - result.offsets.instructions = instrs; - return result; -} - static inline int tools_data_offset() { return 0; @@ -830,7 +901,7 @@ initialize_tools(PyCodeObject *code) assert(event > 0); } assert(event >= 0); - tools[i] = code->_co_instrumentation.monitoring_matrix.tools[event]; + tools[i] = code->_co_instrumentation.monitoring_data->matrix.tools[event]; assert(tools[i] != 0); } else { @@ -843,18 +914,13 @@ initialize_tools(PyCodeObject *code) static bool is_new_line(int index, int current_line, PyCodeAddressRange *bounds, int opcode) { + int line = _PyCode_CheckLineNumber(index*(int)sizeof(_Py_CODEUNIT), bounds); /* END_FOR cannot start a line, as it is skipped by FOR_ITER */ if (opcode == END_FOR) { + bounds->ar_line = -1; return false; } - while (bounds->ar_end <= index*(int)sizeof(_Py_CODEUNIT)) { - if (!_PyLineTable_NextAddressRange(bounds)) { - return false; - } - } - assert(index*(int)sizeof(_Py_CODEUNIT) >= bounds->ar_start && - index*(int)sizeof(_Py_CODEUNIT) < bounds->ar_end); - return bounds->ar_line >= 0 && bounds->ar_line != current_line; + return line >= 0 && line != current_line; } static void @@ -865,12 +931,12 @@ initialize_lines(PyCodeObject *code) int code_len = (int)Py_SIZE(code); PyCodeAddressRange range; _PyCode_InitAddressRange(code, &range); - for (int i = 0; i <= code->_co_firsttraceable && i < code_len; i++) { + for (int i = 0; i < code->_co_firsttraceable && i < code_len; i++) { line_data[i].original_opcode = 0; line_data[i].line_delta = -128; } int current_line = -1; - for (int i = code->_co_firsttraceable + 1; i < code_len; i++) { + for (int i = code->_co_firsttraceable; i < code_len; i++) { int opcode = _Py_GetBaseOpcode(code, i); if (is_new_line(i, current_line, &range, opcode)) { line_data[i].original_opcode = 255; @@ -880,7 +946,7 @@ initialize_lines(PyCodeObject *code) /* Mark as not being an instrumentation point */ line_data[i].original_opcode = 0; } - assert(range.ar_line < 0 || range.ar_line >= code->co_firstlineno); + // assert(range.ar_line < 0 || range.ar_line >= code->co_firstlineno); line_data[i].line_delta = compute_line_delta(code, i, range.ar_line); for (int j = 0; j < _PyOpcode_Caches[opcode]; j++) { i++; @@ -904,7 +970,6 @@ initialize_line_tools(PyCodeObject *code, PyInterpreterState *interp) int update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) { - _PyInstrumentationLayout new_layout = interp->instrumentation_layout; bool restarted = interp->last_restart_version > code->_co_instrumentation.monitoring_version; int code_len = (int)Py_SIZE(code); if (code->_co_instrumentation.monitoring_data == NULL) { @@ -913,13 +978,15 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) PyErr_NoMemory(); return -1; } + code->_co_instrumentation.monitoring_data->matrix = interp->monitoring_matrix; code->_co_instrumentation.monitoring_data->tools = NULL; code->_co_instrumentation.monitoring_data->lines = NULL; code->_co_instrumentation.monitoring_data->line_tools = NULL; code->_co_instrumentation.monitoring_data->per_instruction_opcodes = NULL; code->_co_instrumentation.monitoring_data->per_instruction_tools = NULL; } - if (new_layout.offsets.multi_tools && code->_co_instrumentation.monitoring_data->tools == NULL) { + bool multitools = multiple_tools(interp->monitoring_matrix); + if (code->_co_instrumentation.monitoring_data->tools == NULL && multitools) { code->_co_instrumentation.monitoring_data->tools = PyMem_Malloc(code_len); if (code->_co_instrumentation.monitoring_data->tools == NULL) { PyErr_NoMemory(); @@ -931,7 +998,7 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) initialize_tools(code); } } - if (new_layout.offsets.lines) { + if (interp->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE]) { if (code->_co_instrumentation.monitoring_data->lines == NULL) { code->_co_instrumentation.monitoring_data->lines = PyMem_Malloc(code_len * sizeof(_PyCoLineInstrumentationData)); if (code->_co_instrumentation.monitoring_data->lines == NULL) { @@ -940,7 +1007,7 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) } initialize_lines(code); } - if (new_layout.offsets.multi_tools && code->_co_instrumentation.monitoring_data->line_tools == NULL) { + if (multitools && code->_co_instrumentation.monitoring_data->line_tools == NULL) { code->_co_instrumentation.monitoring_data->line_tools = PyMem_Malloc(code_len); if (code->_co_instrumentation.monitoring_data->line_tools == NULL) { PyErr_NoMemory(); @@ -949,8 +1016,6 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) initialize_line_tools(code, interp); } } - assert(new_layout.offsets.instructions == 0); - code->_co_instrumentation.layout.bits = new_layout.bits; return 0; } @@ -973,35 +1038,33 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) { if (is_instrumentation_up_to_date(code, interp)) { - assert(matrix_equals(code->_co_instrumentation.monitoring_matrix, interp->monitoring_matrix)); + assert(interp->monitoring_version == 0 || + matrix_equals(code->_co_instrumentation.monitoring_data->matrix, interp->monitoring_matrix) + ); return 0; } bool restarted = interp->last_restart_version > code->_co_instrumentation.monitoring_version; assert(interp->monitoring_matrix.tools[PY_MONITORING_EVENT_INSTRUCTION] == 0); int code_len = (int)Py_SIZE(code); - _PyInstrumentationLayout new_layout = interp->instrumentation_layout; - _PyInstrumentationLayout old_layout = code->_co_instrumentation.layout; - if (new_layout.bits != old_layout.bits) { - if (update_instrumentation_data(code, interp)) { - return -1; - } + if (update_instrumentation_data(code, interp)) { + return -1; } _Py_MonitoringMatrix new_events; _Py_MonitoringMatrix removed_events; if (restarted) { - removed_events = code->_co_instrumentation.monitoring_matrix; + removed_events = code->_co_instrumentation.monitoring_data->matrix; new_events = interp->monitoring_matrix; } else { - removed_events = matrix_sub(code->_co_instrumentation.monitoring_matrix, interp->monitoring_matrix); - new_events = matrix_sub(interp->monitoring_matrix, code->_co_instrumentation.monitoring_matrix); + removed_events = matrix_sub(code->_co_instrumentation.monitoring_data->matrix, interp->monitoring_matrix); + new_events = matrix_sub(interp->monitoring_matrix, code->_co_instrumentation.monitoring_data->matrix); assert(matrix_empty(matrix_and(new_events, removed_events))); } - code->_co_instrumentation.monitoring_matrix = interp->monitoring_matrix; - assert(code->_co_instrumentation.layout.bits == interp->instrumentation_layout.bits); + code->_co_instrumentation.monitoring_data->matrix = interp->monitoring_matrix; code->_co_instrumentation.monitoring_version = interp->monitoring_version; if (matrix_empty(new_events) && matrix_empty(removed_events)) { + sanity_check_instrumentation(code); return 0; } /* Insert instrumentation */ @@ -1069,6 +1132,7 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) } } } + sanity_check_instrumentation(code); return 0; } @@ -1104,20 +1168,12 @@ _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) if (existing_events == events) { return; } - bool multitools = false; for (int e = 0; e < PY_MONITORING_EVENTS; e++) { int val = (events >> e) & 1; matrix_set_bit(&interp->monitoring_matrix, e, tool_id, val); - uint8_t tools = interp->monitoring_matrix.tools[e]; - if (_Py_popcount32(tools) > 1) { - multitools = true; - } } interp->monitoring_version++; - bool line_monitoring = interp->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE] != 0; assert(interp->monitoring_matrix.tools[PY_MONITORING_EVENT_INSTRUCTION] == 0); - bool instruction_monitoring = false; - interp->instrumentation_layout = layout_from_flags(multitools, line_monitoring, instruction_monitoring); instrument_all_executing_code_objects(interp); } diff --git a/Python/pystate.c b/Python/pystate.c index f76ef37c70c3eb..8e39759b8ae9a9 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -315,7 +315,6 @@ init_interpreter(PyInterpreterState *interp, interp->tools[t].instrument_callables[i] = NULL; } } - interp->instrumentation_layout.offsets = (_PyInstrumentationOffsets){ 0, 0, 0 }; interp->f_opcode_trace_set = false; interp->_initialized = 1; } From 6c8be7e0217af4576859b1c435661ad3170147c8 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 18 Jan 2023 21:38:39 +0000 Subject: [PATCH 014/116] Add more tests. Get those and some other passing. --- Lib/test/test_monitoring.py | 59 ++++- Objects/codeobject.c | 2 +- Python/instrumentation.c | 448 +++++++++++++++++++++--------------- Python/legacy_tracing.c | 6 + 4 files changed, 324 insertions(+), 191 deletions(-) diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index bab69d55446dcc..fabf3ab8ae6d05 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -8,6 +8,8 @@ import types import collections +PAIR = (0,1) + def f1(): pass @@ -15,6 +17,10 @@ def f2(): len([]) sys.getsizeof(0) +def floop(): + for item in PAIR: + pass + def gen(): yield yield @@ -162,6 +168,7 @@ def caught(): "start", "raise", "exception_handled", + "branch", "return", ] @@ -311,7 +318,6 @@ def __init__(self, events): def __call__(self, code, event): self.events.append(event) - print(len(self.events)) if self.disable: return sys.monitoring.DISABLE @@ -481,16 +487,61 @@ def test_lines_single(self): try: self.assertEqual(sys.monitoring._all_events(), {}) events = [] - counter = RecorderWithDisable(events) - sys.monitoring.register_callback(TEST_TOOL, E.LINE, counter) + recorder = RecorderWithDisable(events) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, recorder) sys.monitoring.set_events(TEST_TOOL, E.LINE) f1() sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) - self.assertEqual(events, [487, 12, 488]) + self.assertEqual(events, [493, 14, 494]) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) + self.assertEqual(sys.monitoring._all_events(), {}) + sys.monitoring.restart_events() + + def test_lines_loop(self): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + events = [] + recorder = RecorderWithDisable(events) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, recorder) + sys.monitoring.set_events(TEST_TOOL, E.LINE) + floop() + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) + self.assertEqual(events, [510, 21, 22, 22, 21, 511]) finally: sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) self.assertEqual(sys.monitoring._all_events(), {}) sys.monitoring.restart_events() + def test_lines_two(self): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + events = [] + recorder = RecorderWithDisable(events) + events2 = [] + recorder2 = RecorderWithDisable(events2) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, recorder) + sys.monitoring.register_callback(TEST_TOOL2, E.LINE, recorder2) + sys.monitoring.set_events(TEST_TOOL, E.LINE); sys.monitoring.set_events(TEST_TOOL2, E.LINE) + f1() + sys.monitoring.set_events(TEST_TOOL, 0); sys.monitoring.set_events(TEST_TOOL2, 0) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) + sys.monitoring.register_callback(TEST_TOOL2, E.LINE, None) + expected = [530, 14, 531] + self.assertEqual(events, expected) + self.assertEqual(events2, expected) + finally: + sys.monitoring.set_events(TEST_TOOL2, 0) + sys.monitoring.set_events(TEST_TOOL2, 0) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) + sys.monitoring.register_callback(TEST_TOOL2, E.LINE, None) + self.assertEqual(sys.monitoring._all_events(), {}) + sys.monitoring.restart_events() + + + + diff --git a/Objects/codeobject.c b/Objects/codeobject.c index e96b0ca5d48c88..535a521e3170b0 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1849,7 +1849,7 @@ code_hash(PyCodeObject *co) SCRAMBLE_IN(co->co_firstlineno); SCRAMBLE_IN(Py_SIZE(co)); for (int i = 0; i < Py_SIZE(co); i++) { - int deop = _PyOpcode_Deopt[_Py_OPCODE(_PyCode_CODE(co)[i])]; + int deop = _Py_GetBaseOpcode(co, i); SCRAMBLE_IN(deop); SCRAMBLE_IN(_Py_OPARG(_PyCode_CODE(co)[i])); i += _PyOpcode_Caches[deop]; diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 9934e03aeed6c5..797df8ebc315cf 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -14,6 +14,67 @@ static PyObject DISABLE = &PyBaseObject_Type }; +static const int8_t EVENT_FOR_OPCODE[256] = { + [RETURN_VALUE] = PY_MONITORING_EVENT_PY_RETURN, + [INSTRUMENTED_RETURN_VALUE] = PY_MONITORING_EVENT_PY_RETURN, + [CALL] = PY_MONITORING_EVENT_CALL, + [INSTRUMENTED_CALL] = PY_MONITORING_EVENT_CALL, + [CALL_FUNCTION_EX] = PY_MONITORING_EVENT_CALL, + [INSTRUMENTED_CALL_FUNCTION_EX] = PY_MONITORING_EVENT_CALL, + [RESUME] = -1, + [YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD, + [INSTRUMENTED_YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD, + [JUMP_FORWARD] = PY_MONITORING_EVENT_JUMP, + [JUMP_BACKWARD] = PY_MONITORING_EVENT_JUMP, + [JUMP_IF_FALSE_OR_POP] = PY_MONITORING_EVENT_BRANCH, + [JUMP_IF_TRUE_OR_POP] = PY_MONITORING_EVENT_BRANCH, + [POP_JUMP_IF_FALSE] = PY_MONITORING_EVENT_BRANCH, + [POP_JUMP_IF_TRUE] = PY_MONITORING_EVENT_BRANCH, + [POP_JUMP_IF_NONE] = PY_MONITORING_EVENT_BRANCH, + [POP_JUMP_IF_NOT_NONE] = PY_MONITORING_EVENT_BRANCH, + [COMPARE_AND_BRANCH] = PY_MONITORING_EVENT_BRANCH, + [INSTRUMENTED_JUMP_FORWARD] = PY_MONITORING_EVENT_JUMP, + [INSTRUMENTED_JUMP_BACKWARD] = PY_MONITORING_EVENT_JUMP, + [INSTRUMENTED_JUMP_IF_FALSE_OR_POP] = PY_MONITORING_EVENT_BRANCH, + [INSTRUMENTED_JUMP_IF_TRUE_OR_POP] = PY_MONITORING_EVENT_BRANCH, + [INSTRUMENTED_POP_JUMP_IF_FALSE] = PY_MONITORING_EVENT_BRANCH, + [INSTRUMENTED_POP_JUMP_IF_TRUE] = PY_MONITORING_EVENT_BRANCH, + [INSTRUMENTED_POP_JUMP_IF_NONE] = PY_MONITORING_EVENT_BRANCH, + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = PY_MONITORING_EVENT_BRANCH, + [INSTRUMENTED_COMPARE_AND_BRANCH] = PY_MONITORING_EVENT_BRANCH, +}; + +static const bool OPCODE_HAS_EVENT[256] = { + [RETURN_VALUE] = true, + [INSTRUMENTED_RETURN_VALUE] = true, + [CALL] = true, + [INSTRUMENTED_CALL] = true, + [CALL_FUNCTION_EX] = true, + [INSTRUMENTED_CALL_FUNCTION_EX] = true, + [RESUME] = true, + [INSTRUMENTED_RESUME] = true, + [YIELD_VALUE] = true, + [INSTRUMENTED_YIELD_VALUE] = true, + [JUMP_FORWARD] = true, + [JUMP_BACKWARD] = true, + [JUMP_IF_FALSE_OR_POP] = true, + [JUMP_IF_TRUE_OR_POP] = true, + [POP_JUMP_IF_FALSE] = true, + [POP_JUMP_IF_TRUE] = true, + [POP_JUMP_IF_NONE] = true, + [POP_JUMP_IF_NOT_NONE] = true, + [COMPARE_AND_BRANCH] = true, + [INSTRUMENTED_JUMP_FORWARD] = true, + [INSTRUMENTED_JUMP_BACKWARD] = true, + [INSTRUMENTED_JUMP_IF_FALSE_OR_POP] = true, + [INSTRUMENTED_JUMP_IF_TRUE_OR_POP] = true, + [INSTRUMENTED_POP_JUMP_IF_FALSE] = true, + [INSTRUMENTED_POP_JUMP_IF_TRUE] = true, + [INSTRUMENTED_POP_JUMP_IF_NONE] = true, + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = true, + [INSTRUMENTED_COMPARE_AND_BRANCH] = true, +}; + static const uint8_t DE_INSTRUMENT[256] = { [INSTRUMENTED_RESUME] = RESUME, [INSTRUMENTED_RETURN_VALUE] = RETURN_VALUE, @@ -21,7 +82,7 @@ static const uint8_t DE_INSTRUMENT[256] = { [INSTRUMENTED_CALL_FUNCTION_EX] = CALL_FUNCTION_EX, [INSTRUMENTED_YIELD_VALUE] = YIELD_VALUE, [INSTRUMENTED_JUMP_FORWARD] = JUMP_FORWARD, - [INSTRUMENTED_JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD, + [INSTRUMENTED_JUMP_BACKWARD] = JUMP_BACKWARD, [INSTRUMENTED_JUMP_IF_FALSE_OR_POP] = JUMP_IF_FALSE_OR_POP, [INSTRUMENTED_JUMP_IF_TRUE_OR_POP] = JUMP_IF_TRUE_OR_POP, [INSTRUMENTED_POP_JUMP_IF_FALSE] = POP_JUMP_IF_FALSE, @@ -103,6 +164,14 @@ dump_instrumentation_data_line_tools(PyCodeObject *code, uint8_t *line_tools, in } } +static void +dump_matrix(const char *prefix, _Py_MonitoringMatrix matrix, FILE*out) +{ + fprintf(out, "%s matrix:\n", prefix); + for (int event = 0; event < PY_MONITORING_EVENTS; event++) { + fprintf(out, " Event %d: Tools %x\n", event, matrix.tools[event]); + } +} /* No error checking -- Don't use this for anything but experimental debugging */ static void @@ -114,6 +183,8 @@ dump_instrumentation_data(PyCodeObject *code, int star, FILE*out) fprintf(out, "NULL\n"); return; } + dump_matrix("Global", PyInterpreterState_Get()->monitoring_matrix, out); + dump_matrix("Code", data->matrix, out); int code_len = (int)Py_SIZE(code); bool starred = false; for (int i = 0; i < code_len; i++) { @@ -149,7 +220,7 @@ dump_instrumentation_data(PyCodeObject *code, int star, FILE*out) } } -#define OFFSET_SHIFT 5 +#define OFFSET_SHIFT 4 /* Line delta. * 8 bit value. @@ -200,49 +271,6 @@ is_instrumented(int opcode) { return INSTRUMENTED_OPCODES[opcode] == opcode; } -static const int8_t EVENT_FOR_OPCODE[256] = { - [RETURN_VALUE] = PY_MONITORING_EVENT_PY_RETURN, - [INSTRUMENTED_RETURN_VALUE] = PY_MONITORING_EVENT_PY_RETURN, - [CALL] = PY_MONITORING_EVENT_CALL, - [INSTRUMENTED_CALL] = PY_MONITORING_EVENT_CALL, - [CALL_FUNCTION_EX] = PY_MONITORING_EVENT_CALL, - [INSTRUMENTED_CALL_FUNCTION_EX] = PY_MONITORING_EVENT_CALL, - [RESUME] = -1, - [YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD, - [INSTRUMENTED_YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD, - [JUMP_FORWARD] = PY_MONITORING_EVENT_JUMP, - [JUMP_BACKWARD] = PY_MONITORING_EVENT_JUMP, - [JUMP_IF_FALSE_OR_POP] = PY_MONITORING_EVENT_BRANCH, - [JUMP_IF_TRUE_OR_POP] = PY_MONITORING_EVENT_BRANCH, - [POP_JUMP_IF_FALSE] = PY_MONITORING_EVENT_BRANCH, - [POP_JUMP_IF_TRUE] = PY_MONITORING_EVENT_BRANCH, - [POP_JUMP_IF_NONE] = PY_MONITORING_EVENT_BRANCH, - [POP_JUMP_IF_NOT_NONE] = PY_MONITORING_EVENT_BRANCH, - [COMPARE_AND_BRANCH] = PY_MONITORING_EVENT_BRANCH, -}; - -static const bool OPCODE_HAS_EVENT[256] = { - [RETURN_VALUE] = true, - [INSTRUMENTED_RETURN_VALUE] = true, - [CALL] = true, - [INSTRUMENTED_CALL] = true, - [CALL_FUNCTION_EX] = true, - [INSTRUMENTED_CALL_FUNCTION_EX] = true, - [RESUME] = true, - [INSTRUMENTED_RESUME] = true, - [YIELD_VALUE] = true, - [INSTRUMENTED_YIELD_VALUE] = true, - [JUMP_FORWARD] = true, - [JUMP_BACKWARD] = true, - [JUMP_IF_FALSE_OR_POP] = true, - [JUMP_IF_TRUE_OR_POP] = true, - [POP_JUMP_IF_FALSE] = true, - [POP_JUMP_IF_TRUE] = true, - [POP_JUMP_IF_NONE] = true, - [POP_JUMP_IF_NOT_NONE] = true, - [COMPARE_AND_BRANCH] = true, -}; - bool valid_opcode(int opcode) { if (opcode > 0 && opcode < 255 && _PyOpcode_OpName[opcode] && @@ -253,6 +281,81 @@ bool valid_opcode(int opcode) { return false; } +static inline bool +matrix_equals(_Py_MonitoringMatrix a, _Py_MonitoringMatrix b) +{ + for (int i = 0; i < PY_MONITORING_EVENTS; i++) { + if (a.tools[i] != b.tools[i]) { + return false; + } + } + return true; +} + +static inline _Py_MonitoringMatrix +matrix_sub(_Py_MonitoringMatrix a, _Py_MonitoringMatrix b) +{ + _Py_MonitoringMatrix res; + for (int i = 0; i < PY_MONITORING_EVENTS; i++) { + res.tools[i] = a.tools[i] & ~b.tools[i]; + } + return res; +} + +static inline _Py_MonitoringMatrix +matrix_and(_Py_MonitoringMatrix a, _Py_MonitoringMatrix b) +{ + _Py_MonitoringMatrix res; + for (int i = 0; i < PY_MONITORING_EVENTS; i++) { + res.tools[i] = a.tools[i] & b.tools[i]; + } + return res; +} + +static inline bool +matrix_empty(_Py_MonitoringMatrix m) +{ + for (int i = 0; i < PY_MONITORING_EVENTS; i++) { + if (m.tools[i]) { + return false; + } + } + return true; +} + +static inline int +multiple_tools(_Py_MonitoringMatrix m) +{ + for (int i = 0; i < PY_MONITORING_EVENTS; i++) { + if (_Py_popcount32(m.tools[i]) > 1) { + return true; + } + } + return false; +} + +static inline void +matrix_set_bit(_Py_MonitoringMatrix *m, int event, int tool, int val) +{ + assert(0 <= event && event < PY_MONITORING_EVENTS); + assert(0 <= tool && tool < PY_MONITORING_TOOL_IDS); + assert(val == 0 || val == 1); + m->tools[event] &= ~(1 << tool); + m->tools[event] |= (val << tool); +} + +static inline _PyMonitoringEventSet +matrix_get_events(_Py_MonitoringMatrix *m, int tool_id) +{ + _PyMonitoringEventSet result = 0; + for (int e = 0; e < PY_MONITORING_EVENTS; e++) { + if ((m->tools[e] >> tool_id) & 1) { + result |= (1 << e); + } + } + return result; +} + #define CHECK(test) do { \ if (!(test)) { \ dump_instrumentation_data(code, i, stderr); \ @@ -267,15 +370,23 @@ sanity_check_instrumentation(PyCodeObject *code) if (data == NULL) { return; } + _Py_MonitoringMatrix tools_matrix = PyInterpreterState_Get()->monitoring_matrix; + assert(matrix_equals( + code->_co_instrumentation.monitoring_data->matrix, + tools_matrix) + ); int code_len = (int)Py_SIZE(code); for (int i = 0; i < code_len; i++) { _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; int opcode = instr->opcode; /* TO DO -- Check INSTRUMENTED_OPCODE */ if (opcode == INSTRUMENTED_LINE) { - assert(data->lines); - assert(valid_opcode(data->lines[i].original_opcode)); + CHECK(data->lines); + CHECK(valid_opcode(data->lines[i].original_opcode)); opcode = data->lines[i].original_opcode; + CHECK(opcode != END_FOR); + CHECK(opcode != RESUME); + CHECK(opcode != INSTRUMENTED_RESUME); if (!is_instrumented(opcode)) { CHECK(_PyOpcode_Deopt[opcode] == opcode); } @@ -293,6 +404,12 @@ sanity_check_instrumentation(PyCodeObject *code) CHECK(instr[1].cache > 0); } opcode = deinstrumented; + int event = EVENT_FOR_OPCODE[opcode]; + if (event < 0) { + /* RESUME fixup */ + event = instr->oparg; + } + CHECK(tools_matrix.tools[event] != 0); } if (data->lines && opcode != END_FOR) { int line1 = compute_line(code, i, data->lines[i].line_delta); @@ -302,34 +419,25 @@ sanity_check_instrumentation(PyCodeObject *code) CHECK(valid_opcode(opcode)); opcode = _PyOpcode_Deopt[opcode]; CHECK(valid_opcode(opcode)); - if (data->tools && OPCODE_HAS_EVENT[opcode]) { - int event = EVENT_FOR_OPCODE[opcode]; + if (data->tools) { uint8_t local_tools = data->tools[i]; - uint8_t global_tools = code->_co_instrumentation.monitoring_data->matrix.tools[event]; - CHECK((global_tools & local_tools) == local_tools); + if (OPCODE_HAS_EVENT[opcode]) { + int event = EVENT_FOR_OPCODE[opcode]; + if (event == -1) { + /* RESUME fixup */ + event = instr->oparg; + } + uint8_t global_tools = tools_matrix.tools[event]; + CHECK((global_tools & local_tools) == local_tools); + } + else { + CHECK(local_tools == 0xff); + } } i += _PyOpcode_Caches[opcode]; } } -static inline uint8_t -get_tools(PyCodeObject * code, int index, int event) -{ - uint8_t tools; - assert(event != PY_MONITORING_EVENT_LINE); - assert(event != PY_MONITORING_EVENT_INSTRUCTION); - _PyCoInstrumentationData *monitoring = code->_co_instrumentation.monitoring_data; - if (monitoring && monitoring->tools) { - tools = monitoring->tools[index]; - } - else { - tools = code->_co_instrumentation.monitoring_data->matrix.tools[event]; - } - assert((tools & PyInterpreterState_Get()->monitoring_matrix.tools[event]) == tools); - assert((tools & code->_co_instrumentation.monitoring_data->matrix.tools[event]) == tools); - return tools; -} - int _Py_GetBaseOpcode(PyCodeObject *code, int offset) { int opcode = _Py_OPCODE(_PyCode_CODE(code)[offset]); @@ -366,8 +474,7 @@ de_instrument(PyCodeObject *code, int offset, int event) int deinstrumented = DE_INSTRUMENT[opcode]; int base_opcode; if (deinstrumented) { - base_opcode = _PyOpcode_Deopt[deinstrumented]; - assert(base_opcode != 0); + base_opcode = deinstrumented; instr->opcode = base_opcode; } else { @@ -375,11 +482,18 @@ de_instrument(PyCodeObject *code, int offset, int event) assert(opcode == INSTRUMENTED_LINE); _PyCoLineInstrumentationData *lines = &code->_co_instrumentation.monitoring_data->lines[offset]; int orignal_opcode = lines->original_opcode; - base_opcode = _PyOpcode_Deopt[orignal_opcode]; - assert(INSTRUMENTED_OPCODES[orignal_opcode] == orignal_opcode); - assert(base_opcode != 0); - lines->original_opcode = base_opcode; + if (is_instrumented(orignal_opcode)) { + base_opcode = DE_INSTRUMENT[orignal_opcode]; + lines->original_opcode = DE_INSTRUMENT[orignal_opcode]; + assert(lines->original_opcode != 0); + + } + else { + base_opcode = orignal_opcode; + } } + assert(base_opcode != 0); + assert(_PyOpcode_Deopt[base_opcode] == base_opcode); if (_PyOpcode_Caches[base_opcode]) { instr[1].cache = adaptive_counter_warmup(); } @@ -428,9 +542,6 @@ instrument(PyCodeObject *code, int offset) else if (!is_instrumented(opcode)) { opcode = _PyOpcode_Deopt[opcode]; int instrumented = INSTRUMENTED_OPCODES[opcode]; - if (!instrumented) { - printf("OPCODE: %s\n", _PyOpcode_OpName[opcode]); - } assert(instrumented); instr->opcode = instrumented; /* TO DO -- Use instruction length, not cache count */ @@ -469,11 +580,27 @@ instrument_line(PyCodeObject *code, int offset) assert(code->_co_instrumentation.monitoring_data->lines[offset].original_opcode != 0); } +#ifndef NDEBUG +static bool +instruction_has_event(PyCodeObject *code, int offset) +{ + _Py_CODEUNIT instr = _PyCode_CODE(code)[offset]; + int opcode = instr.opcode; + /* TO DO -- Handle INSTRUMENTED_OPCODE */ + if (opcode == INSTRUMENTED_LINE) { + opcode = code->_co_instrumentation.monitoring_data->lines[offset].original_opcode; + } + assert(DE_INSTRUMENT[opcode] == 0 || OPCODE_HAS_EVENT[DE_INSTRUMENT[opcode]] == OPCODE_HAS_EVENT[opcode]); + return OPCODE_HAS_EVENT[opcode]; +} +#endif + static void remove_tools(PyCodeObject * code, int offset, int event, int tools) { assert(event != PY_MONITORING_EVENT_LINE); assert(event != PY_MONITORING_EVENT_INSTRUCTION); + assert(instruction_has_event(code, offset)); _PyCoInstrumentationData *monitoring = code->_co_instrumentation.monitoring_data; if (monitoring && monitoring->tools) { monitoring->tools[offset] &= ~tools; @@ -592,10 +719,29 @@ static inline int most_significant_bit(uint8_t bits) { } } +static inline uint8_t +get_tools(PyCodeObject * code, int index, int event) +{ + uint8_t tools; + assert(event != PY_MONITORING_EVENT_LINE); + assert(event != PY_MONITORING_EVENT_INSTRUCTION); + _PyCoInstrumentationData *monitoring = code->_co_instrumentation.monitoring_data; + if (monitoring && monitoring->tools) { + tools = monitoring->tools[index]; + } + else { + tools = code->_co_instrumentation.monitoring_data->matrix.tools[event]; + } + assert((tools & PyInterpreterState_Get()->monitoring_matrix.tools[event]) == tools); + assert((tools & code->_co_instrumentation.monitoring_data->matrix.tools[event]) == tools); + return tools; +} + static int call_instrument(PyThreadState *tstate, PyCodeObject *code, int event, PyObject **args, Py_ssize_t nargsf, _Py_CODEUNIT *instr) { + //sanity_check_instrumentation(code); if (tstate->tracing) { return 0; } @@ -626,17 +772,6 @@ call_instrument(PyThreadState *tstate, PyCodeObject *code, int event, return 0; } -static inline bool -matrix_equals(_Py_MonitoringMatrix a, _Py_MonitoringMatrix b) -{ - for (int i = 0; i < PY_MONITORING_EVENTS; i++) { - if (a.tools[i] != b.tools[i]) { - return false; - } - } - return true; -} - static bool is_instrumentation_up_to_date(PyCodeObject *code, PyInterpreterState *interp) { @@ -802,75 +937,6 @@ _PyMonitoring_RegisterCallback(int tool_id, int event_id, PyObject *obj) return callback; } -static inline _Py_MonitoringMatrix -matrix_sub(_Py_MonitoringMatrix a, _Py_MonitoringMatrix b) -{ - _Py_MonitoringMatrix res; - for (int i = 0; i < PY_MONITORING_EVENTS; i++) { - res.tools[i] = a.tools[i] & ~b.tools[i]; - } - return res; -} - -static inline _Py_MonitoringMatrix -matrix_and(_Py_MonitoringMatrix a, _Py_MonitoringMatrix b) -{ - _Py_MonitoringMatrix res; - for (int i = 0; i < PY_MONITORING_EVENTS; i++) { - res.tools[i] = a.tools[i] & b.tools[i]; - } - return res; -} - -static inline bool -matrix_empty(_Py_MonitoringMatrix m) -{ - for (int i = 0; i < PY_MONITORING_EVENTS; i++) { - if (m.tools[i]) { - return false; - } - } - return true; -} - -static inline int -multiple_tools(_Py_MonitoringMatrix m) -{ - for (int i = 0; i < PY_MONITORING_EVENTS; i++) { - if (_Py_popcount32(m.tools[i]) > 1) { - return true; - } - } - return false; -} - -static inline void -matrix_set_bit(_Py_MonitoringMatrix *m, int event, int tool, int val) -{ - assert(0 <= event && event < PY_MONITORING_EVENTS); - assert(0 <= tool && tool < PY_MONITORING_TOOL_IDS); - assert(val == 0 || val == 1); - m->tools[event] &= ~(1 << tool); - m->tools[event] |= (val << tool); -} - -static inline _PyMonitoringEventSet -matrix_get_events(_Py_MonitoringMatrix *m, int tool_id) -{ - _PyMonitoringEventSet result = 0; - for (int e = 0; e < PY_MONITORING_EVENTS; e++) { - if ((m->tools[e] >> tool_id) & 1) { - result |= (1 << e); - } - } - return result; -} - -static inline int -tools_data_offset() { - return 0; -} - static void initialize_tools(PyCodeObject *code) { @@ -879,7 +945,7 @@ initialize_tools(PyCodeObject *code) int code_len = (int)Py_SIZE(code); for (int i = 0; i < code_len; i++) { _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; - int opcode = _Py_OPCODE(*instr); + int opcode = instr->opcode; if (opcode == INSTRUMENTED_LINE) { opcode = code->_co_instrumentation.monitoring_data->lines[i].original_opcode; } @@ -893,8 +959,7 @@ initialize_tools(PyCodeObject *code) if (instrumented) { int8_t event; if (opcode == RESUME) { - int oparg = _Py_OPARG(*instr); - event = oparg != 0; + event = instr->oparg != 0; } else { event = EVENT_FOR_OPCODE[opcode]; @@ -902,12 +967,21 @@ initialize_tools(PyCodeObject *code) } assert(event >= 0); tools[i] = code->_co_instrumentation.monitoring_data->matrix.tools[event]; - assert(tools[i] != 0); + CHECK(tools[i] != 0); } else { tools[i] = 0; } } +#ifdef Py_DEBUG + /* Initialize tools for invalid locations to all ones to try to catch errors */ + else { + tools[i] = 0xff; + } + for (int j = 1; j <= _PyOpcode_Caches[opcode]; j++) { + tools[i+j] = 0xff; + } +#endif i += _PyOpcode_Caches[opcode]; } } @@ -915,11 +989,6 @@ initialize_tools(PyCodeObject *code) static bool is_new_line(int index, int current_line, PyCodeAddressRange *bounds, int opcode) { int line = _PyCode_CheckLineNumber(index*(int)sizeof(_Py_CODEUNIT), bounds); - /* END_FOR cannot start a line, as it is skipped by FOR_ITER */ - if (opcode == END_FOR) { - bounds->ar_line = -1; - return false; - } return line >= 0 && line != current_line; } @@ -933,21 +1002,29 @@ initialize_lines(PyCodeObject *code) _PyCode_InitAddressRange(code, &range); for (int i = 0; i < code->_co_firsttraceable && i < code_len; i++) { line_data[i].original_opcode = 0; - line_data[i].line_delta = -128; + line_data[i].line_delta = -127; } int current_line = -1; for (int i = code->_co_firsttraceable; i < code_len; i++) { int opcode = _Py_GetBaseOpcode(code, i); - if (is_new_line(i, current_line, &range, opcode)) { - line_data[i].original_opcode = 255; - current_line = range.ar_line; + /* END_FOR cannot start a line, as it is skipped by FOR_ITER + * RESUME must not be instrumented with INSTRUMENT_LINE */ + if (opcode == END_FOR || opcode == RESUME) { + line_data[i].original_opcode = 0; + line_data[i].line_delta = -127; } else { - /* Mark as not being an instrumentation point */ - line_data[i].original_opcode = 0; + if (is_new_line(i, current_line, &range, opcode)) { + line_data[i].original_opcode = 255; + current_line = range.ar_line; + } + else { + /* Mark as not being an instrumentation point */ + line_data[i].original_opcode = 0; + } + // assert(range.ar_line < 0 || range.ar_line >= code->co_firstlineno); + line_data[i].line_delta = compute_line_delta(code, i, range.ar_line); } - // assert(range.ar_line < 0 || range.ar_line >= code->co_firstlineno); - line_data[i].line_delta = compute_line_delta(code, i, range.ar_line); for (int j = 0; j < _PyOpcode_Caches[opcode]; j++) { i++; line_data[i].original_opcode = 0; @@ -970,7 +1047,6 @@ initialize_line_tools(PyCodeObject *code, PyInterpreterState *interp) int update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) { - bool restarted = interp->last_restart_version > code->_co_instrumentation.monitoring_version; int code_len = (int)Py_SIZE(code); if (code->_co_instrumentation.monitoring_data == NULL) { code->_co_instrumentation.monitoring_data = PyMem_Malloc(sizeof(_PyCoInstrumentationData)); @@ -978,7 +1054,7 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) PyErr_NoMemory(); return -1; } - code->_co_instrumentation.monitoring_data->matrix = interp->monitoring_matrix; + code->_co_instrumentation.monitoring_data->matrix = (_Py_MonitoringMatrix){ 0 }; code->_co_instrumentation.monitoring_data->tools = NULL; code->_co_instrumentation.monitoring_data->lines = NULL; code->_co_instrumentation.monitoring_data->line_tools = NULL; @@ -992,11 +1068,7 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) PyErr_NoMemory(); return -1; } - if (restarted) { - memset(code->_co_instrumentation.monitoring_data->tools, 0, code_len); - } else { - initialize_tools(code); - } + initialize_tools(code); } if (interp->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE]) { if (code->_co_instrumentation.monitoring_data->lines == NULL) { @@ -1043,7 +1115,6 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) ); return 0; } - bool restarted = interp->last_restart_version > code->_co_instrumentation.monitoring_version; assert(interp->monitoring_matrix.tools[PY_MONITORING_EVENT_INSTRUCTION] == 0); int code_len = (int)Py_SIZE(code); if (update_instrumentation_data(code, interp)) { @@ -1052,6 +1123,7 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) _Py_MonitoringMatrix new_events; _Py_MonitoringMatrix removed_events; + bool restarted = interp->last_restart_version > code->_co_instrumentation.monitoring_version; if (restarted) { removed_events = code->_co_instrumentation.monitoring_data->matrix; new_events = interp->monitoring_matrix; @@ -1064,13 +1136,17 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) code->_co_instrumentation.monitoring_data->matrix = interp->monitoring_matrix; code->_co_instrumentation.monitoring_version = interp->monitoring_version; if (matrix_empty(new_events) && matrix_empty(removed_events)) { - sanity_check_instrumentation(code); + //sanity_check_instrumentation(code); return 0; } /* Insert instrumentation */ for (int i = 0; i < code_len; i++) { _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; int opcode = _Py_OPCODE(*instr); + /* TO DO handle INSTRUMENTED_OPCODE */ + if (opcode == INSTRUMENTED_LINE) { + opcode = code->_co_instrumentation.monitoring_data->lines[i].original_opcode; + } int base_opcode; if (DE_INSTRUMENT[opcode]) { base_opcode = _PyOpcode_Deopt[DE_INSTRUMENT[opcode]]; @@ -1114,7 +1190,7 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) uint8_t removed_line_tools = removed_events.tools[PY_MONITORING_EVENT_LINE]; if (new_line_tools || removed_line_tools) { _PyCoLineInstrumentationData *line_data = code->_co_instrumentation.monitoring_data->lines; - for (int i = code->_co_firsttraceable + 1; i < code_len; i++) { + for (int i = code->_co_firsttraceable; i < code_len; i++) { if (line_data[i].original_opcode) { if (removed_line_tools) { remove_line_tools(code, i, removed_line_tools); @@ -1132,7 +1208,7 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) } } } - sanity_check_instrumentation(code); + //sanity_check_instrumentation(code); return 0; } diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index 187a3246b71616..5bef8ab9e69ea4 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -259,6 +259,12 @@ sys_trace_branch_func( _PyInterpreterFrame *iframe = _PyEval_GetFrame(); assert(iframe); PyFrameObject* frame = _PyFrame_GetFrameObject(iframe); + if (frame == NULL) { + return NULL; + } + if (!frame->f_trace_lines) { + Py_RETURN_NONE; + } int from = _PyLong_AsInt(args[1]); assert(from >= 0); int to = _PyLong_AsInt(args[2]); From c9e1e21ab4694df3b01d6d43b4e2a48f3b205a26 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 19 Jan 2023 11:45:09 +0000 Subject: [PATCH 015/116] Fix LINE instrumentation and frame.set_lineno support (mostly) --- Include/cpython/pystate.h | 2 +- Include/internal/pycore_instruments.h | 4 +- Lib/test/test_monitoring.py | 2 +- Objects/frameobject.c | 54 ++++++++++++------- Python/bytecodes.c | 18 ++++++- Python/generated_cases.c.h | 18 ++++++- Python/instrumentation.c | 76 +++++++++++++++++++++------ Python/legacy_tracing.c | 52 +++++++----------- Python/pystate.c | 1 + 9 files changed, 153 insertions(+), 74 deletions(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 84a22d2f2f1abd..0c70cbb1b05183 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -129,7 +129,7 @@ struct _ts { This is to prevent the actual trace/profile code from being recorded in the trace/profile. */ int tracing; - int tracing_what; /* The event currently being traced, if any. */ + int what_event; /* The event currently being monitored, if any. */ uint8_t monitoring; /* Pointer to current _PyCFrame in the C stack frame of the currently, diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index 98cba9884b4f60..33c894368d2cac 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -67,8 +67,8 @@ _Py_call_instrumentation(PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr); extern int -_Py_call_instrumentation_line(PyThreadState *tstate, - PyCodeObject *code, _Py_CODEUNIT *instr); +_Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, + _Py_CODEUNIT *instr); int _Py_call_instrumentation_jump( diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index fabf3ab8ae6d05..1e5ef85eb4dcab 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -510,7 +510,7 @@ def test_lines_loop(self): floop() sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) - self.assertEqual(events, [510, 21, 22, 22, 21, 511]) + self.assertEqual(events, [510, 21, 22, 21, 22, 21, 511]) finally: sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index d05b0b7d65e029..6f3f90f13c662e 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -686,32 +686,48 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore * In addition, jumps are forbidden when not tracing, * as this is a debugging feature. */ - switch(PyThreadState_GET()->tracing_what) { - case PyTrace_EXCEPTION: + int what_event = PyThreadState_GET()->what_event; + if (what_event < 0) { + PyErr_Format(PyExc_ValueError, + "f_lineno can only be set by a debugger"); + return -1; + } + switch(what_event) { + case PY_MONITORING_EVENT_PY_START: + PyErr_Format(PyExc_ValueError, + "can't jump from the 'call' event of a new frame"); + return -1; + case PY_MONITORING_EVENT_PY_RESUME: + break; + case PY_MONITORING_EVENT_PY_RETURN: PyErr_SetString(PyExc_ValueError, - "can only jump from a 'line' trace event"); + "can't jump from a 'return' event"); return -1; - case PyTrace_CALL: - PyErr_Format(PyExc_ValueError, - "can't jump from the 'call' trace event of a new frame"); + case PY_MONITORING_EVENT_PY_YIELD: + break; + case PY_MONITORING_EVENT_CALL: + case PY_MONITORING_EVENT_C_RETURN: + PyErr_SetString(PyExc_ValueError, + "can't jump during a call"); return -1; - case PyTrace_LINE: + case PY_MONITORING_EVENT_LINE: break; - case PyTrace_RETURN: - if (state == FRAME_SUSPENDED) { - break; - } - /* fall through */ + case PY_MONITORING_EVENT_PY_UNWIND: + case PY_MONITORING_EVENT_PY_THROW: + case PY_MONITORING_EVENT_RAISE: + case PY_MONITORING_EVENT_C_RAISE: + case PY_MONITORING_EVENT_JUMP: + case PY_MONITORING_EVENT_BRANCH: + case PY_MONITORING_EVENT_INSTRUCTION: + case PY_MONITORING_EVENT_EXCEPTION_HANDLED: + PyErr_Format(PyExc_ValueError, + "can only jump from 'line' event"); + return -1; default: - PyErr_SetString(PyExc_ValueError, - "can only jump from a 'line' trace event"); + PyErr_SetString(PyExc_SystemError, + "unexpected event type"); return -1; } - if (!f->f_trace) { - PyErr_Format(PyExc_ValueError, - "f_lineno can only be set by a trace function"); - return -1; - } int new_lineno; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 31d1c5d2050004..dfd5b7bf304263 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -127,9 +127,17 @@ dummy_func( next_instr--; } else { + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation( tstate, oparg != 0, frame, next_instr-1); ERROR_IF(err, error); + if (frame->prev_instr != next_instr-1) { + fprintf(stderr, "Jump has happened\n"); + /* Instrumentation has jumped */ + next_instr = frame->prev_instr; + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { goto handle_eval_breaker; } @@ -763,10 +771,18 @@ dummy_func( inst(INSTRUMENTED_YIELD_VALUE, ( -- )) { PyObject *val = TOP(); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_YIELD, frame, next_instr-1, val); ERROR_IF(err, error); + if (frame->prev_instr != next_instr-1) { + fprintf(stderr, "Jump has happened\n"); + /* Instrumentation has jumped */ + next_instr = frame->prev_instr; + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } GO_TO_INSTRUCTION(YIELD_VALUE); } @@ -3412,7 +3428,7 @@ dummy_func( inst(INSTRUMENTED_LINE, ( -- )) { _PyFrame_SetStackPointer(frame, stack_pointer); int original_opcode = _Py_call_instrumentation_line( - tstate, frame->f_code, next_instr-1); + tstate, frame, next_instr-1); ERROR_IF(original_opcode < 0, error); next_instr--; if (frame->prev_instr != next_instr) { diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 99f8ba38aa1d72..7a2414958ca1b5 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -16,9 +16,17 @@ next_instr--; } else { + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation( tstate, oparg != 0, frame, next_instr-1); if (err) goto error; + if (frame->prev_instr != next_instr-1) { + fprintf(stderr, "Jump has happened\n"); + /* Instrumentation has jumped */ + next_instr = frame->prev_instr; + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { goto handle_eval_breaker; } @@ -928,10 +936,18 @@ TARGET(INSTRUMENTED_YIELD_VALUE) { PyObject *val = TOP(); + _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_YIELD, frame, next_instr-1, val); if (err) goto error; + if (frame->prev_instr != next_instr-1) { + fprintf(stderr, "Jump has happened\n"); + /* Instrumentation has jumped */ + next_instr = frame->prev_instr; + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } GO_TO_INSTRUCTION(YIELD_VALUE); } @@ -3739,7 +3755,7 @@ TARGET(INSTRUMENTED_LINE) { _PyFrame_SetStackPointer(frame, stack_pointer); int original_opcode = _Py_call_instrumentation_line( - tstate, frame->f_code, next_instr-1); + tstate, frame, next_instr-1); if (original_opcode < 0) goto error; next_instr--; if (frame->prev_instr != next_instr) { diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 797df8ebc315cf..09802dbcf3025a 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -690,9 +690,12 @@ call_one_instrument( if (instrument == NULL) { return 0; } + int old_what = tstate->what_event; + tstate->what_event = event; tstate->tracing++; PyObject *res = PyObject_Vectorcall(instrument, args, nargsf, NULL); tstate->tracing--; + tstate->what_event = old_what; if (res == NULL) { return -1; } @@ -877,8 +880,9 @@ _Py_Instrumentation_GetLine(PyCodeObject *code, int index) } int -_Py_call_instrumentation_line(PyThreadState *tstate, PyCodeObject *code, _Py_CODEUNIT *instr) +_Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr) { + PyCodeObject *code = frame->f_code; assert(is_instrumentation_up_to_date(code, tstate->interp)); assert(matrix_equals(code->_co_instrumentation.monitoring_data->matrix, tstate->interp->monitoring_matrix)); int offset = instr - _PyCode_CODE(code); @@ -891,6 +895,11 @@ _Py_call_instrumentation_line(PyThreadState *tstate, PyCodeObject *code, _Py_COD PyInterpreterState *interp = tstate->interp; int8_t line_delta = line_data->line_delta; int line = compute_line(code, offset, line_delta); + if (tstate->trace_info.code == frame->f_code && + tstate->trace_info.line == line) { + /* Already traced this line */ + return original_opcode; + } uint8_t tools = code->_co_instrumentation.monitoring_data->line_tools != NULL ? code->_co_instrumentation.monitoring_data->line_tools[offset] : interp->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE]; @@ -920,6 +929,8 @@ _Py_call_instrumentation_line(PyThreadState *tstate, PyCodeObject *code, _Py_COD remove_line_tools(code, offset, 1 << tool); } } + tstate->trace_info.code = frame->f_code; + tstate->trace_info.line = line; Py_DECREF(line_obj); assert(original_opcode != 0); assert(_PyOpcode_Deopt[original_opcode] == original_opcode); @@ -992,6 +1003,8 @@ static bool is_new_line(int index, int current_line, PyCodeAddressRange *bounds, return line >= 0 && line != current_line; } +#define LINE_MARKER 255 + static void initialize_lines(PyCodeObject *code) { @@ -1005,31 +1018,62 @@ initialize_lines(PyCodeObject *code) line_data[i].line_delta = -127; } int current_line = -1; + int extended_arg = 0; + _Py_CODEUNIT *bytecode = _PyCode_CODE(code); + for (int i = code->_co_firsttraceable; i < code_len; i++) { + line_data[i].original_opcode = 0; + } for (int i = code->_co_firsttraceable; i < code_len; i++) { int opcode = _Py_GetBaseOpcode(code, i); - /* END_FOR cannot start a line, as it is skipped by FOR_ITER - * RESUME must not be instrumented with INSTRUMENT_LINE */ - if (opcode == END_FOR || opcode == RESUME) { - line_data[i].original_opcode = 0; - line_data[i].line_delta = -127; - } - else { - if (is_new_line(i, current_line, &range, opcode)) { - line_data[i].original_opcode = 255; - current_line = range.ar_line; - } - else { - /* Mark as not being an instrumentation point */ + switch (opcode) { + case END_FOR: + case RESUME: + /* END_FOR cannot start a line, as it is skipped by FOR_ITER + * RESUME must not be instrumented with INSTRUMENT_LINE */ line_data[i].original_opcode = 0; + line_data[i].line_delta = -127; + extended_arg = 0; + continue; + case EXTENDED_ARG: + extended_arg = (extended_arg << 8) + bytecode[i].oparg; + line_data[i].line_delta = -127; + continue; + case JUMP_IF_FALSE_OR_POP: + case JUMP_IF_TRUE_OR_POP: + case POP_JUMP_IF_FALSE: + case POP_JUMP_IF_TRUE: + case JUMP_FORWARD: + /* COMPARE_AND_BRANCH is handled by the following POP_JUMP_IF_FALSE/TRUE */ + { + int offset = (extended_arg << 8) + bytecode[i].oparg; + line_data[i+1+offset].original_opcode = LINE_MARKER; + break; } - // assert(range.ar_line < 0 || range.ar_line >= code->co_firstlineno); - line_data[i].line_delta = compute_line_delta(code, i, range.ar_line); + case FOR_ITER: + { + /* Skip over END_FOR */ + int offset = (extended_arg << 8) + bytecode[i].oparg + 1; + line_data[i+1+offset].original_opcode = LINE_MARKER; + break; + } + case JUMP_BACKWARD: + { + int offset = (extended_arg << 8) + bytecode[i].oparg; + line_data[i+1-offset].original_opcode = LINE_MARKER; + break; + } + } + if (is_new_line(i, current_line, &range, opcode)) { + line_data[i].original_opcode = LINE_MARKER; + current_line = range.ar_line; } + line_data[i].line_delta = compute_line_delta(code, i, range.ar_line); for (int j = 0; j < _PyOpcode_Caches[opcode]; j++) { i++; line_data[i].original_opcode = 0; line_data[i].line_delta = -128; } + extended_arg = 0; } } diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index 5bef8ab9e69ea4..d232917273f8d0 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -163,14 +163,8 @@ trace_line(PyThreadState *tstate, _PyInterpreterFrame *iframe, if (line < 0) { Py_RETURN_NONE; } - if (tstate->trace_info.code == iframe->f_code && - tstate->trace_info.line == line) { - /* Already traced this line */ - Py_RETURN_NONE; - } Py_INCREF(frame); - tstate->trace_info.code = iframe->f_code; - tstate->trace_info.line = frame->f_lineno = line; + frame->f_lineno = line; int err = tstate->c_tracefunc(tstate->c_traceobj, frame, self->event, Py_None); frame->f_lineno = 0; Py_DECREF(frame); @@ -180,7 +174,6 @@ trace_line(PyThreadState *tstate, _PyInterpreterFrame *iframe, Py_RETURN_NONE; } - static PyObject * sys_trace_line_func( _PyLegacyEventHandler *self, PyObject *const *args, @@ -245,7 +238,7 @@ sys_trace_handled_exception( static PyObject * -sys_trace_branch_func( +sys_trace_jump_func( _PyLegacyEventHandler *self, PyObject *const *args, size_t nargsf, PyObject *kwnames ) { @@ -256,6 +249,19 @@ sys_trace_branch_func( } Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); assert(nargs == 3); + int from = _PyLong_AsInt(args[1]); + assert(from >= 0); + int to = _PyLong_AsInt(args[2]); + assert(to >= 0); + /* Forward jumps should have been handled by the line instrumentation. + * We need to handle backward jumps, as the semantics differ. + * PEP 669 only generates events when a new line is seen. + * Legacy tracing generates events for all backward jumps even on + * the same line */ + if (to > from) { + /* Forwards jump */ + Py_RETURN_NONE; + } _PyInterpreterFrame *iframe = _PyEval_GetFrame(); assert(iframe); PyFrameObject* frame = _PyFrame_GetFrameObject(iframe); @@ -265,23 +271,9 @@ sys_trace_branch_func( if (!frame->f_trace_lines) { Py_RETURN_NONE; } - int from = _PyLong_AsInt(args[1]); - assert(from >= 0); - int to = _PyLong_AsInt(args[2]); - assert(to >= 0); /* We can call _Py_Instrumentation_GetLine because we always set * line events for tracing */ int to_line = _Py_Instrumentation_GetLine(iframe->f_code, to); - if (from > to) { - /* Backwards jump -- always trace */ - tstate->trace_info.code = NULL; - } - else { - int from_line = _Py_Instrumentation_GetLine(iframe->f_code, from); - if (to_line == from_line) { - Py_RETURN_NONE; - } - } return trace_line(tstate, iframe, self, frame, to_line); } @@ -459,8 +451,8 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) return -1; } if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, - (vectorcallfunc)sys_trace_branch_func, PyTrace_LINE, - PY_MONITORING_EVENT_JUMP, PY_MONITORING_EVENT_BRANCH)) { + (vectorcallfunc)sys_trace_jump_func, PyTrace_LINE, + PY_MONITORING_EVENT_JUMP, -1)) { return -1; } if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, @@ -468,11 +460,6 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) PY_MONITORING_EVENT_INSTRUCTION, -1)) { return -1; } - if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, - (vectorcallfunc)sys_trace_handled_exception, PyTrace_LINE, - PY_MONITORING_EVENT_EXCEPTION_HANDLED, -1)) { - return -1; - } /* TO DO: Set up callback for PY_MONITORING_EVENT_EXCEPTION_HANDLED */ } if (tstate->interp->sys_tracing_threads) { @@ -480,9 +467,8 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) (1 << PY_MONITORING_EVENT_PY_START) | (1 << PY_MONITORING_EVENT_PY_RESUME) | (1 << PY_MONITORING_EVENT_PY_RETURN) | (1 << PY_MONITORING_EVENT_PY_YIELD) | (1 << PY_MONITORING_EVENT_RAISE) | (1 << PY_MONITORING_EVENT_LINE) | - (1 << PY_MONITORING_EVENT_JUMP) | (1 << PY_MONITORING_EVENT_BRANCH) | - (1 << PY_MONITORING_EVENT_EXCEPTION_HANDLED | (1 << PY_MONITORING_EVENT_PY_UNWIND) | - (1 << PY_MONITORING_EVENT_PY_THROW)); + (1 << PY_MONITORING_EVENT_JUMP)| (1 << PY_MONITORING_EVENT_PY_UNWIND) | + (1 << PY_MONITORING_EVENT_PY_THROW); /* TO DO -- opcode events */ if (tstate->interp->f_opcode_trace_set && false) { events |= (1 << PY_MONITORING_EVENT_INSTRUCTION); diff --git a/Python/pystate.c b/Python/pystate.c index 8e39759b8ae9a9..30a30360778629 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -830,6 +830,7 @@ init_threadstate(PyThreadState *tstate, tstate->datastack_top = NULL; tstate->datastack_limit = NULL; tstate->monitoring = 0; + tstate->what_event = -1; tstate->_initialized = 1; } From e52e8d32812bc4eb097fa1d3a3cc7c26ebf579fc Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 20 Jan 2023 10:02:24 +0000 Subject: [PATCH 016/116] Refining line event generation. --- Include/internal/pycore_instruments.h | 4 + Include/internal/pycore_opcode.h | 4 +- Include/opcode.h | 1 + Lib/opcode.py | 2 +- Objects/frameobject.c | 14 +- Python/bytecodes.c | 14 + Python/generated_cases.c.h | 14 + Python/instrumentation.c | 676 ++++++++++++++++++-------- Python/legacy_tracing.c | 67 +-- Python/opcode_metadata.h | 1 + Python/opcode_targets.h | 2 +- 11 files changed, 530 insertions(+), 269 deletions(-) diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index 33c894368d2cac..6c76027889457d 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -70,6 +70,10 @@ extern int _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr); +extern int +_Py_call_instrumentation_instruction( + PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr); + int _Py_call_instrumentation_jump( PyThreadState *tstate, int event, diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index 05a9d2e4c4673b..fb1359a93b8325 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -139,6 +139,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [INSTRUMENTED_CALL] = INSTRUMENTED_CALL, [INSTRUMENTED_CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, [INSTRUMENTED_COMPARE_AND_BRANCH] = INSTRUMENTED_COMPARE_AND_BRANCH, + [INSTRUMENTED_INSTRUCTION] = INSTRUMENTED_INSTRUCTION, [INSTRUMENTED_JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD, [INSTRUMENTED_JUMP_FORWARD] = INSTRUMENTED_JUMP_FORWARD, [INSTRUMENTED_JUMP_IF_FALSE_OR_POP] = INSTRUMENTED_JUMP_IF_FALSE_OR_POP, @@ -481,7 +482,7 @@ static const char *const _PyOpcode_OpName[263] = { [235] = "<235>", [236] = "<236>", [237] = "<237>", - [238] = "<238>", + [INSTRUMENTED_INSTRUCTION] = "INSTRUMENTED_INSTRUCTION", [INSTRUMENTED_COMPARE_AND_BRANCH] = "INSTRUMENTED_COMPARE_AND_BRANCH", [INSTRUMENTED_RESUME] = "INSTRUMENTED_RESUME", [INSTRUMENTED_CALL] = "INSTRUMENTED_CALL", @@ -580,7 +581,6 @@ static const char *const _PyOpcode_OpName[263] = { case 235: \ case 236: \ case 237: \ - case 238: \ case 248: \ ; diff --git a/Include/opcode.h b/Include/opcode.h index e5adadf8fcb7d0..88ebea7c0efc20 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -116,6 +116,7 @@ extern "C" { #define CALL 171 #define KW_NAMES 172 #define CALL_INTRINSIC_1 173 +#define INSTRUMENTED_INSTRUCTION 238 #define INSTRUMENTED_COMPARE_AND_BRANCH 239 #define INSTRUMENTED_RESUME 240 #define INSTRUMENTED_CALL 241 diff --git a/Lib/opcode.py b/Lib/opcode.py index 254adef46cc837..7d4091ed1178bd 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -222,6 +222,7 @@ def pseudo_op(name, op, real_ops): # Instrumented instructions +def_op('INSTRUMENTED_INSTRUCTION', 238) def_op('INSTRUMENTED_COMPARE_AND_BRANCH', 239) def_op('INSTRUMENTED_RESUME', 240) def_op('INSTRUMENTED_CALL', 241) @@ -236,7 +237,6 @@ def pseudo_op(name, op, real_ops): def_op('INSTRUMENTED_POP_JUMP_IF_TRUE', 251) def_op('INSTRUMENTED_POP_JUMP_IF_NONE', 252) def_op('INSTRUMENTED_POP_JUMP_IF_NOT_NONE', 253) - def_op('INSTRUMENTED_LINE', 254) # 255 is reserved. diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 6f3f90f13c662e..dc196846d57ab4 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -689,20 +689,17 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore int what_event = PyThreadState_GET()->what_event; if (what_event < 0) { PyErr_Format(PyExc_ValueError, - "f_lineno can only be set by a debugger"); + "f_lineno can only be set in a trace function"); return -1; } switch(what_event) { case PY_MONITORING_EVENT_PY_START: PyErr_Format(PyExc_ValueError, - "can't jump from the 'call' event of a new frame"); + "can't jump from the 'call' trace event of a new frame"); return -1; case PY_MONITORING_EVENT_PY_RESUME: break; - case PY_MONITORING_EVENT_PY_RETURN: - PyErr_SetString(PyExc_ValueError, - "can't jump from a 'return' event"); - return -1; + case PY_MONITORING_EVENT_LINE: case PY_MONITORING_EVENT_PY_YIELD: break; case PY_MONITORING_EVENT_CALL: @@ -710,8 +707,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore PyErr_SetString(PyExc_ValueError, "can't jump during a call"); return -1; - case PY_MONITORING_EVENT_LINE: - break; + case PY_MONITORING_EVENT_PY_RETURN: case PY_MONITORING_EVENT_PY_UNWIND: case PY_MONITORING_EVENT_PY_THROW: case PY_MONITORING_EVENT_RAISE: @@ -721,7 +717,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore case PY_MONITORING_EVENT_INSTRUCTION: case PY_MONITORING_EVENT_EXCEPTION_HANDLED: PyErr_Format(PyExc_ValueError, - "can only jump from 'line' event"); + "can only jump from a 'line' trace event"); return -1; default: PyErr_SetString(PyExc_SystemError, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index dfd5b7bf304263..4b3b50bee3f4c4 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3448,6 +3448,20 @@ dummy_func( DISPATCH_GOTO(); } + inst(INSTRUMENTED_INSTRUCTION, ( -- )) { + int next_opcode = _Py_call_instrumentation_instruction( + tstate, frame, next_instr-1); + ERROR_IF(next_opcode < 0, error); + next_instr--; + if (_PyOpcode_Caches[next_opcode]) { + _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); + INCREMENT_ADAPTIVE_COUNTER(cache->counter); + } + assert(next_opcode > 0 && next_opcode < 256); + opcode = next_opcode; + DISPATCH_GOTO(); + } + // stack effect: ( -- ) inst(INSTRUMENTED_JUMP_FORWARD, ( -- )) { int err = _Py_call_instrumentation_jump( diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 7a2414958ca1b5..317485009c8193 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3775,6 +3775,20 @@ DISPATCH_GOTO(); } + TARGET(INSTRUMENTED_INSTRUCTION) { + int next_opcode = _Py_call_instrumentation_instruction( + tstate, frame, next_instr-1); + if (next_opcode < 0) goto error; + next_instr--; + if (_PyOpcode_Caches[next_opcode]) { + _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); + INCREMENT_ADAPTIVE_COUNTER(cache->counter); + } + assert(next_opcode > 0 && next_opcode < 256); + opcode = next_opcode; + DISPATCH_GOTO(); + } + TARGET(INSTRUMENTED_JUMP_FORWARD) { int err = _Py_call_instrumentation_jump( tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 09802dbcf3025a..0ade817707bc48 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -123,8 +123,11 @@ static const uint8_t INSTRUMENTED_OPCODES[256] = { [INSTRUMENTED_COMPARE_AND_BRANCH] = INSTRUMENTED_COMPARE_AND_BRANCH, [INSTRUMENTED_LINE] = INSTRUMENTED_LINE, + [INSTRUMENTED_INSTRUCTION] = INSTRUMENTED_INSTRUCTION, }; +#ifdef INSTRUMENT_DEBUG + static void dump_instrumentation_data_tools(PyCodeObject *code, uint8_t *tools, int i, FILE*out) { @@ -164,6 +167,25 @@ dump_instrumentation_data_line_tools(PyCodeObject *code, uint8_t *line_tools, in } } +static void +dump_instrumentation_data_per_instruction(PyCodeObject *code, _PyCoInstrumentationData *data, int i, FILE*out) +{ + if (data->per_instruction_opcodes == NULL) { + fprintf(out, ", per-inst opcode = NULL"); + } + else { + fprintf(out, ", per-inst opcode = %s", _PyOpcode_OpName[data->per_instruction_opcodes[i]]); + } + if (data->per_instruction_tools == NULL) { + fprintf(out, ", per-inst tools = NULL"); + } + else { + fprintf(out, ", per-inst tools = %d", data->per_instruction_tools[i]); + } +} + + + static void dump_matrix(const char *prefix, _Py_MonitoringMatrix matrix, FILE*out) { @@ -173,6 +195,38 @@ dump_matrix(const char *prefix, _Py_MonitoringMatrix matrix, FILE*out) } } +/* Like _Py_GetBaseOpcode but without asserts. + * Does its best to give the right answer, but won't abort + * if something is wrong */ +int get_base_opcode_best_attempt(PyCodeObject *code, int offset) +{ + int opcode = _Py_OPCODE(_PyCode_CODE(code)[offset]); + if (INSTRUMENTED_OPCODES[opcode] != opcode) { + /* Not instrumented */ + return _PyOpcode_Deopt[opcode] == 0 ? opcode : _PyOpcode_Deopt[opcode]; + } + if (opcode == INSTRUMENTED_INSTRUCTION) { + if (code->_co_instrumentation.monitoring_data->per_instruction_opcodes[offset] == 0) { + return opcode; + } + opcode = code->_co_instrumentation.monitoring_data->per_instruction_opcodes[offset]; + } + if (opcode == INSTRUMENTED_LINE) { + if (code->_co_instrumentation.monitoring_data->lines[offset].original_opcode == 0) { + return opcode; + } + opcode = code->_co_instrumentation.monitoring_data->lines[offset].original_opcode; + } + int deinstrumented = DE_INSTRUMENT[opcode]; + if (deinstrumented) { + return deinstrumented; + } + if (_PyOpcode_Deopt[opcode] == 0) { + return opcode; + } + return _PyOpcode_Deopt[opcode]; +} + /* No error checking -- Don't use this for anything but experimental debugging */ static void dump_instrumentation_data(PyCodeObject *code, int star, FILE*out) @@ -190,13 +244,6 @@ dump_instrumentation_data(PyCodeObject *code, int star, FILE*out) for (int i = 0; i < code_len; i++) { _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; int opcode = instr->opcode; - int base_opcode; - if (DE_INSTRUMENT[opcode]) { - base_opcode = _PyOpcode_Deopt[DE_INSTRUMENT[opcode]]; - } - else { - base_opcode = _PyOpcode_Deopt[opcode]; - } if (i == star) { fprintf(out, "** "); starred = true; @@ -205,9 +252,11 @@ dump_instrumentation_data(PyCodeObject *code, int star, FILE*out) dump_instrumentation_data_tools(code, data->tools, i, out); dump_instrumentation_data_lines(code, data->lines, i, out); dump_instrumentation_data_line_tools(code, data->line_tools, i, out); + dump_instrumentation_data_per_instruction(code, data, i, out); /* TO DO -- per instruction data */ fprintf(out, "\n"); /* TO DO -- Use instruction length, not cache count */ + int base_opcode = get_base_opcode_best_attempt(code, i); i += _PyOpcode_Caches[base_opcode]; } if (!starred && star >= 0) { @@ -216,10 +265,108 @@ dump_instrumentation_data(PyCodeObject *code, int star, FILE*out) dump_instrumentation_data_tools(code, data->tools, star, out); dump_instrumentation_data_lines(code, data->lines, star, out); dump_instrumentation_data_line_tools(code, data->line_tools, star, out); + dump_instrumentation_data_per_instruction(code, data, star, out); fprintf(out, "\n"); } } +#define CHECK(test) do { \ + if (!(test)) { \ + dump_instrumentation_data(code, i, stderr); \ + } \ + assert(test); \ +} while (0) + +bool valid_opcode(int opcode) { + if (opcode > 0 && opcode < 255 && + _PyOpcode_OpName[opcode] && + _PyOpcode_OpName[opcode][0] != '<' + ) { + return true; + } + return false; +} + +static void +sanity_check_instrumentation(PyCodeObject *code) +{ + _PyCoInstrumentationData *data = code->_co_instrumentation.monitoring_data; + if (data == NULL) { + return; + } + _Py_MonitoringMatrix tools_matrix = PyInterpreterState_Get()->monitoring_matrix; + assert(matrix_equals( + code->_co_instrumentation.monitoring_data->matrix, + tools_matrix) + ); + int code_len = (int)Py_SIZE(code); + for (int i = 0; i < code_len; i++) { + _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; + int opcode = instr->opcode; + /* TO DO -- Check INSTRUMENTED_OPCODE */ + if (opcode == INSTRUMENTED_LINE) { + CHECK(data->lines); + CHECK(valid_opcode(data->lines[i].original_opcode)); + opcode = data->lines[i].original_opcode; + CHECK(opcode != END_FOR); + CHECK(opcode != RESUME); + CHECK(opcode != INSTRUMENTED_RESUME); + if (!is_instrumented(opcode)) { + CHECK(_PyOpcode_Deopt[opcode] == opcode); + } + CHECK(opcode != INSTRUMENTED_LINE); + } + else if (data->lines) { + CHECK(data->lines[i].original_opcode == 0 || + data->lines[i].original_opcode == 255); + } + if (is_instrumented(opcode)) { + int deinstrumented = DE_INSTRUMENT[opcode]; + CHECK(valid_opcode(deinstrumented)); + CHECK(_PyOpcode_Deopt[deinstrumented] == deinstrumented); + if (_PyOpcode_Caches[deinstrumented]) { + CHECK(instr[1].cache > 0); + } + opcode = deinstrumented; + int event = EVENT_FOR_OPCODE[opcode]; + if (event < 0) { + /* RESUME fixup */ + event = instr->oparg; + } + CHECK(tools_matrix.tools[event] != 0); + } + if (data->lines && opcode != END_FOR) { + int line1 = compute_line(code, i, data->lines[i].line_delta); + int line2 = PyCode_Addr2Line(code, i*sizeof(_Py_CODEUNIT)); + CHECK(line1 == line2); + } + CHECK(valid_opcode(opcode)); + opcode = _PyOpcode_Deopt[opcode]; + CHECK(valid_opcode(opcode)); + if (data->tools) { + uint8_t local_tools = data->tools[i]; + if (OPCODE_HAS_EVENT[opcode]) { + int event = EVENT_FOR_OPCODE[opcode]; + if (event == -1) { + /* RESUME fixup */ + event = instr->oparg; + } + uint8_t global_tools = tools_matrix.tools[event]; + CHECK((global_tools & local_tools) == local_tools); + } + else { + CHECK(local_tools == 0xff); + } + } + i += _PyOpcode_Caches[opcode]; + } +} +#else + +#define CHECK(test) assert(test) + +#endif + #define OFFSET_SHIFT 4 /* Line delta. @@ -268,19 +415,10 @@ compute_line(PyCodeObject *code, int offset, int8_t line_delta) static inline bool is_instrumented(int opcode) { + assert(opcode != 0); return INSTRUMENTED_OPCODES[opcode] == opcode; } -bool valid_opcode(int opcode) { - if (opcode > 0 && opcode < 255 && - _PyOpcode_OpName[opcode] && - _PyOpcode_OpName[opcode][0] != '<' - ) { - return true; - } - return false; -} - static inline bool matrix_equals(_Py_MonitoringMatrix a, _Py_MonitoringMatrix b) { @@ -356,142 +494,66 @@ matrix_get_events(_Py_MonitoringMatrix *m, int tool_id) return result; } -#define CHECK(test) do { \ - if (!(test)) { \ - dump_instrumentation_data(code, i, stderr); \ - } \ - assert(test); \ -} while (0) - -static void -sanity_check_instrumentation(PyCodeObject *code) -{ - _PyCoInstrumentationData *data = code->_co_instrumentation.monitoring_data; - if (data == NULL) { - return; - } - _Py_MonitoringMatrix tools_matrix = PyInterpreterState_Get()->monitoring_matrix; - assert(matrix_equals( - code->_co_instrumentation.monitoring_data->matrix, - tools_matrix) - ); - int code_len = (int)Py_SIZE(code); - for (int i = 0; i < code_len; i++) { - _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; - int opcode = instr->opcode; - /* TO DO -- Check INSTRUMENTED_OPCODE */ - if (opcode == INSTRUMENTED_LINE) { - CHECK(data->lines); - CHECK(valid_opcode(data->lines[i].original_opcode)); - opcode = data->lines[i].original_opcode; - CHECK(opcode != END_FOR); - CHECK(opcode != RESUME); - CHECK(opcode != INSTRUMENTED_RESUME); - if (!is_instrumented(opcode)) { - CHECK(_PyOpcode_Deopt[opcode] == opcode); - } - CHECK(opcode != INSTRUMENTED_LINE); - } - else if (data->lines) { - CHECK(data->lines[i].original_opcode == 0 || - data->lines[i].original_opcode == 255); - } - if (is_instrumented(opcode)) { - int deinstrumented = DE_INSTRUMENT[opcode]; - CHECK(valid_opcode(deinstrumented)); - CHECK(_PyOpcode_Deopt[deinstrumented] == deinstrumented); - if (_PyOpcode_Caches[deinstrumented]) { - CHECK(instr[1].cache > 0); - } - opcode = deinstrumented; - int event = EVENT_FOR_OPCODE[opcode]; - if (event < 0) { - /* RESUME fixup */ - event = instr->oparg; - } - CHECK(tools_matrix.tools[event] != 0); - } - if (data->lines && opcode != END_FOR) { - int line1 = compute_line(code, i, data->lines[i].line_delta); - int line2 = PyCode_Addr2Line(code, i*sizeof(_Py_CODEUNIT)); - CHECK(line1 == line2); - } - CHECK(valid_opcode(opcode)); - opcode = _PyOpcode_Deopt[opcode]; - CHECK(valid_opcode(opcode)); - if (data->tools) { - uint8_t local_tools = data->tools[i]; - if (OPCODE_HAS_EVENT[opcode]) { - int event = EVENT_FOR_OPCODE[opcode]; - if (event == -1) { - /* RESUME fixup */ - event = instr->oparg; - } - uint8_t global_tools = tools_matrix.tools[event]; - CHECK((global_tools & local_tools) == local_tools); - } - else { - CHECK(local_tools == 0xff); - } - } - i += _PyOpcode_Caches[opcode]; - } -} - int _Py_GetBaseOpcode(PyCodeObject *code, int offset) { int opcode = _Py_OPCODE(_PyCode_CODE(code)[offset]); - int deinstrumented = DE_INSTRUMENT[opcode]; - if (deinstrumented) { - return deinstrumented; + if (!is_instrumented(opcode)) { + assert(_PyOpcode_Deopt[opcode] != 0); + return _PyOpcode_Deopt[opcode]; } - /* To do -- Handle INSTRUMENTED_INSTRUCTION if (opcode == INSTRUMENTED_INSTRUCTION) { - opcode = code->_co_instrumentation.monitoring_data->per_instruction_opcode[offset]; + opcode = code->_co_instrumentation.monitoring_data->per_instruction_opcodes[offset]; } - */ if (opcode == INSTRUMENTED_LINE) { opcode = code->_co_instrumentation.monitoring_data->lines[offset].original_opcode; - deinstrumented = DE_INSTRUMENT[opcode]; - if (deinstrumented) { - return deinstrumented; - } } - assert(!is_instrumented(opcode)); - assert(_PyOpcode_Deopt[opcode]); + int deinstrumented = DE_INSTRUMENT[opcode]; + if (deinstrumented) { + return deinstrumented; + } + assert(_PyOpcode_Deopt[opcode] != 0); return _PyOpcode_Deopt[opcode]; } static void de_instrument(PyCodeObject *code, int offset, int event) { + assert(event != PY_MONITORING_EVENT_INSTRUCTION); assert(event != PY_MONITORING_EVENT_LINE); _Py_CODEUNIT *instr = &_PyCode_CODE(code)[offset]; int opcode = _Py_OPCODE(*instr); if (!is_instrumented(opcode)) { return; } - int deinstrumented = DE_INSTRUMENT[opcode]; - int base_opcode; - if (deinstrumented) { - base_opcode = deinstrumented; - instr->opcode = base_opcode; - } - else { - /* TO DO -- handle INSTRUMENTED_INSTRUCTION */ - assert(opcode == INSTRUMENTED_LINE); - _PyCoLineInstrumentationData *lines = &code->_co_instrumentation.monitoring_data->lines[offset]; - int orignal_opcode = lines->original_opcode; - if (is_instrumented(orignal_opcode)) { - base_opcode = DE_INSTRUMENT[orignal_opcode]; - lines->original_opcode = DE_INSTRUMENT[orignal_opcode]; - assert(lines->original_opcode != 0); - + switch(opcode) { + case INSTRUMENTED_INSTRUCTION: + opcode = code->_co_instrumentation.monitoring_data->per_instruction_opcodes[offset]; + if (opcode != INSTRUMENTED_LINE) { + int deinstrumented = DE_INSTRUMENT[opcode]; + if (deinstrumented) { + code->_co_instrumentation.monitoring_data->per_instruction_opcodes[offset] = deinstrumented; + } + break; + } + /* Intentional fall through */ + case INSTRUMENTED_LINE: + { + _PyCoLineInstrumentationData *lines = &code->_co_instrumentation.monitoring_data->lines[offset]; + int orignal_opcode = lines->original_opcode; + int deinstrumented = DE_INSTRUMENT[orignal_opcode]; + if (deinstrumented) { + lines->original_opcode = deinstrumented; + } + break; } - else { - base_opcode = orignal_opcode; + default: + { + int deinstrumented = DE_INSTRUMENT[opcode]; + assert(deinstrumented); + instr->opcode = deinstrumented; } } + int base_opcode = _Py_GetBaseOpcode(code, offset); assert(base_opcode != 0); assert(_PyOpcode_Deopt[base_opcode] == base_opcode); if (_PyOpcode_Caches[base_opcode]) { @@ -500,21 +562,58 @@ de_instrument(PyCodeObject *code, int offset, int event) } static void -de_instrument_line(PyCodeObject *code, int offset) +de_instrument_line(PyCodeObject *code, int i) { /* TO DO -- handle INSTRUMENTED_INSTRUCTION */ - _Py_CODEUNIT *instr = &_PyCode_CODE(code)[offset]; + _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; int opcode = _Py_OPCODE(*instr); + if (opcode == INSTRUMENTED_INSTRUCTION) { + opcode = code->_co_instrumentation.monitoring_data->per_instruction_opcodes[i]; + } if (opcode != INSTRUMENTED_LINE) { return; } - _PyCoLineInstrumentationData *lines = &code->_co_instrumentation.monitoring_data->lines[offset]; + _PyCoLineInstrumentationData *lines = &code->_co_instrumentation.monitoring_data->lines[i]; int original_opcode = lines->original_opcode; assert(is_instrumented(original_opcode) || _PyOpcode_Deopt[original_opcode]); if (is_instrumented(original_opcode)) { /* Instrumented original */ instr->opcode = original_opcode; } + else { + original_opcode = _PyOpcode_Deopt[original_opcode]; + assert(original_opcode != 0); + if (_PyOpcode_Caches[original_opcode]) { + instr[1].cache = adaptive_counter_warmup(); + } + } + if (instr->opcode == INSTRUMENTED_INSTRUCTION) { + code->_co_instrumentation.monitoring_data->per_instruction_opcodes[i] = original_opcode; + } + else { + instr->opcode = original_opcode; + } + assert(instr->opcode != INSTRUMENTED_LINE); + /* Mark instruction as candidate for line instrumentation */ + lines->original_opcode = 255; +} + + +static void +de_instrument_per_instruction(PyCodeObject *code, int offset) +{ + /* TO DO -- handle INSTRUMENTED_INSTRUCTION */ + _Py_CODEUNIT *instr = &_PyCode_CODE(code)[offset]; + int opcode = _Py_OPCODE(*instr); + if (opcode != INSTRUMENTED_INSTRUCTION) { + return; + } + int original_opcode = code->_co_instrumentation.monitoring_data->per_instruction_opcodes[offset]; + assert(is_instrumented(original_opcode) || _PyOpcode_Deopt[original_opcode]); + if (is_instrumented(original_opcode)) { + /* Instrumented original */ + instr->opcode = original_opcode; + } else { int base_opcode = _PyOpcode_Deopt[original_opcode]; assert(base_opcode != 0); @@ -523,10 +622,12 @@ de_instrument_line(PyCodeObject *code, int offset) instr[1].cache = adaptive_counter_warmup(); } } - /* Mark instruction as candidate for line instrumentation */ - lines->original_opcode = 255; + assert(instr->opcode != INSTRUMENTED_INSTRUCTION); + /* Keep things clean for snaity check */ + code->_co_instrumentation.monitoring_data->per_instruction_opcodes[offset] = 0; } + static void instrument(PyCodeObject *code, int offset) { @@ -552,19 +653,16 @@ instrument(PyCodeObject *code, int offset) } static void -instrument_line(PyCodeObject *code, int offset) +instrument_line(PyCodeObject *code, int i) { - _Py_CODEUNIT *instr = &_PyCode_CODE(code)[offset]; + _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; int opcode = _Py_OPCODE(*instr); /* TO DO -- handle INSTRUMENTED_INSTRUCTION */ if (opcode == INSTRUMENTED_LINE) { return; } - _PyCoLineInstrumentationData *lines = &code->_co_instrumentation.monitoring_data->lines[offset]; - if (lines->original_opcode != 255) { - dump_instrumentation_data(code, offset, stderr); - } - assert(lines->original_opcode == 255); + _PyCoLineInstrumentationData *lines = &code->_co_instrumentation.monitoring_data->lines[i]; + CHECK(lines->original_opcode == 255); if (is_instrumented(opcode)) { lines->original_opcode = opcode; } @@ -577,7 +675,30 @@ instrument_line(PyCodeObject *code, int offset) } assert(lines->original_opcode > 0); instr->opcode = INSTRUMENTED_LINE; - assert(code->_co_instrumentation.monitoring_data->lines[offset].original_opcode != 0); + assert(code->_co_instrumentation.monitoring_data->lines[i].original_opcode != 0); +} + +static void +instrument_per_instruction(PyCodeObject *code, int i) +{ + _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; + int opcode = instr->opcode; + /* TO DO -- handle INSTRUMENTED_INSTRUCTION */ + if (opcode == INSTRUMENTED_INSTRUCTION) { + return; + } + CHECK(opcode != 0); + if (is_instrumented(opcode)) { + code->_co_instrumentation.monitoring_data->per_instruction_opcodes[i] = opcode; + } + else { + assert(opcode != 0); + assert(_PyOpcode_Deopt[opcode] != 0); + assert(_PyOpcode_Deopt[opcode] != RESUME); + code->_co_instrumentation.monitoring_data->per_instruction_opcodes[i] = _PyOpcode_Deopt[opcode]; + } + assert(code->_co_instrumentation.monitoring_data->per_instruction_opcodes[i] > 0); + instr->opcode = INSTRUMENTED_INSTRUCTION; } #ifndef NDEBUG @@ -586,7 +707,9 @@ instruction_has_event(PyCodeObject *code, int offset) { _Py_CODEUNIT instr = _PyCode_CODE(code)[offset]; int opcode = instr.opcode; - /* TO DO -- Handle INSTRUMENTED_OPCODE */ + if (opcode == INSTRUMENTED_INSTRUCTION) { + opcode = code->_co_instrumentation.monitoring_data->per_instruction_opcodes[offset]; + } if (opcode == INSTRUMENTED_LINE) { opcode = code->_co_instrumentation.monitoring_data->lines[offset].original_opcode; } @@ -678,6 +801,47 @@ add_line_tools(PyCodeObject * code, int offset, int tools) instrument_line(code, offset); } + +static void +add_per_instruction_tools(PyCodeObject * code, int offset, int tools) +{ + assert((PyInterpreterState_Get()->monitoring_matrix.tools[PY_MONITORING_EVENT_INSTRUCTION] & tools) == tools); + assert(code->_co_instrumentation.monitoring_data); + if (code->_co_instrumentation.monitoring_data->per_instruction_tools + ) { + code->_co_instrumentation.monitoring_data->per_instruction_tools[offset] |= tools; + } + else { + /* Single tool */ + assert(_Py_popcount32(tools) == 1); + } + instrument_per_instruction(code, offset); +} + + +static void +remove_per_instruction_tools(PyCodeObject * code, int offset, int tools) +{ + assert(code->_co_instrumentation.monitoring_data); + if (code->_co_instrumentation.monitoring_data->per_instruction_tools) + { + uint8_t *toolsptr = &code->_co_instrumentation.monitoring_data->per_instruction_tools[offset]; + *toolsptr &= ~tools; + if (*toolsptr == 0 ) { + de_instrument_per_instruction(code, offset); + } + } + else { + /* Single tool */ + uint8_t single_tool = code->_co_instrumentation.monitoring_data->matrix.tools[PY_MONITORING_EVENT_INSTRUCTION]; + assert(_Py_popcount32(single_tool) <= 1); + if (((single_tool & tools) == single_tool)) { + de_instrument_per_instruction(code, offset); + } + } +} + + /* Return 1 if DISABLE returned, -1 if error, 0 otherwise */ static int call_one_instrument( @@ -751,8 +915,6 @@ call_instrument(PyThreadState *tstate, PyCodeObject *code, int event, PyInterpreterState *interp = tstate->interp; int offset = instr - _PyCode_CODE(code); uint8_t tools = get_tools(code, offset, event); - /* No per-instruction monitoring yet */ - assert(code->_co_instrumentation.monitoring_data->per_instruction_opcodes == NULL); // assert(tools); while (tools) { int tool = most_significant_bit(tools); @@ -895,11 +1057,6 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, PyInterpreterState *interp = tstate->interp; int8_t line_delta = line_data->line_delta; int line = compute_line(code, offset, line_delta); - if (tstate->trace_info.code == frame->f_code && - tstate->trace_info.line == line) { - /* Already traced this line */ - return original_opcode; - } uint8_t tools = code->_co_instrumentation.monitoring_data->line_tools != NULL ? code->_co_instrumentation.monitoring_data->line_tools[offset] : interp->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE]; @@ -929,14 +1086,62 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, remove_line_tools(code, offset, 1 << tool); } } - tstate->trace_info.code = frame->f_code; - tstate->trace_info.line = line; Py_DECREF(line_obj); assert(original_opcode != 0); assert(_PyOpcode_Deopt[original_opcode] == original_opcode); return original_opcode; } +int +_Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr) +{ + PyCodeObject *code = frame->f_code; + assert(is_instrumentation_up_to_date(code, tstate->interp)); + assert(matrix_equals(code->_co_instrumentation.monitoring_data->matrix, tstate->interp->monitoring_matrix)); + int offset = instr - _PyCode_CODE(code); + _PyCoInstrumentationData *instrumentation_data = code->_co_instrumentation.monitoring_data; + assert(instrumentation_data->per_instruction_opcodes); + int next_opcode = instrumentation_data->per_instruction_opcodes[offset]; + if (tstate->tracing) { + return next_opcode; + } + PyInterpreterState *interp = tstate->interp; + uint8_t tools = instrumentation_data->per_instruction_tools != NULL ? + instrumentation_data->per_instruction_tools[offset] : + interp->monitoring_matrix.tools[PY_MONITORING_EVENT_INSTRUCTION]; + + PyObject *offset_obj = PyLong_FromSsize_t(offset); + if (offset_obj == NULL) { + return -1; + } + PyObject *args[3] = { NULL, (PyObject *)code, offset_obj }; + while (tools) { + int tool = most_significant_bit(tools); + assert(tool < 8); + assert(tools & (1 << tool)); + tools &= ~(1 << tool); + int res = call_one_instrument(interp, tstate, &args[1], + 2 | PY_VECTORCALL_ARGUMENTS_OFFSET, + tool, PY_MONITORING_EVENT_INSTRUCTION); + if (res == 0) { + /* Nothing to do */ + } + else if (res < 0) { + /* error */ + Py_DECREF(offset_obj); + return -1; + } + else { + /* DISABLE */ + remove_line_tools(code, offset, 1 << tool); + } + } + Py_DECREF(offset_obj); + assert(next_opcode != 0); + return next_opcode; +} + + PyObject * _PyMonitoring_RegisterCallback(int tool_id, int event_id, PyObject *obj) { @@ -997,14 +1202,10 @@ initialize_tools(PyCodeObject *code) } } -static bool is_new_line(int index, int current_line, PyCodeAddressRange *bounds, int opcode) -{ - int line = _PyCode_CheckLineNumber(index*(int)sizeof(_Py_CODEUNIT), bounds); - return line >= 0 && line != current_line; -} - #define LINE_MARKER 255 +#define NO_LINE -128 + static void initialize_lines(PyCodeObject *code) { @@ -1021,58 +1222,69 @@ initialize_lines(PyCodeObject *code) int extended_arg = 0; _Py_CODEUNIT *bytecode = _PyCode_CODE(code); for (int i = code->_co_firsttraceable; i < code_len; i++) { - line_data[i].original_opcode = 0; + int line = _PyCode_CheckLineNumber(i*(int)sizeof(_Py_CODEUNIT), &range); + if (line != current_line && line >= 0) { + line_data[i].original_opcode = LINE_MARKER; + } + else { + line_data[i].original_opcode = 0; + } + line_data[i].line_delta = compute_line_delta(code, i, line); + current_line = line; + int opcode = _Py_GetBaseOpcode(code, i); + for (int j = 0; j < _PyOpcode_Caches[opcode]; j++) { + i++; + line_data[i].original_opcode = 0; + line_data[i].line_delta = NO_LINE; + } } for (int i = code->_co_firsttraceable; i < code_len; i++) { int opcode = _Py_GetBaseOpcode(code, i); switch (opcode) { + case END_ASYNC_FOR: case END_FOR: case RESUME: /* END_FOR cannot start a line, as it is skipped by FOR_ITER * RESUME must not be instrumented with INSTRUMENT_LINE */ line_data[i].original_opcode = 0; - line_data[i].line_delta = -127; + line_data[i].line_delta = NO_LINE; extended_arg = 0; continue; - case EXTENDED_ARG: - extended_arg = (extended_arg << 8) + bytecode[i].oparg; - line_data[i].line_delta = -127; - continue; - case JUMP_IF_FALSE_OR_POP: - case JUMP_IF_TRUE_OR_POP: - case POP_JUMP_IF_FALSE: - case POP_JUMP_IF_TRUE: - case JUMP_FORWARD: - /* COMPARE_AND_BRANCH is handled by the following POP_JUMP_IF_FALSE/TRUE */ - { - int offset = (extended_arg << 8) + bytecode[i].oparg; - line_data[i+1+offset].original_opcode = LINE_MARKER; - break; - } - case FOR_ITER: - { - /* Skip over END_FOR */ - int offset = (extended_arg << 8) + bytecode[i].oparg + 1; - line_data[i+1+offset].original_opcode = LINE_MARKER; - break; - } - case JUMP_BACKWARD: - { - int offset = (extended_arg << 8) + bytecode[i].oparg; - line_data[i+1-offset].original_opcode = LINE_MARKER; - break; - } - } - if (is_new_line(i, current_line, &range, opcode)) { - line_data[i].original_opcode = LINE_MARKER; - current_line = range.ar_line; - } - line_data[i].line_delta = compute_line_delta(code, i, range.ar_line); - for (int j = 0; j < _PyOpcode_Caches[opcode]; j++) { - i++; - line_data[i].original_opcode = 0; - line_data[i].line_delta = -128; +// case EXTENDED_ARG: +// extended_arg = (extended_arg << 8) + bytecode[i].oparg; +// continue; +// case JUMP_IF_FALSE_OR_POP: +// case JUMP_IF_TRUE_OR_POP: +// case JUMP_FORWARD: +// /* COMPARE_AND_BRANCH is handled by the following POP_JUMP_IF_FALSE/TRUE */ +// case POP_JUMP_IF_FALSE: +// case POP_JUMP_IF_TRUE: +// { +// int target = i + 1 + (extended_arg << 8) + bytecode[i].oparg; +// if (line_data[target].line_delta != NO_LINE) { +// line_data[target].original_opcode = LINE_MARKER; +// } +// break; +// } +// case FOR_ITER: +// { +// /* Skip over END_FOR */ +// int target = i + 2 + (extended_arg << 8) + bytecode[i].oparg; +// if (line_data[target].line_delta != NO_LINE) { +// line_data[target].original_opcode = LINE_MARKER; +// } +// break; +// } +// case JUMP_BACKWARD: +// { +// int target = i + 1 - (extended_arg << 8) - bytecode[i].oparg; +// if (line_data[target].line_delta != NO_LINE) { +// line_data[target].original_opcode = LINE_MARKER; +// } +// break; +// } } + i+= _PyOpcode_Caches[opcode]; extended_arg = 0; } } @@ -1132,6 +1344,30 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) initialize_line_tools(code, interp); } } + if (interp->monitoring_matrix.tools[PY_MONITORING_EVENT_INSTRUCTION]) { + if (code->_co_instrumentation.monitoring_data->per_instruction_opcodes == NULL) { + code->_co_instrumentation.monitoring_data->per_instruction_opcodes = PyMem_Malloc(code_len * sizeof(_PyCoLineInstrumentationData)); + if (code->_co_instrumentation.monitoring_data->per_instruction_opcodes == NULL) { + PyErr_NoMemory(); + return -1; + } + /* This may not be necessary, as we can initialize this memory lazily, but it helps catch errors. */ + for (int i = 0; i < code_len; i++) { + code->_co_instrumentation.monitoring_data->per_instruction_opcodes[i] = 0; + } + } + if (multitools && code->_co_instrumentation.monitoring_data->per_instruction_tools == NULL) { + code->_co_instrumentation.monitoring_data->per_instruction_tools = PyMem_Malloc(code_len); + if (code->_co_instrumentation.monitoring_data->per_instruction_tools == NULL) { + PyErr_NoMemory(); + return -1; + } + /* This may not be necessary, as we can initialize this memory lazily, but it helps catch errors. */ + for (int i = 0; i < code_len; i++) { + code->_co_instrumentation.monitoring_data->per_instruction_tools[i] = 0; + } + } + } return 0; } @@ -1159,7 +1395,6 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) ); return 0; } - assert(interp->monitoring_matrix.tools[PY_MONITORING_EVENT_INSTRUCTION] == 0); int code_len = (int)Py_SIZE(code); if (update_instrumentation_data(code, interp)) { return -1; @@ -1187,20 +1422,10 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) for (int i = 0; i < code_len; i++) { _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; int opcode = _Py_OPCODE(*instr); - /* TO DO handle INSTRUMENTED_OPCODE */ - if (opcode == INSTRUMENTED_LINE) { - opcode = code->_co_instrumentation.monitoring_data->lines[i].original_opcode; - } - int base_opcode; - if (DE_INSTRUMENT[opcode]) { - base_opcode = _PyOpcode_Deopt[DE_INSTRUMENT[opcode]]; - } - else { - base_opcode = _PyOpcode_Deopt[opcode]; - } if (is_super_instruction(opcode)) { - instr->opcode = base_opcode; + instr->opcode = _PyOpcode_Deopt[opcode]; } + int base_opcode = _Py_GetBaseOpcode(code, i); if (OPCODE_HAS_EVENT[base_opcode]) { int8_t event; if (base_opcode == RESUME) { @@ -1221,18 +1446,18 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) } } /* TO DO -- Use instruction length, not cache count */ - i += _PyOpcode_Caches[base_opcode]; if (base_opcode == COMPARE_AND_BRANCH) { /* Skip over following POP_JUMP_IF */ - assert(instr[2].opcode == POP_JUMP_IF_FALSE || + CHECK(instr[2].opcode == POP_JUMP_IF_FALSE || instr[2].opcode == POP_JUMP_IF_TRUE); i++; } + i += _PyOpcode_Caches[base_opcode]; } uint8_t new_line_tools = new_events.tools[PY_MONITORING_EVENT_LINE]; uint8_t removed_line_tools = removed_events.tools[PY_MONITORING_EVENT_LINE]; - if (new_line_tools || removed_line_tools) { + if (new_line_tools | removed_line_tools) { _PyCoLineInstrumentationData *line_data = code->_co_instrumentation.monitoring_data->lines; for (int i = code->_co_firsttraceable; i < code_len; i++) { if (line_data[i].original_opcode) { @@ -1252,6 +1477,28 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) } } } + uint8_t new_per_instruction_tools = new_events.tools[PY_MONITORING_EVENT_INSTRUCTION]; + uint8_t removed_per_instruction_tools = removed_events.tools[PY_MONITORING_EVENT_INSTRUCTION]; + if (new_per_instruction_tools | removed_per_instruction_tools) { + for (int i = code->_co_firsttraceable; i < code_len; i++) { + int opcode = _Py_GetBaseOpcode(code, i); + if (opcode == RESUME || opcode == END_FOR) { + continue; + } + if (removed_per_instruction_tools) { + remove_per_instruction_tools(code, i, removed_per_instruction_tools); + } + if (new_per_instruction_tools) { + add_per_instruction_tools(code, i, new_per_instruction_tools); + } + /* TO DO -- Use instruction length, not cache count */ + i += _PyOpcode_Caches[opcode]; + if (opcode == COMPARE_AND_BRANCH) { + /* Skip over following POP_JUMP_IF */ + i++; + } + } + } //sanity_check_instrumentation(code); return 0; } @@ -1293,7 +1540,6 @@ _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) matrix_set_bit(&interp->monitoring_matrix, e, tool_id, val); } interp->monitoring_version++; - assert(interp->monitoring_matrix.tools[PY_MONITORING_EVENT_INSTRUCTION] == 0); instrument_all_executing_code_objects(interp); } diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index d232917273f8d0..9d78a910096179 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -146,14 +146,29 @@ sys_trace_func3( } static PyObject * -sys_trace_none_func( +sys_trace_instruction_func( _PyLegacyEventHandler *self, PyObject *const *args, size_t nargsf, PyObject *kwnames ) { assert(kwnames == NULL); Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - assert(nargs == 3); - return call_trace_func(self, Py_None); + assert(nargs == 2); + PyFrameObject* frame = PyEval_GetFrame(); + if (frame == NULL) { + return NULL; + } + if (!frame->f_trace_opcodes) { + Py_RETURN_NONE; + } + Py_INCREF(frame); + PyThreadState *tstate = _PyThreadState_GET(); + int err = tstate->c_tracefunc(tstate->c_traceobj, frame, self->event, Py_None); + frame->f_lineno = 0; + Py_DECREF(frame); + if (err) { + return NULL; + } + Py_RETURN_NONE; } static PyObject * @@ -163,6 +178,8 @@ trace_line(PyThreadState *tstate, _PyInterpreterFrame *iframe, if (line < 0) { Py_RETURN_NONE; } + tstate->trace_info.code = iframe->f_code; + tstate->trace_info.line = line; Py_INCREF(frame); frame->f_lineno = line; int err = tstate->c_tracefunc(tstate->c_traceobj, frame, self->event, Py_None); @@ -188,33 +205,14 @@ sys_trace_line_func( assert(nargs == 2); _PyInterpreterFrame *iframe = _PyEval_GetFrame(); assert(iframe); - PyFrameObject* frame = _PyFrame_GetFrameObject(iframe); - if (frame == NULL) { - return NULL; - } - if (!frame->f_trace_lines) { - Py_RETURN_NONE; - } assert(args[0] == (PyObject *)iframe->f_code); int line = _PyLong_AsInt(args[1]); assert(line >= 0); - return trace_line(tstate, iframe, self, frame, line); -} - -static PyObject * -sys_trace_handled_exception( - _PyLegacyEventHandler *self, PyObject *const *args, - size_t nargsf, PyObject *kwnames -) { - assert(kwnames == NULL); - PyThreadState *tstate = _PyThreadState_GET(); - if (tstate->c_tracefunc == NULL) { + if (tstate->trace_info.code == iframe->f_code && + tstate->trace_info.line == line) { + /* Already traced this line */ Py_RETURN_NONE; } - Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - assert(nargs == 3); - _PyInterpreterFrame *iframe = _PyEval_GetFrame(); - assert(iframe); PyFrameObject* frame = _PyFrame_GetFrameObject(iframe); if (frame == NULL) { return NULL; @@ -222,21 +220,10 @@ sys_trace_handled_exception( if (!frame->f_trace_lines) { Py_RETURN_NONE; } - assert(args[0] == (PyObject *)iframe->f_code); - int offset = _PyLong_AsInt(args[1]); - assert(offset >= 0); - if (_PyCode_CODE(iframe->f_code)[offset].opcode == END_ASYNC_FOR) { - Py_RETURN_NONE; - } - int line = _Py_Instrumentation_GetLine(iframe->f_code, offset); - if (line < 0) { - Py_RETURN_NONE; - } return trace_line(tstate, iframe, self, frame, line); } - static PyObject * sys_trace_jump_func( _PyLegacyEventHandler *self, PyObject *const *args, @@ -254,10 +241,9 @@ sys_trace_jump_func( int to = _PyLong_AsInt(args[2]); assert(to >= 0); /* Forward jumps should have been handled by the line instrumentation. - * We need to handle backward jumps, as the semantics differ. * PEP 669 only generates events when a new line is seen. * Legacy tracing generates events for all backward jumps even on - * the same line */ + * the same line. We handle that case here. */ if (to > from) { /* Forwards jump */ Py_RETURN_NONE; @@ -456,7 +442,7 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) return -1; } if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, - (vectorcallfunc)sys_trace_none_func, PyTrace_OPCODE, + (vectorcallfunc)sys_trace_instruction_func, PyTrace_OPCODE, PY_MONITORING_EVENT_INSTRUCTION, -1)) { return -1; } @@ -469,8 +455,7 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) (1 << PY_MONITORING_EVENT_RAISE) | (1 << PY_MONITORING_EVENT_LINE) | (1 << PY_MONITORING_EVENT_JUMP)| (1 << PY_MONITORING_EVENT_PY_UNWIND) | (1 << PY_MONITORING_EVENT_PY_THROW); - /* TO DO -- opcode events */ - if (tstate->interp->f_opcode_trace_set && false) { + if (tstate->interp->f_opcode_trace_set) { events |= (1 << PY_MONITORING_EVENT_INSTRUCTION); } _PyMonitoring_SetEvents(PY_INSTRUMENT_SYS_TRACE, events); diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index daef17b1569ec1..21f69220f20d10 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -185,6 +185,7 @@ static const struct { [BINARY_OP] = { 2, 1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC }, [SWAP] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [INSTRUMENTED_LINE] = { 0, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, + [INSTRUMENTED_INSTRUCTION] = { 0, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [INSTRUMENTED_JUMP_FORWARD] = { 0, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [INSTRUMENTED_JUMP_BACKWARD] = { 0, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [INSTRUMENTED_JUMP_IF_TRUE_OR_POP] = { 0, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 2750da3f642179..0c4ea0ec2b4a3d 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -237,7 +237,7 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, + &&TARGET_INSTRUMENTED_INSTRUCTION, &&TARGET_INSTRUMENTED_COMPARE_AND_BRANCH, &&TARGET_INSTRUMENTED_RESUME, &&TARGET_INSTRUMENTED_CALL, From f434ec7669e6d973eba93547a2a4e8922a951a74 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 20 Jan 2023 15:45:48 +0000 Subject: [PATCH 017/116] Get sys.settrace tests passing. --- Lib/test/test_monitoring.py | 2 +- Lib/test/test_sys_settrace.py | 1 + Objects/frameobject.c | 14 +- Python/bytecodes.c | 62 +++---- Python/ceval.c | 30 ++++ Python/generated_cases.c.h | 68 +++---- Python/instrumentation.c | 330 ++++++++++++++++------------------ Python/legacy_tracing.c | 65 +++---- Python/opcode_metadata.h | 2 +- 9 files changed, 273 insertions(+), 301 deletions(-) diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 1e5ef85eb4dcab..fabf3ab8ae6d05 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -510,7 +510,7 @@ def test_lines_loop(self): floop() sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) - self.assertEqual(events, [510, 21, 22, 21, 22, 21, 511]) + self.assertEqual(events, [510, 21, 22, 22, 21, 511]) finally: sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 35b1a0b629d406..9079f0c0ceb4c8 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -1705,6 +1705,7 @@ def f(): def g(frame, event, arg): if (event == 'exception'): type, exception, trace = arg + print(arg) self.assertIsInstance(exception, Exception) return g diff --git a/Objects/frameobject.c b/Objects/frameobject.c index dc196846d57ab4..0c15ffb2f8740b 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -693,15 +693,17 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore return -1; } switch(what_event) { - case PY_MONITORING_EVENT_PY_START: - PyErr_Format(PyExc_ValueError, - "can't jump from the 'call' trace event of a new frame"); - return -1; case PY_MONITORING_EVENT_PY_RESUME: - break; + case PY_MONITORING_EVENT_JUMP: + case PY_MONITORING_EVENT_BRANCH: case PY_MONITORING_EVENT_LINE: case PY_MONITORING_EVENT_PY_YIELD: + /* OK */ break; + case PY_MONITORING_EVENT_PY_START: + PyErr_Format(PyExc_ValueError, + "can't jump from the 'call' trace event of a new frame"); + return -1; case PY_MONITORING_EVENT_CALL: case PY_MONITORING_EVENT_C_RETURN: PyErr_SetString(PyExc_ValueError, @@ -712,8 +714,6 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore case PY_MONITORING_EVENT_PY_THROW: case PY_MONITORING_EVENT_RAISE: case PY_MONITORING_EVENT_C_RAISE: - case PY_MONITORING_EVENT_JUMP: - case PY_MONITORING_EVENT_BRANCH: case PY_MONITORING_EVENT_INSTRUCTION: case PY_MONITORING_EVENT_EXCEPTION_HANDLED: PyErr_Format(PyExc_ValueError, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 4b3b50bee3f4c4..17dca31131b5a4 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -776,13 +776,6 @@ dummy_func( tstate, PY_MONITORING_EVENT_PY_YIELD, frame, next_instr-1, val); ERROR_IF(err, error); - if (frame->prev_instr != next_instr-1) { - fprintf(stderr, "Jump has happened\n"); - /* Instrumentation has jumped */ - next_instr = frame->prev_instr; - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH(); - } GO_TO_INSTRUCTION(YIELD_VALUE); } @@ -1891,8 +1884,10 @@ dummy_func( } } - inst(INSTRUMENTED_COMPARE_AND_BRANCH, (unused/2, left, right -- )) { + inst(INSTRUMENTED_COMPARE_AND_BRANCH, ( -- )) { assert((oparg >> 4) <= Py_GE); + PyObject *right = POP(); + PyObject *left = POP(); PyObject *cond = PyObject_RichCompare(left, right, oparg>>4); Py_DECREF(left); Py_DECREF(right); @@ -1900,21 +1895,15 @@ dummy_func( assert(_Py_OPCODE(next_instr[1]) == POP_JUMP_IF_FALSE || _Py_OPCODE(next_instr[1]) == POP_JUMP_IF_TRUE); bool jump_on_true = _Py_OPCODE(next_instr[1]) == POP_JUMP_IF_TRUE; - int offset = _Py_OPARG(next_instr[1]); + int offset = 0; int err = PyObject_IsTrue(cond); + _Py_CODEUNIT *here = next_instr-1; Py_DECREF(cond); ERROR_IF(err < 0, error); - _Py_CODEUNIT *target; if (jump_on_true == (err != 0)) { - target = next_instr + offset + 2; - JUMPBY(offset); - } - else { - target = next_instr + 2; + offset = _Py_OPARG(next_instr[1]); } - err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_BRANCH, frame, next_instr-1, target); - ERROR_IF(err, error); + INSTRUMENTED_JUMP(here, next_instr + 2 + offset, PY_MONITORING_EVENT_BRANCH); } inst(COMPARE_AND_BRANCH_FLOAT, (unused/2, left, right -- )) { @@ -3426,19 +3415,19 @@ dummy_func( } inst(INSTRUMENTED_LINE, ( -- )) { + _Py_CODEUNIT *here = next_instr-1; _PyFrame_SetStackPointer(frame, stack_pointer); int original_opcode = _Py_call_instrumentation_line( - tstate, frame, next_instr-1); - ERROR_IF(original_opcode < 0, error); - next_instr--; - if (frame->prev_instr != next_instr) { - fprintf(stderr, "Jump has happened\n"); - /* Instrumentation has jumped */ - next_instr = frame->prev_instr; - stack_pointer = _PyFrame_GetStackPointer(frame); + tstate, frame, here); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (original_opcode < 0) { + next_instr = here+1; + goto error; + } + next_instr = frame->prev_instr; + if (next_instr != here) { DISPATCH(); } - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); if (_PyOpcode_Caches[original_opcode]) { _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); INCREMENT_ADAPTIVE_COUNTER(cache->counter); @@ -3464,19 +3453,12 @@ dummy_func( // stack effect: ( -- ) inst(INSTRUMENTED_JUMP_FORWARD, ( -- )) { - int err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); - ERROR_IF(err, error); - JUMPBY(oparg); + INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); } // stack effect: ( -- ) inst(INSTRUMENTED_JUMP_BACKWARD, ( -- )) { - assert(oparg < INSTR_OFFSET()); - int err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr-oparg); - ERROR_IF(err, error); - JUMPBY(-oparg); + INSTRUMENTED_JUMP(next_instr-1, next_instr-oparg, PY_MONITORING_EVENT_JUMP); CHECK_EVAL_BREAKER(); } @@ -3487,8 +3469,10 @@ dummy_func( _Py_CODEUNIT *here = next_instr-1; assert(err == 0 || err == 1); _Py_CODEUNIT *target = next_instr + err*oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); err = _Py_call_instrumentation_jump( tstate, PY_MONITORING_EVENT_BRANCH, frame, here, target); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); ERROR_IF(err, error); if (err) { JUMPBY(oparg); @@ -3506,8 +3490,10 @@ dummy_func( _Py_CODEUNIT *here = next_instr-1; assert(err == 0 || err == 1); _Py_CODEUNIT *target = next_instr + err*oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); err = _Py_call_instrumentation_jump( tstate, PY_MONITORING_EVENT_BRANCH, frame, here, target); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); ERROR_IF(err, error); if (err == 0) { JUMPBY(oparg); @@ -3525,8 +3511,10 @@ dummy_func( _Py_CODEUNIT *here = next_instr-1; assert(err == 0 || err == 1); int offset = err*oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); err = _Py_call_instrumentation_jump( tstate, PY_MONITORING_EVENT_BRANCH, frame, here, next_instr + offset); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); ERROR_IF(err, error); JUMPBY(offset); } @@ -3538,8 +3526,10 @@ dummy_func( _Py_CODEUNIT *here = next_instr-1; assert(err == 0 || err == 1); int offset = (1-err)*oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); err = _Py_call_instrumentation_jump( tstate, PY_MONITORING_EVENT_BRANCH, frame, here, next_instr + offset); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); ERROR_IF(err, error); JUMPBY(offset); } diff --git a/Python/ceval.c b/Python/ceval.c index 1e2a08555e3f3a..8716ab82dad5bb 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -217,6 +217,19 @@ _PyEvalFrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame); "cannot access free variable '%s' where it is not associated with a" \ " value in enclosing scope" +#define UPDATE_NEXT_OR_JUMP(offset) \ +do { \ + assert(frame->stacktop >= 0); \ + if (frame->prev_instr != next_instr-1) { \ + next_instr = frame->prev_instr; \ + stack_pointer = _PyFrame_GetStackPointer(frame); \ + } \ + else { \ + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); \ + JUMPBY(offset); \ + } \ + frame->stacktop = -1; \ +} while (0) #ifdef HAVE_ERRNO_H #include @@ -913,6 +926,22 @@ static inline void _Py_LeaveRecursiveCallPy(PyThreadState *tstate) { tstate->py_recursion_remaining++; } +// If a trace function sets a new f_lineno and +// *then* raises, we use the destination when searching +// for an exception handler, displaying the traceback, and so on +#define INSTRUMENTED_JUMP(src, dest, event) \ +do { \ + _PyFrame_SetStackPointer(frame, stack_pointer); \ + int err = _Py_call_instrumentation_jump(tstate, event, frame, src, dest); \ + stack_pointer = _PyFrame_GetStackPointer(frame); \ + frame->stacktop = -1; \ + if (err) { \ + next_instr = (dest)+1; \ + goto error; \ + } \ + next_instr = frame->prev_instr; \ +} while (0); + // GH-89279: Must be a macro to be sure it's inlined by MSVC. #define is_method(stack_pointer, args) (PEEK((args)+2) != NULL) @@ -2234,6 +2263,7 @@ monitor_raise(PyThreadState *tstate, } _PyErr_NormalizeException(tstate, &type, &value, &orig_traceback); traceback = (orig_traceback != NULL) ? orig_traceback : Py_None; + assert(value != NULL && value != Py_None); arg = PyTuple_Pack(3, type, value, traceback); if (arg == NULL) { _PyErr_Restore(tstate, type, value, orig_traceback); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 317485009c8193..fd894d477c9294 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -941,13 +941,6 @@ tstate, PY_MONITORING_EVENT_PY_YIELD, frame, next_instr-1, val); if (err) goto error; - if (frame->prev_instr != next_instr-1) { - fprintf(stderr, "Jump has happened\n"); - /* Instrumentation has jumped */ - next_instr = frame->prev_instr; - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH(); - } GO_TO_INSTRUCTION(YIELD_VALUE); } @@ -2151,33 +2144,25 @@ } TARGET(INSTRUMENTED_COMPARE_AND_BRANCH) { - PyObject *right = PEEK(1); - PyObject *left = PEEK(2); assert((oparg >> 4) <= Py_GE); + PyObject *right = POP(); + PyObject *left = POP(); PyObject *cond = PyObject_RichCompare(left, right, oparg>>4); Py_DECREF(left); Py_DECREF(right); - if (cond == NULL) goto pop_2_error; + if (cond == NULL) goto error; assert(_Py_OPCODE(next_instr[1]) == POP_JUMP_IF_FALSE || _Py_OPCODE(next_instr[1]) == POP_JUMP_IF_TRUE); bool jump_on_true = _Py_OPCODE(next_instr[1]) == POP_JUMP_IF_TRUE; - int offset = _Py_OPARG(next_instr[1]); + int offset = 0; int err = PyObject_IsTrue(cond); + _Py_CODEUNIT *here = next_instr-1; Py_DECREF(cond); - if (err < 0) goto pop_2_error; - _Py_CODEUNIT *target; + if (err < 0) goto error; if (jump_on_true == (err != 0)) { - target = next_instr + offset + 2; - JUMPBY(offset); - } - else { - target = next_instr + 2; + offset = _Py_OPARG(next_instr[1]); } - err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_BRANCH, frame, next_instr-1, target); - if (err) goto pop_2_error; - STACK_SHRINK(2); - JUMPBY(2); + INSTRUMENTED_JUMP(here, next_instr + 2 + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } @@ -3753,19 +3738,19 @@ } TARGET(INSTRUMENTED_LINE) { + _Py_CODEUNIT *here = next_instr-1; _PyFrame_SetStackPointer(frame, stack_pointer); int original_opcode = _Py_call_instrumentation_line( - tstate, frame, next_instr-1); - if (original_opcode < 0) goto error; - next_instr--; - if (frame->prev_instr != next_instr) { - fprintf(stderr, "Jump has happened\n"); - /* Instrumentation has jumped */ - next_instr = frame->prev_instr; - stack_pointer = _PyFrame_GetStackPointer(frame); + tstate, frame, here); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (original_opcode < 0) { + next_instr = here+1; + goto error; + } + next_instr = frame->prev_instr; + if (next_instr != here) { DISPATCH(); } - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); if (_PyOpcode_Caches[original_opcode]) { _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); INCREMENT_ADAPTIVE_COUNTER(cache->counter); @@ -3790,19 +3775,12 @@ } TARGET(INSTRUMENTED_JUMP_FORWARD) { - int err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr+oparg); - if (err) goto error; - JUMPBY(oparg); + INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); DISPATCH(); } TARGET(INSTRUMENTED_JUMP_BACKWARD) { - assert(oparg < INSTR_OFFSET()); - int err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_JUMP, frame, next_instr-1, next_instr-oparg); - if (err) goto error; - JUMPBY(-oparg); + INSTRUMENTED_JUMP(next_instr-1, next_instr-oparg, PY_MONITORING_EVENT_JUMP); CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -3814,8 +3792,10 @@ _Py_CODEUNIT *here = next_instr-1; assert(err == 0 || err == 1); _Py_CODEUNIT *target = next_instr + err*oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); err = _Py_call_instrumentation_jump( tstate, PY_MONITORING_EVENT_BRANCH, frame, here, target); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); if (err) goto error; if (err) { JUMPBY(oparg); @@ -3834,8 +3814,10 @@ _Py_CODEUNIT *here = next_instr-1; assert(err == 0 || err == 1); _Py_CODEUNIT *target = next_instr + err*oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); err = _Py_call_instrumentation_jump( tstate, PY_MONITORING_EVENT_BRANCH, frame, here, target); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); if (err) goto error; if (err == 0) { JUMPBY(oparg); @@ -3854,8 +3836,10 @@ _Py_CODEUNIT *here = next_instr-1; assert(err == 0 || err == 1); int offset = err*oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); err = _Py_call_instrumentation_jump( tstate, PY_MONITORING_EVENT_BRANCH, frame, here, next_instr + offset); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); if (err) goto error; JUMPBY(offset); DISPATCH(); @@ -3868,8 +3852,10 @@ _Py_CODEUNIT *here = next_instr-1; assert(err == 0 || err == 1); int offset = (1-err)*oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); err = _Py_call_instrumentation_jump( tstate, PY_MONITORING_EVENT_BRANCH, frame, here, next_instr + offset); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); if (err) goto error; JUMPBY(offset); DISPATCH(); diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 0ade817707bc48..4e8ed0d00ac2af 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -7,6 +7,7 @@ #include "pycore_opcode.h" #include "pycore_pyerrors.h" +#define INSTRUMENT_DEBUG 1 static PyObject DISABLE = { @@ -126,6 +127,133 @@ static const uint8_t INSTRUMENTED_OPCODES[256] = { [INSTRUMENTED_INSTRUCTION] = INSTRUMENTED_INSTRUCTION, }; +static inline bool +is_instrumented(int opcode) { + assert(opcode != 0); + return INSTRUMENTED_OPCODES[opcode] == opcode; +} + +static inline bool +matrix_equals(_Py_MonitoringMatrix a, _Py_MonitoringMatrix b) +{ + for (int i = 0; i < PY_MONITORING_EVENTS; i++) { + if (a.tools[i] != b.tools[i]) { + return false; + } + } + return true; +} + +static inline _Py_MonitoringMatrix +matrix_sub(_Py_MonitoringMatrix a, _Py_MonitoringMatrix b) +{ + _Py_MonitoringMatrix res; + for (int i = 0; i < PY_MONITORING_EVENTS; i++) { + res.tools[i] = a.tools[i] & ~b.tools[i]; + } + return res; +} + +static inline _Py_MonitoringMatrix +matrix_and(_Py_MonitoringMatrix a, _Py_MonitoringMatrix b) +{ + _Py_MonitoringMatrix res; + for (int i = 0; i < PY_MONITORING_EVENTS; i++) { + res.tools[i] = a.tools[i] & b.tools[i]; + } + return res; +} + +static inline bool +matrix_empty(_Py_MonitoringMatrix m) +{ + for (int i = 0; i < PY_MONITORING_EVENTS; i++) { + if (m.tools[i]) { + return false; + } + } + return true; +} + +static inline int +multiple_tools(_Py_MonitoringMatrix m) +{ + for (int i = 0; i < PY_MONITORING_EVENTS; i++) { + if (_Py_popcount32(m.tools[i]) > 1) { + return true; + } + } + return false; +} + +static inline void +matrix_set_bit(_Py_MonitoringMatrix *m, int event, int tool, int val) +{ + assert(0 <= event && event < PY_MONITORING_EVENTS); + assert(0 <= tool && tool < PY_MONITORING_TOOL_IDS); + assert(val == 0 || val == 1); + m->tools[event] &= ~(1 << tool); + m->tools[event] |= (val << tool); +} + +static inline _PyMonitoringEventSet +matrix_get_events(_Py_MonitoringMatrix *m, int tool_id) +{ + _PyMonitoringEventSet result = 0; + for (int e = 0; e < PY_MONITORING_EVENTS; e++) { + if ((m->tools[e] >> tool_id) & 1) { + result |= (1 << e); + } + } + return result; +} + +/* Line delta. + * 8 bit value. + * if line_delta == -128: + * line = None # represented as -1 + * elif line == -127: + * line = PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT)); + * else: + * line = first_line + (offset >> OFFSET_SHIFT) + line_delta; + */ + +#define OFFSET_SHIFT 4 + +static int8_t +compute_line_delta(PyCodeObject *code, int offset, int line) +{ + if (line < 0) { + return -128; + } + // assert(line >= code->co_firstlineno); + // assert(offset >= code->_co_firsttraceable); + // assert(offset < Py_SIZE(code)); + int delta = line - code->co_firstlineno - (offset >> OFFSET_SHIFT); + if (delta < 128 && delta > -128) { + return delta; + } + return -127; +} + +static int +compute_line(PyCodeObject *code, int offset, int8_t line_delta) +{ + if (line_delta > -127) { + // assert((offset >> OFFSET_SHIFT) + line_delta >= 0); + return code->co_firstlineno + (offset >> OFFSET_SHIFT) + line_delta; + } + if (line_delta == -128) { + return -1; + } + else { + assert(line_delta == -127); + /* Compute from table */ + return PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT)); + } +} + + #ifdef INSTRUMENT_DEBUG static void @@ -367,133 +495,6 @@ sanity_check_instrumentation(PyCodeObject *code) #endif -#define OFFSET_SHIFT 4 - -/* Line delta. - * 8 bit value. - * if line_delta == -128: - * line = None # represented as -1 - * elif line == -127: - * line = PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT)); - * else: - * line = first_line + (offset >> OFFSET_SHIFT) + line_delta; - */ - -static int8_t -compute_line_delta(PyCodeObject *code, int offset, int line) -{ - if (line < 0) { - return -128; - } - // assert(line >= code->co_firstlineno); - // assert(offset >= code->_co_firsttraceable); - // assert(offset < Py_SIZE(code)); - int delta = line - code->co_firstlineno - (offset >> OFFSET_SHIFT); - if (delta < 128 && delta > -128) { - return delta; - } - return -127; -} - -static int -compute_line(PyCodeObject *code, int offset, int8_t line_delta) -{ - if (line_delta > -127) { - // assert((offset >> OFFSET_SHIFT) + line_delta >= 0); - return code->co_firstlineno + (offset >> OFFSET_SHIFT) + line_delta; - } - if (line_delta == -128) { - return -1; - } - else { - assert(line_delta == -127); - /* Compute from table */ - return PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT)); - } -} - - -static inline bool -is_instrumented(int opcode) { - assert(opcode != 0); - return INSTRUMENTED_OPCODES[opcode] == opcode; -} - -static inline bool -matrix_equals(_Py_MonitoringMatrix a, _Py_MonitoringMatrix b) -{ - for (int i = 0; i < PY_MONITORING_EVENTS; i++) { - if (a.tools[i] != b.tools[i]) { - return false; - } - } - return true; -} - -static inline _Py_MonitoringMatrix -matrix_sub(_Py_MonitoringMatrix a, _Py_MonitoringMatrix b) -{ - _Py_MonitoringMatrix res; - for (int i = 0; i < PY_MONITORING_EVENTS; i++) { - res.tools[i] = a.tools[i] & ~b.tools[i]; - } - return res; -} - -static inline _Py_MonitoringMatrix -matrix_and(_Py_MonitoringMatrix a, _Py_MonitoringMatrix b) -{ - _Py_MonitoringMatrix res; - for (int i = 0; i < PY_MONITORING_EVENTS; i++) { - res.tools[i] = a.tools[i] & b.tools[i]; - } - return res; -} - -static inline bool -matrix_empty(_Py_MonitoringMatrix m) -{ - for (int i = 0; i < PY_MONITORING_EVENTS; i++) { - if (m.tools[i]) { - return false; - } - } - return true; -} - -static inline int -multiple_tools(_Py_MonitoringMatrix m) -{ - for (int i = 0; i < PY_MONITORING_EVENTS; i++) { - if (_Py_popcount32(m.tools[i]) > 1) { - return true; - } - } - return false; -} - -static inline void -matrix_set_bit(_Py_MonitoringMatrix *m, int event, int tool, int val) -{ - assert(0 <= event && event < PY_MONITORING_EVENTS); - assert(0 <= tool && tool < PY_MONITORING_TOOL_IDS); - assert(val == 0 || val == 1); - m->tools[event] &= ~(1 << tool); - m->tools[event] |= (val << tool); -} - -static inline _PyMonitoringEventSet -matrix_get_events(_Py_MonitoringMatrix *m, int tool_id) -{ - _PyMonitoringEventSet result = 0; - for (int e = 0; e < PY_MONITORING_EVENTS; e++) { - if ((m->tools[e] >> tool_id) & 1) { - result |= (1 << e); - } - } - return result; -} - int _Py_GetBaseOpcode(PyCodeObject *code, int offset) { int opcode = _Py_OPCODE(_PyCode_CODE(code)[offset]); @@ -675,7 +676,7 @@ instrument_line(PyCodeObject *code, int i) } assert(lines->original_opcode > 0); instr->opcode = INSTRUMENTED_LINE; - assert(code->_co_instrumentation.monitoring_data->lines[i].original_opcode != 0); + CHECK(code->_co_instrumentation.monitoring_data->lines[i].original_opcode != 0); } static void @@ -980,6 +981,7 @@ _Py_call_instrumentation_jump( PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *target ) { + frame->prev_instr = target; PyCodeObject *code = frame->f_code; int from = instr - _PyCode_CODE(code); int to = target - _PyCode_CODE(code); @@ -1044,6 +1046,7 @@ _Py_Instrumentation_GetLine(PyCodeObject *code, int index) int _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr) { + frame->prev_instr = instr; PyCodeObject *code = frame->f_code; assert(is_instrumentation_up_to_date(code, tstate->interp)); assert(matrix_equals(code->_co_instrumentation.monitoring_data->matrix, tstate->interp->monitoring_matrix)); @@ -1219,9 +1222,18 @@ initialize_lines(PyCodeObject *code) line_data[i].line_delta = -127; } int current_line = -1; - int extended_arg = 0; - _Py_CODEUNIT *bytecode = _PyCode_CODE(code); for (int i = code->_co_firsttraceable; i < code_len; i++) { + int opcode = _Py_GetBaseOpcode(code, i); + switch (opcode) { + case END_ASYNC_FOR: + case END_FOR: + case RESUME: + /* END_FOR cannot start a line, as it is skipped by FOR_ITER + * RESUME must not be instrumented with INSTRUMENT_LINE */ + line_data[i].original_opcode = 0; + line_data[i].line_delta = NO_LINE; + continue; + } int line = _PyCode_CheckLineNumber(i*(int)sizeof(_Py_CODEUNIT), &range); if (line != current_line && line >= 0) { line_data[i].original_opcode = LINE_MARKER; @@ -1230,62 +1242,22 @@ initialize_lines(PyCodeObject *code) line_data[i].original_opcode = 0; } line_data[i].line_delta = compute_line_delta(code, i, line); - current_line = line; - int opcode = _Py_GetBaseOpcode(code, i); + if (line >= 0) { + current_line = line; + } for (int j = 0; j < _PyOpcode_Caches[opcode]; j++) { i++; line_data[i].original_opcode = 0; line_data[i].line_delta = NO_LINE; } - } - for (int i = code->_co_firsttraceable; i < code_len; i++) { - int opcode = _Py_GetBaseOpcode(code, i); switch (opcode) { - case END_ASYNC_FOR: - case END_FOR: - case RESUME: - /* END_FOR cannot start a line, as it is skipped by FOR_ITER - * RESUME must not be instrumented with INSTRUMENT_LINE */ - line_data[i].original_opcode = 0; - line_data[i].line_delta = NO_LINE; - extended_arg = 0; - continue; -// case EXTENDED_ARG: -// extended_arg = (extended_arg << 8) + bytecode[i].oparg; -// continue; -// case JUMP_IF_FALSE_OR_POP: -// case JUMP_IF_TRUE_OR_POP: -// case JUMP_FORWARD: -// /* COMPARE_AND_BRANCH is handled by the following POP_JUMP_IF_FALSE/TRUE */ -// case POP_JUMP_IF_FALSE: -// case POP_JUMP_IF_TRUE: -// { -// int target = i + 1 + (extended_arg << 8) + bytecode[i].oparg; -// if (line_data[target].line_delta != NO_LINE) { -// line_data[target].original_opcode = LINE_MARKER; -// } -// break; -// } -// case FOR_ITER: -// { -// /* Skip over END_FOR */ -// int target = i + 2 + (extended_arg << 8) + bytecode[i].oparg; -// if (line_data[target].line_delta != NO_LINE) { -// line_data[target].original_opcode = LINE_MARKER; -// } -// break; -// } -// case JUMP_BACKWARD: -// { -// int target = i + 1 - (extended_arg << 8) - bytecode[i].oparg; -// if (line_data[target].line_delta != NO_LINE) { -// line_data[target].original_opcode = LINE_MARKER; -// } -// break; -// } + case RETURN_VALUE: + case RAISE_VARARGS: + case RERAISE: + /* Blocks of code after these terminators + * should be treated as different lines */ + current_line = -1; } - i+= _PyOpcode_Caches[opcode]; - extended_arg = 0; } } diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index 9d78a910096179..989ce7852b5edb 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -56,8 +56,7 @@ sys_profile_func2( size_t nargsf, PyObject *kwnames ) { assert(kwnames == NULL); - Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - assert(nargs == 2); + assert(PyVectorcall_NARGS(nargsf) == 2); return call_profile_func(self, Py_None); } @@ -67,8 +66,7 @@ sys_profile_func3( size_t nargsf, PyObject *kwnames ) { assert(kwnames == NULL); - Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - assert(nargs == 3); + assert(PyVectorcall_NARGS(nargsf) == 3); return call_profile_func(self, args[2]); } @@ -78,8 +76,7 @@ sys_profile_call_or_return( size_t nargsf, PyObject *kwnames ) { assert(kwnames == NULL); - Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - assert(nargs == 3); + assert(PyVectorcall_NARGS(nargsf) == 3); PyObject *callable = args[2]; if (!PyCFunction_Check(callable) && Py_TYPE(callable) != &PyMethodDescr_Type) { Py_RETURN_NONE; @@ -111,15 +108,10 @@ sys_trace_exception_func( size_t nargsf, PyObject *kwnames ) { assert(kwnames == NULL); - Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - assert(nargs == 3); - _PyErr_StackItem *err_info = _PyErr_GetTopmostException(_PyThreadState_GET()); - PyObject *arg = _PyErr_StackItemToExcInfoTuple(err_info); - if (arg == NULL) { - return NULL; - } + assert(PyVectorcall_NARGS(nargsf) == 3); + PyObject *arg = args[2]; + assert(PyTuple_CheckExact(arg)); PyObject *res = call_trace_func(self, arg); - Py_DECREF(arg); return res; } @@ -129,8 +121,7 @@ sys_trace_func2( size_t nargsf, PyObject *kwnames ) { assert(kwnames == NULL); - Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - assert(nargs == 2); + assert(PyVectorcall_NARGS(nargsf) == 2); return call_trace_func(self, Py_None); } @@ -140,8 +131,7 @@ sys_trace_func3( size_t nargsf, PyObject *kwnames ) { assert(kwnames == NULL); - Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - assert(nargs == 3); + assert(PyVectorcall_NARGS(nargsf) == 3); return call_trace_func(self, args[2]); } @@ -151,8 +141,7 @@ sys_trace_instruction_func( size_t nargsf, PyObject *kwnames ) { assert(kwnames == NULL); - Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - assert(nargs == 2); + assert(PyVectorcall_NARGS(nargsf) == 2); PyFrameObject* frame = PyEval_GetFrame(); if (frame == NULL) { return NULL; @@ -201,8 +190,7 @@ sys_trace_line_func( if (tstate->c_tracefunc == NULL) { Py_RETURN_NONE; } - Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - assert(nargs == 2); + assert(PyVectorcall_NARGS(nargsf) == 2); _PyInterpreterFrame *iframe = _PyEval_GetFrame(); assert(iframe); assert(args[0] == (PyObject *)iframe->f_code); @@ -234,20 +222,11 @@ sys_trace_jump_func( if (tstate->c_tracefunc == NULL) { Py_RETURN_NONE; } - Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - assert(nargs == 3); + assert(PyVectorcall_NARGS(nargsf) == 3); int from = _PyLong_AsInt(args[1]); assert(from >= 0); int to = _PyLong_AsInt(args[2]); assert(to >= 0); - /* Forward jumps should have been handled by the line instrumentation. - * PEP 669 only generates events when a new line is seen. - * Legacy tracing generates events for all backward jumps even on - * the same line. We handle that case here. */ - if (to > from) { - /* Forwards jump */ - Py_RETURN_NONE; - } _PyInterpreterFrame *iframe = _PyEval_GetFrame(); assert(iframe); PyFrameObject* frame = _PyFrame_GetFrameObject(iframe); @@ -258,8 +237,22 @@ sys_trace_jump_func( Py_RETURN_NONE; } /* We can call _Py_Instrumentation_GetLine because we always set - * line events for tracing */ + * line events for tracing */ int to_line = _Py_Instrumentation_GetLine(iframe->f_code, to); + /* Backward jump: Always generate event + * Forward jump: Only generate event if jumping to different line. */ + if (to > from) { + /* Forwards jump */ + if (tstate->trace_info.code == iframe->f_code && + tstate->trace_info.line == to_line) { + /* Already traced this line */ + Py_RETURN_NONE; + } + int from_line = _Py_Instrumentation_GetLine(iframe->f_code, from); + if (from_line == to_line) { + Py_RETURN_NONE; + } + } return trace_line(tstate, iframe, self, frame, to_line); } @@ -438,7 +431,7 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) } if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, (vectorcallfunc)sys_trace_jump_func, PyTrace_LINE, - PY_MONITORING_EVENT_JUMP, -1)) { + PY_MONITORING_EVENT_JUMP, PY_MONITORING_EVENT_BRANCH)) { return -1; } if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, @@ -453,8 +446,8 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) (1 << PY_MONITORING_EVENT_PY_START) | (1 << PY_MONITORING_EVENT_PY_RESUME) | (1 << PY_MONITORING_EVENT_PY_RETURN) | (1 << PY_MONITORING_EVENT_PY_YIELD) | (1 << PY_MONITORING_EVENT_RAISE) | (1 << PY_MONITORING_EVENT_LINE) | - (1 << PY_MONITORING_EVENT_JUMP)| (1 << PY_MONITORING_EVENT_PY_UNWIND) | - (1 << PY_MONITORING_EVENT_PY_THROW); + (1 << PY_MONITORING_EVENT_JUMP) | (1 << PY_MONITORING_EVENT_BRANCH) | + (1 << PY_MONITORING_EVENT_PY_UNWIND) | (1 << PY_MONITORING_EVENT_PY_THROW); if (tstate->interp->f_opcode_trace_set) { events |= (1 << PY_MONITORING_EVENT_INSTRUCTION); } diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index 21f69220f20d10..9ad33884808bf7 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -117,7 +117,7 @@ static const struct { [STORE_ATTR_SLOT] = { 2, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IXC000 }, [COMPARE_OP] = { 2, 1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC }, [COMPARE_AND_BRANCH] = { 2, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC0 }, - [INSTRUMENTED_COMPARE_AND_BRANCH] = { 2, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC0 }, + [INSTRUMENTED_COMPARE_AND_BRANCH] = { 0, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [COMPARE_AND_BRANCH_FLOAT] = { 2, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC0 }, [COMPARE_AND_BRANCH_INT] = { 2, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC0 }, [COMPARE_AND_BRANCH_STR] = { 2, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC0 }, From b680084a913234528c3586c624005e3a39c97fbf Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 20 Jan 2023 16:59:29 +0000 Subject: [PATCH 018/116] Monitor 'internal' StopIteration raises. --- Python/bytecodes.c | 8 +++----- Python/generated_cases.c.h | 8 +++----- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 17dca31131b5a4..a935769db19f29 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -737,9 +737,9 @@ dummy_func( retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v); } if (retval == NULL) { - if (tstate->c_tracefunc != NULL - && _PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) + if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { monitor_raise(tstate, frame, next_instr-1); + } if (_PyGen_FetchStopIterationValue(&retval) == 0) { gen_status = PYGEN_RETURN; } @@ -2311,9 +2311,7 @@ dummy_func( if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { goto error; } - else if (tstate->c_tracefunc != NULL) { - monitor_raise(tstate, frame, next_instr-1); - } + monitor_raise(tstate, frame, next_instr-1); _PyErr_Clear(tstate); } /* iterator ended normally */ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index fd894d477c9294..e08c74a1b7124d 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -901,9 +901,9 @@ retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v); } if (retval == NULL) { - if (tstate->c_tracefunc != NULL - && _PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) + if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { monitor_raise(tstate, frame, next_instr-1); + } if (_PyGen_FetchStopIterationValue(&retval) == 0) { gen_status = PYGEN_RETURN; } @@ -2619,9 +2619,7 @@ if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { goto error; } - else if (tstate->c_tracefunc != NULL) { - monitor_raise(tstate, frame, next_instr-1); - } + monitor_raise(tstate, frame, next_instr-1); _PyErr_Clear(tstate); } /* iterator ended normally */ From e6e7cf1f21567cbe2bb506b83a05913a83589d8f Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 20 Jan 2023 17:14:02 +0000 Subject: [PATCH 019/116] Check for NULLs. --- Python/instrumentation.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 4e8ed0d00ac2af..350e35bd25da6a 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -954,6 +954,9 @@ _Py_call_instrumentation( assert(matrix_equals(code->_co_instrumentation.monitoring_data->matrix, tstate->interp->monitoring_matrix)); int instruction_offset = instr - _PyCode_CODE(code); PyObject *instruction_offset_obj = PyLong_FromSsize_t(instruction_offset); + if (instruction_offset_obj == NULL) { + return -1; + } PyObject *args[3] = { NULL, (PyObject *)code, instruction_offset_obj }; int err = call_instrument(tstate, code, event, &args[1], 2 | PY_VECTORCALL_ARGUMENTS_OFFSET, instr); Py_DECREF(instruction_offset_obj); @@ -970,6 +973,9 @@ _Py_call_instrumentation_arg( assert(matrix_equals(code->_co_instrumentation.monitoring_data->matrix, tstate->interp->monitoring_matrix)); int instruction_offset = instr - _PyCode_CODE(code); PyObject *instruction_offset_obj = PyLong_FromSsize_t(instruction_offset); + if (instruction_offset_obj == NULL) { + return -1; + } PyObject *args[4] = { NULL, (PyObject *)code, instruction_offset_obj, arg }; int err = call_instrument(tstate, code, event, &args[1], 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, instr); Py_DECREF(instruction_offset_obj); From 5bbc83e4d45c32f3f525ea9e44d40539cf620b37 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 20 Jan 2023 17:30:41 +0000 Subject: [PATCH 020/116] Fix up a few tests --- Lib/test/test__opcode.py | 4 ++-- Lib/test/test_code.py | 8 ++++---- Lib/test/test_dis.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Lib/test/test__opcode.py b/Lib/test/test__opcode.py index 4e5860dfbe6ca3..5df05b6f57ded6 100644 --- a/Lib/test/test__opcode.py +++ b/Lib/test/test__opcode.py @@ -20,7 +20,7 @@ def test_stack_effect(self): # All defined opcodes has_arg = dis.hasarg for name, code in filter(lambda item: item[0] not in dis.deoptmap, dis.opmap.items()): - if code >= 240: + if code >= 238: # Opcodes 240 and up are internal instrumentation instructions continue with self.subTest(opname=name): @@ -54,7 +54,7 @@ def test_stack_effect_jump(self): has_exc = dis.hasexc has_jump = dis.hasjabs + dis.hasjrel for name, code in filter(lambda item: item[0] not in dis.deoptmap, dis.opmap.items()): - if code >= 240: + if code >= 238: # Opcodes 240 and up are internal instrumentation instructions continue with self.subTest(opname=name): diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 67ed1694205cd6..bf5a6d39281e04 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -343,13 +343,13 @@ def foo(): pass # assert that opcode 238 is invalid - self.assertEqual(opname[238], '<238>') + self.assertEqual(opname[235], '<235>') - # change first opcode to 0xee (=238) + # change first opcode to 0xeb (=235) foo.__code__ = foo.__code__.replace( - co_code=b'\xee' + foo.__code__.co_code[1:]) + co_code=b'\xeb' + foo.__code__.co_code[1:]) - msg = f"unknown opcode 238" + msg = f"unknown opcode 235" with self.assertRaisesRegex(SystemError, msg): foo() diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 894c412cac7704..bc5ce2bfe9f976 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -895,7 +895,7 @@ def test_widths(self): long_opcodes = set(['JUMP_BACKWARD_NO_INTERRUPT', 'INSTRUMENTED_CALL_FUNCTION_EX']) for opcode, opname in enumerate(dis.opname): - if opname in long_opcodes: + if opname in long_opcodes or opname.startswith("INSTRUMENTED"): continue with self.subTest(opname=opname): width = dis._OPNAME_WIDTH From 7fe9a4354e69be761a9bb0124275389daa53dddb Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 23 Jan 2023 10:42:44 +0000 Subject: [PATCH 021/116] Turn off debugging output by default. --- Python/instrumentation.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 350e35bd25da6a..5ca720f2e4301e 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -7,7 +7,8 @@ #include "pycore_opcode.h" #include "pycore_pyerrors.h" -#define INSTRUMENT_DEBUG 1 +/* Uncomment this to dump debugging output when assertions fail */ +//#define INSTRUMENT_DEBUG 1 static PyObject DISABLE = { From 8b9f996afbf7df561b3b32c78ce22bd85a6878fb Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 26 Jan 2023 13:56:28 +0000 Subject: [PATCH 022/116] Remove debugging printfs --- Objects/frameobject.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 0c15ffb2f8740b..30a083da09025a 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -839,7 +839,6 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore } while (start_stack > best_stack) { if (top_of_stack(start_stack) == Except) { - fprintf(stderr, "Popping exception\n"); /* Pop exception stack as well as the evaluation stack */ PyThreadState *tstate = _PyThreadState_GET(); _PyErr_StackItem *exc_info = tstate->exc_info; @@ -850,13 +849,11 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore Py_XDECREF(value); } else { - fprintf(stderr, "Popping object\n"); PyObject *v = _PyFrame_StackPop(f->f_frame); Py_XDECREF(v); } start_stack = pop_value(start_stack); } - fprintf(stderr, "Jumping to %d\n", best_addr); /* Finally set the new lasti and return OK. */ f->f_lineno = 0; f->f_frame->prev_instr = _PyCode_CODE(f->f_frame->f_code) + best_addr; From 9b026404be3ab31e0fe3ea6468a94e3e64c7019c Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 26 Jan 2023 14:38:31 +0000 Subject: [PATCH 023/116] Avoid refleak. --- Python/legacy_tracing.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index 989ce7852b5edb..f8b6a4d98fe2a9 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -144,6 +144,7 @@ sys_trace_instruction_func( assert(PyVectorcall_NARGS(nargsf) == 2); PyFrameObject* frame = PyEval_GetFrame(); if (frame == NULL) { + /* No frame */ return NULL; } if (!frame->f_trace_opcodes) { @@ -387,15 +388,7 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) return -1; } - int delta = (func != NULL) - (tstate->c_tracefunc != NULL); - tstate->c_tracefunc = func; - PyObject *old_traceobj = tstate->c_traceobj; - tstate->c_traceobj = Py_XNewRef(arg); - /* Flag that tracing or profiling is turned on */ - _PyThreadState_UpdateTracingState(tstate); - /* Setup PEP 669 monitoring callbacks and events. */ - tstate->interp->sys_tracing_threads += delta; assert(tstate->interp->sys_tracing_threads >= 0); if (!tstate->interp->sys_trace_initialized) { tstate->interp->sys_trace_initialized = true; @@ -441,6 +434,14 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) } /* TO DO: Set up callback for PY_MONITORING_EVENT_EXCEPTION_HANDLED */ } + + tstate->c_tracefunc = func; + PyObject *old_traceobj = tstate->c_traceobj; + tstate->c_traceobj = Py_XNewRef(arg); + Py_XDECREF(old_traceobj); + + int delta = (func != NULL) - (tstate->c_tracefunc != NULL); + tstate->interp->sys_tracing_threads += delta; if (tstate->interp->sys_tracing_threads) { uint32_t events = (1 << PY_MONITORING_EVENT_PY_START) | (1 << PY_MONITORING_EVENT_PY_RESUME) | @@ -456,6 +457,6 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) else { _PyMonitoring_SetEvents(PY_INSTRUMENT_SYS_TRACE, 0); } - Py_XDECREF(old_traceobj); + return 0; } From 2cadf32e57228d4a8e84af8bf415630fa5f52ac0 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 26 Jan 2023 16:56:13 +0000 Subject: [PATCH 024/116] Record last traced line on frame object. --- Include/cpython/pystate.h | 8 ------ Include/internal/pycore_frame.h | 1 + Objects/frameobject.c | 1 + Python/legacy_tracing.c | 44 ++++++++++++++------------------- 4 files changed, 20 insertions(+), 34 deletions(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 0c70cbb1b05183..0a0f117b782b8d 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -55,12 +55,6 @@ typedef int (*Py_tracefunc)(PyObject *, PyFrameObject *, int, PyObject *); #define PyTrace_C_RETURN 6 #define PyTrace_OPCODE 7 - -typedef struct { - PyCodeObject *code; // The code object for the line number. May be NULL. - int line; // Only valid if code != NULL. -} PyTraceInfo; - // Internal structure: you should not use it directly, but use public functions // like PyThreadState_EnterTracing() and PyThreadState_LeaveTracing(). typedef struct _PyCFrame { @@ -204,8 +198,6 @@ struct _ts { /* Unique thread state id. */ uint64_t id; - PyTraceInfo trace_info; - _PyStackChunk *datastack_chunk; PyObject **datastack_top; PyObject **datastack_limit; diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index f12b225ebfccf2..76ef105cb4c475 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -19,6 +19,7 @@ struct _frame { struct _PyInterpreterFrame *f_frame; /* points to the frame data */ PyObject *f_trace; /* Trace function */ int f_lineno; /* Current line number. Only valid if non-zero */ + int f_last_traced_line; /* The last line traced for this frame */ char f_trace_lines; /* Emit per-line trace events? */ char f_trace_opcodes; /* Emit per-opcode trace events? */ char f_fast_as_locals; /* Have the fast locals of this frame been converted to a dict? */ diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 30a083da09025a..6104549f0c8454 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1076,6 +1076,7 @@ _PyFrame_New_NoTrack(PyCodeObject *code) f->f_trace_opcodes = 0; f->f_fast_as_locals = 0; f->f_lineno = 0; + f->f_last_traced_line = -1; return f; } diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index f8b6a4d98fe2a9..6ca35d8171ac48 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -168,8 +168,7 @@ trace_line(PyThreadState *tstate, _PyInterpreterFrame *iframe, if (line < 0) { Py_RETURN_NONE; } - tstate->trace_info.code = iframe->f_code; - tstate->trace_info.line = line; + frame ->f_last_traced_line = line; Py_INCREF(frame); frame->f_lineno = line; int err = tstate->c_tracefunc(tstate->c_traceobj, frame, self->event, Py_None); @@ -197,16 +196,12 @@ sys_trace_line_func( assert(args[0] == (PyObject *)iframe->f_code); int line = _PyLong_AsInt(args[1]); assert(line >= 0); - if (tstate->trace_info.code == iframe->f_code && - tstate->trace_info.line == line) { - /* Already traced this line */ - Py_RETURN_NONE; - } PyFrameObject* frame = _PyFrame_GetFrameObject(iframe); if (frame == NULL) { return NULL; } - if (!frame->f_trace_lines) { + if (frame ->f_last_traced_line == line || !frame->f_trace_lines) { + /* Already traced this line */ Py_RETURN_NONE; } return trace_line(tstate, iframe, self, frame, line); @@ -244,8 +239,7 @@ sys_trace_jump_func( * Forward jump: Only generate event if jumping to different line. */ if (to > from) { /* Forwards jump */ - if (tstate->trace_info.code == iframe->f_code && - tstate->trace_info.line == to_line) { + if (frame->f_last_traced_line == to_line) { /* Already traced this line */ Py_RETURN_NONE; } @@ -314,16 +308,7 @@ _PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) if (_PySys_Audit(current_tstate, "sys.setprofile", NULL) < 0) { return -1; } - int delta = (func != NULL) - (tstate->c_profilefunc != NULL); - tstate->c_profilefunc = func; - PyObject *old_profileobj = tstate->c_profileobj; - tstate->c_profileobj = Py_XNewRef(arg); - /* Flag that tracing or profiling is turned on */ - _PyThreadState_UpdateTracingState(tstate); - /* Setup PEP 669 monitoring callbacks and events. */ - tstate->interp->sys_profiling_threads += delta; - assert(tstate->interp->sys_profiling_threads >= 0); if (!tstate->interp->sys_profile_initialized) { tstate->interp->sys_profile_initialized = true; if (set_callbacks(PY_INSTRUMENT_SYS_PROFILE, @@ -357,7 +342,16 @@ _PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) return -1; } } - if (tstate->interp->sys_profiling_threads && delta) { + + int delta = (func != NULL) - (tstate->c_profilefunc != NULL); + tstate->c_profilefunc = func; + PyObject *old_profileobj = tstate->c_profileobj; + tstate->c_profileobj = Py_XNewRef(arg); + Py_XDECREF(old_profileobj); + tstate->interp->sys_profiling_threads += delta; + assert(tstate->interp->sys_profiling_threads >= 0); + + if (tstate->interp->sys_profiling_threads) { uint32_t events = (1 << PY_MONITORING_EVENT_PY_START) | (1 << PY_MONITORING_EVENT_PY_RESUME) | (1 << PY_MONITORING_EVENT_PY_RETURN) | (1 << PY_MONITORING_EVENT_PY_YIELD) | @@ -368,9 +362,6 @@ _PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) else if (tstate->interp->sys_profiling_threads == 0) { _PyMonitoring_SetEvents(PY_INSTRUMENT_SYS_PROFILE, 0); } - // gh-98257: Only call Py_XDECREF() once the new profile function is fully - // set, so it's safe to call sys.setprofile() again (reentrant call). - Py_XDECREF(old_profileobj); return 0; } @@ -388,8 +379,8 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) return -1; } - /* Setup PEP 669 monitoring callbacks and events. */ assert(tstate->interp->sys_tracing_threads >= 0); + /* Setup PEP 669 monitoring callbacks and events. */ if (!tstate->interp->sys_trace_initialized) { tstate->interp->sys_trace_initialized = true; if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, @@ -435,13 +426,14 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) /* TO DO: Set up callback for PY_MONITORING_EVENT_EXCEPTION_HANDLED */ } + int delta = (func != NULL) - (tstate->c_tracefunc != NULL); tstate->c_tracefunc = func; PyObject *old_traceobj = tstate->c_traceobj; tstate->c_traceobj = Py_XNewRef(arg); Py_XDECREF(old_traceobj); - - int delta = (func != NULL) - (tstate->c_tracefunc != NULL); tstate->interp->sys_tracing_threads += delta; + assert(tstate->interp->sys_tracing_threads >= 0); + if (tstate->interp->sys_tracing_threads) { uint32_t events = (1 << PY_MONITORING_EVENT_PY_START) | (1 << PY_MONITORING_EVENT_PY_RESUME) | From 1f54d7721aab4b968db9607cbfa4ba24a27d9db5 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 30 Jan 2023 10:09:10 +0000 Subject: [PATCH 025/116] Get a couple more top-level tests passing. --- Lib/test/test_sys.py | 2 +- Objects/codeobject.c | 18 ++++++++ Python/bytecodes.c | 1 - Python/generated_cases.c.h | 1 - Python/instrumentation.c | 87 +++++++++++++++++++++----------------- Python/legacy_tracing.c | 33 +++++++-------- 6 files changed, 84 insertions(+), 58 deletions(-) diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index ab1a0659471857..52f4cc886dfd80 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1443,7 +1443,7 @@ class C(object): pass def func(): return sys._getframe() x = func() - check(x, size('3Pi3c7P2ic??2P')) + check(x, size('3Pii3c7P2ic??2P')) # function def func(): pass check(func, size('14Pi')) diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 535a521e3170b0..1d23689bdcac7c 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1703,6 +1703,24 @@ code_dealloc(PyCodeObject *co) if (co->co_weakreflist != NULL) { PyObject_ClearWeakRefs((PyObject*)co); } + _PyCoInstrumentationData *data = co->_co_instrumentation.monitoring_data; + if (data) { + if (data->tools) { + PyMem_Free(data->tools); + } + if (data->lines) { + PyMem_Free(data->lines); + } + if (data->line_tools) { + PyMem_Free(data->line_tools); + } + if (data->per_instruction_opcodes) { + PyMem_Free(data->per_instruction_opcodes); + } + if (data->per_instruction_tools) { + PyMem_Free(data->per_instruction_tools); + } + } if (co->_co_linearray) { PyMem_Free(co->_co_linearray); } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index a935769db19f29..a4566f574f393f 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3430,7 +3430,6 @@ dummy_func( _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); INCREMENT_ADAPTIVE_COUNTER(cache->counter); } - assert(original_opcode > 0 && original_opcode < 256); opcode = original_opcode; DISPATCH_GOTO(); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index e08c74a1b7124d..c6fff9076bfc14 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3753,7 +3753,6 @@ _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); INCREMENT_ADAPTIVE_COUNTER(cache->counter); } - assert(original_opcode > 0 && original_opcode < 256); opcode = original_opcode; DISPATCH_GOTO(); } diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 5ca720f2e4301e..2bc7e9ee6beb51 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -8,7 +8,7 @@ #include "pycore_pyerrors.h" /* Uncomment this to dump debugging output when assertions fail */ -//#define INSTRUMENT_DEBUG 1 +#define INSTRUMENT_DEBUG 1 static PyObject DISABLE = { @@ -889,35 +889,32 @@ static inline int most_significant_bit(uint8_t bits) { } static inline uint8_t -get_tools(PyCodeObject * code, int index, int event) +get_tools_for_instruction(PyCodeObject * code, int i, int event) { uint8_t tools; assert(event != PY_MONITORING_EVENT_LINE); assert(event != PY_MONITORING_EVENT_INSTRUCTION); _PyCoInstrumentationData *monitoring = code->_co_instrumentation.monitoring_data; - if (monitoring && monitoring->tools) { - tools = monitoring->tools[index]; + if (event < PY_MONITORING_INSTRUMENTED_EVENTS && monitoring && monitoring->tools) { + tools = monitoring->tools[i]; } else { tools = code->_co_instrumentation.monitoring_data->matrix.tools[event]; } - assert((tools & PyInterpreterState_Get()->monitoring_matrix.tools[event]) == tools); - assert((tools & code->_co_instrumentation.monitoring_data->matrix.tools[event]) == tools); + CHECK((tools & PyInterpreterState_Get()->monitoring_matrix.tools[event]) == tools); + CHECK((tools & code->_co_instrumentation.monitoring_data->matrix.tools[event]) == tools); return tools; } static int call_instrument(PyThreadState *tstate, PyCodeObject *code, int event, - PyObject **args, Py_ssize_t nargsf, _Py_CODEUNIT *instr) + PyObject **args, Py_ssize_t nargsf, int offset, uint8_t tools) { //sanity_check_instrumentation(code); if (tstate->tracing) { return 0; } PyInterpreterState *interp = tstate->interp; - int offset = instr - _PyCode_CODE(code); - uint8_t tools = get_tools(code, offset, event); - // assert(tools); while (tools) { int tool = most_significant_bit(tools); assert(tool >= 0 && tool < 8); @@ -953,14 +950,15 @@ _Py_call_instrumentation( PyCodeObject *code = frame->f_code; assert(is_instrumentation_up_to_date(code, tstate->interp)); assert(matrix_equals(code->_co_instrumentation.monitoring_data->matrix, tstate->interp->monitoring_matrix)); - int instruction_offset = instr - _PyCode_CODE(code); - PyObject *instruction_offset_obj = PyLong_FromSsize_t(instruction_offset); - if (instruction_offset_obj == NULL) { + int offset = instr - _PyCode_CODE(code); + PyObject *offset_obj = PyLong_FromSsize_t(offset); + if (offset_obj == NULL) { return -1; } - PyObject *args[3] = { NULL, (PyObject *)code, instruction_offset_obj }; - int err = call_instrument(tstate, code, event, &args[1], 2 | PY_VECTORCALL_ARGUMENTS_OFFSET, instr); - Py_DECREF(instruction_offset_obj); + uint8_t tools = get_tools_for_instruction(code, offset, event); + PyObject *args[3] = { NULL, (PyObject *)code, offset_obj }; + int err = call_instrument(tstate, code, event, &args[1], 2 | PY_VECTORCALL_ARGUMENTS_OFFSET, offset, tools); + Py_DECREF(offset_obj); return err; } @@ -972,14 +970,15 @@ _Py_call_instrumentation_arg( PyCodeObject *code = frame->f_code; assert(is_instrumentation_up_to_date(code, tstate->interp)); assert(matrix_equals(code->_co_instrumentation.monitoring_data->matrix, tstate->interp->monitoring_matrix)); - int instruction_offset = instr - _PyCode_CODE(code); - PyObject *instruction_offset_obj = PyLong_FromSsize_t(instruction_offset); - if (instruction_offset_obj == NULL) { + int offset = instr - _PyCode_CODE(code); + PyObject *offset_obj = PyLong_FromSsize_t(offset); + if (offset_obj == NULL) { return -1; } - PyObject *args[4] = { NULL, (PyObject *)code, instruction_offset_obj, arg }; - int err = call_instrument(tstate, code, event, &args[1], 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, instr); - Py_DECREF(instruction_offset_obj); + uint8_t tools = get_tools_for_instruction(code, offset, event); + PyObject *args[4] = { NULL, (PyObject *)code, offset_obj, arg }; + int err = call_instrument(tstate, code, event, &args[1], 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, offset, tools); + Py_DECREF(offset_obj); return err; } @@ -988,21 +987,24 @@ _Py_call_instrumentation_jump( PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *target ) { + assert(event == PY_MONITORING_EVENT_JUMP || + event == PY_MONITORING_EVENT_BRANCH); frame->prev_instr = target; PyCodeObject *code = frame->f_code; - int from = instr - _PyCode_CODE(code); + int offset = instr - _PyCode_CODE(code); int to = target - _PyCode_CODE(code); PyObject *to_obj = PyLong_FromLong(to); if (to_obj == NULL) { return -1; } - PyObject *from_obj = PyLong_FromLong(from); + PyObject *from_obj = PyLong_FromLong(offset); if (from_obj == NULL) { Py_DECREF(to_obj); return -1; } + uint8_t tools = get_tools_for_instruction(code, offset, event); PyObject *args[4] = { NULL, (PyObject *)code, from_obj, to_obj }; - int err = call_instrument(tstate, code, event, &args[1], 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, instr); + int err = call_instrument(tstate, code, event, &args[1], 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, offset, tools); Py_DECREF(to_obj); Py_DECREF(from_obj); return err; @@ -1019,11 +1021,19 @@ _Py_call_instrumentation_exc( PyCodeObject *code = frame->f_code; assert(code->_co_instrumentation.monitoring_version == tstate->interp->monitoring_version); assert(matrix_equals(code->_co_instrumentation.monitoring_data->matrix, tstate->interp->monitoring_matrix)); - int instruction_offset = instr - _PyCode_CODE(code); - PyObject *instruction_offset_obj = PyLong_FromSsize_t(instruction_offset); - PyObject *args[4] = { NULL, (PyObject *)code, instruction_offset_obj, arg }; - Py_ssize_t nargsf = (2 | PY_VECTORCALL_ARGUMENTS_OFFSET) + (arg != NULL); - int err = call_instrument(tstate, code, event, &args[1], nargsf, instr); + int offset = instr - _PyCode_CODE(code); + PyObject *offset_obj = PyLong_FromSsize_t(offset); + int err; + if (offset_obj == NULL) { + err = -1; + } + else { + PyObject *args[4] = { NULL, (PyObject *)code, offset_obj, arg }; + Py_ssize_t nargsf = (2 | PY_VECTORCALL_ARGUMENTS_OFFSET) + (arg != NULL); + uint8_t tools = code->_co_instrumentation.monitoring_data->matrix.tools[event]; + err = call_instrument(tstate, code, event, &args[1], nargsf, offset, tools); + Py_DECREF(offset_obj); + } if (err) { Py_XDECREF(type); Py_XDECREF(value); @@ -1033,7 +1043,6 @@ _Py_call_instrumentation_exc( _PyErr_Restore(tstate, type, value, traceback); } assert(_PyErr_Occurred(tstate)); - Py_DECREF(instruction_offset_obj); } int @@ -1057,18 +1066,18 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, PyCodeObject *code = frame->f_code; assert(is_instrumentation_up_to_date(code, tstate->interp)); assert(matrix_equals(code->_co_instrumentation.monitoring_data->matrix, tstate->interp->monitoring_matrix)); - int offset = instr - _PyCode_CODE(code); + int i = instr - _PyCode_CODE(code); _PyCoInstrumentation *instrumentation = &code->_co_instrumentation; - _PyCoLineInstrumentationData *line_data = &instrumentation->monitoring_data->lines[offset]; + _PyCoLineInstrumentationData *line_data = &instrumentation->monitoring_data->lines[i]; uint8_t original_opcode = line_data->original_opcode; if (tstate->tracing) { - return original_opcode; + goto done; } PyInterpreterState *interp = tstate->interp; int8_t line_delta = line_data->line_delta; - int line = compute_line(code, offset, line_delta); + int line = compute_line(code, i, line_delta); uint8_t tools = code->_co_instrumentation.monitoring_data->line_tools != NULL ? - code->_co_instrumentation.monitoring_data->line_tools[offset] : + code->_co_instrumentation.monitoring_data->line_tools[i] : interp->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE]; PyObject *line_obj = PyLong_FromSsize_t(line); if (line_obj == NULL) { @@ -1093,11 +1102,13 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, } else { /* DISABLE */ - remove_line_tools(code, offset, 1 << tool); + remove_line_tools(code, i, 1 << tool); } } Py_DECREF(line_obj); +done: assert(original_opcode != 0); + assert(original_opcode < INSTRUMENTED_LINE); assert(_PyOpcode_Deopt[original_opcode] == original_opcode); return original_opcode; } @@ -1143,7 +1154,7 @@ _Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame* } else { /* DISABLE */ - remove_line_tools(code, offset, 1 << tool); + remove_per_instruction_tools(code, offset, 1 << tool); } } Py_DECREF(offset_obj); diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index 6ca35d8171ac48..0f176c0b1b0e74 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -162,8 +162,9 @@ sys_trace_instruction_func( } static PyObject * -trace_line(PyThreadState *tstate, _PyInterpreterFrame *iframe, - _PyLegacyEventHandler *self, PyFrameObject* frame, int line +trace_line( + PyThreadState *tstate, _PyLegacyEventHandler *self, + PyFrameObject* frame, int line ) { if (line < 0) { Py_RETURN_NONE; @@ -191,20 +192,21 @@ sys_trace_line_func( Py_RETURN_NONE; } assert(PyVectorcall_NARGS(nargsf) == 2); - _PyInterpreterFrame *iframe = _PyEval_GetFrame(); - assert(iframe); - assert(args[0] == (PyObject *)iframe->f_code); int line = _PyLong_AsInt(args[1]); assert(line >= 0); - PyFrameObject* frame = _PyFrame_GetFrameObject(iframe); + PyFrameObject* frame = PyEval_GetFrame(); if (frame == NULL) { return NULL; } - if (frame ->f_last_traced_line == line || !frame->f_trace_lines) { + assert(args[0] == (PyObject *)frame->f_frame->f_code); + if (!frame->f_trace_lines) { + Py_RETURN_NONE; + } + if (frame ->f_last_traced_line == line) { /* Already traced this line */ Py_RETURN_NONE; } - return trace_line(tstate, iframe, self, frame, line); + return trace_line(tstate, self, frame, line); } @@ -223,18 +225,19 @@ sys_trace_jump_func( assert(from >= 0); int to = _PyLong_AsInt(args[2]); assert(to >= 0); - _PyInterpreterFrame *iframe = _PyEval_GetFrame(); - assert(iframe); - PyFrameObject* frame = _PyFrame_GetFrameObject(iframe); + PyFrameObject* frame = PyEval_GetFrame(); if (frame == NULL) { return NULL; } if (!frame->f_trace_lines) { Py_RETURN_NONE; } + PyCodeObject *code = (PyCodeObject *)args[0]; + assert(PyCode_Check(code)); + assert(code == frame->f_frame->f_code); /* We can call _Py_Instrumentation_GetLine because we always set * line events for tracing */ - int to_line = _Py_Instrumentation_GetLine(iframe->f_code, to); + int to_line = _Py_Instrumentation_GetLine(code, to); /* Backward jump: Always generate event * Forward jump: Only generate event if jumping to different line. */ if (to > from) { @@ -243,12 +246,8 @@ sys_trace_jump_func( /* Already traced this line */ Py_RETURN_NONE; } - int from_line = _Py_Instrumentation_GetLine(iframe->f_code, from); - if (from_line == to_line) { - Py_RETURN_NONE; - } } - return trace_line(tstate, iframe, self, frame, to_line); + return trace_line(tstate, self, frame, to_line); } From 43a3f3eeed96b1f5612c12d4cfb11efa34f1a109 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 30 Jan 2023 15:41:04 +0000 Subject: [PATCH 026/116] Update magic number --- Lib/importlib/_bootstrap_external.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 5a8cbb98ed3f56..208e37c657c8fc 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -431,6 +431,8 @@ def _write_atomic(path, data, mode=0o666): # Python 3.12a1 3515 (Embed jump mask in COMPARE_OP oparg) # Python 3.12a1 3516 (Add COMAPRE_AND_BRANCH instruction) +# Python 3.12a? 3530 (Add instrumentation support) + # Python 3.13 will start with 3550 # MAGIC must change whenever the bytecode emitted by the compiler may no @@ -442,7 +444,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3516).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3530).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c From 3d436cfa91f57b1d0a96bb2c379532bf7958ac1e Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 30 Jan 2023 15:41:31 +0000 Subject: [PATCH 027/116] Remove debug print statement. --- Lib/profile.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/profile.py b/Lib/profile.py index 747ae117dae437..43a7d5ed5f6516 100755 --- a/Lib/profile.py +++ b/Lib/profile.py @@ -187,7 +187,6 @@ def trace_dispatch(self, frame, event, arg): if event == "c_call": self.c_func_name = arg.__name__ - print(event, file = sys.stderr) if self.dispatch[event](self, frame,t): t = timer() self.t = t[0] + t[1] From 691bcf51b368abf2ba4342e16ed74e572844e835 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 30 Jan 2023 15:41:56 +0000 Subject: [PATCH 028/116] Raise SystemError if frame is missing. --- Python/legacy_tracing.c | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index 0f176c0b1b0e74..22af735304df4a 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -16,7 +16,6 @@ typedef struct _PyLegacyEventHandler { PyObject_HEAD vectorcallfunc vectorcall; int event; - PyCodeObject *last_code; } _PyLegacyEventHandler; static void @@ -40,8 +39,12 @@ call_profile_func(_PyLegacyEventHandler *self, PyObject *arg) Py_RETURN_NONE; } PyFrameObject* frame = PyEval_GetFrame(); + if (frame == NULL) { + PyErr_SetString(PyExc_SystemError, + "Missing frame when calling profile function."); + return NULL; + } Py_INCREF(frame); - assert(frame != NULL); int err = tstate->c_profilefunc(tstate->c_profileobj, frame, self->event, arg); Py_DECREF(frame); if (err) { @@ -92,8 +95,12 @@ call_trace_func(_PyLegacyEventHandler *self, PyObject *arg) Py_RETURN_NONE; } PyFrameObject* frame = PyEval_GetFrame(); + if (frame == NULL) { + PyErr_SetString(PyExc_SystemError, + "Missing frame when calling trace function."); + return NULL; + } Py_INCREF(frame); - assert(frame != NULL); int err = tstate->c_tracefunc(tstate->c_traceobj, frame, self->event, arg); Py_DECREF(frame); if (err) { @@ -144,7 +151,8 @@ sys_trace_instruction_func( assert(PyVectorcall_NARGS(nargsf) == 2); PyFrameObject* frame = PyEval_GetFrame(); if (frame == NULL) { - /* No frame */ + PyErr_SetString(PyExc_SystemError, + "Missing frame when calling trace function."); return NULL; } if (!frame->f_trace_opcodes) { @@ -196,6 +204,8 @@ sys_trace_line_func( assert(line >= 0); PyFrameObject* frame = PyEval_GetFrame(); if (frame == NULL) { + PyErr_SetString(PyExc_SystemError, + "Missing frame when calling trace function."); return NULL; } assert(args[0] == (PyObject *)frame->f_frame->f_code); @@ -227,6 +237,8 @@ sys_trace_jump_func( assert(to >= 0); PyFrameObject* frame = PyEval_GetFrame(); if (frame == NULL) { + PyErr_SetString(PyExc_SystemError, + "Missing frame when calling trace function."); return NULL; } if (!frame->f_trace_lines) { @@ -254,7 +266,7 @@ sys_trace_jump_func( PyTypeObject _PyLegacyEventHandler_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) - "sys.profile_event_handler", + "sys.legacy_event_handler", sizeof(_PyLegacyEventHandler), .tp_dealloc = (destructor)dealloc, .tp_vectorcall_offset = offsetof(_PyLegacyEventHandler, vectorcall), @@ -358,7 +370,7 @@ _PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) (1 << PY_MONITORING_EVENT_C_RETURN) | (1 << PY_MONITORING_EVENT_C_RAISE); _PyMonitoring_SetEvents(PY_INSTRUMENT_SYS_PROFILE, events); } - else if (tstate->interp->sys_profiling_threads == 0) { + else { _PyMonitoring_SetEvents(PY_INSTRUMENT_SYS_PROFILE, 0); } return 0; From f07be070fa4970cf4eaa8883ce08ca24a3f8f782 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 31 Jan 2023 16:36:09 +0000 Subject: [PATCH 029/116] Restore a few tests and better handle EXTENDED_ARG. --- Include/cpython/pystate.h | 1 - Lib/profile.py | 3 +- Lib/test/test_monitoring.py | 2 +- Lib/test/test_sys_settrace.py | 13 ++- Python/instrumentation.c | 191 ++++++++++++++++++++++++---------- Python/pystate.c | 1 - 6 files changed, 145 insertions(+), 66 deletions(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 0a0f117b782b8d..f8ec16e2b37282 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -124,7 +124,6 @@ struct _ts { the trace/profile. */ int tracing; int what_event; /* The event currently being monitored, if any. */ - uint8_t monitoring; /* Pointer to current _PyCFrame in the C stack frame of the currently, * or most recently, executing _PyEval_EvalFrameDefault. */ diff --git a/Lib/profile.py b/Lib/profile.py index 43a7d5ed5f6516..453e56285c510c 100755 --- a/Lib/profile.py +++ b/Lib/profile.py @@ -187,6 +187,7 @@ def trace_dispatch(self, frame, event, arg): if event == "c_call": self.c_func_name = arg.__name__ + if self.dispatch[event](self, frame,t): t = timer() self.t = t[0] + t[1] @@ -288,7 +289,7 @@ def trace_dispatch_c_call (self, frame, t): def trace_dispatch_return(self, frame, t): if frame is not self.cur[-2]: - assert frame is self.cur[-2].f_back, ("Bad return", self.cur[-3], t) + assert frame is self.cur[-2].f_back, ("Bad return", self.cur[-3]) self.trace_dispatch_return(self.cur[-2], 0) # Prefix "r" means part of the Returning or exiting frame. diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index fabf3ab8ae6d05..1388a745b16992 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -510,7 +510,7 @@ def test_lines_loop(self): floop() sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) - self.assertEqual(events, [510, 21, 22, 22, 21, 511]) + self.assertEqual(events, [510, 21, 22, 22, 511]) finally: sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 9079f0c0ceb4c8..a251b2272e95eb 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -1705,7 +1705,6 @@ def f(): def g(frame, event, arg): if (event == 'exception'): type, exception, trace = arg - print(arg) self.assertIsInstance(exception, Exception) return g @@ -1975,12 +1974,12 @@ def test_no_jump_over_return_try_finally_in_finally_block(output): pass output.append(12) - #@jump_test(3, 4, [1], (ValueError, 'after')) - #def test_no_jump_infinite_while_loop(output): - #output.append(1) - #while True: - #output.append(3) - #output.append(4) + @jump_test(3, 4, [1], (ValueError, 'after')) + def test_no_jump_infinite_while_loop(output): + output.append(1) + while True: + output.append(3) + output.append(4) @jump_test(2, 4, [4, 4]) def test_jump_forwards_into_while_block(output): diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 2bc7e9ee6beb51..e92f1405687bcb 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -6,6 +6,7 @@ #include "pycore_namespace.h" #include "pycore_opcode.h" #include "pycore_pyerrors.h" +#include "pycore_pystate.h" /* Uncomment this to dump debugging output when assertions fail */ #define INSTRUMENT_DEBUG 1 @@ -254,6 +255,97 @@ compute_line(PyCodeObject *code, int offset, int8_t line_delta) } } +/* We need to parse instructions from code objects a lot, + * so add a couple of utilities */ +int _Py_GetBaseOpcode(PyCodeObject *code, int offset) +{ + int opcode = _Py_OPCODE(_PyCode_CODE(code)[offset]); + if (!is_instrumented(opcode)) { + assert(_PyOpcode_Deopt[opcode] != 0); + return _PyOpcode_Deopt[opcode]; + } + if (opcode == INSTRUMENTED_INSTRUCTION) { + opcode = code->_co_instrumentation.monitoring_data->per_instruction_opcodes[offset]; + } + if (opcode == INSTRUMENTED_LINE) { + opcode = code->_co_instrumentation.monitoring_data->lines[offset].original_opcode; + } + int deinstrumented = DE_INSTRUMENT[opcode]; + if (deinstrumented) { + return deinstrumented; + } + assert(_PyOpcode_Deopt[opcode] != 0); + return _PyOpcode_Deopt[opcode]; +} + +uint8_t deinstrument(PyCodeObject *code, int offset) +{ + int opcode = _Py_OPCODE(_PyCode_CODE(code)[offset]); + if (!is_instrumented(opcode)) { + assert(_PyOpcode_Deopt[opcode] != 0); + return _PyOpcode_Deopt[opcode]; + } + if (opcode == INSTRUMENTED_INSTRUCTION) { + opcode = code->_co_instrumentation.monitoring_data->per_instruction_opcodes[offset]; + } + if (opcode == INSTRUMENTED_LINE) { + opcode = code->_co_instrumentation.monitoring_data->lines[offset].original_opcode; + } + int deinstrumented = DE_INSTRUMENT[opcode]; + if (deinstrumented) { + return deinstrumented; + } + return opcode; +} + +typedef struct _Instruction { + uint8_t extended_args; + uint8_t deinstrumented_opcode; + uint8_t actual_opcode; + uint8_t length; + /* Whether a prefix instrument like INSTRUMENTED_INSTRUCTION is attached */ + bool is_prefix_instrumented; + /* Whether a specialized instruction like INSTRUMENTED_RETURN_VALUE is used */ + bool is_specialized_instrumented; +} Instruction; + +Instruction read_instruction(PyCodeObject *code, int offset) +{ + Instruction result = (Instruction){0}; + int opcode = result.actual_opcode = _PyCode_CODE(code)[offset].opcode; + if (opcode == INSTRUMENTED_INSTRUCTION) { + opcode = code->_co_instrumentation.monitoring_data->per_instruction_opcodes[offset]; + result.is_prefix_instrumented = true; + } + + if (opcode == INSTRUMENTED_LINE) { + opcode = code->_co_instrumentation.monitoring_data->lines[offset].original_opcode; + result.is_prefix_instrumented = true; + } + while (opcode == EXTENDED_ARG) { + result.extended_args += 1; + offset++; + opcode = _PyCode_CODE(code)[offset].opcode; + } + result.length = result.extended_args + 1; + int deinstrumented = DE_INSTRUMENT[opcode]; + if (deinstrumented) { + result.deinstrumented_opcode = deinstrumented; + result.is_specialized_instrumented = true; + } + else { + result.deinstrumented_opcode = opcode; + } + int base = _PyOpcode_Deopt[result.deinstrumented_opcode]; + if (base == COMPARE_AND_BRANCH) { + /* Skip over following POP_JUMP_IF */ + result.length += 1; + } + else { + result.length += _PyOpcode_Caches[base]; + } + return result; +} #ifdef INSTRUMENT_DEBUG @@ -429,14 +521,14 @@ sanity_check_instrumentation(PyCodeObject *code) tools_matrix) ); int code_len = (int)Py_SIZE(code); - for (int i = 0; i < code_len; i++) { - _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; - int opcode = instr->opcode; + Instruction inst; + for (int i = 0; i < code_len;) { + inst = read_instruction(code, i); /* TO DO -- Check INSTRUMENTED_OPCODE */ - if (opcode == INSTRUMENTED_LINE) { + if (inst.actual_opcode == INSTRUMENTED_LINE) { CHECK(data->lines); CHECK(valid_opcode(data->lines[i].original_opcode)); - opcode = data->lines[i].original_opcode; + int opcode = data->lines[i].original_opcode; CHECK(opcode != END_FOR); CHECK(opcode != RESUME); CHECK(opcode != INSTRUMENTED_RESUME); @@ -449,36 +541,32 @@ sanity_check_instrumentation(PyCodeObject *code) CHECK(data->lines[i].original_opcode == 0 || data->lines[i].original_opcode == 255); } - if (is_instrumented(opcode)) { - int deinstrumented = DE_INSTRUMENT[opcode]; - CHECK(valid_opcode(deinstrumented)); - CHECK(_PyOpcode_Deopt[deinstrumented] == deinstrumented); - if (_PyOpcode_Caches[deinstrumented]) { - CHECK(instr[1].cache > 0); - } - opcode = deinstrumented; - int event = EVENT_FOR_OPCODE[opcode]; + if (inst.is_specialized_instrumented) { + int event = EVENT_FOR_OPCODE[inst.deinstrumented_opcode]; if (event < 0) { /* RESUME fixup */ - event = instr->oparg; + event = _PyCode_CODE(code)[i + inst.extended_args].oparg; } CHECK(tools_matrix.tools[event] != 0); + CHECK(inst.deinstrumented_opcode != END_FOR); } - if (data->lines && opcode != END_FOR) { + if (data->lines && inst.deinstrumented_opcode != END_FOR) { int line1 = compute_line(code, i, data->lines[i].line_delta); int line2 = PyCode_Addr2Line(code, i*sizeof(_Py_CODEUNIT)); CHECK(line1 == line2); } - CHECK(valid_opcode(opcode)); - opcode = _PyOpcode_Deopt[opcode]; - CHECK(valid_opcode(opcode)); + CHECK(valid_opcode(inst.deinstrumented_opcode)); + if (inst.is_prefix_instrumented || inst.is_specialized_instrumented) { + CHECK(_PyOpcode_Deopt[inst.deinstrumented_opcode] == inst.deinstrumented_opcode); + } if (data->tools) { - uint8_t local_tools = data->tools[i]; - if (OPCODE_HAS_EVENT[opcode]) { - int event = EVENT_FOR_OPCODE[opcode]; + uint8_t local_tools = data->tools[i + inst.extended_args]; + int deopt = _PyOpcode_Deopt[inst.deinstrumented_opcode]; + if (OPCODE_HAS_EVENT[deopt]) { + int event = EVENT_FOR_OPCODE[deopt]; if (event == -1) { /* RESUME fixup */ - event = instr->oparg; + event = _PyCode_CODE(code)[i + inst.extended_args].oparg; } uint8_t global_tools = tools_matrix.tools[event]; CHECK((global_tools & local_tools) == local_tools); @@ -487,7 +575,8 @@ sanity_check_instrumentation(PyCodeObject *code) CHECK(local_tools == 0xff); } } - i += _PyOpcode_Caches[opcode]; + i += inst.length; + assert(i <= code_len); } } #else @@ -496,27 +585,6 @@ sanity_check_instrumentation(PyCodeObject *code) #endif -int _Py_GetBaseOpcode(PyCodeObject *code, int offset) -{ - int opcode = _Py_OPCODE(_PyCode_CODE(code)[offset]); - if (!is_instrumented(opcode)) { - assert(_PyOpcode_Deopt[opcode] != 0); - return _PyOpcode_Deopt[opcode]; - } - if (opcode == INSTRUMENTED_INSTRUCTION) { - opcode = code->_co_instrumentation.monitoring_data->per_instruction_opcodes[offset]; - } - if (opcode == INSTRUMENTED_LINE) { - opcode = code->_co_instrumentation.monitoring_data->lines[offset].original_opcode; - } - int deinstrumented = DE_INSTRUMENT[opcode]; - if (deinstrumented) { - return deinstrumented; - } - assert(_PyOpcode_Deopt[opcode] != 0); - return _PyOpcode_Deopt[opcode]; -} - static void de_instrument(PyCodeObject *code, int offset, int event) { @@ -1242,6 +1310,13 @@ initialize_lines(PyCodeObject *code) int current_line = -1; for (int i = code->_co_firsttraceable; i < code_len; i++) { int opcode = _Py_GetBaseOpcode(code, i); + int line = _PyCode_CheckLineNumber(i*(int)sizeof(_Py_CODEUNIT), &range); + if (line != current_line && line >= 0) { + line_data[i].original_opcode = LINE_MARKER; + } + else { + line_data[i].original_opcode = 0; + } switch (opcode) { case END_ASYNC_FOR: case END_FOR: @@ -1249,20 +1324,17 @@ initialize_lines(PyCodeObject *code) /* END_FOR cannot start a line, as it is skipped by FOR_ITER * RESUME must not be instrumented with INSTRUMENT_LINE */ line_data[i].original_opcode = 0; - line_data[i].line_delta = NO_LINE; - continue; - } - int line = _PyCode_CheckLineNumber(i*(int)sizeof(_Py_CODEUNIT), &range); - if (line != current_line && line >= 0) { - line_data[i].original_opcode = LINE_MARKER; - } - else { - line_data[i].original_opcode = 0; } line_data[i].line_delta = compute_line_delta(code, i, line); if (line >= 0) { current_line = line; } + while (opcode == EXTENDED_ARG) { + i++; + line_data[i].original_opcode = 0; + line_data[i].line_delta = NO_LINE; + opcode = _Py_GetBaseOpcode(code, i); + } for (int j = 0; j < _PyOpcode_Caches[opcode]; j++) { i++; line_data[i].original_opcode = 0; @@ -1435,6 +1507,10 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) add_tools(code, i, event, new_tools); } } + while (base_opcode == EXTENDED_ARG) { + i++; + base_opcode = _Py_GetBaseOpcode(code, i); + } /* TO DO -- Use instruction length, not cache count */ if (base_opcode == COMPARE_AND_BRANCH) { /* Skip over following POP_JUMP_IF */ @@ -1489,7 +1565,7 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) } } } - //sanity_check_instrumentation(code); + sanity_check_instrumentation(code); return 0; } @@ -1503,7 +1579,10 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) static void instrument_all_executing_code_objects(PyInterpreterState *interp) { + _PyRuntimeState *runtime = &_PyRuntime; + HEAD_LOCK(runtime); PyThreadState* ts = PyInterpreterState_ThreadHead(interp); + HEAD_UNLOCK(runtime); while (ts) { _PyInterpreterFrame *frame = ts->cframe->current_frame; while (frame) { @@ -1512,7 +1591,9 @@ instrument_all_executing_code_objects(PyInterpreterState *interp) { } frame = frame->previous; } + HEAD_LOCK(runtime); ts = PyThreadState_Next(ts); + HEAD_UNLOCK(runtime); } } diff --git a/Python/pystate.c b/Python/pystate.c index 30a30360778629..5c9339c312fa3d 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -829,7 +829,6 @@ init_threadstate(PyThreadState *tstate, tstate->datastack_chunk = NULL; tstate->datastack_top = NULL; tstate->datastack_limit = NULL; - tstate->monitoring = 0; tstate->what_event = -1; tstate->_initialized = 1; From 8b8f67eae53fce0185a634d79e64b8e44d51e054 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 1 Feb 2023 09:58:07 +0000 Subject: [PATCH 030/116] Make sure instrumentation respects instruction boundaries. --- Python/instrumentation.c | 99 ++++++++++++++++++++-------------------- 1 file changed, 50 insertions(+), 49 deletions(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index e92f1405687bcb..40c91ba4d7e75f 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -337,13 +337,12 @@ Instruction read_instruction(PyCodeObject *code, int offset) result.deinstrumented_opcode = opcode; } int base = _PyOpcode_Deopt[result.deinstrumented_opcode]; + /* TO DO -- Use instruction length, not cache count table */ + result.length += _PyOpcode_Caches[base]; if (base == COMPARE_AND_BRANCH) { /* Skip over following POP_JUMP_IF */ result.length += 1; } - else { - result.length += _PyOpcode_Caches[base]; - } return result; } @@ -521,11 +520,17 @@ sanity_check_instrumentation(PyCodeObject *code) tools_matrix) ); int code_len = (int)Py_SIZE(code); - Instruction inst; for (int i = 0; i < code_len;) { - inst = read_instruction(code, i); - /* TO DO -- Check INSTRUMENTED_OPCODE */ - if (inst.actual_opcode == INSTRUMENTED_LINE) { + Instruction inst = read_instruction(code, i); + int opcode = inst.actual_opcode; + if (opcode == INSTRUMENTED_INSTRUCTION) { + opcode = data->per_instruction_opcodes[i]; + if (!is_instrumented(opcode)) { + CHECK(_PyOpcode_Deopt[opcode] == opcode); + } + /* TO DO -- check tools */ + } + if (opcode == INSTRUMENTED_LINE) { CHECK(data->lines); CHECK(valid_opcode(data->lines[i].original_opcode)); int opcode = data->lines[i].original_opcode; @@ -550,13 +555,20 @@ sanity_check_instrumentation(PyCodeObject *code) CHECK(tools_matrix.tools[event] != 0); CHECK(inst.deinstrumented_opcode != END_FOR); } + if (_PyOpcode_Deopt[inst.deinstrumented_opcode] == COMPARE_AND_BRANCH) { + CHECK(_PyCode_CODE(code)[i+2].opcode == POP_JUMP_IF_FALSE || + _PyCode_CODE(code)[i+2].opcode == POP_JUMP_IF_TRUE); + } if (data->lines && inst.deinstrumented_opcode != END_FOR) { int line1 = compute_line(code, i, data->lines[i].line_delta); int line2 = PyCode_Addr2Line(code, i*sizeof(_Py_CODEUNIT)); CHECK(line1 == line2); } CHECK(valid_opcode(inst.deinstrumented_opcode)); - if (inst.is_prefix_instrumented || inst.is_specialized_instrumented) { + if (inst.is_prefix_instrumented) { + CHECK(inst.extended_args || _PyOpcode_Deopt[inst.deinstrumented_opcode] == inst.deinstrumented_opcode); + } + if (inst.is_specialized_instrumented) { CHECK(_PyOpcode_Deopt[inst.deinstrumented_opcode] == inst.deinstrumented_opcode); } if (data->tools) { @@ -753,7 +765,6 @@ instrument_per_instruction(PyCodeObject *code, int i) { _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; int opcode = instr->opcode; - /* TO DO -- handle INSTRUMENTED_INSTRUCTION */ if (opcode == INSTRUMENTED_INSTRUCTION) { return; } @@ -1308,39 +1319,34 @@ initialize_lines(PyCodeObject *code) line_data[i].line_delta = -127; } int current_line = -1; - for (int i = code->_co_firsttraceable; i < code_len; i++) { - int opcode = _Py_GetBaseOpcode(code, i); + for (int i = code->_co_firsttraceable; i < code_len; ) { + Instruction inst = read_instruction(code, i); int line = _PyCode_CheckLineNumber(i*(int)sizeof(_Py_CODEUNIT), &range); - if (line != current_line && line >= 0) { - line_data[i].original_opcode = LINE_MARKER; - } - else { - line_data[i].original_opcode = 0; - } - switch (opcode) { + line_data[i].line_delta = compute_line_delta(code, i, line); + switch (inst.deinstrumented_opcode) { case END_ASYNC_FOR: case END_FOR: case RESUME: /* END_FOR cannot start a line, as it is skipped by FOR_ITER * RESUME must not be instrumented with INSTRUMENT_LINE */ line_data[i].original_opcode = 0; + break; + default: + if (line != current_line && line >= 0) { + line_data[i].original_opcode = LINE_MARKER; + } + else { + line_data[i].original_opcode = 0; + } + if (line >= 0) { + current_line = line; + } } - line_data[i].line_delta = compute_line_delta(code, i, line); - if (line >= 0) { - current_line = line; - } - while (opcode == EXTENDED_ARG) { - i++; - line_data[i].original_opcode = 0; - line_data[i].line_delta = NO_LINE; - opcode = _Py_GetBaseOpcode(code, i); - } - for (int j = 0; j < _PyOpcode_Caches[opcode]; j++) { - i++; - line_data[i].original_opcode = 0; - line_data[i].line_delta = NO_LINE; + for (int j = 1; j < inst.length; j++) { + line_data[i+j].original_opcode = 0; + line_data[i+j].line_delta = NO_LINE; } - switch (opcode) { + switch (inst.deinstrumented_opcode) { case RETURN_VALUE: case RAISE_VARARGS: case RERAISE: @@ -1348,6 +1354,7 @@ initialize_lines(PyCodeObject *code) * should be treated as different lines */ current_line = -1; } + i += inst.length; } } @@ -1525,7 +1532,8 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) uint8_t removed_line_tools = removed_events.tools[PY_MONITORING_EVENT_LINE]; if (new_line_tools | removed_line_tools) { _PyCoLineInstrumentationData *line_data = code->_co_instrumentation.monitoring_data->lines; - for (int i = code->_co_firsttraceable; i < code_len; i++) { + for (int i = code->_co_firsttraceable; i < code_len;) { + Instruction inst = read_instruction(code, i); if (line_data[i].original_opcode) { if (removed_line_tools) { remove_line_tools(code, i, removed_line_tools); @@ -1534,21 +1542,17 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) add_line_tools(code, i, new_line_tools); } } - int opcode = _Py_GetBaseOpcode(code, i); - /* TO DO -- Use instruction length, not cache count */ - i += _PyOpcode_Caches[opcode]; - if (opcode == COMPARE_AND_BRANCH) { - /* Skip over following POP_JUMP_IF */ - i++; - } + i += inst.length; } } uint8_t new_per_instruction_tools = new_events.tools[PY_MONITORING_EVENT_INSTRUCTION]; uint8_t removed_per_instruction_tools = removed_events.tools[PY_MONITORING_EVENT_INSTRUCTION]; if (new_per_instruction_tools | removed_per_instruction_tools) { - for (int i = code->_co_firsttraceable; i < code_len; i++) { - int opcode = _Py_GetBaseOpcode(code, i); + for (int i = code->_co_firsttraceable; i < code_len;) { + Instruction inst = read_instruction(code, i); + int opcode = _PyOpcode_Deopt[inst.deinstrumented_opcode]; if (opcode == RESUME || opcode == END_FOR) { + i += inst.length; continue; } if (removed_per_instruction_tools) { @@ -1557,15 +1561,12 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) if (new_per_instruction_tools) { add_per_instruction_tools(code, i, new_per_instruction_tools); } - /* TO DO -- Use instruction length, not cache count */ - i += _PyOpcode_Caches[opcode]; - if (opcode == COMPARE_AND_BRANCH) { - /* Skip over following POP_JUMP_IF */ - i++; - } + i += inst.length; } } +#ifdef INSTRUMENT_DEBUG sanity_check_instrumentation(code); +#endif return 0; } From d0a2228ca5b008d2b87787c90dbaaff6b8f3b92a Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 1 Feb 2023 12:33:20 +0000 Subject: [PATCH 031/116] Fix handling of errors in jump instruments --- Lib/test/test_monitoring.py | 9 ++++--- Python/bytecodes.c | 50 ++++++++----------------------------- Python/generated_cases.c.h | 50 ++++++++----------------------------- 3 files changed, 28 insertions(+), 81 deletions(-) diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 1388a745b16992..11be2fe7fe1677 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -493,7 +493,8 @@ def test_lines_single(self): f1() sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) - self.assertEqual(events, [493, 14, 494]) + start = LineMontoringTest.test_lines_single.__code__.co_firstlineno + self.assertEqual(events, [start+7, 14, start+8]) finally: sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) @@ -510,7 +511,8 @@ def test_lines_loop(self): floop() sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) - self.assertEqual(events, [510, 21, 22, 22, 511]) + start = LineMontoringTest.test_lines_loop.__code__.co_firstlineno + self.assertEqual(events, [start+7, 21, 22, 22, 21, start+8]) finally: sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) @@ -531,7 +533,8 @@ def test_lines_two(self): sys.monitoring.set_events(TEST_TOOL, 0); sys.monitoring.set_events(TEST_TOOL2, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) sys.monitoring.register_callback(TEST_TOOL2, E.LINE, None) - expected = [530, 14, 531] + start = LineMontoringTest.test_lines_two.__code__.co_firstlineno + expected = [start+10, 14, start+11] self.assertEqual(events, expected) self.assertEqual(events2, expected) finally: diff --git a/Python/bytecodes.c b/Python/bytecodes.c index a4566f574f393f..5bcbde155e2771 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3466,15 +3466,9 @@ dummy_func( _Py_CODEUNIT *here = next_instr-1; assert(err == 0 || err == 1); _Py_CODEUNIT *target = next_instr + err*oparg; - _PyFrame_SetStackPointer(frame, stack_pointer); - err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_BRANCH, frame, here, target); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - ERROR_IF(err, error); - if (err) { - JUMPBY(oparg); - } - else { + int shrink = 1 - err; + INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); + if (shrink) { STACK_SHRINK(1); Py_DECREF(cond); } @@ -3486,16 +3480,10 @@ dummy_func( ERROR_IF(err < 0, error); _Py_CODEUNIT *here = next_instr-1; assert(err == 0 || err == 1); - _Py_CODEUNIT *target = next_instr + err*oparg; - _PyFrame_SetStackPointer(frame, stack_pointer); - err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_BRANCH, frame, here, target); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - ERROR_IF(err, error); - if (err == 0) { - JUMPBY(oparg); - } - else { + _Py_CODEUNIT *target = next_instr + (1-err)*oparg; + int shrink = err; + INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); + if (shrink) { STACK_SHRINK(1); Py_DECREF(cond); } @@ -3508,12 +3496,7 @@ dummy_func( _Py_CODEUNIT *here = next_instr-1; assert(err == 0 || err == 1); int offset = err*oparg; - _PyFrame_SetStackPointer(frame, stack_pointer); - err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_BRANCH, frame, here, next_instr + offset); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - ERROR_IF(err, error); - JUMPBY(offset); + INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); } inst(INSTRUMENTED_POP_JUMP_IF_FALSE, ( -- )) { @@ -3523,12 +3506,7 @@ dummy_func( _Py_CODEUNIT *here = next_instr-1; assert(err == 0 || err == 1); int offset = (1-err)*oparg; - _PyFrame_SetStackPointer(frame, stack_pointer); - err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_BRANCH, frame, here, next_instr + offset); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - ERROR_IF(err, error); - JUMPBY(offset); + INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); } inst(INSTRUMENTED_POP_JUMP_IF_NONE, ( -- )) { @@ -3543,10 +3521,7 @@ dummy_func( Py_DECREF(value); offset = 0; } - int err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_BRANCH, frame, here, next_instr + offset); - ERROR_IF(err, error); - JUMPBY(offset); + INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); } inst(INSTRUMENTED_POP_JUMP_IF_NOT_NONE, ( -- )) { @@ -3561,10 +3536,7 @@ dummy_func( Py_DECREF(value); offset = oparg; } - int err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_BRANCH, frame, here, next_instr + offset); - ERROR_IF(err, error); - JUMPBY(offset); + INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); } // stack effect: ( -- ) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index c6fff9076bfc14..c91dd0fef592ea 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3789,15 +3789,9 @@ _Py_CODEUNIT *here = next_instr-1; assert(err == 0 || err == 1); _Py_CODEUNIT *target = next_instr + err*oparg; - _PyFrame_SetStackPointer(frame, stack_pointer); - err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_BRANCH, frame, here, target); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - if (err) goto error; - if (err) { - JUMPBY(oparg); - } - else { + int shrink = 1 - err; + INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); + if (shrink) { STACK_SHRINK(1); Py_DECREF(cond); } @@ -3810,16 +3804,10 @@ if (err < 0) goto error; _Py_CODEUNIT *here = next_instr-1; assert(err == 0 || err == 1); - _Py_CODEUNIT *target = next_instr + err*oparg; - _PyFrame_SetStackPointer(frame, stack_pointer); - err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_BRANCH, frame, here, target); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - if (err) goto error; - if (err == 0) { - JUMPBY(oparg); - } - else { + _Py_CODEUNIT *target = next_instr + (1-err)*oparg; + int shrink = err; + INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); + if (shrink) { STACK_SHRINK(1); Py_DECREF(cond); } @@ -3833,12 +3821,7 @@ _Py_CODEUNIT *here = next_instr-1; assert(err == 0 || err == 1); int offset = err*oparg; - _PyFrame_SetStackPointer(frame, stack_pointer); - err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_BRANCH, frame, here, next_instr + offset); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - if (err) goto error; - JUMPBY(offset); + INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } @@ -3849,12 +3832,7 @@ _Py_CODEUNIT *here = next_instr-1; assert(err == 0 || err == 1); int offset = (1-err)*oparg; - _PyFrame_SetStackPointer(frame, stack_pointer); - err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_BRANCH, frame, here, next_instr + offset); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - if (err) goto error; - JUMPBY(offset); + INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } @@ -3870,10 +3848,7 @@ Py_DECREF(value); offset = 0; } - int err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_BRANCH, frame, here, next_instr + offset); - if (err) goto error; - JUMPBY(offset); + INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } @@ -3889,10 +3864,7 @@ Py_DECREF(value); offset = oparg; } - int err = _Py_call_instrumentation_jump( - tstate, PY_MONITORING_EVENT_BRANCH, frame, here, next_instr + offset); - if (err) goto error; - JUMPBY(offset); + INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } From 2a3a85e8b80056a353fab818215ce6bfa1f5724f Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 1 Feb 2023 12:33:42 +0000 Subject: [PATCH 032/116] Fix memory leaks. --- Python/instrumentation.c | 12 ++++++++++-- Python/pystate.c | 8 +++++++- Python/sysmodule.c | 4 +++- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 40c91ba4d7e75f..07a3330d232b93 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -9,7 +9,7 @@ #include "pycore_pystate.h" /* Uncomment this to dump debugging output when assertions fail */ -#define INSTRUMENT_DEBUG 1 +// #define INSTRUMENT_DEBUG 1 static PyObject DISABLE = { @@ -1068,6 +1068,7 @@ _Py_call_instrumentation_jump( ) { assert(event == PY_MONITORING_EVENT_JUMP || event == PY_MONITORING_EVENT_BRANCH); + assert(frame->prev_instr == instr); frame->prev_instr = target; PyCodeObject *code = frame->f_code; int offset = instr - _PyCode_CODE(code); @@ -1086,6 +1087,11 @@ _Py_call_instrumentation_jump( int err = call_instrument(tstate, code, event, &args[1], 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, offset, tools); Py_DECREF(to_obj); Py_DECREF(from_obj); + if (err) { + /* Error handling expects next_instr to point to instruction + 1. + * So we add one here. */ + frame->prev_instr++; + } return err; } @@ -1914,7 +1920,9 @@ PyObject *_Py_CreateMonitoringObject(void) if (events == NULL) { goto error; } - if (PyObject_SetAttrString(mod, "events", events)) { + int err = PyObject_SetAttrString(mod, "events", events); + Py_DECREF(events); + if (err) { goto error; } for (int i = 0; i < PY_MONITORING_EVENTS; i++) { diff --git a/Python/pystate.c b/Python/pystate.c index 5c9339c312fa3d..bcc48480aa98ab 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -421,6 +421,13 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) Py_CLEAR(interp->audit_hooks); + for(int i = 0; i < PY_MONITORING_EVENTS; i++) { + interp->monitoring_matrix.tools[i] = 0; + for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { + Py_CLEAR(interp->tools[t].instrument_callables[i]); + } + } + PyConfig_Clear(&interp->config); Py_CLEAR(interp->codec_search_path); Py_CLEAR(interp->codec_search_cache); @@ -475,7 +482,6 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) interp->code_watchers[i] = NULL; } interp->active_code_watchers = 0; - // XXX Once we have one allocator per interpreter (i.e. // per-interpreter GC) we must ensure that all of the interpreter's // objects have been cleaned up at the point. diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 1291694f06d8b9..4c8cea23576a6d 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -3452,7 +3452,9 @@ _PySys_Create(PyThreadState *tstate, PyObject **sysmod_p) if (monitoring == NULL) { goto error; } - if (PyDict_SetItemString(sysdict, "monitoring", monitoring) < 0) { + int err = PyDict_SetItemString(sysdict, "monitoring", monitoring); + Py_DECREF(monitoring); + if (err < 0) { goto error; } From c3724ab3c1adeda46c3235fa058dce1e6d47433a Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 1 Feb 2023 16:51:35 +0000 Subject: [PATCH 033/116] Instrument FOR_ITER. --- Include/internal/pycore_opcode.h | 4 ++-- Include/opcode.h | 1 + Lib/opcode.py | 1 + Lib/test/test__opcode.py | 8 ++++---- Python/bytecodes.c | 31 +++++++++++++++++++++++++++++-- Python/ceval.c | 11 ++--------- Python/generated_cases.c.h | 31 +++++++++++++++++++++++++++++-- Python/instrumentation.c | 7 +++++-- Python/legacy_tracing.c | 17 ++++++++++++++--- Python/opcode_metadata.h | 1 + Python/opcode_targets.h | 2 +- 11 files changed, 89 insertions(+), 25 deletions(-) diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index fb1359a93b8325..bbf777cf82f7de 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -139,6 +139,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [INSTRUMENTED_CALL] = INSTRUMENTED_CALL, [INSTRUMENTED_CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, [INSTRUMENTED_COMPARE_AND_BRANCH] = INSTRUMENTED_COMPARE_AND_BRANCH, + [INSTRUMENTED_FOR_ITER] = INSTRUMENTED_FOR_ITER, [INSTRUMENTED_INSTRUCTION] = INSTRUMENTED_INSTRUCTION, [INSTRUMENTED_JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD, [INSTRUMENTED_JUMP_FORWARD] = INSTRUMENTED_JUMP_FORWARD, @@ -481,7 +482,7 @@ static const char *const _PyOpcode_OpName[263] = { [234] = "<234>", [235] = "<235>", [236] = "<236>", - [237] = "<237>", + [INSTRUMENTED_FOR_ITER] = "INSTRUMENTED_FOR_ITER", [INSTRUMENTED_INSTRUCTION] = "INSTRUMENTED_INSTRUCTION", [INSTRUMENTED_COMPARE_AND_BRANCH] = "INSTRUMENTED_COMPARE_AND_BRANCH", [INSTRUMENTED_RESUME] = "INSTRUMENTED_RESUME", @@ -580,7 +581,6 @@ static const char *const _PyOpcode_OpName[263] = { case 234: \ case 235: \ case 236: \ - case 237: \ case 248: \ ; diff --git a/Include/opcode.h b/Include/opcode.h index 88ebea7c0efc20..c2c6faebe7f7a4 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -116,6 +116,7 @@ extern "C" { #define CALL 171 #define KW_NAMES 172 #define CALL_INTRINSIC_1 173 +#define INSTRUMENTED_FOR_ITER 237 #define INSTRUMENTED_INSTRUCTION 238 #define INSTRUMENTED_COMPARE_AND_BRANCH 239 #define INSTRUMENTED_RESUME 240 diff --git a/Lib/opcode.py b/Lib/opcode.py index 7d4091ed1178bd..d1f95928a085bc 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -222,6 +222,7 @@ def pseudo_op(name, op, real_ops): # Instrumented instructions +def_op('INSTRUMENTED_FOR_ITER', 237) def_op('INSTRUMENTED_INSTRUCTION', 238) def_op('INSTRUMENTED_COMPARE_AND_BRANCH', 239) def_op('INSTRUMENTED_RESUME', 240) diff --git a/Lib/test/test__opcode.py b/Lib/test/test__opcode.py index 5df05b6f57ded6..03850f68263a3a 100644 --- a/Lib/test/test__opcode.py +++ b/Lib/test/test__opcode.py @@ -20,8 +20,8 @@ def test_stack_effect(self): # All defined opcodes has_arg = dis.hasarg for name, code in filter(lambda item: item[0] not in dis.deoptmap, dis.opmap.items()): - if code >= 238: - # Opcodes 240 and up are internal instrumentation instructions + if code >= 237: + # Opcodes 237 and up are internal instrumentation instructions continue with self.subTest(opname=name): if code not in has_arg: @@ -54,8 +54,8 @@ def test_stack_effect_jump(self): has_exc = dis.hasexc has_jump = dis.hasjabs + dis.hasjrel for name, code in filter(lambda item: item[0] not in dis.deoptmap, dis.opmap.items()): - if code >= 238: - # Opcodes 240 and up are internal instrumentation instructions + if code >= 237: + # Opcodes 237 and up are internal instrumentation instructions continue with self.subTest(opname=name): if code not in has_arg: diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 5bcbde155e2771..f773ae2ad27d91 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -129,10 +129,9 @@ dummy_func( else { _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation( - tstate, oparg != 0, frame, next_instr-1); + tstate, oparg > 0, frame, next_instr-1); ERROR_IF(err, error); if (frame->prev_instr != next_instr-1) { - fprintf(stderr, "Jump has happened\n"); /* Instrumentation has jumped */ next_instr = frame->prev_instr; stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2323,6 +2322,34 @@ dummy_func( } } + // stack effect: ( -- __0) + inst(INSTRUMENTED_FOR_ITER) { + _Py_CODEUNIT *here = next_instr-1; + _Py_CODEUNIT *target; + PyObject *iter = TOP(); + PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); + if (next != NULL) { + PUSH(next); + target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER; + } + else { + if (_PyErr_Occurred(tstate)) { + if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + goto error; + } + monitor_raise(tstate, frame, here); + _PyErr_Clear(tstate); + } + /* iterator ended normally */ + assert(_Py_OPCODE(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg]) == END_FOR); + STACK_SHRINK(1); + Py_DECREF(iter); + /* Skip END_FOR */ + target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1; + } + INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); + } + // stack effect: ( -- __0) inst(FOR_ITER_LIST) { _PyListIterObject *it = (_PyListIterObject *)TOP(); diff --git a/Python/ceval.c b/Python/ceval.c index 8716ab82dad5bb..18f044f6fbb80a 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2255,22 +2255,15 @@ monitor_raise(PyThreadState *tstate, if (tstate->interp->monitoring_matrix.tools[PY_MONITORING_EVENT_RAISE] == 0) { return; } - PyObject *type, *value, *traceback, *orig_traceback, *arg; + PyObject *type, *value, *orig_traceback; int err; _PyErr_Fetch(tstate, &type, &value, &orig_traceback); if (value == NULL) { value = Py_NewRef(Py_None); } _PyErr_NormalizeException(tstate, &type, &value, &orig_traceback); - traceback = (orig_traceback != NULL) ? orig_traceback : Py_None; assert(value != NULL && value != Py_None); - arg = PyTuple_Pack(3, type, value, traceback); - if (arg == NULL) { - _PyErr_Restore(tstate, type, value, orig_traceback); - return; - } - err = _Py_call_instrumentation_arg(tstate, PY_MONITORING_EVENT_RAISE, frame, instr, arg); - Py_DECREF(arg); + err = _Py_call_instrumentation_arg(tstate, PY_MONITORING_EVENT_RAISE, frame, instr, value); if (err == 0) { _PyErr_Restore(tstate, type, value, orig_traceback); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index c91dd0fef592ea..b21057b8e94998 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -18,10 +18,9 @@ else { _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation( - tstate, oparg != 0, frame, next_instr-1); + tstate, oparg > 0, frame, next_instr-1); if (err) goto error; if (frame->prev_instr != next_instr-1) { - fprintf(stderr, "Jump has happened\n"); /* Instrumentation has jumped */ next_instr = frame->prev_instr; stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2632,6 +2631,34 @@ DISPATCH(); } + TARGET(INSTRUMENTED_FOR_ITER) { + _Py_CODEUNIT *here = next_instr-1; + _Py_CODEUNIT *target; + PyObject *iter = TOP(); + PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); + if (next != NULL) { + PUSH(next); + target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER; + } + else { + if (_PyErr_Occurred(tstate)) { + if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + goto error; + } + monitor_raise(tstate, frame, here); + _PyErr_Clear(tstate); + } + /* iterator ended normally */ + assert(_Py_OPCODE(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg]) == END_FOR); + STACK_SHRINK(1); + Py_DECREF(iter); + /* Skip END_FOR */ + target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1; + } + INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); + DISPATCH(); + } + TARGET(FOR_ITER_LIST) { _PyListIterObject *it = (_PyListIterObject *)TOP(); DEOPT_IF(Py_TYPE(it) != &PyListIter_Type, FOR_ITER); diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 07a3330d232b93..c6adcdd1c6c805 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -45,6 +45,8 @@ static const int8_t EVENT_FOR_OPCODE[256] = { [INSTRUMENTED_POP_JUMP_IF_NONE] = PY_MONITORING_EVENT_BRANCH, [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = PY_MONITORING_EVENT_BRANCH, [INSTRUMENTED_COMPARE_AND_BRANCH] = PY_MONITORING_EVENT_BRANCH, + [FOR_ITER] = PY_MONITORING_EVENT_BRANCH, + [INSTRUMENTED_FOR_ITER] = PY_MONITORING_EVENT_BRANCH, }; static const bool OPCODE_HAS_EVENT[256] = { @@ -76,6 +78,7 @@ static const bool OPCODE_HAS_EVENT[256] = { [INSTRUMENTED_POP_JUMP_IF_NONE] = true, [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = true, [INSTRUMENTED_COMPARE_AND_BRANCH] = true, + [INSTRUMENTED_FOR_ITER] = true, }; static const uint8_t DE_INSTRUMENT[256] = { @@ -93,6 +96,7 @@ static const uint8_t DE_INSTRUMENT[256] = { [INSTRUMENTED_POP_JUMP_IF_NONE] = POP_JUMP_IF_NONE, [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = POP_JUMP_IF_NOT_NONE, [INSTRUMENTED_COMPARE_AND_BRANCH] = COMPARE_AND_BRANCH, + [INSTRUMENTED_FOR_ITER] = FOR_ITER, }; static const uint8_t INSTRUMENTED_OPCODES[256] = { @@ -1504,8 +1508,7 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) if (OPCODE_HAS_EVENT[base_opcode]) { int8_t event; if (base_opcode == RESUME) { - int oparg = _Py_OPARG(*instr); - event = oparg > 0; + event = instr->oparg > 0; } else { event = EVENT_FOR_OPCODE[base_opcode]; diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index 22af735304df4a..12580036036cb1 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -116,9 +116,20 @@ sys_trace_exception_func( ) { assert(kwnames == NULL); assert(PyVectorcall_NARGS(nargsf) == 3); - PyObject *arg = args[2]; - assert(PyTuple_CheckExact(arg)); - PyObject *res = call_trace_func(self, arg); + PyObject *exc = args[2]; + assert(PyExceptionInstance_Check(exc)); + PyObject *type = (PyObject *)Py_TYPE(exc); + PyObject *tb = PyException_GetTraceback(exc); + if (tb == NULL) { + tb = Py_NewRef(Py_None); + } + PyObject * tuple = PyTuple_Pack(3, type, exc, tb); + Py_DECREF(tb); + if (tuple == NULL) { + return NULL; + } + PyObject *res = call_trace_func(self, tuple); + Py_DECREF(tuple); return res; } diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index 9ad33884808bf7..41dc9ef0ba0d71 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -144,6 +144,7 @@ static const struct { [GET_ITER] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [GET_YIELD_FROM_ITER] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [FOR_ITER] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, + [INSTRUMENTED_FOR_ITER] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [FOR_ITER_LIST] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [FOR_ITER_TUPLE] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [FOR_ITER_RANGE] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 0c4ea0ec2b4a3d..113b664fffccf1 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -236,7 +236,7 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, + &&TARGET_INSTRUMENTED_FOR_ITER, &&TARGET_INSTRUMENTED_INSTRUCTION, &&TARGET_INSTRUMENTED_COMPARE_AND_BRANCH, &&TARGET_INSTRUMENTED_RESUME, From d64823c79598fccf159eef3e7bca3d23c27f5f0a Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 2 Feb 2023 18:54:36 +0000 Subject: [PATCH 034/116] Fix legacy profiling of calls. --- Include/internal/pycore_instruments.h | 12 +++++- Lib/test/test_monitoring.py | 4 +- Modules/_lsprof.c | 4 ++ Python/bytecodes.c | 34 +++++++++------- Python/ceval.c | 4 +- Python/generated_cases.c.h | 34 +++++++++------- Python/instrumentation.c | 56 ++++++++++++++++++++++++--- Python/legacy_tracing.c | 26 +++++++++++-- 8 files changed, 131 insertions(+), 43 deletions(-) diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index 6c76027889457d..a64633aa1edb47 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -83,9 +83,17 @@ extern int _Py_call_instrumentation_arg(PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg); +extern int +_Py_call_instrumentation_2args(PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg0, PyObject *arg1); + extern void -_Py_call_instrumentation_exc(PyThreadState *tstate, int event, - _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg); +_Py_call_instrumentation_exc0(PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr); + +extern void +_Py_call_instrumentation_exc2(PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg0, PyObject *arg1); extern int _Py_Instrumentation_GetLine(PyCodeObject *code, int index); diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 11be2fe7fe1677..c6a813fe4b55ae 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -206,7 +206,7 @@ def gather_events(self, func): def record(*args, event_name=event_name): events.append(event_name) sys.monitoring.register_callback(TEST_TOOL, event, record) - def record_call(code, offset, obj): + def record_call(code, offset, obj, arg): if isinstance(obj, PY_CALLABLES): events.append("py_call") else: @@ -282,7 +282,7 @@ def down(*args): frame = sys._getframe(1) stack.append(frame) seen.add(frame.f_code) - def call(code, offset, callable): + def call(code, offset, callable, arg): if not isinstance(callable, PY_CALLABLES): stack.append(sys._getframe(1)) for event in UP_EVENTS: diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c index 37170bbea56ad3..d0a483e6b1d704 100644 --- a/Modules/_lsprof.c +++ b/Modules/_lsprof.c @@ -126,6 +126,10 @@ call_timer(ProfilerObject *pObj) static PyObject * normalizeUserObj(PyObject *obj) { + //if (Py_TYPE(obj) == &PyMethodDescr_Type) { + // assert(0); + // return PyObject_Repr(obj); + //} PyCFunctionObject *fn; if (!PyCFunction_Check(obj)) { return Py_NewRef(obj); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index f773ae2ad27d91..f97a14b17c8a71 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2335,9 +2335,9 @@ dummy_func( else { if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + monitor_raise(tstate, frame, here); goto error; } - monitor_raise(tstate, frame, here); _PyErr_Clear(tstate); } /* iterator ended normally */ @@ -2634,9 +2634,10 @@ dummy_func( int is_meth = is_method(stack_pointer, oparg); int total_args = oparg + is_meth; PyObject *function = PEEK(total_args + 1); - int err = _Py_call_instrumentation_arg( + PyObject *arg = total_args == 0 ? Py_None : PEEK(total_args); + int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, - frame, next_instr-1, function); + frame, next_instr-1, function, arg); ERROR_IF(err, error); _PyCallCache *cache = (_PyCallCache *)next_instr; INCREMENT_ADAPTIVE_COUNTER(cache->counter); @@ -2699,15 +2700,16 @@ dummy_func( positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames); if (opcode == INSTRUMENTED_CALL) { + PyObject *arg = total_args == 0 ? Py_None : PEEK(total_args); if (res == NULL) { - _Py_call_instrumentation_exc( + _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, - frame, next_instr-1, function); + frame, next_instr-1, function, arg); } else { - int err = _Py_call_instrumentation_arg( + int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, - frame, next_instr-1, function); + frame, next_instr-1, function, arg); if (err < 0) { Py_CLEAR(res); } @@ -3236,21 +3238,25 @@ dummy_func( } assert(PyTuple_CheckExact(callargs)); EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); - if (opcode == INSTRUMENTED_CALL_FUNCTION_EX && !PyFunction_Check(func) && !PyMethod_Check(func)) { - int err = _Py_call_instrumentation_arg( + if (opcode == INSTRUMENTED_CALL_FUNCTION_EX && + !PyFunction_Check(func) && !PyMethod_Check(func) + ) { + PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? + PyTuple_GET_ITEM(callargs, 0) : Py_None; + int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, - frame, next_instr-1, func); + frame, next_instr-1, func, arg); ERROR_IF(err, error); result = PyObject_Call(func, callargs, kwargs); if (result == NULL) { - _Py_call_instrumentation_exc( + _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, - frame, next_instr-1, func); + frame, next_instr-1, func, arg); } else { - int err = _Py_call_instrumentation_arg( + int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, - frame, next_instr-1, func); + frame, next_instr-1, func, arg); if (err < 0) { Py_CLEAR(result); } diff --git a/Python/ceval.c b/Python/ceval.c index 18f044f6fbb80a..d7e4a9f9a3049e 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2283,7 +2283,7 @@ monitor_unwind(PyThreadState *tstate, if (tstate->interp->monitoring_matrix.tools[PY_MONITORING_EVENT_PY_UNWIND] == 0) { return; } - _Py_call_instrumentation_exc(tstate, PY_MONITORING_EVENT_PY_UNWIND, frame, instr, NULL); + _Py_call_instrumentation_exc0(tstate, PY_MONITORING_EVENT_PY_UNWIND, frame, instr); } @@ -2306,7 +2306,7 @@ monitor_throw(PyThreadState *tstate, if (tstate->interp->monitoring_matrix.tools[PY_MONITORING_EVENT_PY_THROW] == 0) { return; } - _Py_call_instrumentation_exc(tstate, PY_MONITORING_EVENT_PY_THROW, frame, instr, NULL); + _Py_call_instrumentation_exc0(tstate, PY_MONITORING_EVENT_PY_THROW, frame, instr); } void diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index b21057b8e94998..eb42b76d2e193b 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2643,9 +2643,9 @@ else { if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + monitor_raise(tstate, frame, here); goto error; } - monitor_raise(tstate, frame, here); _PyErr_Clear(tstate); } /* iterator ended normally */ @@ -2948,9 +2948,10 @@ int is_meth = is_method(stack_pointer, oparg); int total_args = oparg + is_meth; PyObject *function = PEEK(total_args + 1); - int err = _Py_call_instrumentation_arg( + PyObject *arg = total_args == 0 ? Py_None : PEEK(total_args); + int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, - frame, next_instr-1, function); + frame, next_instr-1, function, arg); if (err) goto error; _PyCallCache *cache = (_PyCallCache *)next_instr; INCREMENT_ADAPTIVE_COUNTER(cache->counter); @@ -3013,15 +3014,16 @@ positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames); if (opcode == INSTRUMENTED_CALL) { + PyObject *arg = total_args == 0 ? Py_None : PEEK(total_args); if (res == NULL) { - _Py_call_instrumentation_exc( + _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, - frame, next_instr-1, function); + frame, next_instr-1, function, arg); } else { - int err = _Py_call_instrumentation_arg( + int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, - frame, next_instr-1, function); + frame, next_instr-1, function, arg); if (err < 0) { Py_CLEAR(res); } @@ -3550,21 +3552,25 @@ } assert(PyTuple_CheckExact(callargs)); EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); - if (opcode == INSTRUMENTED_CALL_FUNCTION_EX && !PyFunction_Check(func) && !PyMethod_Check(func)) { - int err = _Py_call_instrumentation_arg( + if (opcode == INSTRUMENTED_CALL_FUNCTION_EX && + !PyFunction_Check(func) && !PyMethod_Check(func) + ) { + PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? + PyTuple_GET_ITEM(callargs, 0) : Py_None; + int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, - frame, next_instr-1, func); + frame, next_instr-1, func, arg); if (err) goto error; result = PyObject_Call(func, callargs, kwargs); if (result == NULL) { - _Py_call_instrumentation_exc( + _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, - frame, next_instr-1, func); + frame, next_instr-1, func, arg); } else { - int err = _Py_call_instrumentation_arg( + int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, - frame, next_instr-1, func); + frame, next_instr-1, func, arg); if (err < 0) { Py_CLEAR(result); } diff --git a/Python/instrumentation.c b/Python/instrumentation.c index c6adcdd1c6c805..ee62afb103a913 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1065,6 +1065,26 @@ _Py_call_instrumentation_arg( return err; } +int +_Py_call_instrumentation_2args( + PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg0, PyObject *arg1) +{ + PyCodeObject *code = frame->f_code; + assert(is_instrumentation_up_to_date(code, tstate->interp)); + assert(matrix_equals(code->_co_instrumentation.monitoring_data->matrix, tstate->interp->monitoring_matrix)); + int offset = instr - _PyCode_CODE(code); + PyObject *offset_obj = PyLong_FromSsize_t(offset); + if (offset_obj == NULL) { + return -1; + } + uint8_t tools = get_tools_for_instruction(code, offset, event); + PyObject *args[5] = { NULL, (PyObject *)code, offset_obj, arg0, arg1 }; + int err = call_instrument(tstate, code, event, &args[1], 4 | PY_VECTORCALL_ARGUMENTS_OFFSET, offset, tools); + Py_DECREF(offset_obj); + return err; +} + int _Py_call_instrumentation_jump( PyThreadState *tstate, int event, @@ -1100,26 +1120,29 @@ _Py_call_instrumentation_jump( } void -_Py_call_instrumentation_exc( +_Py_call_instrumentation_exc_vector( PyThreadState *tstate, int event, - _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg) + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, Py_ssize_t nargsf, PyObject *args[]) { assert(_PyErr_Occurred(tstate)); + assert(args[0] == NULL); PyObject *type, *value, *traceback; _PyErr_Fetch(tstate, &type, &value, &traceback); PyCodeObject *code = frame->f_code; + assert(args[1] == NULL); + args[1] = (PyObject *)code; assert(code->_co_instrumentation.monitoring_version == tstate->interp->monitoring_version); assert(matrix_equals(code->_co_instrumentation.monitoring_data->matrix, tstate->interp->monitoring_matrix)); int offset = instr - _PyCode_CODE(code); + uint8_t tools = code->_co_instrumentation.monitoring_data->matrix.tools[event]; PyObject *offset_obj = PyLong_FromSsize_t(offset); int err; if (offset_obj == NULL) { err = -1; } else { - PyObject *args[4] = { NULL, (PyObject *)code, offset_obj, arg }; - Py_ssize_t nargsf = (2 | PY_VECTORCALL_ARGUMENTS_OFFSET) + (arg != NULL); - uint8_t tools = code->_co_instrumentation.monitoring_data->matrix.tools[event]; + assert(args[2] == NULL); + args[2] = offset_obj; err = call_instrument(tstate, code, event, &args[1], nargsf, offset, tools); Py_DECREF(offset_obj); } @@ -1134,6 +1157,29 @@ _Py_call_instrumentation_exc( assert(_PyErr_Occurred(tstate)); } +void +_Py_call_instrumentation_exc0( + PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) +{ + assert(_PyErr_Occurred(tstate)); + PyObject *args[3] = { NULL, NULL, NULL }; + Py_ssize_t nargsf = 2 | PY_VECTORCALL_ARGUMENTS_OFFSET; + _Py_call_instrumentation_exc_vector(tstate, event, frame, instr, nargsf, args); +} + +void +_Py_call_instrumentation_exc2( + PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg0, PyObject *arg1) +{ + assert(_PyErr_Occurred(tstate)); + PyObject *args[5] = { NULL, NULL, NULL, arg0, arg1 }; + Py_ssize_t nargsf = 4 | PY_VECTORCALL_ARGUMENTS_OFFSET; + _Py_call_instrumentation_exc_vector(tstate, event, frame, instr, nargsf, args); +} + + int _Py_Instrumentation_GetLine(PyCodeObject *code, int index) { diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index 12580036036cb1..f494134d9340c0 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -79,12 +79,30 @@ sys_profile_call_or_return( size_t nargsf, PyObject *kwnames ) { assert(kwnames == NULL); - assert(PyVectorcall_NARGS(nargsf) == 3); + assert(PyVectorcall_NARGS(nargsf) == 4); PyObject *callable = args[2]; - if (!PyCFunction_Check(callable) && Py_TYPE(callable) != &PyMethodDescr_Type) { - Py_RETURN_NONE; + if (PyCFunction_Check(callable)) { + return call_profile_func(self, callable); + } + if (Py_TYPE(callable) == &PyMethodDescr_Type) { + PyObject *self_arg = args[3]; + /* For backwards compatibility need to + * convert to builtin method */ + + /* If no arg, skip */ + if (self_arg == Py_None) { + Py_RETURN_NONE; + } + PyObject *meth = Py_TYPE(callable)->tp_descr_get( + callable, self_arg, (PyObject*)Py_TYPE(self_arg)); + if (meth == NULL) { + return NULL; + } + PyObject *res = call_profile_func(self, meth); + Py_DECREF(meth); + return res; } - return call_profile_func(self, callable); + Py_RETURN_NONE; } static PyObject * From 284d0b113045c5d7f43e6474433cbc6a1e889fad Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 3 Feb 2023 15:10:34 +0000 Subject: [PATCH 035/116] Fix up instrumented yield. --- Python/bytecodes.c | 23 ++++++++++++++++------- Python/generated_cases.c.h | 25 +++++++++++++++++-------- Python/opcode_metadata.h | 2 +- 3 files changed, 34 insertions(+), 16 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index f97a14b17c8a71..ec05dff27b18fc 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -768,14 +768,24 @@ dummy_func( } } - inst(INSTRUMENTED_YIELD_VALUE, ( -- )) { - PyObject *val = TOP(); - _PyFrame_SetStackPointer(frame, stack_pointer); + inst(INSTRUMENTED_YIELD_VALUE, (retval -- unused)) { + assert(frame != &entry_frame); + PyGenObject *gen = _PyFrame_GetGenerator(frame); + gen->gi_frame_state = FRAME_SUSPENDED; + _PyFrame_SetStackPointer(frame, stack_pointer - 1); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_YIELD, - frame, next_instr-1, val); + frame, next_instr-1, retval); ERROR_IF(err, error); - GO_TO_INSTRUCTION(YIELD_VALUE); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = cframe.current_frame = frame->previous; + gen_frame->previous = NULL; + frame->prev_instr -= frame->yield_offset; + _PyFrame_StackPush(frame, retval); + goto resume_frame; } inst(YIELD_VALUE, (retval -- unused)) { @@ -2694,8 +2704,7 @@ dummy_func( DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ - PyObject *res; - res = PyObject_Vectorcall( + PyObject *res = PyObject_Vectorcall( function, stack_pointer-total_args, positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index eb42b76d2e193b..f242d9cfce51c4 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -934,17 +934,27 @@ } TARGET(INSTRUMENTED_YIELD_VALUE) { - PyObject *val = TOP(); - _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *retval = PEEK(1); + assert(frame != &entry_frame); + PyGenObject *gen = _PyFrame_GetGenerator(frame); + gen->gi_frame_state = FRAME_SUSPENDED; + _PyFrame_SetStackPointer(frame, stack_pointer - 1); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_YIELD, - frame, next_instr-1, val); - if (err) goto error; - GO_TO_INSTRUCTION(YIELD_VALUE); + frame, next_instr-1, retval); + if (err) goto pop_1_error; + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = cframe.current_frame = frame->previous; + gen_frame->previous = NULL; + frame->prev_instr -= frame->yield_offset; + _PyFrame_StackPush(frame, retval); + goto resume_frame; } TARGET(YIELD_VALUE) { - PREDICTED(YIELD_VALUE); PyObject *retval = PEEK(1); // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() @@ -3008,8 +3018,7 @@ DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ - PyObject *res; - res = PyObject_Vectorcall( + PyObject *res = PyObject_Vectorcall( function, stack_pointer-total_args, positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames); diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index 41dc9ef0ba0d71..e7b5bde0c0475f 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -61,7 +61,7 @@ static const struct { [GET_ANEXT] = { 1, 2, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [GET_AWAITABLE] = { 1, 1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [SEND] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [INSTRUMENTED_YIELD_VALUE] = { 0, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, + [INSTRUMENTED_YIELD_VALUE] = { 1, 1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [YIELD_VALUE] = { 1, 1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [POP_EXCEPT] = { 1, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [RERAISE] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, From 1440473a071784b0753420dccb7ee0171253b248 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 9 Feb 2023 10:33:39 +0000 Subject: [PATCH 036/116] Fix instrumentation of SEND and tidy up instrumented bytecode numbering. --- Include/internal/pycore_opcode.h | 4 +- Include/opcode.h | 6 +-- Lib/importlib/_bootstrap_external.py | 2 +- Lib/opcode.py | 8 ++-- Lib/test/test_sys_settrace.py | 3 -- Python/bytecodes.c | 27 +---------- Python/ceval.c | 17 ++----- Python/generated_cases.c.h | 27 +---------- Python/legacy_tracing.c | 71 ++++++++++++++++++++++++++-- Python/opcode_targets.h | 2 +- 10 files changed, 88 insertions(+), 79 deletions(-) diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index 42c3cbcb5e6496..af565176279c79 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -495,10 +495,10 @@ static const char *const _PyOpcode_OpName[263] = { [INSTRUMENTED_JUMP_FORWARD] = "INSTRUMENTED_JUMP_FORWARD", [INSTRUMENTED_JUMP_BACKWARD] = "INSTRUMENTED_JUMP_BACKWARD", [INSTRUMENTED_JUMP_IF_FALSE_OR_POP] = "INSTRUMENTED_JUMP_IF_FALSE_OR_POP", - [248] = "<248>", [INSTRUMENTED_JUMP_IF_TRUE_OR_POP] = "INSTRUMENTED_JUMP_IF_TRUE_OR_POP", [INSTRUMENTED_POP_JUMP_IF_FALSE] = "INSTRUMENTED_POP_JUMP_IF_FALSE", [INSTRUMENTED_POP_JUMP_IF_TRUE] = "INSTRUMENTED_POP_JUMP_IF_TRUE", + [251] = "<251>", [252] = "<252>", [INSTRUMENTED_INSTRUCTION] = "INSTRUMENTED_INSTRUCTION", [INSTRUMENTED_LINE] = "INSTRUMENTED_LINE", @@ -580,7 +580,7 @@ static const char *const _PyOpcode_OpName[263] = { case 232: \ case 233: \ case 234: \ - case 248: \ + case 251: \ case 252: \ ; diff --git a/Include/opcode.h b/Include/opcode.h index c2068edc908388..3eda340f1c4d24 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -130,9 +130,9 @@ extern "C" { #define INSTRUMENTED_JUMP_FORWARD 245 #define INSTRUMENTED_JUMP_BACKWARD 246 #define INSTRUMENTED_JUMP_IF_FALSE_OR_POP 247 -#define INSTRUMENTED_JUMP_IF_TRUE_OR_POP 249 -#define INSTRUMENTED_POP_JUMP_IF_FALSE 250 -#define INSTRUMENTED_POP_JUMP_IF_TRUE 251 +#define INSTRUMENTED_JUMP_IF_TRUE_OR_POP 248 +#define INSTRUMENTED_POP_JUMP_IF_FALSE 249 +#define INSTRUMENTED_POP_JUMP_IF_TRUE 250 #define INSTRUMENTED_INSTRUCTION 253 #define INSTRUMENTED_LINE 254 #define MIN_PSEUDO_OPCODE 256 diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 347bda99054101..a7cc58ca3d6a32 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -446,7 +446,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3529).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3528).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c diff --git a/Lib/opcode.py b/Lib/opcode.py index 879291569cb32f..71e5b43a67c697 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -240,13 +240,13 @@ def pseudo_op(name, op, real_ops): def_op('INSTRUMENTED_JUMP_FORWARD', 245) def_op('INSTRUMENTED_JUMP_BACKWARD', 246) def_op('INSTRUMENTED_JUMP_IF_FALSE_OR_POP', 247) -def_op('INSTRUMENTED_JUMP_IF_TRUE_OR_POP', 249) -def_op('INSTRUMENTED_POP_JUMP_IF_FALSE', 250) -def_op('INSTRUMENTED_POP_JUMP_IF_TRUE', 251) +def_op('INSTRUMENTED_JUMP_IF_TRUE_OR_POP', 248) +def_op('INSTRUMENTED_POP_JUMP_IF_FALSE', 249) +def_op('INSTRUMENTED_POP_JUMP_IF_TRUE', 250) def_op('INSTRUMENTED_INSTRUCTION', 253) def_op('INSTRUMENTED_LINE', 254) -# 255 will be MARKER +# 255 is reserved hasarg.extend([op for op in opmap.values() if op >= HAVE_ARGUMENT]) diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index a251b2272e95eb..88a2b941b99ad4 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -636,21 +636,18 @@ async def f(): (-2, 'line'), (-1, 'line'), (-1, 'return'), - (1, 'exception'), (2, 'line'), (1, 'line'), (-1, 'call'), (-2, 'line'), (-1, 'line'), (-1, 'return'), - (1, 'exception'), (2, 'line'), (1, 'line'), (-1, 'call'), (-2, 'line'), (-1, 'line'), (-1, 'return'), - (1, 'exception'), (2, 'line'), (1, 'line'), (-1, 'call'), diff --git a/Python/bytecodes.c b/Python/bytecodes.c index e2c3504a643d86..4a92c1a65dc402 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -733,31 +733,8 @@ dummy_func( inst(SEND, (receiver, v -- receiver if (!jump), retval)) { assert(frame != &entry_frame); bool jump = false; - PySendResult gen_status; - if (tstate->c_tracefunc == NULL) { - gen_status = PyIter_Send(receiver, v, &retval); - } else { - if (Py_IsNone(v) && PyIter_Check(receiver)) { - retval = Py_TYPE(receiver)->tp_iternext(receiver); - } - else { - retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v); - } - if (retval == NULL) { - if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - monitor_raise(tstate, frame, next_instr-1); - } - if (_PyGen_FetchStopIterationValue(&retval) == 0) { - gen_status = PYGEN_RETURN; - } - else { - gen_status = PYGEN_ERROR; - } - } - else { - gen_status = PYGEN_NEXT; - } - } + + PySendResult gen_status = PyIter_Send(receiver, v, &retval); if (gen_status == PYGEN_ERROR) { assert(retval == NULL); goto error; diff --git a/Python/ceval.c b/Python/ceval.c index 4813342d9173b8..fe6a7eb019c7c1 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1949,22 +1949,15 @@ monitor_raise(PyThreadState *tstate, if (tstate->interp->monitoring_matrix.tools[PY_MONITORING_EVENT_RAISE] == 0) { return; } - PyObject *type, *value, *orig_traceback; + PyObject *exc = PyErr_GetRaisedException(); + assert(exc != NULL); int err; - _PyErr_Fetch(tstate, &type, &value, &orig_traceback); - if (value == NULL) { - value = Py_NewRef(Py_None); - } - _PyErr_NormalizeException(tstate, &type, &value, &orig_traceback); - assert(value != NULL && value != Py_None); - err = _Py_call_instrumentation_arg(tstate, PY_MONITORING_EVENT_RAISE, frame, instr, value); + err = _Py_call_instrumentation_arg(tstate, PY_MONITORING_EVENT_RAISE, frame, instr, exc); if (err == 0) { - _PyErr_Restore(tstate, type, value, orig_traceback); + PyErr_SetRaisedException(exc); } else { - Py_XDECREF(type); - Py_XDECREF(value); - Py_XDECREF(orig_traceback); + Py_DECREF(exc); } } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 3bfa7c627b985f..21f25bf0589e94 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -938,31 +938,8 @@ PyObject *retval; assert(frame != &entry_frame); bool jump = false; - PySendResult gen_status; - if (tstate->c_tracefunc == NULL) { - gen_status = PyIter_Send(receiver, v, &retval); - } else { - if (Py_IsNone(v) && PyIter_Check(receiver)) { - retval = Py_TYPE(receiver)->tp_iternext(receiver); - } - else { - retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v); - } - if (retval == NULL) { - if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - monitor_raise(tstate, frame, next_instr-1); - } - if (_PyGen_FetchStopIterationValue(&retval) == 0) { - gen_status = PYGEN_RETURN; - } - else { - gen_status = PYGEN_ERROR; - } - } - else { - gen_status = PYGEN_NEXT; - } - } + + PySendResult gen_status = PyIter_Send(receiver, v, &retval); if (gen_status == PYGEN_ERROR) { assert(retval == NULL); goto error; diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index f494134d9340c0..140ae4ec8bb94a 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -105,6 +105,8 @@ sys_profile_call_or_return( Py_RETURN_NONE; } + + static PyObject * call_trace_func(_PyLegacyEventHandler *self, PyObject *arg) { @@ -162,7 +164,65 @@ sys_trace_func2( } static PyObject * -sys_trace_func3( +sys_trace_return( + _PyLegacyEventHandler *self, PyObject *const *args, + size_t nargsf, PyObject *kwnames +) { + PyThreadState *tstate = _PyThreadState_GET(); + if (tstate->c_tracefunc == NULL) { + Py_RETURN_NONE; + } + assert(!PyErr_Occurred()); + assert(kwnames == NULL); + assert(PyVectorcall_NARGS(nargsf) == 3); + assert(PyCode_Check(args[0])); + PyCodeObject *code = (PyCodeObject *)args[0]; + PyObject *val = args[2]; + PyObject *res = call_trace_func(self, val); + if ((code->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) && + res != NULL + ) { + /* Fake PEP 380 StopIteration exception event */ + Py_DECREF(res); + PyObject *stop_iter_args[2] = { NULL, val }; + PyObject *exc = PyObject_Vectorcall(PyExc_StopIteration, + &stop_iter_args[1], + 1 | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + if (exc == NULL) { + return NULL; + } + /* The event occurs in the caller frame, for historical reasons */ + _PyInterpreterFrame *frame = _PyEval_GetFrame(); + frame = frame->previous; + while (frame && _PyFrame_IsIncomplete(frame)) { + frame = frame->previous; + } + PyFrameObject* frame_obj = NULL; + if (frame != NULL) { + frame_obj = _PyFrame_GetFrameObject(frame); + } else { + PyErr_SetString(PyExc_SystemError, + "Missing frame when calling trace function."); + return NULL; + } + if (frame_obj == NULL) { + Py_DECREF(exc); + return NULL; + } + Py_INCREF(frame_obj); + int err = tstate->c_tracefunc(tstate->c_traceobj, frame_obj, PyTrace_EXCEPTION, exc); + Py_DECREF(exc); + Py_DECREF(frame_obj); + if (err) { + return NULL; + } + } + return res; +} + +static PyObject * +sys_trace_yield( _PyLegacyEventHandler *self, PyObject *const *args, size_t nargsf, PyObject *kwnames ) { @@ -434,8 +494,13 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) return -1; } if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, - (vectorcallfunc)sys_trace_func3, PyTrace_RETURN, - PY_MONITORING_EVENT_PY_RETURN, PY_MONITORING_EVENT_PY_YIELD)) { + (vectorcallfunc)sys_trace_return, PyTrace_RETURN, + PY_MONITORING_EVENT_PY_RETURN, -1)) { + return -1; + } + if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, + (vectorcallfunc)sys_trace_yield, PyTrace_RETURN, + PY_MONITORING_EVENT_PY_YIELD, -1)) { return -1; } if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index c7f985981689a5..6112dc790f4553 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -247,11 +247,11 @@ static void *opcode_targets[256] = { &&TARGET_INSTRUMENTED_JUMP_FORWARD, &&TARGET_INSTRUMENTED_JUMP_BACKWARD, &&TARGET_INSTRUMENTED_JUMP_IF_FALSE_OR_POP, - &&_unknown_opcode, &&TARGET_INSTRUMENTED_JUMP_IF_TRUE_OR_POP, &&TARGET_INSTRUMENTED_POP_JUMP_IF_FALSE, &&TARGET_INSTRUMENTED_POP_JUMP_IF_TRUE, &&_unknown_opcode, + &&_unknown_opcode, &&TARGET_INSTRUMENTED_INSTRUCTION, &&TARGET_INSTRUMENTED_LINE, &&TARGET_DO_TRACING From 825f42a294ddda8a68e8eae5d708fd3e291d5672 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 9 Feb 2023 15:09:32 +0000 Subject: [PATCH 037/116] Handle line numbers in exception handled event. --- Lib/test/test_bdb.py | 3 +- Lib/test/test_monitoring.py | 58 +++++++++++++- Lib/test/test_sys_settrace.py | 3 + Python/bytecodes.c | 32 ++++---- Python/generated_cases.c.h | 32 ++++---- Python/legacy_tracing.c | 147 +++++++++++++++++++++------------- 6 files changed, 191 insertions(+), 84 deletions(-) diff --git a/Lib/test/test_bdb.py b/Lib/test/test_bdb.py index 87a5ac308a12df..9f2f964f7f54bb 100644 --- a/Lib/test/test_bdb.py +++ b/Lib/test/test_bdb.py @@ -433,8 +433,9 @@ def __exit__(self, type_=None, value=None, traceback=None): not_empty = '' if self.tracer.set_list: not_empty += 'All paired tuples have not been processed, ' - not_empty += ('the last one was number %d' % + not_empty += ('the last one was number %d\n' % self.tracer.expect_set_no) + not_empty += repr(self.tracer.set_list) # Make a BdbNotExpectedError a unittest failure. if type_ is not None and issubclass(BdbNotExpectedError, type_): diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index c6a813fe4b55ae..27308f9a6fc1ad 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -538,13 +538,69 @@ def test_lines_two(self): self.assertEqual(events, expected) self.assertEqual(events2, expected) finally: - sys.monitoring.set_events(TEST_TOOL2, 0) + sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.set_events(TEST_TOOL2, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) sys.monitoring.register_callback(TEST_TOOL2, E.LINE, None) self.assertEqual(sys.monitoring._all_events(), {}) sys.monitoring.restart_events() + def check_lines(self, func, expected, tool=TEST_TOOL): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + events = [] + recorder = RecorderWithDisable(events) + sys.monitoring.register_callback(tool, E.LINE, recorder) + sys.monitoring.set_events(tool, E.LINE) + func() + sys.monitoring.set_events(tool, 0) + sys.monitoring.register_callback(tool, E.LINE, None) + lines = [ line - func.__code__.co_firstlineno for line in events[1:-1] ] + self.assertEqual(lines, expected) + finally: + sys.monitoring.set_events(tool, 0) + + + def test_linear(self): + + def func(): + line = 1 + line = 2 + line = 3 + line = 4 + line = 5 + + self.check_lines(func, [1,2,3,4,5]) + + def test_branch(self): + def func(): + if "true".startswith("t"): + line = 2 + line = 3 + else: + line = 5 + line = 6 + + self.check_lines(func, [1,2,3,6]) + + def test_try_except(self): + + def func1(): + try: + line = 2 + line = 3 + except: + line = 5 + line = 6 + self.check_lines(func1, [1,2,3,6]) + def func2(): + try: + line = 2 + raise 3 + except: + line = 5 + line = 6 + self.check_lines(func2, [1,2,3,4,5,6]) diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 88a2b941b99ad4..a251b2272e95eb 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -636,18 +636,21 @@ async def f(): (-2, 'line'), (-1, 'line'), (-1, 'return'), + (1, 'exception'), (2, 'line'), (1, 'line'), (-1, 'call'), (-2, 'line'), (-1, 'line'), (-1, 'return'), + (1, 'exception'), (2, 'line'), (1, 'line'), (-1, 'call'), (-2, 'line'), (-1, 'line'), (-1, 'return'), + (1, 'exception'), (2, 'line'), (1, 'line'), (-1, 'call'), diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 4a92c1a65dc402..8763cf82a79b03 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -733,22 +733,26 @@ dummy_func( inst(SEND, (receiver, v -- receiver if (!jump), retval)) { assert(frame != &entry_frame); bool jump = false; - - PySendResult gen_status = PyIter_Send(receiver, v, &retval); - if (gen_status == PYGEN_ERROR) { - assert(retval == NULL); - goto error; - } - Py_DECREF(v); - if (gen_status == PYGEN_RETURN) { - assert(retval != NULL); - Py_DECREF(receiver); - JUMPBY(oparg); - jump = true; + if (Py_IsNone(v) && PyIter_Check(receiver)) { + retval = Py_TYPE(receiver)->tp_iternext(receiver); } else { - assert(gen_status == PYGEN_NEXT); - assert(retval != NULL); + retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v); + } + if (retval == NULL) { + if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) + ) { + monitor_raise(tstate, frame, next_instr-1); + } + if (_PyGen_FetchStopIterationValue(&retval) == 0) { + assert(retval != NULL); + Py_DECREF(receiver); + JUMPBY(oparg); + jump = true; + } + else { + goto error; + } } } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 21f25bf0589e94..cc5ff8ec3c5a82 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -938,22 +938,26 @@ PyObject *retval; assert(frame != &entry_frame); bool jump = false; - - PySendResult gen_status = PyIter_Send(receiver, v, &retval); - if (gen_status == PYGEN_ERROR) { - assert(retval == NULL); - goto error; - } - Py_DECREF(v); - if (gen_status == PYGEN_RETURN) { - assert(retval != NULL); - Py_DECREF(receiver); - JUMPBY(oparg); - jump = true; + if (Py_IsNone(v) && PyIter_Check(receiver)) { + retval = Py_TYPE(receiver)->tp_iternext(receiver); } else { - assert(gen_status == PYGEN_NEXT); - assert(retval != NULL); + retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v); + } + if (retval == NULL) { + if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) + ) { + monitor_raise(tstate, frame, next_instr-1); + } + if (_PyGen_FetchStopIterationValue(&retval) == 0) { + assert(retval != NULL); + Py_DECREF(receiver); + JUMPBY(oparg); + jump = true; + } + else { + goto error; + } } STACK_SHRINK(1); STACK_GROW(((!jump) ? 1 : 0)); diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index 140ae4ec8bb94a..12183820bb0d35 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -169,9 +169,6 @@ sys_trace_return( size_t nargsf, PyObject *kwnames ) { PyThreadState *tstate = _PyThreadState_GET(); - if (tstate->c_tracefunc == NULL) { - Py_RETURN_NONE; - } assert(!PyErr_Occurred()); assert(kwnames == NULL); assert(PyVectorcall_NARGS(nargsf) == 3); @@ -179,45 +176,57 @@ sys_trace_return( PyCodeObject *code = (PyCodeObject *)args[0]; PyObject *val = args[2]; PyObject *res = call_trace_func(self, val); - if ((code->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) && - res != NULL - ) { - /* Fake PEP 380 StopIteration exception event */ - Py_DECREF(res); - PyObject *stop_iter_args[2] = { NULL, val }; - PyObject *exc = PyObject_Vectorcall(PyExc_StopIteration, - &stop_iter_args[1], - 1 | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); - if (exc == NULL) { - return NULL; - } - /* The event occurs in the caller frame, for historical reasons */ - _PyInterpreterFrame *frame = _PyEval_GetFrame(); - frame = frame->previous; - while (frame && _PyFrame_IsIncomplete(frame)) { - frame = frame->previous; - } - PyFrameObject* frame_obj = NULL; - if (frame != NULL) { - frame_obj = _PyFrame_GetFrameObject(frame); - } else { - PyErr_SetString(PyExc_SystemError, - "Missing frame when calling trace function."); - return NULL; - } - if (frame_obj == NULL) { - Py_DECREF(exc); - return NULL; - } - Py_INCREF(frame_obj); - int err = tstate->c_tracefunc(tstate->c_traceobj, frame_obj, PyTrace_EXCEPTION, exc); - Py_DECREF(exc); - Py_DECREF(frame_obj); - if (err) { - return NULL; - } - } +// if ((code->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) && +// res != NULL && tstate->c_tracefunc != NULL +// ) { +// /* Fake PEP 380 StopIteration exception event */ +// Py_DECREF(res); +// PyObject *stop_iter_args[2] = { NULL, val }; +// PyObject *exc = PyObject_Vectorcall(PyExc_StopIteration, +// &stop_iter_args[1], +// 1 | PY_VECTORCALL_ARGUMENTS_OFFSET, +// NULL); +// if (exc == NULL) { +// return NULL; +// } +// /* The event occurs in the caller frame, for historical reasons */ +// _PyInterpreterFrame *frame = _PyEval_GetFrame(); +// frame = frame->previous; +// while (frame && _PyFrame_IsIncomplete(frame)) { +// frame = frame->previous; +// } +// PyFrameObject* frame_obj = NULL; +// if (frame != NULL) { +// frame_obj = _PyFrame_GetFrameObject(frame); +// } else { +// PyErr_SetString(PyExc_SystemError, +// "Missing frame when calling trace function."); +// return NULL; +// } +// if (frame_obj == NULL) { +// Py_DECREF(exc); +// return NULL; +// } +// PyObject *type = (PyObject *)Py_TYPE(exc); +// PyObject *tb = PyException_GetTraceback(exc); +// if (tb == NULL) { +// tb = Py_NewRef(Py_None); +// } +// PyObject * tuple = PyTuple_Pack(3, type, exc, tb); +// Py_DECREF(tb); +// if (tuple == NULL) { +// Py_DECREF(exc); +// return NULL; +// } +// Py_INCREF(frame_obj); +// int err = tstate->c_tracefunc(tstate->c_traceobj, frame_obj, PyTrace_EXCEPTION, tuple); +// Py_DECREF(frame_obj); +// Py_DECREF(tuple); +// Py_DECREF(exc); +// if (err) { +// return NULL; +// } +// } return res; } @@ -263,6 +272,9 @@ trace_line( PyThreadState *tstate, _PyLegacyEventHandler *self, PyFrameObject* frame, int line ) { + if (!frame->f_trace_lines) { + Py_RETURN_NONE; + } if (line < 0) { Py_RETURN_NONE; } @@ -298,9 +310,6 @@ sys_trace_line_func( return NULL; } assert(args[0] == (PyObject *)frame->f_frame->f_code); - if (!frame->f_trace_lines) { - Py_RETURN_NONE; - } if (frame ->f_last_traced_line == line) { /* Already traced this line */ Py_RETURN_NONE; @@ -341,19 +350,44 @@ sys_trace_jump_func( int to_line = _Py_Instrumentation_GetLine(code, to); /* Backward jump: Always generate event * Forward jump: Only generate event if jumping to different line. */ - if (to > from) { - /* Forwards jump */ - if (frame->f_last_traced_line == to_line) { - /* Already traced this line */ - Py_RETURN_NONE; - } + if (to > from && frame->f_last_traced_line == to_line) { + /* Already traced this line */ + Py_RETURN_NONE; } return trace_line(tstate, self, frame, to_line); } +/* We don't care about the exception here, + * we just treat it as a possible new line + */ +static PyObject * +sys_trace_exception_handled( + _PyLegacyEventHandler *self, PyObject *const *args, + size_t nargsf, PyObject *kwnames +) { + assert(kwnames == NULL); + PyThreadState *tstate = _PyThreadState_GET(); + if (tstate->c_tracefunc == NULL) { + Py_RETURN_NONE; + } + assert(PyVectorcall_NARGS(nargsf) == 3); + PyFrameObject* frame = PyEval_GetFrame(); + PyCodeObject *code = (PyCodeObject *)args[0]; + assert(PyCode_Check(code)); + assert(code == frame->f_frame->f_code); + int offset = _PyLong_AsInt(args[1]); + /* We can call _Py_Instrumentation_GetLine because we always set + * line events for tracing */ + int line = _Py_Instrumentation_GetLine(code, offset); + if (frame->f_last_traced_line == line) { + /* Already traced this line */ + Py_RETURN_NONE; + } + return trace_line(tstate, self, frame, line); +} + PyTypeObject _PyLegacyEventHandler_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) "sys.legacy_event_handler", sizeof(_PyLegacyEventHandler), @@ -528,7 +562,11 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) PY_MONITORING_EVENT_INSTRUCTION, -1)) { return -1; } - /* TO DO: Set up callback for PY_MONITORING_EVENT_EXCEPTION_HANDLED */ + if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, + (vectorcallfunc)sys_trace_exception_handled, PyTrace_LINE, + PY_MONITORING_EVENT_EXCEPTION_HANDLED, -1)) { + return -1; + } } int delta = (func != NULL) - (tstate->c_tracefunc != NULL); @@ -545,7 +583,8 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) (1 << PY_MONITORING_EVENT_PY_RETURN) | (1 << PY_MONITORING_EVENT_PY_YIELD) | (1 << PY_MONITORING_EVENT_RAISE) | (1 << PY_MONITORING_EVENT_LINE) | (1 << PY_MONITORING_EVENT_JUMP) | (1 << PY_MONITORING_EVENT_BRANCH) | - (1 << PY_MONITORING_EVENT_PY_UNWIND) | (1 << PY_MONITORING_EVENT_PY_THROW); + (1 << PY_MONITORING_EVENT_PY_UNWIND) | (1 << PY_MONITORING_EVENT_PY_THROW) | + (1 << PY_MONITORING_EVENT_EXCEPTION_HANDLED); if (tstate->interp->f_opcode_trace_set) { events |= (1 << PY_MONITORING_EVENT_INSTRUCTION); } From ce5ddb32aa5f7d3ad7fe6858aaf36ebdf5210473 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 10 Feb 2023 09:50:48 +0000 Subject: [PATCH 038/116] Instrument END_FOR to mimic PEP 380 StopIteration and add more tests. --- Include/internal/pycore_instruments.h | 10 +++--- Include/internal/pycore_opcode.h | 4 +-- Include/opcode.h | 1 + Lib/opcode.py | 1 + Lib/test/test_monitoring.py | 49 +++++++++++++++++++++++++++ Python/bytecodes.c | 24 +++++++++---- Python/ceval.c | 10 +++--- Python/generated_cases.c.h | 29 ++++++++++++---- Python/instrumentation.c | 9 ++++- Python/legacy_tracing.c | 3 +- Python/opcode_metadata.h | 5 +++ Python/opcode_targets.h | 2 +- Python/specialize.c | 6 ++-- 13 files changed, 125 insertions(+), 28 deletions(-) diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index a64633aa1edb47..34dd2551f8aaa6 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -25,20 +25,22 @@ extern "C" { #define PY_MONITORING_EVENT_INSTRUCTION 6 #define PY_MONITORING_EVENT_JUMP 7 #define PY_MONITORING_EVENT_BRANCH 8 +#define PY_MONITORING_EVENT_STOP_ITERATION 9 -#define PY_MONITORING_INSTRUMENTED_EVENTS 9 +#define PY_MONITORING_INSTRUMENTED_EVENTS 10 /* Grouped events */ -#define PY_MONITORING_EVENT_C_RETURN 9 -#define PY_MONITORING_EVENT_C_RAISE 10 +#define PY_MONITORING_EVENT_C_RETURN 10 +#define PY_MONITORING_EVENT_C_RAISE 11 /* Exceptional events */ -#define PY_MONITORING_EVENT_PY_THROW 11 #define PY_MONITORING_EVENT_RAISE 12 #define PY_MONITORING_EVENT_EXCEPTION_HANDLED 13 #define PY_MONITORING_EVENT_PY_UNWIND 14 +#define PY_MONITORING_EVENT_PY_THROW 15 + /* #define INSTRUMENT_EVENT_BRANCH_NOT_TAKEN xxx -- If we can afford this */ diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index af565176279c79..b2d8e57b3c8024 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -139,6 +139,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [INSTRUMENTED_CALL] = INSTRUMENTED_CALL, [INSTRUMENTED_CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, [INSTRUMENTED_COMPARE_AND_BRANCH] = INSTRUMENTED_COMPARE_AND_BRANCH, + [INSTRUMENTED_END_FOR] = INSTRUMENTED_END_FOR, [INSTRUMENTED_FOR_ITER] = INSTRUMENTED_FOR_ITER, [INSTRUMENTED_INSTRUCTION] = INSTRUMENTED_INSTRUCTION, [INSTRUMENTED_JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD, @@ -498,7 +499,7 @@ static const char *const _PyOpcode_OpName[263] = { [INSTRUMENTED_JUMP_IF_TRUE_OR_POP] = "INSTRUMENTED_JUMP_IF_TRUE_OR_POP", [INSTRUMENTED_POP_JUMP_IF_FALSE] = "INSTRUMENTED_POP_JUMP_IF_FALSE", [INSTRUMENTED_POP_JUMP_IF_TRUE] = "INSTRUMENTED_POP_JUMP_IF_TRUE", - [251] = "<251>", + [INSTRUMENTED_END_FOR] = "INSTRUMENTED_END_FOR", [252] = "<252>", [INSTRUMENTED_INSTRUCTION] = "INSTRUMENTED_INSTRUCTION", [INSTRUMENTED_LINE] = "INSTRUMENTED_LINE", @@ -580,7 +581,6 @@ static const char *const _PyOpcode_OpName[263] = { case 232: \ case 233: \ case 234: \ - case 251: \ case 252: \ ; diff --git a/Include/opcode.h b/Include/opcode.h index 3eda340f1c4d24..8dc6ef9655760d 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -133,6 +133,7 @@ extern "C" { #define INSTRUMENTED_JUMP_IF_TRUE_OR_POP 248 #define INSTRUMENTED_POP_JUMP_IF_FALSE 249 #define INSTRUMENTED_POP_JUMP_IF_TRUE 250 +#define INSTRUMENTED_END_FOR 251 #define INSTRUMENTED_INSTRUCTION 253 #define INSTRUMENTED_LINE 254 #define MIN_PSEUDO_OPCODE 256 diff --git a/Lib/opcode.py b/Lib/opcode.py index 71e5b43a67c697..022a8885eef43a 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -243,6 +243,7 @@ def pseudo_op(name, op, real_ops): def_op('INSTRUMENTED_JUMP_IF_TRUE_OR_POP', 248) def_op('INSTRUMENTED_POP_JUMP_IF_FALSE', 249) def_op('INSTRUMENTED_POP_JUMP_IF_TRUE', 250) +def_op('INSTRUMENTED_END_FOR', 251) def_op('INSTRUMENTED_INSTRUCTION', 253) def_op('INSTRUMENTED_LINE', 254) diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 27308f9a6fc1ad..37961cc981fb83 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -604,3 +604,52 @@ def func2(): line = 6 self.check_lines(func2, [1,2,3,4,5,6]) + + +class ExceptionRecorder: + + def __init__(self, events): + self.events = events + + def __call__(self, code, offset, exc): + self.events.append(type(exc)) + +class ExceptionMontoringTest(unittest.TestCase): + + def check_events(self, func, expected, tool=TEST_TOOL, events=E.RAISE): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + event_list = [] + recorder = ExceptionRecorder(event_list) + sys.monitoring.register_callback(tool, events, recorder) + sys.monitoring.set_events(tool, events) + func() + sys.monitoring.set_events(tool, 0) + sys.monitoring.register_callback(tool, events, None) + self.assertEqual(event_list, expected) + finally: + sys.monitoring.set_events(tool, 0) + sys.monitoring.register_callback(tool, events, None) + + + def test_simple_try_except(self): + + def func1(): + try: + line = 2 + raise KeyError + except: + line = 5 + line = 6 + + self.check_events(func1, [KeyError]) + + def gen(): + yield 1 + return 2 + + def implicit_stop_iteration(): + for _ in gen(): + pass + + self.check_events(implicit_stop_iteration, [StopIteration], events=E.STOP_ITERATION) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 8763cf82a79b03..4ac79ea25acacb 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -172,6 +172,15 @@ dummy_func( macro(END_FOR) = POP_TOP + POP_TOP; + inst(INSTRUMENTED_END_FOR, (pop1, pop2 --)) { + /* Need to create a fake StopIteration error here, + * to conform to PEP 380 */ + PyErr_SetNone(PyExc_StopIteration); + monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_STOP_ITERATION); + PyErr_SetRaisedException(NULL); + DECREF_INPUTS(); + } + inst(UNARY_NEGATIVE, (value -- res)) { res = PyNumber_Negative(value); DECREF_INPUTS(); @@ -742,7 +751,7 @@ dummy_func( if (retval == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) ) { - monitor_raise(tstate, frame, next_instr-1); + monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_RAISE); } if (_PyGen_FetchStopIterationValue(&retval) == 0) { assert(retval != NULL); @@ -2147,11 +2156,12 @@ dummy_func( if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { goto error; } - monitor_raise(tstate, frame, next_instr-1); + monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_RAISE); _PyErr_Clear(tstate); } /* iterator ended normally */ - assert(_Py_OPCODE(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg]) == END_FOR); + assert(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].opcode == END_FOR || + next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].opcode == INSTRUMENTED_END_FOR); Py_DECREF(iter); STACK_SHRINK(1); /* Jump forward oparg, then skip following END_FOR instruction */ @@ -2173,13 +2183,14 @@ dummy_func( else { if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - monitor_raise(tstate, frame, here); goto error; } + monitor_raise(tstate, frame, here, PY_MONITORING_EVENT_RAISE); _PyErr_Clear(tstate); } /* iterator ended normally */ - assert(_Py_OPCODE(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg]) == END_FOR); + assert(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].opcode == END_FOR || + next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].opcode == INSTRUMENTED_END_FOR); STACK_SHRINK(1); Py_DECREF(iter); /* Skip END_FOR */ @@ -2272,7 +2283,8 @@ dummy_func( gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER + oparg); - assert(_Py_OPCODE(*next_instr) == END_FOR); + assert(next_instr->opcode == END_FOR || + next_instr->opcode == INSTRUMENTED_END_FOR); DISPATCH_INLINED(gen_frame); } diff --git a/Python/ceval.c b/Python/ceval.c index fe6a7eb019c7c1..5a870eee02c107 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -178,7 +178,7 @@ lltrace_resume_frame(_PyInterpreterFrame *frame) static void monitor_raise(PyThreadState *tstate, _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr); + _Py_CODEUNIT *instr, int event); static void monitor_unwind(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr); @@ -905,7 +905,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int PyTraceBack_Here(f); } } - monitor_raise(tstate, frame, next_instr-1); + monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_RAISE); exception_unwind: { @@ -1944,15 +1944,15 @@ unpack_iterable(PyThreadState *tstate, PyObject *v, static void monitor_raise(PyThreadState *tstate, _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr) + _Py_CODEUNIT *instr, int event) { - if (tstate->interp->monitoring_matrix.tools[PY_MONITORING_EVENT_RAISE] == 0) { + if (tstate->interp->monitoring_matrix.tools[event] == 0) { return; } PyObject *exc = PyErr_GetRaisedException(); assert(exc != NULL); int err; - err = _Py_call_instrumentation_arg(tstate, PY_MONITORING_EVENT_RAISE, frame, instr, exc); + err = _Py_call_instrumentation_arg(tstate, event, frame, instr, exc); if (err == 0) { PyErr_SetRaisedException(exc); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index cc5ff8ec3c5a82..ceea168c763258 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -235,6 +235,20 @@ DISPATCH(); } + TARGET(INSTRUMENTED_END_FOR) { + PyObject *pop2 = PEEK(1); + PyObject *pop1 = PEEK(2); + /* Need to create a fake StopIteration error here, + * to conform to PEP 380 */ + PyErr_SetNone(PyExc_StopIteration); + monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_STOP_ITERATION); + PyErr_SetRaisedException(NULL); + Py_DECREF(pop1); + Py_DECREF(pop2); + STACK_SHRINK(2); + DISPATCH(); + } + TARGET(UNARY_NEGATIVE) { PyObject *value = PEEK(1); PyObject *res; @@ -947,7 +961,7 @@ if (retval == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) ) { - monitor_raise(tstate, frame, next_instr-1); + monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_RAISE); } if (_PyGen_FetchStopIterationValue(&retval) == 0) { assert(retval != NULL); @@ -2711,11 +2725,12 @@ if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { goto error; } - monitor_raise(tstate, frame, next_instr-1); + monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_RAISE); _PyErr_Clear(tstate); } /* iterator ended normally */ - assert(_Py_OPCODE(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg]) == END_FOR); + assert(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].opcode == END_FOR || + next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].opcode == INSTRUMENTED_END_FOR); Py_DECREF(iter); STACK_SHRINK(1); /* Jump forward oparg, then skip following END_FOR instruction */ @@ -2741,13 +2756,14 @@ else { if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - monitor_raise(tstate, frame, here); goto error; } + monitor_raise(tstate, frame, here, PY_MONITORING_EVENT_RAISE); _PyErr_Clear(tstate); } /* iterator ended normally */ - assert(_Py_OPCODE(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg]) == END_FOR); + assert(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].opcode == END_FOR || + next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].opcode == INSTRUMENTED_END_FOR); STACK_SHRINK(1); Py_DECREF(iter); /* Skip END_FOR */ @@ -2852,7 +2868,8 @@ gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER + oparg); - assert(_Py_OPCODE(*next_instr) == END_FOR); + assert(next_instr->opcode == END_FOR || + next_instr->opcode == INSTRUMENTED_END_FOR); DISPATCH_INLINED(gen_frame); } diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 25a38d70e492fb..3c5750c4b40aa2 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -49,6 +49,8 @@ static const int8_t EVENT_FOR_OPCODE[256] = { [INSTRUMENTED_COMPARE_AND_BRANCH] = PY_MONITORING_EVENT_BRANCH, [FOR_ITER] = PY_MONITORING_EVENT_BRANCH, [INSTRUMENTED_FOR_ITER] = PY_MONITORING_EVENT_BRANCH, + [END_FOR] = PY_MONITORING_EVENT_STOP_ITERATION, + [INSTRUMENTED_END_FOR] = PY_MONITORING_EVENT_STOP_ITERATION, }; static const bool OPCODE_HAS_EVENT[256] = { @@ -83,6 +85,8 @@ static const bool OPCODE_HAS_EVENT[256] = { [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = true, [INSTRUMENTED_COMPARE_AND_BRANCH] = true, [INSTRUMENTED_FOR_ITER] = true, + [END_FOR] = true, + [INSTRUMENTED_END_FOR] = true, }; static const uint8_t DE_INSTRUMENT[256] = { @@ -102,6 +106,7 @@ static const uint8_t DE_INSTRUMENT[256] = { [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = POP_JUMP_IF_NOT_NONE, [INSTRUMENTED_COMPARE_AND_BRANCH] = COMPARE_AND_BRANCH, [INSTRUMENTED_FOR_ITER] = FOR_ITER, + [INSTRUMENTED_END_FOR] = END_FOR, }; static const uint8_t INSTRUMENTED_OPCODES[256] = { @@ -135,6 +140,8 @@ static const uint8_t INSTRUMENTED_OPCODES[256] = { [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE, [COMPARE_AND_BRANCH] = INSTRUMENTED_COMPARE_AND_BRANCH, [INSTRUMENTED_COMPARE_AND_BRANCH] = INSTRUMENTED_COMPARE_AND_BRANCH, + [END_FOR] = INSTRUMENTED_END_FOR, + [INSTRUMENTED_END_FOR] = INSTRUMENTED_END_FOR, [INSTRUMENTED_LINE] = INSTRUMENTED_LINE, [INSTRUMENTED_INSTRUCTION] = INSTRUMENTED_INSTRUCTION, @@ -1909,7 +1916,7 @@ const char *event_names[] = { [PY_MONITORING_EVENT_EXCEPTION_HANDLED] = "EXCEPTION_HANDLED", [PY_MONITORING_EVENT_C_RAISE] = "C_RAISE", [PY_MONITORING_EVENT_PY_UNWIND] = "PY_UNWIND", - [15] = "Unused", + [PY_MONITORING_EVENT_STOP_ITERATION] = "STOP_ITERATION", }; /*[clinic input] diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index 12183820bb0d35..d9f4b4adfa48e0 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -539,7 +539,7 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) } if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, (vectorcallfunc)sys_trace_exception_func, PyTrace_EXCEPTION, - PY_MONITORING_EVENT_RAISE, -1)) { + PY_MONITORING_EVENT_RAISE, PY_MONITORING_EVENT_STOP_ITERATION)) { return -1; } if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, @@ -584,6 +584,7 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) (1 << PY_MONITORING_EVENT_RAISE) | (1 << PY_MONITORING_EVENT_LINE) | (1 << PY_MONITORING_EVENT_JUMP) | (1 << PY_MONITORING_EVENT_BRANCH) | (1 << PY_MONITORING_EVENT_PY_UNWIND) | (1 << PY_MONITORING_EVENT_PY_THROW) | + (1 << PY_MONITORING_EVENT_STOP_ITERATION) | (1 << PY_MONITORING_EVENT_EXCEPTION_HANDLED); if (tstate->interp->f_opcode_trace_set) { events |= (1 << PY_MONITORING_EVENT_INSTRUCTION); diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index 2ea8b1027cbc80..bcc628f0ab05e2 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -38,6 +38,8 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 0; case END_FOR: return 1+1; + case INSTRUMENTED_END_FOR: + return 2; case UNARY_NEGATIVE: return 1; case UNARY_NOT: @@ -422,6 +424,8 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 1; case END_FOR: return 0+0; + case INSTRUMENTED_END_FOR: + return 0; case UNARY_NEGATIVE: return 1; case UNARY_NOT: @@ -794,6 +798,7 @@ struct opcode_metadata { [POP_TOP] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [PUSH_NULL] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [END_FOR] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, + [INSTRUMENTED_END_FOR] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [UNARY_NEGATIVE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [UNARY_NOT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [UNARY_INVERT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 6112dc790f4553..874231dc5dc12f 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -250,7 +250,7 @@ static void *opcode_targets[256] = { &&TARGET_INSTRUMENTED_JUMP_IF_TRUE_OR_POP, &&TARGET_INSTRUMENTED_POP_JUMP_IF_FALSE, &&TARGET_INSTRUMENTED_POP_JUMP_IF_TRUE, - &&_unknown_opcode, + &&TARGET_INSTRUMENTED_END_FOR, &&_unknown_opcode, &&TARGET_INSTRUMENTED_INSTRUCTION, &&TARGET_INSTRUMENTED_LINE, diff --git a/Python/specialize.c b/Python/specialize.c index 1bf90f429268fb..940de62e33d51e 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1084,7 +1084,7 @@ PyObject *descr, DescriptorClassification kind) if (dict) { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_NOT_MANAGED_DICT); return 0; - } + } assert(owner_cls->tp_dictoffset > 0); assert(owner_cls->tp_dictoffset <= INT16_MAX); _py_set_opcode(instr, LOAD_ATTR_METHOD_LAZY_DICT); @@ -2170,7 +2170,9 @@ _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr, int oparg) goto success; } else if (tp == &PyGen_Type && oparg <= SHRT_MAX) { - assert(_Py_OPCODE(instr[oparg + INLINE_CACHE_ENTRIES_FOR_ITER + 1]) == END_FOR); + assert(instr[oparg + INLINE_CACHE_ENTRIES_FOR_ITER + 1].opcode == END_FOR || + instr[oparg + INLINE_CACHE_ENTRIES_FOR_ITER + 1].opcode == INSTRUMENTED_END_FOR + ); _py_set_opcode(instr, FOR_ITER_GEN); goto success; } From da83abead70680e9f96523aae8529ec53f070a44 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 14 Feb 2023 12:30:52 +0000 Subject: [PATCH 039/116] Add END_SEND for instrumentation. --- Include/internal/pycore_opcode.h | 44 +++++------ Include/opcode.h | 126 ++++++++++++++++--------------- Lib/opcode.py | 2 + Python/bytecodes.c | 41 ++++++++-- Python/compile.c | 3 +- Python/generated_cases.c.h | 57 +++++++++++--- Python/instrumentation.c | 7 ++ Python/legacy_tracing.c | 4 + Python/opcode_metadata.h | 10 +++ Python/opcode_targets.h | 40 +++++----- 10 files changed, 213 insertions(+), 121 deletions(-) diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index e3b9e03b6f3e25..f566b2673c5f26 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -122,6 +122,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [DICT_UPDATE] = DICT_UPDATE, [END_ASYNC_FOR] = END_ASYNC_FOR, [END_FOR] = END_FOR, + [END_SEND] = END_SEND, [EXTENDED_ARG] = EXTENDED_ARG, [FORMAT_VALUE] = FORMAT_VALUE, [FOR_ITER] = FOR_ITER, @@ -141,6 +142,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [INSTRUMENTED_CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, [INSTRUMENTED_COMPARE_AND_BRANCH] = INSTRUMENTED_COMPARE_AND_BRANCH, [INSTRUMENTED_END_FOR] = INSTRUMENTED_END_FOR, + [INSTRUMENTED_END_SEND] = INSTRUMENTED_END_SEND, [INSTRUMENTED_FOR_ITER] = INSTRUMENTED_FOR_ITER, [INSTRUMENTED_INSTRUCTION] = INSTRUMENTED_INSTRUCTION, [INSTRUMENTED_JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD, @@ -255,17 +257,18 @@ static const char *const _PyOpcode_OpName[263] = { [PUSH_NULL] = "PUSH_NULL", [INTERPRETER_EXIT] = "INTERPRETER_EXIT", [END_FOR] = "END_FOR", + [END_SEND] = "END_SEND", [BINARY_OP_ADD_FLOAT] = "BINARY_OP_ADD_FLOAT", [BINARY_OP_ADD_INT] = "BINARY_OP_ADD_INT", [BINARY_OP_ADD_UNICODE] = "BINARY_OP_ADD_UNICODE", - [BINARY_OP_INPLACE_ADD_UNICODE] = "BINARY_OP_INPLACE_ADD_UNICODE", [NOP] = "NOP", - [BINARY_OP_MULTIPLY_FLOAT] = "BINARY_OP_MULTIPLY_FLOAT", + [BINARY_OP_INPLACE_ADD_UNICODE] = "BINARY_OP_INPLACE_ADD_UNICODE", [UNARY_NEGATIVE] = "UNARY_NEGATIVE", [UNARY_NOT] = "UNARY_NOT", + [BINARY_OP_MULTIPLY_FLOAT] = "BINARY_OP_MULTIPLY_FLOAT", [BINARY_OP_MULTIPLY_INT] = "BINARY_OP_MULTIPLY_INT", - [BINARY_OP_SUBTRACT_FLOAT] = "BINARY_OP_SUBTRACT_FLOAT", [UNARY_INVERT] = "UNARY_INVERT", + [BINARY_OP_SUBTRACT_FLOAT] = "BINARY_OP_SUBTRACT_FLOAT", [BINARY_OP_SUBTRACT_INT] = "BINARY_OP_SUBTRACT_INT", [BINARY_SUBSCR_DICT] = "BINARY_SUBSCR_DICT", [BINARY_SUBSCR_GETITEM] = "BINARY_SUBSCR_GETITEM", @@ -274,20 +277,20 @@ static const char *const _PyOpcode_OpName[263] = { [CALL_PY_EXACT_ARGS] = "CALL_PY_EXACT_ARGS", [CALL_PY_WITH_DEFAULTS] = "CALL_PY_WITH_DEFAULTS", [CALL_BOUND_METHOD_EXACT_ARGS] = "CALL_BOUND_METHOD_EXACT_ARGS", - [CALL_BUILTIN_CLASS] = "CALL_BUILTIN_CLASS", [BINARY_SUBSCR] = "BINARY_SUBSCR", [BINARY_SLICE] = "BINARY_SLICE", [STORE_SLICE] = "STORE_SLICE", + [CALL_BUILTIN_CLASS] = "CALL_BUILTIN_CLASS", [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", [GET_LEN] = "GET_LEN", [MATCH_MAPPING] = "MATCH_MAPPING", [MATCH_SEQUENCE] = "MATCH_SEQUENCE", [MATCH_KEYS] = "MATCH_KEYS", - [CALL_NO_KW_BUILTIN_FAST] = "CALL_NO_KW_BUILTIN_FAST", + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", [PUSH_EXC_INFO] = "PUSH_EXC_INFO", [CHECK_EXC_MATCH] = "CHECK_EXC_MATCH", [CHECK_EG_MATCH] = "CHECK_EG_MATCH", + [CALL_NO_KW_BUILTIN_FAST] = "CALL_NO_KW_BUILTIN_FAST", [CALL_NO_KW_BUILTIN_O] = "CALL_NO_KW_BUILTIN_O", [CALL_NO_KW_ISINSTANCE] = "CALL_NO_KW_ISINSTANCE", [CALL_NO_KW_LEN] = "CALL_NO_KW_LEN", @@ -298,7 +301,6 @@ static const char *const _PyOpcode_OpName[263] = { [CALL_NO_KW_STR_1] = "CALL_NO_KW_STR_1", [CALL_NO_KW_TUPLE_1] = "CALL_NO_KW_TUPLE_1", [CALL_NO_KW_TYPE_1] = "CALL_NO_KW_TYPE_1", - [COMPARE_AND_BRANCH_FLOAT] = "COMPARE_AND_BRANCH_FLOAT", [WITH_EXCEPT_START] = "WITH_EXCEPT_START", [GET_AITER] = "GET_AITER", [GET_ANEXT] = "GET_ANEXT", @@ -306,38 +308,38 @@ static const char *const _PyOpcode_OpName[263] = { [BEFORE_WITH] = "BEFORE_WITH", [END_ASYNC_FOR] = "END_ASYNC_FOR", [CLEANUP_THROW] = "CLEANUP_THROW", + [COMPARE_AND_BRANCH_FLOAT] = "COMPARE_AND_BRANCH_FLOAT", [COMPARE_AND_BRANCH_INT] = "COMPARE_AND_BRANCH_INT", [COMPARE_AND_BRANCH_STR] = "COMPARE_AND_BRANCH_STR", [FOR_ITER_LIST] = "FOR_ITER_LIST", - [FOR_ITER_TUPLE] = "FOR_ITER_TUPLE", [STORE_SUBSCR] = "STORE_SUBSCR", [DELETE_SUBSCR] = "DELETE_SUBSCR", + [FOR_ITER_TUPLE] = "FOR_ITER_TUPLE", [FOR_ITER_RANGE] = "FOR_ITER_RANGE", [FOR_ITER_GEN] = "FOR_ITER_GEN", [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", - [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", [GET_ITER] = "GET_ITER", [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", - [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", + [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS", + [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", - [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR", [RETURN_GENERATOR] = "RETURN_GENERATOR", + [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST", [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST", - [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", [RETURN_VALUE] = "RETURN_VALUE", - [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", + [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", + [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", - [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", [PREP_RERAISE_STAR] = "PREP_RERAISE_STAR", [POP_EXCEPT] = "POP_EXCEPT", [STORE_NAME] = "STORE_NAME", @@ -363,7 +365,7 @@ static const char *const _PyOpcode_OpName[263] = { [JUMP_FORWARD] = "JUMP_FORWARD", [JUMP_IF_FALSE_OR_POP] = "JUMP_IF_FALSE_OR_POP", [JUMP_IF_TRUE_OR_POP] = "JUMP_IF_TRUE_OR_POP", - [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", + [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE", [POP_JUMP_IF_TRUE] = "POP_JUMP_IF_TRUE", [LOAD_GLOBAL] = "LOAD_GLOBAL", @@ -393,7 +395,7 @@ static const char *const _PyOpcode_OpName[263] = { [JUMP_BACKWARD] = "JUMP_BACKWARD", [COMPARE_AND_BRANCH] = "COMPARE_AND_BRANCH", [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX", - [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST", + [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", [EXTENDED_ARG] = "EXTENDED_ARG", [LIST_APPEND] = "LIST_APPEND", [SET_ADD] = "SET_ADD", @@ -403,21 +405,21 @@ static const char *const _PyOpcode_OpName[263] = { [YIELD_VALUE] = "YIELD_VALUE", [RESUME] = "RESUME", [MATCH_CLASS] = "MATCH_CLASS", + [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST", [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", - [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", [FORMAT_VALUE] = "FORMAT_VALUE", [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", [BUILD_STRING] = "BUILD_STRING", + [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", - [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", [LIST_EXTEND] = "LIST_EXTEND", [SET_UPDATE] = "SET_UPDATE", [DICT_MERGE] = "DICT_MERGE", [DICT_UPDATE] = "DICT_UPDATE", + [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", [SEND_GEN] = "SEND_GEN", - [167] = "<167>", [168] = "<168>", [169] = "<169>", [170] = "<170>", @@ -502,7 +504,7 @@ static const char *const _PyOpcode_OpName[263] = { [INSTRUMENTED_POP_JUMP_IF_FALSE] = "INSTRUMENTED_POP_JUMP_IF_FALSE", [INSTRUMENTED_POP_JUMP_IF_TRUE] = "INSTRUMENTED_POP_JUMP_IF_TRUE", [INSTRUMENTED_END_FOR] = "INSTRUMENTED_END_FOR", - [252] = "<252>", + [INSTRUMENTED_END_SEND] = "INSTRUMENTED_END_SEND", [INSTRUMENTED_INSTRUCTION] = "INSTRUMENTED_INSTRUCTION", [INSTRUMENTED_LINE] = "INSTRUMENTED_LINE", [DO_TRACING] = "DO_TRACING", @@ -517,7 +519,6 @@ static const char *const _PyOpcode_OpName[263] = { #endif #define EXTRA_CASES \ - case 167: \ case 168: \ case 169: \ case 170: \ @@ -582,7 +583,6 @@ static const char *const _PyOpcode_OpName[263] = { case 232: \ case 233: \ case 234: \ - case 252: \ ; #ifdef __cplusplus diff --git a/Include/opcode.h b/Include/opcode.h index cd10052b7aadf2..efd91df2d730cd 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -13,6 +13,7 @@ extern "C" { #define PUSH_NULL 2 #define INTERPRETER_EXIT 3 #define END_FOR 4 +#define END_SEND 5 #define NOP 9 #define UNARY_NEGATIVE 11 #define UNARY_NOT 12 @@ -134,6 +135,7 @@ extern "C" { #define INSTRUMENTED_POP_JUMP_IF_FALSE 249 #define INSTRUMENTED_POP_JUMP_IF_TRUE 250 #define INSTRUMENTED_END_FOR 251 +#define INSTRUMENTED_END_SEND 252 #define INSTRUMENTED_INSTRUCTION 253 #define INSTRUMENTED_LINE 254 #define MIN_PSEUDO_OPCODE 256 @@ -145,68 +147,68 @@ extern "C" { #define JUMP_NO_INTERRUPT 261 #define LOAD_METHOD 262 #define MAX_PSEUDO_OPCODE 262 -#define BINARY_OP_ADD_FLOAT 5 -#define BINARY_OP_ADD_INT 6 -#define BINARY_OP_ADD_UNICODE 7 -#define BINARY_OP_INPLACE_ADD_UNICODE 8 -#define BINARY_OP_MULTIPLY_FLOAT 10 -#define BINARY_OP_MULTIPLY_INT 13 -#define BINARY_OP_SUBTRACT_FLOAT 14 -#define BINARY_OP_SUBTRACT_INT 16 -#define BINARY_SUBSCR_DICT 17 -#define BINARY_SUBSCR_GETITEM 18 -#define BINARY_SUBSCR_LIST_INT 19 -#define BINARY_SUBSCR_TUPLE_INT 20 -#define CALL_PY_EXACT_ARGS 21 -#define CALL_PY_WITH_DEFAULTS 22 -#define CALL_BOUND_METHOD_EXACT_ARGS 23 -#define CALL_BUILTIN_CLASS 24 -#define CALL_BUILTIN_FAST_WITH_KEYWORDS 28 -#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 29 -#define CALL_NO_KW_BUILTIN_FAST 34 -#define CALL_NO_KW_BUILTIN_O 38 -#define CALL_NO_KW_ISINSTANCE 39 -#define CALL_NO_KW_LEN 40 -#define CALL_NO_KW_LIST_APPEND 41 -#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 42 -#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 43 -#define CALL_NO_KW_METHOD_DESCRIPTOR_O 44 -#define CALL_NO_KW_STR_1 45 -#define CALL_NO_KW_TUPLE_1 46 -#define CALL_NO_KW_TYPE_1 47 -#define COMPARE_AND_BRANCH_FLOAT 48 -#define COMPARE_AND_BRANCH_INT 56 -#define COMPARE_AND_BRANCH_STR 57 -#define FOR_ITER_LIST 58 -#define FOR_ITER_TUPLE 59 -#define FOR_ITER_RANGE 62 -#define FOR_ITER_GEN 63 -#define LOAD_ATTR_CLASS 64 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 65 -#define LOAD_ATTR_INSTANCE_VALUE 66 -#define LOAD_ATTR_MODULE 67 -#define LOAD_ATTR_PROPERTY 70 -#define LOAD_ATTR_SLOT 72 -#define LOAD_ATTR_WITH_HINT 73 -#define LOAD_ATTR_METHOD_LAZY_DICT 76 -#define LOAD_ATTR_METHOD_NO_DICT 77 -#define LOAD_ATTR_METHOD_WITH_VALUES 78 -#define LOAD_CONST__LOAD_FAST 79 -#define LOAD_FAST__LOAD_CONST 80 -#define LOAD_FAST__LOAD_FAST 81 -#define LOAD_GLOBAL_BUILTIN 82 -#define LOAD_GLOBAL_MODULE 84 -#define STORE_ATTR_INSTANCE_VALUE 86 -#define STORE_ATTR_SLOT 87 -#define STORE_ATTR_WITH_HINT 113 -#define STORE_FAST__LOAD_FAST 143 -#define STORE_FAST__STORE_FAST 153 -#define STORE_SUBSCR_DICT 154 -#define STORE_SUBSCR_LIST_INT 158 -#define UNPACK_SEQUENCE_LIST 159 -#define UNPACK_SEQUENCE_TUPLE 160 -#define UNPACK_SEQUENCE_TWO_TUPLE 161 -#define SEND_GEN 166 +#define BINARY_OP_ADD_FLOAT 6 +#define BINARY_OP_ADD_INT 7 +#define BINARY_OP_ADD_UNICODE 8 +#define BINARY_OP_INPLACE_ADD_UNICODE 10 +#define BINARY_OP_MULTIPLY_FLOAT 13 +#define BINARY_OP_MULTIPLY_INT 14 +#define BINARY_OP_SUBTRACT_FLOAT 16 +#define BINARY_OP_SUBTRACT_INT 17 +#define BINARY_SUBSCR_DICT 18 +#define BINARY_SUBSCR_GETITEM 19 +#define BINARY_SUBSCR_LIST_INT 20 +#define BINARY_SUBSCR_TUPLE_INT 21 +#define CALL_PY_EXACT_ARGS 22 +#define CALL_PY_WITH_DEFAULTS 23 +#define CALL_BOUND_METHOD_EXACT_ARGS 24 +#define CALL_BUILTIN_CLASS 28 +#define CALL_BUILTIN_FAST_WITH_KEYWORDS 29 +#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 34 +#define CALL_NO_KW_BUILTIN_FAST 38 +#define CALL_NO_KW_BUILTIN_O 39 +#define CALL_NO_KW_ISINSTANCE 40 +#define CALL_NO_KW_LEN 41 +#define CALL_NO_KW_LIST_APPEND 42 +#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 43 +#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 44 +#define CALL_NO_KW_METHOD_DESCRIPTOR_O 45 +#define CALL_NO_KW_STR_1 46 +#define CALL_NO_KW_TUPLE_1 47 +#define CALL_NO_KW_TYPE_1 48 +#define COMPARE_AND_BRANCH_FLOAT 56 +#define COMPARE_AND_BRANCH_INT 57 +#define COMPARE_AND_BRANCH_STR 58 +#define FOR_ITER_LIST 59 +#define FOR_ITER_TUPLE 62 +#define FOR_ITER_RANGE 63 +#define FOR_ITER_GEN 64 +#define LOAD_ATTR_CLASS 65 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 66 +#define LOAD_ATTR_INSTANCE_VALUE 67 +#define LOAD_ATTR_MODULE 70 +#define LOAD_ATTR_PROPERTY 72 +#define LOAD_ATTR_SLOT 73 +#define LOAD_ATTR_WITH_HINT 76 +#define LOAD_ATTR_METHOD_LAZY_DICT 77 +#define LOAD_ATTR_METHOD_NO_DICT 78 +#define LOAD_ATTR_METHOD_WITH_VALUES 79 +#define LOAD_CONST__LOAD_FAST 80 +#define LOAD_FAST__LOAD_CONST 81 +#define LOAD_FAST__LOAD_FAST 82 +#define LOAD_GLOBAL_BUILTIN 84 +#define LOAD_GLOBAL_MODULE 86 +#define STORE_ATTR_INSTANCE_VALUE 87 +#define STORE_ATTR_SLOT 113 +#define STORE_ATTR_WITH_HINT 143 +#define STORE_FAST__LOAD_FAST 153 +#define STORE_FAST__STORE_FAST 154 +#define STORE_SUBSCR_DICT 158 +#define STORE_SUBSCR_LIST_INT 159 +#define UNPACK_SEQUENCE_LIST 160 +#define UNPACK_SEQUENCE_TUPLE 161 +#define UNPACK_SEQUENCE_TWO_TUPLE 166 +#define SEND_GEN 167 #define DO_TRACING 255 #define HAS_ARG(op) ((((op) >= HAVE_ARGUMENT) && (!IS_PSEUDO_OPCODE(op)))\ diff --git a/Lib/opcode.py b/Lib/opcode.py index 582b008bc28e1d..55756c552ad1de 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -83,6 +83,7 @@ def pseudo_op(name, op, real_ops): def_op('INTERPRETER_EXIT', 3) def_op('END_FOR', 4) +def_op('END_SEND', 5) def_op('NOP', 9) @@ -244,6 +245,7 @@ def pseudo_op(name, op, real_ops): def_op('INSTRUMENTED_POP_JUMP_IF_FALSE', 249) def_op('INSTRUMENTED_POP_JUMP_IF_TRUE', 250) def_op('INSTRUMENTED_END_FOR', 251) +def_op('INSTRUMENTED_END_SEND', 252) def_op('INSTRUMENTED_INSTRUCTION', 253) def_op('INSTRUMENTED_LINE', 254) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 4ba7be06429435..5b1ee975cb0944 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -172,15 +172,30 @@ dummy_func( macro(END_FOR) = POP_TOP + POP_TOP; - inst(INSTRUMENTED_END_FOR, (pop1, pop2 --)) { + inst(INSTRUMENTED_END_FOR, (receiver, discard --)) { /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ - PyErr_SetNone(PyExc_StopIteration); - monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_STOP_ITERATION); - PyErr_SetRaisedException(NULL); + if (PyGen_Check(receiver)) { + PyErr_SetNone(PyExc_StopIteration); + monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_STOP_ITERATION); + PyErr_SetRaisedException(NULL); + } DECREF_INPUTS(); } + inst(END_SEND, (receiver, value -- value)) { + Py_DECREF(receiver); + } + + inst(INSTRUMENTED_END_SEND, (receiver, value -- value)) { + if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { + PyErr_SetObject(PyExc_StopIteration, value); + monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_STOP_ITERATION); + PyErr_SetRaisedException(NULL); + } + Py_DECREF(receiver); + } + inst(UNARY_NEGATIVE, (value -- res)) { res = PyNumber_Negative(value); DECREF_INPUTS(); @@ -756,6 +771,20 @@ dummy_func( DECREMENT_ADAPTIVE_COUNTER(cache->counter); #endif /* ENABLE_SPECIALIZATION */ assert(frame != &entry_frame); + if ((Py_TYPE(receiver) == &PyGen_Type || + Py_TYPE(receiver) == &PyCoro_Type) && ((PyGenObject *)receiver)->gi_frame_state < FRAME_EXECUTING) + { + PyGenObject *gen = (PyGenObject *)receiver; + _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; + frame->yield_offset = oparg; + STACK_SHRINK(1); + _PyFrame_StackPush(gen_frame, v); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + JUMPBY(INLINE_CACHE_ENTRIES_SEND + oparg); + DISPATCH_INLINED(gen_frame); + } if (Py_IsNone(v) && PyIter_Check(receiver)) { retval = Py_TYPE(receiver)->tp_iternext(receiver); } @@ -766,10 +795,10 @@ dummy_func( if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) ) { monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_RAISE); - assert(retval != NULL); - JUMPBY(oparg); } if (_PyGen_FetchStopIterationValue(&retval) == 0) { + assert(retval != NULL); + JUMPBY(oparg); } else { goto error; diff --git a/Python/compile.c b/Python/compile.c index b49eda314eeef1..a1771b77ceb3a8 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1789,8 +1789,7 @@ compiler_add_yield_from(struct compiler *c, location loc, int await) ADDOP(c, loc, CLEANUP_THROW); USE_LABEL(c, exit); - ADDOP_I(c, loc, SWAP, 2); - ADDOP(c, loc, POP_TOP); + ADDOP(c, loc, END_SEND); return SUCCESS; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 7470803dd52a03..1c1e469aaa19fd 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -236,19 +236,44 @@ } TARGET(INSTRUMENTED_END_FOR) { - PyObject *pop2 = PEEK(1); - PyObject *pop1 = PEEK(2); + PyObject *discard = PEEK(1); + PyObject *receiver = PEEK(2); /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ - PyErr_SetNone(PyExc_StopIteration); - monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_STOP_ITERATION); - PyErr_SetRaisedException(NULL); - Py_DECREF(pop1); - Py_DECREF(pop2); + if (PyGen_Check(receiver)) { + PyErr_SetNone(PyExc_StopIteration); + monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_STOP_ITERATION); + PyErr_SetRaisedException(NULL); + } + Py_DECREF(receiver); + Py_DECREF(discard); STACK_SHRINK(2); DISPATCH(); } + TARGET(END_SEND) { + PyObject *value = PEEK(1); + PyObject *receiver = PEEK(2); + Py_DECREF(receiver); + STACK_SHRINK(1); + POKE(1, value); + DISPATCH(); + } + + TARGET(INSTRUMENTED_END_SEND) { + PyObject *value = PEEK(1); + PyObject *receiver = PEEK(2); + if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { + PyErr_SetObject(PyExc_StopIteration, value); + monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_STOP_ITERATION); + PyErr_SetRaisedException(NULL); + } + Py_DECREF(receiver); + STACK_SHRINK(1); + POKE(1, value); + DISPATCH(); + } + TARGET(UNARY_NEGATIVE) { PyObject *value = PEEK(1); PyObject *res; @@ -962,6 +987,20 @@ DECREMENT_ADAPTIVE_COUNTER(cache->counter); #endif /* ENABLE_SPECIALIZATION */ assert(frame != &entry_frame); + if ((Py_TYPE(receiver) == &PyGen_Type || + Py_TYPE(receiver) == &PyCoro_Type) && ((PyGenObject *)receiver)->gi_frame_state < FRAME_EXECUTING) + { + PyGenObject *gen = (PyGenObject *)receiver; + _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; + frame->yield_offset = oparg; + STACK_SHRINK(1); + _PyFrame_StackPush(gen_frame, v); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + JUMPBY(INLINE_CACHE_ENTRIES_SEND + oparg); + DISPATCH_INLINED(gen_frame); + } if (Py_IsNone(v) && PyIter_Check(receiver)) { retval = Py_TYPE(receiver)->tp_iternext(receiver); } @@ -972,10 +1011,10 @@ if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) ) { monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_RAISE); - assert(retval != NULL); - JUMPBY(oparg); } if (_PyGen_FetchStopIterationValue(&retval) == 0) { + assert(retval != NULL); + JUMPBY(oparg); } else { goto error; diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 3c5750c4b40aa2..ab254aff29679a 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -51,6 +51,8 @@ static const int8_t EVENT_FOR_OPCODE[256] = { [INSTRUMENTED_FOR_ITER] = PY_MONITORING_EVENT_BRANCH, [END_FOR] = PY_MONITORING_EVENT_STOP_ITERATION, [INSTRUMENTED_END_FOR] = PY_MONITORING_EVENT_STOP_ITERATION, + [END_SEND] = PY_MONITORING_EVENT_STOP_ITERATION, + [INSTRUMENTED_END_SEND] = PY_MONITORING_EVENT_STOP_ITERATION, }; static const bool OPCODE_HAS_EVENT[256] = { @@ -87,6 +89,8 @@ static const bool OPCODE_HAS_EVENT[256] = { [INSTRUMENTED_FOR_ITER] = true, [END_FOR] = true, [INSTRUMENTED_END_FOR] = true, + [END_SEND] = true, + [INSTRUMENTED_END_SEND] = true, }; static const uint8_t DE_INSTRUMENT[256] = { @@ -107,6 +111,7 @@ static const uint8_t DE_INSTRUMENT[256] = { [INSTRUMENTED_COMPARE_AND_BRANCH] = COMPARE_AND_BRANCH, [INSTRUMENTED_FOR_ITER] = FOR_ITER, [INSTRUMENTED_END_FOR] = END_FOR, + [INSTRUMENTED_END_SEND] = END_SEND, }; static const uint8_t INSTRUMENTED_OPCODES[256] = { @@ -142,6 +147,8 @@ static const uint8_t INSTRUMENTED_OPCODES[256] = { [INSTRUMENTED_COMPARE_AND_BRANCH] = INSTRUMENTED_COMPARE_AND_BRANCH, [END_FOR] = INSTRUMENTED_END_FOR, [INSTRUMENTED_END_FOR] = INSTRUMENTED_END_FOR, + [END_SEND] = INSTRUMENTED_END_SEND, + [INSTRUMENTED_END_SEND] = INSTRUMENTED_END_SEND, [INSTRUMENTED_LINE] = INSTRUMENTED_LINE, [INSTRUMENTED_INSTRUCTION] = INSTRUMENTED_INSTRUCTION, diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index d9f4b4adfa48e0..def18489556adc 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -39,6 +39,7 @@ call_profile_func(_PyLegacyEventHandler *self, PyObject *arg) Py_RETURN_NONE; } PyFrameObject* frame = PyEval_GetFrame(); + assert(frame); if (frame == NULL) { PyErr_SetString(PyExc_SystemError, "Missing frame when calling profile function."); @@ -248,6 +249,7 @@ sys_trace_instruction_func( assert(kwnames == NULL); assert(PyVectorcall_NARGS(nargsf) == 2); PyFrameObject* frame = PyEval_GetFrame(); + assert(frame); if (frame == NULL) { PyErr_SetString(PyExc_SystemError, "Missing frame when calling trace function."); @@ -304,6 +306,7 @@ sys_trace_line_func( int line = _PyLong_AsInt(args[1]); assert(line >= 0); PyFrameObject* frame = PyEval_GetFrame(); + assert(frame); if (frame == NULL) { PyErr_SetString(PyExc_SystemError, "Missing frame when calling trace function."); @@ -334,6 +337,7 @@ sys_trace_jump_func( int to = _PyLong_AsInt(args[2]); assert(to >= 0); PyFrameObject* frame = PyEval_GetFrame(); + assert(frame); if (frame == NULL) { PyErr_SetString(PyExc_SystemError, "Missing frame when calling trace function."); diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index 52822c6a99a46c..842dbbb49d4ba3 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -42,6 +42,10 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 1+1; case INSTRUMENTED_END_FOR: return 2; + case END_SEND: + return 2; + case INSTRUMENTED_END_SEND: + return 2; case UNARY_NEGATIVE: return 1; case UNARY_NOT: @@ -432,6 +436,10 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 0+0; case INSTRUMENTED_END_FOR: return 0; + case END_SEND: + return 1; + case INSTRUMENTED_END_SEND: + return 1; case UNARY_NEGATIVE: return 1; case UNARY_NOT: @@ -813,6 +821,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[256] = { [PUSH_NULL] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [END_FOR] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [INSTRUMENTED_END_FOR] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, + [END_SEND] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, + [INSTRUMENTED_END_SEND] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [UNARY_NEGATIVE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [UNARY_NOT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [UNARY_INVERT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 9281ce70d735a4..48825405f39748 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -4,17 +4,18 @@ static void *opcode_targets[256] = { &&TARGET_PUSH_NULL, &&TARGET_INTERPRETER_EXIT, &&TARGET_END_FOR, + &&TARGET_END_SEND, &&TARGET_BINARY_OP_ADD_FLOAT, &&TARGET_BINARY_OP_ADD_INT, &&TARGET_BINARY_OP_ADD_UNICODE, - &&TARGET_BINARY_OP_INPLACE_ADD_UNICODE, &&TARGET_NOP, - &&TARGET_BINARY_OP_MULTIPLY_FLOAT, + &&TARGET_BINARY_OP_INPLACE_ADD_UNICODE, &&TARGET_UNARY_NEGATIVE, &&TARGET_UNARY_NOT, + &&TARGET_BINARY_OP_MULTIPLY_FLOAT, &&TARGET_BINARY_OP_MULTIPLY_INT, - &&TARGET_BINARY_OP_SUBTRACT_FLOAT, &&TARGET_UNARY_INVERT, + &&TARGET_BINARY_OP_SUBTRACT_FLOAT, &&TARGET_BINARY_OP_SUBTRACT_INT, &&TARGET_BINARY_SUBSCR_DICT, &&TARGET_BINARY_SUBSCR_GETITEM, @@ -23,20 +24,20 @@ static void *opcode_targets[256] = { &&TARGET_CALL_PY_EXACT_ARGS, &&TARGET_CALL_PY_WITH_DEFAULTS, &&TARGET_CALL_BOUND_METHOD_EXACT_ARGS, - &&TARGET_CALL_BUILTIN_CLASS, &&TARGET_BINARY_SUBSCR, &&TARGET_BINARY_SLICE, &&TARGET_STORE_SLICE, + &&TARGET_CALL_BUILTIN_CLASS, &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS, - &&TARGET_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, &&TARGET_GET_LEN, &&TARGET_MATCH_MAPPING, &&TARGET_MATCH_SEQUENCE, &&TARGET_MATCH_KEYS, - &&TARGET_CALL_NO_KW_BUILTIN_FAST, + &&TARGET_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, &&TARGET_PUSH_EXC_INFO, &&TARGET_CHECK_EXC_MATCH, &&TARGET_CHECK_EG_MATCH, + &&TARGET_CALL_NO_KW_BUILTIN_FAST, &&TARGET_CALL_NO_KW_BUILTIN_O, &&TARGET_CALL_NO_KW_ISINSTANCE, &&TARGET_CALL_NO_KW_LEN, @@ -47,7 +48,6 @@ static void *opcode_targets[256] = { &&TARGET_CALL_NO_KW_STR_1, &&TARGET_CALL_NO_KW_TUPLE_1, &&TARGET_CALL_NO_KW_TYPE_1, - &&TARGET_COMPARE_AND_BRANCH_FLOAT, &&TARGET_WITH_EXCEPT_START, &&TARGET_GET_AITER, &&TARGET_GET_ANEXT, @@ -55,38 +55,38 @@ static void *opcode_targets[256] = { &&TARGET_BEFORE_WITH, &&TARGET_END_ASYNC_FOR, &&TARGET_CLEANUP_THROW, + &&TARGET_COMPARE_AND_BRANCH_FLOAT, &&TARGET_COMPARE_AND_BRANCH_INT, &&TARGET_COMPARE_AND_BRANCH_STR, &&TARGET_FOR_ITER_LIST, - &&TARGET_FOR_ITER_TUPLE, &&TARGET_STORE_SUBSCR, &&TARGET_DELETE_SUBSCR, + &&TARGET_FOR_ITER_TUPLE, &&TARGET_FOR_ITER_RANGE, &&TARGET_FOR_ITER_GEN, &&TARGET_LOAD_ATTR_CLASS, &&TARGET_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, &&TARGET_LOAD_ATTR_INSTANCE_VALUE, - &&TARGET_LOAD_ATTR_MODULE, &&TARGET_GET_ITER, &&TARGET_GET_YIELD_FROM_ITER, - &&TARGET_LOAD_ATTR_PROPERTY, + &&TARGET_LOAD_ATTR_MODULE, &&TARGET_LOAD_BUILD_CLASS, + &&TARGET_LOAD_ATTR_PROPERTY, &&TARGET_LOAD_ATTR_SLOT, - &&TARGET_LOAD_ATTR_WITH_HINT, &&TARGET_LOAD_ASSERTION_ERROR, &&TARGET_RETURN_GENERATOR, + &&TARGET_LOAD_ATTR_WITH_HINT, &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, &&TARGET_LOAD_ATTR_METHOD_NO_DICT, &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, &&TARGET_LOAD_CONST__LOAD_FAST, &&TARGET_LOAD_FAST__LOAD_CONST, &&TARGET_LOAD_FAST__LOAD_FAST, - &&TARGET_LOAD_GLOBAL_BUILTIN, &&TARGET_RETURN_VALUE, - &&TARGET_LOAD_GLOBAL_MODULE, + &&TARGET_LOAD_GLOBAL_BUILTIN, &&TARGET_SETUP_ANNOTATIONS, + &&TARGET_LOAD_GLOBAL_MODULE, &&TARGET_STORE_ATTR_INSTANCE_VALUE, - &&TARGET_STORE_ATTR_SLOT, &&TARGET_PREP_RERAISE_STAR, &&TARGET_POP_EXCEPT, &&TARGET_STORE_NAME, @@ -112,7 +112,7 @@ static void *opcode_targets[256] = { &&TARGET_JUMP_FORWARD, &&TARGET_JUMP_IF_FALSE_OR_POP, &&TARGET_JUMP_IF_TRUE_OR_POP, - &&TARGET_STORE_ATTR_WITH_HINT, + &&TARGET_STORE_ATTR_SLOT, &&TARGET_POP_JUMP_IF_FALSE, &&TARGET_POP_JUMP_IF_TRUE, &&TARGET_LOAD_GLOBAL, @@ -142,7 +142,7 @@ static void *opcode_targets[256] = { &&TARGET_JUMP_BACKWARD, &&TARGET_COMPARE_AND_BRANCH, &&TARGET_CALL_FUNCTION_EX, - &&TARGET_STORE_FAST__LOAD_FAST, + &&TARGET_STORE_ATTR_WITH_HINT, &&TARGET_EXTENDED_ARG, &&TARGET_LIST_APPEND, &&TARGET_SET_ADD, @@ -152,24 +152,24 @@ static void *opcode_targets[256] = { &&TARGET_YIELD_VALUE, &&TARGET_RESUME, &&TARGET_MATCH_CLASS, + &&TARGET_STORE_FAST__LOAD_FAST, &&TARGET_STORE_FAST__STORE_FAST, - &&TARGET_STORE_SUBSCR_DICT, &&TARGET_FORMAT_VALUE, &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_STRING, + &&TARGET_STORE_SUBSCR_DICT, &&TARGET_STORE_SUBSCR_LIST_INT, &&TARGET_UNPACK_SEQUENCE_LIST, &&TARGET_UNPACK_SEQUENCE_TUPLE, - &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, &&TARGET_LIST_EXTEND, &&TARGET_SET_UPDATE, &&TARGET_DICT_MERGE, &&TARGET_DICT_UPDATE, + &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, &&TARGET_SEND_GEN, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_CALL, &&TARGET_KW_NAMES, &&TARGET_CALL_INTRINSIC_1, @@ -251,7 +251,7 @@ static void *opcode_targets[256] = { &&TARGET_INSTRUMENTED_POP_JUMP_IF_FALSE, &&TARGET_INSTRUMENTED_POP_JUMP_IF_TRUE, &&TARGET_INSTRUMENTED_END_FOR, - &&_unknown_opcode, + &&TARGET_INSTRUMENTED_END_SEND, &&TARGET_INSTRUMENTED_INSTRUCTION, &&TARGET_INSTRUMENTED_LINE, &&TARGET_DO_TRACING From cdb2bda5648b072713bddacceaed5c168f45f23e Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 14 Feb 2023 13:48:11 +0000 Subject: [PATCH 040/116] Correctly set StopIteration in INSTRUMENTED_END_FOR. --- Python/bytecodes.c | 4 ++-- Python/generated_cases.c.h | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 5b1ee975cb0944..5449784fc5d498 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -172,11 +172,11 @@ dummy_func( macro(END_FOR) = POP_TOP + POP_TOP; - inst(INSTRUMENTED_END_FOR, (receiver, discard --)) { + inst(INSTRUMENTED_END_FOR, (receiver, value --)) { /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ if (PyGen_Check(receiver)) { - PyErr_SetNone(PyExc_StopIteration); + PyErr_SetObject(PyExc_StopIteration, value); monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_STOP_ITERATION); PyErr_SetRaisedException(NULL); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 1c1e469aaa19fd..2c932829014f31 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -236,17 +236,17 @@ } TARGET(INSTRUMENTED_END_FOR) { - PyObject *discard = PEEK(1); + PyObject *value = PEEK(1); PyObject *receiver = PEEK(2); /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ if (PyGen_Check(receiver)) { - PyErr_SetNone(PyExc_StopIteration); + PyErr_SetObject(PyExc_StopIteration, value); monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_STOP_ITERATION); PyErr_SetRaisedException(NULL); } Py_DECREF(receiver); - Py_DECREF(discard); + Py_DECREF(value); STACK_SHRINK(2); DISPATCH(); } From 0148fa35da11156db5d6efb8c8ec2b4521534680 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 14 Feb 2023 15:35:48 +0000 Subject: [PATCH 041/116] Set last traced line when jumping in debugger. --- Objects/frameobject.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 3e5d347ff5dca8..ced90a91d7b738 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -857,6 +857,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore start_stack = pop_value(start_stack); } /* Finally set the new lasti and return OK. */ + f->f_last_traced_line = new_lineno; f->f_lineno = 0; f->f_frame->prev_instr = _PyCode_CODE(f->f_frame->f_code) + best_addr; return 0; From c5fb4f48d58eb28b2b244e57160ee487a351bdc9 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 17 Feb 2023 13:32:03 +0000 Subject: [PATCH 042/116] Fix test_dis to account for END_SEND. --- Lib/test/test_dis.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index b80281580c4083..92ead0e71fec08 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -479,8 +479,7 @@ async def _asyncwith(c): YIELD_VALUE 2 RESUME 3 JUMP_BACKWARD_NO_INTERRUPT 5 (to 14) - >> SWAP 2 - POP_TOP + >> END_SEND POP_TOP %3d LOAD_CONST 1 (1) @@ -492,11 +491,11 @@ async def _asyncwith(c): CALL 2 GET_AWAITABLE 2 LOAD_CONST 0 (None) - >> SEND 3 (to 64) + >> SEND 3 (to 62) YIELD_VALUE 2 RESUME 3 - JUMP_BACKWARD_NO_INTERRUPT 5 (to 54) - >> POP_TOP + JUMP_BACKWARD_NO_INTERRUPT 5 (to 52) + >> END_SEND POP_TOP %3d LOAD_CONST 2 (2) @@ -504,21 +503,20 @@ async def _asyncwith(c): RETURN_CONST 0 (None) %3d >> CLEANUP_THROW - JUMP_BACKWARD 27 (to 24) + JUMP_BACKWARD 26 (to 24) >> CLEANUP_THROW - JUMP_BACKWARD 9 (to 64) + JUMP_BACKWARD 9 (to 62) >> PUSH_EXC_INFO WITH_EXCEPT_START GET_AWAITABLE 2 LOAD_CONST 0 (None) - >> SEND 4 (to 102) + >> SEND 4 (to 100) YIELD_VALUE 3 RESUME 3 - JUMP_BACKWARD_NO_INTERRUPT 5 (to 90) + JUMP_BACKWARD_NO_INTERRUPT 5 (to 88) >> CLEANUP_THROW - >> SWAP 2 - POP_TOP - POP_JUMP_IF_TRUE 1 (to 110) + >> END_SEND + POP_JUMP_IF_TRUE 1 (to 106) RERAISE 2 >> POP_TOP POP_EXCEPT From 7fa431b603992cfb4917c9e11450d2ce0acb8baf Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 17 Feb 2023 13:32:47 +0000 Subject: [PATCH 043/116] Add a few more tests. --- Lib/test/test_monitoring.py | 134 +++++++++++++++++++++++++++++++++--- Python/instrumentation.c | 4 +- 2 files changed, 127 insertions(+), 11 deletions(-) diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 37961cc981fb83..faa6f6b9073c81 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -608,29 +608,43 @@ def func2(): class ExceptionRecorder: + event_type = E.RAISE + def __init__(self, events): self.events = events def __call__(self, code, offset, exc): - self.events.append(type(exc)) + self.events.append(("raise", type(exc))) -class ExceptionMontoringTest(unittest.TestCase): +class CheckEvents(unittest.TestCase): - def check_events(self, func, expected, tool=TEST_TOOL, events=E.RAISE): + def check_events(self, func, expected, tool=TEST_TOOL, recorders=(ExceptionRecorder,)): try: self.assertEqual(sys.monitoring._all_events(), {}) event_list = [] - recorder = ExceptionRecorder(event_list) - sys.monitoring.register_callback(tool, events, recorder) - sys.monitoring.set_events(tool, events) + all_events = 0 + for recorder in recorders: + ev = recorder.event_type + sys.monitoring.register_callback(tool, ev, recorder(event_list)) + all_events |= ev + sys.monitoring.set_events(tool, all_events) func() sys.monitoring.set_events(tool, 0) - sys.monitoring.register_callback(tool, events, None) + for recorder in recorders: + sys.monitoring.register_callback(tool, recorder.event_type, None) self.assertEqual(event_list, expected) finally: sys.monitoring.set_events(tool, 0) - sys.monitoring.register_callback(tool, events, None) + for recorder in recorders: + sys.monitoring.register_callback(tool, recorder.event_type, None) + +class StopiterationRecorder(ExceptionRecorder): + + event_type = E.STOP_ITERATION +class ExceptionMontoringTest(CheckEvents): + + recorder = ExceptionRecorder def test_simple_try_except(self): @@ -642,7 +656,7 @@ def func1(): line = 5 line = 6 - self.check_events(func1, [KeyError]) + self.check_events(func1, [("raise", KeyError)]) def gen(): yield 1 @@ -652,4 +666,104 @@ def implicit_stop_iteration(): for _ in gen(): pass - self.check_events(implicit_stop_iteration, [StopIteration], events=E.STOP_ITERATION) + self.check_events(implicit_stop_iteration, [("raise", StopIteration)], recorders=(StopiterationRecorder,)) + +class LineRecorder: + + event_type = E.LINE + + + def __init__(self, events): + self.events = events + + def __call__(self, code, line): + self.events.append(("line", code.co_name, line - code.co_firstlineno)) + +class CallRecorder: + + event_type = E.CALL + + def __init__(self, events): + self.events = events + + def __call__(self, code, offset, func, arg): + self.events.append(("call", func.__name__, arg)) + +class CEventRecorder: + + def __init__(self, events): + self.events = events + + def __call__(self, code, offset, func, arg): + self.events.append((self.event_name, func.__name__, arg)) + +class CReturnRecorder(CEventRecorder): + + event_type = E.C_RETURN + event_name = "C return" + +class CRaiseRecorder(CEventRecorder): + + event_type = E.C_RAISE + event_name = "C raise" + +MANY_RECORDERS = ExceptionRecorder, CallRecorder, LineRecorder, CReturnRecorder, CRaiseRecorder + +class TestManyEvents(CheckEvents): + + def test_simple(self): + + def func1(): + line1 = 1 + line2 = 2 + line3 = 3 + + self.check_events(func1, recorders = MANY_RECORDERS, expected = [ + ('line', 'check_events', 10), + ('call', 'func1', None), + ('line', 'func1', 1), + ('line', 'func1', 2), + ('line', 'func1', 3), + ('line', 'check_events', 11), + ('call', 'set_events', 2)]) + + def test_c_call(self): + + def func2(): + line1 = 1 + [].append(2) + line3 = 3 + + self.check_events(func2, recorders = MANY_RECORDERS, expected = [ + ('line', 'check_events', 10), + ('call', 'func2', None), + ('line', 'func2', 1), + ('line', 'func2', 2), + ('call', 'append', [2]), + ('C return', 'append', [2]), + ('line', 'func2', 3), + ('line', 'check_events', 11), + ('call', 'set_events', 2)]) + + def test_try_except(self): + + def func3(): + try: + line = 2 + raise KeyError + except: + line = 5 + line = 6 + + self.check_events(func3, recorders = MANY_RECORDERS, expected = [ + ('line', 'check_events', 10), + ('call', 'func3', None), + ('line', 'func3', 1), + ('line', 'func3', 2), + ('line', 'func3', 3), + ('raise', KeyError), + ('line', 'func3', 4), + ('line', 'func3', 5), + ('line', 'func3', 6), + ('line', 'check_events', 11), + ('call', 'set_events', 2)]) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index ab254aff29679a..9051eb4e550c60 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1403,8 +1403,10 @@ initialize_lines(PyCodeObject *code) switch (inst.deinstrumented_opcode) { case END_ASYNC_FOR: case END_FOR: + case END_SEND: case RESUME: /* END_FOR cannot start a line, as it is skipped by FOR_ITER + * END_SEND cannot start a line, as it is skipped by SEND * RESUME must not be instrumented with INSTRUMENT_LINE */ line_data[i].original_opcode = 0; break; @@ -1807,7 +1809,7 @@ monitoring_register_callback_impl(PyObject *module, int tool_id, int event, return NULL; } if (_Py_popcount32(event) != 1) { - PyErr_SetString(PyExc_ValueError, "The callaback can only be set for one event at a time"); + PyErr_SetString(PyExc_ValueError, "The callback can only be set for one event at a time"); } int event_id = _Py_bit_length(event)-1; if (event_id < 0 || event_id >= PY_MONITORING_EVENTS) { From 25bbc614ee6575c1d5bc58c05530d5371acd1c17 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 17 Feb 2023 16:00:29 +0000 Subject: [PATCH 044/116] delete commented out code --- Python/legacy_tracing.c | 53 ----------------------------------------- 1 file changed, 53 deletions(-) diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index def18489556adc..5d9fc6917e5a80 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -169,65 +169,12 @@ sys_trace_return( _PyLegacyEventHandler *self, PyObject *const *args, size_t nargsf, PyObject *kwnames ) { - PyThreadState *tstate = _PyThreadState_GET(); assert(!PyErr_Occurred()); assert(kwnames == NULL); assert(PyVectorcall_NARGS(nargsf) == 3); assert(PyCode_Check(args[0])); - PyCodeObject *code = (PyCodeObject *)args[0]; PyObject *val = args[2]; PyObject *res = call_trace_func(self, val); -// if ((code->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) && -// res != NULL && tstate->c_tracefunc != NULL -// ) { -// /* Fake PEP 380 StopIteration exception event */ -// Py_DECREF(res); -// PyObject *stop_iter_args[2] = { NULL, val }; -// PyObject *exc = PyObject_Vectorcall(PyExc_StopIteration, -// &stop_iter_args[1], -// 1 | PY_VECTORCALL_ARGUMENTS_OFFSET, -// NULL); -// if (exc == NULL) { -// return NULL; -// } -// /* The event occurs in the caller frame, for historical reasons */ -// _PyInterpreterFrame *frame = _PyEval_GetFrame(); -// frame = frame->previous; -// while (frame && _PyFrame_IsIncomplete(frame)) { -// frame = frame->previous; -// } -// PyFrameObject* frame_obj = NULL; -// if (frame != NULL) { -// frame_obj = _PyFrame_GetFrameObject(frame); -// } else { -// PyErr_SetString(PyExc_SystemError, -// "Missing frame when calling trace function."); -// return NULL; -// } -// if (frame_obj == NULL) { -// Py_DECREF(exc); -// return NULL; -// } -// PyObject *type = (PyObject *)Py_TYPE(exc); -// PyObject *tb = PyException_GetTraceback(exc); -// if (tb == NULL) { -// tb = Py_NewRef(Py_None); -// } -// PyObject * tuple = PyTuple_Pack(3, type, exc, tb); -// Py_DECREF(tb); -// if (tuple == NULL) { -// Py_DECREF(exc); -// return NULL; -// } -// Py_INCREF(frame_obj); -// int err = tstate->c_tracefunc(tstate->c_traceobj, frame_obj, PyTrace_EXCEPTION, tuple); -// Py_DECREF(frame_obj); -// Py_DECREF(tuple); -// Py_DECREF(exc); -// if (err) { -// return NULL; -// } -// } return res; } From 5629a3e859f6e34755957098b6780f67d164896b Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 17 Feb 2023 16:01:37 +0000 Subject: [PATCH 045/116] Fix last (known) inconsistency in sys.settrace behaviour --- Lib/test/test_sys_settrace.py | 17 +++++++++++++++++ Objects/frameobject.c | 1 + 2 files changed, 18 insertions(+) diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index a251b2272e95eb..9cf5d3e9e19b60 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -2809,5 +2809,22 @@ def foo(*args): sys.settrace(sys.gettrace()) +class TestLinesAfterTraceStarted(TraceTestCase): + + def test_events(self): + tracer = Tracer() + sys._getframe().f_trace = tracer.trace + sys.settrace(tracer.trace) + line = 4 + line = 5 + sys.settrace(None) + self.compare_events( + TestLinesAfterTraceStarted.test_events.__code__.co_firstlineno, + tracer.events, [ + (4, 'line'), + (5, 'line'), + (6, 'line')]) + + if __name__ == "__main__": unittest.main() diff --git a/Objects/frameobject.c b/Objects/frameobject.c index ced90a91d7b738..fb8e6423b6e6fe 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -879,6 +879,7 @@ frame_settrace(PyFrameObject *f, PyObject* v, void *closure) v = NULL; } Py_XSETREF(f->f_trace, Py_XNewRef(v)); + f->f_last_traced_line = -1; return 0; } From 477cc53e5709d21e8fb92297dc68043751d33767 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 20 Feb 2023 17:25:07 +0000 Subject: [PATCH 046/116] More clearly differentiate between instrumented events and non-instrumented events. --- Include/cpython/code.h | 10 +- Include/internal/pycore_instruments.h | 1 - Include/internal/pycore_interp.h | 18 +++- Python/ceval.c | 18 +++- Python/instrumentation.c | 141 +++++++++++++++----------- Python/pystate.c | 14 ++- 6 files changed, 127 insertions(+), 75 deletions(-) diff --git a/Include/cpython/code.h b/Include/cpython/code.h index ffb6d513fcf7fd..b6d737f7c3fbcb 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -9,10 +9,11 @@ extern "C" { #endif #define PY_MONITORING_EVENTS 16 +#define PY_MONITORING_INSTRUMENTED_EVENTS 10 -typedef struct _Py_MonitoringMatrix { - uint8_t tools[PY_MONITORING_EVENTS]; -} _Py_MonitoringMatrix; +typedef struct _Py_InstrumentationMatrix { + uint8_t tools[PY_MONITORING_INSTRUMENTED_EVENTS]; +} _Py_InstrumentationMatrix; /* Each instruction in a code object is a fixed-width value, * currently 2 bytes: 1-byte opcode + 1-byte oparg. The EXTENDED_ARG @@ -55,7 +56,8 @@ typedef struct { } _PyCoLineInstrumentationData; typedef struct { - _Py_MonitoringMatrix matrix; + _Py_InstrumentationMatrix local_instrumentation; + _Py_InstrumentationMatrix current_instrumentation; uint8_t *tools; _PyCoLineInstrumentationData *lines; uint8_t *line_tools; diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index 34dd2551f8aaa6..c7c438ff05ca71 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -27,7 +27,6 @@ extern "C" { #define PY_MONITORING_EVENT_BRANCH 8 #define PY_MONITORING_EVENT_STOP_ITERATION 9 -#define PY_MONITORING_INSTRUMENTED_EVENTS 10 /* Grouped events */ diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 184db0505864e6..1cf517864a8abc 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -190,14 +190,14 @@ struct _is { struct callable_cache callable_cache; PyCodeObject *interpreter_trampoline; - _Py_MonitoringMatrix monitoring_matrix; - uint8_t required_monitoring_bytes; - /* Tools numbered 1-8. 0 is the dispatcher/sole tool */ - struct _instrumentation_tool tools[PY_MONITORING_TOOL_IDS]; - PyObject *monitoring_tool_names[PY_MONITORING_TOOL_IDS]; + _Py_InstrumentationMatrix instrumented_events; + uint8_t other_events[PY_MONITORING_EVENTS - PY_MONITORING_INSTRUMENTED_EVENTS]; bool f_opcode_trace_set; bool sys_profile_initialized; bool sys_trace_initialized; + /* Tools numbered 0-7 */ + struct _instrumentation_tool tools[PY_MONITORING_TOOL_IDS]; + PyObject *monitoring_tool_names[PY_MONITORING_TOOL_IDS]; Py_ssize_t sys_profiling_threads; Py_ssize_t sys_tracing_threads; @@ -248,6 +248,14 @@ PyAPI_FUNC(int) _PyInterpreterState_IDInitref(PyInterpreterState *); PyAPI_FUNC(int) _PyInterpreterState_IDIncref(PyInterpreterState *); PyAPI_FUNC(void) _PyInterpreterState_IDDecref(PyInterpreterState *); +static inline uint8_t +_PyInterpreterState_GetTools(PyInterpreterState * is, int event) +{ + assert(event >= PY_MONITORING_INSTRUMENTED_EVENTS && event < PY_MONITORING_EVENTS); + return is->other_events[event-PY_MONITORING_INSTRUMENTED_EVENTS]; +} + + #ifdef __cplusplus } #endif diff --git a/Python/ceval.c b/Python/ceval.c index 3c6b701f120e99..3f329d5df9ca44 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1935,13 +1935,21 @@ unpack_iterable(PyThreadState *tstate, PyObject *v, return 0; } + static void monitor_raise(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, int event) { - if (tstate->interp->monitoring_matrix.tools[event] == 0) { - return; + if (event < PY_MONITORING_INSTRUMENTED_EVENTS) { + if (frame->f_code->_co_instrumentation.monitoring_data->current_instrumentation.tools[event] == 0) { + return; + } + } + else { + if (_PyInterpreterState_GetTools(tstate->interp, event) == 0) { + return; + } } PyObject *exc = PyErr_GetRaisedException(); assert(exc != NULL); @@ -1961,7 +1969,7 @@ monitor_unwind(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) { - if (tstate->interp->monitoring_matrix.tools[PY_MONITORING_EVENT_PY_UNWIND] == 0) { + if (_PyInterpreterState_GetTools(tstate->interp, PY_MONITORING_EVENT_PY_UNWIND) == 0) { return; } _Py_call_instrumentation_exc0(tstate, PY_MONITORING_EVENT_PY_UNWIND, frame, instr); @@ -1973,7 +1981,7 @@ monitor_handled(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *exc) { - if (tstate->interp->monitoring_matrix.tools[PY_MONITORING_EVENT_EXCEPTION_HANDLED] == 0) { + if (_PyInterpreterState_GetTools(tstate->interp, PY_MONITORING_EVENT_EXCEPTION_HANDLED) == 0) { return; } _Py_call_instrumentation_arg(tstate, PY_MONITORING_EVENT_EXCEPTION_HANDLED, frame, instr, exc); @@ -1984,7 +1992,7 @@ monitor_throw(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) { - if (tstate->interp->monitoring_matrix.tools[PY_MONITORING_EVENT_PY_THROW] == 0) { + if (_PyInterpreterState_GetTools(tstate->interp, PY_MONITORING_EVENT_PY_THROW) == 0) { return; } _Py_call_instrumentation_exc0(tstate, PY_MONITORING_EVENT_PY_THROW, frame, instr); diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 9051eb4e550c60..e6886c30a8b373 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -161,9 +161,9 @@ is_instrumented(int opcode) { } static inline bool -matrix_equals(_Py_MonitoringMatrix a, _Py_MonitoringMatrix b) +matrix_equals(_Py_InstrumentationMatrix a, _Py_InstrumentationMatrix b) { - for (int i = 0; i < PY_MONITORING_EVENTS; i++) { + for (int i = 0; i < PY_MONITORING_INSTRUMENTED_EVENTS; i++) { if (a.tools[i] != b.tools[i]) { return false; } @@ -171,30 +171,30 @@ matrix_equals(_Py_MonitoringMatrix a, _Py_MonitoringMatrix b) return true; } -static inline _Py_MonitoringMatrix -matrix_sub(_Py_MonitoringMatrix a, _Py_MonitoringMatrix b) +static inline _Py_InstrumentationMatrix +matrix_sub(_Py_InstrumentationMatrix a, _Py_InstrumentationMatrix b) { - _Py_MonitoringMatrix res; - for (int i = 0; i < PY_MONITORING_EVENTS; i++) { + _Py_InstrumentationMatrix res; + for (int i = 0; i < PY_MONITORING_INSTRUMENTED_EVENTS; i++) { res.tools[i] = a.tools[i] & ~b.tools[i]; } return res; } -static inline _Py_MonitoringMatrix -matrix_and(_Py_MonitoringMatrix a, _Py_MonitoringMatrix b) +static inline _Py_InstrumentationMatrix +matrix_and(_Py_InstrumentationMatrix a, _Py_InstrumentationMatrix b) { - _Py_MonitoringMatrix res; - for (int i = 0; i < PY_MONITORING_EVENTS; i++) { + _Py_InstrumentationMatrix res; + for (int i = 0; i < PY_MONITORING_INSTRUMENTED_EVENTS; i++) { res.tools[i] = a.tools[i] & b.tools[i]; } return res; } static inline bool -matrix_empty(_Py_MonitoringMatrix m) +matrix_empty(_Py_InstrumentationMatrix m) { - for (int i = 0; i < PY_MONITORING_EVENTS; i++) { + for (int i = 0; i < PY_MONITORING_INSTRUMENTED_EVENTS; i++) { if (m.tools[i]) { return false; } @@ -203,10 +203,10 @@ matrix_empty(_Py_MonitoringMatrix m) } static inline int -multiple_tools(_Py_MonitoringMatrix m) +multiple_tools(_Py_InstrumentationMatrix *m) { - for (int i = 0; i < PY_MONITORING_EVENTS; i++) { - if (_Py_popcount32(m.tools[i]) > 1) { + for (int i = 0; i < PY_MONITORING_INSTRUMENTED_EVENTS; i++) { + if (_Py_popcount32(m->tools[i]) > 1) { return true; } } @@ -214,21 +214,33 @@ multiple_tools(_Py_MonitoringMatrix m) } static inline void -matrix_set_bit(_Py_MonitoringMatrix *m, int event, int tool, int val) +interpreter_set_tool_event(PyInterpreterState *interp, int event, int tool, int val) { assert(0 <= event && event < PY_MONITORING_EVENTS); assert(0 <= tool && tool < PY_MONITORING_TOOL_IDS); assert(val == 0 || val == 1); - m->tools[event] &= ~(1 << tool); - m->tools[event] |= (val << tool); + uint8_t *tools; + if (event < PY_MONITORING_INSTRUMENTED_EVENTS) { + tools = &interp->instrumented_events.tools[event]; + } + else { + tools = &interp->other_events[event-PY_MONITORING_INSTRUMENTED_EVENTS]; + } + *tools &= ~(1 << tool); + *tools |= (val << tool); } static inline _PyMonitoringEventSet -matrix_get_events(_Py_MonitoringMatrix *m, int tool_id) +interpeter_get_events(PyInterpreterState *interp, int tool_id) { _PyMonitoringEventSet result = 0; - for (int e = 0; e < PY_MONITORING_EVENTS; e++) { - if ((m->tools[e] >> tool_id) & 1) { + for (int e = 0; e < PY_MONITORING_INSTRUMENTED_EVENTS; e++) { + if ((interp->instrumented_events.tools[e] >> tool_id) & 1) { + result |= (1 << e); + } + } + for (int e = PY_MONITORING_INSTRUMENTED_EVENTS; e < PY_MONITORING_EVENTS; e++) { + if ((interp->other_events[e] >> tool_id) & 1) { result |= (1 << e); } } @@ -829,6 +841,7 @@ remove_tools(PyCodeObject * code, int offset, int event, int tools) { assert(event != PY_MONITORING_EVENT_LINE); assert(event != PY_MONITORING_EVENT_INSTRUCTION); + assert(event < PY_MONITORING_INSTRUMENTED_EVENTS); assert(instruction_has_event(code, offset)); _PyCoInstrumentationData *monitoring = code->_co_instrumentation.monitoring_data; if (monitoring && monitoring->tools) { @@ -839,7 +852,7 @@ remove_tools(PyCodeObject * code, int offset, int event, int tools) } else { /* Single tool */ - uint8_t single_tool = code->_co_instrumentation.monitoring_data->matrix.tools[event]; + uint8_t single_tool = code->_co_instrumentation.monitoring_data->current_instrumentation.tools[event]; assert(_Py_popcount32(single_tool) <= 1); if (((single_tool & tools) == single_tool) && event < PY_MONITORING_INSTRUMENTED_EVENTS) { de_instrument(code, offset, event); @@ -861,7 +874,7 @@ remove_line_tools(PyCodeObject * code, int offset, int tools) } else { /* Single tool */ - uint8_t single_tool = code->_co_instrumentation.monitoring_data->matrix.tools[PY_MONITORING_EVENT_LINE]; + uint8_t single_tool = code->_co_instrumentation.monitoring_data->current_instrumentation.tools[PY_MONITORING_EVENT_LINE]; assert(_Py_popcount32(single_tool) <= 1); if (((single_tool & tools) == single_tool)) { de_instrument_line(code, offset); @@ -875,6 +888,7 @@ add_tools(PyCodeObject * code, int offset, int event, int tools) { assert(event != PY_MONITORING_EVENT_LINE); assert(event != PY_MONITORING_EVENT_INSTRUCTION); + assert(event < PY_MONITORING_INSTRUMENTED_EVENTS); assert(code->_co_instrumentation.monitoring_data); if (code->_co_instrumentation.monitoring_data && code->_co_instrumentation.monitoring_data->tools @@ -883,7 +897,7 @@ add_tools(PyCodeObject * code, int offset, int event, int tools) } else { /* Single tool */ - assert(PyInterpreterState_Get()->monitoring_matrix.tools[event] == tools); + assert(PyInterpreterState_Get()->instrumented_events.tools[event] == tools); assert(_Py_popcount32(tools) == 1); } if (event < PY_MONITORING_INSTRUMENTED_EVENTS) { @@ -894,7 +908,7 @@ add_tools(PyCodeObject * code, int offset, int event, int tools) static void add_line_tools(PyCodeObject * code, int offset, int tools) { - assert((PyInterpreterState_Get()->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE] & tools) == tools); + assert((PyInterpreterState_Get()->instrumented_events.tools[PY_MONITORING_EVENT_LINE] & tools) == tools); assert(code->_co_instrumentation.monitoring_data); if (code->_co_instrumentation.monitoring_data->line_tools ) { @@ -911,7 +925,7 @@ add_line_tools(PyCodeObject * code, int offset, int tools) static void add_per_instruction_tools(PyCodeObject * code, int offset, int tools) { - assert((PyInterpreterState_Get()->monitoring_matrix.tools[PY_MONITORING_EVENT_INSTRUCTION] & tools) == tools); + assert((PyInterpreterState_Get()->instrumented_events.tools[PY_MONITORING_EVENT_INSTRUCTION] & tools) == tools); assert(code->_co_instrumentation.monitoring_data); if (code->_co_instrumentation.monitoring_data->per_instruction_tools ) { @@ -939,7 +953,7 @@ remove_per_instruction_tools(PyCodeObject * code, int offset, int tools) } else { /* Single tool */ - uint8_t single_tool = code->_co_instrumentation.monitoring_data->matrix.tools[PY_MONITORING_EVENT_INSTRUCTION]; + uint8_t single_tool = code->_co_instrumentation.monitoring_data->current_instrumentation.tools[PY_MONITORING_EVENT_INSTRUCTION]; assert(_Py_popcount32(single_tool) <= 1); if (((single_tool & tools) == single_tool)) { de_instrument_per_instruction(code, offset); @@ -999,14 +1013,18 @@ get_tools_for_instruction(PyCodeObject * code, int i, int event) assert(event != PY_MONITORING_EVENT_LINE); assert(event != PY_MONITORING_EVENT_INSTRUCTION); _PyCoInstrumentationData *monitoring = code->_co_instrumentation.monitoring_data; - if (event < PY_MONITORING_INSTRUMENTED_EVENTS && monitoring && monitoring->tools) { + assert(monitoring != NULL); + if (event >= PY_MONITORING_INSTRUMENTED_EVENTS) { + return _PyInterpreterState_GetTools(PyInterpreterState_Get(), event); + } + else if (monitoring->tools) { tools = monitoring->tools[i]; } else { - tools = code->_co_instrumentation.monitoring_data->matrix.tools[event]; + tools = code->_co_instrumentation.monitoring_data->current_instrumentation.tools[event]; } - CHECK((tools & PyInterpreterState_Get()->monitoring_matrix.tools[event]) == tools); - CHECK((tools & code->_co_instrumentation.monitoring_data->matrix.tools[event]) == tools); + CHECK((tools & PyInterpreterState_Get()->instrumented_events.tools[event]) == tools); + CHECK((tools & code->_co_instrumentation.monitoring_data->current_instrumentation.tools[event]) == tools); return tools; } @@ -1053,7 +1071,7 @@ _Py_call_instrumentation( { PyCodeObject *code = frame->f_code; assert(is_instrumentation_up_to_date(code, tstate->interp)); - assert(matrix_equals(code->_co_instrumentation.monitoring_data->matrix, tstate->interp->monitoring_matrix)); + assert(matrix_equals(code->_co_instrumentation.monitoring_data->current_instrumentation, tstate->interp->instrumented_events)); int offset = instr - _PyCode_CODE(code); PyObject *offset_obj = PyLong_FromSsize_t(offset); if (offset_obj == NULL) { @@ -1073,7 +1091,7 @@ _Py_call_instrumentation_arg( { PyCodeObject *code = frame->f_code; assert(is_instrumentation_up_to_date(code, tstate->interp)); - assert(matrix_equals(code->_co_instrumentation.monitoring_data->matrix, tstate->interp->monitoring_matrix)); + assert(matrix_equals(code->_co_instrumentation.monitoring_data->current_instrumentation, tstate->interp->instrumented_events)); int offset = instr - _PyCode_CODE(code); PyObject *offset_obj = PyLong_FromSsize_t(offset); if (offset_obj == NULL) { @@ -1093,7 +1111,7 @@ _Py_call_instrumentation_2args( { PyCodeObject *code = frame->f_code; assert(is_instrumentation_up_to_date(code, tstate->interp)); - assert(matrix_equals(code->_co_instrumentation.monitoring_data->matrix, tstate->interp->monitoring_matrix)); + assert(matrix_equals(code->_co_instrumentation.monitoring_data->current_instrumentation, tstate->interp->instrumented_events)); int offset = instr - _PyCode_CODE(code); PyObject *offset_obj = PyLong_FromSsize_t(offset); if (offset_obj == NULL) { @@ -1153,9 +1171,9 @@ _Py_call_instrumentation_exc_vector( assert(args[1] == NULL); args[1] = (PyObject *)code; assert(code->_co_instrumentation.monitoring_version == tstate->interp->monitoring_version); - assert(matrix_equals(code->_co_instrumentation.monitoring_data->matrix, tstate->interp->monitoring_matrix)); int offset = instr - _PyCode_CODE(code); - uint8_t tools = code->_co_instrumentation.monitoring_data->matrix.tools[event]; + assert(event >= PY_MONITORING_INSTRUMENTED_EVENTS); + uint8_t tools = tstate->interp->instrumented_events.tools[event]; PyObject *offset_obj = PyLong_FromSsize_t(offset); int err; if (offset_obj == NULL) { @@ -1221,7 +1239,7 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, frame->prev_instr = instr; PyCodeObject *code = frame->f_code; assert(is_instrumentation_up_to_date(code, tstate->interp)); - assert(matrix_equals(code->_co_instrumentation.monitoring_data->matrix, tstate->interp->monitoring_matrix)); + assert(matrix_equals(code->_co_instrumentation.monitoring_data->current_instrumentation, tstate->interp->instrumented_events)); int i = instr - _PyCode_CODE(code); _PyCoInstrumentation *instrumentation = &code->_co_instrumentation; _PyCoLineInstrumentationData *line_data = &instrumentation->monitoring_data->lines[i]; @@ -1234,7 +1252,7 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, int line = compute_line(code, i, line_delta); uint8_t tools = code->_co_instrumentation.monitoring_data->line_tools != NULL ? code->_co_instrumentation.monitoring_data->line_tools[i] : - interp->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE]; + interp->instrumented_events.tools[PY_MONITORING_EVENT_LINE]; PyObject *line_obj = PyLong_FromSsize_t(line); if (line_obj == NULL) { return -1; @@ -1274,7 +1292,7 @@ _Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame* { PyCodeObject *code = frame->f_code; assert(is_instrumentation_up_to_date(code, tstate->interp)); - assert(matrix_equals(code->_co_instrumentation.monitoring_data->matrix, tstate->interp->monitoring_matrix)); + assert(matrix_equals(code->_co_instrumentation.monitoring_data->current_instrumentation, tstate->interp->instrumented_events)); int offset = instr - _PyCode_CODE(code); _PyCoInstrumentationData *instrumentation_data = code->_co_instrumentation.monitoring_data; assert(instrumentation_data->per_instruction_opcodes); @@ -1285,7 +1303,7 @@ _Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame* PyInterpreterState *interp = tstate->interp; uint8_t tools = instrumentation_data->per_instruction_tools != NULL ? instrumentation_data->per_instruction_tools[offset] : - interp->monitoring_matrix.tools[PY_MONITORING_EVENT_INSTRUCTION]; + interp->instrumented_events.tools[PY_MONITORING_EVENT_INSTRUCTION]; PyObject *offset_obj = PyLong_FromSsize_t(offset); if (offset_obj == NULL) { @@ -1359,7 +1377,8 @@ initialize_tools(PyCodeObject *code) assert(event > 0); } assert(event >= 0); - tools[i] = code->_co_instrumentation.monitoring_data->matrix.tools[event]; + assert(event < PY_MONITORING_INSTRUMENTED_EVENTS); + tools[i] = code->_co_instrumentation.monitoring_data->current_instrumentation.tools[event]; CHECK(tools[i] != 0); } else { @@ -1444,7 +1463,7 @@ initialize_line_tools(PyCodeObject *code, PyInterpreterState *interp) assert(line_tools != NULL); int code_len = (int)Py_SIZE(code); for (int i = 0; i < code_len; i++) { - line_tools[i] = interp->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE]; + line_tools[i] = interp->instrumented_events.tools[PY_MONITORING_EVENT_LINE]; } } @@ -1458,14 +1477,14 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) PyErr_NoMemory(); return -1; } - code->_co_instrumentation.monitoring_data->matrix = (_Py_MonitoringMatrix){ 0 }; + code->_co_instrumentation.monitoring_data->current_instrumentation = (_Py_InstrumentationMatrix){ 0 }; code->_co_instrumentation.monitoring_data->tools = NULL; code->_co_instrumentation.monitoring_data->lines = NULL; code->_co_instrumentation.monitoring_data->line_tools = NULL; code->_co_instrumentation.monitoring_data->per_instruction_opcodes = NULL; code->_co_instrumentation.monitoring_data->per_instruction_tools = NULL; } - bool multitools = multiple_tools(interp->monitoring_matrix); + bool multitools = multiple_tools(&interp->instrumented_events); if (code->_co_instrumentation.monitoring_data->tools == NULL && multitools) { code->_co_instrumentation.monitoring_data->tools = PyMem_Malloc(code_len); if (code->_co_instrumentation.monitoring_data->tools == NULL) { @@ -1474,7 +1493,7 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) } initialize_tools(code); } - if (interp->monitoring_matrix.tools[PY_MONITORING_EVENT_LINE]) { + if (interp->instrumented_events.tools[PY_MONITORING_EVENT_LINE]) { if (code->_co_instrumentation.monitoring_data->lines == NULL) { code->_co_instrumentation.monitoring_data->lines = PyMem_Malloc(code_len * sizeof(_PyCoLineInstrumentationData)); if (code->_co_instrumentation.monitoring_data->lines == NULL) { @@ -1492,7 +1511,7 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) initialize_line_tools(code, interp); } } - if (interp->monitoring_matrix.tools[PY_MONITORING_EVENT_INSTRUCTION]) { + if (interp->instrumented_events.tools[PY_MONITORING_EVENT_INSTRUCTION]) { if (code->_co_instrumentation.monitoring_data->per_instruction_opcodes == NULL) { code->_co_instrumentation.monitoring_data->per_instruction_opcodes = PyMem_Malloc(code_len * sizeof(_PyCoLineInstrumentationData)); if (code->_co_instrumentation.monitoring_data->per_instruction_opcodes == NULL) { @@ -1539,7 +1558,7 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) if (is_instrumentation_up_to_date(code, interp)) { assert(interp->monitoring_version == 0 || - matrix_equals(code->_co_instrumentation.monitoring_data->matrix, interp->monitoring_matrix) + matrix_equals(code->_co_instrumentation.monitoring_data->current_instrumentation, interp->instrumented_events) ); return 0; } @@ -1547,20 +1566,20 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) if (update_instrumentation_data(code, interp)) { return -1; } - _Py_MonitoringMatrix new_events; - _Py_MonitoringMatrix removed_events; + _Py_InstrumentationMatrix new_events; + _Py_InstrumentationMatrix removed_events; bool restarted = interp->last_restart_version > code->_co_instrumentation.monitoring_version; if (restarted) { - removed_events = code->_co_instrumentation.monitoring_data->matrix; - new_events = interp->monitoring_matrix; + removed_events = code->_co_instrumentation.monitoring_data->current_instrumentation; + new_events = interp->instrumented_events; } else { - removed_events = matrix_sub(code->_co_instrumentation.monitoring_data->matrix, interp->monitoring_matrix); - new_events = matrix_sub(interp->monitoring_matrix, code->_co_instrumentation.monitoring_data->matrix); + removed_events = matrix_sub(code->_co_instrumentation.monitoring_data->current_instrumentation, interp->instrumented_events); + new_events = matrix_sub(interp->instrumented_events, code->_co_instrumentation.monitoring_data->current_instrumentation); assert(matrix_empty(matrix_and(new_events, removed_events))); } - code->_co_instrumentation.monitoring_data->matrix = interp->monitoring_matrix; + code->_co_instrumentation.monitoring_data->current_instrumentation = interp->instrumented_events; code->_co_instrumentation.monitoring_version = interp->monitoring_version; if (matrix_empty(new_events) && matrix_empty(removed_events)) { //sanity_check_instrumentation(code); @@ -1681,13 +1700,13 @@ _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) { PyInterpreterState *interp = _PyInterpreterState_Get(); assert((events & C_CALL_EVENTS) == 0 || (events & C_CALL_EVENTS) == C_CALL_EVENTS); - uint32_t existing_events = matrix_get_events(&interp->monitoring_matrix, tool_id); + uint32_t existing_events = interpeter_get_events(interp, tool_id); if (existing_events == events) { return; } for (int e = 0; e < PY_MONITORING_EVENTS; e++) { int val = (events >> e) & 1; - matrix_set_bit(&interp->monitoring_matrix, e, tool_id, val); + interpreter_set_tool_event(interp, e, tool_id, val); } interp->monitoring_version++; @@ -1842,7 +1861,7 @@ monitoring_get_events_impl(PyObject *module, int tool_id) return NULL; } PyInterpreterState *interp = _PyInterpreterState_Get(); - _PyMonitoringEventSet event_set = matrix_get_events(&interp->monitoring_matrix, tool_id); + _PyMonitoringEventSet event_set = interpeter_get_events(interp, tool_id); return PyLong_FromUnsignedLong(event_set); } @@ -1942,7 +1961,13 @@ monitoring__all_events_impl(PyObject *module) return NULL; } for (int e = 0; e < PY_MONITORING_EVENTS; e++) { - uint8_t tools = interp->monitoring_matrix.tools[e]; + uint8_t tools; + if (e < PY_MONITORING_INSTRUMENTED_EVENTS) { + tools = interp->instrumented_events.tools[e]; + } + else { + tools = interp->other_events[e]; + } if (tools == 0) { continue; } diff --git a/Python/pystate.c b/Python/pystate.c index 84ae45874dbb97..aebe4ae9e1e267 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -641,8 +641,13 @@ init_interpreter(PyInterpreterState *interp, _PyGC_InitState(&interp->gc); PyConfig_InitPythonConfig(&interp->config); _PyType_InitCache(interp); + for(int i = 0; i < PY_MONITORING_INSTRUMENTED_EVENTS; i++) { + interp->instrumented_events.tools[i] = 0; + } + for(int i = 0; i < PY_MONITORING_EVENTS-PY_MONITORING_INSTRUMENTED_EVENTS; i++) { + interp->other_events[i] = 0; + } for(int i = 0; i < PY_MONITORING_EVENTS; i++) { - interp->monitoring_matrix.tools[i] = 0; for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { interp->tools[t].instrument_callables[i] = NULL; } @@ -773,8 +778,13 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) Py_CLEAR(interp->audit_hooks); + for(int i = 0; i < PY_MONITORING_INSTRUMENTED_EVENTS; i++) { + interp->instrumented_events.tools[i] = 0; + } + for(int i = 0; i < PY_MONITORING_EVENTS-PY_MONITORING_INSTRUMENTED_EVENTS; i++) { + interp->other_events[i] = 0; + } for(int i = 0; i < PY_MONITORING_EVENTS; i++) { - interp->monitoring_matrix.tools[i] = 0; for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { Py_CLEAR(interp->tools[t].instrument_callables[i]); } From 23b5f5ed14684fb4d8315a5817dfd42d4fe582a4 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 21 Feb 2023 14:48:18 +0000 Subject: [PATCH 047/116] Add support for sys.monitoring.MISSING --- Include/internal/pycore_instruments.h | 6 ++---- Lib/test/test_monitoring.py | 3 +-- Python/bytecodes.c | 6 ++++-- Python/instrumentation.c | 12 +++++++++++- Python/legacy_tracing.c | 2 +- 5 files changed, 19 insertions(+), 10 deletions(-) diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index c7c438ff05ca71..91fba602827b03 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -43,10 +43,6 @@ extern "C" { /* #define INSTRUMENT_EVENT_BRANCH_NOT_TAKEN xxx -- If we can afford this */ -/* Temporary and internal events */ - -// #define PY_INSTRUMENT_PEP_523 50 -/* #define PY_INSTRUMENT_JIT_API 17 -- Reserved */ typedef uint32_t _PyMonitoringEventSet; @@ -99,6 +95,8 @@ _Py_call_instrumentation_exc2(PyThreadState *tstate, int event, extern int _Py_Instrumentation_GetLine(PyCodeObject *code, int index); +extern PyObject _PyInstrumentation_MISSING; + #ifdef __cplusplus } #endif diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index faa6f6b9073c81..3e9a27d2984157 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -46,10 +46,9 @@ def test_has_objects(self): # m.get_local_events # m.set_local_events m.register_callback - # m.insert_marker - # m.remove_marker m.restart_events m.DISABLE + m.MISSING def test_tool(self): sys.monitoring.use_tool(TEST_TOOL, "MonitoringTest.Tool") diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 5449784fc5d498..4f8761dff8e035 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2507,7 +2507,8 @@ dummy_func( int is_meth = PEEK(oparg+2) != NULL; int total_args = oparg + is_meth; PyObject *function = PEEK(total_args + 1); - PyObject *arg = total_args == 0 ? Py_None : PEEK(total_args); + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PEEK(total_args); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, next_instr-1, function, arg); @@ -2605,7 +2606,8 @@ dummy_func( positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames); if (opcode == INSTRUMENTED_CALL) { - PyObject *arg = total_args == 0 ? Py_None : PEEK(total_args); + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PEEK(total_args); if (res == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, diff --git a/Python/instrumentation.c b/Python/instrumentation.c index e6886c30a8b373..ce3d89ee2901bc 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -4,6 +4,7 @@ #include "pycore_frame.h" #include "pycore_interp.h" #include "pycore_namespace.h" +#include "pycore_object.h" #include "pycore_opcode.h" #include "pycore_pyerrors.h" #include "pycore_pystate.h" @@ -13,7 +14,13 @@ static PyObject DISABLE = { - 1 << 30, + _PyObject_IMMORTAL_REFCNT, + &PyBaseObject_Type +}; + +PyObject _PyInstrumentation_MISSING = +{ + _PyObject_IMMORTAL_REFCNT, &PyBaseObject_Type }; @@ -2013,6 +2020,9 @@ PyObject *_Py_CreateMonitoringObject(void) if (PyObject_SetAttrString(mod, "DISABLE", &DISABLE)) { goto error; } + if (PyObject_SetAttrString(mod, "MISSING", &_PyInstrumentation_MISSING)) { + goto error; + } PyObject *events = _PyNamespace_New(NULL); if (events == NULL) { goto error; diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index 5d9fc6917e5a80..348138e3649d19 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -91,7 +91,7 @@ sys_profile_call_or_return( * convert to builtin method */ /* If no arg, skip */ - if (self_arg == Py_None) { + if (self_arg == &_PyInstrumentation_MISSING) { Py_RETURN_NONE; } PyObject *meth = Py_TYPE(callable)->tp_descr_get( From dfc18c53610af938c797be7adef18a6df4ba11b4 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 23 Feb 2023 10:23:22 +0000 Subject: [PATCH 048/116] Add support for local (per-code-object) event monitoring. --- Include/cpython/code.h | 12 +- Include/internal/pycore_instruments.h | 18 +- Include/internal/pycore_interp.h | 10 +- Lib/test/test_monitoring.py | 78 +++++- Python/bytecodes.c | 14 +- Python/ceval.c | 73 ++++-- Python/clinic/instrumentation.c.h | 73 +++++- Python/generated_cases.c.h | 20 +- Python/instrumentation.c | 362 ++++++++++++++++++-------- Python/pystate.c | 10 +- 10 files changed, 485 insertions(+), 185 deletions(-) diff --git a/Include/cpython/code.h b/Include/cpython/code.h index b6d737f7c3fbcb..f2e31087e58d66 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -9,11 +9,11 @@ extern "C" { #endif #define PY_MONITORING_EVENTS 16 -#define PY_MONITORING_INSTRUMENTED_EVENTS 10 +#define PY_MONITORING_UNGROUPED_EVENTS 14 -typedef struct _Py_InstrumentationMatrix { - uint8_t tools[PY_MONITORING_INSTRUMENTED_EVENTS]; -} _Py_InstrumentationMatrix; +typedef struct _Py_Monitors { + uint8_t tools[PY_MONITORING_UNGROUPED_EVENTS]; +} _Py_Monitors; /* Each instruction in a code object is a fixed-width value, * currently 2 bytes: 1-byte opcode + 1-byte oparg. The EXTENDED_ARG @@ -56,8 +56,8 @@ typedef struct { } _PyCoLineInstrumentationData; typedef struct { - _Py_InstrumentationMatrix local_instrumentation; - _Py_InstrumentationMatrix current_instrumentation; + _Py_Monitors local_instrumentation; + _Py_Monitors current_instrumentation; uint8_t *tools; _PyCoLineInstrumentationData *lines; uint8_t *line_tools; diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index 91fba602827b03..f34bd074e6a92f 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -27,18 +27,20 @@ extern "C" { #define PY_MONITORING_EVENT_BRANCH 8 #define PY_MONITORING_EVENT_STOP_ITERATION 9 +#define PY_MONITORING_INSTRUMENTED_EVENTS 10 -/* Grouped events */ +/* Exceptional events */ -#define PY_MONITORING_EVENT_C_RETURN 10 -#define PY_MONITORING_EVENT_C_RAISE 11 +#define PY_MONITORING_EVENT_RAISE 10 +#define PY_MONITORING_EVENT_EXCEPTION_HANDLED 11 +#define PY_MONITORING_EVENT_PY_UNWIND 12 +#define PY_MONITORING_EVENT_PY_THROW 13 -/* Exceptional events */ -#define PY_MONITORING_EVENT_RAISE 12 -#define PY_MONITORING_EVENT_EXCEPTION_HANDLED 13 -#define PY_MONITORING_EVENT_PY_UNWIND 14 -#define PY_MONITORING_EVENT_PY_THROW 15 +/* Grouped events */ + +#define PY_MONITORING_EVENT_C_RETURN 14 +#define PY_MONITORING_EVENT_C_RAISE 15 /* #define INSTRUMENT_EVENT_BRANCH_NOT_TAKEN xxx -- If we can afford this */ diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 1cf517864a8abc..964e4f5e908ff9 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -190,8 +190,7 @@ struct _is { struct callable_cache callable_cache; PyCodeObject *interpreter_trampoline; - _Py_InstrumentationMatrix instrumented_events; - uint8_t other_events[PY_MONITORING_EVENTS - PY_MONITORING_INSTRUMENTED_EVENTS]; + _Py_Monitors instrumented_events; bool f_opcode_trace_set; bool sys_profile_initialized; bool sys_trace_initialized; @@ -248,13 +247,6 @@ PyAPI_FUNC(int) _PyInterpreterState_IDInitref(PyInterpreterState *); PyAPI_FUNC(int) _PyInterpreterState_IDIncref(PyInterpreterState *); PyAPI_FUNC(void) _PyInterpreterState_IDDecref(PyInterpreterState *); -static inline uint8_t -_PyInterpreterState_GetTools(PyInterpreterState * is, int event) -{ - assert(event >= PY_MONITORING_INSTRUMENTED_EVENTS && event < PY_MONITORING_EVENTS); - return is->other_events[event-PY_MONITORING_INSTRUMENTED_EVENTS]; -} - #ifdef __cplusplus } diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 3e9a27d2984157..9278a474ed66fe 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -43,8 +43,8 @@ def test_has_objects(self): m.get_tool m.get_events m.set_events - # m.get_local_events - # m.set_local_events + m.get_local_events + m.set_local_events m.register_callback m.restart_events m.DISABLE @@ -719,7 +719,7 @@ def func1(): self.check_events(func1, recorders = MANY_RECORDERS, expected = [ ('line', 'check_events', 10), - ('call', 'func1', None), + ('call', 'func1', sys.monitoring.MISSING), ('line', 'func1', 1), ('line', 'func1', 2), ('line', 'func1', 3), @@ -735,7 +735,7 @@ def func2(): self.check_events(func2, recorders = MANY_RECORDERS, expected = [ ('line', 'check_events', 10), - ('call', 'func2', None), + ('call', 'func2', sys.monitoring.MISSING), ('line', 'func2', 1), ('line', 'func2', 2), ('call', 'append', [2]), @@ -756,7 +756,7 @@ def func3(): self.check_events(func3, recorders = MANY_RECORDERS, expected = [ ('line', 'check_events', 10), - ('call', 'func3', None), + ('call', 'func3', sys.monitoring.MISSING), ('line', 'func3', 1), ('line', 'func3', 2), ('line', 'func3', 3), @@ -766,3 +766,71 @@ def func3(): ('line', 'func3', 6), ('line', 'check_events', 11), ('call', 'set_events', 2)]) + +class TestLocalEvents(unittest.TestCase): + + def check_events(self, func, expected, tool=TEST_TOOL, recorders=(ExceptionRecorder,)): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + event_list = [] + all_events = 0 + for recorder in recorders: + ev = recorder.event_type + sys.monitoring.register_callback(tool, ev, recorder(event_list)) + all_events |= ev + sys.monitoring.set_local_events(func.__code__, tool, all_events) + func() + sys.monitoring.set_local_events(func.__code__, tool, 0) + for recorder in recorders: + sys.monitoring.register_callback(tool, recorder.event_type, None) + self.assertEqual(event_list, expected) + finally: + sys.monitoring.set_local_events(func.__code__, tool, 0) + for recorder in recorders: + sys.monitoring.register_callback(tool, recorder.event_type, None) + + + def test_simple(self): + + def func1(): + line1 = 1 + line2 = 2 + line3 = 3 + + self.check_events(func1, recorders = MANY_RECORDERS, expected = [ + ('line', 'func1', 1), + ('line', 'func1', 2), + ('line', 'func1', 3)]) + + def test_c_call(self): + + def func2(): + line1 = 1 + [].append(2) + line3 = 3 + + self.check_events(func2, recorders = MANY_RECORDERS, expected = [ + ('line', 'func2', 1), + ('line', 'func2', 2), + ('call', 'append', [2]), + ('C return', 'append', [2]), + ('line', 'func2', 3)]) + + def test_try_except(self): + + def func3(): + try: + line = 2 + raise KeyError + except: + line = 5 + line = 6 + + self.check_events(func3, recorders = MANY_RECORDERS, expected = [ + ('line', 'func3', 1), + ('line', 'func3', 2), + ('line', 'func3', 3), + ('raise', KeyError), + ('line', 'func3', 4), + ('line', 'func3', 5), + ('line', 'func3', 6)]) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 4f8761dff8e035..6d1d8cd7f4495b 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -177,7 +177,9 @@ dummy_func( * to conform to PEP 380 */ if (PyGen_Check(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); - monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_STOP_ITERATION); + if (monitor_stop_iteration(tstate, frame, next_instr-1)) { + goto error; + } PyErr_SetRaisedException(NULL); } DECREF_INPUTS(); @@ -190,7 +192,9 @@ dummy_func( inst(INSTRUMENTED_END_SEND, (receiver, value -- value)) { if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); - monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_STOP_ITERATION); + if (monitor_stop_iteration(tstate, frame, next_instr-1)) { + goto error; + } PyErr_SetRaisedException(NULL); } Py_DECREF(receiver); @@ -794,7 +798,7 @@ dummy_func( if (retval == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) ) { - monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_RAISE); + monitor_raise(tstate, frame, next_instr-1); } if (_PyGen_FetchStopIterationValue(&retval) == 0) { assert(retval != NULL); @@ -2215,7 +2219,7 @@ dummy_func( if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { goto error; } - monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_RAISE); + monitor_raise(tstate, frame, next_instr-1); _PyErr_Clear(tstate); } /* iterator ended normally */ @@ -2244,7 +2248,7 @@ dummy_func( if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { goto error; } - monitor_raise(tstate, frame, here, PY_MONITORING_EVENT_RAISE); + monitor_raise(tstate, frame, here); _PyErr_Clear(tstate); } /* iterator ended normally */ diff --git a/Python/ceval.c b/Python/ceval.c index 3f329d5df9ca44..0c1276e13c21ef 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -178,7 +178,10 @@ lltrace_resume_frame(_PyInterpreterFrame *frame) static void monitor_raise(PyThreadState *tstate, _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr, int event); + _Py_CODEUNIT *instr); +static int monitor_stop_iteration(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr); static void monitor_unwind(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr); @@ -899,7 +902,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int PyTraceBack_Here(f); } } - monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_RAISE); + monitor_raise(tstate, frame, next_instr-1); exception_unwind: { @@ -1935,22 +1938,12 @@ unpack_iterable(PyThreadState *tstate, PyObject *v, return 0; } - -static void -monitor_raise(PyThreadState *tstate, - _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr, int event) -{ - if (event < PY_MONITORING_INSTRUMENTED_EVENTS) { - if (frame->f_code->_co_instrumentation.monitoring_data->current_instrumentation.tools[event] == 0) { - return; - } - } - else { - if (_PyInterpreterState_GetTools(tstate->interp, event) == 0) { - return; - } - } +static int +do_monitor_exc( + PyThreadState *tstate, _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr, int event +) { + assert(event < PY_MONITORING_UNGROUPED_EVENTS); PyObject *exc = PyErr_GetRaisedException(); assert(exc != NULL); int err; @@ -1961,15 +1954,53 @@ monitor_raise(PyThreadState *tstate, else { Py_DECREF(exc); } + return err; +} + +static inline int +no_tools_for_event(PyThreadState *tstate, _PyInterpreterFrame *frame, int event) +{ + _PyCoInstrumentationData *data = frame->f_code->_co_instrumentation.monitoring_data; + if (data) { + if (data->current_instrumentation.tools[event] == 0) { + return 1; + } + } + else { + if (tstate->interp->instrumented_events.tools[event] == 0) { + return 1; + } + } + return 0; } +static void +monitor_raise( + PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) +{ + if (no_tools_for_event(tstate, frame, PY_MONITORING_EVENT_RAISE)) { + return; + } + do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_RAISE); +} + +static int +monitor_stop_iteration( + PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) +{ + if (no_tools_for_event(tstate, frame, PY_MONITORING_EVENT_STOP_ITERATION)) { + return 0; + } + return do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_STOP_ITERATION); +} static void monitor_unwind(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) { - if (_PyInterpreterState_GetTools(tstate->interp, PY_MONITORING_EVENT_PY_UNWIND) == 0) { + + if (no_tools_for_event(tstate, frame, PY_MONITORING_EVENT_PY_UNWIND)) { return; } _Py_call_instrumentation_exc0(tstate, PY_MONITORING_EVENT_PY_UNWIND, frame, instr); @@ -1981,7 +2012,7 @@ monitor_handled(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *exc) { - if (_PyInterpreterState_GetTools(tstate->interp, PY_MONITORING_EVENT_EXCEPTION_HANDLED) == 0) { + if (no_tools_for_event(tstate, frame, PY_MONITORING_EVENT_EXCEPTION_HANDLED)) { return; } _Py_call_instrumentation_arg(tstate, PY_MONITORING_EVENT_EXCEPTION_HANDLED, frame, instr, exc); @@ -1992,7 +2023,7 @@ monitor_throw(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) { - if (_PyInterpreterState_GetTools(tstate->interp, PY_MONITORING_EVENT_PY_THROW) == 0) { + if (no_tools_for_event(tstate, frame, PY_MONITORING_EVENT_PY_THROW)) { return; } _Py_call_instrumentation_exc0(tstate, PY_MONITORING_EVENT_PY_THROW, frame, instr); diff --git a/Python/clinic/instrumentation.c.h b/Python/clinic/instrumentation.c.h index feecf7695a2a2a..57021191e29df6 100644 --- a/Python/clinic/instrumentation.c.h +++ b/Python/clinic/instrumentation.c.h @@ -194,6 +194,77 @@ monitoring_set_events(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +PyDoc_STRVAR(monitoring_get_local_events__doc__, +"get_local_events($module, code, tool_id, /)\n" +"--\n" +"\n"); + +#define MONITORING_GET_LOCAL_EVENTS_METHODDEF \ + {"get_local_events", _PyCFunction_CAST(monitoring_get_local_events), METH_FASTCALL, monitoring_get_local_events__doc__}, + +static PyObject * +monitoring_get_local_events_impl(PyObject *module, PyObject *code, + int tool_id); + +static PyObject * +monitoring_get_local_events(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *code; + int tool_id; + + if (!_PyArg_CheckPositional("get_local_events", nargs, 2, 2)) { + goto exit; + } + code = args[0]; + tool_id = _PyLong_AsInt(args[1]); + if (tool_id == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = monitoring_get_local_events_impl(module, code, tool_id); + +exit: + return return_value; +} + +PyDoc_STRVAR(monitoring_set_local_events__doc__, +"set_local_events($module, code, tool_id, event_set, /)\n" +"--\n" +"\n"); + +#define MONITORING_SET_LOCAL_EVENTS_METHODDEF \ + {"set_local_events", _PyCFunction_CAST(monitoring_set_local_events), METH_FASTCALL, monitoring_set_local_events__doc__}, + +static PyObject * +monitoring_set_local_events_impl(PyObject *module, PyObject *code, + int tool_id, int event_set); + +static PyObject * +monitoring_set_local_events(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *code; + int tool_id; + int event_set; + + if (!_PyArg_CheckPositional("set_local_events", nargs, 3, 3)) { + goto exit; + } + code = args[0]; + tool_id = _PyLong_AsInt(args[1]); + if (tool_id == -1 && PyErr_Occurred()) { + goto exit; + } + event_set = _PyLong_AsInt(args[2]); + if (event_set == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = monitoring_set_local_events_impl(module, code, tool_id, event_set); + +exit: + return return_value; +} + PyDoc_STRVAR(monitoring_restart_events__doc__, "restart_events($module, /)\n" "--\n" @@ -227,4 +298,4 @@ monitoring__all_events(PyObject *module, PyObject *Py_UNUSED(ignored)) { return monitoring__all_events_impl(module); } -/*[clinic end generated code: output=d4c412a002392e2b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9e5f270f3ce1e945 input=a9049054013a1b77]*/ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 2c932829014f31..74b69f697ccc00 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -242,7 +242,9 @@ * to conform to PEP 380 */ if (PyGen_Check(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); - monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_STOP_ITERATION); + if (monitor_stop_iteration(tstate, frame, next_instr-1)) { + goto error; + } PyErr_SetRaisedException(NULL); } Py_DECREF(receiver); @@ -265,7 +267,9 @@ PyObject *receiver = PEEK(2); if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); - monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_STOP_ITERATION); + if (monitor_stop_iteration(tstate, frame, next_instr-1)) { + goto error; + } PyErr_SetRaisedException(NULL); } Py_DECREF(receiver); @@ -1010,7 +1014,7 @@ if (retval == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) ) { - monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_RAISE); + monitor_raise(tstate, frame, next_instr-1); } if (_PyGen_FetchStopIterationValue(&retval) == 0) { assert(retval != NULL); @@ -2793,7 +2797,7 @@ if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { goto error; } - monitor_raise(tstate, frame, next_instr-1, PY_MONITORING_EVENT_RAISE); + monitor_raise(tstate, frame, next_instr-1); _PyErr_Clear(tstate); } /* iterator ended normally */ @@ -2826,7 +2830,7 @@ if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { goto error; } - monitor_raise(tstate, frame, here, PY_MONITORING_EVENT_RAISE); + monitor_raise(tstate, frame, here); _PyErr_Clear(tstate); } /* iterator ended normally */ @@ -3160,7 +3164,8 @@ int is_meth = PEEK(oparg+2) != NULL; int total_args = oparg + is_meth; PyObject *function = PEEK(total_args + 1); - PyObject *arg = total_args == 0 ? Py_None : PEEK(total_args); + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PEEK(total_args); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, next_instr-1, function, arg); @@ -3234,7 +3239,8 @@ positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames); if (opcode == INSTRUMENTED_CALL) { - PyObject *arg = total_args == 0 ? Py_None : PEEK(total_args); + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PEEK(total_args); if (res == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, diff --git a/Python/instrumentation.c b/Python/instrumentation.c index ce3d89ee2901bc..2619716fef95f8 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -168,9 +168,9 @@ is_instrumented(int opcode) { } static inline bool -matrix_equals(_Py_InstrumentationMatrix a, _Py_InstrumentationMatrix b) +matrix_equals(_Py_Monitors a, _Py_Monitors b) { - for (int i = 0; i < PY_MONITORING_INSTRUMENTED_EVENTS; i++) { + for (int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { if (a.tools[i] != b.tools[i]) { return false; } @@ -178,30 +178,40 @@ matrix_equals(_Py_InstrumentationMatrix a, _Py_InstrumentationMatrix b) return true; } -static inline _Py_InstrumentationMatrix -matrix_sub(_Py_InstrumentationMatrix a, _Py_InstrumentationMatrix b) +static inline _Py_Monitors +matrix_sub(_Py_Monitors a, _Py_Monitors b) { - _Py_InstrumentationMatrix res; - for (int i = 0; i < PY_MONITORING_INSTRUMENTED_EVENTS; i++) { + _Py_Monitors res; + for (int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { res.tools[i] = a.tools[i] & ~b.tools[i]; } return res; } -static inline _Py_InstrumentationMatrix -matrix_and(_Py_InstrumentationMatrix a, _Py_InstrumentationMatrix b) +static inline _Py_Monitors +matrix_and(_Py_Monitors a, _Py_Monitors b) { - _Py_InstrumentationMatrix res; - for (int i = 0; i < PY_MONITORING_INSTRUMENTED_EVENTS; i++) { + _Py_Monitors res; + for (int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { res.tools[i] = a.tools[i] & b.tools[i]; } return res; } +static inline _Py_Monitors +matrix_or(_Py_Monitors a, _Py_Monitors b) +{ + _Py_Monitors res; + for (int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { + res.tools[i] = a.tools[i] | b.tools[i]; + } + return res; +} + static inline bool -matrix_empty(_Py_InstrumentationMatrix m) +matrix_empty(_Py_Monitors m) { - for (int i = 0; i < PY_MONITORING_INSTRUMENTED_EVENTS; i++) { + for (int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { if (m.tools[i]) { return false; } @@ -210,9 +220,9 @@ matrix_empty(_Py_InstrumentationMatrix m) } static inline int -multiple_tools(_Py_InstrumentationMatrix *m) +multiple_tools(_Py_Monitors *m) { - for (int i = 0; i < PY_MONITORING_INSTRUMENTED_EVENTS; i++) { + for (int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { if (_Py_popcount32(m->tools[i]) > 1) { return true; } @@ -220,34 +230,12 @@ multiple_tools(_Py_InstrumentationMatrix *m) return false; } -static inline void -interpreter_set_tool_event(PyInterpreterState *interp, int event, int tool, int val) -{ - assert(0 <= event && event < PY_MONITORING_EVENTS); - assert(0 <= tool && tool < PY_MONITORING_TOOL_IDS); - assert(val == 0 || val == 1); - uint8_t *tools; - if (event < PY_MONITORING_INSTRUMENTED_EVENTS) { - tools = &interp->instrumented_events.tools[event]; - } - else { - tools = &interp->other_events[event-PY_MONITORING_INSTRUMENTED_EVENTS]; - } - *tools &= ~(1 << tool); - *tools |= (val << tool); -} - static inline _PyMonitoringEventSet -interpeter_get_events(PyInterpreterState *interp, int tool_id) +get_events(_Py_Monitors *m, int tool_id) { _PyMonitoringEventSet result = 0; - for (int e = 0; e < PY_MONITORING_INSTRUMENTED_EVENTS; e++) { - if ((interp->instrumented_events.tools[e] >> tool_id) & 1) { - result |= (1 << e); - } - } - for (int e = PY_MONITORING_INSTRUMENTED_EVENTS; e < PY_MONITORING_EVENTS; e++) { - if ((interp->other_events[e] >> tool_id) & 1) { + for (int e = 0; e < PY_MONITORING_UNGROUPED_EVENTS; e++) { + if ((m->tools[e] >> tool_id) & 1) { result |= (1 << e); } } @@ -853,7 +841,7 @@ remove_tools(PyCodeObject * code, int offset, int event, int tools) _PyCoInstrumentationData *monitoring = code->_co_instrumentation.monitoring_data; if (monitoring && monitoring->tools) { monitoring->tools[offset] &= ~tools; - if (monitoring->tools[offset] == 0 && event < PY_MONITORING_INSTRUMENTED_EVENTS) { + if (monitoring->tools[offset] == 0) { de_instrument(code, offset, event); } } @@ -861,12 +849,21 @@ remove_tools(PyCodeObject * code, int offset, int event, int tools) /* Single tool */ uint8_t single_tool = code->_co_instrumentation.monitoring_data->current_instrumentation.tools[event]; assert(_Py_popcount32(single_tool) <= 1); - if (((single_tool & tools) == single_tool) && event < PY_MONITORING_INSTRUMENTED_EVENTS) { + if (((single_tool & tools) == single_tool)) { de_instrument(code, offset, event); } } } +#ifndef NDEBUG +bool tools_is_subset_for_event(PyCodeObject * code, int event, int tools) +{ + int global_tools = PyInterpreterState_Get()->instrumented_events.tools[event]; + int local_tools = code->_co_instrumentation.monitoring_data->local_instrumentation.tools[event]; + return tools == ((global_tools | local_tools) & tools); +} +#endif + static void remove_line_tools(PyCodeObject * code, int offset, int tools) { @@ -889,7 +886,6 @@ remove_line_tools(PyCodeObject * code, int offset, int tools) } } - static void add_tools(PyCodeObject * code, int offset, int event, int tools) { @@ -904,18 +900,17 @@ add_tools(PyCodeObject * code, int offset, int event, int tools) } else { /* Single tool */ - assert(PyInterpreterState_Get()->instrumented_events.tools[event] == tools); + assert(_Py_popcount32(tools) == 1); + assert(tools_is_subset_for_event(code, event, tools)); assert(_Py_popcount32(tools) == 1); } - if (event < PY_MONITORING_INSTRUMENTED_EVENTS) { - instrument(code, offset); - } + instrument(code, offset); } static void add_line_tools(PyCodeObject * code, int offset, int tools) { - assert((PyInterpreterState_Get()->instrumented_events.tools[PY_MONITORING_EVENT_LINE] & tools) == tools); + assert(tools_is_subset_for_event(code, PY_MONITORING_EVENT_LINE, tools)); assert(code->_co_instrumentation.monitoring_data); if (code->_co_instrumentation.monitoring_data->line_tools ) { @@ -932,7 +927,7 @@ add_line_tools(PyCodeObject * code, int offset, int tools) static void add_per_instruction_tools(PyCodeObject * code, int offset, int tools) { - assert((PyInterpreterState_Get()->instrumented_events.tools[PY_MONITORING_EVENT_INSTRUCTION] & tools) == tools); + assert(tools_is_subset_for_event(code, PY_MONITORING_EVENT_INSTRUCTION, tools)); assert(code->_co_instrumentation.monitoring_data); if (code->_co_instrumentation.monitoring_data->per_instruction_tools ) { @@ -1013,24 +1008,44 @@ static inline int most_significant_bit(uint8_t bits) { } } +static bool +is_version_up_to_date(PyCodeObject *code, PyInterpreterState *interp) +{ + return interp->monitoring_version == code->_co_instrumentation.monitoring_version; +} + +#ifndef NDEBUG +static bool +instrumentation_cross_checks(PyInterpreterState *interp, PyCodeObject *code) +{ + _Py_Monitors expected = matrix_or( + interp->instrumented_events, + code->_co_instrumentation.monitoring_data->local_instrumentation); + return matrix_equals(code->_co_instrumentation.monitoring_data->current_instrumentation, + expected); +} +#endif + static inline uint8_t get_tools_for_instruction(PyCodeObject * code, int i, int event) { uint8_t tools; assert(event != PY_MONITORING_EVENT_LINE); assert(event != PY_MONITORING_EVENT_INSTRUCTION); + assert(instrumentation_cross_checks(PyThreadState_GET()->interp, code)); _PyCoInstrumentationData *monitoring = code->_co_instrumentation.monitoring_data; - assert(monitoring != NULL); - if (event >= PY_MONITORING_INSTRUMENTED_EVENTS) { - return _PyInterpreterState_GetTools(PyInterpreterState_Get(), event); - } - else if (monitoring->tools) { + if (monitoring->tools) { tools = monitoring->tools[i]; } else { + if (event >= PY_MONITORING_UNGROUPED_EVENTS) { + assert(event == PY_MONITORING_EVENT_C_RAISE || + event == PY_MONITORING_EVENT_C_RETURN); + event = PY_MONITORING_EVENT_CALL; + } tools = code->_co_instrumentation.monitoring_data->current_instrumentation.tools[event]; } - CHECK((tools & PyInterpreterState_Get()->instrumented_events.tools[event]) == tools); + CHECK(tools_is_subset_for_event(code, event, tools)); CHECK((tools & code->_co_instrumentation.monitoring_data->current_instrumentation.tools[event]) == tools); return tools; } @@ -1065,20 +1080,14 @@ call_instrument(PyThreadState *tstate, PyCodeObject *code, int event, return 0; } -static bool -is_instrumentation_up_to_date(PyCodeObject *code, PyInterpreterState *interp) -{ - return interp->monitoring_version == code->_co_instrumentation.monitoring_version; -} - int _Py_call_instrumentation( PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) { PyCodeObject *code = frame->f_code; - assert(is_instrumentation_up_to_date(code, tstate->interp)); - assert(matrix_equals(code->_co_instrumentation.monitoring_data->current_instrumentation, tstate->interp->instrumented_events)); + assert(is_version_up_to_date(code, tstate->interp)); + assert(instrumentation_cross_checks(tstate->interp, code)); int offset = instr - _PyCode_CODE(code); PyObject *offset_obj = PyLong_FromSsize_t(offset); if (offset_obj == NULL) { @@ -1097,8 +1106,8 @@ _Py_call_instrumentation_arg( _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg) { PyCodeObject *code = frame->f_code; - assert(is_instrumentation_up_to_date(code, tstate->interp)); - assert(matrix_equals(code->_co_instrumentation.monitoring_data->current_instrumentation, tstate->interp->instrumented_events)); + assert(is_version_up_to_date(code, tstate->interp)); + assert(instrumentation_cross_checks(tstate->interp, code)); int offset = instr - _PyCode_CODE(code); PyObject *offset_obj = PyLong_FromSsize_t(offset); if (offset_obj == NULL) { @@ -1117,8 +1126,8 @@ _Py_call_instrumentation_2args( _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg0, PyObject *arg1) { PyCodeObject *code = frame->f_code; - assert(is_instrumentation_up_to_date(code, tstate->interp)); - assert(matrix_equals(code->_co_instrumentation.monitoring_data->current_instrumentation, tstate->interp->instrumented_events)); + assert(is_version_up_to_date(code, tstate->interp)); + assert(instrumentation_cross_checks(tstate->interp, code)); int offset = instr - _PyCode_CODE(code); PyObject *offset_obj = PyLong_FromSsize_t(offset); if (offset_obj == NULL) { @@ -1179,8 +1188,7 @@ _Py_call_instrumentation_exc_vector( args[1] = (PyObject *)code; assert(code->_co_instrumentation.monitoring_version == tstate->interp->monitoring_version); int offset = instr - _PyCode_CODE(code); - assert(event >= PY_MONITORING_INSTRUMENTED_EVENTS); - uint8_t tools = tstate->interp->instrumented_events.tools[event]; + uint8_t tools = code->_co_instrumentation.monitoring_data->current_instrumentation.tools[event]; PyObject *offset_obj = PyLong_FromSsize_t(offset); int err; if (offset_obj == NULL) { @@ -1245,8 +1253,8 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, { frame->prev_instr = instr; PyCodeObject *code = frame->f_code; - assert(is_instrumentation_up_to_date(code, tstate->interp)); - assert(matrix_equals(code->_co_instrumentation.monitoring_data->current_instrumentation, tstate->interp->instrumented_events)); + assert(is_version_up_to_date(code, tstate->interp)); + assert(instrumentation_cross_checks(tstate->interp, code)); int i = instr - _PyCode_CODE(code); _PyCoInstrumentation *instrumentation = &code->_co_instrumentation; _PyCoLineInstrumentationData *line_data = &instrumentation->monitoring_data->lines[i]; @@ -1259,7 +1267,9 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, int line = compute_line(code, i, line_delta); uint8_t tools = code->_co_instrumentation.monitoring_data->line_tools != NULL ? code->_co_instrumentation.monitoring_data->line_tools[i] : - interp->instrumented_events.tools[PY_MONITORING_EVENT_LINE]; + (interp->instrumented_events.tools[PY_MONITORING_EVENT_LINE] | + code->_co_instrumentation.monitoring_data->local_instrumentation.tools[PY_MONITORING_EVENT_LINE] + ); PyObject *line_obj = PyLong_FromSsize_t(line); if (line_obj == NULL) { return -1; @@ -1298,8 +1308,8 @@ int _Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr) { PyCodeObject *code = frame->f_code; - assert(is_instrumentation_up_to_date(code, tstate->interp)); - assert(matrix_equals(code->_co_instrumentation.monitoring_data->current_instrumentation, tstate->interp->instrumented_events)); + assert(is_version_up_to_date(code, tstate->interp)); + assert(instrumentation_cross_checks(tstate->interp, code)); int offset = instr - _PyCode_CODE(code); _PyCoInstrumentationData *instrumentation_data = code->_co_instrumentation.monitoring_data; assert(instrumentation_data->per_instruction_opcodes); @@ -1310,8 +1320,9 @@ _Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame* PyInterpreterState *interp = tstate->interp; uint8_t tools = instrumentation_data->per_instruction_tools != NULL ? instrumentation_data->per_instruction_tools[offset] : - interp->instrumented_events.tools[PY_MONITORING_EVENT_INSTRUCTION]; - + (interp->instrumented_events.tools[PY_MONITORING_EVENT_INSTRUCTION] | + code->_co_instrumentation.monitoring_data->local_instrumentation.tools[PY_MONITORING_EVENT_INSTRUCTION] + ); PyObject *offset_obj = PyLong_FromSsize_t(offset); if (offset_obj == NULL) { return -1; @@ -1464,34 +1475,48 @@ initialize_lines(PyCodeObject *code) } static void -initialize_line_tools(PyCodeObject *code, PyInterpreterState *interp) +initialize_line_tools(PyCodeObject *code, _Py_Monitors *all_events) { uint8_t *line_tools = code->_co_instrumentation.monitoring_data->line_tools; assert(line_tools != NULL); int code_len = (int)Py_SIZE(code); for (int i = 0; i < code_len; i++) { - line_tools[i] = interp->instrumented_events.tools[PY_MONITORING_EVENT_LINE]; + line_tools[i] = all_events->tools[PY_MONITORING_EVENT_LINE]; } } -int -update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) +static +int allocate_instrumentation_data(PyCodeObject *code) { - int code_len = (int)Py_SIZE(code); + if (code->_co_instrumentation.monitoring_data == NULL) { code->_co_instrumentation.monitoring_data = PyMem_Malloc(sizeof(_PyCoInstrumentationData)); if (code->_co_instrumentation.monitoring_data == NULL) { PyErr_NoMemory(); return -1; } - code->_co_instrumentation.monitoring_data->current_instrumentation = (_Py_InstrumentationMatrix){ 0 }; + code->_co_instrumentation.monitoring_data->local_instrumentation = (_Py_Monitors){ 0 }; + code->_co_instrumentation.monitoring_data->current_instrumentation = (_Py_Monitors){ 0 }; code->_co_instrumentation.monitoring_data->tools = NULL; code->_co_instrumentation.monitoring_data->lines = NULL; code->_co_instrumentation.monitoring_data->line_tools = NULL; code->_co_instrumentation.monitoring_data->per_instruction_opcodes = NULL; code->_co_instrumentation.monitoring_data->per_instruction_tools = NULL; } - bool multitools = multiple_tools(&interp->instrumented_events); + return 0; +} + +static int +update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) +{ + int code_len = (int)Py_SIZE(code); + if (allocate_instrumentation_data(code)) { + return -1; + } + _Py_Monitors all_events = matrix_or( + interp->instrumented_events, + code->_co_instrumentation.monitoring_data->local_instrumentation); + bool multitools = multiple_tools(&all_events); if (code->_co_instrumentation.monitoring_data->tools == NULL && multitools) { code->_co_instrumentation.monitoring_data->tools = PyMem_Malloc(code_len); if (code->_co_instrumentation.monitoring_data->tools == NULL) { @@ -1500,7 +1525,7 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) } initialize_tools(code); } - if (interp->instrumented_events.tools[PY_MONITORING_EVENT_LINE]) { + if (all_events.tools[PY_MONITORING_EVENT_LINE]) { if (code->_co_instrumentation.monitoring_data->lines == NULL) { code->_co_instrumentation.monitoring_data->lines = PyMem_Malloc(code_len * sizeof(_PyCoLineInstrumentationData)); if (code->_co_instrumentation.monitoring_data->lines == NULL) { @@ -1515,10 +1540,10 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) PyErr_NoMemory(); return -1; } - initialize_line_tools(code, interp); + initialize_line_tools(code, &all_events); } } - if (interp->instrumented_events.tools[PY_MONITORING_EVENT_INSTRUCTION]) { + if (all_events.tools[PY_MONITORING_EVENT_INSTRUCTION]) { if (code->_co_instrumentation.monitoring_data->per_instruction_opcodes == NULL) { code->_co_instrumentation.monitoring_data->per_instruction_opcodes = PyMem_Malloc(code_len * sizeof(_PyCoLineInstrumentationData)); if (code->_co_instrumentation.monitoring_data->per_instruction_opcodes == NULL) { @@ -1563,9 +1588,10 @@ int _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) { - if (is_instrumentation_up_to_date(code, interp)) { - assert(interp->monitoring_version == 0 || - matrix_equals(code->_co_instrumentation.monitoring_data->current_instrumentation, interp->instrumented_events) + if (is_version_up_to_date(code, interp)) { + assert( + interp->monitoring_version == 0 || + instrumentation_cross_checks(interp, code) ); return 0; } @@ -1573,23 +1599,28 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) if (update_instrumentation_data(code, interp)) { return -1; } - _Py_InstrumentationMatrix new_events; - _Py_InstrumentationMatrix removed_events; + _Py_Monitors active_events = matrix_or( + interp->instrumented_events, + code->_co_instrumentation.monitoring_data->local_instrumentation); + _Py_Monitors new_events; + _Py_Monitors removed_events; bool restarted = interp->last_restart_version > code->_co_instrumentation.monitoring_version; if (restarted) { removed_events = code->_co_instrumentation.monitoring_data->current_instrumentation; - new_events = interp->instrumented_events; + new_events = active_events; } else { - removed_events = matrix_sub(code->_co_instrumentation.monitoring_data->current_instrumentation, interp->instrumented_events); - new_events = matrix_sub(interp->instrumented_events, code->_co_instrumentation.monitoring_data->current_instrumentation); + removed_events = matrix_sub(code->_co_instrumentation.monitoring_data->current_instrumentation, active_events); + new_events = matrix_sub(active_events, code->_co_instrumentation.monitoring_data->current_instrumentation); assert(matrix_empty(matrix_and(new_events, removed_events))); } - code->_co_instrumentation.monitoring_data->current_instrumentation = interp->instrumented_events; + code->_co_instrumentation.monitoring_data->current_instrumentation = active_events; code->_co_instrumentation.monitoring_version = interp->monitoring_version; if (matrix_empty(new_events) && matrix_empty(removed_events)) { - //sanity_check_instrumentation(code); +#ifdef INSTRUMENT_DEBUG + sanity_check_instrumentation(code); +#endif return 0; } /* Insert instrumentation */ @@ -1702,24 +1733,56 @@ instrument_all_executing_code_objects(PyInterpreterState *interp) { } } +static void +set_events(_Py_Monitors *m, int tool_id, _PyMonitoringEventSet events) +{ + assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); + for (int e = 0; e < PY_MONITORING_UNGROUPED_EVENTS; e++) { + uint8_t *tools = &m->tools[e]; + int val = (events >> e) & 1; + *tools &= ~(1 << tool_id); + *tools |= (val << tool_id); + } +} + void _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) { + assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); PyInterpreterState *interp = _PyInterpreterState_Get(); - assert((events & C_CALL_EVENTS) == 0 || (events & C_CALL_EVENTS) == C_CALL_EVENTS); - uint32_t existing_events = interpeter_get_events(interp, tool_id); + assert(events < (1 << PY_MONITORING_UNGROUPED_EVENTS)); + uint32_t existing_events = get_events(&interp->instrumented_events, tool_id); if (existing_events == events) { return; } - for (int e = 0; e < PY_MONITORING_EVENTS; e++) { - int val = (events >> e) & 1; - interpreter_set_tool_event(interp, e, tool_id, val); - } + set_events(&interp->instrumented_events, tool_id, events); interp->monitoring_version++; - instrument_all_executing_code_objects(interp); } +int +_PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEventSet events) +{ + assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); + PyInterpreterState *interp = _PyInterpreterState_Get(); + assert(events < (1 << PY_MONITORING_UNGROUPED_EVENTS)); + if (allocate_instrumentation_data(code)) { + return -1; + } + _Py_Monitors *local = &code->_co_instrumentation.monitoring_data->local_instrumentation; + uint32_t existing_events = get_events(local, tool_id); + if (existing_events == events) { + return 0; + } + set_events(local, tool_id, events); + if (is_version_up_to_date(code, interp)) { + /* Force instrumentation update */ + code->_co_instrumentation.monitoring_version = UINT64_MAX; + } + _Py_Instrument(code, interp); + return 0; +} + /*[clinic input] module monitoring [clinic start generated code]*/ @@ -1867,8 +1930,8 @@ monitoring_get_events_impl(PyObject *module, int tool_id) if (check_valid_tool(tool_id)) { return NULL; } - PyInterpreterState *interp = _PyInterpreterState_Get(); - _PyMonitoringEventSet event_set = interpeter_get_events(interp, tool_id); + _Py_Monitors *m = &_PyInterpreterState_Get()->instrumented_events; + _PyMonitoringEventSet event_set = get_events(m, tool_id); return PyLong_FromUnsignedLong(event_set); } @@ -1896,13 +1959,86 @@ monitoring_set_events_impl(PyObject *module, int tool_id, int event_set) PyErr_Format(PyExc_ValueError, "cannot set C_RETURN or C_RAISE events independently"); return NULL; } - if (event_set & (1 << PY_MONITORING_EVENT_CALL)) { - event_set |= C_RETURN_EVENTS; - } + event_set &= ~C_RETURN_EVENTS; _PyMonitoring_SetEvents(tool_id, event_set); Py_RETURN_NONE; } +/*[clinic input] +monitoring.get_local_events + + code: object + tool_id: int + / + +[clinic start generated code]*/ + +static PyObject * +monitoring_get_local_events_impl(PyObject *module, PyObject *code, + int tool_id) +/*[clinic end generated code: output=22fad50d2e6404a7 input=c14650d27de48264]*/ +{ + if (!PyCode_Check(code)) { + PyErr_Format( + PyExc_TypeError, + "code must be a code object" + ); + return NULL; + } + if (check_valid_tool(tool_id)) { + return NULL; + } + _PyMonitoringEventSet event_set = 0; + for (int e = 0; e < PY_MONITORING_UNGROUPED_EVENTS; e++) { + _Py_Monitors *m = &((PyCodeObject *)code)-> + _co_instrumentation.monitoring_data->local_instrumentation; + if ((m->tools[e] >> tool_id) & 1) { + event_set |= (1 << e); + } + } + return PyLong_FromUnsignedLong(event_set); +} + +/*[clinic input] +monitoring.set_local_events + + code: object + tool_id: int + event_set: int + / + +[clinic start generated code]*/ + +static PyObject * +monitoring_set_local_events_impl(PyObject *module, PyObject *code, + int tool_id, int event_set) +/*[clinic end generated code: output=65d616f95cbb76d8 input=2706fbfe062404bf]*/ +{ + if (!PyCode_Check(code)) { + PyErr_Format( + PyExc_TypeError, + "code must be a code object" + ); + return NULL; + } + if (check_valid_tool(tool_id)) { + return NULL; + } + if (event_set < 0 || event_set >= (1 << PY_MONITORING_EVENTS)) { + PyErr_Format(PyExc_ValueError, "invalid event set 0x%x", event_set); + return NULL; + } + if ((event_set & C_RETURN_EVENTS) && (event_set & C_CALL_EVENTS) != C_CALL_EVENTS) { + PyErr_Format(PyExc_ValueError, "cannot set C_RETURN or C_RAISE events independently"); + return NULL; + } + event_set &= ~C_RETURN_EVENTS; + if (_PyMonitoring_SetLocalEvents((PyCodeObject*)code, tool_id, event_set)) { + return NULL; + } + Py_RETURN_NONE; +} + /*[clinic input] monitoring.restart_events @@ -1967,14 +2103,8 @@ monitoring__all_events_impl(PyObject *module) if (res == NULL) { return NULL; } - for (int e = 0; e < PY_MONITORING_EVENTS; e++) { - uint8_t tools; - if (e < PY_MONITORING_INSTRUMENTED_EVENTS) { - tools = interp->instrumented_events.tools[e]; - } - else { - tools = interp->other_events[e]; - } + for (int e = 0; e < PY_MONITORING_UNGROUPED_EVENTS; e++) { + uint8_t tools = interp->instrumented_events.tools[e]; if (tools == 0) { continue; } @@ -1994,6 +2124,8 @@ static PyMethodDef methods[] = { MONITORING_REGISTER_CALLBACK_METHODDEF MONITORING_GET_EVENTS_METHODDEF MONITORING_SET_EVENTS_METHODDEF + MONITORING_GET_LOCAL_EVENTS_METHODDEF + MONITORING_SET_LOCAL_EVENTS_METHODDEF MONITORING_RESTART_EVENTS_METHODDEF MONITORING__ALL_EVENTS_METHODDEF {NULL, NULL} // sentinel diff --git a/Python/pystate.c b/Python/pystate.c index aebe4ae9e1e267..2edba326886331 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -641,12 +641,9 @@ init_interpreter(PyInterpreterState *interp, _PyGC_InitState(&interp->gc); PyConfig_InitPythonConfig(&interp->config); _PyType_InitCache(interp); - for(int i = 0; i < PY_MONITORING_INSTRUMENTED_EVENTS; i++) { + for(int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { interp->instrumented_events.tools[i] = 0; } - for(int i = 0; i < PY_MONITORING_EVENTS-PY_MONITORING_INSTRUMENTED_EVENTS; i++) { - interp->other_events[i] = 0; - } for(int i = 0; i < PY_MONITORING_EVENTS; i++) { for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { interp->tools[t].instrument_callables[i] = NULL; @@ -778,12 +775,9 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) Py_CLEAR(interp->audit_hooks); - for(int i = 0; i < PY_MONITORING_INSTRUMENTED_EVENTS; i++) { + for(int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { interp->instrumented_events.tools[i] = 0; } - for(int i = 0; i < PY_MONITORING_EVENTS-PY_MONITORING_INSTRUMENTED_EVENTS; i++) { - interp->other_events[i] = 0; - } for(int i = 0; i < PY_MONITORING_EVENTS; i++) { for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { Py_CLEAR(interp->tools[t].instrument_callables[i]); From cfb17ed861b18892d5ec574992a7f1a5a3e0444c Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 23 Feb 2023 11:14:35 +0000 Subject: [PATCH 049/116] Make sure that tracing/profiling thread counts are correct. --- Include/internal/pycore_interp.h | 4 ++-- Python/pystate.c | 10 ++++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 5c4b710de08efa..0037fbf0e5a96d 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -169,8 +169,8 @@ struct _is { /* Tools numbered 0-7 */ struct _instrumentation_tool tools[PY_MONITORING_TOOL_IDS]; PyObject *monitoring_tool_names[PY_MONITORING_TOOL_IDS]; - Py_ssize_t sys_profiling_threads; - Py_ssize_t sys_tracing_threads; + Py_ssize_t sys_profiling_threads; /* Count of threads with c_profilefunc set */ + Py_ssize_t sys_tracing_threads; /* Count of threads with c_tracefunc set */ struct _Py_interp_cached_objects cached_objects; struct _Py_interp_static_objects static_objects; diff --git a/Python/pystate.c b/Python/pystate.c index 8daece9f4569b1..67c479272631d9 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1380,8 +1380,14 @@ PyThreadState_Clear(PyThreadState *tstate) "PyThreadState_Clear: warning: thread still has a generator\n"); } - tstate->c_profilefunc = NULL; - tstate->c_tracefunc = NULL; + if (tstate->c_profilefunc != NULL) { + tstate->interp->sys_profiling_threads--; + tstate->c_profilefunc = NULL; + } + if (tstate->c_tracefunc != NULL) { + tstate->interp->sys_tracing_threads--; + tstate->c_tracefunc = NULL; + } Py_CLEAR(tstate->c_profileobj); Py_CLEAR(tstate->c_traceobj); From c535f76636be9860be55fad3e882c268f3749077 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 23 Feb 2023 11:43:21 +0000 Subject: [PATCH 050/116] Remove linearray field --- Include/cpython/code.h | 2 -- Include/internal/pycore_code.h | 26 --------------- Objects/codeobject.c | 60 ---------------------------------- Python/instrumentation.c | 2 +- Tools/build/deepfreeze.py | 2 -- 5 files changed, 1 insertion(+), 91 deletions(-) diff --git a/Include/cpython/code.h b/Include/cpython/code.h index 5e62233e906f5e..b2e32f15d5ca3e 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -115,7 +115,6 @@ typedef struct { PyObject *co_exceptiontable; /* Byte string encoding exception handling \ table */ \ int co_flags; /* CO_..., see below */ \ - short _co_linearray_entry_size; /* Size of each entry in _co_linearray */ \ \ /* The rest are not so impactful on performance. */ \ int co_argcount; /* #arguments, except *args */ \ @@ -144,7 +143,6 @@ typedef struct { _PyCoCached *_co_cached; /* cached co_* attributes */ \ _PyCoInstrumentation _co_instrumentation; /* Instrumentation */ \ int _co_firsttraceable; /* index of first traceable instruction */ \ - char *_co_linearray; /* array of line offsets */ \ /* Scratch space for extra data relating to the code object. \ Type is a void* to keep the format private in codeobject.c to force \ people to go through the proper APIs. */ \ diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 70533e47be4dfb..6fa79a121e2b7b 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -444,32 +444,6 @@ adaptive_counter_backoff(uint16_t counter) { /* Line array cache for tracing */ -extern int _PyCode_CreateLineArray(PyCodeObject *co); - -static inline int -_PyCode_InitLineArray(PyCodeObject *co) -{ - if (co->_co_linearray) { - return 0; - } - return _PyCode_CreateLineArray(co); -} - -static inline int -_PyCode_LineNumberFromArray(PyCodeObject *co, int index) -{ - assert(co->_co_linearray != NULL); - assert(index >= 0); - assert(index < Py_SIZE(co)); - if (co->_co_linearray_entry_size == 2) { - return ((int16_t *)co->_co_linearray)[index]; - } - else { - assert(co->_co_linearray_entry_size == 4); - return ((int32_t *)co->_co_linearray)[index]; - } -} - typedef struct _PyShimCodeDef { const uint8_t *code; int codelen; diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 1774519cdbbf1f..367cf3095fee93 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -409,8 +409,6 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) co->co_extra = NULL; co->_co_cached = NULL; - co->_co_linearray_entry_size = 0; - co->_co_linearray = NULL; memcpy(_PyCode_CODE(co), PyBytes_AS_STRING(con->code), PyBytes_GET_SIZE(con->code)); int entry_point = 0; @@ -788,54 +786,6 @@ PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno) * source location tracking (co_lines/co_positions) ******************/ -/* Use co_linetable to compute the line number from a bytecode index, addrq. See - lnotab_notes.txt for the details of the lnotab representation. -*/ - -int -_PyCode_CreateLineArray(PyCodeObject *co) -{ - assert(co->_co_linearray == NULL); - PyCodeAddressRange bounds; - int size; - int max_line = 0; - _PyCode_InitAddressRange(co, &bounds); - while(_PyLineTable_NextAddressRange(&bounds)) { - if (bounds.ar_line > max_line) { - max_line = bounds.ar_line; - } - } - if (max_line < (1 << 15)) { - size = 2; - } - else { - size = 4; - } - co->_co_linearray = PyMem_Malloc(Py_SIZE(co)*size); - if (co->_co_linearray == NULL) { - PyErr_NoMemory(); - return -1; - } - co->_co_linearray_entry_size = size; - _PyCode_InitAddressRange(co, &bounds); - while(_PyLineTable_NextAddressRange(&bounds)) { - int start = bounds.ar_start / sizeof(_Py_CODEUNIT); - int end = bounds.ar_end / sizeof(_Py_CODEUNIT); - for (int index = start; index < end; index++) { - assert(index < (int)Py_SIZE(co)); - if (size == 2) { - assert(((int16_t)bounds.ar_line) == bounds.ar_line); - ((int16_t *)co->_co_linearray)[index] = bounds.ar_line; - } - else { - assert(size == 4); - ((int32_t *)co->_co_linearray)[index] = bounds.ar_line; - } - } - } - return 0; -} - int PyCode_Addr2Line(PyCodeObject *co, int addrq) { @@ -843,9 +793,6 @@ PyCode_Addr2Line(PyCodeObject *co, int addrq) return co->co_firstlineno; } assert(addrq >= 0 && addrq < _PyCode_NBYTES(co)); - if (co->_co_linearray) { - return _PyCode_LineNumberFromArray(co, addrq / sizeof(_Py_CODEUNIT)); - } PyCodeAddressRange bounds; _PyCode_InitAddressRange(co, &bounds); return _PyCode_CheckLineNumber(addrq, &bounds); @@ -1722,9 +1669,6 @@ code_dealloc(PyCodeObject *co) PyMem_Free(data->per_instruction_tools); } } - if (co->_co_linearray) { - PyMem_Free(co->_co_linearray); - } PyObject_Free(co); } @@ -2307,10 +2251,6 @@ _PyStaticCode_Fini(PyCodeObject *co) PyObject_ClearWeakRefs((PyObject *)co); co->co_weakreflist = NULL; } - if (co->_co_linearray) { - PyMem_Free(co->_co_linearray); - co->_co_linearray = NULL; - } } int diff --git a/Python/instrumentation.c b/Python/instrumentation.c index f02e33faf55b25..747f699e5df308 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -282,7 +282,7 @@ compute_line(PyCodeObject *code, int offset, int8_t line_delta) } else { assert(line_delta == -127); - /* Compute from table */ + /* Look it up */ return PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT)); } } diff --git a/Tools/build/deepfreeze.py b/Tools/build/deepfreeze.py index 511b26a5ce3dc7..bfdaf8ac41c451 100644 --- a/Tools/build/deepfreeze.py +++ b/Tools/build/deepfreeze.py @@ -255,7 +255,6 @@ def generate_code(self, name: str, code: types.CodeType) -> str: self.write(f".co_names = {co_names},") self.write(f".co_exceptiontable = {co_exceptiontable},") self.field(code, "co_flags") - self.write("._co_linearray_entry_size = 0,") self.field(code, "co_argcount") self.field(code, "co_posonlyargcount") self.field(code, "co_kwonlyargcount") @@ -276,7 +275,6 @@ def generate_code(self, name: str, code: types.CodeType) -> str: self.write(f".co_qualname = {co_qualname},") self.write(f".co_linetable = {co_linetable},") self.write(f"._co_cached = NULL,") - self.write("._co_linearray = NULL,") self.write(f".co_code_adaptive = {co_code_adaptive},") for i, op in enumerate(code.co_code[::2]): if op == RESUME: From d579d2e73eb77cbd1bd676604f50151663ac73f5 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 23 Feb 2023 12:16:00 +0000 Subject: [PATCH 051/116] Rename some fields, shortening names. --- Include/cpython/code.h | 14 +- Include/internal/pycore_interp.h | 11 +- Objects/codeobject.c | 6 +- Python/bytecodes.c | 4 +- Python/ceval.c | 6 +- Python/generated_cases.c.h | 4 +- Python/instrumentation.c | 306 +++++++++++++++---------------- Python/pystate.c | 16 +- 8 files changed, 179 insertions(+), 188 deletions(-) diff --git a/Include/cpython/code.h b/Include/cpython/code.h index b2e32f15d5ca3e..ebc832e79974bc 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -70,19 +70,14 @@ typedef struct { } _PyCoLineInstrumentationData; typedef struct { - _Py_Monitors local_instrumentation; - _Py_Monitors current_instrumentation; + _Py_Monitors local_monitors; + _Py_Monitors active_monitors; uint8_t *tools; _PyCoLineInstrumentationData *lines; uint8_t *line_tools; uint8_t *per_instruction_opcodes; uint8_t *per_instruction_tools; -} _PyCoInstrumentationData; - -typedef struct { - uint64_t monitoring_version; /* current instrumentation version */ - _PyCoInstrumentationData *monitoring_data; /* data for monitoring */ -} _PyCoInstrumentation; +} _PyCoMonitoringData; // To avoid repeating ourselves in deepfreeze.py, all PyCodeObject members are // defined in this macro: @@ -141,7 +136,8 @@ typedef struct { PyObject *co_linetable; /* bytes object that holds location info */ \ PyObject *co_weakreflist; /* to support weakrefs to code objects */ \ _PyCoCached *_co_cached; /* cached co_* attributes */ \ - _PyCoInstrumentation _co_instrumentation; /* Instrumentation */ \ + uint64_t _co_instrumentation_version; /* current instrumentation version */ \ + _PyCoMonitoringData *_co_monitoring; /* Monitoring data */ \ int _co_firsttraceable; /* index of first traceable instruction */ \ /* Scratch space for extra data relating to the code object. \ Type is a void* to keep the format private in codeobject.c to force \ diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 0037fbf0e5a96d..36198d2c41bd33 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -49,10 +49,6 @@ struct _Py_long_state { int max_str_digits; }; -struct _instrumentation_tool { - PyObject *instrument_callables[PY_MONITORING_EVENTS]; -}; - /* interpreter state */ /* PyInterpreterState holds the global state for one of the runtime's @@ -162,15 +158,14 @@ struct _is { struct callable_cache callable_cache; PyCodeObject *interpreter_trampoline; - _Py_Monitors instrumented_events; + _Py_Monitors monitors; bool f_opcode_trace_set; bool sys_profile_initialized; bool sys_trace_initialized; - /* Tools numbered 0-7 */ - struct _instrumentation_tool tools[PY_MONITORING_TOOL_IDS]; - PyObject *monitoring_tool_names[PY_MONITORING_TOOL_IDS]; Py_ssize_t sys_profiling_threads; /* Count of threads with c_profilefunc set */ Py_ssize_t sys_tracing_threads; /* Count of threads with c_tracefunc set */ + PyObject *monitoring_callables[PY_MONITORING_TOOL_IDS][PY_MONITORING_EVENTS]; + PyObject *monitoring_tool_names[PY_MONITORING_TOOL_IDS]; struct _Py_interp_cached_objects cached_objects; struct _Py_interp_static_objects static_objects; diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 367cf3095fee93..44cb31a1fcb689 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -402,8 +402,8 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) if (_Py_next_func_version != 0) { _Py_next_func_version++; } - co->_co_instrumentation.monitoring_data = NULL; - co->_co_instrumentation.monitoring_version = 0; + co->_co_monitoring = NULL; + co->_co_instrumentation_version = 0; /* not set */ co->co_weakreflist = NULL; co->co_extra = NULL; @@ -1651,7 +1651,7 @@ code_dealloc(PyCodeObject *co) if (co->co_weakreflist != NULL) { PyObject_ClearWeakRefs((PyObject*)co); } - _PyCoInstrumentationData *data = co->_co_instrumentation.monitoring_data; + _PyCoMonitoringData *data = co->_co_monitoring; if (data) { if (data->tools) { PyMem_Free(data->tools); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 3ee972cbb395d0..df4686fa033a59 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -91,7 +91,7 @@ dummy_func( inst(INSTRUMENTED_RESUME, (--)) { /* Check monitoring *before* calling instrument */ /* Possibly combine this with eval breaker */ - if (frame->f_code->_co_instrumentation.monitoring_version != tstate->interp->monitoring_version) { + if (frame->f_code->_co_instrumentation_version != tstate->interp->monitoring_version) { if (_Py_Instrument(frame->f_code, tstate->interp)) { goto error; } @@ -118,7 +118,7 @@ dummy_func( assert(tstate->cframe == &cframe); assert(frame == cframe.current_frame); /* Possibly combine this with eval breaker */ - if (frame->f_code->_co_instrumentation.monitoring_version != tstate->interp->monitoring_version) { + if (frame->f_code->_co_instrumentation_version != tstate->interp->monitoring_version) { int err = _Py_Instrument(frame->f_code, tstate->interp); ERROR_IF(err, error); next_instr--; diff --git a/Python/ceval.c b/Python/ceval.c index e1eb73935fedc7..8c0a80487985fc 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1961,14 +1961,14 @@ do_monitor_exc( static inline int no_tools_for_event(PyThreadState *tstate, _PyInterpreterFrame *frame, int event) { - _PyCoInstrumentationData *data = frame->f_code->_co_instrumentation.monitoring_data; + _PyCoMonitoringData *data = frame->f_code->_co_monitoring; if (data) { - if (data->current_instrumentation.tools[event] == 0) { + if (data->active_monitors.tools[event] == 0) { return 1; } } else { - if (tstate->interp->instrumented_events.tools[event] == 0) { + if (tstate->interp->monitors.tools[event] == 0) { return 1; } } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 5c125bd824c30e..64cc0aaa581c71 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -9,7 +9,7 @@ TARGET(INSTRUMENTED_RESUME) { /* Check monitoring *before* calling instrument */ /* Possibly combine this with eval breaker */ - if (frame->f_code->_co_instrumentation.monitoring_version != tstate->interp->monitoring_version) { + if (frame->f_code->_co_instrumentation_version != tstate->interp->monitoring_version) { if (_Py_Instrument(frame->f_code, tstate->interp)) { goto error; } @@ -37,7 +37,7 @@ assert(tstate->cframe == &cframe); assert(frame == cframe.current_frame); /* Possibly combine this with eval breaker */ - if (frame->f_code->_co_instrumentation.monitoring_version != tstate->interp->monitoring_version) { + if (frame->f_code->_co_instrumentation_version != tstate->interp->monitoring_version) { int err = _Py_Instrument(frame->f_code, tstate->interp); if (err) goto error; next_instr--; diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 747f699e5df308..c0f4d4920cda82 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -168,7 +168,7 @@ is_instrumented(int opcode) { } static inline bool -matrix_equals(_Py_Monitors a, _Py_Monitors b) +monitors_equals(_Py_Monitors a, _Py_Monitors b) { for (int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { if (a.tools[i] != b.tools[i]) { @@ -179,7 +179,7 @@ matrix_equals(_Py_Monitors a, _Py_Monitors b) } static inline _Py_Monitors -matrix_sub(_Py_Monitors a, _Py_Monitors b) +monitors_sub(_Py_Monitors a, _Py_Monitors b) { _Py_Monitors res; for (int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { @@ -189,7 +189,7 @@ matrix_sub(_Py_Monitors a, _Py_Monitors b) } static inline _Py_Monitors -matrix_and(_Py_Monitors a, _Py_Monitors b) +monitors_and(_Py_Monitors a, _Py_Monitors b) { _Py_Monitors res; for (int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { @@ -199,7 +199,7 @@ matrix_and(_Py_Monitors a, _Py_Monitors b) } static inline _Py_Monitors -matrix_or(_Py_Monitors a, _Py_Monitors b) +monitors_or(_Py_Monitors a, _Py_Monitors b) { _Py_Monitors res; for (int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { @@ -209,7 +209,7 @@ matrix_or(_Py_Monitors a, _Py_Monitors b) } static inline bool -matrix_empty(_Py_Monitors m) +monitors_empty(_Py_Monitors m) { for (int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { if (m.tools[i]) { @@ -297,10 +297,10 @@ int _Py_GetBaseOpcode(PyCodeObject *code, int offset) return _PyOpcode_Deopt[opcode]; } if (opcode == INSTRUMENTED_INSTRUCTION) { - opcode = code->_co_instrumentation.monitoring_data->per_instruction_opcodes[offset]; + opcode = code->_co_monitoring->per_instruction_opcodes[offset]; } if (opcode == INSTRUMENTED_LINE) { - opcode = code->_co_instrumentation.monitoring_data->lines[offset].original_opcode; + opcode = code->_co_monitoring->lines[offset].original_opcode; } int deinstrumented = DE_INSTRUMENT[opcode]; if (deinstrumented) { @@ -318,10 +318,10 @@ uint8_t deinstrument(PyCodeObject *code, int offset) return _PyOpcode_Deopt[opcode]; } if (opcode == INSTRUMENTED_INSTRUCTION) { - opcode = code->_co_instrumentation.monitoring_data->per_instruction_opcodes[offset]; + opcode = code->_co_monitoring->per_instruction_opcodes[offset]; } if (opcode == INSTRUMENTED_LINE) { - opcode = code->_co_instrumentation.monitoring_data->lines[offset].original_opcode; + opcode = code->_co_monitoring->lines[offset].original_opcode; } int deinstrumented = DE_INSTRUMENT[opcode]; if (deinstrumented) { @@ -346,12 +346,12 @@ Instruction read_instruction(PyCodeObject *code, int offset) Instruction result = (Instruction){0}; int opcode = result.actual_opcode = _PyCode_CODE(code)[offset].op.code; if (opcode == INSTRUMENTED_INSTRUCTION) { - opcode = code->_co_instrumentation.monitoring_data->per_instruction_opcodes[offset]; + opcode = code->_co_monitoring->per_instruction_opcodes[offset]; result.is_prefix_instrumented = true; } if (opcode == INSTRUMENTED_LINE) { - opcode = code->_co_instrumentation.monitoring_data->lines[offset].original_opcode; + opcode = code->_co_monitoring->lines[offset].original_opcode; result.is_prefix_instrumented = true; } while (opcode == EXTENDED_ARG) { @@ -420,7 +420,7 @@ dump_instrumentation_data_line_tools(PyCodeObject *code, uint8_t *line_tools, in } static void -dump_instrumentation_data_per_instruction(PyCodeObject *code, _PyCoInstrumentationData *data, int i, FILE*out) +dump_instrumentation_data_per_instruction(PyCodeObject *code, _PyCoMonitoringData *data, int i, FILE*out) { if (data->per_instruction_opcodes == NULL) { fprintf(out, ", per-inst opcode = NULL"); @@ -436,14 +436,12 @@ dump_instrumentation_data_per_instruction(PyCodeObject *code, _PyCoInstrumentati } } - - static void -dump_matrix(const char *prefix, _Py_MonitoringMatrix matrix, FILE*out) +dump_monitors(const char *prefix, _Py_Monitors monitors, FILE*out) { - fprintf(out, "%s matrix:\n", prefix); - for (int event = 0; event < PY_MONITORING_EVENTS; event++) { - fprintf(out, " Event %d: Tools %x\n", event, matrix.tools[event]); + fprintf(out, "%s monitors:\n", prefix); + for (int event = 0; event < PY_MONITORING_UNGROUPED_EVENTS; event++) { + fprintf(out, " Event %d: Tools %x\n", event, monitors.tools[event]); } } @@ -458,16 +456,16 @@ int get_base_opcode_best_attempt(PyCodeObject *code, int offset) return _PyOpcode_Deopt[opcode] == 0 ? opcode : _PyOpcode_Deopt[opcode]; } if (opcode == INSTRUMENTED_INSTRUCTION) { - if (code->_co_instrumentation.monitoring_data->per_instruction_opcodes[offset] == 0) { + if (code->_co_monitoring->per_instruction_opcodes[offset] == 0) { return opcode; } - opcode = code->_co_instrumentation.monitoring_data->per_instruction_opcodes[offset]; + opcode = code->_co_monitoring->per_instruction_opcodes[offset]; } if (opcode == INSTRUMENTED_LINE) { - if (code->_co_instrumentation.monitoring_data->lines[offset].original_opcode == 0) { + if (code->_co_monitoring->lines[offset].original_opcode == 0) { return opcode; } - opcode = code->_co_instrumentation.monitoring_data->lines[offset].original_opcode; + opcode = code->_co_monitoring->lines[offset].original_opcode; } int deinstrumented = DE_INSTRUMENT[opcode]; if (deinstrumented) { @@ -483,14 +481,15 @@ int get_base_opcode_best_attempt(PyCodeObject *code, int offset) static void dump_instrumentation_data(PyCodeObject *code, int star, FILE*out) { - _PyCoInstrumentationData *data = code->_co_instrumentation.monitoring_data; + _PyCoMonitoringData *data = code->_co_monitoring; fprintf(out, "\n"); if (data == NULL) { fprintf(out, "NULL\n"); return; } - dump_matrix("Global", PyInterpreterState_Get()->monitoring_matrix, out); - dump_matrix("Code", data->matrix, out); + dump_monitors("Global", PyInterpreterState_Get()->monitors, out); + dump_monitors("Code", data->local_monitors, out); + dump_monitors("Active", data->active_monitors, out); int code_len = (int)Py_SIZE(code); bool starred = false; for (int i = 0; i < code_len; i++) { @@ -542,14 +541,18 @@ bool valid_opcode(int opcode) { static void sanity_check_instrumentation(PyCodeObject *code) { - _PyCoInstrumentationData *data = code->_co_instrumentation.monitoring_data; + _PyCoMonitoringData *data = code->_co_monitoring; if (data == NULL) { return; } - _Py_MonitoringMatrix tools_matrix = PyInterpreterState_Get()->monitoring_matrix; - assert(matrix_equals( - code->_co_instrumentation.monitoring_data->matrix, - tools_matrix) + _Py_Monitors active_monitors = PyInterpreterState_Get()->monitors; + if (code->_co_monitoring) { + _Py_Monitors local_monitors = code->_co_monitoring->local_monitors; + active_monitors = monitors_or(active_monitors, local_monitors); + } + assert(monitors_equals( + code->_co_monitoring->active_monitors, + active_monitors) ); int code_len = (int)Py_SIZE(code); for (int i = 0; i < code_len;) { @@ -584,7 +587,7 @@ sanity_check_instrumentation(PyCodeObject *code) /* RESUME fixup */ event = _PyCode_CODE(code)[i + inst.extended_args].op.arg; } - CHECK(tools_matrix.tools[event] != 0); + CHECK(active_monitors.tools[event] != 0); CHECK(inst.deinstrumented_opcode != END_FOR); } if (_PyOpcode_Deopt[inst.deinstrumented_opcode] == COMPARE_AND_BRANCH) { @@ -612,8 +615,7 @@ sanity_check_instrumentation(PyCodeObject *code) /* RESUME fixup */ event = _PyCode_CODE(code)[i + inst.extended_args].op.arg; } - uint8_t global_tools = tools_matrix.tools[event]; - CHECK((global_tools & local_tools) == local_tools); + CHECK((active_monitors.tools[event] & local_tools) == local_tools); } else { CHECK(local_tools == 0xff); @@ -641,18 +643,18 @@ de_instrument(PyCodeObject *code, int offset, int event) } switch(opcode) { case INSTRUMENTED_INSTRUCTION: - opcode = code->_co_instrumentation.monitoring_data->per_instruction_opcodes[offset]; + opcode = code->_co_monitoring->per_instruction_opcodes[offset]; if (opcode != INSTRUMENTED_LINE) { int deinstrumented = DE_INSTRUMENT[opcode]; if (deinstrumented) { - code->_co_instrumentation.monitoring_data->per_instruction_opcodes[offset] = deinstrumented; + code->_co_monitoring->per_instruction_opcodes[offset] = deinstrumented; } break; } /* Intentional fall through */ case INSTRUMENTED_LINE: { - _PyCoLineInstrumentationData *lines = &code->_co_instrumentation.monitoring_data->lines[offset]; + _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[offset]; int orignal_opcode = lines->original_opcode; int deinstrumented = DE_INSTRUMENT[orignal_opcode]; if (deinstrumented) { @@ -682,12 +684,12 @@ de_instrument_line(PyCodeObject *code, int i) _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; int opcode = _Py_OPCODE(*instr); if (opcode == INSTRUMENTED_INSTRUCTION) { - opcode = code->_co_instrumentation.monitoring_data->per_instruction_opcodes[i]; + opcode = code->_co_monitoring->per_instruction_opcodes[i]; } if (opcode != INSTRUMENTED_LINE) { return; } - _PyCoLineInstrumentationData *lines = &code->_co_instrumentation.monitoring_data->lines[i]; + _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[i]; int original_opcode = lines->original_opcode; assert(is_instrumented(original_opcode) || _PyOpcode_Deopt[original_opcode]); if (is_instrumented(original_opcode)) { @@ -702,7 +704,7 @@ de_instrument_line(PyCodeObject *code, int i) } } if (instr->op.code == INSTRUMENTED_INSTRUCTION) { - code->_co_instrumentation.monitoring_data->per_instruction_opcodes[i] = original_opcode; + code->_co_monitoring->per_instruction_opcodes[i] = original_opcode; } else { instr->op.code = original_opcode; @@ -722,7 +724,7 @@ de_instrument_per_instruction(PyCodeObject *code, int offset) if (opcode != INSTRUMENTED_INSTRUCTION) { return; } - int original_opcode = code->_co_instrumentation.monitoring_data->per_instruction_opcodes[offset]; + int original_opcode = code->_co_monitoring->per_instruction_opcodes[offset]; assert(is_instrumented(original_opcode) || _PyOpcode_Deopt[original_opcode]); if (is_instrumented(original_opcode)) { /* Instrumented original */ @@ -738,7 +740,7 @@ de_instrument_per_instruction(PyCodeObject *code, int offset) } assert(instr->op.code != INSTRUMENTED_INSTRUCTION); /* Keep things clean for snaity check */ - code->_co_instrumentation.monitoring_data->per_instruction_opcodes[offset] = 0; + code->_co_monitoring->per_instruction_opcodes[offset] = 0; } @@ -749,7 +751,7 @@ instrument(PyCodeObject *code, int offset) _Py_CODEUNIT *instr = &_PyCode_CODE(code)[offset]; int opcode = _Py_OPCODE(*instr); if (opcode == INSTRUMENTED_LINE) { - _PyCoLineInstrumentationData *lines = &code->_co_instrumentation.monitoring_data->lines[offset]; + _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[offset]; opcode = lines->original_opcode; assert(!is_instrumented(opcode)); lines->original_opcode = INSTRUMENTED_OPCODES[opcode]; @@ -775,7 +777,7 @@ instrument_line(PyCodeObject *code, int i) if (opcode == INSTRUMENTED_LINE) { return; } - _PyCoLineInstrumentationData *lines = &code->_co_instrumentation.monitoring_data->lines[i]; + _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[i]; CHECK(lines->original_opcode == 255); if (is_instrumented(opcode)) { lines->original_opcode = opcode; @@ -789,7 +791,7 @@ instrument_line(PyCodeObject *code, int i) } assert(lines->original_opcode > 0); instr->op.code = INSTRUMENTED_LINE; - CHECK(code->_co_instrumentation.monitoring_data->lines[i].original_opcode != 0); + CHECK(code->_co_monitoring->lines[i].original_opcode != 0); } static void @@ -802,15 +804,15 @@ instrument_per_instruction(PyCodeObject *code, int i) } CHECK(opcode != 0); if (is_instrumented(opcode)) { - code->_co_instrumentation.monitoring_data->per_instruction_opcodes[i] = opcode; + code->_co_monitoring->per_instruction_opcodes[i] = opcode; } else { assert(opcode != 0); assert(_PyOpcode_Deopt[opcode] != 0); assert(_PyOpcode_Deopt[opcode] != RESUME); - code->_co_instrumentation.monitoring_data->per_instruction_opcodes[i] = _PyOpcode_Deopt[opcode]; + code->_co_monitoring->per_instruction_opcodes[i] = _PyOpcode_Deopt[opcode]; } - assert(code->_co_instrumentation.monitoring_data->per_instruction_opcodes[i] > 0); + assert(code->_co_monitoring->per_instruction_opcodes[i] > 0); instr->op.code = INSTRUMENTED_INSTRUCTION; } @@ -821,10 +823,10 @@ instruction_has_event(PyCodeObject *code, int offset) _Py_CODEUNIT instr = _PyCode_CODE(code)[offset]; int opcode = instr.op.code; if (opcode == INSTRUMENTED_INSTRUCTION) { - opcode = code->_co_instrumentation.monitoring_data->per_instruction_opcodes[offset]; + opcode = code->_co_monitoring->per_instruction_opcodes[offset]; } if (opcode == INSTRUMENTED_LINE) { - opcode = code->_co_instrumentation.monitoring_data->lines[offset].original_opcode; + opcode = code->_co_monitoring->lines[offset].original_opcode; } assert(DE_INSTRUMENT[opcode] == 0 || OPCODE_HAS_EVENT[DE_INSTRUMENT[opcode]] == OPCODE_HAS_EVENT[opcode]); return OPCODE_HAS_EVENT[opcode]; @@ -838,7 +840,7 @@ remove_tools(PyCodeObject * code, int offset, int event, int tools) assert(event != PY_MONITORING_EVENT_INSTRUCTION); assert(event < PY_MONITORING_INSTRUMENTED_EVENTS); assert(instruction_has_event(code, offset)); - _PyCoInstrumentationData *monitoring = code->_co_instrumentation.monitoring_data; + _PyCoMonitoringData *monitoring = code->_co_monitoring; if (monitoring && monitoring->tools) { monitoring->tools[offset] &= ~tools; if (monitoring->tools[offset] == 0) { @@ -847,7 +849,7 @@ remove_tools(PyCodeObject * code, int offset, int event, int tools) } else { /* Single tool */ - uint8_t single_tool = code->_co_instrumentation.monitoring_data->current_instrumentation.tools[event]; + uint8_t single_tool = code->_co_monitoring->active_monitors.tools[event]; assert(_Py_popcount32(single_tool) <= 1); if (((single_tool & tools) == single_tool)) { de_instrument(code, offset, event); @@ -858,8 +860,8 @@ remove_tools(PyCodeObject * code, int offset, int event, int tools) #ifndef NDEBUG bool tools_is_subset_for_event(PyCodeObject * code, int event, int tools) { - int global_tools = PyInterpreterState_Get()->instrumented_events.tools[event]; - int local_tools = code->_co_instrumentation.monitoring_data->local_instrumentation.tools[event]; + int global_tools = PyInterpreterState_Get()->monitors.tools[event]; + int local_tools = code->_co_monitoring->local_monitors.tools[event]; return tools == ((global_tools | local_tools) & tools); } #endif @@ -867,10 +869,10 @@ bool tools_is_subset_for_event(PyCodeObject * code, int event, int tools) static void remove_line_tools(PyCodeObject * code, int offset, int tools) { - assert(code->_co_instrumentation.monitoring_data); - if (code->_co_instrumentation.monitoring_data->line_tools) + assert(code->_co_monitoring); + if (code->_co_monitoring->line_tools) { - uint8_t *toolsptr = &code->_co_instrumentation.monitoring_data->line_tools[offset]; + uint8_t *toolsptr = &code->_co_monitoring->line_tools[offset]; *toolsptr &= ~tools; if (*toolsptr == 0 ) { de_instrument_line(code, offset); @@ -878,7 +880,7 @@ remove_line_tools(PyCodeObject * code, int offset, int tools) } else { /* Single tool */ - uint8_t single_tool = code->_co_instrumentation.monitoring_data->current_instrumentation.tools[PY_MONITORING_EVENT_LINE]; + uint8_t single_tool = code->_co_monitoring->active_monitors.tools[PY_MONITORING_EVENT_LINE]; assert(_Py_popcount32(single_tool) <= 1); if (((single_tool & tools) == single_tool)) { de_instrument_line(code, offset); @@ -892,11 +894,11 @@ add_tools(PyCodeObject * code, int offset, int event, int tools) assert(event != PY_MONITORING_EVENT_LINE); assert(event != PY_MONITORING_EVENT_INSTRUCTION); assert(event < PY_MONITORING_INSTRUMENTED_EVENTS); - assert(code->_co_instrumentation.monitoring_data); - if (code->_co_instrumentation.monitoring_data && - code->_co_instrumentation.monitoring_data->tools + assert(code->_co_monitoring); + if (code->_co_monitoring && + code->_co_monitoring->tools ) { - code->_co_instrumentation.monitoring_data->tools[offset] |= tools; + code->_co_monitoring->tools[offset] |= tools; } else { /* Single tool */ @@ -911,10 +913,10 @@ static void add_line_tools(PyCodeObject * code, int offset, int tools) { assert(tools_is_subset_for_event(code, PY_MONITORING_EVENT_LINE, tools)); - assert(code->_co_instrumentation.monitoring_data); - if (code->_co_instrumentation.monitoring_data->line_tools + assert(code->_co_monitoring); + if (code->_co_monitoring->line_tools ) { - code->_co_instrumentation.monitoring_data->line_tools[offset] |= tools; + code->_co_monitoring->line_tools[offset] |= tools; } else { /* Single tool */ @@ -928,10 +930,10 @@ static void add_per_instruction_tools(PyCodeObject * code, int offset, int tools) { assert(tools_is_subset_for_event(code, PY_MONITORING_EVENT_INSTRUCTION, tools)); - assert(code->_co_instrumentation.monitoring_data); - if (code->_co_instrumentation.monitoring_data->per_instruction_tools + assert(code->_co_monitoring); + if (code->_co_monitoring->per_instruction_tools ) { - code->_co_instrumentation.monitoring_data->per_instruction_tools[offset] |= tools; + code->_co_monitoring->per_instruction_tools[offset] |= tools; } else { /* Single tool */ @@ -944,10 +946,10 @@ add_per_instruction_tools(PyCodeObject * code, int offset, int tools) static void remove_per_instruction_tools(PyCodeObject * code, int offset, int tools) { - assert(code->_co_instrumentation.monitoring_data); - if (code->_co_instrumentation.monitoring_data->per_instruction_tools) + assert(code->_co_monitoring); + if (code->_co_monitoring->per_instruction_tools) { - uint8_t *toolsptr = &code->_co_instrumentation.monitoring_data->per_instruction_tools[offset]; + uint8_t *toolsptr = &code->_co_monitoring->per_instruction_tools[offset]; *toolsptr &= ~tools; if (*toolsptr == 0 ) { de_instrument_per_instruction(code, offset); @@ -955,7 +957,7 @@ remove_per_instruction_tools(PyCodeObject * code, int offset, int tools) } else { /* Single tool */ - uint8_t single_tool = code->_co_instrumentation.monitoring_data->current_instrumentation.tools[PY_MONITORING_EVENT_INSTRUCTION]; + uint8_t single_tool = code->_co_monitoring->active_monitors.tools[PY_MONITORING_EVENT_INSTRUCTION]; assert(_Py_popcount32(single_tool) <= 1); if (((single_tool & tools) == single_tool)) { de_instrument_per_instruction(code, offset); @@ -972,7 +974,7 @@ call_one_instrument( { assert(0 <= tool && tool < 8); assert(tstate->tracing == 0); - PyObject *instrument = interp->tools[tool].instrument_callables[event]; + PyObject *instrument = interp->monitoring_callables[tool][event]; if (instrument == NULL) { return 0; } @@ -1011,18 +1013,17 @@ static inline int most_significant_bit(uint8_t bits) { static bool is_version_up_to_date(PyCodeObject *code, PyInterpreterState *interp) { - return interp->monitoring_version == code->_co_instrumentation.monitoring_version; + return interp->monitoring_version == code->_co_instrumentation_version; } #ifndef NDEBUG static bool instrumentation_cross_checks(PyInterpreterState *interp, PyCodeObject *code) { - _Py_Monitors expected = matrix_or( - interp->instrumented_events, - code->_co_instrumentation.monitoring_data->local_instrumentation); - return matrix_equals(code->_co_instrumentation.monitoring_data->current_instrumentation, - expected); + _Py_Monitors expected = monitors_or( + interp->monitors, + code->_co_monitoring->local_monitors); + return monitors_equals(code->_co_monitoring->active_monitors, expected); } #endif @@ -1033,7 +1034,7 @@ get_tools_for_instruction(PyCodeObject * code, int i, int event) assert(event != PY_MONITORING_EVENT_LINE); assert(event != PY_MONITORING_EVENT_INSTRUCTION); assert(instrumentation_cross_checks(PyThreadState_GET()->interp, code)); - _PyCoInstrumentationData *monitoring = code->_co_instrumentation.monitoring_data; + _PyCoMonitoringData *monitoring = code->_co_monitoring; if (monitoring->tools) { tools = monitoring->tools[i]; } @@ -1043,10 +1044,10 @@ get_tools_for_instruction(PyCodeObject * code, int i, int event) event == PY_MONITORING_EVENT_C_RETURN); event = PY_MONITORING_EVENT_CALL; } - tools = code->_co_instrumentation.monitoring_data->current_instrumentation.tools[event]; + tools = code->_co_monitoring->active_monitors.tools[event]; } CHECK(tools_is_subset_for_event(code, event, tools)); - CHECK((tools & code->_co_instrumentation.monitoring_data->current_instrumentation.tools[event]) == tools); + CHECK((tools & code->_co_monitoring->active_monitors.tools[event]) == tools); return tools; } @@ -1186,9 +1187,9 @@ _Py_call_instrumentation_exc_vector( PyCodeObject *code = frame->f_code; assert(args[1] == NULL); args[1] = (PyObject *)code; - assert(code->_co_instrumentation.monitoring_version == tstate->interp->monitoring_version); + assert(code->_co_instrumentation_version == tstate->interp->monitoring_version); int offset = instr - _PyCode_CODE(code); - uint8_t tools = code->_co_instrumentation.monitoring_data->current_instrumentation.tools[event]; + uint8_t tools = code->_co_monitoring->active_monitors.tools[event]; PyObject *offset_obj = PyLong_FromSsize_t(offset); int err; if (offset_obj == NULL) { @@ -1237,12 +1238,12 @@ _Py_call_instrumentation_exc2( int _Py_Instrumentation_GetLine(PyCodeObject *code, int index) { - _PyCoInstrumentation *instrumentation = &code->_co_instrumentation; - assert(instrumentation->monitoring_data != NULL); - assert(instrumentation->monitoring_data->lines != NULL); + _PyCoMonitoringData *monitoring = code->_co_monitoring; + assert(monitoring != NULL); + assert(monitoring->lines != NULL); assert(index >= code->_co_firsttraceable); assert(index < Py_SIZE(code)); - _PyCoLineInstrumentationData *line_data = &instrumentation->monitoring_data->lines[index]; + _PyCoLineInstrumentationData *line_data = &monitoring->lines[index]; int8_t line_delta = line_data->line_delta; int line = compute_line(code, index, line_delta); return line; @@ -1256,8 +1257,8 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, assert(is_version_up_to_date(code, tstate->interp)); assert(instrumentation_cross_checks(tstate->interp, code)); int i = instr - _PyCode_CODE(code); - _PyCoInstrumentation *instrumentation = &code->_co_instrumentation; - _PyCoLineInstrumentationData *line_data = &instrumentation->monitoring_data->lines[i]; + _PyCoMonitoringData *monitoring = code->_co_monitoring; + _PyCoLineInstrumentationData *line_data = &monitoring->lines[i]; uint8_t original_opcode = line_data->original_opcode; if (tstate->tracing) { goto done; @@ -1265,10 +1266,10 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, PyInterpreterState *interp = tstate->interp; int8_t line_delta = line_data->line_delta; int line = compute_line(code, i, line_delta); - uint8_t tools = code->_co_instrumentation.monitoring_data->line_tools != NULL ? - code->_co_instrumentation.monitoring_data->line_tools[i] : - (interp->instrumented_events.tools[PY_MONITORING_EVENT_LINE] | - code->_co_instrumentation.monitoring_data->local_instrumentation.tools[PY_MONITORING_EVENT_LINE] + uint8_t tools = code->_co_monitoring->line_tools != NULL ? + code->_co_monitoring->line_tools[i] : + (interp->monitors.tools[PY_MONITORING_EVENT_LINE] | + code->_co_monitoring->local_monitors.tools[PY_MONITORING_EVENT_LINE] ); PyObject *line_obj = PyLong_FromSsize_t(line); if (line_obj == NULL) { @@ -1311,7 +1312,7 @@ _Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame* assert(is_version_up_to_date(code, tstate->interp)); assert(instrumentation_cross_checks(tstate->interp, code)); int offset = instr - _PyCode_CODE(code); - _PyCoInstrumentationData *instrumentation_data = code->_co_instrumentation.monitoring_data; + _PyCoMonitoringData *instrumentation_data = code->_co_monitoring; assert(instrumentation_data->per_instruction_opcodes); int next_opcode = instrumentation_data->per_instruction_opcodes[offset]; if (tstate->tracing) { @@ -1320,8 +1321,8 @@ _Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame* PyInterpreterState *interp = tstate->interp; uint8_t tools = instrumentation_data->per_instruction_tools != NULL ? instrumentation_data->per_instruction_tools[offset] : - (interp->instrumented_events.tools[PY_MONITORING_EVENT_INSTRUCTION] | - code->_co_instrumentation.monitoring_data->local_instrumentation.tools[PY_MONITORING_EVENT_INSTRUCTION] + (interp->monitors.tools[PY_MONITORING_EVENT_INSTRUCTION] | + code->_co_monitoring->local_monitors.tools[PY_MONITORING_EVENT_INSTRUCTION] ); PyObject *offset_obj = PyLong_FromSsize_t(offset); if (offset_obj == NULL) { @@ -1361,22 +1362,22 @@ _PyMonitoring_RegisterCallback(int tool_id, int event_id, PyObject *obj) PyInterpreterState *is = _PyInterpreterState_Get(); assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); assert(0 <= event_id && event_id < PY_MONITORING_EVENTS); - PyObject *callback = is->tools[tool_id].instrument_callables[event_id]; - is->tools[tool_id].instrument_callables[event_id] = Py_XNewRef(obj); + PyObject *callback = is->monitoring_callables[tool_id][event_id]; + is->monitoring_callables[tool_id][event_id] = Py_XNewRef(obj); return callback; } static void initialize_tools(PyCodeObject *code) { - uint8_t* tools = code->_co_instrumentation.monitoring_data->tools; + uint8_t* tools = code->_co_monitoring->tools; assert(tools != NULL); int code_len = (int)Py_SIZE(code); for (int i = 0; i < code_len; i++) { _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; int opcode = instr->op.code; if (opcode == INSTRUMENTED_LINE) { - opcode = code->_co_instrumentation.monitoring_data->lines[i].original_opcode; + opcode = code->_co_monitoring->lines[i].original_opcode; } bool instrumented = is_instrumented(opcode); if (instrumented) { @@ -1396,7 +1397,7 @@ initialize_tools(PyCodeObject *code) } assert(event >= 0); assert(event < PY_MONITORING_INSTRUMENTED_EVENTS); - tools[i] = code->_co_instrumentation.monitoring_data->current_instrumentation.tools[event]; + tools[i] = code->_co_monitoring->active_monitors.tools[event]; CHECK(tools[i] != 0); } else { @@ -1423,7 +1424,7 @@ initialize_tools(PyCodeObject *code) static void initialize_lines(PyCodeObject *code) { - _PyCoLineInstrumentationData *line_data = code->_co_instrumentation.monitoring_data->lines; + _PyCoLineInstrumentationData *line_data = code->_co_monitoring->lines; assert(line_data != NULL); int code_len = (int)Py_SIZE(code); PyCodeAddressRange range; @@ -1477,7 +1478,7 @@ initialize_lines(PyCodeObject *code) static void initialize_line_tools(PyCodeObject *code, _Py_Monitors *all_events) { - uint8_t *line_tools = code->_co_instrumentation.monitoring_data->line_tools; + uint8_t *line_tools = code->_co_monitoring->line_tools; assert(line_tools != NULL); int code_len = (int)Py_SIZE(code); for (int i = 0; i < code_len; i++) { @@ -1489,19 +1490,19 @@ static int allocate_instrumentation_data(PyCodeObject *code) { - if (code->_co_instrumentation.monitoring_data == NULL) { - code->_co_instrumentation.monitoring_data = PyMem_Malloc(sizeof(_PyCoInstrumentationData)); - if (code->_co_instrumentation.monitoring_data == NULL) { + if (code->_co_monitoring == NULL) { + code->_co_monitoring = PyMem_Malloc(sizeof(_PyCoMonitoringData)); + if (code->_co_monitoring == NULL) { PyErr_NoMemory(); return -1; } - code->_co_instrumentation.monitoring_data->local_instrumentation = (_Py_Monitors){ 0 }; - code->_co_instrumentation.monitoring_data->current_instrumentation = (_Py_Monitors){ 0 }; - code->_co_instrumentation.monitoring_data->tools = NULL; - code->_co_instrumentation.monitoring_data->lines = NULL; - code->_co_instrumentation.monitoring_data->line_tools = NULL; - code->_co_instrumentation.monitoring_data->per_instruction_opcodes = NULL; - code->_co_instrumentation.monitoring_data->per_instruction_tools = NULL; + code->_co_monitoring->local_monitors = (_Py_Monitors){ 0 }; + code->_co_monitoring->active_monitors = (_Py_Monitors){ 0 }; + code->_co_monitoring->tools = NULL; + code->_co_monitoring->lines = NULL; + code->_co_monitoring->line_tools = NULL; + code->_co_monitoring->per_instruction_opcodes = NULL; + code->_co_monitoring->per_instruction_tools = NULL; } return 0; } @@ -1513,30 +1514,30 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) if (allocate_instrumentation_data(code)) { return -1; } - _Py_Monitors all_events = matrix_or( - interp->instrumented_events, - code->_co_instrumentation.monitoring_data->local_instrumentation); + _Py_Monitors all_events = monitors_or( + interp->monitors, + code->_co_monitoring->local_monitors); bool multitools = multiple_tools(&all_events); - if (code->_co_instrumentation.monitoring_data->tools == NULL && multitools) { - code->_co_instrumentation.monitoring_data->tools = PyMem_Malloc(code_len); - if (code->_co_instrumentation.monitoring_data->tools == NULL) { + if (code->_co_monitoring->tools == NULL && multitools) { + code->_co_monitoring->tools = PyMem_Malloc(code_len); + if (code->_co_monitoring->tools == NULL) { PyErr_NoMemory(); return -1; } initialize_tools(code); } if (all_events.tools[PY_MONITORING_EVENT_LINE]) { - if (code->_co_instrumentation.monitoring_data->lines == NULL) { - code->_co_instrumentation.monitoring_data->lines = PyMem_Malloc(code_len * sizeof(_PyCoLineInstrumentationData)); - if (code->_co_instrumentation.monitoring_data->lines == NULL) { + if (code->_co_monitoring->lines == NULL) { + code->_co_monitoring->lines = PyMem_Malloc(code_len * sizeof(_PyCoLineInstrumentationData)); + if (code->_co_monitoring->lines == NULL) { PyErr_NoMemory(); return -1; } initialize_lines(code); } - if (multitools && code->_co_instrumentation.monitoring_data->line_tools == NULL) { - code->_co_instrumentation.monitoring_data->line_tools = PyMem_Malloc(code_len); - if (code->_co_instrumentation.monitoring_data->line_tools == NULL) { + if (multitools && code->_co_monitoring->line_tools == NULL) { + code->_co_monitoring->line_tools = PyMem_Malloc(code_len); + if (code->_co_monitoring->line_tools == NULL) { PyErr_NoMemory(); return -1; } @@ -1544,26 +1545,26 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) } } if (all_events.tools[PY_MONITORING_EVENT_INSTRUCTION]) { - if (code->_co_instrumentation.monitoring_data->per_instruction_opcodes == NULL) { - code->_co_instrumentation.monitoring_data->per_instruction_opcodes = PyMem_Malloc(code_len * sizeof(_PyCoLineInstrumentationData)); - if (code->_co_instrumentation.monitoring_data->per_instruction_opcodes == NULL) { + if (code->_co_monitoring->per_instruction_opcodes == NULL) { + code->_co_monitoring->per_instruction_opcodes = PyMem_Malloc(code_len * sizeof(_PyCoLineInstrumentationData)); + if (code->_co_monitoring->per_instruction_opcodes == NULL) { PyErr_NoMemory(); return -1; } /* This may not be necessary, as we can initialize this memory lazily, but it helps catch errors. */ for (int i = 0; i < code_len; i++) { - code->_co_instrumentation.monitoring_data->per_instruction_opcodes[i] = 0; + code->_co_monitoring->per_instruction_opcodes[i] = 0; } } - if (multitools && code->_co_instrumentation.monitoring_data->per_instruction_tools == NULL) { - code->_co_instrumentation.monitoring_data->per_instruction_tools = PyMem_Malloc(code_len); - if (code->_co_instrumentation.monitoring_data->per_instruction_tools == NULL) { + if (multitools && code->_co_monitoring->per_instruction_tools == NULL) { + code->_co_monitoring->per_instruction_tools = PyMem_Malloc(code_len); + if (code->_co_monitoring->per_instruction_tools == NULL) { PyErr_NoMemory(); return -1; } /* This may not be necessary, as we can initialize this memory lazily, but it helps catch errors. */ for (int i = 0; i < code_len; i++) { - code->_co_instrumentation.monitoring_data->per_instruction_tools[i] = 0; + code->_co_monitoring->per_instruction_tools[i] = 0; } } } @@ -1599,25 +1600,25 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) if (update_instrumentation_data(code, interp)) { return -1; } - _Py_Monitors active_events = matrix_or( - interp->instrumented_events, - code->_co_instrumentation.monitoring_data->local_instrumentation); + _Py_Monitors active_events = monitors_or( + interp->monitors, + code->_co_monitoring->local_monitors); _Py_Monitors new_events; _Py_Monitors removed_events; - bool restarted = interp->last_restart_version > code->_co_instrumentation.monitoring_version; + bool restarted = interp->last_restart_version > code->_co_instrumentation_version; if (restarted) { - removed_events = code->_co_instrumentation.monitoring_data->current_instrumentation; + removed_events = code->_co_monitoring->active_monitors; new_events = active_events; } else { - removed_events = matrix_sub(code->_co_instrumentation.monitoring_data->current_instrumentation, active_events); - new_events = matrix_sub(active_events, code->_co_instrumentation.monitoring_data->current_instrumentation); - assert(matrix_empty(matrix_and(new_events, removed_events))); + removed_events = monitors_sub(code->_co_monitoring->active_monitors, active_events); + new_events = monitors_sub(active_events, code->_co_monitoring->active_monitors); + assert(monitors_empty(monitors_and(new_events, removed_events))); } - code->_co_instrumentation.monitoring_data->current_instrumentation = active_events; - code->_co_instrumentation.monitoring_version = interp->monitoring_version; - if (matrix_empty(new_events) && matrix_empty(removed_events)) { + code->_co_monitoring->active_monitors = active_events; + code->_co_instrumentation_version = interp->monitoring_version; + if (monitors_empty(new_events) && monitors_empty(removed_events)) { #ifdef INSTRUMENT_DEBUG sanity_check_instrumentation(code); #endif @@ -1666,7 +1667,7 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) uint8_t new_line_tools = new_events.tools[PY_MONITORING_EVENT_LINE]; uint8_t removed_line_tools = removed_events.tools[PY_MONITORING_EVENT_LINE]; if (new_line_tools | removed_line_tools) { - _PyCoLineInstrumentationData *line_data = code->_co_instrumentation.monitoring_data->lines; + _PyCoLineInstrumentationData *line_data = code->_co_monitoring->lines; for (int i = code->_co_firsttraceable; i < code_len;) { Instruction inst = read_instruction(code, i); if (line_data[i].original_opcode) { @@ -1751,11 +1752,11 @@ _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); PyInterpreterState *interp = _PyInterpreterState_Get(); assert(events < (1 << PY_MONITORING_UNGROUPED_EVENTS)); - uint32_t existing_events = get_events(&interp->instrumented_events, tool_id); + uint32_t existing_events = get_events(&interp->monitors, tool_id); if (existing_events == events) { return; } - set_events(&interp->instrumented_events, tool_id, events); + set_events(&interp->monitors, tool_id, events); interp->monitoring_version++; instrument_all_executing_code_objects(interp); } @@ -1769,7 +1770,7 @@ _PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEvent if (allocate_instrumentation_data(code)) { return -1; } - _Py_Monitors *local = &code->_co_instrumentation.monitoring_data->local_instrumentation; + _Py_Monitors *local = &code->_co_monitoring->local_monitors; uint32_t existing_events = get_events(local, tool_id); if (existing_events == events) { return 0; @@ -1777,7 +1778,7 @@ _PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEvent set_events(local, tool_id, events); if (is_version_up_to_date(code, interp)) { /* Force instrumentation update */ - code->_co_instrumentation.monitoring_version = UINT64_MAX; + code->_co_instrumentation_version = UINT64_MAX; } _Py_Instrument(code, interp); return 0; @@ -1930,7 +1931,7 @@ monitoring_get_events_impl(PyObject *module, int tool_id) if (check_valid_tool(tool_id)) { return NULL; } - _Py_Monitors *m = &_PyInterpreterState_Get()->instrumented_events; + _Py_Monitors *m = &_PyInterpreterState_Get()->monitors; _PyMonitoringEventSet event_set = get_events(m, tool_id); return PyLong_FromUnsignedLong(event_set); } @@ -1990,8 +1991,7 @@ monitoring_get_local_events_impl(PyObject *module, PyObject *code, } _PyMonitoringEventSet event_set = 0; for (int e = 0; e < PY_MONITORING_UNGROUPED_EVENTS; e++) { - _Py_Monitors *m = &((PyCodeObject *)code)-> - _co_instrumentation.monitoring_data->local_instrumentation; + _Py_Monitors *m = &((PyCodeObject *)code)->_co_monitoring->local_monitors; if ((m->tools[e] >> tool_id) & 1) { event_set |= (1 << e); } @@ -2104,7 +2104,7 @@ monitoring__all_events_impl(PyObject *module) return NULL; } for (int e = 0; e < PY_MONITORING_UNGROUPED_EVENTS; e++) { - uint8_t tools = interp->instrumented_events.tools[e]; + uint8_t tools = interp->monitors.tools[e]; if (tools == 0) { continue; } diff --git a/Python/pystate.c b/Python/pystate.c index 67c479272631d9..611922df14e299 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -644,11 +644,11 @@ init_interpreter(PyInterpreterState *interp, PyConfig_InitPythonConfig(&interp->config); _PyType_InitCache(interp); for(int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { - interp->instrumented_events.tools[i] = 0; + interp->monitors.tools[i] = 0; } - for(int i = 0; i < PY_MONITORING_EVENTS; i++) { - for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { - interp->tools[t].instrument_callables[i] = NULL; + for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { + for(int e = 0; e < PY_MONITORING_EVENTS; e++) { + interp->monitoring_callables[t][e] = NULL; } } interp->f_opcode_trace_set = false; @@ -778,11 +778,11 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) Py_CLEAR(interp->audit_hooks); for(int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { - interp->instrumented_events.tools[i] = 0; + interp->monitors.tools[i] = 0; } - for(int i = 0; i < PY_MONITORING_EVENTS; i++) { - for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { - Py_CLEAR(interp->tools[t].instrument_callables[i]); + for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { + for(int e = 0; e < PY_MONITORING_EVENTS; e++) { + Py_CLEAR(interp->monitoring_callables[t][e]); } } From 0982e5ea6143ff64d0af90a4aff26f7f4cca409d Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 23 Feb 2023 17:04:13 +0000 Subject: [PATCH 052/116] Remove out of dat comments. --- Python/instrumentation.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index c0f4d4920cda82..c6a1ee67cd21f2 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -680,7 +680,6 @@ de_instrument(PyCodeObject *code, int offset, int event) static void de_instrument_line(PyCodeObject *code, int i) { - /* TO DO -- handle INSTRUMENTED_INSTRUCTION */ _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; int opcode = _Py_OPCODE(*instr); if (opcode == INSTRUMENTED_INSTRUCTION) { @@ -718,7 +717,6 @@ de_instrument_line(PyCodeObject *code, int i) static void de_instrument_per_instruction(PyCodeObject *code, int offset) { - /* TO DO -- handle INSTRUMENTED_INSTRUCTION */ _Py_CODEUNIT *instr = &_PyCode_CODE(code)[offset]; int opcode = _Py_OPCODE(*instr); if (opcode != INSTRUMENTED_INSTRUCTION) { From 0693423091aa055ace9c8efa1dc5640ca1b94100 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 22 Mar 2023 19:35:28 +0000 Subject: [PATCH 053/116] Remove instrumentation for JUMP_IF_[FALS|TRUE]_OR_POP --- Lib/opcode.py | 3 +-- Python/bytecodes.c | 30 ------------------------------ Python/instrumentation.c | 10 ---------- 3 files changed, 1 insertion(+), 42 deletions(-) diff --git a/Lib/opcode.py b/Lib/opcode.py index 91d2afebf08ab6..49f86890fa599a 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -240,8 +240,7 @@ def pseudo_op(name, op, real_ops): def_op('INSTRUMENTED_CALL_FUNCTION_EX', 244) def_op('INSTRUMENTED_JUMP_FORWARD', 245) def_op('INSTRUMENTED_JUMP_BACKWARD', 246) -def_op('INSTRUMENTED_JUMP_IF_FALSE_OR_POP', 247) -def_op('INSTRUMENTED_JUMP_IF_TRUE_OR_POP', 248) + def_op('INSTRUMENTED_POP_JUMP_IF_FALSE', 249) def_op('INSTRUMENTED_POP_JUMP_IF_TRUE', 250) def_op('INSTRUMENTED_END_FOR', 251) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index df4686fa033a59..1acf2fa55463b3 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3325,36 +3325,6 @@ dummy_func( CHECK_EVAL_BREAKER(); } - inst(INSTRUMENTED_JUMP_IF_TRUE_OR_POP, ( -- )) { - PyObject *cond = TOP(); - int err = PyObject_IsTrue(cond); - ERROR_IF(err < 0, error); - _Py_CODEUNIT *here = next_instr-1; - assert(err == 0 || err == 1); - _Py_CODEUNIT *target = next_instr + err*oparg; - int shrink = 1 - err; - INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); - if (shrink) { - STACK_SHRINK(1); - Py_DECREF(cond); - } - } - - inst(INSTRUMENTED_JUMP_IF_FALSE_OR_POP, ( -- )) { - PyObject *cond = TOP(); - int err = PyObject_IsTrue(cond); - ERROR_IF(err < 0, error); - _Py_CODEUNIT *here = next_instr-1; - assert(err == 0 || err == 1); - _Py_CODEUNIT *target = next_instr + (1-err)*oparg; - int shrink = err; - INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); - if (shrink) { - STACK_SHRINK(1); - Py_DECREF(cond); - } - } - inst(INSTRUMENTED_POP_JUMP_IF_TRUE, ( -- )) { PyObject *cond = POP(); int err = PyObject_IsTrue(cond); diff --git a/Python/instrumentation.c b/Python/instrumentation.c index c6a1ee67cd21f2..b49b8d9380810a 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -47,8 +47,6 @@ static const int8_t EVENT_FOR_OPCODE[256] = { [COMPARE_AND_BRANCH] = PY_MONITORING_EVENT_BRANCH, [INSTRUMENTED_JUMP_FORWARD] = PY_MONITORING_EVENT_JUMP, [INSTRUMENTED_JUMP_BACKWARD] = PY_MONITORING_EVENT_JUMP, - [INSTRUMENTED_JUMP_IF_FALSE_OR_POP] = PY_MONITORING_EVENT_BRANCH, - [INSTRUMENTED_JUMP_IF_TRUE_OR_POP] = PY_MONITORING_EVENT_BRANCH, [INSTRUMENTED_POP_JUMP_IF_FALSE] = PY_MONITORING_EVENT_BRANCH, [INSTRUMENTED_POP_JUMP_IF_TRUE] = PY_MONITORING_EVENT_BRANCH, [INSTRUMENTED_POP_JUMP_IF_NONE] = PY_MONITORING_EVENT_BRANCH, @@ -86,8 +84,6 @@ static const bool OPCODE_HAS_EVENT[256] = { [COMPARE_AND_BRANCH] = true, [INSTRUMENTED_JUMP_FORWARD] = true, [INSTRUMENTED_JUMP_BACKWARD] = true, - [INSTRUMENTED_JUMP_IF_FALSE_OR_POP] = true, - [INSTRUMENTED_JUMP_IF_TRUE_OR_POP] = true, [INSTRUMENTED_POP_JUMP_IF_FALSE] = true, [INSTRUMENTED_POP_JUMP_IF_TRUE] = true, [INSTRUMENTED_POP_JUMP_IF_NONE] = true, @@ -109,8 +105,6 @@ static const uint8_t DE_INSTRUMENT[256] = { [INSTRUMENTED_YIELD_VALUE] = YIELD_VALUE, [INSTRUMENTED_JUMP_FORWARD] = JUMP_FORWARD, [INSTRUMENTED_JUMP_BACKWARD] = JUMP_BACKWARD, - [INSTRUMENTED_JUMP_IF_FALSE_OR_POP] = JUMP_IF_FALSE_OR_POP, - [INSTRUMENTED_JUMP_IF_TRUE_OR_POP] = JUMP_IF_TRUE_OR_POP, [INSTRUMENTED_POP_JUMP_IF_FALSE] = POP_JUMP_IF_FALSE, [INSTRUMENTED_POP_JUMP_IF_TRUE] = POP_JUMP_IF_TRUE, [INSTRUMENTED_POP_JUMP_IF_NONE] = POP_JUMP_IF_NONE, @@ -138,10 +132,6 @@ static const uint8_t INSTRUMENTED_OPCODES[256] = { [INSTRUMENTED_JUMP_FORWARD] = INSTRUMENTED_JUMP_FORWARD, [JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD, [INSTRUMENTED_JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD, - [JUMP_IF_FALSE_OR_POP] = INSTRUMENTED_JUMP_IF_FALSE_OR_POP, - [INSTRUMENTED_JUMP_IF_FALSE_OR_POP] = INSTRUMENTED_JUMP_IF_FALSE_OR_POP, - [JUMP_IF_TRUE_OR_POP] = INSTRUMENTED_JUMP_IF_TRUE_OR_POP, - [INSTRUMENTED_JUMP_IF_TRUE_OR_POP] = INSTRUMENTED_JUMP_IF_TRUE_OR_POP, [POP_JUMP_IF_FALSE] = INSTRUMENTED_POP_JUMP_IF_FALSE, [INSTRUMENTED_POP_JUMP_IF_FALSE] = INSTRUMENTED_POP_JUMP_IF_FALSE, [POP_JUMP_IF_TRUE] = INSTRUMENTED_POP_JUMP_IF_TRUE, From 0ee2aee8fd375b03feea7b5a1035d68b8d97e89b Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 23 Mar 2023 00:41:37 +0000 Subject: [PATCH 054/116] Minor fixups from merge. --- Python/bytecodes.c | 11 ++--- Python/generated_cases.c.h | 93 +++++++++++++++++++------------------- 2 files changed, 51 insertions(+), 53 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 51b041c85af6f0..6f7729402eda1f 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3002,12 +3002,9 @@ dummy_func( } inst(CALL_FUNCTION_EX, (unused, func, callargs, kwargs if (oparg & 1) -- result)) { - if (oparg & 1) { - - // DICT_MERGE is called before this opcode if there are kwargs. - // It converts all dict subtypes in kwargs into regular dicts. - assert(PyDict_CheckExact(kwargs)); - } + // DICT_MERGE is called before this opcode if there are kwargs. + // It converts all dict subtypes in kwargs into regular dicts. + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); if (!PyTuple_CheckExact(callargs)) { if (check_args_iterable(tstate, func, callargs) < 0) { goto error; @@ -3047,7 +3044,7 @@ dummy_func( else { result = PyObject_Call(func, callargs, kwargs); } - + DECREF_INPUTS(); assert(PEEK(3 + (oparg & 1)) == NULL); ERROR_IF(result == NULL, error); CHECK_EVAL_BREAKER(); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index ffe63fe0ddedf0..7089adfb563926 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -4221,12 +4221,9 @@ PyObject *func = stack_pointer[-(2 + ((oparg & 1) ? 1 : 0))]; PyObject *result; #line 3005 "Python/bytecodes.c" - if (oparg & 1) { - - // DICT_MERGE is called before this opcode if there are kwargs. - // It converts all dict subtypes in kwargs into regular dicts. - assert(PyDict_CheckExact(kwargs)); - } + // DICT_MERGE is called before this opcode if there are kwargs. + // It converts all dict subtypes in kwargs into regular dicts. + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); if (!PyTuple_CheckExact(callargs)) { if (check_args_iterable(tstate, func, callargs) < 0) { goto error; @@ -4266,10 +4263,14 @@ else { result = PyObject_Call(func, callargs, kwargs); } - + #line 4267 "Python/generated_cases.c.h" + Py_DECREF(func); + Py_DECREF(callargs); + Py_XDECREF(kwargs); + #line 3048 "Python/bytecodes.c" assert(PEEK(3 + (oparg & 1)) == NULL); if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } - #line 4273 "Python/generated_cases.c.h" + #line 4274 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 1) ? 1 : 0)); STACK_SHRINK(2); stack_pointer[-1] = result; @@ -4284,7 +4285,7 @@ PyObject *kwdefaults = (oparg & 0x02) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0))] : NULL; PyObject *defaults = (oparg & 0x01) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x01) ? 1 : 0))] : NULL; PyObject *func; - #line 3061 "Python/bytecodes.c" + #line 3058 "Python/bytecodes.c" PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); @@ -4313,14 +4314,14 @@ func_obj->func_version = ((PyCodeObject *)codeobj)->co_version; func = (PyObject *)func_obj; - #line 4317 "Python/generated_cases.c.h" + #line 4318 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 0x01) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x08) ? 1 : 0)); stack_pointer[-1] = func; DISPATCH(); } TARGET(RETURN_GENERATOR) { - #line 3092 "Python/bytecodes.c" + #line 3089 "Python/bytecodes.c" assert(PyFunction_Check(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); @@ -4341,7 +4342,7 @@ frame = cframe.current_frame = prev; _PyFrame_StackPush(frame, (PyObject *)gen); goto resume_frame; - #line 4345 "Python/generated_cases.c.h" + #line 4346 "Python/generated_cases.c.h" } TARGET(BUILD_SLICE) { @@ -4349,15 +4350,15 @@ PyObject *stop = stack_pointer[-(1 + ((oparg == 3) ? 1 : 0))]; PyObject *start = stack_pointer[-(2 + ((oparg == 3) ? 1 : 0))]; PyObject *slice; - #line 3115 "Python/bytecodes.c" + #line 3112 "Python/bytecodes.c" slice = PySlice_New(start, stop, step); - #line 4355 "Python/generated_cases.c.h" + #line 4356 "Python/generated_cases.c.h" Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); - #line 3117 "Python/bytecodes.c" + #line 3114 "Python/bytecodes.c" if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } - #line 4361 "Python/generated_cases.c.h" + #line 4362 "Python/generated_cases.c.h" STACK_SHRINK(((oparg == 3) ? 1 : 0)); STACK_SHRINK(1); stack_pointer[-1] = slice; @@ -4368,7 +4369,7 @@ PyObject *fmt_spec = ((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? stack_pointer[-((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))] : NULL; PyObject *value = stack_pointer[-(1 + (((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))]; PyObject *result; - #line 3121 "Python/bytecodes.c" + #line 3118 "Python/bytecodes.c" /* Handles f-string value formatting. */ PyObject *(*conv_fn)(PyObject *); int which_conversion = oparg & FVC_MASK; @@ -4403,7 +4404,7 @@ Py_DECREF(value); Py_XDECREF(fmt_spec); if (result == NULL) { STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); goto pop_1_error; } - #line 4407 "Python/generated_cases.c.h" + #line 4408 "Python/generated_cases.c.h" STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); stack_pointer[-1] = result; DISPATCH(); @@ -4412,10 +4413,10 @@ TARGET(COPY) { PyObject *bottom = stack_pointer[-(1 + (oparg-1))]; PyObject *top; - #line 3158 "Python/bytecodes.c" + #line 3155 "Python/bytecodes.c" assert(oparg > 0); top = Py_NewRef(bottom); - #line 4419 "Python/generated_cases.c.h" + #line 4420 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = top; DISPATCH(); @@ -4427,7 +4428,7 @@ PyObject *rhs = stack_pointer[-1]; PyObject *lhs = stack_pointer[-2]; PyObject *res; - #line 3163 "Python/bytecodes.c" + #line 3160 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -4442,12 +4443,12 @@ assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); assert(binary_ops[oparg]); res = binary_ops[oparg](lhs, rhs); - #line 4446 "Python/generated_cases.c.h" + #line 4447 "Python/generated_cases.c.h" Py_DECREF(lhs); Py_DECREF(rhs); - #line 3178 "Python/bytecodes.c" + #line 3175 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 4451 "Python/generated_cases.c.h" + #line 4452 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -4457,16 +4458,16 @@ TARGET(SWAP) { PyObject *top = stack_pointer[-1]; PyObject *bottom = stack_pointer[-(2 + (oparg-2))]; - #line 3183 "Python/bytecodes.c" + #line 3180 "Python/bytecodes.c" assert(oparg >= 2); - #line 4463 "Python/generated_cases.c.h" + #line 4464 "Python/generated_cases.c.h" stack_pointer[-1] = bottom; stack_pointer[-(2 + (oparg-2))] = top; DISPATCH(); } TARGET(INSTRUMENTED_LINE) { - #line 3187 "Python/bytecodes.c" + #line 3184 "Python/bytecodes.c" _Py_CODEUNIT *here = next_instr-1; _PyFrame_SetStackPointer(frame, stack_pointer); int original_opcode = _Py_call_instrumentation_line( @@ -4486,11 +4487,11 @@ } opcode = original_opcode; DISPATCH_GOTO(); - #line 4490 "Python/generated_cases.c.h" + #line 4491 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_INSTRUCTION) { - #line 3209 "Python/bytecodes.c" + #line 3206 "Python/bytecodes.c" int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, next_instr-1); if (next_opcode < 0) goto error; @@ -4502,26 +4503,26 @@ assert(next_opcode > 0 && next_opcode < 256); opcode = next_opcode; DISPATCH_GOTO(); - #line 4506 "Python/generated_cases.c.h" + #line 4507 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_JUMP_FORWARD) { - #line 3224 "Python/bytecodes.c" + #line 3221 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); - #line 4512 "Python/generated_cases.c.h" + #line 4513 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_JUMP_BACKWARD) { - #line 3229 "Python/bytecodes.c" + #line 3226 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr-oparg, PY_MONITORING_EVENT_JUMP); - #line 4519 "Python/generated_cases.c.h" + #line 4520 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { - #line 3234 "Python/bytecodes.c" + #line 3231 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); if (err < 0) goto error; @@ -4529,12 +4530,12 @@ assert(err == 0 || err == 1); int offset = err*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4533 "Python/generated_cases.c.h" + #line 4534 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { - #line 3244 "Python/bytecodes.c" + #line 3241 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); if (err < 0) goto error; @@ -4542,12 +4543,12 @@ assert(err == 0 || err == 1); int offset = (1-err)*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4546 "Python/generated_cases.c.h" + #line 4547 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { - #line 3254 "Python/bytecodes.c" + #line 3251 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4560,12 +4561,12 @@ offset = 0; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4564 "Python/generated_cases.c.h" + #line 4565 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { - #line 3269 "Python/bytecodes.c" + #line 3266 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4578,22 +4579,22 @@ offset = oparg; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4582 "Python/generated_cases.c.h" + #line 4583 "Python/generated_cases.c.h" DISPATCH(); } TARGET(EXTENDED_ARG) { - #line 3284 "Python/bytecodes.c" + #line 3281 "Python/bytecodes.c" assert(oparg); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); - #line 4593 "Python/generated_cases.c.h" + #line 4594 "Python/generated_cases.c.h" } TARGET(CACHE) { - #line 3292 "Python/bytecodes.c" + #line 3289 "Python/bytecodes.c" Py_UNREACHABLE(); - #line 4599 "Python/generated_cases.c.h" + #line 4600 "Python/generated_cases.c.h" } From acdca93f1ce4cb738a2f5b2f9916fcf3df996afb Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 23 Mar 2023 01:47:02 +0000 Subject: [PATCH 055/116] Remove instrumentation for COMPARE_AND_BRANCH. --- Include/internal/pycore_opcode.h | 4 +- Include/opcode.h | 1 - Lib/importlib/_bootstrap_external.py | 2 +- Lib/opcode.py | 2 +- Python/bytecodes.c | 22 -- Python/generated_cases.c.h | 401 +++++++++++++-------------- Python/instrumentation.c | 18 -- Python/opcode_metadata.h | 5 - Python/opcode_targets.h | 2 +- 9 files changed, 193 insertions(+), 264 deletions(-) diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index 1e8a00a2b54c3c..acd8c8d3b1143c 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -141,7 +141,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [IMPORT_NAME] = IMPORT_NAME, [INSTRUMENTED_CALL] = INSTRUMENTED_CALL, [INSTRUMENTED_CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, - [INSTRUMENTED_COMPARE_AND_BRANCH] = INSTRUMENTED_COMPARE_AND_BRANCH, [INSTRUMENTED_END_FOR] = INSTRUMENTED_END_FOR, [INSTRUMENTED_END_SEND] = INSTRUMENTED_END_SEND, [INSTRUMENTED_FOR_ITER] = INSTRUMENTED_FOR_ITER, @@ -487,7 +486,7 @@ static const char *const _PyOpcode_OpName[263] = { [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = "INSTRUMENTED_POP_JUMP_IF_NOT_NONE", [INSTRUMENTED_RETURN_CONST] = "INSTRUMENTED_RETURN_CONST", [INSTRUMENTED_FOR_ITER] = "INSTRUMENTED_FOR_ITER", - [INSTRUMENTED_COMPARE_AND_BRANCH] = "INSTRUMENTED_COMPARE_AND_BRANCH", + [239] = "<239>", [INSTRUMENTED_RESUME] = "INSTRUMENTED_RESUME", [INSTRUMENTED_CALL] = "INSTRUMENTED_CALL", [INSTRUMENTED_RETURN_VALUE] = "INSTRUMENTED_RETURN_VALUE", @@ -581,6 +580,7 @@ static const char *const _PyOpcode_OpName[263] = { case 232: \ case 233: \ case 234: \ + case 239: \ case 247: \ case 248: \ ; diff --git a/Include/opcode.h b/Include/opcode.h index 5a54888fe0e037..962b61cdf3bb5d 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -120,7 +120,6 @@ extern "C" { #define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 236 #define INSTRUMENTED_RETURN_CONST 237 #define INSTRUMENTED_FOR_ITER 238 -#define INSTRUMENTED_COMPARE_AND_BRANCH 239 #define INSTRUMENTED_RESUME 240 #define INSTRUMENTED_CALL 241 #define INSTRUMENTED_RETURN_VALUE 242 diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 91c22a98b809bf..bbd5eb36148ba0 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -454,7 +454,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3526).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3525).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c diff --git a/Lib/opcode.py b/Lib/opcode.py index adf65cd629dd0a..ebfc0ee9276d81 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -230,7 +230,7 @@ def pseudo_op(name, op, real_ops): def_op('INSTRUMENTED_POP_JUMP_IF_NOT_NONE', 236) def_op('INSTRUMENTED_RETURN_CONST', 237) def_op('INSTRUMENTED_FOR_ITER', 238) -def_op('INSTRUMENTED_COMPARE_AND_BRANCH', 239) + def_op('INSTRUMENTED_RESUME', 240) def_op('INSTRUMENTED_CALL', 241) def_op('INSTRUMENTED_RETURN_VALUE', 242) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 6f7729402eda1f..34951e8d8f512e 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1819,28 +1819,6 @@ dummy_func( } } - inst(INSTRUMENTED_COMPARE_AND_BRANCH, ( -- )) { - assert((oparg >> 4) <= Py_GE); - PyObject *right = POP(); - PyObject *left = POP(); - PyObject *cond = PyObject_RichCompare(left, right, oparg>>4); - Py_DECREF(left); - Py_DECREF(right); - ERROR_IF(cond == NULL, error); - assert(_Py_OPCODE(next_instr[1]) == POP_JUMP_IF_FALSE || - _Py_OPCODE(next_instr[1]) == POP_JUMP_IF_TRUE); - bool jump_on_true = _Py_OPCODE(next_instr[1]) == POP_JUMP_IF_TRUE; - int offset = 0; - int err = PyObject_IsTrue(cond); - _Py_CODEUNIT *here = next_instr-1; - Py_DECREF(cond); - ERROR_IF(err < 0, error); - if (jump_on_true == (err != 0)) { - offset = _Py_OPARG(next_instr[1]); - } - INSTRUMENTED_JUMP(here, next_instr + 2 + offset, PY_MONITORING_EVENT_BRANCH); - } - inst(COMPARE_AND_BRANCH_FLOAT, (unused/2, left, right -- )) { DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_AND_BRANCH); DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_AND_BRANCH); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 7089adfb563926..99f11245cd23cb 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2597,35 +2597,10 @@ DISPATCH(); } - TARGET(INSTRUMENTED_COMPARE_AND_BRANCH) { - #line 1823 "Python/bytecodes.c" - assert((oparg >> 4) <= Py_GE); - PyObject *right = POP(); - PyObject *left = POP(); - PyObject *cond = PyObject_RichCompare(left, right, oparg>>4); - Py_DECREF(left); - Py_DECREF(right); - if (cond == NULL) goto error; - assert(_Py_OPCODE(next_instr[1]) == POP_JUMP_IF_FALSE || - _Py_OPCODE(next_instr[1]) == POP_JUMP_IF_TRUE); - bool jump_on_true = _Py_OPCODE(next_instr[1]) == POP_JUMP_IF_TRUE; - int offset = 0; - int err = PyObject_IsTrue(cond); - _Py_CODEUNIT *here = next_instr-1; - Py_DECREF(cond); - if (err < 0) goto error; - if (jump_on_true == (err != 0)) { - offset = _Py_OPARG(next_instr[1]); - } - INSTRUMENTED_JUMP(here, next_instr + 2 + offset, PY_MONITORING_EVENT_BRANCH); - #line 2622 "Python/generated_cases.c.h" - DISPATCH(); - } - TARGET(COMPARE_AND_BRANCH_FLOAT) { PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; - #line 1845 "Python/bytecodes.c" + #line 1823 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_AND_BRANCH); DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_AND_BRANCH); STAT_INC(COMPARE_AND_BRANCH, hit); @@ -2639,7 +2614,7 @@ int offset = next_instr[1].op.arg; JUMPBY(offset); } - #line 2643 "Python/generated_cases.c.h" + #line 2618 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 2; DISPATCH(); @@ -2648,7 +2623,7 @@ TARGET(COMPARE_AND_BRANCH_INT) { PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; - #line 1862 "Python/bytecodes.c" + #line 1840 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), COMPARE_AND_BRANCH); DEOPT_IF(!PyLong_CheckExact(right), COMPARE_AND_BRANCH); DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_AND_BRANCH); @@ -2666,7 +2641,7 @@ int offset = next_instr[1].op.arg; JUMPBY(offset); } - #line 2670 "Python/generated_cases.c.h" + #line 2645 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 2; DISPATCH(); @@ -2675,7 +2650,7 @@ TARGET(COMPARE_AND_BRANCH_STR) { PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; - #line 1883 "Python/bytecodes.c" + #line 1861 "Python/bytecodes.c" DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_AND_BRANCH); DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_AND_BRANCH); STAT_INC(COMPARE_AND_BRANCH, hit); @@ -2690,7 +2665,7 @@ int offset = next_instr[1].op.arg; JUMPBY(offset); } - #line 2694 "Python/generated_cases.c.h" + #line 2669 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 2; DISPATCH(); @@ -2700,14 +2675,14 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 1900 "Python/bytecodes.c" + #line 1878 "Python/bytecodes.c" int res = Py_Is(left, right) ^ oparg; - #line 2706 "Python/generated_cases.c.h" + #line 2681 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 1902 "Python/bytecodes.c" + #line 1880 "Python/bytecodes.c" b = Py_NewRef(res ? Py_True : Py_False); - #line 2711 "Python/generated_cases.c.h" + #line 2686 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = b; DISPATCH(); @@ -2717,15 +2692,15 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 1906 "Python/bytecodes.c" + #line 1884 "Python/bytecodes.c" int res = PySequence_Contains(right, left); - #line 2723 "Python/generated_cases.c.h" + #line 2698 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 1908 "Python/bytecodes.c" + #line 1886 "Python/bytecodes.c" if (res < 0) goto pop_2_error; b = Py_NewRef((res^oparg) ? Py_True : Py_False); - #line 2729 "Python/generated_cases.c.h" + #line 2704 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = b; DISPATCH(); @@ -2736,12 +2711,12 @@ PyObject *exc_value = stack_pointer[-2]; PyObject *rest; PyObject *match; - #line 1913 "Python/bytecodes.c" + #line 1891 "Python/bytecodes.c" if (check_except_star_type_valid(tstate, match_type) < 0) { - #line 2742 "Python/generated_cases.c.h" + #line 2717 "Python/generated_cases.c.h" Py_DECREF(exc_value); Py_DECREF(match_type); - #line 1915 "Python/bytecodes.c" + #line 1893 "Python/bytecodes.c" if (true) goto pop_2_error; } @@ -2749,10 +2724,10 @@ rest = NULL; int res = exception_group_match(exc_value, match_type, &match, &rest); - #line 2753 "Python/generated_cases.c.h" + #line 2728 "Python/generated_cases.c.h" Py_DECREF(exc_value); Py_DECREF(match_type); - #line 1923 "Python/bytecodes.c" + #line 1901 "Python/bytecodes.c" if (res < 0) goto pop_2_error; assert((match == NULL) == (rest == NULL)); @@ -2761,7 +2736,7 @@ if (!Py_IsNone(match)) { PyErr_SetExcInfo(NULL, Py_NewRef(match), NULL); } - #line 2765 "Python/generated_cases.c.h" + #line 2740 "Python/generated_cases.c.h" stack_pointer[-1] = match; stack_pointer[-2] = rest; DISPATCH(); @@ -2771,21 +2746,21 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 1934 "Python/bytecodes.c" + #line 1912 "Python/bytecodes.c" assert(PyExceptionInstance_Check(left)); if (check_except_type_valid(tstate, right) < 0) { - #line 2778 "Python/generated_cases.c.h" + #line 2753 "Python/generated_cases.c.h" Py_DECREF(right); - #line 1937 "Python/bytecodes.c" + #line 1915 "Python/bytecodes.c" if (true) goto pop_1_error; } int res = PyErr_GivenExceptionMatches(left, right); - #line 2785 "Python/generated_cases.c.h" + #line 2760 "Python/generated_cases.c.h" Py_DECREF(right); - #line 1942 "Python/bytecodes.c" + #line 1920 "Python/bytecodes.c" b = Py_NewRef(res ? Py_True : Py_False); - #line 2789 "Python/generated_cases.c.h" + #line 2764 "Python/generated_cases.c.h" stack_pointer[-1] = b; DISPATCH(); } @@ -2794,15 +2769,15 @@ PyObject *fromlist = stack_pointer[-1]; PyObject *level = stack_pointer[-2]; PyObject *res; - #line 1946 "Python/bytecodes.c" + #line 1924 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_name(tstate, frame, name, fromlist, level); - #line 2801 "Python/generated_cases.c.h" + #line 2776 "Python/generated_cases.c.h" Py_DECREF(level); Py_DECREF(fromlist); - #line 1949 "Python/bytecodes.c" + #line 1927 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 2806 "Python/generated_cases.c.h" + #line 2781 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); @@ -2811,29 +2786,29 @@ TARGET(IMPORT_FROM) { PyObject *from = stack_pointer[-1]; PyObject *res; - #line 1953 "Python/bytecodes.c" + #line 1931 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_from(tstate, from, name); if (res == NULL) goto error; - #line 2819 "Python/generated_cases.c.h" + #line 2794 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); } TARGET(JUMP_FORWARD) { - #line 1959 "Python/bytecodes.c" + #line 1937 "Python/bytecodes.c" JUMPBY(oparg); - #line 2828 "Python/generated_cases.c.h" + #line 2803 "Python/generated_cases.c.h" DISPATCH(); } TARGET(JUMP_BACKWARD) { PREDICTED(JUMP_BACKWARD); - #line 1963 "Python/bytecodes.c" + #line 1941 "Python/bytecodes.c" assert(oparg < INSTR_OFFSET()); JUMPBY(-oparg); - #line 2837 "Python/generated_cases.c.h" + #line 2812 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -2841,7 +2816,7 @@ TARGET(POP_JUMP_IF_FALSE) { PREDICTED(POP_JUMP_IF_FALSE); PyObject *cond = stack_pointer[-1]; - #line 1969 "Python/bytecodes.c" + #line 1947 "Python/bytecodes.c" if (Py_IsTrue(cond)) { _Py_DECREF_NO_DEALLOC(cond); } @@ -2851,9 +2826,9 @@ } else { int err = PyObject_IsTrue(cond); - #line 2855 "Python/generated_cases.c.h" + #line 2830 "Python/generated_cases.c.h" Py_DECREF(cond); - #line 1979 "Python/bytecodes.c" + #line 1957 "Python/bytecodes.c" if (err == 0) { JUMPBY(oparg); } @@ -2861,14 +2836,14 @@ if (err < 0) goto pop_1_error; } } - #line 2865 "Python/generated_cases.c.h" + #line 2840 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_TRUE) { PyObject *cond = stack_pointer[-1]; - #line 1989 "Python/bytecodes.c" + #line 1967 "Python/bytecodes.c" if (Py_IsFalse(cond)) { _Py_DECREF_NO_DEALLOC(cond); } @@ -2878,9 +2853,9 @@ } else { int err = PyObject_IsTrue(cond); - #line 2882 "Python/generated_cases.c.h" + #line 2857 "Python/generated_cases.c.h" Py_DECREF(cond); - #line 1999 "Python/bytecodes.c" + #line 1977 "Python/bytecodes.c" if (err > 0) { JUMPBY(oparg); } @@ -2888,67 +2863,67 @@ if (err < 0) goto pop_1_error; } } - #line 2892 "Python/generated_cases.c.h" + #line 2867 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_NOT_NONE) { PyObject *value = stack_pointer[-1]; - #line 2009 "Python/bytecodes.c" + #line 1987 "Python/bytecodes.c" if (!Py_IsNone(value)) { - #line 2901 "Python/generated_cases.c.h" + #line 2876 "Python/generated_cases.c.h" Py_DECREF(value); - #line 2011 "Python/bytecodes.c" + #line 1989 "Python/bytecodes.c" JUMPBY(oparg); } else { _Py_DECREF_NO_DEALLOC(value); } - #line 2909 "Python/generated_cases.c.h" + #line 2884 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_NONE) { PyObject *value = stack_pointer[-1]; - #line 2019 "Python/bytecodes.c" + #line 1997 "Python/bytecodes.c" if (Py_IsNone(value)) { _Py_DECREF_NO_DEALLOC(value); JUMPBY(oparg); } else { - #line 2922 "Python/generated_cases.c.h" + #line 2897 "Python/generated_cases.c.h" Py_DECREF(value); - #line 2025 "Python/bytecodes.c" + #line 2003 "Python/bytecodes.c" } - #line 2926 "Python/generated_cases.c.h" + #line 2901 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(JUMP_BACKWARD_NO_INTERRUPT) { - #line 2029 "Python/bytecodes.c" + #line 2007 "Python/bytecodes.c" /* This bytecode is used in the `yield from` or `await` loop. * If there is an interrupt, we want it handled in the innermost * generator or coroutine, so we deliberately do not check it here. * (see bpo-30039). */ JUMPBY(-oparg); - #line 2939 "Python/generated_cases.c.h" + #line 2914 "Python/generated_cases.c.h" DISPATCH(); } TARGET(GET_LEN) { PyObject *obj = stack_pointer[-1]; PyObject *len_o; - #line 2038 "Python/bytecodes.c" + #line 2016 "Python/bytecodes.c" // PUSH(len(TOS)) Py_ssize_t len_i = PyObject_Length(obj); if (len_i < 0) goto error; len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) goto error; - #line 2952 "Python/generated_cases.c.h" + #line 2927 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = len_o; DISPATCH(); @@ -2959,16 +2934,16 @@ PyObject *type = stack_pointer[-2]; PyObject *subject = stack_pointer[-3]; PyObject *attrs; - #line 2046 "Python/bytecodes.c" + #line 2024 "Python/bytecodes.c" // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. assert(PyTuple_CheckExact(names)); attrs = match_class(tstate, subject, type, oparg, names); - #line 2968 "Python/generated_cases.c.h" + #line 2943 "Python/generated_cases.c.h" Py_DECREF(subject); Py_DECREF(type); Py_DECREF(names); - #line 2051 "Python/bytecodes.c" + #line 2029 "Python/bytecodes.c" if (attrs) { assert(PyTuple_CheckExact(attrs)); // Success! } @@ -2976,7 +2951,7 @@ if (_PyErr_Occurred(tstate)) goto pop_3_error; attrs = Py_NewRef(Py_None); // Failure! } - #line 2980 "Python/generated_cases.c.h" + #line 2955 "Python/generated_cases.c.h" STACK_SHRINK(2); stack_pointer[-1] = attrs; DISPATCH(); @@ -2985,10 +2960,10 @@ TARGET(MATCH_MAPPING) { PyObject *subject = stack_pointer[-1]; PyObject *res; - #line 2061 "Python/bytecodes.c" + #line 2039 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; res = Py_NewRef(match ? Py_True : Py_False); - #line 2992 "Python/generated_cases.c.h" + #line 2967 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; PREDICT(POP_JUMP_IF_FALSE); @@ -2998,10 +2973,10 @@ TARGET(MATCH_SEQUENCE) { PyObject *subject = stack_pointer[-1]; PyObject *res; - #line 2067 "Python/bytecodes.c" + #line 2045 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; res = Py_NewRef(match ? Py_True : Py_False); - #line 3005 "Python/generated_cases.c.h" + #line 2980 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; PREDICT(POP_JUMP_IF_FALSE); @@ -3012,11 +2987,11 @@ PyObject *keys = stack_pointer[-1]; PyObject *subject = stack_pointer[-2]; PyObject *values_or_none; - #line 2073 "Python/bytecodes.c" + #line 2051 "Python/bytecodes.c" // On successful match, PUSH(values). Otherwise, PUSH(None). values_or_none = match_keys(tstate, subject, keys); if (values_or_none == NULL) goto error; - #line 3020 "Python/generated_cases.c.h" + #line 2995 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = values_or_none; DISPATCH(); @@ -3025,14 +3000,14 @@ TARGET(GET_ITER) { PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 2079 "Python/bytecodes.c" + #line 2057 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ iter = PyObject_GetIter(iterable); - #line 3032 "Python/generated_cases.c.h" + #line 3007 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 2082 "Python/bytecodes.c" + #line 2060 "Python/bytecodes.c" if (iter == NULL) goto pop_1_error; - #line 3036 "Python/generated_cases.c.h" + #line 3011 "Python/generated_cases.c.h" stack_pointer[-1] = iter; DISPATCH(); } @@ -3040,7 +3015,7 @@ TARGET(GET_YIELD_FROM_ITER) { PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 2086 "Python/bytecodes.c" + #line 2064 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ if (PyCoro_CheckExact(iterable)) { /* `iterable` is a coroutine */ @@ -3063,11 +3038,11 @@ if (iter == NULL) { goto error; } - #line 3067 "Python/generated_cases.c.h" + #line 3042 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 2109 "Python/bytecodes.c" + #line 2087 "Python/bytecodes.c" } - #line 3071 "Python/generated_cases.c.h" + #line 3046 "Python/generated_cases.c.h" stack_pointer[-1] = iter; PREDICT(LOAD_CONST); DISPATCH(); @@ -3078,7 +3053,7 @@ static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2128 "Python/bytecodes.c" + #line 2106 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyForIterCache *cache = (_PyForIterCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -3109,7 +3084,7 @@ DISPATCH(); } // Common case: no jump, leave it to the code generator - #line 3113 "Python/generated_cases.c.h" + #line 3088 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3117,7 +3092,7 @@ } TARGET(INSTRUMENTED_FOR_ITER) { - #line 2161 "Python/bytecodes.c" + #line 2139 "Python/bytecodes.c" _Py_CODEUNIT *here = next_instr-1; _Py_CODEUNIT *target; PyObject *iter = TOP(); @@ -3143,14 +3118,14 @@ target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1; } INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); - #line 3147 "Python/generated_cases.c.h" + #line 3122 "Python/generated_cases.c.h" DISPATCH(); } TARGET(FOR_ITER_LIST) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2189 "Python/bytecodes.c" + #line 2167 "Python/bytecodes.c" DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); _PyListIterObject *it = (_PyListIterObject *)iter; STAT_INC(FOR_ITER, hit); @@ -3170,7 +3145,7 @@ DISPATCH(); end_for_iter_list: // Common case: no jump, leave it to the code generator - #line 3174 "Python/generated_cases.c.h" + #line 3149 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3180,7 +3155,7 @@ TARGET(FOR_ITER_TUPLE) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2211 "Python/bytecodes.c" + #line 2189 "Python/bytecodes.c" _PyTupleIterObject *it = (_PyTupleIterObject *)iter; DEOPT_IF(Py_TYPE(it) != &PyTupleIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -3200,7 +3175,7 @@ DISPATCH(); end_for_iter_tuple: // Common case: no jump, leave it to the code generator - #line 3204 "Python/generated_cases.c.h" + #line 3179 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3210,7 +3185,7 @@ TARGET(FOR_ITER_RANGE) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2233 "Python/bytecodes.c" + #line 2211 "Python/bytecodes.c" _PyRangeIterObject *r = (_PyRangeIterObject *)iter; DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -3228,7 +3203,7 @@ if (next == NULL) { goto error; } - #line 3232 "Python/generated_cases.c.h" + #line 3207 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3237,7 +3212,7 @@ TARGET(FOR_ITER_GEN) { PyObject *iter = stack_pointer[-1]; - #line 2253 "Python/bytecodes.c" + #line 2231 "Python/bytecodes.c" PyGenObject *gen = (PyGenObject *)iter; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); @@ -3252,14 +3227,14 @@ assert(next_instr->op.code == END_FOR || next_instr->op.code == INSTRUMENTED_END_FOR); DISPATCH_INLINED(gen_frame); - #line 3256 "Python/generated_cases.c.h" + #line 3231 "Python/generated_cases.c.h" } TARGET(BEFORE_ASYNC_WITH) { PyObject *mgr = stack_pointer[-1]; PyObject *exit; PyObject *res; - #line 2270 "Python/bytecodes.c" + #line 2248 "Python/bytecodes.c" PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); if (enter == NULL) { if (!_PyErr_Occurred(tstate)) { @@ -3282,16 +3257,16 @@ Py_DECREF(enter); goto error; } - #line 3286 "Python/generated_cases.c.h" + #line 3261 "Python/generated_cases.c.h" Py_DECREF(mgr); - #line 2293 "Python/bytecodes.c" + #line 2271 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); if (true) goto pop_1_error; } - #line 3295 "Python/generated_cases.c.h" + #line 3270 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -3303,7 +3278,7 @@ PyObject *mgr = stack_pointer[-1]; PyObject *exit; PyObject *res; - #line 2303 "Python/bytecodes.c" + #line 2281 "Python/bytecodes.c" /* pop the context manager, push its __exit__ and the * value returned from calling its __enter__ */ @@ -3329,16 +3304,16 @@ Py_DECREF(enter); goto error; } - #line 3333 "Python/generated_cases.c.h" + #line 3308 "Python/generated_cases.c.h" Py_DECREF(mgr); - #line 2329 "Python/bytecodes.c" + #line 2307 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); if (true) goto pop_1_error; } - #line 3342 "Python/generated_cases.c.h" + #line 3317 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -3350,7 +3325,7 @@ PyObject *lasti = stack_pointer[-3]; PyObject *exit_func = stack_pointer[-4]; PyObject *res; - #line 2338 "Python/bytecodes.c" + #line 2316 "Python/bytecodes.c" /* At the top of the stack are 4 values: - val: TOP = exc_info() - unused: SECOND = previous exception @@ -3371,7 +3346,7 @@ res = PyObject_Vectorcall(exit_func, stack + 1, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); if (res == NULL) goto error; - #line 3375 "Python/generated_cases.c.h" + #line 3350 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); @@ -3380,7 +3355,7 @@ TARGET(PUSH_EXC_INFO) { PyObject *new_exc = stack_pointer[-1]; PyObject *prev_exc; - #line 2361 "Python/bytecodes.c" + #line 2339 "Python/bytecodes.c" _PyErr_StackItem *exc_info = tstate->exc_info; if (exc_info->exc_value != NULL) { prev_exc = exc_info->exc_value; @@ -3390,7 +3365,7 @@ } assert(PyExceptionInstance_Check(new_exc)); exc_info->exc_value = Py_NewRef(new_exc); - #line 3394 "Python/generated_cases.c.h" + #line 3369 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = new_exc; stack_pointer[-2] = prev_exc; @@ -3404,7 +3379,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t keys_version = read_u32(&next_instr[3].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2373 "Python/bytecodes.c" + #line 2351 "Python/bytecodes.c" /* Cached method object */ PyTypeObject *self_cls = Py_TYPE(self); assert(type_version != 0); @@ -3421,7 +3396,7 @@ assert(_PyType_HasFeature(Py_TYPE(res2), Py_TPFLAGS_METHOD_DESCRIPTOR)); res = self; assert(oparg & 1); - #line 3425 "Python/generated_cases.c.h" + #line 3400 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3435,7 +3410,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2392 "Python/bytecodes.c" + #line 2370 "Python/bytecodes.c" PyTypeObject *self_cls = Py_TYPE(self); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); assert(self_cls->tp_dictoffset == 0); @@ -3445,7 +3420,7 @@ res2 = Py_NewRef(descr); res = self; assert(oparg & 1); - #line 3449 "Python/generated_cases.c.h" + #line 3424 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3459,7 +3434,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2404 "Python/bytecodes.c" + #line 2382 "Python/bytecodes.c" PyTypeObject *self_cls = Py_TYPE(self); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); Py_ssize_t dictoffset = self_cls->tp_dictoffset; @@ -3473,7 +3448,7 @@ res2 = Py_NewRef(descr); res = self; assert(oparg & 1); - #line 3477 "Python/generated_cases.c.h" + #line 3452 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3482,16 +3457,16 @@ } TARGET(KW_NAMES) { - #line 2420 "Python/bytecodes.c" + #line 2398 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg < PyTuple_GET_SIZE(frame->f_code->co_consts)); kwnames = GETITEM(frame->f_code->co_consts, oparg); - #line 3490 "Python/generated_cases.c.h" + #line 3465 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_CALL) { - #line 2426 "Python/bytecodes.c" + #line 2404 "Python/bytecodes.c" int is_meth = PEEK(oparg+2) != NULL; int total_args = oparg + is_meth; PyObject *function = PEEK(total_args + 1); @@ -3504,7 +3479,7 @@ _PyCallCache *cache = (_PyCallCache *)next_instr; INCREMENT_ADAPTIVE_COUNTER(cache->counter); GO_TO_INSTRUCTION(CALL); - #line 3508 "Python/generated_cases.c.h" + #line 3483 "Python/generated_cases.c.h" } TARGET(CALL) { @@ -3514,7 +3489,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2471 "Python/bytecodes.c" + #line 2449 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3595,7 +3570,7 @@ Py_DECREF(args[i]); } if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3599 "Python/generated_cases.c.h" + #line 3574 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3607,7 +3582,7 @@ TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; - #line 2558 "Python/bytecodes.c" + #line 2536 "Python/bytecodes.c" DEOPT_IF(method != NULL, CALL); DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); STAT_INC(CALL, hit); @@ -3617,7 +3592,7 @@ PEEK(oparg + 2) = Py_NewRef(meth); // method Py_DECREF(callable); GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); - #line 3621 "Python/generated_cases.c.h" + #line 3596 "Python/generated_cases.c.h" } TARGET(CALL_PY_EXACT_ARGS) { @@ -3626,7 +3601,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); - #line 2570 "Python/bytecodes.c" + #line 2548 "Python/bytecodes.c" assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; @@ -3651,7 +3626,7 @@ STACK_SHRINK(oparg + 2); JUMPBY(INLINE_CACHE_ENTRIES_CALL); DISPATCH_INLINED(new_frame); - #line 3655 "Python/generated_cases.c.h" + #line 3630 "Python/generated_cases.c.h" } TARGET(CALL_PY_WITH_DEFAULTS) { @@ -3660,7 +3635,7 @@ PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); uint16_t min_args = read_u16(&next_instr[3].cache); - #line 2597 "Python/bytecodes.c" + #line 2575 "Python/bytecodes.c" assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; @@ -3690,7 +3665,7 @@ STACK_SHRINK(oparg + 2); JUMPBY(INLINE_CACHE_ENTRIES_CALL); DISPATCH_INLINED(new_frame); - #line 3694 "Python/generated_cases.c.h" + #line 3669 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_TYPE_1) { @@ -3698,7 +3673,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2629 "Python/bytecodes.c" + #line 2607 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3708,7 +3683,7 @@ res = Py_NewRef(Py_TYPE(obj)); Py_DECREF(obj); Py_DECREF(&PyType_Type); // I.e., callable - #line 3712 "Python/generated_cases.c.h" + #line 3687 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3721,7 +3696,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2641 "Python/bytecodes.c" + #line 2619 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3732,7 +3707,7 @@ Py_DECREF(arg); Py_DECREF(&PyUnicode_Type); // I.e., callable if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3736 "Python/generated_cases.c.h" + #line 3711 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3746,7 +3721,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2655 "Python/bytecodes.c" + #line 2633 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3757,7 +3732,7 @@ Py_DECREF(arg); Py_DECREF(&PyTuple_Type); // I.e., tuple if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3761 "Python/generated_cases.c.h" + #line 3736 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3771,7 +3746,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2669 "Python/bytecodes.c" + #line 2647 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3793,7 +3768,7 @@ } Py_DECREF(tp); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3797 "Python/generated_cases.c.h" + #line 3772 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3807,7 +3782,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2694 "Python/bytecodes.c" + #line 2672 "Python/bytecodes.c" /* Builtin METH_O functions */ assert(kwnames == NULL); int is_meth = method != NULL; @@ -3835,7 +3810,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3839 "Python/generated_cases.c.h" + #line 3814 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3849,7 +3824,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2725 "Python/bytecodes.c" + #line 2703 "Python/bytecodes.c" /* Builtin METH_FASTCALL functions, without keywords */ assert(kwnames == NULL); int is_meth = method != NULL; @@ -3881,7 +3856,7 @@ 'invalid'). In those cases an exception is set, so we must handle it. */ - #line 3885 "Python/generated_cases.c.h" + #line 3860 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3895,7 +3870,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2760 "Python/bytecodes.c" + #line 2738 "Python/bytecodes.c" /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ int is_meth = method != NULL; int total_args = oparg; @@ -3927,7 +3902,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3931 "Python/generated_cases.c.h" + #line 3906 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3941,7 +3916,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2795 "Python/bytecodes.c" + #line 2773 "Python/bytecodes.c" assert(kwnames == NULL); /* len(o) */ int is_meth = method != NULL; @@ -3966,7 +3941,7 @@ Py_DECREF(callable); Py_DECREF(arg); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3970 "Python/generated_cases.c.h" + #line 3945 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3979,7 +3954,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2822 "Python/bytecodes.c" + #line 2800 "Python/bytecodes.c" assert(kwnames == NULL); /* isinstance(o, o2) */ int is_meth = method != NULL; @@ -4006,7 +3981,7 @@ Py_DECREF(cls); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4010 "Python/generated_cases.c.h" + #line 3985 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4018,7 +3993,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *self = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; - #line 2852 "Python/bytecodes.c" + #line 2830 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); assert(method != NULL); @@ -4036,14 +4011,14 @@ JUMPBY(INLINE_CACHE_ENTRIES_CALL + 1); assert(next_instr[-1].op.code == POP_TOP); DISPATCH(); - #line 4040 "Python/generated_cases.c.h" + #line 4015 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) { PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2872 "Python/bytecodes.c" + #line 2850 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -4074,7 +4049,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4078 "Python/generated_cases.c.h" + #line 4053 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4087,7 +4062,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2906 "Python/bytecodes.c" + #line 2884 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -4116,7 +4091,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4120 "Python/generated_cases.c.h" + #line 4095 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4129,7 +4104,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2938 "Python/bytecodes.c" + #line 2916 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 0 || oparg == 1); int is_meth = method != NULL; @@ -4158,7 +4133,7 @@ Py_DECREF(self); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4162 "Python/generated_cases.c.h" + #line 4137 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4171,7 +4146,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2970 "Python/bytecodes.c" + #line 2948 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -4199,7 +4174,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4203 "Python/generated_cases.c.h" + #line 4178 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4209,9 +4184,9 @@ } TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { - #line 3001 "Python/bytecodes.c" + #line 2979 "Python/bytecodes.c" GO_TO_INSTRUCTION(CALL_FUNCTION_EX); - #line 4215 "Python/generated_cases.c.h" + #line 4190 "Python/generated_cases.c.h" } TARGET(CALL_FUNCTION_EX) { @@ -4220,7 +4195,7 @@ PyObject *callargs = stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))]; PyObject *func = stack_pointer[-(2 + ((oparg & 1) ? 1 : 0))]; PyObject *result; - #line 3005 "Python/bytecodes.c" + #line 2983 "Python/bytecodes.c" // DICT_MERGE is called before this opcode if there are kwargs. // It converts all dict subtypes in kwargs into regular dicts. assert(kwargs == NULL || PyDict_CheckExact(kwargs)); @@ -4263,14 +4238,14 @@ else { result = PyObject_Call(func, callargs, kwargs); } - #line 4267 "Python/generated_cases.c.h" + #line 4242 "Python/generated_cases.c.h" Py_DECREF(func); Py_DECREF(callargs); Py_XDECREF(kwargs); - #line 3048 "Python/bytecodes.c" + #line 3026 "Python/bytecodes.c" assert(PEEK(3 + (oparg & 1)) == NULL); if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } - #line 4274 "Python/generated_cases.c.h" + #line 4249 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 1) ? 1 : 0)); STACK_SHRINK(2); stack_pointer[-1] = result; @@ -4285,7 +4260,7 @@ PyObject *kwdefaults = (oparg & 0x02) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0))] : NULL; PyObject *defaults = (oparg & 0x01) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x01) ? 1 : 0))] : NULL; PyObject *func; - #line 3058 "Python/bytecodes.c" + #line 3036 "Python/bytecodes.c" PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); @@ -4314,14 +4289,14 @@ func_obj->func_version = ((PyCodeObject *)codeobj)->co_version; func = (PyObject *)func_obj; - #line 4318 "Python/generated_cases.c.h" + #line 4293 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 0x01) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x08) ? 1 : 0)); stack_pointer[-1] = func; DISPATCH(); } TARGET(RETURN_GENERATOR) { - #line 3089 "Python/bytecodes.c" + #line 3067 "Python/bytecodes.c" assert(PyFunction_Check(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); @@ -4342,7 +4317,7 @@ frame = cframe.current_frame = prev; _PyFrame_StackPush(frame, (PyObject *)gen); goto resume_frame; - #line 4346 "Python/generated_cases.c.h" + #line 4321 "Python/generated_cases.c.h" } TARGET(BUILD_SLICE) { @@ -4350,15 +4325,15 @@ PyObject *stop = stack_pointer[-(1 + ((oparg == 3) ? 1 : 0))]; PyObject *start = stack_pointer[-(2 + ((oparg == 3) ? 1 : 0))]; PyObject *slice; - #line 3112 "Python/bytecodes.c" + #line 3090 "Python/bytecodes.c" slice = PySlice_New(start, stop, step); - #line 4356 "Python/generated_cases.c.h" + #line 4331 "Python/generated_cases.c.h" Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); - #line 3114 "Python/bytecodes.c" + #line 3092 "Python/bytecodes.c" if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } - #line 4362 "Python/generated_cases.c.h" + #line 4337 "Python/generated_cases.c.h" STACK_SHRINK(((oparg == 3) ? 1 : 0)); STACK_SHRINK(1); stack_pointer[-1] = slice; @@ -4369,7 +4344,7 @@ PyObject *fmt_spec = ((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? stack_pointer[-((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))] : NULL; PyObject *value = stack_pointer[-(1 + (((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))]; PyObject *result; - #line 3118 "Python/bytecodes.c" + #line 3096 "Python/bytecodes.c" /* Handles f-string value formatting. */ PyObject *(*conv_fn)(PyObject *); int which_conversion = oparg & FVC_MASK; @@ -4404,7 +4379,7 @@ Py_DECREF(value); Py_XDECREF(fmt_spec); if (result == NULL) { STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); goto pop_1_error; } - #line 4408 "Python/generated_cases.c.h" + #line 4383 "Python/generated_cases.c.h" STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); stack_pointer[-1] = result; DISPATCH(); @@ -4413,10 +4388,10 @@ TARGET(COPY) { PyObject *bottom = stack_pointer[-(1 + (oparg-1))]; PyObject *top; - #line 3155 "Python/bytecodes.c" + #line 3133 "Python/bytecodes.c" assert(oparg > 0); top = Py_NewRef(bottom); - #line 4420 "Python/generated_cases.c.h" + #line 4395 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = top; DISPATCH(); @@ -4428,7 +4403,7 @@ PyObject *rhs = stack_pointer[-1]; PyObject *lhs = stack_pointer[-2]; PyObject *res; - #line 3160 "Python/bytecodes.c" + #line 3138 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -4443,12 +4418,12 @@ assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); assert(binary_ops[oparg]); res = binary_ops[oparg](lhs, rhs); - #line 4447 "Python/generated_cases.c.h" + #line 4422 "Python/generated_cases.c.h" Py_DECREF(lhs); Py_DECREF(rhs); - #line 3175 "Python/bytecodes.c" + #line 3153 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 4452 "Python/generated_cases.c.h" + #line 4427 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -4458,16 +4433,16 @@ TARGET(SWAP) { PyObject *top = stack_pointer[-1]; PyObject *bottom = stack_pointer[-(2 + (oparg-2))]; - #line 3180 "Python/bytecodes.c" + #line 3158 "Python/bytecodes.c" assert(oparg >= 2); - #line 4464 "Python/generated_cases.c.h" + #line 4439 "Python/generated_cases.c.h" stack_pointer[-1] = bottom; stack_pointer[-(2 + (oparg-2))] = top; DISPATCH(); } TARGET(INSTRUMENTED_LINE) { - #line 3184 "Python/bytecodes.c" + #line 3162 "Python/bytecodes.c" _Py_CODEUNIT *here = next_instr-1; _PyFrame_SetStackPointer(frame, stack_pointer); int original_opcode = _Py_call_instrumentation_line( @@ -4487,11 +4462,11 @@ } opcode = original_opcode; DISPATCH_GOTO(); - #line 4491 "Python/generated_cases.c.h" + #line 4466 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_INSTRUCTION) { - #line 3206 "Python/bytecodes.c" + #line 3184 "Python/bytecodes.c" int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, next_instr-1); if (next_opcode < 0) goto error; @@ -4503,26 +4478,26 @@ assert(next_opcode > 0 && next_opcode < 256); opcode = next_opcode; DISPATCH_GOTO(); - #line 4507 "Python/generated_cases.c.h" + #line 4482 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_JUMP_FORWARD) { - #line 3221 "Python/bytecodes.c" + #line 3199 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); - #line 4513 "Python/generated_cases.c.h" + #line 4488 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_JUMP_BACKWARD) { - #line 3226 "Python/bytecodes.c" + #line 3204 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr-oparg, PY_MONITORING_EVENT_JUMP); - #line 4520 "Python/generated_cases.c.h" + #line 4495 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { - #line 3231 "Python/bytecodes.c" + #line 3209 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); if (err < 0) goto error; @@ -4530,12 +4505,12 @@ assert(err == 0 || err == 1); int offset = err*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4534 "Python/generated_cases.c.h" + #line 4509 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { - #line 3241 "Python/bytecodes.c" + #line 3219 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); if (err < 0) goto error; @@ -4543,12 +4518,12 @@ assert(err == 0 || err == 1); int offset = (1-err)*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4547 "Python/generated_cases.c.h" + #line 4522 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { - #line 3251 "Python/bytecodes.c" + #line 3229 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4561,12 +4536,12 @@ offset = 0; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4565 "Python/generated_cases.c.h" + #line 4540 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { - #line 3266 "Python/bytecodes.c" + #line 3244 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4579,22 +4554,22 @@ offset = oparg; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4583 "Python/generated_cases.c.h" + #line 4558 "Python/generated_cases.c.h" DISPATCH(); } TARGET(EXTENDED_ARG) { - #line 3281 "Python/bytecodes.c" + #line 3259 "Python/bytecodes.c" assert(oparg); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); - #line 4594 "Python/generated_cases.c.h" + #line 4569 "Python/generated_cases.c.h" } TARGET(CACHE) { - #line 3289 "Python/bytecodes.c" + #line 3267 "Python/bytecodes.c" Py_UNREACHABLE(); - #line 4600 "Python/generated_cases.c.h" + #line 4575 "Python/generated_cases.c.h" } diff --git a/Python/instrumentation.c b/Python/instrumentation.c index fee0356e19b455..f6e8353fdd14b5 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -42,14 +42,12 @@ static const int8_t EVENT_FOR_OPCODE[256] = { [POP_JUMP_IF_TRUE] = PY_MONITORING_EVENT_BRANCH, [POP_JUMP_IF_NONE] = PY_MONITORING_EVENT_BRANCH, [POP_JUMP_IF_NOT_NONE] = PY_MONITORING_EVENT_BRANCH, - [COMPARE_AND_BRANCH] = PY_MONITORING_EVENT_BRANCH, [INSTRUMENTED_JUMP_FORWARD] = PY_MONITORING_EVENT_JUMP, [INSTRUMENTED_JUMP_BACKWARD] = PY_MONITORING_EVENT_JUMP, [INSTRUMENTED_POP_JUMP_IF_FALSE] = PY_MONITORING_EVENT_BRANCH, [INSTRUMENTED_POP_JUMP_IF_TRUE] = PY_MONITORING_EVENT_BRANCH, [INSTRUMENTED_POP_JUMP_IF_NONE] = PY_MONITORING_EVENT_BRANCH, [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = PY_MONITORING_EVENT_BRANCH, - [INSTRUMENTED_COMPARE_AND_BRANCH] = PY_MONITORING_EVENT_BRANCH, [FOR_ITER] = PY_MONITORING_EVENT_BRANCH, [INSTRUMENTED_FOR_ITER] = PY_MONITORING_EVENT_BRANCH, [END_FOR] = PY_MONITORING_EVENT_STOP_ITERATION, @@ -77,14 +75,12 @@ static const bool OPCODE_HAS_EVENT[256] = { [POP_JUMP_IF_TRUE] = true, [POP_JUMP_IF_NONE] = true, [POP_JUMP_IF_NOT_NONE] = true, - [COMPARE_AND_BRANCH] = true, [INSTRUMENTED_JUMP_FORWARD] = true, [INSTRUMENTED_JUMP_BACKWARD] = true, [INSTRUMENTED_POP_JUMP_IF_FALSE] = true, [INSTRUMENTED_POP_JUMP_IF_TRUE] = true, [INSTRUMENTED_POP_JUMP_IF_NONE] = true, [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = true, - [INSTRUMENTED_COMPARE_AND_BRANCH] = true, [INSTRUMENTED_FOR_ITER] = true, [END_FOR] = true, [INSTRUMENTED_END_FOR] = true, @@ -105,7 +101,6 @@ static const uint8_t DE_INSTRUMENT[256] = { [INSTRUMENTED_POP_JUMP_IF_TRUE] = POP_JUMP_IF_TRUE, [INSTRUMENTED_POP_JUMP_IF_NONE] = POP_JUMP_IF_NONE, [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = POP_JUMP_IF_NOT_NONE, - [INSTRUMENTED_COMPARE_AND_BRANCH] = COMPARE_AND_BRANCH, [INSTRUMENTED_FOR_ITER] = FOR_ITER, [INSTRUMENTED_END_FOR] = END_FOR, [INSTRUMENTED_END_SEND] = END_SEND, @@ -136,8 +131,6 @@ static const uint8_t INSTRUMENTED_OPCODES[256] = { [INSTRUMENTED_POP_JUMP_IF_NONE] = INSTRUMENTED_POP_JUMP_IF_NONE, [POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE, [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE, - [COMPARE_AND_BRANCH] = INSTRUMENTED_COMPARE_AND_BRANCH, - [INSTRUMENTED_COMPARE_AND_BRANCH] = INSTRUMENTED_COMPARE_AND_BRANCH, [END_FOR] = INSTRUMENTED_END_FOR, [INSTRUMENTED_END_FOR] = INSTRUMENTED_END_FOR, [END_SEND] = INSTRUMENTED_END_SEND, @@ -357,10 +350,6 @@ Instruction read_instruction(PyCodeObject *code, int offset) int base = _PyOpcode_Deopt[result.deinstrumented_opcode]; /* TO DO -- Use instruction length, not cache count table */ result.length += _PyOpcode_Caches[base]; - if (base == COMPARE_AND_BRANCH) { - /* Skip over following POP_JUMP_IF */ - result.length += 1; - } return result; } @@ -1638,13 +1627,6 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) i++; base_opcode = _Py_GetBaseOpcode(code, i); } - /* TO DO -- Use instruction length, not cache count */ - if (base_opcode == COMPARE_AND_BRANCH) { - /* Skip over following POP_JUMP_IF */ - CHECK(instr[2].op.code == POP_JUMP_IF_FALSE || - instr[2].op.code == POP_JUMP_IF_TRUE); - i++; - } i += _PyOpcode_Caches[base_opcode]; } diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index 7d6eb5bcd29977..df842e5a3316d0 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -231,8 +231,6 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 2; case COMPARE_AND_BRANCH: return 2; - case INSTRUMENTED_COMPARE_AND_BRANCH: - return 0; case COMPARE_AND_BRANCH_FLOAT: return 2; case COMPARE_AND_BRANCH_INT: @@ -617,8 +615,6 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 1; case COMPARE_AND_BRANCH: return 0; - case INSTRUMENTED_COMPARE_AND_BRANCH: - return 0; case COMPARE_AND_BRANCH_FLOAT: return 0; case COMPARE_AND_BRANCH_INT: @@ -896,7 +892,6 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[256] = { [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000 }, [COMPARE_OP] = { true, INSTR_FMT_IBC }, [COMPARE_AND_BRANCH] = { true, INSTR_FMT_IBC0 }, - [INSTRUMENTED_COMPARE_AND_BRANCH] = { true, INSTR_FMT_IB }, [COMPARE_AND_BRANCH_FLOAT] = { true, INSTR_FMT_IBC0 }, [COMPARE_AND_BRANCH_INT] = { true, INSTR_FMT_IBC0 }, [COMPARE_AND_BRANCH_STR] = { true, INSTR_FMT_IBC0 }, diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 9018e84e805793..146e41c0a00432 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -238,7 +238,7 @@ static void *opcode_targets[256] = { &&TARGET_INSTRUMENTED_POP_JUMP_IF_NOT_NONE, &&TARGET_INSTRUMENTED_RETURN_CONST, &&TARGET_INSTRUMENTED_FOR_ITER, - &&TARGET_INSTRUMENTED_COMPARE_AND_BRANCH, + &&_unknown_opcode, &&TARGET_INSTRUMENTED_RESUME, &&TARGET_INSTRUMENTED_CALL, &&TARGET_INSTRUMENTED_RETURN_VALUE, From e7f6c374deb32642aa3fd423e8ea149534cda9b0 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 23 Mar 2023 02:02:23 +0000 Subject: [PATCH 056/116] Group instrumented opcodes. --- Include/internal/pycore_opcode.h | 16 ++++++++-------- Include/opcode.h | 8 ++++---- Lib/opcode.py | 11 ++++------- Python/opcode_targets.h | 10 +++++----- 4 files changed, 21 insertions(+), 24 deletions(-) diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index 344b3da287f68f..0ebad1ae45623c 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -480,11 +480,11 @@ static const char *const _PyOpcode_OpName[263] = { [232] = "<232>", [233] = "<233>", [234] = "<234>", + [235] = "<235>", + [236] = "<236>", + [237] = "<237>", [INSTRUMENTED_POP_JUMP_IF_NONE] = "INSTRUMENTED_POP_JUMP_IF_NONE", [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = "INSTRUMENTED_POP_JUMP_IF_NOT_NONE", - [INSTRUMENTED_RETURN_CONST] = "INSTRUMENTED_RETURN_CONST", - [INSTRUMENTED_FOR_ITER] = "INSTRUMENTED_FOR_ITER", - [239] = "<239>", [INSTRUMENTED_RESUME] = "INSTRUMENTED_RESUME", [INSTRUMENTED_CALL] = "INSTRUMENTED_CALL", [INSTRUMENTED_RETURN_VALUE] = "INSTRUMENTED_RETURN_VALUE", @@ -492,8 +492,8 @@ static const char *const _PyOpcode_OpName[263] = { [INSTRUMENTED_CALL_FUNCTION_EX] = "INSTRUMENTED_CALL_FUNCTION_EX", [INSTRUMENTED_JUMP_FORWARD] = "INSTRUMENTED_JUMP_FORWARD", [INSTRUMENTED_JUMP_BACKWARD] = "INSTRUMENTED_JUMP_BACKWARD", - [247] = "<247>", - [248] = "<248>", + [INSTRUMENTED_RETURN_CONST] = "INSTRUMENTED_RETURN_CONST", + [INSTRUMENTED_FOR_ITER] = "INSTRUMENTED_FOR_ITER", [INSTRUMENTED_POP_JUMP_IF_FALSE] = "INSTRUMENTED_POP_JUMP_IF_FALSE", [INSTRUMENTED_POP_JUMP_IF_TRUE] = "INSTRUMENTED_POP_JUMP_IF_TRUE", [INSTRUMENTED_END_FOR] = "INSTRUMENTED_END_FOR", @@ -579,9 +579,9 @@ static const char *const _PyOpcode_OpName[263] = { case 232: \ case 233: \ case 234: \ - case 239: \ - case 247: \ - case 248: \ + case 235: \ + case 236: \ + case 237: \ ; #ifdef __cplusplus diff --git a/Include/opcode.h b/Include/opcode.h index ea5646d25e6d3a..fcca0daa6de179 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -115,10 +115,8 @@ extern "C" { #define KW_NAMES 172 #define CALL_INTRINSIC_1 173 #define CALL_INTRINSIC_2 174 -#define INSTRUMENTED_POP_JUMP_IF_NONE 235 -#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 236 -#define INSTRUMENTED_RETURN_CONST 237 -#define INSTRUMENTED_FOR_ITER 238 +#define INSTRUMENTED_POP_JUMP_IF_NONE 238 +#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 239 #define INSTRUMENTED_RESUME 240 #define INSTRUMENTED_CALL 241 #define INSTRUMENTED_RETURN_VALUE 242 @@ -126,6 +124,8 @@ extern "C" { #define INSTRUMENTED_CALL_FUNCTION_EX 244 #define INSTRUMENTED_JUMP_FORWARD 245 #define INSTRUMENTED_JUMP_BACKWARD 246 +#define INSTRUMENTED_RETURN_CONST 247 +#define INSTRUMENTED_FOR_ITER 248 #define INSTRUMENTED_POP_JUMP_IF_FALSE 249 #define INSTRUMENTED_POP_JUMP_IF_TRUE 250 #define INSTRUMENTED_END_FOR 251 diff --git a/Lib/opcode.py b/Lib/opcode.py index 5f13ca6f0a8ed8..2057d451f0eb26 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -224,11 +224,8 @@ def pseudo_op(name, op, real_ops): # Instrumented instructions -def_op('INSTRUMENTED_POP_JUMP_IF_NONE', 235) -def_op('INSTRUMENTED_POP_JUMP_IF_NOT_NONE', 236) -def_op('INSTRUMENTED_RETURN_CONST', 237) -def_op('INSTRUMENTED_FOR_ITER', 238) - +def_op('INSTRUMENTED_POP_JUMP_IF_NONE', 238) +def_op('INSTRUMENTED_POP_JUMP_IF_NOT_NONE', 239) def_op('INSTRUMENTED_RESUME', 240) def_op('INSTRUMENTED_CALL', 241) def_op('INSTRUMENTED_RETURN_VALUE', 242) @@ -236,12 +233,12 @@ def pseudo_op(name, op, real_ops): def_op('INSTRUMENTED_CALL_FUNCTION_EX', 244) def_op('INSTRUMENTED_JUMP_FORWARD', 245) def_op('INSTRUMENTED_JUMP_BACKWARD', 246) - +def_op('INSTRUMENTED_RETURN_CONST', 247) +def_op('INSTRUMENTED_FOR_ITER', 248) def_op('INSTRUMENTED_POP_JUMP_IF_FALSE', 249) def_op('INSTRUMENTED_POP_JUMP_IF_TRUE', 250) def_op('INSTRUMENTED_END_FOR', 251) def_op('INSTRUMENTED_END_SEND', 252) - def_op('INSTRUMENTED_INSTRUCTION', 253) def_op('INSTRUMENTED_LINE', 254) # 255 is reserved diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index b03da52b61271f..a9ab58a42beaf9 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -234,11 +234,11 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, &&TARGET_INSTRUMENTED_POP_JUMP_IF_NONE, &&TARGET_INSTRUMENTED_POP_JUMP_IF_NOT_NONE, - &&TARGET_INSTRUMENTED_RETURN_CONST, - &&TARGET_INSTRUMENTED_FOR_ITER, - &&_unknown_opcode, &&TARGET_INSTRUMENTED_RESUME, &&TARGET_INSTRUMENTED_CALL, &&TARGET_INSTRUMENTED_RETURN_VALUE, @@ -246,8 +246,8 @@ static void *opcode_targets[256] = { &&TARGET_INSTRUMENTED_CALL_FUNCTION_EX, &&TARGET_INSTRUMENTED_JUMP_FORWARD, &&TARGET_INSTRUMENTED_JUMP_BACKWARD, - &&_unknown_opcode, - &&_unknown_opcode, + &&TARGET_INSTRUMENTED_RETURN_CONST, + &&TARGET_INSTRUMENTED_FOR_ITER, &&TARGET_INSTRUMENTED_POP_JUMP_IF_FALSE, &&TARGET_INSTRUMENTED_POP_JUMP_IF_TRUE, &&TARGET_INSTRUMENTED_END_FOR, From e44ebc59af7dc645acc918d0e48283087c1cbe5f Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 23 Mar 2023 03:17:26 +0000 Subject: [PATCH 057/116] Remove last references to DO_TRACING --- Include/internal/pycore_opcode.h | 3 ++- Include/opcode.h | 1 - Python/ceval.c | 33 -------------------------------- Python/makeopcodetargets.py | 1 - Python/opcode_targets.h | 2 +- Tools/build/generate_opcode_h.py | 3 --- 6 files changed, 3 insertions(+), 40 deletions(-) diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index 0ebad1ae45623c..53a446e378d143 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -500,7 +500,7 @@ static const char *const _PyOpcode_OpName[263] = { [INSTRUMENTED_END_SEND] = "INSTRUMENTED_END_SEND", [INSTRUMENTED_INSTRUCTION] = "INSTRUMENTED_INSTRUCTION", [INSTRUMENTED_LINE] = "INSTRUMENTED_LINE", - [DO_TRACING] = "DO_TRACING", + [255] = "<255>", [SETUP_FINALLY] = "SETUP_FINALLY", [SETUP_CLEANUP] = "SETUP_CLEANUP", [SETUP_WITH] = "SETUP_WITH", @@ -582,6 +582,7 @@ static const char *const _PyOpcode_OpName[263] = { case 235: \ case 236: \ case 237: \ + case 255: \ ; #ifdef __cplusplus diff --git a/Include/opcode.h b/Include/opcode.h index fcca0daa6de179..e619ed968a8e0a 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -203,7 +203,6 @@ extern "C" { #define UNPACK_SEQUENCE_TUPLE 154 #define UNPACK_SEQUENCE_TWO_TUPLE 158 #define SEND_GEN 159 -#define DO_TRACING 255 #define HAS_ARG(op) ((((op) >= HAVE_ARGUMENT) && (!IS_PSEUDO_OPCODE(op)))\ || ((op) == JUMP) \ diff --git a/Python/ceval.c b/Python/ceval.c index 7b9a70e07225a3..004410fe4dc1e6 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -802,39 +802,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #include "generated_cases.c.h" -#if USE_COMPUTED_GOTOS - TARGET_DO_TRACING: -#else - case DO_TRACING: -#endif - { - assert(0); - NEXTOPARG(); - PRE_DISPATCH_GOTO(); - // No _PyOpcode_Deopt here, since EXTENDED_ARG has no optimized forms: - while (opcode == EXTENDED_ARG) { - // CPython hasn't ever traced the instruction after an EXTENDED_ARG. - // Inline the EXTENDED_ARG here, so we can avoid branching there: - INSTRUCTION_START(EXTENDED_ARG); - opcode = next_instr->op.code; - oparg = oparg << 8 | next_instr->op.arg; - // Make sure the next instruction isn't a RESUME, since that needs - // to trace properly (and shouldn't have an EXTENDED_ARG, anyways): - assert(opcode != RESUME); - PRE_DISPATCH_GOTO(); - } - opcode = _PyOpcode_Deopt[opcode]; - if (_PyOpcode_Caches[opcode]) { - uint16_t *counter = &next_instr[1].cache; - // The instruction is going to decrement the counter, so we need to - // increment it here to make sure it doesn't try to specialize: - if (!ADAPTIVE_COUNTER_IS_MAX(*counter)) { - INCREMENT_ADAPTIVE_COUNTER(*counter); - } - } - DISPATCH_GOTO(); - } - #if USE_COMPUTED_GOTOS _unknown_opcode: #else diff --git a/Python/makeopcodetargets.py b/Python/makeopcodetargets.py index 33a4b4a76a1253..5aa31803397ce4 100755 --- a/Python/makeopcodetargets.py +++ b/Python/makeopcodetargets.py @@ -32,7 +32,6 @@ def write_contents(f): """ opcode = find_module('opcode') targets = ['_unknown_opcode'] * 256 - targets[255] = "TARGET_DO_TRACING" for opname, op in opcode.opmap.items(): if not opcode.is_pseudo(op): targets[op] = "TARGET_%s" % opname diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index a9ab58a42beaf9..81882e08ebb9d1 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -254,5 +254,5 @@ static void *opcode_targets[256] = { &&TARGET_INSTRUMENTED_END_SEND, &&TARGET_INSTRUMENTED_INSTRUCTION, &&TARGET_INSTRUMENTED_LINE, - &&TARGET_DO_TRACING + &&_unknown_opcode }; diff --git a/Tools/build/generate_opcode_h.py b/Tools/build/generate_opcode_h.py index 9b2112f7f5f31d..7aaef6b16f3443 100644 --- a/Tools/build/generate_opcode_h.py +++ b/Tools/build/generate_opcode_h.py @@ -105,9 +105,6 @@ def main(opcode_py, outfile='Include/opcode.h', internaloutfile='Include/interna specialized_opmap[name] = next_op opname_including_specialized[next_op] = name used[next_op] = True - specialized_opmap['DO_TRACING'] = 255 - opname_including_specialized[255] = 'DO_TRACING' - used[255] = True with open(outfile, 'w') as fobj, open(internaloutfile, 'w') as iobj: fobj.write(header) From e88921e2ccc3156a643257a30873bd6e2a59ab93 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 23 Mar 2023 18:32:22 +0000 Subject: [PATCH 058/116] Remove unused function. --- Python/instrumentation.c | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index f6e8353fdd14b5..243490d517ec8b 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -289,26 +289,6 @@ int _Py_GetBaseOpcode(PyCodeObject *code, int offset) return _PyOpcode_Deopt[opcode]; } -uint8_t deinstrument(PyCodeObject *code, int offset) -{ - int opcode = _Py_OPCODE(_PyCode_CODE(code)[offset]); - if (!is_instrumented(opcode)) { - assert(_PyOpcode_Deopt[opcode] != 0); - return _PyOpcode_Deopt[opcode]; - } - if (opcode == INSTRUMENTED_INSTRUCTION) { - opcode = code->_co_monitoring->per_instruction_opcodes[offset]; - } - if (opcode == INSTRUMENTED_LINE) { - opcode = code->_co_monitoring->lines[offset].original_opcode; - } - int deinstrumented = DE_INSTRUMENT[opcode]; - if (deinstrumented) { - return deinstrumented; - } - return opcode; -} - typedef struct _Instruction { uint8_t extended_args; uint8_t deinstrumented_opcode; From 2d9f22cbe3855ee2f77737b27d761de47e62d450 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 23 Mar 2023 18:38:37 +0000 Subject: [PATCH 059/116] Streamline de-instrumentation a little. --- Python/instrumentation.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 243490d517ec8b..d5ed9d3c1bce28 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -602,7 +602,7 @@ de_instrument(PyCodeObject *code, int offset, int event) if (opcode != INSTRUMENTED_LINE) { int deinstrumented = DE_INSTRUMENT[opcode]; if (deinstrumented) { - code->_co_monitoring->per_instruction_opcodes[offset] = deinstrumented; + code->_co_monitoring->per_instruction_opcodes[offset] = opcode = deinstrumented; } break; } @@ -613,7 +613,7 @@ de_instrument(PyCodeObject *code, int offset, int event) int orignal_opcode = lines->original_opcode; int deinstrumented = DE_INSTRUMENT[orignal_opcode]; if (deinstrumented) { - lines->original_opcode = deinstrumented; + lines->original_opcode = opcode = deinstrumented; } break; } @@ -621,12 +621,12 @@ de_instrument(PyCodeObject *code, int offset, int event) { int deinstrumented = DE_INSTRUMENT[opcode]; assert(deinstrumented); - instr->op.code = deinstrumented; + instr->op.code = opcode = deinstrumented; } } - int base_opcode = _Py_GetBaseOpcode(code, offset); + assert(!is_instrumented(opcode)); + int base_opcode = _PyOpcode_Deopt[opcode]; assert(base_opcode != 0); - assert(_PyOpcode_Deopt[base_opcode] == base_opcode); if (_PyOpcode_Caches[base_opcode]) { instr[1].cache = adaptive_counter_warmup(); } From b7579ac8be7ee92e9c7a8096f2e8ab31dda75ea1 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 23 Mar 2023 18:44:53 +0000 Subject: [PATCH 060/116] Use modern API for saving and restoring exception. --- Python/instrumentation.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index d5ed9d3c1bce28..06e65b48eb0d29 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1135,8 +1135,7 @@ _Py_call_instrumentation_exc_vector( { assert(_PyErr_Occurred(tstate)); assert(args[0] == NULL); - PyObject *type, *value, *traceback; - _PyErr_Fetch(tstate, &type, &value, &traceback); + PyObject *exc = _PyErr_GetRaisedException(tstate); PyCodeObject *code = frame->f_code; assert(args[1] == NULL); args[1] = (PyObject *)code; @@ -1155,12 +1154,10 @@ _Py_call_instrumentation_exc_vector( Py_DECREF(offset_obj); } if (err) { - Py_XDECREF(type); - Py_XDECREF(value); - Py_XDECREF(traceback); + Py_XDECREF(exc); } else { - _PyErr_Restore(tstate, type, value, traceback); + _PyErr_SetRaisedException(tstate, exc); } assert(_PyErr_Occurred(tstate)); } From 5a089a64b725170c04839fa188a9016922e99406 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 24 Mar 2023 02:17:21 +0000 Subject: [PATCH 061/116] Refactor instrumentation calls to reduce duplication. --- Python/instrumentation.c | 113 +++++++++++++-------------------------- 1 file changed, 37 insertions(+), 76 deletions(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 06e65b48eb0d29..76cb031f27c198 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -988,15 +988,15 @@ get_tools_for_instruction(PyCodeObject * code, int i, int event) assert(event != PY_MONITORING_EVENT_INSTRUCTION); assert(instrumentation_cross_checks(PyThreadState_GET()->interp, code)); _PyCoMonitoringData *monitoring = code->_co_monitoring; - if (monitoring->tools) { + if (event >= PY_MONITORING_UNGROUPED_EVENTS) { + assert(event == PY_MONITORING_EVENT_C_RAISE || + event == PY_MONITORING_EVENT_C_RETURN); + event = PY_MONITORING_EVENT_CALL; + } + if (event < PY_MONITORING_INSTRUMENTED_EVENTS && monitoring->tools) { tools = monitoring->tools[i]; } else { - if (event >= PY_MONITORING_UNGROUPED_EVENTS) { - assert(event == PY_MONITORING_EVENT_C_RAISE || - event == PY_MONITORING_EVENT_C_RETURN); - event = PY_MONITORING_EVENT_CALL; - } tools = code->_co_monitoring->active_monitors.tools[event]; } CHECK(tools_is_subset_for_event(code, event, tools)); @@ -1034,44 +1034,48 @@ call_instrument(PyThreadState *tstate, PyCodeObject *code, int event, return 0; } -int -_Py_call_instrumentation( +static int +call_instrumentation_vector( PyThreadState *tstate, int event, - _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, Py_ssize_t nargsf, PyObject *args[]) { + assert(!_PyErr_Occurred(tstate)); + assert(args[0] == NULL); PyCodeObject *code = frame->f_code; + assert(code->_co_instrumentation_version == tstate->interp->monitoring_version); assert(is_version_up_to_date(code, tstate->interp)); assert(instrumentation_cross_checks(tstate->interp, code)); + assert(args[1] == NULL); + args[1] = (PyObject *)code; int offset = instr - _PyCode_CODE(code); PyObject *offset_obj = PyLong_FromSsize_t(offset); if (offset_obj == NULL) { return -1; } + assert(args[2] == NULL); + args[2] = offset_obj; uint8_t tools = get_tools_for_instruction(code, offset, event); - PyObject *args[3] = { NULL, (PyObject *)code, offset_obj }; - int err = call_instrument(tstate, code, event, &args[1], 2 | PY_VECTORCALL_ARGUMENTS_OFFSET, offset, tools); + int err = call_instrument(tstate, code, event, &args[1], nargsf | PY_VECTORCALL_ARGUMENTS_OFFSET, offset, tools); Py_DECREF(offset_obj); return err; } +int +_Py_call_instrumentation( + PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) +{ + PyObject *args[3] = { NULL, NULL, NULL }; + return call_instrumentation_vector(tstate, event, frame, instr, 2, args); +} + int _Py_call_instrumentation_arg( PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg) { - PyCodeObject *code = frame->f_code; - assert(is_version_up_to_date(code, tstate->interp)); - assert(instrumentation_cross_checks(tstate->interp, code)); - int offset = instr - _PyCode_CODE(code); - PyObject *offset_obj = PyLong_FromSsize_t(offset); - if (offset_obj == NULL) { - return -1; - } - uint8_t tools = get_tools_for_instruction(code, offset, event); - PyObject *args[4] = { NULL, (PyObject *)code, offset_obj, arg }; - int err = call_instrument(tstate, code, event, &args[1], 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, offset, tools); - Py_DECREF(offset_obj); - return err; + PyObject *args[4] = { NULL, NULL, NULL, arg }; + return call_instrumentation_vector(tstate, event, frame, instr, 3, args); } int @@ -1079,19 +1083,8 @@ _Py_call_instrumentation_2args( PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg0, PyObject *arg1) { - PyCodeObject *code = frame->f_code; - assert(is_version_up_to_date(code, tstate->interp)); - assert(instrumentation_cross_checks(tstate->interp, code)); - int offset = instr - _PyCode_CODE(code); - PyObject *offset_obj = PyLong_FromSsize_t(offset); - if (offset_obj == NULL) { - return -1; - } - uint8_t tools = get_tools_for_instruction(code, offset, event); - PyObject *args[5] = { NULL, (PyObject *)code, offset_obj, arg0, arg1 }; - int err = call_instrument(tstate, code, event, &args[1], 4 | PY_VECTORCALL_ARGUMENTS_OFFSET, offset, tools); - Py_DECREF(offset_obj); - return err; + PyObject *args[5] = { NULL, NULL, NULL, arg0, arg1 }; + return call_instrumentation_vector(tstate, event, frame, instr, 4, args); } int @@ -1104,55 +1097,25 @@ _Py_call_instrumentation_jump( assert(frame->prev_instr == instr); frame->prev_instr = target; PyCodeObject *code = frame->f_code; - int offset = instr - _PyCode_CODE(code); int to = target - _PyCode_CODE(code); PyObject *to_obj = PyLong_FromLong(to); if (to_obj == NULL) { return -1; } - PyObject *from_obj = PyLong_FromLong(offset); - if (from_obj == NULL) { - Py_DECREF(to_obj); - return -1; - } - uint8_t tools = get_tools_for_instruction(code, offset, event); - PyObject *args[4] = { NULL, (PyObject *)code, from_obj, to_obj }; - int err = call_instrument(tstate, code, event, &args[1], 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, offset, tools); + PyObject *args[4] = { NULL, NULL, NULL, to_obj }; + int err = call_instrumentation_vector(tstate, event, frame, instr, 3, args); Py_DECREF(to_obj); - Py_DECREF(from_obj); - if (err) { - /* Error handling expects next_instr to point to instruction + 1. - * So we add one here. */ - frame->prev_instr++; - } return err; } -void -_Py_call_instrumentation_exc_vector( +static void +call_instrumentation_vector_protected( PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, Py_ssize_t nargsf, PyObject *args[]) { assert(_PyErr_Occurred(tstate)); - assert(args[0] == NULL); PyObject *exc = _PyErr_GetRaisedException(tstate); - PyCodeObject *code = frame->f_code; - assert(args[1] == NULL); - args[1] = (PyObject *)code; - assert(code->_co_instrumentation_version == tstate->interp->monitoring_version); - int offset = instr - _PyCode_CODE(code); - uint8_t tools = code->_co_monitoring->active_monitors.tools[event]; - PyObject *offset_obj = PyLong_FromSsize_t(offset); - int err; - if (offset_obj == NULL) { - err = -1; - } - else { - assert(args[2] == NULL); - args[2] = offset_obj; - err = call_instrument(tstate, code, event, &args[1], nargsf, offset, tools); - Py_DECREF(offset_obj); - } + int err = call_instrumentation_vector(tstate, event, frame, instr, nargsf, args); if (err) { Py_XDECREF(exc); } @@ -1169,8 +1132,7 @@ _Py_call_instrumentation_exc0( { assert(_PyErr_Occurred(tstate)); PyObject *args[3] = { NULL, NULL, NULL }; - Py_ssize_t nargsf = 2 | PY_VECTORCALL_ARGUMENTS_OFFSET; - _Py_call_instrumentation_exc_vector(tstate, event, frame, instr, nargsf, args); + call_instrumentation_vector_protected(tstate, event, frame, instr, 2, args); } void @@ -1180,8 +1142,7 @@ _Py_call_instrumentation_exc2( { assert(_PyErr_Occurred(tstate)); PyObject *args[5] = { NULL, NULL, NULL, arg0, arg1 }; - Py_ssize_t nargsf = 4 | PY_VECTORCALL_ARGUMENTS_OFFSET; - _Py_call_instrumentation_exc_vector(tstate, event, frame, instr, nargsf, args); + call_instrumentation_vector_protected(tstate, event, frame, instr, 4, args); } From d5fdec8339b9064ce20817c250f38f6a531288ef Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 24 Mar 2023 03:34:56 +0000 Subject: [PATCH 062/116] Fix refleaks --- Python/bytecodes.c | 2 ++ Python/generated_cases.c.h | 24 +++++++++++++----------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 0f500cb802daa0..f392266edfdbdd 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3183,6 +3183,7 @@ dummy_func( inst(INSTRUMENTED_POP_JUMP_IF_TRUE, ( -- )) { PyObject *cond = POP(); int err = PyObject_IsTrue(cond); + Py_DECREF(cond); ERROR_IF(err < 0, error); _Py_CODEUNIT *here = next_instr-1; assert(err == 0 || err == 1); @@ -3193,6 +3194,7 @@ dummy_func( inst(INSTRUMENTED_POP_JUMP_IF_FALSE, ( -- )) { PyObject *cond = POP(); int err = PyObject_IsTrue(cond); + Py_DECREF(cond); ERROR_IF(err < 0, error); _Py_CODEUNIT *here = next_instr-1; assert(err == 0 || err == 1); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 107aac46019491..b7064a71ce3d09 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -4473,30 +4473,32 @@ #line 3184 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); + Py_DECREF(cond); if (err < 0) goto error; _Py_CODEUNIT *here = next_instr-1; assert(err == 0 || err == 1); int offset = err*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4482 "Python/generated_cases.c.h" + #line 4483 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { - #line 3194 "Python/bytecodes.c" + #line 3195 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); + Py_DECREF(cond); if (err < 0) goto error; _Py_CODEUNIT *here = next_instr-1; assert(err == 0 || err == 1); int offset = (1-err)*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4495 "Python/generated_cases.c.h" + #line 4497 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { - #line 3204 "Python/bytecodes.c" + #line 3206 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4509,12 +4511,12 @@ offset = 0; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4513 "Python/generated_cases.c.h" + #line 4515 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { - #line 3219 "Python/bytecodes.c" + #line 3221 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4527,22 +4529,22 @@ offset = oparg; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4531 "Python/generated_cases.c.h" + #line 4533 "Python/generated_cases.c.h" DISPATCH(); } TARGET(EXTENDED_ARG) { - #line 3234 "Python/bytecodes.c" + #line 3236 "Python/bytecodes.c" assert(oparg); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); - #line 4542 "Python/generated_cases.c.h" + #line 4544 "Python/generated_cases.c.h" } TARGET(CACHE) { - #line 3242 "Python/bytecodes.c" + #line 3244 "Python/bytecodes.c" Py_UNREACHABLE(); - #line 4548 "Python/generated_cases.c.h" + #line 4550 "Python/generated_cases.c.h" } From c88741df3751fd6b91b9d8601eaa37d04976d9f2 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 24 Mar 2023 04:13:44 +0000 Subject: [PATCH 063/116] Remove commented out code. --- Modules/_lsprof.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c index 47a917bff843dc..83d034ae7eed78 100644 --- a/Modules/_lsprof.c +++ b/Modules/_lsprof.c @@ -126,10 +126,6 @@ call_timer(ProfilerObject *pObj) static PyObject * normalizeUserObj(PyObject *obj) { - //if (Py_TYPE(obj) == &PyMethodDescr_Type) { - // assert(0); - // return PyObject_Repr(obj); - //} PyCFunctionObject *fn; if (!PyCFunction_Check(obj)) { return Py_NewRef(obj); From b6744cac1d5831144cd97b03ca4b4d21854e6cdb Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 24 Mar 2023 04:15:02 +0000 Subject: [PATCH 064/116] Make sure that stacktop == -1 when stack_pointer is cached. --- Python/bytecodes.c | 4 +- Python/generated_cases.c.h | 966 +++++++++++++++++++------------------ 2 files changed, 487 insertions(+), 483 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index f392266edfdbdd..fd213f8e7282e7 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -99,11 +99,12 @@ dummy_func( _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation( tstate, oparg > 0, frame, next_instr-1); + stack_pointer = _PyFrame_GetStackPointer(frame); + frame->stacktop = -1; ERROR_IF(err, error); if (frame->prev_instr != next_instr-1) { /* Instrumentation has jumped */ next_instr = frame->prev_instr; - stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { @@ -3139,6 +3140,7 @@ dummy_func( int original_opcode = _Py_call_instrumentation_line( tstate, frame, here); stack_pointer = _PyFrame_GetStackPointer(frame); + frame->stacktop = -1; if (original_opcode < 0) { next_instr = here+1; goto error; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index b7064a71ce3d09..6531f4fd764ca6 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -21,23 +21,24 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation( tstate, oparg > 0, frame, next_instr-1); + stack_pointer = _PyFrame_GetStackPointer(frame); + frame->stacktop = -1; if (err) goto error; if (frame->prev_instr != next_instr-1) { /* Instrumentation has jumped */ next_instr = frame->prev_instr; - stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { goto handle_eval_breaker; } } - #line 36 "Python/generated_cases.c.h" + #line 37 "Python/generated_cases.c.h" DISPATCH(); } TARGET(RESUME) { - #line 116 "Python/bytecodes.c" + #line 117 "Python/bytecodes.c" assert(tstate->cframe == &cframe); assert(frame == cframe.current_frame); /* Possibly combine this with eval breaker */ @@ -49,18 +50,18 @@ else if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { goto handle_eval_breaker; } - #line 53 "Python/generated_cases.c.h" + #line 54 "Python/generated_cases.c.h" DISPATCH(); } TARGET(LOAD_CLOSURE) { PyObject *value; - #line 130 "Python/bytecodes.c" + #line 131 "Python/bytecodes.c" /* We keep LOAD_CLOSURE so that the bytecode stays more readable. */ value = GETLOCAL(oparg); if (value == NULL) goto unbound_local_error; Py_INCREF(value); - #line 64 "Python/generated_cases.c.h" + #line 65 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -68,11 +69,11 @@ TARGET(LOAD_FAST_CHECK) { PyObject *value; - #line 137 "Python/bytecodes.c" + #line 138 "Python/bytecodes.c" value = GETLOCAL(oparg); if (value == NULL) goto unbound_local_error; Py_INCREF(value); - #line 76 "Python/generated_cases.c.h" + #line 77 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -80,11 +81,11 @@ TARGET(LOAD_FAST) { PyObject *value; - #line 143 "Python/bytecodes.c" + #line 144 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - #line 88 "Python/generated_cases.c.h" + #line 89 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -93,10 +94,10 @@ TARGET(LOAD_CONST) { PREDICTED(LOAD_CONST); PyObject *value; - #line 149 "Python/bytecodes.c" + #line 150 "Python/bytecodes.c" value = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(value); - #line 100 "Python/generated_cases.c.h" + #line 101 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -104,9 +105,9 @@ TARGET(STORE_FAST) { PyObject *value = stack_pointer[-1]; - #line 154 "Python/bytecodes.c" + #line 155 "Python/bytecodes.c" SETLOCAL(oparg, value); - #line 110 "Python/generated_cases.c.h" + #line 111 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -116,21 +117,21 @@ PyObject *_tmp_2; { PyObject *value; - #line 143 "Python/bytecodes.c" + #line 144 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - #line 124 "Python/generated_cases.c.h" + #line 125 "Python/generated_cases.c.h" _tmp_2 = value; } oparg = (next_instr++)->op.arg; { PyObject *value; - #line 143 "Python/bytecodes.c" + #line 144 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - #line 134 "Python/generated_cases.c.h" + #line 135 "Python/generated_cases.c.h" _tmp_1 = value; } STACK_GROW(2); @@ -144,20 +145,20 @@ PyObject *_tmp_2; { PyObject *value; - #line 143 "Python/bytecodes.c" + #line 144 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - #line 152 "Python/generated_cases.c.h" + #line 153 "Python/generated_cases.c.h" _tmp_2 = value; } oparg = (next_instr++)->op.arg; { PyObject *value; - #line 149 "Python/bytecodes.c" + #line 150 "Python/bytecodes.c" value = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(value); - #line 161 "Python/generated_cases.c.h" + #line 162 "Python/generated_cases.c.h" _tmp_1 = value; } STACK_GROW(2); @@ -170,18 +171,18 @@ PyObject *_tmp_1 = stack_pointer[-1]; { PyObject *value = _tmp_1; - #line 154 "Python/bytecodes.c" + #line 155 "Python/bytecodes.c" SETLOCAL(oparg, value); - #line 176 "Python/generated_cases.c.h" + #line 177 "Python/generated_cases.c.h" } oparg = (next_instr++)->op.arg; { PyObject *value; - #line 143 "Python/bytecodes.c" + #line 144 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - #line 185 "Python/generated_cases.c.h" + #line 186 "Python/generated_cases.c.h" _tmp_1 = value; } stack_pointer[-1] = _tmp_1; @@ -193,16 +194,16 @@ PyObject *_tmp_2 = stack_pointer[-2]; { PyObject *value = _tmp_1; - #line 154 "Python/bytecodes.c" + #line 155 "Python/bytecodes.c" SETLOCAL(oparg, value); - #line 199 "Python/generated_cases.c.h" + #line 200 "Python/generated_cases.c.h" } oparg = (next_instr++)->op.arg; { PyObject *value = _tmp_2; - #line 154 "Python/bytecodes.c" + #line 155 "Python/bytecodes.c" SETLOCAL(oparg, value); - #line 206 "Python/generated_cases.c.h" + #line 207 "Python/generated_cases.c.h" } STACK_SHRINK(2); DISPATCH(); @@ -213,20 +214,20 @@ PyObject *_tmp_2; { PyObject *value; - #line 149 "Python/bytecodes.c" + #line 150 "Python/bytecodes.c" value = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(value); - #line 220 "Python/generated_cases.c.h" + #line 221 "Python/generated_cases.c.h" _tmp_2 = value; } oparg = (next_instr++)->op.arg; { PyObject *value; - #line 143 "Python/bytecodes.c" + #line 144 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - #line 230 "Python/generated_cases.c.h" + #line 231 "Python/generated_cases.c.h" _tmp_1 = value; } STACK_GROW(2); @@ -237,8 +238,8 @@ TARGET(POP_TOP) { PyObject *value = stack_pointer[-1]; - #line 164 "Python/bytecodes.c" - #line 242 "Python/generated_cases.c.h" + #line 165 "Python/bytecodes.c" + #line 243 "Python/generated_cases.c.h" Py_DECREF(value); STACK_SHRINK(1); DISPATCH(); @@ -246,9 +247,9 @@ TARGET(PUSH_NULL) { PyObject *res; - #line 168 "Python/bytecodes.c" + #line 169 "Python/bytecodes.c" res = NULL; - #line 252 "Python/generated_cases.c.h" + #line 253 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); @@ -259,14 +260,14 @@ PyObject *_tmp_2 = stack_pointer[-2]; { PyObject *value = _tmp_1; - #line 164 "Python/bytecodes.c" - #line 264 "Python/generated_cases.c.h" + #line 165 "Python/bytecodes.c" + #line 265 "Python/generated_cases.c.h" Py_DECREF(value); } { PyObject *value = _tmp_2; - #line 164 "Python/bytecodes.c" - #line 270 "Python/generated_cases.c.h" + #line 165 "Python/bytecodes.c" + #line 271 "Python/generated_cases.c.h" Py_DECREF(value); } STACK_SHRINK(2); @@ -276,7 +277,7 @@ TARGET(INSTRUMENTED_END_FOR) { PyObject *value = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; - #line 174 "Python/bytecodes.c" + #line 175 "Python/bytecodes.c" /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ if (PyGen_Check(receiver)) { @@ -286,7 +287,7 @@ } PyErr_SetRaisedException(NULL); } - #line 290 "Python/generated_cases.c.h" + #line 291 "Python/generated_cases.c.h" Py_DECREF(receiver); Py_DECREF(value); STACK_SHRINK(2); @@ -296,9 +297,9 @@ TARGET(END_SEND) { PyObject *value = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; - #line 187 "Python/bytecodes.c" + #line 188 "Python/bytecodes.c" Py_DECREF(receiver); - #line 302 "Python/generated_cases.c.h" + #line 303 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = value; DISPATCH(); @@ -307,7 +308,7 @@ TARGET(INSTRUMENTED_END_SEND) { PyObject *value = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; - #line 191 "Python/bytecodes.c" + #line 192 "Python/bytecodes.c" if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); if (monitor_stop_iteration(tstate, frame, next_instr-1)) { @@ -316,7 +317,7 @@ PyErr_SetRaisedException(NULL); } Py_DECREF(receiver); - #line 320 "Python/generated_cases.c.h" + #line 321 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = value; DISPATCH(); @@ -325,13 +326,13 @@ TARGET(UNARY_NEGATIVE) { PyObject *value = stack_pointer[-1]; PyObject *res; - #line 202 "Python/bytecodes.c" + #line 203 "Python/bytecodes.c" res = PyNumber_Negative(value); - #line 331 "Python/generated_cases.c.h" + #line 332 "Python/generated_cases.c.h" Py_DECREF(value); - #line 204 "Python/bytecodes.c" + #line 205 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; - #line 335 "Python/generated_cases.c.h" + #line 336 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -339,11 +340,11 @@ TARGET(UNARY_NOT) { PyObject *value = stack_pointer[-1]; PyObject *res; - #line 208 "Python/bytecodes.c" + #line 209 "Python/bytecodes.c" int err = PyObject_IsTrue(value); - #line 345 "Python/generated_cases.c.h" + #line 346 "Python/generated_cases.c.h" Py_DECREF(value); - #line 210 "Python/bytecodes.c" + #line 211 "Python/bytecodes.c" if (err < 0) goto pop_1_error; if (err == 0) { res = Py_True; @@ -352,7 +353,7 @@ res = Py_False; } Py_INCREF(res); - #line 356 "Python/generated_cases.c.h" + #line 357 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -360,13 +361,13 @@ TARGET(UNARY_INVERT) { PyObject *value = stack_pointer[-1]; PyObject *res; - #line 221 "Python/bytecodes.c" + #line 222 "Python/bytecodes.c" res = PyNumber_Invert(value); - #line 366 "Python/generated_cases.c.h" + #line 367 "Python/generated_cases.c.h" Py_DECREF(value); - #line 223 "Python/bytecodes.c" + #line 224 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; - #line 370 "Python/generated_cases.c.h" + #line 371 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -375,7 +376,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *prod; - #line 240 "Python/bytecodes.c" + #line 241 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -383,7 +384,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (prod == NULL) goto pop_2_error; - #line 387 "Python/generated_cases.c.h" + #line 388 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = prod; next_instr += 1; @@ -394,14 +395,14 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *prod; - #line 250 "Python/bytecodes.c" + #line 251 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); double dprod = ((PyFloatObject *)left)->ob_fval * ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dprod, prod); - #line 405 "Python/generated_cases.c.h" + #line 406 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = prod; next_instr += 1; @@ -412,7 +413,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *sub; - #line 259 "Python/bytecodes.c" + #line 260 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -420,7 +421,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (sub == NULL) goto pop_2_error; - #line 424 "Python/generated_cases.c.h" + #line 425 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = sub; next_instr += 1; @@ -431,13 +432,13 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *sub; - #line 269 "Python/bytecodes.c" + #line 270 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); double dsub = ((PyFloatObject *)left)->ob_fval - ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dsub, sub); - #line 441 "Python/generated_cases.c.h" + #line 442 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = sub; next_instr += 1; @@ -448,7 +449,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 277 "Python/bytecodes.c" + #line 278 "Python/bytecodes.c" DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -456,7 +457,7 @@ _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); if (res == NULL) goto pop_2_error; - #line 460 "Python/generated_cases.c.h" + #line 461 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -466,7 +467,7 @@ TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; - #line 293 "Python/bytecodes.c" + #line 294 "Python/bytecodes.c" DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); _Py_CODEUNIT true_next = next_instr[INLINE_CACHE_ENTRIES_BINARY_OP]; @@ -493,7 +494,7 @@ if (*target_local == NULL) goto pop_2_error; // The STORE_FAST is already done. JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP + 1); - #line 497 "Python/generated_cases.c.h" + #line 498 "Python/generated_cases.c.h" STACK_SHRINK(2); DISPATCH(); } @@ -502,14 +503,14 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *sum; - #line 322 "Python/bytecodes.c" + #line 323 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); double dsum = ((PyFloatObject *)left)->ob_fval + ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dsum, sum); - #line 513 "Python/generated_cases.c.h" + #line 514 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = sum; next_instr += 1; @@ -520,7 +521,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *sum; - #line 331 "Python/bytecodes.c" + #line 332 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -528,7 +529,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (sum == NULL) goto pop_2_error; - #line 532 "Python/generated_cases.c.h" + #line 533 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = sum; next_instr += 1; @@ -541,7 +542,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *container = stack_pointer[-2]; PyObject *res; - #line 349 "Python/bytecodes.c" + #line 350 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -553,12 +554,12 @@ DECREMENT_ADAPTIVE_COUNTER(cache->counter); #endif /* ENABLE_SPECIALIZATION */ res = PyObject_GetItem(container, sub); - #line 557 "Python/generated_cases.c.h" + #line 558 "Python/generated_cases.c.h" Py_DECREF(container); Py_DECREF(sub); - #line 361 "Python/bytecodes.c" + #line 362 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 562 "Python/generated_cases.c.h" + #line 563 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 4; @@ -570,7 +571,7 @@ PyObject *start = stack_pointer[-2]; PyObject *container = stack_pointer[-3]; PyObject *res; - #line 365 "Python/bytecodes.c" + #line 366 "Python/bytecodes.c" PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); // Can't use ERROR_IF() here, because we haven't // DECREF'ed container yet, and we still own slice. @@ -583,7 +584,7 @@ } Py_DECREF(container); if (res == NULL) goto pop_3_error; - #line 587 "Python/generated_cases.c.h" + #line 588 "Python/generated_cases.c.h" STACK_SHRINK(2); stack_pointer[-1] = res; DISPATCH(); @@ -594,7 +595,7 @@ PyObject *start = stack_pointer[-2]; PyObject *container = stack_pointer[-3]; PyObject *v = stack_pointer[-4]; - #line 380 "Python/bytecodes.c" + #line 381 "Python/bytecodes.c" PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); int err; if (slice == NULL) { @@ -607,7 +608,7 @@ Py_DECREF(v); Py_DECREF(container); if (err) goto pop_4_error; - #line 611 "Python/generated_cases.c.h" + #line 612 "Python/generated_cases.c.h" STACK_SHRINK(4); DISPATCH(); } @@ -616,7 +617,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *list = stack_pointer[-2]; PyObject *res; - #line 395 "Python/bytecodes.c" + #line 396 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); @@ -630,7 +631,7 @@ Py_INCREF(res); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(list); - #line 634 "Python/generated_cases.c.h" + #line 635 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 4; @@ -641,7 +642,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *tuple = stack_pointer[-2]; PyObject *res; - #line 411 "Python/bytecodes.c" + #line 412 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); @@ -655,7 +656,7 @@ Py_INCREF(res); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(tuple); - #line 659 "Python/generated_cases.c.h" + #line 660 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 4; @@ -666,7 +667,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *dict = stack_pointer[-2]; PyObject *res; - #line 427 "Python/bytecodes.c" + #line 428 "Python/bytecodes.c" DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); res = PyDict_GetItemWithError(dict, sub); @@ -674,14 +675,14 @@ if (!_PyErr_Occurred(tstate)) { _PyErr_SetKeyError(sub); } - #line 678 "Python/generated_cases.c.h" + #line 679 "Python/generated_cases.c.h" Py_DECREF(dict); Py_DECREF(sub); - #line 435 "Python/bytecodes.c" + #line 436 "Python/bytecodes.c" if (true) goto pop_2_error; } Py_INCREF(res); // Do this before DECREF'ing dict, sub - #line 685 "Python/generated_cases.c.h" + #line 686 "Python/generated_cases.c.h" Py_DECREF(dict); Py_DECREF(sub); STACK_SHRINK(1); @@ -695,7 +696,7 @@ PyObject *container = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t func_version = read_u16(&next_instr[3].cache); - #line 442 "Python/bytecodes.c" + #line 443 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(container); DEOPT_IF(tp->tp_version_tag != type_version, BINARY_SUBSCR); assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); @@ -714,15 +715,15 @@ new_frame->localsplus[1] = sub; JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR); DISPATCH_INLINED(new_frame); - #line 718 "Python/generated_cases.c.h" + #line 719 "Python/generated_cases.c.h" } TARGET(LIST_APPEND) { PyObject *v = stack_pointer[-1]; PyObject *list = stack_pointer[-(2 + (oparg-1))]; - #line 463 "Python/bytecodes.c" + #line 464 "Python/bytecodes.c" if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; - #line 726 "Python/generated_cases.c.h" + #line 727 "Python/generated_cases.c.h" STACK_SHRINK(1); PREDICT(JUMP_BACKWARD); DISPATCH(); @@ -731,13 +732,13 @@ TARGET(SET_ADD) { PyObject *v = stack_pointer[-1]; PyObject *set = stack_pointer[-(2 + (oparg-1))]; - #line 468 "Python/bytecodes.c" + #line 469 "Python/bytecodes.c" int err = PySet_Add(set, v); - #line 737 "Python/generated_cases.c.h" + #line 738 "Python/generated_cases.c.h" Py_DECREF(v); - #line 470 "Python/bytecodes.c" + #line 471 "Python/bytecodes.c" if (err) goto pop_1_error; - #line 741 "Python/generated_cases.c.h" + #line 742 "Python/generated_cases.c.h" STACK_SHRINK(1); PREDICT(JUMP_BACKWARD); DISPATCH(); @@ -750,7 +751,7 @@ PyObject *container = stack_pointer[-2]; PyObject *v = stack_pointer[-3]; uint16_t counter = read_u16(&next_instr[0].cache); - #line 481 "Python/bytecodes.c" + #line 482 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr--; @@ -765,13 +766,13 @@ #endif /* ENABLE_SPECIALIZATION */ /* container[sub] = v */ int err = PyObject_SetItem(container, sub, v); - #line 769 "Python/generated_cases.c.h" + #line 770 "Python/generated_cases.c.h" Py_DECREF(v); Py_DECREF(container); Py_DECREF(sub); - #line 496 "Python/bytecodes.c" + #line 497 "Python/bytecodes.c" if (err) goto pop_3_error; - #line 775 "Python/generated_cases.c.h" + #line 776 "Python/generated_cases.c.h" STACK_SHRINK(3); next_instr += 1; DISPATCH(); @@ -781,7 +782,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *list = stack_pointer[-2]; PyObject *value = stack_pointer[-3]; - #line 500 "Python/bytecodes.c" + #line 501 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); @@ -798,7 +799,7 @@ Py_DECREF(old_value); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(list); - #line 802 "Python/generated_cases.c.h" + #line 803 "Python/generated_cases.c.h" STACK_SHRINK(3); next_instr += 1; DISPATCH(); @@ -808,13 +809,13 @@ PyObject *sub = stack_pointer[-1]; PyObject *dict = stack_pointer[-2]; PyObject *value = stack_pointer[-3]; - #line 519 "Python/bytecodes.c" + #line 520 "Python/bytecodes.c" DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); Py_DECREF(dict); if (err) goto pop_3_error; - #line 818 "Python/generated_cases.c.h" + #line 819 "Python/generated_cases.c.h" STACK_SHRINK(3); next_instr += 1; DISPATCH(); @@ -823,15 +824,15 @@ TARGET(DELETE_SUBSCR) { PyObject *sub = stack_pointer[-1]; PyObject *container = stack_pointer[-2]; - #line 527 "Python/bytecodes.c" + #line 528 "Python/bytecodes.c" /* del container[sub] */ int err = PyObject_DelItem(container, sub); - #line 830 "Python/generated_cases.c.h" + #line 831 "Python/generated_cases.c.h" Py_DECREF(container); Py_DECREF(sub); - #line 530 "Python/bytecodes.c" + #line 531 "Python/bytecodes.c" if (err) goto pop_2_error; - #line 835 "Python/generated_cases.c.h" + #line 836 "Python/generated_cases.c.h" STACK_SHRINK(2); DISPATCH(); } @@ -839,14 +840,14 @@ TARGET(CALL_INTRINSIC_1) { PyObject *value = stack_pointer[-1]; PyObject *res; - #line 534 "Python/bytecodes.c" + #line 535 "Python/bytecodes.c" assert(oparg <= MAX_INTRINSIC_1); res = _PyIntrinsics_UnaryFunctions[oparg](tstate, value); - #line 846 "Python/generated_cases.c.h" + #line 847 "Python/generated_cases.c.h" Py_DECREF(value); - #line 537 "Python/bytecodes.c" + #line 538 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; - #line 850 "Python/generated_cases.c.h" + #line 851 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -855,15 +856,15 @@ PyObject *value1 = stack_pointer[-1]; PyObject *value2 = stack_pointer[-2]; PyObject *res; - #line 541 "Python/bytecodes.c" + #line 542 "Python/bytecodes.c" assert(oparg <= MAX_INTRINSIC_2); res = _PyIntrinsics_BinaryFunctions[oparg](tstate, value2, value1); - #line 862 "Python/generated_cases.c.h" + #line 863 "Python/generated_cases.c.h" Py_DECREF(value2); Py_DECREF(value1); - #line 544 "Python/bytecodes.c" + #line 545 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 867 "Python/generated_cases.c.h" + #line 868 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); @@ -871,7 +872,7 @@ TARGET(RAISE_VARARGS) { PyObject **args = (stack_pointer - oparg); - #line 548 "Python/bytecodes.c" + #line 549 "Python/bytecodes.c" PyObject *cause = NULL, *exc = NULL; switch (oparg) { case 2: @@ -889,12 +890,12 @@ break; } if (true) { STACK_SHRINK(oparg); goto error; } - #line 893 "Python/generated_cases.c.h" + #line 894 "Python/generated_cases.c.h" } TARGET(INTERPRETER_EXIT) { PyObject *retval = stack_pointer[-1]; - #line 568 "Python/bytecodes.c" + #line 569 "Python/bytecodes.c" assert(frame == &entry_frame); assert(_PyFrame_IsIncomplete(frame)); STACK_SHRINK(1); // Since we're not going to DISPATCH() @@ -905,12 +906,12 @@ assert(!_PyErr_Occurred(tstate)); _Py_LeaveRecursiveCallTstate(tstate); return retval; - #line 909 "Python/generated_cases.c.h" + #line 910 "Python/generated_cases.c.h" } TARGET(RETURN_VALUE) { PyObject *retval = stack_pointer[-1]; - #line 581 "Python/bytecodes.c" + #line 582 "Python/bytecodes.c" STACK_SHRINK(1); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -922,12 +923,12 @@ _PyEvalFrameClearAndPop(tstate, dying); _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 926 "Python/generated_cases.c.h" + #line 927 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_RETURN_VALUE) { PyObject *retval = stack_pointer[-1]; - #line 595 "Python/bytecodes.c" + #line 596 "Python/bytecodes.c" int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, next_instr-1, retval); @@ -943,11 +944,11 @@ _PyEvalFrameClearAndPop(tstate, dying); _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 947 "Python/generated_cases.c.h" + #line 948 "Python/generated_cases.c.h" } TARGET(RETURN_CONST) { - #line 613 "Python/bytecodes.c" + #line 614 "Python/bytecodes.c" PyObject *retval = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(retval); assert(EMPTY()); @@ -960,11 +961,11 @@ _PyEvalFrameClearAndPop(tstate, dying); _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 964 "Python/generated_cases.c.h" + #line 965 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_RETURN_CONST) { - #line 628 "Python/bytecodes.c" + #line 629 "Python/bytecodes.c" PyObject *retval = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(retval); int err = _Py_call_instrumentation_arg( @@ -981,13 +982,13 @@ _PyEvalFrameClearAndPop(tstate, dying); _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 985 "Python/generated_cases.c.h" + #line 986 "Python/generated_cases.c.h" } TARGET(GET_AITER) { PyObject *obj = stack_pointer[-1]; PyObject *iter; - #line 647 "Python/bytecodes.c" + #line 648 "Python/bytecodes.c" unaryfunc getter = NULL; PyTypeObject *type = Py_TYPE(obj); @@ -1000,16 +1001,16 @@ "'async for' requires an object with " "__aiter__ method, got %.100s", type->tp_name); - #line 1004 "Python/generated_cases.c.h" + #line 1005 "Python/generated_cases.c.h" Py_DECREF(obj); - #line 660 "Python/bytecodes.c" + #line 661 "Python/bytecodes.c" if (true) goto pop_1_error; } iter = (*getter)(obj); - #line 1011 "Python/generated_cases.c.h" + #line 1012 "Python/generated_cases.c.h" Py_DECREF(obj); - #line 665 "Python/bytecodes.c" + #line 666 "Python/bytecodes.c" if (iter == NULL) goto pop_1_error; if (Py_TYPE(iter)->tp_as_async == NULL || @@ -1022,7 +1023,7 @@ Py_DECREF(iter); if (true) goto pop_1_error; } - #line 1026 "Python/generated_cases.c.h" + #line 1027 "Python/generated_cases.c.h" stack_pointer[-1] = iter; DISPATCH(); } @@ -1030,7 +1031,7 @@ TARGET(GET_ANEXT) { PyObject *aiter = stack_pointer[-1]; PyObject *awaitable; - #line 680 "Python/bytecodes.c" + #line 681 "Python/bytecodes.c" unaryfunc getter = NULL; PyObject *next_iter = NULL; PyTypeObject *type = Py_TYPE(aiter); @@ -1074,7 +1075,7 @@ } } - #line 1078 "Python/generated_cases.c.h" + #line 1079 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = awaitable; PREDICT(LOAD_CONST); @@ -1085,16 +1086,16 @@ PREDICTED(GET_AWAITABLE); PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 727 "Python/bytecodes.c" + #line 728 "Python/bytecodes.c" iter = _PyCoro_GetAwaitableIter(iterable); if (iter == NULL) { format_awaitable_error(tstate, Py_TYPE(iterable), oparg); } - #line 1096 "Python/generated_cases.c.h" + #line 1097 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 734 "Python/bytecodes.c" + #line 735 "Python/bytecodes.c" if (iter != NULL && PyCoro_CheckExact(iter)) { PyObject *yf = _PyGen_yf((PyGenObject*)iter); @@ -1112,7 +1113,7 @@ if (iter == NULL) goto pop_1_error; - #line 1116 "Python/generated_cases.c.h" + #line 1117 "Python/generated_cases.c.h" stack_pointer[-1] = iter; PREDICT(LOAD_CONST); DISPATCH(); @@ -1123,7 +1124,7 @@ PyObject *v = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; PyObject *retval; - #line 760 "Python/bytecodes.c" + #line 761 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PySendCache *cache = (_PySendCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1169,7 +1170,7 @@ } } Py_DECREF(v); - #line 1173 "Python/generated_cases.c.h" + #line 1174 "Python/generated_cases.c.h" stack_pointer[-1] = retval; next_instr += 1; DISPATCH(); @@ -1178,7 +1179,7 @@ TARGET(SEND_GEN) { PyObject *v = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; - #line 808 "Python/bytecodes.c" + #line 809 "Python/bytecodes.c" PyGenObject *gen = (PyGenObject *)receiver; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); @@ -1193,12 +1194,12 @@ tstate->exc_info = &gen->gi_exc_state; JUMPBY(INLINE_CACHE_ENTRIES_SEND + oparg); DISPATCH_INLINED(gen_frame); - #line 1197 "Python/generated_cases.c.h" + #line 1198 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_YIELD_VALUE) { PyObject *retval = stack_pointer[-1]; - #line 825 "Python/bytecodes.c" + #line 826 "Python/bytecodes.c" assert(frame != &entry_frame); PyGenObject *gen = _PyFrame_GetGenerator(frame); gen->gi_frame_state = FRAME_SUSPENDED; @@ -1216,12 +1217,12 @@ frame->prev_instr -= frame->yield_offset; _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 1220 "Python/generated_cases.c.h" + #line 1221 "Python/generated_cases.c.h" } TARGET(YIELD_VALUE) { PyObject *retval = stack_pointer[-1]; - #line 845 "Python/bytecodes.c" + #line 846 "Python/bytecodes.c" // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. @@ -1238,15 +1239,15 @@ frame->prev_instr -= frame->yield_offset; _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 1242 "Python/generated_cases.c.h" + #line 1243 "Python/generated_cases.c.h" } TARGET(POP_EXCEPT) { PyObject *exc_value = stack_pointer[-1]; - #line 864 "Python/bytecodes.c" + #line 865 "Python/bytecodes.c" _PyErr_StackItem *exc_info = tstate->exc_info; Py_XSETREF(exc_info->exc_value, exc_value); - #line 1250 "Python/generated_cases.c.h" + #line 1251 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -1254,7 +1255,7 @@ TARGET(RERAISE) { PyObject *exc = stack_pointer[-1]; PyObject **values = (stack_pointer - (1 + oparg)); - #line 869 "Python/bytecodes.c" + #line 870 "Python/bytecodes.c" assert(oparg >= 0 && oparg <= 2); if (oparg) { PyObject *lasti = values[0]; @@ -1272,26 +1273,26 @@ Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); goto exception_unwind; - #line 1276 "Python/generated_cases.c.h" + #line 1277 "Python/generated_cases.c.h" } TARGET(END_ASYNC_FOR) { PyObject *exc = stack_pointer[-1]; PyObject *awaitable = stack_pointer[-2]; - #line 889 "Python/bytecodes.c" + #line 890 "Python/bytecodes.c" assert(exc && PyExceptionInstance_Check(exc)); if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { - #line 1285 "Python/generated_cases.c.h" + #line 1286 "Python/generated_cases.c.h" Py_DECREF(awaitable); Py_DECREF(exc); - #line 892 "Python/bytecodes.c" + #line 893 "Python/bytecodes.c" } else { Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); goto exception_unwind; } - #line 1295 "Python/generated_cases.c.h" + #line 1296 "Python/generated_cases.c.h" STACK_SHRINK(2); DISPATCH(); } @@ -1302,23 +1303,23 @@ PyObject *sub_iter = stack_pointer[-3]; PyObject *none; PyObject *value; - #line 901 "Python/bytecodes.c" + #line 902 "Python/bytecodes.c" assert(throwflag); assert(exc_value && PyExceptionInstance_Check(exc_value)); if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { value = Py_NewRef(((PyStopIterationObject *)exc_value)->value); - #line 1311 "Python/generated_cases.c.h" + #line 1312 "Python/generated_cases.c.h" Py_DECREF(sub_iter); Py_DECREF(last_sent_val); Py_DECREF(exc_value); - #line 906 "Python/bytecodes.c" + #line 907 "Python/bytecodes.c" none = Py_NewRef(Py_None); } else { _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); goto exception_unwind; } - #line 1322 "Python/generated_cases.c.h" + #line 1323 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = value; stack_pointer[-2] = none; @@ -1327,9 +1328,9 @@ TARGET(LOAD_ASSERTION_ERROR) { PyObject *value; - #line 915 "Python/bytecodes.c" + #line 916 "Python/bytecodes.c" value = Py_NewRef(PyExc_AssertionError); - #line 1333 "Python/generated_cases.c.h" + #line 1334 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -1337,7 +1338,7 @@ TARGET(LOAD_BUILD_CLASS) { PyObject *bc; - #line 919 "Python/bytecodes.c" + #line 920 "Python/bytecodes.c" if (PyDict_CheckExact(BUILTINS())) { bc = _PyDict_GetItemWithError(BUILTINS(), &_Py_ID(__build_class__)); @@ -1359,7 +1360,7 @@ if (true) goto error; } } - #line 1363 "Python/generated_cases.c.h" + #line 1364 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = bc; DISPATCH(); @@ -1367,33 +1368,33 @@ TARGET(STORE_NAME) { PyObject *v = stack_pointer[-1]; - #line 943 "Python/bytecodes.c" + #line 944 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); PyObject *ns = LOCALS(); int err; if (ns == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals found when storing %R", name); - #line 1378 "Python/generated_cases.c.h" + #line 1379 "Python/generated_cases.c.h" Py_DECREF(v); - #line 950 "Python/bytecodes.c" + #line 951 "Python/bytecodes.c" if (true) goto pop_1_error; } if (PyDict_CheckExact(ns)) err = PyDict_SetItem(ns, name, v); else err = PyObject_SetItem(ns, name, v); - #line 1387 "Python/generated_cases.c.h" + #line 1388 "Python/generated_cases.c.h" Py_DECREF(v); - #line 957 "Python/bytecodes.c" + #line 958 "Python/bytecodes.c" if (err) goto pop_1_error; - #line 1391 "Python/generated_cases.c.h" + #line 1392 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(DELETE_NAME) { - #line 961 "Python/bytecodes.c" + #line 962 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); PyObject *ns = LOCALS(); int err; @@ -1410,7 +1411,7 @@ name); goto error; } - #line 1414 "Python/generated_cases.c.h" + #line 1415 "Python/generated_cases.c.h" DISPATCH(); } @@ -1418,7 +1419,7 @@ PREDICTED(UNPACK_SEQUENCE); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); PyObject *seq = stack_pointer[-1]; - #line 987 "Python/bytecodes.c" + #line 988 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1431,11 +1432,11 @@ #endif /* ENABLE_SPECIALIZATION */ PyObject **top = stack_pointer + oparg - 1; int res = unpack_iterable(tstate, seq, oparg, -1, top); - #line 1435 "Python/generated_cases.c.h" + #line 1436 "Python/generated_cases.c.h" Py_DECREF(seq); - #line 1000 "Python/bytecodes.c" + #line 1001 "Python/bytecodes.c" if (res == 0) goto pop_1_error; - #line 1439 "Python/generated_cases.c.h" + #line 1440 "Python/generated_cases.c.h" STACK_SHRINK(1); STACK_GROW(oparg); next_instr += 1; @@ -1445,14 +1446,14 @@ TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { PyObject *seq = stack_pointer[-1]; PyObject **values = stack_pointer - (1); - #line 1004 "Python/bytecodes.c" + #line 1005 "Python/bytecodes.c" DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); assert(oparg == 2); STAT_INC(UNPACK_SEQUENCE, hit); values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); - #line 1456 "Python/generated_cases.c.h" + #line 1457 "Python/generated_cases.c.h" Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); @@ -1463,7 +1464,7 @@ TARGET(UNPACK_SEQUENCE_TUPLE) { PyObject *seq = stack_pointer[-1]; PyObject **values = stack_pointer - (1); - #line 1014 "Python/bytecodes.c" + #line 1015 "Python/bytecodes.c" DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); @@ -1471,7 +1472,7 @@ for (int i = oparg; --i >= 0; ) { *values++ = Py_NewRef(items[i]); } - #line 1475 "Python/generated_cases.c.h" + #line 1476 "Python/generated_cases.c.h" Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); @@ -1482,7 +1483,7 @@ TARGET(UNPACK_SEQUENCE_LIST) { PyObject *seq = stack_pointer[-1]; PyObject **values = stack_pointer - (1); - #line 1025 "Python/bytecodes.c" + #line 1026 "Python/bytecodes.c" DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); @@ -1490,7 +1491,7 @@ for (int i = oparg; --i >= 0; ) { *values++ = Py_NewRef(items[i]); } - #line 1494 "Python/generated_cases.c.h" + #line 1495 "Python/generated_cases.c.h" Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); @@ -1500,15 +1501,15 @@ TARGET(UNPACK_EX) { PyObject *seq = stack_pointer[-1]; - #line 1036 "Python/bytecodes.c" + #line 1037 "Python/bytecodes.c" int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); PyObject **top = stack_pointer + totalargs - 1; int res = unpack_iterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); - #line 1508 "Python/generated_cases.c.h" + #line 1509 "Python/generated_cases.c.h" Py_DECREF(seq); - #line 1040 "Python/bytecodes.c" + #line 1041 "Python/bytecodes.c" if (res == 0) goto pop_1_error; - #line 1512 "Python/generated_cases.c.h" + #line 1513 "Python/generated_cases.c.h" STACK_GROW((oparg & 0xFF) + (oparg >> 8)); DISPATCH(); } @@ -1519,7 +1520,7 @@ PyObject *owner = stack_pointer[-1]; PyObject *v = stack_pointer[-2]; uint16_t counter = read_u16(&next_instr[0].cache); - #line 1051 "Python/bytecodes.c" + #line 1052 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { PyObject *name = GETITEM(frame->f_code->co_names, oparg); @@ -1535,12 +1536,12 @@ #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyObject_SetAttr(owner, name, v); - #line 1539 "Python/generated_cases.c.h" + #line 1540 "Python/generated_cases.c.h" Py_DECREF(v); Py_DECREF(owner); - #line 1067 "Python/bytecodes.c" + #line 1068 "Python/bytecodes.c" if (err) goto pop_2_error; - #line 1544 "Python/generated_cases.c.h" + #line 1545 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -1548,34 +1549,34 @@ TARGET(DELETE_ATTR) { PyObject *owner = stack_pointer[-1]; - #line 1071 "Python/bytecodes.c" + #line 1072 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyObject_SetAttr(owner, name, (PyObject *)NULL); - #line 1555 "Python/generated_cases.c.h" + #line 1556 "Python/generated_cases.c.h" Py_DECREF(owner); - #line 1074 "Python/bytecodes.c" + #line 1075 "Python/bytecodes.c" if (err) goto pop_1_error; - #line 1559 "Python/generated_cases.c.h" + #line 1560 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(STORE_GLOBAL) { PyObject *v = stack_pointer[-1]; - #line 1078 "Python/bytecodes.c" + #line 1079 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyDict_SetItem(GLOBALS(), name, v); - #line 1569 "Python/generated_cases.c.h" + #line 1570 "Python/generated_cases.c.h" Py_DECREF(v); - #line 1081 "Python/bytecodes.c" + #line 1082 "Python/bytecodes.c" if (err) goto pop_1_error; - #line 1573 "Python/generated_cases.c.h" + #line 1574 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(DELETE_GLOBAL) { - #line 1085 "Python/bytecodes.c" + #line 1086 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err; err = PyDict_DelItem(GLOBALS(), name); @@ -1587,13 +1588,13 @@ } goto error; } - #line 1591 "Python/generated_cases.c.h" + #line 1592 "Python/generated_cases.c.h" DISPATCH(); } TARGET(LOAD_NAME) { PyObject *v; - #line 1099 "Python/bytecodes.c" + #line 1100 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); PyObject *locals = LOCALS(); if (locals == NULL) { @@ -1652,7 +1653,7 @@ } } } - #line 1656 "Python/generated_cases.c.h" + #line 1657 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = v; DISPATCH(); @@ -1663,7 +1664,7 @@ static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyObject *null = NULL; PyObject *v; - #line 1166 "Python/bytecodes.c" + #line 1167 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1715,7 +1716,7 @@ } } null = NULL; - #line 1719 "Python/generated_cases.c.h" + #line 1720 "Python/generated_cases.c.h" STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = v; @@ -1729,7 +1730,7 @@ PyObject *res; uint16_t index = read_u16(&next_instr[1].cache); uint16_t version = read_u16(&next_instr[2].cache); - #line 1220 "Python/bytecodes.c" + #line 1221 "Python/bytecodes.c" DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); PyDictObject *dict = (PyDictObject *)GLOBALS(); DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); @@ -1740,7 +1741,7 @@ Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; - #line 1744 "Python/generated_cases.c.h" + #line 1745 "Python/generated_cases.c.h" STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -1755,7 +1756,7 @@ uint16_t index = read_u16(&next_instr[1].cache); uint16_t mod_version = read_u16(&next_instr[2].cache); uint16_t bltn_version = read_u16(&next_instr[3].cache); - #line 1233 "Python/bytecodes.c" + #line 1234 "Python/bytecodes.c" DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); DEOPT_IF(!PyDict_CheckExact(BUILTINS()), LOAD_GLOBAL); PyDictObject *mdict = (PyDictObject *)GLOBALS(); @@ -1769,7 +1770,7 @@ Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; - #line 1773 "Python/generated_cases.c.h" + #line 1774 "Python/generated_cases.c.h" STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -1779,16 +1780,16 @@ } TARGET(DELETE_FAST) { - #line 1249 "Python/bytecodes.c" + #line 1250 "Python/bytecodes.c" PyObject *v = GETLOCAL(oparg); if (v == NULL) goto unbound_local_error; SETLOCAL(oparg, NULL); - #line 1787 "Python/generated_cases.c.h" + #line 1788 "Python/generated_cases.c.h" DISPATCH(); } TARGET(MAKE_CELL) { - #line 1255 "Python/bytecodes.c" + #line 1256 "Python/bytecodes.c" // "initial" is probably NULL but not if it's an arg (or set // via PyFrame_LocalsToFast() before MAKE_CELL has run). PyObject *initial = GETLOCAL(oparg); @@ -1797,12 +1798,12 @@ goto resume_with_error; } SETLOCAL(oparg, cell); - #line 1801 "Python/generated_cases.c.h" + #line 1802 "Python/generated_cases.c.h" DISPATCH(); } TARGET(DELETE_DEREF) { - #line 1266 "Python/bytecodes.c" + #line 1267 "Python/bytecodes.c" PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); // Can't use ERROR_IF here. @@ -1813,13 +1814,13 @@ } PyCell_SET(cell, NULL); Py_DECREF(oldobj); - #line 1817 "Python/generated_cases.c.h" + #line 1818 "Python/generated_cases.c.h" DISPATCH(); } TARGET(LOAD_CLASSDEREF) { PyObject *value; - #line 1279 "Python/bytecodes.c" + #line 1280 "Python/bytecodes.c" PyObject *name, *locals = LOCALS(); assert(locals); assert(oparg >= 0 && oparg < frame->f_code->co_nlocalsplus); @@ -1851,7 +1852,7 @@ } Py_INCREF(value); } - #line 1855 "Python/generated_cases.c.h" + #line 1856 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -1859,7 +1860,7 @@ TARGET(LOAD_DEREF) { PyObject *value; - #line 1313 "Python/bytecodes.c" + #line 1314 "Python/bytecodes.c" PyObject *cell = GETLOCAL(oparg); value = PyCell_GET(cell); if (value == NULL) { @@ -1867,7 +1868,7 @@ if (true) goto error; } Py_INCREF(value); - #line 1871 "Python/generated_cases.c.h" + #line 1872 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -1875,18 +1876,18 @@ TARGET(STORE_DEREF) { PyObject *v = stack_pointer[-1]; - #line 1323 "Python/bytecodes.c" + #line 1324 "Python/bytecodes.c" PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); PyCell_SET(cell, v); Py_XDECREF(oldobj); - #line 1884 "Python/generated_cases.c.h" + #line 1885 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(COPY_FREE_VARS) { - #line 1330 "Python/bytecodes.c" + #line 1331 "Python/bytecodes.c" /* Copy closure variables to free variables */ PyCodeObject *co = frame->f_code; assert(PyFunction_Check(frame->f_funcobj)); @@ -1897,22 +1898,22 @@ PyObject *o = PyTuple_GET_ITEM(closure, i); frame->localsplus[offset + i] = Py_NewRef(o); } - #line 1901 "Python/generated_cases.c.h" + #line 1902 "Python/generated_cases.c.h" DISPATCH(); } TARGET(BUILD_STRING) { PyObject **pieces = (stack_pointer - oparg); PyObject *str; - #line 1343 "Python/bytecodes.c" + #line 1344 "Python/bytecodes.c" str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); - #line 1910 "Python/generated_cases.c.h" + #line 1911 "Python/generated_cases.c.h" for (int _i = oparg; --_i >= 0;) { Py_DECREF(pieces[_i]); } - #line 1345 "Python/bytecodes.c" + #line 1346 "Python/bytecodes.c" if (str == NULL) { STACK_SHRINK(oparg); goto error; } - #line 1916 "Python/generated_cases.c.h" + #line 1917 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = str; @@ -1922,10 +1923,10 @@ TARGET(BUILD_TUPLE) { PyObject **values = (stack_pointer - oparg); PyObject *tup; - #line 1349 "Python/bytecodes.c" + #line 1350 "Python/bytecodes.c" tup = _PyTuple_FromArraySteal(values, oparg); if (tup == NULL) { STACK_SHRINK(oparg); goto error; } - #line 1929 "Python/generated_cases.c.h" + #line 1930 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = tup; @@ -1935,10 +1936,10 @@ TARGET(BUILD_LIST) { PyObject **values = (stack_pointer - oparg); PyObject *list; - #line 1354 "Python/bytecodes.c" + #line 1355 "Python/bytecodes.c" list = _PyList_FromArraySteal(values, oparg); if (list == NULL) { STACK_SHRINK(oparg); goto error; } - #line 1942 "Python/generated_cases.c.h" + #line 1943 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = list; @@ -1948,7 +1949,7 @@ TARGET(LIST_EXTEND) { PyObject *iterable = stack_pointer[-1]; PyObject *list = stack_pointer[-(2 + (oparg-1))]; - #line 1359 "Python/bytecodes.c" + #line 1360 "Python/bytecodes.c" PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); if (none_val == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && @@ -1959,13 +1960,13 @@ "Value after * must be an iterable, not %.200s", Py_TYPE(iterable)->tp_name); } - #line 1963 "Python/generated_cases.c.h" + #line 1964 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 1370 "Python/bytecodes.c" + #line 1371 "Python/bytecodes.c" if (true) goto pop_1_error; } Py_DECREF(none_val); - #line 1969 "Python/generated_cases.c.h" + #line 1970 "Python/generated_cases.c.h" Py_DECREF(iterable); STACK_SHRINK(1); DISPATCH(); @@ -1974,13 +1975,13 @@ TARGET(SET_UPDATE) { PyObject *iterable = stack_pointer[-1]; PyObject *set = stack_pointer[-(2 + (oparg-1))]; - #line 1377 "Python/bytecodes.c" + #line 1378 "Python/bytecodes.c" int err = _PySet_Update(set, iterable); - #line 1980 "Python/generated_cases.c.h" + #line 1981 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 1379 "Python/bytecodes.c" + #line 1380 "Python/bytecodes.c" if (err < 0) goto pop_1_error; - #line 1984 "Python/generated_cases.c.h" + #line 1985 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -1988,7 +1989,7 @@ TARGET(BUILD_SET) { PyObject **values = (stack_pointer - oparg); PyObject *set; - #line 1383 "Python/bytecodes.c" + #line 1384 "Python/bytecodes.c" set = PySet_New(NULL); if (set == NULL) goto error; @@ -2003,7 +2004,7 @@ Py_DECREF(set); if (true) { STACK_SHRINK(oparg); goto error; } } - #line 2007 "Python/generated_cases.c.h" + #line 2008 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = set; @@ -2013,7 +2014,7 @@ TARGET(BUILD_MAP) { PyObject **values = (stack_pointer - oparg*2); PyObject *map; - #line 1400 "Python/bytecodes.c" + #line 1401 "Python/bytecodes.c" map = _PyDict_FromItems( values, 2, values+1, 2, @@ -2021,13 +2022,13 @@ if (map == NULL) goto error; - #line 2025 "Python/generated_cases.c.h" + #line 2026 "Python/generated_cases.c.h" for (int _i = oparg*2; --_i >= 0;) { Py_DECREF(values[_i]); } - #line 1408 "Python/bytecodes.c" + #line 1409 "Python/bytecodes.c" if (map == NULL) { STACK_SHRINK(oparg*2); goto error; } - #line 2031 "Python/generated_cases.c.h" + #line 2032 "Python/generated_cases.c.h" STACK_SHRINK(oparg*2); STACK_GROW(1); stack_pointer[-1] = map; @@ -2035,7 +2036,7 @@ } TARGET(SETUP_ANNOTATIONS) { - #line 1412 "Python/bytecodes.c" + #line 1413 "Python/bytecodes.c" int err; PyObject *ann_dict; if (LOCALS() == NULL) { @@ -2075,7 +2076,7 @@ Py_DECREF(ann_dict); } } - #line 2079 "Python/generated_cases.c.h" + #line 2080 "Python/generated_cases.c.h" DISPATCH(); } @@ -2083,7 +2084,7 @@ PyObject *keys = stack_pointer[-1]; PyObject **values = (stack_pointer - (1 + oparg)); PyObject *map; - #line 1454 "Python/bytecodes.c" + #line 1455 "Python/bytecodes.c" if (!PyTuple_CheckExact(keys) || PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { _PyErr_SetString(tstate, PyExc_SystemError, @@ -2093,14 +2094,14 @@ map = _PyDict_FromItems( &PyTuple_GET_ITEM(keys, 0), 1, values, 1, oparg); - #line 2097 "Python/generated_cases.c.h" + #line 2098 "Python/generated_cases.c.h" for (int _i = oparg; --_i >= 0;) { Py_DECREF(values[_i]); } Py_DECREF(keys); - #line 1464 "Python/bytecodes.c" + #line 1465 "Python/bytecodes.c" if (map == NULL) { STACK_SHRINK(oparg); goto pop_1_error; } - #line 2104 "Python/generated_cases.c.h" + #line 2105 "Python/generated_cases.c.h" STACK_SHRINK(oparg); stack_pointer[-1] = map; DISPATCH(); @@ -2108,7 +2109,7 @@ TARGET(DICT_UPDATE) { PyObject *update = stack_pointer[-1]; - #line 1468 "Python/bytecodes.c" + #line 1469 "Python/bytecodes.c" PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (PyDict_Update(dict, update) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { @@ -2116,12 +2117,12 @@ "'%.200s' object is not a mapping", Py_TYPE(update)->tp_name); } - #line 2120 "Python/generated_cases.c.h" + #line 2121 "Python/generated_cases.c.h" Py_DECREF(update); - #line 1476 "Python/bytecodes.c" + #line 1477 "Python/bytecodes.c" if (true) goto pop_1_error; } - #line 2125 "Python/generated_cases.c.h" + #line 2126 "Python/generated_cases.c.h" Py_DECREF(update); STACK_SHRINK(1); DISPATCH(); @@ -2129,17 +2130,17 @@ TARGET(DICT_MERGE) { PyObject *update = stack_pointer[-1]; - #line 1482 "Python/bytecodes.c" + #line 1483 "Python/bytecodes.c" PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (_PyDict_MergeEx(dict, update, 2) < 0) { format_kwargs_error(tstate, PEEK(3 + oparg), update); - #line 2138 "Python/generated_cases.c.h" + #line 2139 "Python/generated_cases.c.h" Py_DECREF(update); - #line 1487 "Python/bytecodes.c" + #line 1488 "Python/bytecodes.c" if (true) goto pop_1_error; } - #line 2143 "Python/generated_cases.c.h" + #line 2144 "Python/generated_cases.c.h" Py_DECREF(update); STACK_SHRINK(1); PREDICT(CALL_FUNCTION_EX); @@ -2149,13 +2150,13 @@ TARGET(MAP_ADD) { PyObject *value = stack_pointer[-1]; PyObject *key = stack_pointer[-2]; - #line 1494 "Python/bytecodes.c" + #line 1495 "Python/bytecodes.c" PyObject *dict = PEEK(oparg + 2); // key, value are still on the stack assert(PyDict_CheckExact(dict)); /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; - #line 2159 "Python/generated_cases.c.h" + #line 2160 "Python/generated_cases.c.h" STACK_SHRINK(2); PREDICT(JUMP_BACKWARD); DISPATCH(); @@ -2167,7 +2168,7 @@ PyObject *owner = stack_pointer[-1]; PyObject *res2 = NULL; PyObject *res; - #line 1517 "Python/bytecodes.c" + #line 1518 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -2201,9 +2202,9 @@ NULL | meth | arg1 | ... | argN */ - #line 2205 "Python/generated_cases.c.h" + #line 2206 "Python/generated_cases.c.h" Py_DECREF(owner); - #line 1551 "Python/bytecodes.c" + #line 1552 "Python/bytecodes.c" if (meth == NULL) goto pop_1_error; res2 = NULL; res = meth; @@ -2212,12 +2213,12 @@ else { /* Classic, pushes one value. */ res = PyObject_GetAttr(owner, name); - #line 2216 "Python/generated_cases.c.h" + #line 2217 "Python/generated_cases.c.h" Py_DECREF(owner); - #line 1560 "Python/bytecodes.c" + #line 1561 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; } - #line 2221 "Python/generated_cases.c.h" + #line 2222 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -2231,7 +2232,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1565 "Python/bytecodes.c" + #line 1566 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2244,7 +2245,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2248 "Python/generated_cases.c.h" + #line 2249 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2259,7 +2260,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1581 "Python/bytecodes.c" + #line 1582 "Python/bytecodes.c" DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict != NULL); @@ -2272,7 +2273,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2276 "Python/generated_cases.c.h" + #line 2277 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2287,7 +2288,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1597 "Python/bytecodes.c" + #line 1598 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2314,7 +2315,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2318 "Python/generated_cases.c.h" + #line 2319 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2329,7 +2330,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1627 "Python/bytecodes.c" + #line 1628 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2339,7 +2340,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2343 "Python/generated_cases.c.h" + #line 2344 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2354,7 +2355,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 1640 "Python/bytecodes.c" + #line 1641 "Python/bytecodes.c" DEOPT_IF(!PyType_Check(cls), LOAD_ATTR); DEOPT_IF(((PyTypeObject *)cls)->tp_version_tag != type_version, @@ -2366,7 +2367,7 @@ res = descr; assert(res != NULL); Py_INCREF(res); - #line 2370 "Python/generated_cases.c.h" + #line 2371 "Python/generated_cases.c.h" Py_DECREF(cls); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2380,7 +2381,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t func_version = read_u32(&next_instr[3].cache); PyObject *fget = read_obj(&next_instr[5].cache); - #line 1655 "Python/bytecodes.c" + #line 1656 "Python/bytecodes.c" DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); @@ -2403,7 +2404,7 @@ new_frame->localsplus[0] = owner; JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); DISPATCH_INLINED(new_frame); - #line 2407 "Python/generated_cases.c.h" + #line 2408 "Python/generated_cases.c.h" } TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { @@ -2411,7 +2412,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t func_version = read_u32(&next_instr[3].cache); PyObject *getattribute = read_obj(&next_instr[5].cache); - #line 1680 "Python/bytecodes.c" + #line 1681 "Python/bytecodes.c" DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); @@ -2436,7 +2437,7 @@ new_frame->localsplus[1] = Py_NewRef(name); JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); DISPATCH_INLINED(new_frame); - #line 2440 "Python/generated_cases.c.h" + #line 2441 "Python/generated_cases.c.h" } TARGET(STORE_ATTR_INSTANCE_VALUE) { @@ -2444,7 +2445,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1707 "Python/bytecodes.c" + #line 1708 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2462,7 +2463,7 @@ Py_DECREF(old_value); } Py_DECREF(owner); - #line 2466 "Python/generated_cases.c.h" + #line 2467 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2473,7 +2474,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t hint = read_u16(&next_instr[3].cache); - #line 1727 "Python/bytecodes.c" + #line 1728 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2512,7 +2513,7 @@ /* PEP 509 */ dict->ma_version_tag = new_version; Py_DECREF(owner); - #line 2516 "Python/generated_cases.c.h" + #line 2517 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2523,7 +2524,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1768 "Python/bytecodes.c" + #line 1769 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2533,7 +2534,7 @@ *(PyObject **)addr = value; Py_XDECREF(old_value); Py_DECREF(owner); - #line 2537 "Python/generated_cases.c.h" + #line 2538 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2545,7 +2546,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1787 "Python/bytecodes.c" + #line 1788 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -2558,12 +2559,12 @@ #endif /* ENABLE_SPECIALIZATION */ assert((oparg >> 4) <= Py_GE); res = PyObject_RichCompare(left, right, oparg>>4); - #line 2562 "Python/generated_cases.c.h" + #line 2563 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 1800 "Python/bytecodes.c" + #line 1801 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 2567 "Python/generated_cases.c.h" + #line 2568 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2574,7 +2575,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1804 "Python/bytecodes.c" + #line 1805 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); @@ -2586,7 +2587,7 @@ _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); res = (sign_ish & oparg) ? Py_True : Py_False; Py_INCREF(res); - #line 2590 "Python/generated_cases.c.h" + #line 2591 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2597,7 +2598,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1819 "Python/bytecodes.c" + #line 1820 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); @@ -2613,7 +2614,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); res = (sign_ish & oparg) ? Py_True : Py_False; Py_INCREF(res); - #line 2617 "Python/generated_cases.c.h" + #line 2618 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2624,7 +2625,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1838 "Python/bytecodes.c" + #line 1839 "Python/bytecodes.c" DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); @@ -2637,7 +2638,7 @@ assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; Py_INCREF(res); - #line 2641 "Python/generated_cases.c.h" + #line 2642 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2648,14 +2649,14 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 1853 "Python/bytecodes.c" + #line 1854 "Python/bytecodes.c" int res = Py_Is(left, right) ^ oparg; - #line 2654 "Python/generated_cases.c.h" + #line 2655 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 1855 "Python/bytecodes.c" + #line 1856 "Python/bytecodes.c" b = Py_NewRef(res ? Py_True : Py_False); - #line 2659 "Python/generated_cases.c.h" + #line 2660 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = b; DISPATCH(); @@ -2665,15 +2666,15 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 1859 "Python/bytecodes.c" + #line 1860 "Python/bytecodes.c" int res = PySequence_Contains(right, left); - #line 2671 "Python/generated_cases.c.h" + #line 2672 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 1861 "Python/bytecodes.c" + #line 1862 "Python/bytecodes.c" if (res < 0) goto pop_2_error; b = Py_NewRef((res^oparg) ? Py_True : Py_False); - #line 2677 "Python/generated_cases.c.h" + #line 2678 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = b; DISPATCH(); @@ -2684,12 +2685,12 @@ PyObject *exc_value = stack_pointer[-2]; PyObject *rest; PyObject *match; - #line 1866 "Python/bytecodes.c" + #line 1867 "Python/bytecodes.c" if (check_except_star_type_valid(tstate, match_type) < 0) { - #line 2690 "Python/generated_cases.c.h" + #line 2691 "Python/generated_cases.c.h" Py_DECREF(exc_value); Py_DECREF(match_type); - #line 1868 "Python/bytecodes.c" + #line 1869 "Python/bytecodes.c" if (true) goto pop_2_error; } @@ -2697,10 +2698,10 @@ rest = NULL; int res = exception_group_match(exc_value, match_type, &match, &rest); - #line 2701 "Python/generated_cases.c.h" + #line 2702 "Python/generated_cases.c.h" Py_DECREF(exc_value); Py_DECREF(match_type); - #line 1876 "Python/bytecodes.c" + #line 1877 "Python/bytecodes.c" if (res < 0) goto pop_2_error; assert((match == NULL) == (rest == NULL)); @@ -2709,7 +2710,7 @@ if (!Py_IsNone(match)) { PyErr_SetExcInfo(NULL, Py_NewRef(match), NULL); } - #line 2713 "Python/generated_cases.c.h" + #line 2714 "Python/generated_cases.c.h" stack_pointer[-1] = match; stack_pointer[-2] = rest; DISPATCH(); @@ -2719,21 +2720,21 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 1887 "Python/bytecodes.c" + #line 1888 "Python/bytecodes.c" assert(PyExceptionInstance_Check(left)); if (check_except_type_valid(tstate, right) < 0) { - #line 2726 "Python/generated_cases.c.h" + #line 2727 "Python/generated_cases.c.h" Py_DECREF(right); - #line 1890 "Python/bytecodes.c" + #line 1891 "Python/bytecodes.c" if (true) goto pop_1_error; } int res = PyErr_GivenExceptionMatches(left, right); - #line 2733 "Python/generated_cases.c.h" + #line 2734 "Python/generated_cases.c.h" Py_DECREF(right); - #line 1895 "Python/bytecodes.c" + #line 1896 "Python/bytecodes.c" b = Py_NewRef(res ? Py_True : Py_False); - #line 2737 "Python/generated_cases.c.h" + #line 2738 "Python/generated_cases.c.h" stack_pointer[-1] = b; DISPATCH(); } @@ -2742,15 +2743,15 @@ PyObject *fromlist = stack_pointer[-1]; PyObject *level = stack_pointer[-2]; PyObject *res; - #line 1899 "Python/bytecodes.c" + #line 1900 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_name(tstate, frame, name, fromlist, level); - #line 2749 "Python/generated_cases.c.h" + #line 2750 "Python/generated_cases.c.h" Py_DECREF(level); Py_DECREF(fromlist); - #line 1902 "Python/bytecodes.c" + #line 1903 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 2754 "Python/generated_cases.c.h" + #line 2755 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); @@ -2759,29 +2760,29 @@ TARGET(IMPORT_FROM) { PyObject *from = stack_pointer[-1]; PyObject *res; - #line 1906 "Python/bytecodes.c" + #line 1907 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_from(tstate, from, name); if (res == NULL) goto error; - #line 2767 "Python/generated_cases.c.h" + #line 2768 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); } TARGET(JUMP_FORWARD) { - #line 1912 "Python/bytecodes.c" + #line 1913 "Python/bytecodes.c" JUMPBY(oparg); - #line 2776 "Python/generated_cases.c.h" + #line 2777 "Python/generated_cases.c.h" DISPATCH(); } TARGET(JUMP_BACKWARD) { PREDICTED(JUMP_BACKWARD); - #line 1916 "Python/bytecodes.c" + #line 1917 "Python/bytecodes.c" assert(oparg < INSTR_OFFSET()); JUMPBY(-oparg); - #line 2785 "Python/generated_cases.c.h" + #line 2786 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -2789,7 +2790,7 @@ TARGET(POP_JUMP_IF_FALSE) { PREDICTED(POP_JUMP_IF_FALSE); PyObject *cond = stack_pointer[-1]; - #line 1922 "Python/bytecodes.c" + #line 1923 "Python/bytecodes.c" if (Py_IsTrue(cond)) { _Py_DECREF_NO_DEALLOC(cond); } @@ -2799,9 +2800,9 @@ } else { int err = PyObject_IsTrue(cond); - #line 2803 "Python/generated_cases.c.h" + #line 2804 "Python/generated_cases.c.h" Py_DECREF(cond); - #line 1932 "Python/bytecodes.c" + #line 1933 "Python/bytecodes.c" if (err == 0) { JUMPBY(oparg); } @@ -2809,14 +2810,14 @@ if (err < 0) goto pop_1_error; } } - #line 2813 "Python/generated_cases.c.h" + #line 2814 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_TRUE) { PyObject *cond = stack_pointer[-1]; - #line 1942 "Python/bytecodes.c" + #line 1943 "Python/bytecodes.c" if (Py_IsFalse(cond)) { _Py_DECREF_NO_DEALLOC(cond); } @@ -2826,9 +2827,9 @@ } else { int err = PyObject_IsTrue(cond); - #line 2830 "Python/generated_cases.c.h" + #line 2831 "Python/generated_cases.c.h" Py_DECREF(cond); - #line 1952 "Python/bytecodes.c" + #line 1953 "Python/bytecodes.c" if (err > 0) { JUMPBY(oparg); } @@ -2836,67 +2837,67 @@ if (err < 0) goto pop_1_error; } } - #line 2840 "Python/generated_cases.c.h" + #line 2841 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_NOT_NONE) { PyObject *value = stack_pointer[-1]; - #line 1962 "Python/bytecodes.c" + #line 1963 "Python/bytecodes.c" if (!Py_IsNone(value)) { - #line 2849 "Python/generated_cases.c.h" + #line 2850 "Python/generated_cases.c.h" Py_DECREF(value); - #line 1964 "Python/bytecodes.c" + #line 1965 "Python/bytecodes.c" JUMPBY(oparg); } else { _Py_DECREF_NO_DEALLOC(value); } - #line 2857 "Python/generated_cases.c.h" + #line 2858 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_NONE) { PyObject *value = stack_pointer[-1]; - #line 1972 "Python/bytecodes.c" + #line 1973 "Python/bytecodes.c" if (Py_IsNone(value)) { _Py_DECREF_NO_DEALLOC(value); JUMPBY(oparg); } else { - #line 2870 "Python/generated_cases.c.h" + #line 2871 "Python/generated_cases.c.h" Py_DECREF(value); - #line 1978 "Python/bytecodes.c" + #line 1979 "Python/bytecodes.c" } - #line 2874 "Python/generated_cases.c.h" + #line 2875 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(JUMP_BACKWARD_NO_INTERRUPT) { - #line 1982 "Python/bytecodes.c" + #line 1983 "Python/bytecodes.c" /* This bytecode is used in the `yield from` or `await` loop. * If there is an interrupt, we want it handled in the innermost * generator or coroutine, so we deliberately do not check it here. * (see bpo-30039). */ JUMPBY(-oparg); - #line 2887 "Python/generated_cases.c.h" + #line 2888 "Python/generated_cases.c.h" DISPATCH(); } TARGET(GET_LEN) { PyObject *obj = stack_pointer[-1]; PyObject *len_o; - #line 1991 "Python/bytecodes.c" + #line 1992 "Python/bytecodes.c" // PUSH(len(TOS)) Py_ssize_t len_i = PyObject_Length(obj); if (len_i < 0) goto error; len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) goto error; - #line 2900 "Python/generated_cases.c.h" + #line 2901 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = len_o; DISPATCH(); @@ -2907,16 +2908,16 @@ PyObject *type = stack_pointer[-2]; PyObject *subject = stack_pointer[-3]; PyObject *attrs; - #line 1999 "Python/bytecodes.c" + #line 2000 "Python/bytecodes.c" // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. assert(PyTuple_CheckExact(names)); attrs = match_class(tstate, subject, type, oparg, names); - #line 2916 "Python/generated_cases.c.h" + #line 2917 "Python/generated_cases.c.h" Py_DECREF(subject); Py_DECREF(type); Py_DECREF(names); - #line 2004 "Python/bytecodes.c" + #line 2005 "Python/bytecodes.c" if (attrs) { assert(PyTuple_CheckExact(attrs)); // Success! } @@ -2924,7 +2925,7 @@ if (_PyErr_Occurred(tstate)) goto pop_3_error; attrs = Py_NewRef(Py_None); // Failure! } - #line 2928 "Python/generated_cases.c.h" + #line 2929 "Python/generated_cases.c.h" STACK_SHRINK(2); stack_pointer[-1] = attrs; DISPATCH(); @@ -2933,10 +2934,10 @@ TARGET(MATCH_MAPPING) { PyObject *subject = stack_pointer[-1]; PyObject *res; - #line 2014 "Python/bytecodes.c" + #line 2015 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; res = Py_NewRef(match ? Py_True : Py_False); - #line 2940 "Python/generated_cases.c.h" + #line 2941 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; PREDICT(POP_JUMP_IF_FALSE); @@ -2946,10 +2947,10 @@ TARGET(MATCH_SEQUENCE) { PyObject *subject = stack_pointer[-1]; PyObject *res; - #line 2020 "Python/bytecodes.c" + #line 2021 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; res = Py_NewRef(match ? Py_True : Py_False); - #line 2953 "Python/generated_cases.c.h" + #line 2954 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; PREDICT(POP_JUMP_IF_FALSE); @@ -2960,11 +2961,11 @@ PyObject *keys = stack_pointer[-1]; PyObject *subject = stack_pointer[-2]; PyObject *values_or_none; - #line 2026 "Python/bytecodes.c" + #line 2027 "Python/bytecodes.c" // On successful match, PUSH(values). Otherwise, PUSH(None). values_or_none = match_keys(tstate, subject, keys); if (values_or_none == NULL) goto error; - #line 2968 "Python/generated_cases.c.h" + #line 2969 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = values_or_none; DISPATCH(); @@ -2973,14 +2974,14 @@ TARGET(GET_ITER) { PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 2032 "Python/bytecodes.c" + #line 2033 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ iter = PyObject_GetIter(iterable); - #line 2980 "Python/generated_cases.c.h" + #line 2981 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 2035 "Python/bytecodes.c" + #line 2036 "Python/bytecodes.c" if (iter == NULL) goto pop_1_error; - #line 2984 "Python/generated_cases.c.h" + #line 2985 "Python/generated_cases.c.h" stack_pointer[-1] = iter; DISPATCH(); } @@ -2988,7 +2989,7 @@ TARGET(GET_YIELD_FROM_ITER) { PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 2039 "Python/bytecodes.c" + #line 2040 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ if (PyCoro_CheckExact(iterable)) { /* `iterable` is a coroutine */ @@ -3011,11 +3012,11 @@ if (iter == NULL) { goto error; } - #line 3015 "Python/generated_cases.c.h" + #line 3016 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 2062 "Python/bytecodes.c" + #line 2063 "Python/bytecodes.c" } - #line 3019 "Python/generated_cases.c.h" + #line 3020 "Python/generated_cases.c.h" stack_pointer[-1] = iter; PREDICT(LOAD_CONST); DISPATCH(); @@ -3026,7 +3027,7 @@ static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2081 "Python/bytecodes.c" + #line 2082 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyForIterCache *cache = (_PyForIterCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -3057,7 +3058,7 @@ DISPATCH(); } // Common case: no jump, leave it to the code generator - #line 3061 "Python/generated_cases.c.h" + #line 3062 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3065,7 +3066,7 @@ } TARGET(INSTRUMENTED_FOR_ITER) { - #line 2114 "Python/bytecodes.c" + #line 2115 "Python/bytecodes.c" _Py_CODEUNIT *here = next_instr-1; _Py_CODEUNIT *target; PyObject *iter = TOP(); @@ -3091,14 +3092,14 @@ target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1; } INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); - #line 3095 "Python/generated_cases.c.h" + #line 3096 "Python/generated_cases.c.h" DISPATCH(); } TARGET(FOR_ITER_LIST) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2142 "Python/bytecodes.c" + #line 2143 "Python/bytecodes.c" DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); _PyListIterObject *it = (_PyListIterObject *)iter; STAT_INC(FOR_ITER, hit); @@ -3118,7 +3119,7 @@ DISPATCH(); end_for_iter_list: // Common case: no jump, leave it to the code generator - #line 3122 "Python/generated_cases.c.h" + #line 3123 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3128,7 +3129,7 @@ TARGET(FOR_ITER_TUPLE) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2164 "Python/bytecodes.c" + #line 2165 "Python/bytecodes.c" _PyTupleIterObject *it = (_PyTupleIterObject *)iter; DEOPT_IF(Py_TYPE(it) != &PyTupleIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -3148,7 +3149,7 @@ DISPATCH(); end_for_iter_tuple: // Common case: no jump, leave it to the code generator - #line 3152 "Python/generated_cases.c.h" + #line 3153 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3158,7 +3159,7 @@ TARGET(FOR_ITER_RANGE) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2186 "Python/bytecodes.c" + #line 2187 "Python/bytecodes.c" _PyRangeIterObject *r = (_PyRangeIterObject *)iter; DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -3176,7 +3177,7 @@ if (next == NULL) { goto error; } - #line 3180 "Python/generated_cases.c.h" + #line 3181 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3185,7 +3186,7 @@ TARGET(FOR_ITER_GEN) { PyObject *iter = stack_pointer[-1]; - #line 2206 "Python/bytecodes.c" + #line 2207 "Python/bytecodes.c" PyGenObject *gen = (PyGenObject *)iter; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); @@ -3200,14 +3201,14 @@ assert(next_instr->op.code == END_FOR || next_instr->op.code == INSTRUMENTED_END_FOR); DISPATCH_INLINED(gen_frame); - #line 3204 "Python/generated_cases.c.h" + #line 3205 "Python/generated_cases.c.h" } TARGET(BEFORE_ASYNC_WITH) { PyObject *mgr = stack_pointer[-1]; PyObject *exit; PyObject *res; - #line 2223 "Python/bytecodes.c" + #line 2224 "Python/bytecodes.c" PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); if (enter == NULL) { if (!_PyErr_Occurred(tstate)) { @@ -3230,16 +3231,16 @@ Py_DECREF(enter); goto error; } - #line 3234 "Python/generated_cases.c.h" + #line 3235 "Python/generated_cases.c.h" Py_DECREF(mgr); - #line 2246 "Python/bytecodes.c" + #line 2247 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); if (true) goto pop_1_error; } - #line 3243 "Python/generated_cases.c.h" + #line 3244 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -3251,7 +3252,7 @@ PyObject *mgr = stack_pointer[-1]; PyObject *exit; PyObject *res; - #line 2256 "Python/bytecodes.c" + #line 2257 "Python/bytecodes.c" /* pop the context manager, push its __exit__ and the * value returned from calling its __enter__ */ @@ -3277,16 +3278,16 @@ Py_DECREF(enter); goto error; } - #line 3281 "Python/generated_cases.c.h" + #line 3282 "Python/generated_cases.c.h" Py_DECREF(mgr); - #line 2282 "Python/bytecodes.c" + #line 2283 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); if (true) goto pop_1_error; } - #line 3290 "Python/generated_cases.c.h" + #line 3291 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -3298,7 +3299,7 @@ PyObject *lasti = stack_pointer[-3]; PyObject *exit_func = stack_pointer[-4]; PyObject *res; - #line 2291 "Python/bytecodes.c" + #line 2292 "Python/bytecodes.c" /* At the top of the stack are 4 values: - val: TOP = exc_info() - unused: SECOND = previous exception @@ -3319,7 +3320,7 @@ res = PyObject_Vectorcall(exit_func, stack + 1, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); if (res == NULL) goto error; - #line 3323 "Python/generated_cases.c.h" + #line 3324 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); @@ -3328,7 +3329,7 @@ TARGET(PUSH_EXC_INFO) { PyObject *new_exc = stack_pointer[-1]; PyObject *prev_exc; - #line 2314 "Python/bytecodes.c" + #line 2315 "Python/bytecodes.c" _PyErr_StackItem *exc_info = tstate->exc_info; if (exc_info->exc_value != NULL) { prev_exc = exc_info->exc_value; @@ -3338,7 +3339,7 @@ } assert(PyExceptionInstance_Check(new_exc)); exc_info->exc_value = Py_NewRef(new_exc); - #line 3342 "Python/generated_cases.c.h" + #line 3343 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = new_exc; stack_pointer[-2] = prev_exc; @@ -3352,7 +3353,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t keys_version = read_u32(&next_instr[3].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2326 "Python/bytecodes.c" + #line 2327 "Python/bytecodes.c" /* Cached method object */ PyTypeObject *self_cls = Py_TYPE(self); assert(type_version != 0); @@ -3369,7 +3370,7 @@ assert(_PyType_HasFeature(Py_TYPE(res2), Py_TPFLAGS_METHOD_DESCRIPTOR)); res = self; assert(oparg & 1); - #line 3373 "Python/generated_cases.c.h" + #line 3374 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3383,7 +3384,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2345 "Python/bytecodes.c" + #line 2346 "Python/bytecodes.c" PyTypeObject *self_cls = Py_TYPE(self); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); assert(self_cls->tp_dictoffset == 0); @@ -3393,7 +3394,7 @@ res2 = Py_NewRef(descr); res = self; assert(oparg & 1); - #line 3397 "Python/generated_cases.c.h" + #line 3398 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3407,7 +3408,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2357 "Python/bytecodes.c" + #line 2358 "Python/bytecodes.c" PyTypeObject *self_cls = Py_TYPE(self); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); Py_ssize_t dictoffset = self_cls->tp_dictoffset; @@ -3421,7 +3422,7 @@ res2 = Py_NewRef(descr); res = self; assert(oparg & 1); - #line 3425 "Python/generated_cases.c.h" + #line 3426 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3430,16 +3431,16 @@ } TARGET(KW_NAMES) { - #line 2373 "Python/bytecodes.c" + #line 2374 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg < PyTuple_GET_SIZE(frame->f_code->co_consts)); kwnames = GETITEM(frame->f_code->co_consts, oparg); - #line 3438 "Python/generated_cases.c.h" + #line 3439 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_CALL) { - #line 2379 "Python/bytecodes.c" + #line 2380 "Python/bytecodes.c" int is_meth = PEEK(oparg+2) != NULL; int total_args = oparg + is_meth; PyObject *function = PEEK(total_args + 1); @@ -3452,7 +3453,7 @@ _PyCallCache *cache = (_PyCallCache *)next_instr; INCREMENT_ADAPTIVE_COUNTER(cache->counter); GO_TO_INSTRUCTION(CALL); - #line 3456 "Python/generated_cases.c.h" + #line 3457 "Python/generated_cases.c.h" } TARGET(CALL) { @@ -3462,7 +3463,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2424 "Python/bytecodes.c" + #line 2425 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3543,7 +3544,7 @@ Py_DECREF(args[i]); } if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3547 "Python/generated_cases.c.h" + #line 3548 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3555,7 +3556,7 @@ TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; - #line 2511 "Python/bytecodes.c" + #line 2512 "Python/bytecodes.c" DEOPT_IF(method != NULL, CALL); DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); STAT_INC(CALL, hit); @@ -3565,7 +3566,7 @@ PEEK(oparg + 2) = Py_NewRef(meth); // method Py_DECREF(callable); GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); - #line 3569 "Python/generated_cases.c.h" + #line 3570 "Python/generated_cases.c.h" } TARGET(CALL_PY_EXACT_ARGS) { @@ -3574,7 +3575,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); - #line 2523 "Python/bytecodes.c" + #line 2524 "Python/bytecodes.c" assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; @@ -3599,7 +3600,7 @@ STACK_SHRINK(oparg + 2); JUMPBY(INLINE_CACHE_ENTRIES_CALL); DISPATCH_INLINED(new_frame); - #line 3603 "Python/generated_cases.c.h" + #line 3604 "Python/generated_cases.c.h" } TARGET(CALL_PY_WITH_DEFAULTS) { @@ -3608,7 +3609,7 @@ PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); uint16_t min_args = read_u16(&next_instr[3].cache); - #line 2550 "Python/bytecodes.c" + #line 2551 "Python/bytecodes.c" assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; @@ -3638,7 +3639,7 @@ STACK_SHRINK(oparg + 2); JUMPBY(INLINE_CACHE_ENTRIES_CALL); DISPATCH_INLINED(new_frame); - #line 3642 "Python/generated_cases.c.h" + #line 3643 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_TYPE_1) { @@ -3646,7 +3647,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2582 "Python/bytecodes.c" + #line 2583 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3656,7 +3657,7 @@ res = Py_NewRef(Py_TYPE(obj)); Py_DECREF(obj); Py_DECREF(&PyType_Type); // I.e., callable - #line 3660 "Python/generated_cases.c.h" + #line 3661 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3669,7 +3670,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2594 "Python/bytecodes.c" + #line 2595 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3680,7 +3681,7 @@ Py_DECREF(arg); Py_DECREF(&PyUnicode_Type); // I.e., callable if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3684 "Python/generated_cases.c.h" + #line 3685 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3694,7 +3695,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2608 "Python/bytecodes.c" + #line 2609 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3705,7 +3706,7 @@ Py_DECREF(arg); Py_DECREF(&PyTuple_Type); // I.e., tuple if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3709 "Python/generated_cases.c.h" + #line 3710 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3719,7 +3720,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2622 "Python/bytecodes.c" + #line 2623 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3741,7 +3742,7 @@ } Py_DECREF(tp); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3745 "Python/generated_cases.c.h" + #line 3746 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3755,7 +3756,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2647 "Python/bytecodes.c" + #line 2648 "Python/bytecodes.c" /* Builtin METH_O functions */ assert(kwnames == NULL); int is_meth = method != NULL; @@ -3783,7 +3784,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3787 "Python/generated_cases.c.h" + #line 3788 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3797,7 +3798,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2678 "Python/bytecodes.c" + #line 2679 "Python/bytecodes.c" /* Builtin METH_FASTCALL functions, without keywords */ assert(kwnames == NULL); int is_meth = method != NULL; @@ -3829,7 +3830,7 @@ 'invalid'). In those cases an exception is set, so we must handle it. */ - #line 3833 "Python/generated_cases.c.h" + #line 3834 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3843,7 +3844,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2713 "Python/bytecodes.c" + #line 2714 "Python/bytecodes.c" /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ int is_meth = method != NULL; int total_args = oparg; @@ -3875,7 +3876,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3879 "Python/generated_cases.c.h" + #line 3880 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3889,7 +3890,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2748 "Python/bytecodes.c" + #line 2749 "Python/bytecodes.c" assert(kwnames == NULL); /* len(o) */ int is_meth = method != NULL; @@ -3914,7 +3915,7 @@ Py_DECREF(callable); Py_DECREF(arg); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3918 "Python/generated_cases.c.h" + #line 3919 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3927,7 +3928,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2775 "Python/bytecodes.c" + #line 2776 "Python/bytecodes.c" assert(kwnames == NULL); /* isinstance(o, o2) */ int is_meth = method != NULL; @@ -3954,7 +3955,7 @@ Py_DECREF(cls); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3958 "Python/generated_cases.c.h" + #line 3959 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3966,7 +3967,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *self = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; - #line 2805 "Python/bytecodes.c" + #line 2806 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); assert(method != NULL); @@ -3984,14 +3985,14 @@ JUMPBY(INLINE_CACHE_ENTRIES_CALL + 1); assert(next_instr[-1].op.code == POP_TOP); DISPATCH(); - #line 3988 "Python/generated_cases.c.h" + #line 3989 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) { PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2825 "Python/bytecodes.c" + #line 2826 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -4022,7 +4023,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4026 "Python/generated_cases.c.h" + #line 4027 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4035,7 +4036,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2859 "Python/bytecodes.c" + #line 2860 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -4064,7 +4065,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4068 "Python/generated_cases.c.h" + #line 4069 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4077,7 +4078,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2891 "Python/bytecodes.c" + #line 2892 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 0 || oparg == 1); int is_meth = method != NULL; @@ -4106,7 +4107,7 @@ Py_DECREF(self); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4110 "Python/generated_cases.c.h" + #line 4111 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4119,7 +4120,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2923 "Python/bytecodes.c" + #line 2924 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -4147,7 +4148,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4151 "Python/generated_cases.c.h" + #line 4152 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4157,9 +4158,9 @@ } TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { - #line 2954 "Python/bytecodes.c" + #line 2955 "Python/bytecodes.c" GO_TO_INSTRUCTION(CALL_FUNCTION_EX); - #line 4163 "Python/generated_cases.c.h" + #line 4164 "Python/generated_cases.c.h" } TARGET(CALL_FUNCTION_EX) { @@ -4168,7 +4169,7 @@ PyObject *callargs = stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))]; PyObject *func = stack_pointer[-(2 + ((oparg & 1) ? 1 : 0))]; PyObject *result; - #line 2958 "Python/bytecodes.c" + #line 2959 "Python/bytecodes.c" // DICT_MERGE is called before this opcode if there are kwargs. // It converts all dict subtypes in kwargs into regular dicts. assert(kwargs == NULL || PyDict_CheckExact(kwargs)); @@ -4211,14 +4212,14 @@ else { result = PyObject_Call(func, callargs, kwargs); } - #line 4215 "Python/generated_cases.c.h" + #line 4216 "Python/generated_cases.c.h" Py_DECREF(func); Py_DECREF(callargs); Py_XDECREF(kwargs); - #line 3001 "Python/bytecodes.c" + #line 3002 "Python/bytecodes.c" assert(PEEK(3 + (oparg & 1)) == NULL); if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } - #line 4222 "Python/generated_cases.c.h" + #line 4223 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 1) ? 1 : 0)); STACK_SHRINK(2); stack_pointer[-1] = result; @@ -4233,7 +4234,7 @@ PyObject *kwdefaults = (oparg & 0x02) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0))] : NULL; PyObject *defaults = (oparg & 0x01) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x01) ? 1 : 0))] : NULL; PyObject *func; - #line 3011 "Python/bytecodes.c" + #line 3012 "Python/bytecodes.c" PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); @@ -4262,14 +4263,14 @@ func_obj->func_version = ((PyCodeObject *)codeobj)->co_version; func = (PyObject *)func_obj; - #line 4266 "Python/generated_cases.c.h" + #line 4267 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 0x01) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x08) ? 1 : 0)); stack_pointer[-1] = func; DISPATCH(); } TARGET(RETURN_GENERATOR) { - #line 3042 "Python/bytecodes.c" + #line 3043 "Python/bytecodes.c" assert(PyFunction_Check(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); @@ -4290,7 +4291,7 @@ frame = cframe.current_frame = prev; _PyFrame_StackPush(frame, (PyObject *)gen); goto resume_frame; - #line 4294 "Python/generated_cases.c.h" + #line 4295 "Python/generated_cases.c.h" } TARGET(BUILD_SLICE) { @@ -4298,15 +4299,15 @@ PyObject *stop = stack_pointer[-(1 + ((oparg == 3) ? 1 : 0))]; PyObject *start = stack_pointer[-(2 + ((oparg == 3) ? 1 : 0))]; PyObject *slice; - #line 3065 "Python/bytecodes.c" + #line 3066 "Python/bytecodes.c" slice = PySlice_New(start, stop, step); - #line 4304 "Python/generated_cases.c.h" + #line 4305 "Python/generated_cases.c.h" Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); - #line 3067 "Python/bytecodes.c" + #line 3068 "Python/bytecodes.c" if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } - #line 4310 "Python/generated_cases.c.h" + #line 4311 "Python/generated_cases.c.h" STACK_SHRINK(((oparg == 3) ? 1 : 0)); STACK_SHRINK(1); stack_pointer[-1] = slice; @@ -4317,7 +4318,7 @@ PyObject *fmt_spec = ((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? stack_pointer[-((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))] : NULL; PyObject *value = stack_pointer[-(1 + (((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))]; PyObject *result; - #line 3071 "Python/bytecodes.c" + #line 3072 "Python/bytecodes.c" /* Handles f-string value formatting. */ PyObject *(*conv_fn)(PyObject *); int which_conversion = oparg & FVC_MASK; @@ -4352,7 +4353,7 @@ Py_DECREF(value); Py_XDECREF(fmt_spec); if (result == NULL) { STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); goto pop_1_error; } - #line 4356 "Python/generated_cases.c.h" + #line 4357 "Python/generated_cases.c.h" STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); stack_pointer[-1] = result; DISPATCH(); @@ -4361,10 +4362,10 @@ TARGET(COPY) { PyObject *bottom = stack_pointer[-(1 + (oparg-1))]; PyObject *top; - #line 3108 "Python/bytecodes.c" + #line 3109 "Python/bytecodes.c" assert(oparg > 0); top = Py_NewRef(bottom); - #line 4368 "Python/generated_cases.c.h" + #line 4369 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = top; DISPATCH(); @@ -4376,7 +4377,7 @@ PyObject *rhs = stack_pointer[-1]; PyObject *lhs = stack_pointer[-2]; PyObject *res; - #line 3113 "Python/bytecodes.c" + #line 3114 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -4391,12 +4392,12 @@ assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); assert(binary_ops[oparg]); res = binary_ops[oparg](lhs, rhs); - #line 4395 "Python/generated_cases.c.h" + #line 4396 "Python/generated_cases.c.h" Py_DECREF(lhs); Py_DECREF(rhs); - #line 3128 "Python/bytecodes.c" + #line 3129 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 4400 "Python/generated_cases.c.h" + #line 4401 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -4406,21 +4407,22 @@ TARGET(SWAP) { PyObject *top = stack_pointer[-1]; PyObject *bottom = stack_pointer[-(2 + (oparg-2))]; - #line 3133 "Python/bytecodes.c" + #line 3134 "Python/bytecodes.c" assert(oparg >= 2); - #line 4412 "Python/generated_cases.c.h" + #line 4413 "Python/generated_cases.c.h" stack_pointer[-1] = bottom; stack_pointer[-(2 + (oparg-2))] = top; DISPATCH(); } TARGET(INSTRUMENTED_LINE) { - #line 3137 "Python/bytecodes.c" + #line 3138 "Python/bytecodes.c" _Py_CODEUNIT *here = next_instr-1; _PyFrame_SetStackPointer(frame, stack_pointer); int original_opcode = _Py_call_instrumentation_line( tstate, frame, here); stack_pointer = _PyFrame_GetStackPointer(frame); + frame->stacktop = -1; if (original_opcode < 0) { next_instr = here+1; goto error; @@ -4435,11 +4437,11 @@ } opcode = original_opcode; DISPATCH_GOTO(); - #line 4439 "Python/generated_cases.c.h" + #line 4441 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_INSTRUCTION) { - #line 3159 "Python/bytecodes.c" + #line 3161 "Python/bytecodes.c" int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, next_instr-1); if (next_opcode < 0) goto error; @@ -4451,26 +4453,26 @@ assert(next_opcode > 0 && next_opcode < 256); opcode = next_opcode; DISPATCH_GOTO(); - #line 4455 "Python/generated_cases.c.h" + #line 4457 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_JUMP_FORWARD) { - #line 3174 "Python/bytecodes.c" + #line 3176 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); - #line 4461 "Python/generated_cases.c.h" + #line 4463 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_JUMP_BACKWARD) { - #line 3179 "Python/bytecodes.c" + #line 3181 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr-oparg, PY_MONITORING_EVENT_JUMP); - #line 4468 "Python/generated_cases.c.h" + #line 4470 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { - #line 3184 "Python/bytecodes.c" + #line 3186 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); Py_DECREF(cond); @@ -4479,12 +4481,12 @@ assert(err == 0 || err == 1); int offset = err*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4483 "Python/generated_cases.c.h" + #line 4485 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { - #line 3195 "Python/bytecodes.c" + #line 3197 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); Py_DECREF(cond); @@ -4493,12 +4495,12 @@ assert(err == 0 || err == 1); int offset = (1-err)*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4497 "Python/generated_cases.c.h" + #line 4499 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { - #line 3206 "Python/bytecodes.c" + #line 3208 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4511,12 +4513,12 @@ offset = 0; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4515 "Python/generated_cases.c.h" + #line 4517 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { - #line 3221 "Python/bytecodes.c" + #line 3223 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4529,22 +4531,22 @@ offset = oparg; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4533 "Python/generated_cases.c.h" + #line 4535 "Python/generated_cases.c.h" DISPATCH(); } TARGET(EXTENDED_ARG) { - #line 3236 "Python/bytecodes.c" + #line 3238 "Python/bytecodes.c" assert(oparg); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); - #line 4544 "Python/generated_cases.c.h" + #line 4546 "Python/generated_cases.c.h" } TARGET(CACHE) { - #line 3244 "Python/bytecodes.c" + #line 3246 "Python/bytecodes.c" Py_UNREACHABLE(); - #line 4550 "Python/generated_cases.c.h" + #line 4552 "Python/generated_cases.c.h" } From 6c3473ab9b21b22338033c63068eeaf7b003a8a2 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 24 Mar 2023 04:41:59 +0000 Subject: [PATCH 065/116] Minor cleanups. --- Python/instrumentation.c | 65 +++++++++++++++++++--------------------- 1 file changed, 30 insertions(+), 35 deletions(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 76cb031f27c198..e2c54c32185e50 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1,6 +1,7 @@ #include "Python.h" +#include "pycore_call.h" #include "pycore_frame.h" #include "pycore_interp.h" #include "pycore_namespace.h" @@ -857,7 +858,6 @@ add_tools(PyCodeObject * code, int offset, int event, int tools) /* Single tool */ assert(_Py_popcount32(tools) == 1); assert(tools_is_subset_for_event(code, event, tools)); - assert(_Py_popcount32(tools) == 1); } instrument(code, offset); } @@ -934,7 +934,7 @@ call_one_instrument( int old_what = tstate->what_event; tstate->what_event = event; tstate->tracing++; - PyObject *res = PyObject_Vectorcall(instrument, args, nargsf, NULL); + PyObject *res = _PyObject_VectorcallTstate(tstate, instrument, args, nargsf, NULL); tstate->tracing--; tstate->what_event = old_what; if (res == NULL) { @@ -1005,40 +1005,13 @@ get_tools_for_instruction(PyCodeObject * code, int i, int event) } static int -call_instrument(PyThreadState *tstate, PyCodeObject *code, int event, - PyObject **args, Py_ssize_t nargsf, int offset, uint8_t tools) +call_instrumentation_vector( + PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, Py_ssize_t nargs, PyObject *args[]) { - //sanity_check_instrumentation(code); if (tstate->tracing) { return 0; } - PyInterpreterState *interp = tstate->interp; - while (tools) { - int tool = most_significant_bit(tools); - assert(tool >= 0 && tool < 8); - assert(tools & (1 << tool)); - tools &= ~(1 << tool); - int res = call_one_instrument(interp, tstate, args, nargsf, tool, event); - if (res == 0) { - /* Nothing to do */ - } - else if (res < 0) { - /* error */ - return -1; - } - else { - /* DISABLE */ - remove_tools(code, offset, event, 1 << tool); - } - } - return 0; -} - -static int -call_instrumentation_vector( - PyThreadState *tstate, int event, - _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, Py_ssize_t nargsf, PyObject *args[]) -{ assert(!_PyErr_Occurred(tstate)); assert(args[0] == NULL); PyCodeObject *code = frame->f_code; @@ -1055,7 +1028,29 @@ call_instrumentation_vector( assert(args[2] == NULL); args[2] = offset_obj; uint8_t tools = get_tools_for_instruction(code, offset, event); - int err = call_instrument(tstate, code, event, &args[1], nargsf | PY_VECTORCALL_ARGUMENTS_OFFSET, offset, tools); + Py_ssize_t nargsf = nargs | PY_VECTORCALL_ARGUMENTS_OFFSET; + PyObject **callargs = &args[1]; + int err = 0; + PyInterpreterState *interp = tstate->interp; + while (tools) { + int tool = most_significant_bit(tools); + assert(tool >= 0 && tool < 8); + assert(tools & (1 << tool)); + tools ^= (1 << tool); + int res = call_one_instrument(interp, tstate, callargs, nargsf, tool, event); + if (res == 0) { + /* Nothing to do */ + } + else if (res < 0) { + /* error */ + err = -1; + break; + } + else { + /* DISABLE */ + remove_tools(code, offset, event, 1 << tool); + } + } Py_DECREF(offset_obj); return err; } @@ -1111,11 +1106,11 @@ _Py_call_instrumentation_jump( static void call_instrumentation_vector_protected( PyThreadState *tstate, int event, - _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, Py_ssize_t nargsf, PyObject *args[]) + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, Py_ssize_t nargs, PyObject *args[]) { assert(_PyErr_Occurred(tstate)); PyObject *exc = _PyErr_GetRaisedException(tstate); - int err = call_instrumentation_vector(tstate, event, frame, instr, nargsf, args); + int err = call_instrumentation_vector(tstate, event, frame, instr, nargs, args); if (err) { Py_XDECREF(exc); } From 899aecda675eb0041d05720204a9b163522b32af Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 24 Mar 2023 06:17:25 +0000 Subject: [PATCH 066/116] Remove useless asserts. --- Python/legacy_tracing.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index b55a93ab64ab4f..c72af86183bc29 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -196,7 +196,6 @@ sys_trace_instruction_func( assert(kwnames == NULL); assert(PyVectorcall_NARGS(nargsf) == 2); PyFrameObject* frame = PyEval_GetFrame(); - assert(frame); if (frame == NULL) { PyErr_SetString(PyExc_SystemError, "Missing frame when calling trace function."); @@ -253,7 +252,6 @@ sys_trace_line_func( int line = _PyLong_AsInt(args[1]); assert(line >= 0); PyFrameObject* frame = PyEval_GetFrame(); - assert(frame); if (frame == NULL) { PyErr_SetString(PyExc_SystemError, "Missing frame when calling trace function."); @@ -284,7 +282,6 @@ sys_trace_jump_func( int to = _PyLong_AsInt(args[2]); assert(to >= 0); PyFrameObject* frame = PyEval_GetFrame(); - assert(frame); if (frame == NULL) { PyErr_SetString(PyExc_SystemError, "Missing frame when calling trace function."); From 80d2e2e37e5aa6e894f81798a7b5d8990af98176 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 24 Mar 2023 07:48:48 +0000 Subject: [PATCH 067/116] Make functions static --- Python/instrumentation.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index e2c54c32185e50..5b6314021961f9 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -301,7 +301,8 @@ typedef struct _Instruction { bool is_specialized_instrumented; } Instruction; -Instruction read_instruction(PyCodeObject *code, int offset) +static Instruction +read_instruction(PyCodeObject *code, int offset) { Instruction result = (Instruction){0}; int opcode = result.actual_opcode = _PyCode_CODE(code)[offset].op.code; @@ -812,7 +813,8 @@ remove_tools(PyCodeObject * code, int offset, int event, int tools) } #ifndef NDEBUG -bool tools_is_subset_for_event(PyCodeObject * code, int event, int tools) +static bool +tools_is_subset_for_event(PyCodeObject * code, int event, int tools) { int global_tools = PyInterpreterState_Get()->monitors.tools[event]; int local_tools = code->_co_monitoring->local_monitors.tools[event]; From 94d35d829d22775590118f3e5dd4f8b1f9ef6827 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 24 Mar 2023 08:52:53 +0000 Subject: [PATCH 068/116] Update Windows build files. --- PCbuild/_freeze_module.vcxproj | 2 ++ PCbuild/_freeze_module.vcxproj.filters | 6 ++++++ PCbuild/pythoncore.vcxproj | 2 ++ PCbuild/pythoncore.vcxproj.filters | 6 ++++++ 4 files changed, 16 insertions(+) diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 4f39756019e692..9f7bb27ba2a98b 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -207,6 +207,8 @@ + + diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 7d7c4587b9a3f3..77dc9ef7215a02 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -208,6 +208,12 @@ Source Files + + Source Files + + + Source Files + Source Files diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index c754b2165745ff..54c125f34c7697 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -528,6 +528,8 @@ + + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 90ed0602821bff..3b07728e0abc65 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -1169,6 +1169,12 @@ Source Files + + Source Files + + + Source Files + Python From 5aa080575ba92e6a732a13e7a52de4a8f6515932 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 24 Mar 2023 09:11:18 +0000 Subject: [PATCH 069/116] Make _PyLegacyEventHandler_Type immortal --- Python/legacy_tracing.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index c72af86183bc29..514afc00f7adf0 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -7,6 +7,7 @@ #include "opcode.h" #include "pycore_ceval.h" #include "pycore_instruments.h" +#include "pycore_object.h" #include "pycore_pyerrors.h" #include "pycore_pymem.h" #include "pycore_pystate.h" // _PyInterpreterState_GET() @@ -336,7 +337,7 @@ sys_trace_exception_handled( PyTypeObject _PyLegacyEventHandler_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) + _PyVarObject_IMMORTAL_INIT(&PyType_Type, 0), "sys.legacy_event_handler", sizeof(_PyLegacyEventHandler), .tp_dealloc = (destructor)dealloc, From b39edd311ea0e5bf2963a4b307480e6b6cc47d4d Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 24 Mar 2023 09:19:31 +0000 Subject: [PATCH 070/116] Make arrays const. --- Python/instrumentation.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 5b6314021961f9..0147b4c35f1647 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1479,7 +1479,7 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) return 0; } -static bool super_instructions[256] = { +static const bool super_instructions[256] = { [LOAD_FAST__LOAD_FAST] = true, [LOAD_FAST__LOAD_CONST] = true, [STORE_FAST__LOAD_FAST] = true, @@ -1972,7 +1972,7 @@ add_power2_constant(PyObject *obj, const char *name, int i) return err; } -const char *event_names[] = { +const char *const event_names [] = { [PY_MONITORING_EVENT_PY_START] = "PY_START", [PY_MONITORING_EVENT_PY_RESUME] = "PY_RESUME", [PY_MONITORING_EVENT_PY_RETURN] = "PY_RETURN", From c9c40cbd393ed415365aa4c4938942186f26af5e Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 24 Mar 2023 15:26:03 +0000 Subject: [PATCH 071/116] Rename _PyFrame_GetStackPointer to _PyFrame_FetchStackPointer and make it responsible for setting stacktop to -1. --- Include/internal/pycore_frame.h | 10 +- Python/bytecodes.c | 6 +- Python/ceval.c | 27 +- Python/generated_cases.c.h | 968 ++++++++++++++++---------------- 4 files changed, 495 insertions(+), 516 deletions(-) diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index 8af4f13527ecb7..93b37e6fee64b3 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -138,10 +138,16 @@ _PyFrame_GetLocalsArray(_PyInterpreterFrame *frame) return frame->localsplus; } +/* Fetches the stack pointer, and sets stacktop to -1. + Having stacktop <= 0 ensures that invalid + values are not visible to the cycle GC. + We choose -1 rather than 0 to assist debugging. */ static inline PyObject** -_PyFrame_GetStackPointer(_PyInterpreterFrame *frame) +_PyFrame_FetchStackPointer(_PyInterpreterFrame *frame) { - return frame->localsplus+frame->stacktop; + PyObject **sp = frame->localsplus+frame->stacktop; + frame->stacktop = -1; + return sp; } static inline void diff --git a/Python/bytecodes.c b/Python/bytecodes.c index fd213f8e7282e7..f8df0ccb030889 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -99,8 +99,7 @@ dummy_func( _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation( tstate, oparg > 0, frame, next_instr-1); - stack_pointer = _PyFrame_GetStackPointer(frame); - frame->stacktop = -1; + stack_pointer = _PyFrame_FetchStackPointer(frame); ERROR_IF(err, error); if (frame->prev_instr != next_instr-1) { /* Instrumentation has jumped */ @@ -3139,8 +3138,7 @@ dummy_func( _PyFrame_SetStackPointer(frame, stack_pointer); int original_opcode = _Py_call_instrumentation_line( tstate, frame, here); - stack_pointer = _PyFrame_GetStackPointer(frame); - frame->stacktop = -1; + stack_pointer = _PyFrame_FetchStackPointer(frame); if (original_opcode < 0) { next_instr = here+1; goto error; diff --git a/Python/ceval.c b/Python/ceval.c index 004410fe4dc1e6..c2b0dd5ea95776 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -214,20 +214,6 @@ _PyEvalFrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame); "cannot access free variable '%s' where it is not associated with a" \ " value in enclosing scope" -#define UPDATE_NEXT_OR_JUMP(offset) \ -do { \ - assert(frame->stacktop >= 0); \ - if (frame->prev_instr != next_instr-1) { \ - next_instr = frame->prev_instr; \ - stack_pointer = _PyFrame_GetStackPointer(frame); \ - } \ - else { \ - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); \ - JUMPBY(offset); \ - } \ - frame->stacktop = -1; \ -} while (0) - #ifdef HAVE_ERRNO_H #include #endif @@ -631,8 +617,7 @@ static inline void _Py_LeaveRecursiveCallPy(PyThreadState *tstate) { do { \ _PyFrame_SetStackPointer(frame, stack_pointer); \ int err = _Py_call_instrumentation_jump(tstate, event, frame, src, dest); \ - stack_pointer = _PyFrame_GetStackPointer(frame); \ - frame->stacktop = -1; \ + stack_pointer = _PyFrame_FetchStackPointer(frame); \ if (err) { \ next_instr = (dest)+1; \ goto error; \ @@ -738,15 +723,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int assert(_PyInterpreterFrame_LASTI(frame) >= -1); \ /* Jump back to the last instruction executed... */ \ next_instr = frame->prev_instr + 1; \ - stack_pointer = _PyFrame_GetStackPointer(frame); \ - /* Set stackdepth to -1. \ - Update when returning or calling trace function. \ - Having stackdepth <= 0 ensures that invalid \ - values are not visible to the cycle GC. \ - We choose -1 rather than 0 to assist debugging. \ - */ \ - frame->stacktop = -1; - + stack_pointer = _PyFrame_FetchStackPointer(frame); start_frame: if (_Py_EnterRecursivePy(tstate)) { diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 6531f4fd764ca6..08a294e4803e9e 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -21,8 +21,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation( tstate, oparg > 0, frame, next_instr-1); - stack_pointer = _PyFrame_GetStackPointer(frame); - frame->stacktop = -1; + stack_pointer = _PyFrame_FetchStackPointer(frame); if (err) goto error; if (frame->prev_instr != next_instr-1) { /* Instrumentation has jumped */ @@ -33,12 +32,12 @@ goto handle_eval_breaker; } } - #line 37 "Python/generated_cases.c.h" + #line 36 "Python/generated_cases.c.h" DISPATCH(); } TARGET(RESUME) { - #line 117 "Python/bytecodes.c" + #line 116 "Python/bytecodes.c" assert(tstate->cframe == &cframe); assert(frame == cframe.current_frame); /* Possibly combine this with eval breaker */ @@ -50,18 +49,18 @@ else if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { goto handle_eval_breaker; } - #line 54 "Python/generated_cases.c.h" + #line 53 "Python/generated_cases.c.h" DISPATCH(); } TARGET(LOAD_CLOSURE) { PyObject *value; - #line 131 "Python/bytecodes.c" + #line 130 "Python/bytecodes.c" /* We keep LOAD_CLOSURE so that the bytecode stays more readable. */ value = GETLOCAL(oparg); if (value == NULL) goto unbound_local_error; Py_INCREF(value); - #line 65 "Python/generated_cases.c.h" + #line 64 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -69,11 +68,11 @@ TARGET(LOAD_FAST_CHECK) { PyObject *value; - #line 138 "Python/bytecodes.c" + #line 137 "Python/bytecodes.c" value = GETLOCAL(oparg); if (value == NULL) goto unbound_local_error; Py_INCREF(value); - #line 77 "Python/generated_cases.c.h" + #line 76 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -81,11 +80,11 @@ TARGET(LOAD_FAST) { PyObject *value; - #line 144 "Python/bytecodes.c" + #line 143 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - #line 89 "Python/generated_cases.c.h" + #line 88 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -94,10 +93,10 @@ TARGET(LOAD_CONST) { PREDICTED(LOAD_CONST); PyObject *value; - #line 150 "Python/bytecodes.c" + #line 149 "Python/bytecodes.c" value = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(value); - #line 101 "Python/generated_cases.c.h" + #line 100 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -105,9 +104,9 @@ TARGET(STORE_FAST) { PyObject *value = stack_pointer[-1]; - #line 155 "Python/bytecodes.c" + #line 154 "Python/bytecodes.c" SETLOCAL(oparg, value); - #line 111 "Python/generated_cases.c.h" + #line 110 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -117,21 +116,21 @@ PyObject *_tmp_2; { PyObject *value; - #line 144 "Python/bytecodes.c" + #line 143 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - #line 125 "Python/generated_cases.c.h" + #line 124 "Python/generated_cases.c.h" _tmp_2 = value; } oparg = (next_instr++)->op.arg; { PyObject *value; - #line 144 "Python/bytecodes.c" + #line 143 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - #line 135 "Python/generated_cases.c.h" + #line 134 "Python/generated_cases.c.h" _tmp_1 = value; } STACK_GROW(2); @@ -145,20 +144,20 @@ PyObject *_tmp_2; { PyObject *value; - #line 144 "Python/bytecodes.c" + #line 143 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - #line 153 "Python/generated_cases.c.h" + #line 152 "Python/generated_cases.c.h" _tmp_2 = value; } oparg = (next_instr++)->op.arg; { PyObject *value; - #line 150 "Python/bytecodes.c" + #line 149 "Python/bytecodes.c" value = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(value); - #line 162 "Python/generated_cases.c.h" + #line 161 "Python/generated_cases.c.h" _tmp_1 = value; } STACK_GROW(2); @@ -171,18 +170,18 @@ PyObject *_tmp_1 = stack_pointer[-1]; { PyObject *value = _tmp_1; - #line 155 "Python/bytecodes.c" + #line 154 "Python/bytecodes.c" SETLOCAL(oparg, value); - #line 177 "Python/generated_cases.c.h" + #line 176 "Python/generated_cases.c.h" } oparg = (next_instr++)->op.arg; { PyObject *value; - #line 144 "Python/bytecodes.c" + #line 143 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - #line 186 "Python/generated_cases.c.h" + #line 185 "Python/generated_cases.c.h" _tmp_1 = value; } stack_pointer[-1] = _tmp_1; @@ -194,16 +193,16 @@ PyObject *_tmp_2 = stack_pointer[-2]; { PyObject *value = _tmp_1; - #line 155 "Python/bytecodes.c" + #line 154 "Python/bytecodes.c" SETLOCAL(oparg, value); - #line 200 "Python/generated_cases.c.h" + #line 199 "Python/generated_cases.c.h" } oparg = (next_instr++)->op.arg; { PyObject *value = _tmp_2; - #line 155 "Python/bytecodes.c" + #line 154 "Python/bytecodes.c" SETLOCAL(oparg, value); - #line 207 "Python/generated_cases.c.h" + #line 206 "Python/generated_cases.c.h" } STACK_SHRINK(2); DISPATCH(); @@ -214,20 +213,20 @@ PyObject *_tmp_2; { PyObject *value; - #line 150 "Python/bytecodes.c" + #line 149 "Python/bytecodes.c" value = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(value); - #line 221 "Python/generated_cases.c.h" + #line 220 "Python/generated_cases.c.h" _tmp_2 = value; } oparg = (next_instr++)->op.arg; { PyObject *value; - #line 144 "Python/bytecodes.c" + #line 143 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - #line 231 "Python/generated_cases.c.h" + #line 230 "Python/generated_cases.c.h" _tmp_1 = value; } STACK_GROW(2); @@ -238,8 +237,8 @@ TARGET(POP_TOP) { PyObject *value = stack_pointer[-1]; - #line 165 "Python/bytecodes.c" - #line 243 "Python/generated_cases.c.h" + #line 164 "Python/bytecodes.c" + #line 242 "Python/generated_cases.c.h" Py_DECREF(value); STACK_SHRINK(1); DISPATCH(); @@ -247,9 +246,9 @@ TARGET(PUSH_NULL) { PyObject *res; - #line 169 "Python/bytecodes.c" + #line 168 "Python/bytecodes.c" res = NULL; - #line 253 "Python/generated_cases.c.h" + #line 252 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); @@ -260,14 +259,14 @@ PyObject *_tmp_2 = stack_pointer[-2]; { PyObject *value = _tmp_1; - #line 165 "Python/bytecodes.c" - #line 265 "Python/generated_cases.c.h" + #line 164 "Python/bytecodes.c" + #line 264 "Python/generated_cases.c.h" Py_DECREF(value); } { PyObject *value = _tmp_2; - #line 165 "Python/bytecodes.c" - #line 271 "Python/generated_cases.c.h" + #line 164 "Python/bytecodes.c" + #line 270 "Python/generated_cases.c.h" Py_DECREF(value); } STACK_SHRINK(2); @@ -277,7 +276,7 @@ TARGET(INSTRUMENTED_END_FOR) { PyObject *value = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; - #line 175 "Python/bytecodes.c" + #line 174 "Python/bytecodes.c" /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ if (PyGen_Check(receiver)) { @@ -287,7 +286,7 @@ } PyErr_SetRaisedException(NULL); } - #line 291 "Python/generated_cases.c.h" + #line 290 "Python/generated_cases.c.h" Py_DECREF(receiver); Py_DECREF(value); STACK_SHRINK(2); @@ -297,9 +296,9 @@ TARGET(END_SEND) { PyObject *value = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; - #line 188 "Python/bytecodes.c" + #line 187 "Python/bytecodes.c" Py_DECREF(receiver); - #line 303 "Python/generated_cases.c.h" + #line 302 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = value; DISPATCH(); @@ -308,7 +307,7 @@ TARGET(INSTRUMENTED_END_SEND) { PyObject *value = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; - #line 192 "Python/bytecodes.c" + #line 191 "Python/bytecodes.c" if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); if (monitor_stop_iteration(tstate, frame, next_instr-1)) { @@ -317,7 +316,7 @@ PyErr_SetRaisedException(NULL); } Py_DECREF(receiver); - #line 321 "Python/generated_cases.c.h" + #line 320 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = value; DISPATCH(); @@ -326,13 +325,13 @@ TARGET(UNARY_NEGATIVE) { PyObject *value = stack_pointer[-1]; PyObject *res; - #line 203 "Python/bytecodes.c" + #line 202 "Python/bytecodes.c" res = PyNumber_Negative(value); - #line 332 "Python/generated_cases.c.h" + #line 331 "Python/generated_cases.c.h" Py_DECREF(value); - #line 205 "Python/bytecodes.c" + #line 204 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; - #line 336 "Python/generated_cases.c.h" + #line 335 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -340,11 +339,11 @@ TARGET(UNARY_NOT) { PyObject *value = stack_pointer[-1]; PyObject *res; - #line 209 "Python/bytecodes.c" + #line 208 "Python/bytecodes.c" int err = PyObject_IsTrue(value); - #line 346 "Python/generated_cases.c.h" + #line 345 "Python/generated_cases.c.h" Py_DECREF(value); - #line 211 "Python/bytecodes.c" + #line 210 "Python/bytecodes.c" if (err < 0) goto pop_1_error; if (err == 0) { res = Py_True; @@ -353,7 +352,7 @@ res = Py_False; } Py_INCREF(res); - #line 357 "Python/generated_cases.c.h" + #line 356 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -361,13 +360,13 @@ TARGET(UNARY_INVERT) { PyObject *value = stack_pointer[-1]; PyObject *res; - #line 222 "Python/bytecodes.c" + #line 221 "Python/bytecodes.c" res = PyNumber_Invert(value); - #line 367 "Python/generated_cases.c.h" + #line 366 "Python/generated_cases.c.h" Py_DECREF(value); - #line 224 "Python/bytecodes.c" + #line 223 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; - #line 371 "Python/generated_cases.c.h" + #line 370 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -376,7 +375,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *prod; - #line 241 "Python/bytecodes.c" + #line 240 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -384,7 +383,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (prod == NULL) goto pop_2_error; - #line 388 "Python/generated_cases.c.h" + #line 387 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = prod; next_instr += 1; @@ -395,14 +394,14 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *prod; - #line 251 "Python/bytecodes.c" + #line 250 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); double dprod = ((PyFloatObject *)left)->ob_fval * ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dprod, prod); - #line 406 "Python/generated_cases.c.h" + #line 405 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = prod; next_instr += 1; @@ -413,7 +412,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *sub; - #line 260 "Python/bytecodes.c" + #line 259 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -421,7 +420,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (sub == NULL) goto pop_2_error; - #line 425 "Python/generated_cases.c.h" + #line 424 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = sub; next_instr += 1; @@ -432,13 +431,13 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *sub; - #line 270 "Python/bytecodes.c" + #line 269 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); double dsub = ((PyFloatObject *)left)->ob_fval - ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dsub, sub); - #line 442 "Python/generated_cases.c.h" + #line 441 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = sub; next_instr += 1; @@ -449,7 +448,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 278 "Python/bytecodes.c" + #line 277 "Python/bytecodes.c" DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -457,7 +456,7 @@ _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); if (res == NULL) goto pop_2_error; - #line 461 "Python/generated_cases.c.h" + #line 460 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -467,7 +466,7 @@ TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; - #line 294 "Python/bytecodes.c" + #line 293 "Python/bytecodes.c" DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); _Py_CODEUNIT true_next = next_instr[INLINE_CACHE_ENTRIES_BINARY_OP]; @@ -494,7 +493,7 @@ if (*target_local == NULL) goto pop_2_error; // The STORE_FAST is already done. JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP + 1); - #line 498 "Python/generated_cases.c.h" + #line 497 "Python/generated_cases.c.h" STACK_SHRINK(2); DISPATCH(); } @@ -503,14 +502,14 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *sum; - #line 323 "Python/bytecodes.c" + #line 322 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); double dsum = ((PyFloatObject *)left)->ob_fval + ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dsum, sum); - #line 514 "Python/generated_cases.c.h" + #line 513 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = sum; next_instr += 1; @@ -521,7 +520,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *sum; - #line 332 "Python/bytecodes.c" + #line 331 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -529,7 +528,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (sum == NULL) goto pop_2_error; - #line 533 "Python/generated_cases.c.h" + #line 532 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = sum; next_instr += 1; @@ -542,7 +541,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *container = stack_pointer[-2]; PyObject *res; - #line 350 "Python/bytecodes.c" + #line 349 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -554,12 +553,12 @@ DECREMENT_ADAPTIVE_COUNTER(cache->counter); #endif /* ENABLE_SPECIALIZATION */ res = PyObject_GetItem(container, sub); - #line 558 "Python/generated_cases.c.h" + #line 557 "Python/generated_cases.c.h" Py_DECREF(container); Py_DECREF(sub); - #line 362 "Python/bytecodes.c" + #line 361 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 563 "Python/generated_cases.c.h" + #line 562 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 4; @@ -571,7 +570,7 @@ PyObject *start = stack_pointer[-2]; PyObject *container = stack_pointer[-3]; PyObject *res; - #line 366 "Python/bytecodes.c" + #line 365 "Python/bytecodes.c" PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); // Can't use ERROR_IF() here, because we haven't // DECREF'ed container yet, and we still own slice. @@ -584,7 +583,7 @@ } Py_DECREF(container); if (res == NULL) goto pop_3_error; - #line 588 "Python/generated_cases.c.h" + #line 587 "Python/generated_cases.c.h" STACK_SHRINK(2); stack_pointer[-1] = res; DISPATCH(); @@ -595,7 +594,7 @@ PyObject *start = stack_pointer[-2]; PyObject *container = stack_pointer[-3]; PyObject *v = stack_pointer[-4]; - #line 381 "Python/bytecodes.c" + #line 380 "Python/bytecodes.c" PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); int err; if (slice == NULL) { @@ -608,7 +607,7 @@ Py_DECREF(v); Py_DECREF(container); if (err) goto pop_4_error; - #line 612 "Python/generated_cases.c.h" + #line 611 "Python/generated_cases.c.h" STACK_SHRINK(4); DISPATCH(); } @@ -617,7 +616,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *list = stack_pointer[-2]; PyObject *res; - #line 396 "Python/bytecodes.c" + #line 395 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); @@ -631,7 +630,7 @@ Py_INCREF(res); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(list); - #line 635 "Python/generated_cases.c.h" + #line 634 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 4; @@ -642,7 +641,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *tuple = stack_pointer[-2]; PyObject *res; - #line 412 "Python/bytecodes.c" + #line 411 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); @@ -656,7 +655,7 @@ Py_INCREF(res); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(tuple); - #line 660 "Python/generated_cases.c.h" + #line 659 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 4; @@ -667,7 +666,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *dict = stack_pointer[-2]; PyObject *res; - #line 428 "Python/bytecodes.c" + #line 427 "Python/bytecodes.c" DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); res = PyDict_GetItemWithError(dict, sub); @@ -675,14 +674,14 @@ if (!_PyErr_Occurred(tstate)) { _PyErr_SetKeyError(sub); } - #line 679 "Python/generated_cases.c.h" + #line 678 "Python/generated_cases.c.h" Py_DECREF(dict); Py_DECREF(sub); - #line 436 "Python/bytecodes.c" + #line 435 "Python/bytecodes.c" if (true) goto pop_2_error; } Py_INCREF(res); // Do this before DECREF'ing dict, sub - #line 686 "Python/generated_cases.c.h" + #line 685 "Python/generated_cases.c.h" Py_DECREF(dict); Py_DECREF(sub); STACK_SHRINK(1); @@ -696,7 +695,7 @@ PyObject *container = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t func_version = read_u16(&next_instr[3].cache); - #line 443 "Python/bytecodes.c" + #line 442 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(container); DEOPT_IF(tp->tp_version_tag != type_version, BINARY_SUBSCR); assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); @@ -715,15 +714,15 @@ new_frame->localsplus[1] = sub; JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR); DISPATCH_INLINED(new_frame); - #line 719 "Python/generated_cases.c.h" + #line 718 "Python/generated_cases.c.h" } TARGET(LIST_APPEND) { PyObject *v = stack_pointer[-1]; PyObject *list = stack_pointer[-(2 + (oparg-1))]; - #line 464 "Python/bytecodes.c" + #line 463 "Python/bytecodes.c" if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; - #line 727 "Python/generated_cases.c.h" + #line 726 "Python/generated_cases.c.h" STACK_SHRINK(1); PREDICT(JUMP_BACKWARD); DISPATCH(); @@ -732,13 +731,13 @@ TARGET(SET_ADD) { PyObject *v = stack_pointer[-1]; PyObject *set = stack_pointer[-(2 + (oparg-1))]; - #line 469 "Python/bytecodes.c" + #line 468 "Python/bytecodes.c" int err = PySet_Add(set, v); - #line 738 "Python/generated_cases.c.h" + #line 737 "Python/generated_cases.c.h" Py_DECREF(v); - #line 471 "Python/bytecodes.c" + #line 470 "Python/bytecodes.c" if (err) goto pop_1_error; - #line 742 "Python/generated_cases.c.h" + #line 741 "Python/generated_cases.c.h" STACK_SHRINK(1); PREDICT(JUMP_BACKWARD); DISPATCH(); @@ -751,7 +750,7 @@ PyObject *container = stack_pointer[-2]; PyObject *v = stack_pointer[-3]; uint16_t counter = read_u16(&next_instr[0].cache); - #line 482 "Python/bytecodes.c" + #line 481 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr--; @@ -766,13 +765,13 @@ #endif /* ENABLE_SPECIALIZATION */ /* container[sub] = v */ int err = PyObject_SetItem(container, sub, v); - #line 770 "Python/generated_cases.c.h" + #line 769 "Python/generated_cases.c.h" Py_DECREF(v); Py_DECREF(container); Py_DECREF(sub); - #line 497 "Python/bytecodes.c" + #line 496 "Python/bytecodes.c" if (err) goto pop_3_error; - #line 776 "Python/generated_cases.c.h" + #line 775 "Python/generated_cases.c.h" STACK_SHRINK(3); next_instr += 1; DISPATCH(); @@ -782,7 +781,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *list = stack_pointer[-2]; PyObject *value = stack_pointer[-3]; - #line 501 "Python/bytecodes.c" + #line 500 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); @@ -799,7 +798,7 @@ Py_DECREF(old_value); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(list); - #line 803 "Python/generated_cases.c.h" + #line 802 "Python/generated_cases.c.h" STACK_SHRINK(3); next_instr += 1; DISPATCH(); @@ -809,13 +808,13 @@ PyObject *sub = stack_pointer[-1]; PyObject *dict = stack_pointer[-2]; PyObject *value = stack_pointer[-3]; - #line 520 "Python/bytecodes.c" + #line 519 "Python/bytecodes.c" DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); Py_DECREF(dict); if (err) goto pop_3_error; - #line 819 "Python/generated_cases.c.h" + #line 818 "Python/generated_cases.c.h" STACK_SHRINK(3); next_instr += 1; DISPATCH(); @@ -824,15 +823,15 @@ TARGET(DELETE_SUBSCR) { PyObject *sub = stack_pointer[-1]; PyObject *container = stack_pointer[-2]; - #line 528 "Python/bytecodes.c" + #line 527 "Python/bytecodes.c" /* del container[sub] */ int err = PyObject_DelItem(container, sub); - #line 831 "Python/generated_cases.c.h" + #line 830 "Python/generated_cases.c.h" Py_DECREF(container); Py_DECREF(sub); - #line 531 "Python/bytecodes.c" + #line 530 "Python/bytecodes.c" if (err) goto pop_2_error; - #line 836 "Python/generated_cases.c.h" + #line 835 "Python/generated_cases.c.h" STACK_SHRINK(2); DISPATCH(); } @@ -840,14 +839,14 @@ TARGET(CALL_INTRINSIC_1) { PyObject *value = stack_pointer[-1]; PyObject *res; - #line 535 "Python/bytecodes.c" + #line 534 "Python/bytecodes.c" assert(oparg <= MAX_INTRINSIC_1); res = _PyIntrinsics_UnaryFunctions[oparg](tstate, value); - #line 847 "Python/generated_cases.c.h" + #line 846 "Python/generated_cases.c.h" Py_DECREF(value); - #line 538 "Python/bytecodes.c" + #line 537 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; - #line 851 "Python/generated_cases.c.h" + #line 850 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -856,15 +855,15 @@ PyObject *value1 = stack_pointer[-1]; PyObject *value2 = stack_pointer[-2]; PyObject *res; - #line 542 "Python/bytecodes.c" + #line 541 "Python/bytecodes.c" assert(oparg <= MAX_INTRINSIC_2); res = _PyIntrinsics_BinaryFunctions[oparg](tstate, value2, value1); - #line 863 "Python/generated_cases.c.h" + #line 862 "Python/generated_cases.c.h" Py_DECREF(value2); Py_DECREF(value1); - #line 545 "Python/bytecodes.c" + #line 544 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 868 "Python/generated_cases.c.h" + #line 867 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); @@ -872,7 +871,7 @@ TARGET(RAISE_VARARGS) { PyObject **args = (stack_pointer - oparg); - #line 549 "Python/bytecodes.c" + #line 548 "Python/bytecodes.c" PyObject *cause = NULL, *exc = NULL; switch (oparg) { case 2: @@ -890,12 +889,12 @@ break; } if (true) { STACK_SHRINK(oparg); goto error; } - #line 894 "Python/generated_cases.c.h" + #line 893 "Python/generated_cases.c.h" } TARGET(INTERPRETER_EXIT) { PyObject *retval = stack_pointer[-1]; - #line 569 "Python/bytecodes.c" + #line 568 "Python/bytecodes.c" assert(frame == &entry_frame); assert(_PyFrame_IsIncomplete(frame)); STACK_SHRINK(1); // Since we're not going to DISPATCH() @@ -906,12 +905,12 @@ assert(!_PyErr_Occurred(tstate)); _Py_LeaveRecursiveCallTstate(tstate); return retval; - #line 910 "Python/generated_cases.c.h" + #line 909 "Python/generated_cases.c.h" } TARGET(RETURN_VALUE) { PyObject *retval = stack_pointer[-1]; - #line 582 "Python/bytecodes.c" + #line 581 "Python/bytecodes.c" STACK_SHRINK(1); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -923,12 +922,12 @@ _PyEvalFrameClearAndPop(tstate, dying); _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 927 "Python/generated_cases.c.h" + #line 926 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_RETURN_VALUE) { PyObject *retval = stack_pointer[-1]; - #line 596 "Python/bytecodes.c" + #line 595 "Python/bytecodes.c" int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, next_instr-1, retval); @@ -944,11 +943,11 @@ _PyEvalFrameClearAndPop(tstate, dying); _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 948 "Python/generated_cases.c.h" + #line 947 "Python/generated_cases.c.h" } TARGET(RETURN_CONST) { - #line 614 "Python/bytecodes.c" + #line 613 "Python/bytecodes.c" PyObject *retval = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(retval); assert(EMPTY()); @@ -961,11 +960,11 @@ _PyEvalFrameClearAndPop(tstate, dying); _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 965 "Python/generated_cases.c.h" + #line 964 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_RETURN_CONST) { - #line 629 "Python/bytecodes.c" + #line 628 "Python/bytecodes.c" PyObject *retval = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(retval); int err = _Py_call_instrumentation_arg( @@ -982,13 +981,13 @@ _PyEvalFrameClearAndPop(tstate, dying); _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 986 "Python/generated_cases.c.h" + #line 985 "Python/generated_cases.c.h" } TARGET(GET_AITER) { PyObject *obj = stack_pointer[-1]; PyObject *iter; - #line 648 "Python/bytecodes.c" + #line 647 "Python/bytecodes.c" unaryfunc getter = NULL; PyTypeObject *type = Py_TYPE(obj); @@ -1001,16 +1000,16 @@ "'async for' requires an object with " "__aiter__ method, got %.100s", type->tp_name); - #line 1005 "Python/generated_cases.c.h" + #line 1004 "Python/generated_cases.c.h" Py_DECREF(obj); - #line 661 "Python/bytecodes.c" + #line 660 "Python/bytecodes.c" if (true) goto pop_1_error; } iter = (*getter)(obj); - #line 1012 "Python/generated_cases.c.h" + #line 1011 "Python/generated_cases.c.h" Py_DECREF(obj); - #line 666 "Python/bytecodes.c" + #line 665 "Python/bytecodes.c" if (iter == NULL) goto pop_1_error; if (Py_TYPE(iter)->tp_as_async == NULL || @@ -1023,7 +1022,7 @@ Py_DECREF(iter); if (true) goto pop_1_error; } - #line 1027 "Python/generated_cases.c.h" + #line 1026 "Python/generated_cases.c.h" stack_pointer[-1] = iter; DISPATCH(); } @@ -1031,7 +1030,7 @@ TARGET(GET_ANEXT) { PyObject *aiter = stack_pointer[-1]; PyObject *awaitable; - #line 681 "Python/bytecodes.c" + #line 680 "Python/bytecodes.c" unaryfunc getter = NULL; PyObject *next_iter = NULL; PyTypeObject *type = Py_TYPE(aiter); @@ -1075,7 +1074,7 @@ } } - #line 1079 "Python/generated_cases.c.h" + #line 1078 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = awaitable; PREDICT(LOAD_CONST); @@ -1086,16 +1085,16 @@ PREDICTED(GET_AWAITABLE); PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 728 "Python/bytecodes.c" + #line 727 "Python/bytecodes.c" iter = _PyCoro_GetAwaitableIter(iterable); if (iter == NULL) { format_awaitable_error(tstate, Py_TYPE(iterable), oparg); } - #line 1097 "Python/generated_cases.c.h" + #line 1096 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 735 "Python/bytecodes.c" + #line 734 "Python/bytecodes.c" if (iter != NULL && PyCoro_CheckExact(iter)) { PyObject *yf = _PyGen_yf((PyGenObject*)iter); @@ -1113,7 +1112,7 @@ if (iter == NULL) goto pop_1_error; - #line 1117 "Python/generated_cases.c.h" + #line 1116 "Python/generated_cases.c.h" stack_pointer[-1] = iter; PREDICT(LOAD_CONST); DISPATCH(); @@ -1124,7 +1123,7 @@ PyObject *v = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; PyObject *retval; - #line 761 "Python/bytecodes.c" + #line 760 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PySendCache *cache = (_PySendCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1170,7 +1169,7 @@ } } Py_DECREF(v); - #line 1174 "Python/generated_cases.c.h" + #line 1173 "Python/generated_cases.c.h" stack_pointer[-1] = retval; next_instr += 1; DISPATCH(); @@ -1179,7 +1178,7 @@ TARGET(SEND_GEN) { PyObject *v = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; - #line 809 "Python/bytecodes.c" + #line 808 "Python/bytecodes.c" PyGenObject *gen = (PyGenObject *)receiver; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); @@ -1194,12 +1193,12 @@ tstate->exc_info = &gen->gi_exc_state; JUMPBY(INLINE_CACHE_ENTRIES_SEND + oparg); DISPATCH_INLINED(gen_frame); - #line 1198 "Python/generated_cases.c.h" + #line 1197 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_YIELD_VALUE) { PyObject *retval = stack_pointer[-1]; - #line 826 "Python/bytecodes.c" + #line 825 "Python/bytecodes.c" assert(frame != &entry_frame); PyGenObject *gen = _PyFrame_GetGenerator(frame); gen->gi_frame_state = FRAME_SUSPENDED; @@ -1217,12 +1216,12 @@ frame->prev_instr -= frame->yield_offset; _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 1221 "Python/generated_cases.c.h" + #line 1220 "Python/generated_cases.c.h" } TARGET(YIELD_VALUE) { PyObject *retval = stack_pointer[-1]; - #line 846 "Python/bytecodes.c" + #line 845 "Python/bytecodes.c" // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. @@ -1239,15 +1238,15 @@ frame->prev_instr -= frame->yield_offset; _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 1243 "Python/generated_cases.c.h" + #line 1242 "Python/generated_cases.c.h" } TARGET(POP_EXCEPT) { PyObject *exc_value = stack_pointer[-1]; - #line 865 "Python/bytecodes.c" + #line 864 "Python/bytecodes.c" _PyErr_StackItem *exc_info = tstate->exc_info; Py_XSETREF(exc_info->exc_value, exc_value); - #line 1251 "Python/generated_cases.c.h" + #line 1250 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -1255,7 +1254,7 @@ TARGET(RERAISE) { PyObject *exc = stack_pointer[-1]; PyObject **values = (stack_pointer - (1 + oparg)); - #line 870 "Python/bytecodes.c" + #line 869 "Python/bytecodes.c" assert(oparg >= 0 && oparg <= 2); if (oparg) { PyObject *lasti = values[0]; @@ -1273,26 +1272,26 @@ Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); goto exception_unwind; - #line 1277 "Python/generated_cases.c.h" + #line 1276 "Python/generated_cases.c.h" } TARGET(END_ASYNC_FOR) { PyObject *exc = stack_pointer[-1]; PyObject *awaitable = stack_pointer[-2]; - #line 890 "Python/bytecodes.c" + #line 889 "Python/bytecodes.c" assert(exc && PyExceptionInstance_Check(exc)); if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { - #line 1286 "Python/generated_cases.c.h" + #line 1285 "Python/generated_cases.c.h" Py_DECREF(awaitable); Py_DECREF(exc); - #line 893 "Python/bytecodes.c" + #line 892 "Python/bytecodes.c" } else { Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); goto exception_unwind; } - #line 1296 "Python/generated_cases.c.h" + #line 1295 "Python/generated_cases.c.h" STACK_SHRINK(2); DISPATCH(); } @@ -1303,23 +1302,23 @@ PyObject *sub_iter = stack_pointer[-3]; PyObject *none; PyObject *value; - #line 902 "Python/bytecodes.c" + #line 901 "Python/bytecodes.c" assert(throwflag); assert(exc_value && PyExceptionInstance_Check(exc_value)); if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { value = Py_NewRef(((PyStopIterationObject *)exc_value)->value); - #line 1312 "Python/generated_cases.c.h" + #line 1311 "Python/generated_cases.c.h" Py_DECREF(sub_iter); Py_DECREF(last_sent_val); Py_DECREF(exc_value); - #line 907 "Python/bytecodes.c" + #line 906 "Python/bytecodes.c" none = Py_NewRef(Py_None); } else { _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); goto exception_unwind; } - #line 1323 "Python/generated_cases.c.h" + #line 1322 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = value; stack_pointer[-2] = none; @@ -1328,9 +1327,9 @@ TARGET(LOAD_ASSERTION_ERROR) { PyObject *value; - #line 916 "Python/bytecodes.c" + #line 915 "Python/bytecodes.c" value = Py_NewRef(PyExc_AssertionError); - #line 1334 "Python/generated_cases.c.h" + #line 1333 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -1338,7 +1337,7 @@ TARGET(LOAD_BUILD_CLASS) { PyObject *bc; - #line 920 "Python/bytecodes.c" + #line 919 "Python/bytecodes.c" if (PyDict_CheckExact(BUILTINS())) { bc = _PyDict_GetItemWithError(BUILTINS(), &_Py_ID(__build_class__)); @@ -1360,7 +1359,7 @@ if (true) goto error; } } - #line 1364 "Python/generated_cases.c.h" + #line 1363 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = bc; DISPATCH(); @@ -1368,33 +1367,33 @@ TARGET(STORE_NAME) { PyObject *v = stack_pointer[-1]; - #line 944 "Python/bytecodes.c" + #line 943 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); PyObject *ns = LOCALS(); int err; if (ns == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals found when storing %R", name); - #line 1379 "Python/generated_cases.c.h" + #line 1378 "Python/generated_cases.c.h" Py_DECREF(v); - #line 951 "Python/bytecodes.c" + #line 950 "Python/bytecodes.c" if (true) goto pop_1_error; } if (PyDict_CheckExact(ns)) err = PyDict_SetItem(ns, name, v); else err = PyObject_SetItem(ns, name, v); - #line 1388 "Python/generated_cases.c.h" + #line 1387 "Python/generated_cases.c.h" Py_DECREF(v); - #line 958 "Python/bytecodes.c" + #line 957 "Python/bytecodes.c" if (err) goto pop_1_error; - #line 1392 "Python/generated_cases.c.h" + #line 1391 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(DELETE_NAME) { - #line 962 "Python/bytecodes.c" + #line 961 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); PyObject *ns = LOCALS(); int err; @@ -1411,7 +1410,7 @@ name); goto error; } - #line 1415 "Python/generated_cases.c.h" + #line 1414 "Python/generated_cases.c.h" DISPATCH(); } @@ -1419,7 +1418,7 @@ PREDICTED(UNPACK_SEQUENCE); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); PyObject *seq = stack_pointer[-1]; - #line 988 "Python/bytecodes.c" + #line 987 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1432,11 +1431,11 @@ #endif /* ENABLE_SPECIALIZATION */ PyObject **top = stack_pointer + oparg - 1; int res = unpack_iterable(tstate, seq, oparg, -1, top); - #line 1436 "Python/generated_cases.c.h" + #line 1435 "Python/generated_cases.c.h" Py_DECREF(seq); - #line 1001 "Python/bytecodes.c" + #line 1000 "Python/bytecodes.c" if (res == 0) goto pop_1_error; - #line 1440 "Python/generated_cases.c.h" + #line 1439 "Python/generated_cases.c.h" STACK_SHRINK(1); STACK_GROW(oparg); next_instr += 1; @@ -1446,14 +1445,14 @@ TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { PyObject *seq = stack_pointer[-1]; PyObject **values = stack_pointer - (1); - #line 1005 "Python/bytecodes.c" + #line 1004 "Python/bytecodes.c" DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); assert(oparg == 2); STAT_INC(UNPACK_SEQUENCE, hit); values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); - #line 1457 "Python/generated_cases.c.h" + #line 1456 "Python/generated_cases.c.h" Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); @@ -1464,7 +1463,7 @@ TARGET(UNPACK_SEQUENCE_TUPLE) { PyObject *seq = stack_pointer[-1]; PyObject **values = stack_pointer - (1); - #line 1015 "Python/bytecodes.c" + #line 1014 "Python/bytecodes.c" DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); @@ -1472,7 +1471,7 @@ for (int i = oparg; --i >= 0; ) { *values++ = Py_NewRef(items[i]); } - #line 1476 "Python/generated_cases.c.h" + #line 1475 "Python/generated_cases.c.h" Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); @@ -1483,7 +1482,7 @@ TARGET(UNPACK_SEQUENCE_LIST) { PyObject *seq = stack_pointer[-1]; PyObject **values = stack_pointer - (1); - #line 1026 "Python/bytecodes.c" + #line 1025 "Python/bytecodes.c" DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); @@ -1491,7 +1490,7 @@ for (int i = oparg; --i >= 0; ) { *values++ = Py_NewRef(items[i]); } - #line 1495 "Python/generated_cases.c.h" + #line 1494 "Python/generated_cases.c.h" Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); @@ -1501,15 +1500,15 @@ TARGET(UNPACK_EX) { PyObject *seq = stack_pointer[-1]; - #line 1037 "Python/bytecodes.c" + #line 1036 "Python/bytecodes.c" int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); PyObject **top = stack_pointer + totalargs - 1; int res = unpack_iterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); - #line 1509 "Python/generated_cases.c.h" + #line 1508 "Python/generated_cases.c.h" Py_DECREF(seq); - #line 1041 "Python/bytecodes.c" + #line 1040 "Python/bytecodes.c" if (res == 0) goto pop_1_error; - #line 1513 "Python/generated_cases.c.h" + #line 1512 "Python/generated_cases.c.h" STACK_GROW((oparg & 0xFF) + (oparg >> 8)); DISPATCH(); } @@ -1520,7 +1519,7 @@ PyObject *owner = stack_pointer[-1]; PyObject *v = stack_pointer[-2]; uint16_t counter = read_u16(&next_instr[0].cache); - #line 1052 "Python/bytecodes.c" + #line 1051 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { PyObject *name = GETITEM(frame->f_code->co_names, oparg); @@ -1536,12 +1535,12 @@ #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyObject_SetAttr(owner, name, v); - #line 1540 "Python/generated_cases.c.h" + #line 1539 "Python/generated_cases.c.h" Py_DECREF(v); Py_DECREF(owner); - #line 1068 "Python/bytecodes.c" + #line 1067 "Python/bytecodes.c" if (err) goto pop_2_error; - #line 1545 "Python/generated_cases.c.h" + #line 1544 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -1549,34 +1548,34 @@ TARGET(DELETE_ATTR) { PyObject *owner = stack_pointer[-1]; - #line 1072 "Python/bytecodes.c" + #line 1071 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyObject_SetAttr(owner, name, (PyObject *)NULL); - #line 1556 "Python/generated_cases.c.h" + #line 1555 "Python/generated_cases.c.h" Py_DECREF(owner); - #line 1075 "Python/bytecodes.c" + #line 1074 "Python/bytecodes.c" if (err) goto pop_1_error; - #line 1560 "Python/generated_cases.c.h" + #line 1559 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(STORE_GLOBAL) { PyObject *v = stack_pointer[-1]; - #line 1079 "Python/bytecodes.c" + #line 1078 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyDict_SetItem(GLOBALS(), name, v); - #line 1570 "Python/generated_cases.c.h" + #line 1569 "Python/generated_cases.c.h" Py_DECREF(v); - #line 1082 "Python/bytecodes.c" + #line 1081 "Python/bytecodes.c" if (err) goto pop_1_error; - #line 1574 "Python/generated_cases.c.h" + #line 1573 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(DELETE_GLOBAL) { - #line 1086 "Python/bytecodes.c" + #line 1085 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err; err = PyDict_DelItem(GLOBALS(), name); @@ -1588,13 +1587,13 @@ } goto error; } - #line 1592 "Python/generated_cases.c.h" + #line 1591 "Python/generated_cases.c.h" DISPATCH(); } TARGET(LOAD_NAME) { PyObject *v; - #line 1100 "Python/bytecodes.c" + #line 1099 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); PyObject *locals = LOCALS(); if (locals == NULL) { @@ -1653,7 +1652,7 @@ } } } - #line 1657 "Python/generated_cases.c.h" + #line 1656 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = v; DISPATCH(); @@ -1664,7 +1663,7 @@ static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyObject *null = NULL; PyObject *v; - #line 1167 "Python/bytecodes.c" + #line 1166 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1716,7 +1715,7 @@ } } null = NULL; - #line 1720 "Python/generated_cases.c.h" + #line 1719 "Python/generated_cases.c.h" STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = v; @@ -1730,7 +1729,7 @@ PyObject *res; uint16_t index = read_u16(&next_instr[1].cache); uint16_t version = read_u16(&next_instr[2].cache); - #line 1221 "Python/bytecodes.c" + #line 1220 "Python/bytecodes.c" DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); PyDictObject *dict = (PyDictObject *)GLOBALS(); DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); @@ -1741,7 +1740,7 @@ Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; - #line 1745 "Python/generated_cases.c.h" + #line 1744 "Python/generated_cases.c.h" STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -1756,7 +1755,7 @@ uint16_t index = read_u16(&next_instr[1].cache); uint16_t mod_version = read_u16(&next_instr[2].cache); uint16_t bltn_version = read_u16(&next_instr[3].cache); - #line 1234 "Python/bytecodes.c" + #line 1233 "Python/bytecodes.c" DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); DEOPT_IF(!PyDict_CheckExact(BUILTINS()), LOAD_GLOBAL); PyDictObject *mdict = (PyDictObject *)GLOBALS(); @@ -1770,7 +1769,7 @@ Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; - #line 1774 "Python/generated_cases.c.h" + #line 1773 "Python/generated_cases.c.h" STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -1780,16 +1779,16 @@ } TARGET(DELETE_FAST) { - #line 1250 "Python/bytecodes.c" + #line 1249 "Python/bytecodes.c" PyObject *v = GETLOCAL(oparg); if (v == NULL) goto unbound_local_error; SETLOCAL(oparg, NULL); - #line 1788 "Python/generated_cases.c.h" + #line 1787 "Python/generated_cases.c.h" DISPATCH(); } TARGET(MAKE_CELL) { - #line 1256 "Python/bytecodes.c" + #line 1255 "Python/bytecodes.c" // "initial" is probably NULL but not if it's an arg (or set // via PyFrame_LocalsToFast() before MAKE_CELL has run). PyObject *initial = GETLOCAL(oparg); @@ -1798,12 +1797,12 @@ goto resume_with_error; } SETLOCAL(oparg, cell); - #line 1802 "Python/generated_cases.c.h" + #line 1801 "Python/generated_cases.c.h" DISPATCH(); } TARGET(DELETE_DEREF) { - #line 1267 "Python/bytecodes.c" + #line 1266 "Python/bytecodes.c" PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); // Can't use ERROR_IF here. @@ -1814,13 +1813,13 @@ } PyCell_SET(cell, NULL); Py_DECREF(oldobj); - #line 1818 "Python/generated_cases.c.h" + #line 1817 "Python/generated_cases.c.h" DISPATCH(); } TARGET(LOAD_CLASSDEREF) { PyObject *value; - #line 1280 "Python/bytecodes.c" + #line 1279 "Python/bytecodes.c" PyObject *name, *locals = LOCALS(); assert(locals); assert(oparg >= 0 && oparg < frame->f_code->co_nlocalsplus); @@ -1852,7 +1851,7 @@ } Py_INCREF(value); } - #line 1856 "Python/generated_cases.c.h" + #line 1855 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -1860,7 +1859,7 @@ TARGET(LOAD_DEREF) { PyObject *value; - #line 1314 "Python/bytecodes.c" + #line 1313 "Python/bytecodes.c" PyObject *cell = GETLOCAL(oparg); value = PyCell_GET(cell); if (value == NULL) { @@ -1868,7 +1867,7 @@ if (true) goto error; } Py_INCREF(value); - #line 1872 "Python/generated_cases.c.h" + #line 1871 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -1876,18 +1875,18 @@ TARGET(STORE_DEREF) { PyObject *v = stack_pointer[-1]; - #line 1324 "Python/bytecodes.c" + #line 1323 "Python/bytecodes.c" PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); PyCell_SET(cell, v); Py_XDECREF(oldobj); - #line 1885 "Python/generated_cases.c.h" + #line 1884 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(COPY_FREE_VARS) { - #line 1331 "Python/bytecodes.c" + #line 1330 "Python/bytecodes.c" /* Copy closure variables to free variables */ PyCodeObject *co = frame->f_code; assert(PyFunction_Check(frame->f_funcobj)); @@ -1898,22 +1897,22 @@ PyObject *o = PyTuple_GET_ITEM(closure, i); frame->localsplus[offset + i] = Py_NewRef(o); } - #line 1902 "Python/generated_cases.c.h" + #line 1901 "Python/generated_cases.c.h" DISPATCH(); } TARGET(BUILD_STRING) { PyObject **pieces = (stack_pointer - oparg); PyObject *str; - #line 1344 "Python/bytecodes.c" + #line 1343 "Python/bytecodes.c" str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); - #line 1911 "Python/generated_cases.c.h" + #line 1910 "Python/generated_cases.c.h" for (int _i = oparg; --_i >= 0;) { Py_DECREF(pieces[_i]); } - #line 1346 "Python/bytecodes.c" + #line 1345 "Python/bytecodes.c" if (str == NULL) { STACK_SHRINK(oparg); goto error; } - #line 1917 "Python/generated_cases.c.h" + #line 1916 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = str; @@ -1923,10 +1922,10 @@ TARGET(BUILD_TUPLE) { PyObject **values = (stack_pointer - oparg); PyObject *tup; - #line 1350 "Python/bytecodes.c" + #line 1349 "Python/bytecodes.c" tup = _PyTuple_FromArraySteal(values, oparg); if (tup == NULL) { STACK_SHRINK(oparg); goto error; } - #line 1930 "Python/generated_cases.c.h" + #line 1929 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = tup; @@ -1936,10 +1935,10 @@ TARGET(BUILD_LIST) { PyObject **values = (stack_pointer - oparg); PyObject *list; - #line 1355 "Python/bytecodes.c" + #line 1354 "Python/bytecodes.c" list = _PyList_FromArraySteal(values, oparg); if (list == NULL) { STACK_SHRINK(oparg); goto error; } - #line 1943 "Python/generated_cases.c.h" + #line 1942 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = list; @@ -1949,7 +1948,7 @@ TARGET(LIST_EXTEND) { PyObject *iterable = stack_pointer[-1]; PyObject *list = stack_pointer[-(2 + (oparg-1))]; - #line 1360 "Python/bytecodes.c" + #line 1359 "Python/bytecodes.c" PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); if (none_val == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && @@ -1960,13 +1959,13 @@ "Value after * must be an iterable, not %.200s", Py_TYPE(iterable)->tp_name); } - #line 1964 "Python/generated_cases.c.h" + #line 1963 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 1371 "Python/bytecodes.c" + #line 1370 "Python/bytecodes.c" if (true) goto pop_1_error; } Py_DECREF(none_val); - #line 1970 "Python/generated_cases.c.h" + #line 1969 "Python/generated_cases.c.h" Py_DECREF(iterable); STACK_SHRINK(1); DISPATCH(); @@ -1975,13 +1974,13 @@ TARGET(SET_UPDATE) { PyObject *iterable = stack_pointer[-1]; PyObject *set = stack_pointer[-(2 + (oparg-1))]; - #line 1378 "Python/bytecodes.c" + #line 1377 "Python/bytecodes.c" int err = _PySet_Update(set, iterable); - #line 1981 "Python/generated_cases.c.h" + #line 1980 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 1380 "Python/bytecodes.c" + #line 1379 "Python/bytecodes.c" if (err < 0) goto pop_1_error; - #line 1985 "Python/generated_cases.c.h" + #line 1984 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -1989,7 +1988,7 @@ TARGET(BUILD_SET) { PyObject **values = (stack_pointer - oparg); PyObject *set; - #line 1384 "Python/bytecodes.c" + #line 1383 "Python/bytecodes.c" set = PySet_New(NULL); if (set == NULL) goto error; @@ -2004,7 +2003,7 @@ Py_DECREF(set); if (true) { STACK_SHRINK(oparg); goto error; } } - #line 2008 "Python/generated_cases.c.h" + #line 2007 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = set; @@ -2014,7 +2013,7 @@ TARGET(BUILD_MAP) { PyObject **values = (stack_pointer - oparg*2); PyObject *map; - #line 1401 "Python/bytecodes.c" + #line 1400 "Python/bytecodes.c" map = _PyDict_FromItems( values, 2, values+1, 2, @@ -2022,13 +2021,13 @@ if (map == NULL) goto error; - #line 2026 "Python/generated_cases.c.h" + #line 2025 "Python/generated_cases.c.h" for (int _i = oparg*2; --_i >= 0;) { Py_DECREF(values[_i]); } - #line 1409 "Python/bytecodes.c" + #line 1408 "Python/bytecodes.c" if (map == NULL) { STACK_SHRINK(oparg*2); goto error; } - #line 2032 "Python/generated_cases.c.h" + #line 2031 "Python/generated_cases.c.h" STACK_SHRINK(oparg*2); STACK_GROW(1); stack_pointer[-1] = map; @@ -2036,7 +2035,7 @@ } TARGET(SETUP_ANNOTATIONS) { - #line 1413 "Python/bytecodes.c" + #line 1412 "Python/bytecodes.c" int err; PyObject *ann_dict; if (LOCALS() == NULL) { @@ -2076,7 +2075,7 @@ Py_DECREF(ann_dict); } } - #line 2080 "Python/generated_cases.c.h" + #line 2079 "Python/generated_cases.c.h" DISPATCH(); } @@ -2084,7 +2083,7 @@ PyObject *keys = stack_pointer[-1]; PyObject **values = (stack_pointer - (1 + oparg)); PyObject *map; - #line 1455 "Python/bytecodes.c" + #line 1454 "Python/bytecodes.c" if (!PyTuple_CheckExact(keys) || PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { _PyErr_SetString(tstate, PyExc_SystemError, @@ -2094,14 +2093,14 @@ map = _PyDict_FromItems( &PyTuple_GET_ITEM(keys, 0), 1, values, 1, oparg); - #line 2098 "Python/generated_cases.c.h" + #line 2097 "Python/generated_cases.c.h" for (int _i = oparg; --_i >= 0;) { Py_DECREF(values[_i]); } Py_DECREF(keys); - #line 1465 "Python/bytecodes.c" + #line 1464 "Python/bytecodes.c" if (map == NULL) { STACK_SHRINK(oparg); goto pop_1_error; } - #line 2105 "Python/generated_cases.c.h" + #line 2104 "Python/generated_cases.c.h" STACK_SHRINK(oparg); stack_pointer[-1] = map; DISPATCH(); @@ -2109,7 +2108,7 @@ TARGET(DICT_UPDATE) { PyObject *update = stack_pointer[-1]; - #line 1469 "Python/bytecodes.c" + #line 1468 "Python/bytecodes.c" PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (PyDict_Update(dict, update) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { @@ -2117,12 +2116,12 @@ "'%.200s' object is not a mapping", Py_TYPE(update)->tp_name); } - #line 2121 "Python/generated_cases.c.h" + #line 2120 "Python/generated_cases.c.h" Py_DECREF(update); - #line 1477 "Python/bytecodes.c" + #line 1476 "Python/bytecodes.c" if (true) goto pop_1_error; } - #line 2126 "Python/generated_cases.c.h" + #line 2125 "Python/generated_cases.c.h" Py_DECREF(update); STACK_SHRINK(1); DISPATCH(); @@ -2130,17 +2129,17 @@ TARGET(DICT_MERGE) { PyObject *update = stack_pointer[-1]; - #line 1483 "Python/bytecodes.c" + #line 1482 "Python/bytecodes.c" PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (_PyDict_MergeEx(dict, update, 2) < 0) { format_kwargs_error(tstate, PEEK(3 + oparg), update); - #line 2139 "Python/generated_cases.c.h" + #line 2138 "Python/generated_cases.c.h" Py_DECREF(update); - #line 1488 "Python/bytecodes.c" + #line 1487 "Python/bytecodes.c" if (true) goto pop_1_error; } - #line 2144 "Python/generated_cases.c.h" + #line 2143 "Python/generated_cases.c.h" Py_DECREF(update); STACK_SHRINK(1); PREDICT(CALL_FUNCTION_EX); @@ -2150,13 +2149,13 @@ TARGET(MAP_ADD) { PyObject *value = stack_pointer[-1]; PyObject *key = stack_pointer[-2]; - #line 1495 "Python/bytecodes.c" + #line 1494 "Python/bytecodes.c" PyObject *dict = PEEK(oparg + 2); // key, value are still on the stack assert(PyDict_CheckExact(dict)); /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; - #line 2160 "Python/generated_cases.c.h" + #line 2159 "Python/generated_cases.c.h" STACK_SHRINK(2); PREDICT(JUMP_BACKWARD); DISPATCH(); @@ -2168,7 +2167,7 @@ PyObject *owner = stack_pointer[-1]; PyObject *res2 = NULL; PyObject *res; - #line 1518 "Python/bytecodes.c" + #line 1517 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -2202,9 +2201,9 @@ NULL | meth | arg1 | ... | argN */ - #line 2206 "Python/generated_cases.c.h" + #line 2205 "Python/generated_cases.c.h" Py_DECREF(owner); - #line 1552 "Python/bytecodes.c" + #line 1551 "Python/bytecodes.c" if (meth == NULL) goto pop_1_error; res2 = NULL; res = meth; @@ -2213,12 +2212,12 @@ else { /* Classic, pushes one value. */ res = PyObject_GetAttr(owner, name); - #line 2217 "Python/generated_cases.c.h" + #line 2216 "Python/generated_cases.c.h" Py_DECREF(owner); - #line 1561 "Python/bytecodes.c" + #line 1560 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; } - #line 2222 "Python/generated_cases.c.h" + #line 2221 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -2232,7 +2231,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1566 "Python/bytecodes.c" + #line 1565 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2245,7 +2244,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2249 "Python/generated_cases.c.h" + #line 2248 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2260,7 +2259,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1582 "Python/bytecodes.c" + #line 1581 "Python/bytecodes.c" DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict != NULL); @@ -2273,7 +2272,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2277 "Python/generated_cases.c.h" + #line 2276 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2288,7 +2287,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1598 "Python/bytecodes.c" + #line 1597 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2315,7 +2314,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2319 "Python/generated_cases.c.h" + #line 2318 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2330,7 +2329,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1628 "Python/bytecodes.c" + #line 1627 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2340,7 +2339,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2344 "Python/generated_cases.c.h" + #line 2343 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2355,7 +2354,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 1641 "Python/bytecodes.c" + #line 1640 "Python/bytecodes.c" DEOPT_IF(!PyType_Check(cls), LOAD_ATTR); DEOPT_IF(((PyTypeObject *)cls)->tp_version_tag != type_version, @@ -2367,7 +2366,7 @@ res = descr; assert(res != NULL); Py_INCREF(res); - #line 2371 "Python/generated_cases.c.h" + #line 2370 "Python/generated_cases.c.h" Py_DECREF(cls); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2381,7 +2380,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t func_version = read_u32(&next_instr[3].cache); PyObject *fget = read_obj(&next_instr[5].cache); - #line 1656 "Python/bytecodes.c" + #line 1655 "Python/bytecodes.c" DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); @@ -2404,7 +2403,7 @@ new_frame->localsplus[0] = owner; JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); DISPATCH_INLINED(new_frame); - #line 2408 "Python/generated_cases.c.h" + #line 2407 "Python/generated_cases.c.h" } TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { @@ -2412,7 +2411,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t func_version = read_u32(&next_instr[3].cache); PyObject *getattribute = read_obj(&next_instr[5].cache); - #line 1681 "Python/bytecodes.c" + #line 1680 "Python/bytecodes.c" DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); @@ -2437,7 +2436,7 @@ new_frame->localsplus[1] = Py_NewRef(name); JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); DISPATCH_INLINED(new_frame); - #line 2441 "Python/generated_cases.c.h" + #line 2440 "Python/generated_cases.c.h" } TARGET(STORE_ATTR_INSTANCE_VALUE) { @@ -2445,7 +2444,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1708 "Python/bytecodes.c" + #line 1707 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2463,7 +2462,7 @@ Py_DECREF(old_value); } Py_DECREF(owner); - #line 2467 "Python/generated_cases.c.h" + #line 2466 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2474,7 +2473,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t hint = read_u16(&next_instr[3].cache); - #line 1728 "Python/bytecodes.c" + #line 1727 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2513,7 +2512,7 @@ /* PEP 509 */ dict->ma_version_tag = new_version; Py_DECREF(owner); - #line 2517 "Python/generated_cases.c.h" + #line 2516 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2524,7 +2523,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1769 "Python/bytecodes.c" + #line 1768 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2534,7 +2533,7 @@ *(PyObject **)addr = value; Py_XDECREF(old_value); Py_DECREF(owner); - #line 2538 "Python/generated_cases.c.h" + #line 2537 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2546,7 +2545,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1788 "Python/bytecodes.c" + #line 1787 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -2559,12 +2558,12 @@ #endif /* ENABLE_SPECIALIZATION */ assert((oparg >> 4) <= Py_GE); res = PyObject_RichCompare(left, right, oparg>>4); - #line 2563 "Python/generated_cases.c.h" + #line 2562 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 1801 "Python/bytecodes.c" + #line 1800 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 2568 "Python/generated_cases.c.h" + #line 2567 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2575,7 +2574,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1805 "Python/bytecodes.c" + #line 1804 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); @@ -2587,7 +2586,7 @@ _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); res = (sign_ish & oparg) ? Py_True : Py_False; Py_INCREF(res); - #line 2591 "Python/generated_cases.c.h" + #line 2590 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2598,7 +2597,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1820 "Python/bytecodes.c" + #line 1819 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); @@ -2614,7 +2613,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); res = (sign_ish & oparg) ? Py_True : Py_False; Py_INCREF(res); - #line 2618 "Python/generated_cases.c.h" + #line 2617 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2625,7 +2624,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1839 "Python/bytecodes.c" + #line 1838 "Python/bytecodes.c" DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); @@ -2638,7 +2637,7 @@ assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; Py_INCREF(res); - #line 2642 "Python/generated_cases.c.h" + #line 2641 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2649,14 +2648,14 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 1854 "Python/bytecodes.c" + #line 1853 "Python/bytecodes.c" int res = Py_Is(left, right) ^ oparg; - #line 2655 "Python/generated_cases.c.h" + #line 2654 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 1856 "Python/bytecodes.c" + #line 1855 "Python/bytecodes.c" b = Py_NewRef(res ? Py_True : Py_False); - #line 2660 "Python/generated_cases.c.h" + #line 2659 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = b; DISPATCH(); @@ -2666,15 +2665,15 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 1860 "Python/bytecodes.c" + #line 1859 "Python/bytecodes.c" int res = PySequence_Contains(right, left); - #line 2672 "Python/generated_cases.c.h" + #line 2671 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 1862 "Python/bytecodes.c" + #line 1861 "Python/bytecodes.c" if (res < 0) goto pop_2_error; b = Py_NewRef((res^oparg) ? Py_True : Py_False); - #line 2678 "Python/generated_cases.c.h" + #line 2677 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = b; DISPATCH(); @@ -2685,12 +2684,12 @@ PyObject *exc_value = stack_pointer[-2]; PyObject *rest; PyObject *match; - #line 1867 "Python/bytecodes.c" + #line 1866 "Python/bytecodes.c" if (check_except_star_type_valid(tstate, match_type) < 0) { - #line 2691 "Python/generated_cases.c.h" + #line 2690 "Python/generated_cases.c.h" Py_DECREF(exc_value); Py_DECREF(match_type); - #line 1869 "Python/bytecodes.c" + #line 1868 "Python/bytecodes.c" if (true) goto pop_2_error; } @@ -2698,10 +2697,10 @@ rest = NULL; int res = exception_group_match(exc_value, match_type, &match, &rest); - #line 2702 "Python/generated_cases.c.h" + #line 2701 "Python/generated_cases.c.h" Py_DECREF(exc_value); Py_DECREF(match_type); - #line 1877 "Python/bytecodes.c" + #line 1876 "Python/bytecodes.c" if (res < 0) goto pop_2_error; assert((match == NULL) == (rest == NULL)); @@ -2710,7 +2709,7 @@ if (!Py_IsNone(match)) { PyErr_SetExcInfo(NULL, Py_NewRef(match), NULL); } - #line 2714 "Python/generated_cases.c.h" + #line 2713 "Python/generated_cases.c.h" stack_pointer[-1] = match; stack_pointer[-2] = rest; DISPATCH(); @@ -2720,21 +2719,21 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 1888 "Python/bytecodes.c" + #line 1887 "Python/bytecodes.c" assert(PyExceptionInstance_Check(left)); if (check_except_type_valid(tstate, right) < 0) { - #line 2727 "Python/generated_cases.c.h" + #line 2726 "Python/generated_cases.c.h" Py_DECREF(right); - #line 1891 "Python/bytecodes.c" + #line 1890 "Python/bytecodes.c" if (true) goto pop_1_error; } int res = PyErr_GivenExceptionMatches(left, right); - #line 2734 "Python/generated_cases.c.h" + #line 2733 "Python/generated_cases.c.h" Py_DECREF(right); - #line 1896 "Python/bytecodes.c" + #line 1895 "Python/bytecodes.c" b = Py_NewRef(res ? Py_True : Py_False); - #line 2738 "Python/generated_cases.c.h" + #line 2737 "Python/generated_cases.c.h" stack_pointer[-1] = b; DISPATCH(); } @@ -2743,15 +2742,15 @@ PyObject *fromlist = stack_pointer[-1]; PyObject *level = stack_pointer[-2]; PyObject *res; - #line 1900 "Python/bytecodes.c" + #line 1899 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_name(tstate, frame, name, fromlist, level); - #line 2750 "Python/generated_cases.c.h" + #line 2749 "Python/generated_cases.c.h" Py_DECREF(level); Py_DECREF(fromlist); - #line 1903 "Python/bytecodes.c" + #line 1902 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 2755 "Python/generated_cases.c.h" + #line 2754 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); @@ -2760,29 +2759,29 @@ TARGET(IMPORT_FROM) { PyObject *from = stack_pointer[-1]; PyObject *res; - #line 1907 "Python/bytecodes.c" + #line 1906 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_from(tstate, from, name); if (res == NULL) goto error; - #line 2768 "Python/generated_cases.c.h" + #line 2767 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); } TARGET(JUMP_FORWARD) { - #line 1913 "Python/bytecodes.c" + #line 1912 "Python/bytecodes.c" JUMPBY(oparg); - #line 2777 "Python/generated_cases.c.h" + #line 2776 "Python/generated_cases.c.h" DISPATCH(); } TARGET(JUMP_BACKWARD) { PREDICTED(JUMP_BACKWARD); - #line 1917 "Python/bytecodes.c" + #line 1916 "Python/bytecodes.c" assert(oparg < INSTR_OFFSET()); JUMPBY(-oparg); - #line 2786 "Python/generated_cases.c.h" + #line 2785 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -2790,7 +2789,7 @@ TARGET(POP_JUMP_IF_FALSE) { PREDICTED(POP_JUMP_IF_FALSE); PyObject *cond = stack_pointer[-1]; - #line 1923 "Python/bytecodes.c" + #line 1922 "Python/bytecodes.c" if (Py_IsTrue(cond)) { _Py_DECREF_NO_DEALLOC(cond); } @@ -2800,9 +2799,9 @@ } else { int err = PyObject_IsTrue(cond); - #line 2804 "Python/generated_cases.c.h" + #line 2803 "Python/generated_cases.c.h" Py_DECREF(cond); - #line 1933 "Python/bytecodes.c" + #line 1932 "Python/bytecodes.c" if (err == 0) { JUMPBY(oparg); } @@ -2810,14 +2809,14 @@ if (err < 0) goto pop_1_error; } } - #line 2814 "Python/generated_cases.c.h" + #line 2813 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_TRUE) { PyObject *cond = stack_pointer[-1]; - #line 1943 "Python/bytecodes.c" + #line 1942 "Python/bytecodes.c" if (Py_IsFalse(cond)) { _Py_DECREF_NO_DEALLOC(cond); } @@ -2827,9 +2826,9 @@ } else { int err = PyObject_IsTrue(cond); - #line 2831 "Python/generated_cases.c.h" + #line 2830 "Python/generated_cases.c.h" Py_DECREF(cond); - #line 1953 "Python/bytecodes.c" + #line 1952 "Python/bytecodes.c" if (err > 0) { JUMPBY(oparg); } @@ -2837,67 +2836,67 @@ if (err < 0) goto pop_1_error; } } - #line 2841 "Python/generated_cases.c.h" + #line 2840 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_NOT_NONE) { PyObject *value = stack_pointer[-1]; - #line 1963 "Python/bytecodes.c" + #line 1962 "Python/bytecodes.c" if (!Py_IsNone(value)) { - #line 2850 "Python/generated_cases.c.h" + #line 2849 "Python/generated_cases.c.h" Py_DECREF(value); - #line 1965 "Python/bytecodes.c" + #line 1964 "Python/bytecodes.c" JUMPBY(oparg); } else { _Py_DECREF_NO_DEALLOC(value); } - #line 2858 "Python/generated_cases.c.h" + #line 2857 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_NONE) { PyObject *value = stack_pointer[-1]; - #line 1973 "Python/bytecodes.c" + #line 1972 "Python/bytecodes.c" if (Py_IsNone(value)) { _Py_DECREF_NO_DEALLOC(value); JUMPBY(oparg); } else { - #line 2871 "Python/generated_cases.c.h" + #line 2870 "Python/generated_cases.c.h" Py_DECREF(value); - #line 1979 "Python/bytecodes.c" + #line 1978 "Python/bytecodes.c" } - #line 2875 "Python/generated_cases.c.h" + #line 2874 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(JUMP_BACKWARD_NO_INTERRUPT) { - #line 1983 "Python/bytecodes.c" + #line 1982 "Python/bytecodes.c" /* This bytecode is used in the `yield from` or `await` loop. * If there is an interrupt, we want it handled in the innermost * generator or coroutine, so we deliberately do not check it here. * (see bpo-30039). */ JUMPBY(-oparg); - #line 2888 "Python/generated_cases.c.h" + #line 2887 "Python/generated_cases.c.h" DISPATCH(); } TARGET(GET_LEN) { PyObject *obj = stack_pointer[-1]; PyObject *len_o; - #line 1992 "Python/bytecodes.c" + #line 1991 "Python/bytecodes.c" // PUSH(len(TOS)) Py_ssize_t len_i = PyObject_Length(obj); if (len_i < 0) goto error; len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) goto error; - #line 2901 "Python/generated_cases.c.h" + #line 2900 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = len_o; DISPATCH(); @@ -2908,16 +2907,16 @@ PyObject *type = stack_pointer[-2]; PyObject *subject = stack_pointer[-3]; PyObject *attrs; - #line 2000 "Python/bytecodes.c" + #line 1999 "Python/bytecodes.c" // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. assert(PyTuple_CheckExact(names)); attrs = match_class(tstate, subject, type, oparg, names); - #line 2917 "Python/generated_cases.c.h" + #line 2916 "Python/generated_cases.c.h" Py_DECREF(subject); Py_DECREF(type); Py_DECREF(names); - #line 2005 "Python/bytecodes.c" + #line 2004 "Python/bytecodes.c" if (attrs) { assert(PyTuple_CheckExact(attrs)); // Success! } @@ -2925,7 +2924,7 @@ if (_PyErr_Occurred(tstate)) goto pop_3_error; attrs = Py_NewRef(Py_None); // Failure! } - #line 2929 "Python/generated_cases.c.h" + #line 2928 "Python/generated_cases.c.h" STACK_SHRINK(2); stack_pointer[-1] = attrs; DISPATCH(); @@ -2934,10 +2933,10 @@ TARGET(MATCH_MAPPING) { PyObject *subject = stack_pointer[-1]; PyObject *res; - #line 2015 "Python/bytecodes.c" + #line 2014 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; res = Py_NewRef(match ? Py_True : Py_False); - #line 2941 "Python/generated_cases.c.h" + #line 2940 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; PREDICT(POP_JUMP_IF_FALSE); @@ -2947,10 +2946,10 @@ TARGET(MATCH_SEQUENCE) { PyObject *subject = stack_pointer[-1]; PyObject *res; - #line 2021 "Python/bytecodes.c" + #line 2020 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; res = Py_NewRef(match ? Py_True : Py_False); - #line 2954 "Python/generated_cases.c.h" + #line 2953 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; PREDICT(POP_JUMP_IF_FALSE); @@ -2961,11 +2960,11 @@ PyObject *keys = stack_pointer[-1]; PyObject *subject = stack_pointer[-2]; PyObject *values_or_none; - #line 2027 "Python/bytecodes.c" + #line 2026 "Python/bytecodes.c" // On successful match, PUSH(values). Otherwise, PUSH(None). values_or_none = match_keys(tstate, subject, keys); if (values_or_none == NULL) goto error; - #line 2969 "Python/generated_cases.c.h" + #line 2968 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = values_or_none; DISPATCH(); @@ -2974,14 +2973,14 @@ TARGET(GET_ITER) { PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 2033 "Python/bytecodes.c" + #line 2032 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ iter = PyObject_GetIter(iterable); - #line 2981 "Python/generated_cases.c.h" + #line 2980 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 2036 "Python/bytecodes.c" + #line 2035 "Python/bytecodes.c" if (iter == NULL) goto pop_1_error; - #line 2985 "Python/generated_cases.c.h" + #line 2984 "Python/generated_cases.c.h" stack_pointer[-1] = iter; DISPATCH(); } @@ -2989,7 +2988,7 @@ TARGET(GET_YIELD_FROM_ITER) { PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 2040 "Python/bytecodes.c" + #line 2039 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ if (PyCoro_CheckExact(iterable)) { /* `iterable` is a coroutine */ @@ -3012,11 +3011,11 @@ if (iter == NULL) { goto error; } - #line 3016 "Python/generated_cases.c.h" + #line 3015 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 2063 "Python/bytecodes.c" + #line 2062 "Python/bytecodes.c" } - #line 3020 "Python/generated_cases.c.h" + #line 3019 "Python/generated_cases.c.h" stack_pointer[-1] = iter; PREDICT(LOAD_CONST); DISPATCH(); @@ -3027,7 +3026,7 @@ static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2082 "Python/bytecodes.c" + #line 2081 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyForIterCache *cache = (_PyForIterCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -3058,7 +3057,7 @@ DISPATCH(); } // Common case: no jump, leave it to the code generator - #line 3062 "Python/generated_cases.c.h" + #line 3061 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3066,7 +3065,7 @@ } TARGET(INSTRUMENTED_FOR_ITER) { - #line 2115 "Python/bytecodes.c" + #line 2114 "Python/bytecodes.c" _Py_CODEUNIT *here = next_instr-1; _Py_CODEUNIT *target; PyObject *iter = TOP(); @@ -3092,14 +3091,14 @@ target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1; } INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); - #line 3096 "Python/generated_cases.c.h" + #line 3095 "Python/generated_cases.c.h" DISPATCH(); } TARGET(FOR_ITER_LIST) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2143 "Python/bytecodes.c" + #line 2142 "Python/bytecodes.c" DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); _PyListIterObject *it = (_PyListIterObject *)iter; STAT_INC(FOR_ITER, hit); @@ -3119,7 +3118,7 @@ DISPATCH(); end_for_iter_list: // Common case: no jump, leave it to the code generator - #line 3123 "Python/generated_cases.c.h" + #line 3122 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3129,7 +3128,7 @@ TARGET(FOR_ITER_TUPLE) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2165 "Python/bytecodes.c" + #line 2164 "Python/bytecodes.c" _PyTupleIterObject *it = (_PyTupleIterObject *)iter; DEOPT_IF(Py_TYPE(it) != &PyTupleIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -3149,7 +3148,7 @@ DISPATCH(); end_for_iter_tuple: // Common case: no jump, leave it to the code generator - #line 3153 "Python/generated_cases.c.h" + #line 3152 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3159,7 +3158,7 @@ TARGET(FOR_ITER_RANGE) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2187 "Python/bytecodes.c" + #line 2186 "Python/bytecodes.c" _PyRangeIterObject *r = (_PyRangeIterObject *)iter; DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -3177,7 +3176,7 @@ if (next == NULL) { goto error; } - #line 3181 "Python/generated_cases.c.h" + #line 3180 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3186,7 +3185,7 @@ TARGET(FOR_ITER_GEN) { PyObject *iter = stack_pointer[-1]; - #line 2207 "Python/bytecodes.c" + #line 2206 "Python/bytecodes.c" PyGenObject *gen = (PyGenObject *)iter; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); @@ -3201,14 +3200,14 @@ assert(next_instr->op.code == END_FOR || next_instr->op.code == INSTRUMENTED_END_FOR); DISPATCH_INLINED(gen_frame); - #line 3205 "Python/generated_cases.c.h" + #line 3204 "Python/generated_cases.c.h" } TARGET(BEFORE_ASYNC_WITH) { PyObject *mgr = stack_pointer[-1]; PyObject *exit; PyObject *res; - #line 2224 "Python/bytecodes.c" + #line 2223 "Python/bytecodes.c" PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); if (enter == NULL) { if (!_PyErr_Occurred(tstate)) { @@ -3231,16 +3230,16 @@ Py_DECREF(enter); goto error; } - #line 3235 "Python/generated_cases.c.h" + #line 3234 "Python/generated_cases.c.h" Py_DECREF(mgr); - #line 2247 "Python/bytecodes.c" + #line 2246 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); if (true) goto pop_1_error; } - #line 3244 "Python/generated_cases.c.h" + #line 3243 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -3252,7 +3251,7 @@ PyObject *mgr = stack_pointer[-1]; PyObject *exit; PyObject *res; - #line 2257 "Python/bytecodes.c" + #line 2256 "Python/bytecodes.c" /* pop the context manager, push its __exit__ and the * value returned from calling its __enter__ */ @@ -3278,16 +3277,16 @@ Py_DECREF(enter); goto error; } - #line 3282 "Python/generated_cases.c.h" + #line 3281 "Python/generated_cases.c.h" Py_DECREF(mgr); - #line 2283 "Python/bytecodes.c" + #line 2282 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); if (true) goto pop_1_error; } - #line 3291 "Python/generated_cases.c.h" + #line 3290 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -3299,7 +3298,7 @@ PyObject *lasti = stack_pointer[-3]; PyObject *exit_func = stack_pointer[-4]; PyObject *res; - #line 2292 "Python/bytecodes.c" + #line 2291 "Python/bytecodes.c" /* At the top of the stack are 4 values: - val: TOP = exc_info() - unused: SECOND = previous exception @@ -3320,7 +3319,7 @@ res = PyObject_Vectorcall(exit_func, stack + 1, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); if (res == NULL) goto error; - #line 3324 "Python/generated_cases.c.h" + #line 3323 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); @@ -3329,7 +3328,7 @@ TARGET(PUSH_EXC_INFO) { PyObject *new_exc = stack_pointer[-1]; PyObject *prev_exc; - #line 2315 "Python/bytecodes.c" + #line 2314 "Python/bytecodes.c" _PyErr_StackItem *exc_info = tstate->exc_info; if (exc_info->exc_value != NULL) { prev_exc = exc_info->exc_value; @@ -3339,7 +3338,7 @@ } assert(PyExceptionInstance_Check(new_exc)); exc_info->exc_value = Py_NewRef(new_exc); - #line 3343 "Python/generated_cases.c.h" + #line 3342 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = new_exc; stack_pointer[-2] = prev_exc; @@ -3353,7 +3352,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t keys_version = read_u32(&next_instr[3].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2327 "Python/bytecodes.c" + #line 2326 "Python/bytecodes.c" /* Cached method object */ PyTypeObject *self_cls = Py_TYPE(self); assert(type_version != 0); @@ -3370,7 +3369,7 @@ assert(_PyType_HasFeature(Py_TYPE(res2), Py_TPFLAGS_METHOD_DESCRIPTOR)); res = self; assert(oparg & 1); - #line 3374 "Python/generated_cases.c.h" + #line 3373 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3384,7 +3383,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2346 "Python/bytecodes.c" + #line 2345 "Python/bytecodes.c" PyTypeObject *self_cls = Py_TYPE(self); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); assert(self_cls->tp_dictoffset == 0); @@ -3394,7 +3393,7 @@ res2 = Py_NewRef(descr); res = self; assert(oparg & 1); - #line 3398 "Python/generated_cases.c.h" + #line 3397 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3408,7 +3407,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2358 "Python/bytecodes.c" + #line 2357 "Python/bytecodes.c" PyTypeObject *self_cls = Py_TYPE(self); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); Py_ssize_t dictoffset = self_cls->tp_dictoffset; @@ -3422,7 +3421,7 @@ res2 = Py_NewRef(descr); res = self; assert(oparg & 1); - #line 3426 "Python/generated_cases.c.h" + #line 3425 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3431,16 +3430,16 @@ } TARGET(KW_NAMES) { - #line 2374 "Python/bytecodes.c" + #line 2373 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg < PyTuple_GET_SIZE(frame->f_code->co_consts)); kwnames = GETITEM(frame->f_code->co_consts, oparg); - #line 3439 "Python/generated_cases.c.h" + #line 3438 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_CALL) { - #line 2380 "Python/bytecodes.c" + #line 2379 "Python/bytecodes.c" int is_meth = PEEK(oparg+2) != NULL; int total_args = oparg + is_meth; PyObject *function = PEEK(total_args + 1); @@ -3453,7 +3452,7 @@ _PyCallCache *cache = (_PyCallCache *)next_instr; INCREMENT_ADAPTIVE_COUNTER(cache->counter); GO_TO_INSTRUCTION(CALL); - #line 3457 "Python/generated_cases.c.h" + #line 3456 "Python/generated_cases.c.h" } TARGET(CALL) { @@ -3463,7 +3462,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2425 "Python/bytecodes.c" + #line 2424 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3544,7 +3543,7 @@ Py_DECREF(args[i]); } if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3548 "Python/generated_cases.c.h" + #line 3547 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3556,7 +3555,7 @@ TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; - #line 2512 "Python/bytecodes.c" + #line 2511 "Python/bytecodes.c" DEOPT_IF(method != NULL, CALL); DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); STAT_INC(CALL, hit); @@ -3566,7 +3565,7 @@ PEEK(oparg + 2) = Py_NewRef(meth); // method Py_DECREF(callable); GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); - #line 3570 "Python/generated_cases.c.h" + #line 3569 "Python/generated_cases.c.h" } TARGET(CALL_PY_EXACT_ARGS) { @@ -3575,7 +3574,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); - #line 2524 "Python/bytecodes.c" + #line 2523 "Python/bytecodes.c" assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; @@ -3600,7 +3599,7 @@ STACK_SHRINK(oparg + 2); JUMPBY(INLINE_CACHE_ENTRIES_CALL); DISPATCH_INLINED(new_frame); - #line 3604 "Python/generated_cases.c.h" + #line 3603 "Python/generated_cases.c.h" } TARGET(CALL_PY_WITH_DEFAULTS) { @@ -3609,7 +3608,7 @@ PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); uint16_t min_args = read_u16(&next_instr[3].cache); - #line 2551 "Python/bytecodes.c" + #line 2550 "Python/bytecodes.c" assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; @@ -3639,7 +3638,7 @@ STACK_SHRINK(oparg + 2); JUMPBY(INLINE_CACHE_ENTRIES_CALL); DISPATCH_INLINED(new_frame); - #line 3643 "Python/generated_cases.c.h" + #line 3642 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_TYPE_1) { @@ -3647,7 +3646,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2583 "Python/bytecodes.c" + #line 2582 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3657,7 +3656,7 @@ res = Py_NewRef(Py_TYPE(obj)); Py_DECREF(obj); Py_DECREF(&PyType_Type); // I.e., callable - #line 3661 "Python/generated_cases.c.h" + #line 3660 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3670,7 +3669,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2595 "Python/bytecodes.c" + #line 2594 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3681,7 +3680,7 @@ Py_DECREF(arg); Py_DECREF(&PyUnicode_Type); // I.e., callable if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3685 "Python/generated_cases.c.h" + #line 3684 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3695,7 +3694,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2609 "Python/bytecodes.c" + #line 2608 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3706,7 +3705,7 @@ Py_DECREF(arg); Py_DECREF(&PyTuple_Type); // I.e., tuple if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3710 "Python/generated_cases.c.h" + #line 3709 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3720,7 +3719,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2623 "Python/bytecodes.c" + #line 2622 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3742,7 +3741,7 @@ } Py_DECREF(tp); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3746 "Python/generated_cases.c.h" + #line 3745 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3756,7 +3755,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2648 "Python/bytecodes.c" + #line 2647 "Python/bytecodes.c" /* Builtin METH_O functions */ assert(kwnames == NULL); int is_meth = method != NULL; @@ -3784,7 +3783,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3788 "Python/generated_cases.c.h" + #line 3787 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3798,7 +3797,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2679 "Python/bytecodes.c" + #line 2678 "Python/bytecodes.c" /* Builtin METH_FASTCALL functions, without keywords */ assert(kwnames == NULL); int is_meth = method != NULL; @@ -3830,7 +3829,7 @@ 'invalid'). In those cases an exception is set, so we must handle it. */ - #line 3834 "Python/generated_cases.c.h" + #line 3833 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3844,7 +3843,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2714 "Python/bytecodes.c" + #line 2713 "Python/bytecodes.c" /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ int is_meth = method != NULL; int total_args = oparg; @@ -3876,7 +3875,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3880 "Python/generated_cases.c.h" + #line 3879 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3890,7 +3889,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2749 "Python/bytecodes.c" + #line 2748 "Python/bytecodes.c" assert(kwnames == NULL); /* len(o) */ int is_meth = method != NULL; @@ -3915,7 +3914,7 @@ Py_DECREF(callable); Py_DECREF(arg); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3919 "Python/generated_cases.c.h" + #line 3918 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3928,7 +3927,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2776 "Python/bytecodes.c" + #line 2775 "Python/bytecodes.c" assert(kwnames == NULL); /* isinstance(o, o2) */ int is_meth = method != NULL; @@ -3955,7 +3954,7 @@ Py_DECREF(cls); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3959 "Python/generated_cases.c.h" + #line 3958 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3967,7 +3966,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *self = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; - #line 2806 "Python/bytecodes.c" + #line 2805 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); assert(method != NULL); @@ -3985,14 +3984,14 @@ JUMPBY(INLINE_CACHE_ENTRIES_CALL + 1); assert(next_instr[-1].op.code == POP_TOP); DISPATCH(); - #line 3989 "Python/generated_cases.c.h" + #line 3988 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) { PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2826 "Python/bytecodes.c" + #line 2825 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -4023,7 +4022,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4027 "Python/generated_cases.c.h" + #line 4026 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4036,7 +4035,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2860 "Python/bytecodes.c" + #line 2859 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -4065,7 +4064,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4069 "Python/generated_cases.c.h" + #line 4068 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4078,7 +4077,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2892 "Python/bytecodes.c" + #line 2891 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 0 || oparg == 1); int is_meth = method != NULL; @@ -4107,7 +4106,7 @@ Py_DECREF(self); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4111 "Python/generated_cases.c.h" + #line 4110 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4120,7 +4119,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2924 "Python/bytecodes.c" + #line 2923 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -4148,7 +4147,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4152 "Python/generated_cases.c.h" + #line 4151 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4158,9 +4157,9 @@ } TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { - #line 2955 "Python/bytecodes.c" + #line 2954 "Python/bytecodes.c" GO_TO_INSTRUCTION(CALL_FUNCTION_EX); - #line 4164 "Python/generated_cases.c.h" + #line 4163 "Python/generated_cases.c.h" } TARGET(CALL_FUNCTION_EX) { @@ -4169,7 +4168,7 @@ PyObject *callargs = stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))]; PyObject *func = stack_pointer[-(2 + ((oparg & 1) ? 1 : 0))]; PyObject *result; - #line 2959 "Python/bytecodes.c" + #line 2958 "Python/bytecodes.c" // DICT_MERGE is called before this opcode if there are kwargs. // It converts all dict subtypes in kwargs into regular dicts. assert(kwargs == NULL || PyDict_CheckExact(kwargs)); @@ -4212,14 +4211,14 @@ else { result = PyObject_Call(func, callargs, kwargs); } - #line 4216 "Python/generated_cases.c.h" + #line 4215 "Python/generated_cases.c.h" Py_DECREF(func); Py_DECREF(callargs); Py_XDECREF(kwargs); - #line 3002 "Python/bytecodes.c" + #line 3001 "Python/bytecodes.c" assert(PEEK(3 + (oparg & 1)) == NULL); if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } - #line 4223 "Python/generated_cases.c.h" + #line 4222 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 1) ? 1 : 0)); STACK_SHRINK(2); stack_pointer[-1] = result; @@ -4234,7 +4233,7 @@ PyObject *kwdefaults = (oparg & 0x02) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0))] : NULL; PyObject *defaults = (oparg & 0x01) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x01) ? 1 : 0))] : NULL; PyObject *func; - #line 3012 "Python/bytecodes.c" + #line 3011 "Python/bytecodes.c" PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); @@ -4263,14 +4262,14 @@ func_obj->func_version = ((PyCodeObject *)codeobj)->co_version; func = (PyObject *)func_obj; - #line 4267 "Python/generated_cases.c.h" + #line 4266 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 0x01) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x08) ? 1 : 0)); stack_pointer[-1] = func; DISPATCH(); } TARGET(RETURN_GENERATOR) { - #line 3043 "Python/bytecodes.c" + #line 3042 "Python/bytecodes.c" assert(PyFunction_Check(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); @@ -4291,7 +4290,7 @@ frame = cframe.current_frame = prev; _PyFrame_StackPush(frame, (PyObject *)gen); goto resume_frame; - #line 4295 "Python/generated_cases.c.h" + #line 4294 "Python/generated_cases.c.h" } TARGET(BUILD_SLICE) { @@ -4299,15 +4298,15 @@ PyObject *stop = stack_pointer[-(1 + ((oparg == 3) ? 1 : 0))]; PyObject *start = stack_pointer[-(2 + ((oparg == 3) ? 1 : 0))]; PyObject *slice; - #line 3066 "Python/bytecodes.c" + #line 3065 "Python/bytecodes.c" slice = PySlice_New(start, stop, step); - #line 4305 "Python/generated_cases.c.h" + #line 4304 "Python/generated_cases.c.h" Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); - #line 3068 "Python/bytecodes.c" + #line 3067 "Python/bytecodes.c" if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } - #line 4311 "Python/generated_cases.c.h" + #line 4310 "Python/generated_cases.c.h" STACK_SHRINK(((oparg == 3) ? 1 : 0)); STACK_SHRINK(1); stack_pointer[-1] = slice; @@ -4318,7 +4317,7 @@ PyObject *fmt_spec = ((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? stack_pointer[-((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))] : NULL; PyObject *value = stack_pointer[-(1 + (((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))]; PyObject *result; - #line 3072 "Python/bytecodes.c" + #line 3071 "Python/bytecodes.c" /* Handles f-string value formatting. */ PyObject *(*conv_fn)(PyObject *); int which_conversion = oparg & FVC_MASK; @@ -4353,7 +4352,7 @@ Py_DECREF(value); Py_XDECREF(fmt_spec); if (result == NULL) { STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); goto pop_1_error; } - #line 4357 "Python/generated_cases.c.h" + #line 4356 "Python/generated_cases.c.h" STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); stack_pointer[-1] = result; DISPATCH(); @@ -4362,10 +4361,10 @@ TARGET(COPY) { PyObject *bottom = stack_pointer[-(1 + (oparg-1))]; PyObject *top; - #line 3109 "Python/bytecodes.c" + #line 3108 "Python/bytecodes.c" assert(oparg > 0); top = Py_NewRef(bottom); - #line 4369 "Python/generated_cases.c.h" + #line 4368 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = top; DISPATCH(); @@ -4377,7 +4376,7 @@ PyObject *rhs = stack_pointer[-1]; PyObject *lhs = stack_pointer[-2]; PyObject *res; - #line 3114 "Python/bytecodes.c" + #line 3113 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -4392,12 +4391,12 @@ assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); assert(binary_ops[oparg]); res = binary_ops[oparg](lhs, rhs); - #line 4396 "Python/generated_cases.c.h" + #line 4395 "Python/generated_cases.c.h" Py_DECREF(lhs); Py_DECREF(rhs); - #line 3129 "Python/bytecodes.c" + #line 3128 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 4401 "Python/generated_cases.c.h" + #line 4400 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -4407,22 +4406,21 @@ TARGET(SWAP) { PyObject *top = stack_pointer[-1]; PyObject *bottom = stack_pointer[-(2 + (oparg-2))]; - #line 3134 "Python/bytecodes.c" + #line 3133 "Python/bytecodes.c" assert(oparg >= 2); - #line 4413 "Python/generated_cases.c.h" + #line 4412 "Python/generated_cases.c.h" stack_pointer[-1] = bottom; stack_pointer[-(2 + (oparg-2))] = top; DISPATCH(); } TARGET(INSTRUMENTED_LINE) { - #line 3138 "Python/bytecodes.c" + #line 3137 "Python/bytecodes.c" _Py_CODEUNIT *here = next_instr-1; _PyFrame_SetStackPointer(frame, stack_pointer); int original_opcode = _Py_call_instrumentation_line( tstate, frame, here); - stack_pointer = _PyFrame_GetStackPointer(frame); - frame->stacktop = -1; + stack_pointer = _PyFrame_FetchStackPointer(frame); if (original_opcode < 0) { next_instr = here+1; goto error; @@ -4437,11 +4435,11 @@ } opcode = original_opcode; DISPATCH_GOTO(); - #line 4441 "Python/generated_cases.c.h" + #line 4439 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_INSTRUCTION) { - #line 3161 "Python/bytecodes.c" + #line 3159 "Python/bytecodes.c" int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, next_instr-1); if (next_opcode < 0) goto error; @@ -4453,26 +4451,26 @@ assert(next_opcode > 0 && next_opcode < 256); opcode = next_opcode; DISPATCH_GOTO(); - #line 4457 "Python/generated_cases.c.h" + #line 4455 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_JUMP_FORWARD) { - #line 3176 "Python/bytecodes.c" + #line 3174 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); - #line 4463 "Python/generated_cases.c.h" + #line 4461 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_JUMP_BACKWARD) { - #line 3181 "Python/bytecodes.c" + #line 3179 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr-oparg, PY_MONITORING_EVENT_JUMP); - #line 4470 "Python/generated_cases.c.h" + #line 4468 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { - #line 3186 "Python/bytecodes.c" + #line 3184 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); Py_DECREF(cond); @@ -4481,12 +4479,12 @@ assert(err == 0 || err == 1); int offset = err*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4485 "Python/generated_cases.c.h" + #line 4483 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { - #line 3197 "Python/bytecodes.c" + #line 3195 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); Py_DECREF(cond); @@ -4495,12 +4493,12 @@ assert(err == 0 || err == 1); int offset = (1-err)*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4499 "Python/generated_cases.c.h" + #line 4497 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { - #line 3208 "Python/bytecodes.c" + #line 3206 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4513,12 +4511,12 @@ offset = 0; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4517 "Python/generated_cases.c.h" + #line 4515 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { - #line 3223 "Python/bytecodes.c" + #line 3221 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4531,22 +4529,22 @@ offset = oparg; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4535 "Python/generated_cases.c.h" + #line 4533 "Python/generated_cases.c.h" DISPATCH(); } TARGET(EXTENDED_ARG) { - #line 3238 "Python/bytecodes.c" + #line 3236 "Python/bytecodes.c" assert(oparg); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); - #line 4546 "Python/generated_cases.c.h" + #line 4544 "Python/generated_cases.c.h" } TARGET(CACHE) { - #line 3246 "Python/bytecodes.c" + #line 3244 "Python/bytecodes.c" Py_UNREACHABLE(); - #line 4552 "Python/generated_cases.c.h" + #line 4550 "Python/generated_cases.c.h" } From 6611c72929d2acb1915fa1ecc32b5f81c2d347bd Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 24 Mar 2023 15:30:08 +0000 Subject: [PATCH 072/116] Fixups from code review --- Python/instrumentation.c | 6 +++--- Python/legacy_tracing.c | 6 ------ 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 0147b4c35f1647..4a1fde044f10bf 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -612,8 +612,8 @@ de_instrument(PyCodeObject *code, int offset, int event) case INSTRUMENTED_LINE: { _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[offset]; - int orignal_opcode = lines->original_opcode; - int deinstrumented = DE_INSTRUMENT[orignal_opcode]; + int original_opcode = lines->original_opcode; + int deinstrumented = DE_INSTRUMENT[original_opcode]; if (deinstrumented) { lines->original_opcode = opcode = deinstrumented; } @@ -694,7 +694,7 @@ de_instrument_per_instruction(PyCodeObject *code, int offset) } } assert(instr->op.code != INSTRUMENTED_INSTRUCTION); - /* Keep things clean for snaity check */ + /* Keep things clean for sanity check */ code->_co_monitoring->per_instruction_opcodes[offset] = 0; } diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index 514afc00f7adf0..a0a47fa6e6abea 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -4,13 +4,8 @@ #include #include "Python.h" -#include "opcode.h" #include "pycore_ceval.h" -#include "pycore_instruments.h" #include "pycore_object.h" -#include "pycore_pyerrors.h" -#include "pycore_pymem.h" -#include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_sysmodule.h" typedef struct _PyLegacyEventHandler { @@ -40,7 +35,6 @@ call_profile_func(_PyLegacyEventHandler *self, PyObject *arg) Py_RETURN_NONE; } PyFrameObject* frame = PyEval_GetFrame(); - assert(frame); if (frame == NULL) { PyErr_SetString(PyExc_SystemError, "Missing frame when calling profile function."); From 50d28f16f492dbc6e6f70daad3cc1ac1c709b904 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 24 Mar 2023 21:26:27 +0000 Subject: [PATCH 073/116] Improve debugging output and make a few things more robust. --- Include/internal/pycore_opcode.h | 34 +- Include/opcode.h | 111 +++--- Lib/importlib/_bootstrap_external.py | 2 +- Lib/opcode.py | 2 + Objects/codeobject.c | 27 +- Programs/test_frozenmain.h | 16 +- Python/bytecodes.c | 5 + Python/generated_cases.c.h | 550 ++++++++++++++------------- Python/instrumentation.c | 131 ++++--- Python/opcode_metadata.h | 5 + Python/opcode_targets.h | 32 +- 11 files changed, 484 insertions(+), 431 deletions(-) diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index 53a446e378d143..b8d26b663353b0 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -206,6 +206,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [PUSH_NULL] = PUSH_NULL, [RAISE_VARARGS] = RAISE_VARARGS, [RERAISE] = RERAISE, + [RESERVED] = RESERVED, [RESUME] = RESUME, [RETURN_CONST] = RETURN_CONST, [RETURN_GENERATOR] = RETURN_GENERATOR, @@ -262,6 +263,7 @@ static const char *const _PyOpcode_OpName[263] = { [BINARY_OP_MULTIPLY_INT] = "BINARY_OP_MULTIPLY_INT", [UNARY_INVERT] = "UNARY_INVERT", [BINARY_OP_SUBTRACT_FLOAT] = "BINARY_OP_SUBTRACT_FLOAT", + [RESERVED] = "RESERVED", [BINARY_OP_SUBTRACT_INT] = "BINARY_OP_SUBTRACT_INT", [BINARY_SUBSCR_DICT] = "BINARY_SUBSCR_DICT", [BINARY_SUBSCR_GETITEM] = "BINARY_SUBSCR_GETITEM", @@ -269,20 +271,20 @@ static const char *const _PyOpcode_OpName[263] = { [BINARY_SUBSCR_TUPLE_INT] = "BINARY_SUBSCR_TUPLE_INT", [CALL_PY_EXACT_ARGS] = "CALL_PY_EXACT_ARGS", [CALL_PY_WITH_DEFAULTS] = "CALL_PY_WITH_DEFAULTS", - [CALL_BOUND_METHOD_EXACT_ARGS] = "CALL_BOUND_METHOD_EXACT_ARGS", [BINARY_SUBSCR] = "BINARY_SUBSCR", [BINARY_SLICE] = "BINARY_SLICE", [STORE_SLICE] = "STORE_SLICE", + [CALL_BOUND_METHOD_EXACT_ARGS] = "CALL_BOUND_METHOD_EXACT_ARGS", [CALL_BUILTIN_CLASS] = "CALL_BUILTIN_CLASS", - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", [GET_LEN] = "GET_LEN", [MATCH_MAPPING] = "MATCH_MAPPING", [MATCH_SEQUENCE] = "MATCH_SEQUENCE", [MATCH_KEYS] = "MATCH_KEYS", - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", [PUSH_EXC_INFO] = "PUSH_EXC_INFO", [CHECK_EXC_MATCH] = "CHECK_EXC_MATCH", [CHECK_EG_MATCH] = "CHECK_EG_MATCH", + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", [CALL_NO_KW_BUILTIN_FAST] = "CALL_NO_KW_BUILTIN_FAST", [CALL_NO_KW_BUILTIN_O] = "CALL_NO_KW_BUILTIN_O", [CALL_NO_KW_ISINSTANCE] = "CALL_NO_KW_ISINSTANCE", @@ -293,7 +295,6 @@ static const char *const _PyOpcode_OpName[263] = { [CALL_NO_KW_METHOD_DESCRIPTOR_O] = "CALL_NO_KW_METHOD_DESCRIPTOR_O", [CALL_NO_KW_STR_1] = "CALL_NO_KW_STR_1", [CALL_NO_KW_TUPLE_1] = "CALL_NO_KW_TUPLE_1", - [CALL_NO_KW_TYPE_1] = "CALL_NO_KW_TYPE_1", [WITH_EXCEPT_START] = "WITH_EXCEPT_START", [GET_AITER] = "GET_AITER", [GET_ANEXT] = "GET_ANEXT", @@ -301,39 +302,39 @@ static const char *const _PyOpcode_OpName[263] = { [BEFORE_WITH] = "BEFORE_WITH", [END_ASYNC_FOR] = "END_ASYNC_FOR", [CLEANUP_THROW] = "CLEANUP_THROW", + [CALL_NO_KW_TYPE_1] = "CALL_NO_KW_TYPE_1", [COMPARE_OP_FLOAT] = "COMPARE_OP_FLOAT", [COMPARE_OP_INT] = "COMPARE_OP_INT", [COMPARE_OP_STR] = "COMPARE_OP_STR", - [FOR_ITER_LIST] = "FOR_ITER_LIST", [STORE_SUBSCR] = "STORE_SUBSCR", [DELETE_SUBSCR] = "DELETE_SUBSCR", + [FOR_ITER_LIST] = "FOR_ITER_LIST", [FOR_ITER_TUPLE] = "FOR_ITER_TUPLE", [FOR_ITER_RANGE] = "FOR_ITER_RANGE", [FOR_ITER_GEN] = "FOR_ITER_GEN", [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", - [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", [GET_ITER] = "GET_ITER", [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", - [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", + [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS", + [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", - [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR", [RETURN_GENERATOR] = "RETURN_GENERATOR", + [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST", [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", - [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST", [RETURN_VALUE] = "RETURN_VALUE", - [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", + [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST", [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", + [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", - [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", [POP_EXCEPT] = "POP_EXCEPT", [STORE_NAME] = "STORE_NAME", [DELETE_NAME] = "DELETE_NAME", @@ -356,9 +357,9 @@ static const char *const _PyOpcode_OpName[263] = { [IMPORT_NAME] = "IMPORT_NAME", [IMPORT_FROM] = "IMPORT_FROM", [JUMP_FORWARD] = "JUMP_FORWARD", + [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST", - [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE", [POP_JUMP_IF_TRUE] = "POP_JUMP_IF_TRUE", [LOAD_GLOBAL] = "LOAD_GLOBAL", @@ -386,9 +387,9 @@ static const char *const _PyOpcode_OpName[263] = { [STORE_DEREF] = "STORE_DEREF", [DELETE_DEREF] = "DELETE_DEREF", [JUMP_BACKWARD] = "JUMP_BACKWARD", - [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", + [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX", - [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", + [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", [EXTENDED_ARG] = "EXTENDED_ARG", [LIST_APPEND] = "LIST_APPEND", [SET_ADD] = "SET_ADD", @@ -398,14 +399,14 @@ static const char *const _PyOpcode_OpName[263] = { [YIELD_VALUE] = "YIELD_VALUE", [RESUME] = "RESUME", [MATCH_CLASS] = "MATCH_CLASS", + [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", - [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", [FORMAT_VALUE] = "FORMAT_VALUE", [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", [BUILD_STRING] = "BUILD_STRING", + [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", [SEND_GEN] = "SEND_GEN", - [160] = "<160>", [161] = "<161>", [LIST_EXTEND] = "LIST_EXTEND", [SET_UPDATE] = "SET_UPDATE", @@ -512,7 +513,6 @@ static const char *const _PyOpcode_OpName[263] = { #endif #define EXTRA_CASES \ - case 160: \ case 161: \ case 166: \ case 167: \ diff --git a/Include/opcode.h b/Include/opcode.h index e619ed968a8e0a..701d7f81837e5e 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -18,6 +18,7 @@ extern "C" { #define UNARY_NEGATIVE 11 #define UNARY_NOT 12 #define UNARY_INVERT 15 +#define RESERVED 17 #define BINARY_SUBSCR 25 #define BINARY_SLICE 26 #define STORE_SLICE 27 @@ -148,61 +149,61 @@ extern "C" { #define BINARY_OP_MULTIPLY_FLOAT 13 #define BINARY_OP_MULTIPLY_INT 14 #define BINARY_OP_SUBTRACT_FLOAT 16 -#define BINARY_OP_SUBTRACT_INT 17 -#define BINARY_SUBSCR_DICT 18 -#define BINARY_SUBSCR_GETITEM 19 -#define BINARY_SUBSCR_LIST_INT 20 -#define BINARY_SUBSCR_TUPLE_INT 21 -#define CALL_PY_EXACT_ARGS 22 -#define CALL_PY_WITH_DEFAULTS 23 -#define CALL_BOUND_METHOD_EXACT_ARGS 24 -#define CALL_BUILTIN_CLASS 28 -#define CALL_BUILTIN_FAST_WITH_KEYWORDS 29 -#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 34 -#define CALL_NO_KW_BUILTIN_FAST 38 -#define CALL_NO_KW_BUILTIN_O 39 -#define CALL_NO_KW_ISINSTANCE 40 -#define CALL_NO_KW_LEN 41 -#define CALL_NO_KW_LIST_APPEND 42 -#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 43 -#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 44 -#define CALL_NO_KW_METHOD_DESCRIPTOR_O 45 -#define CALL_NO_KW_STR_1 46 -#define CALL_NO_KW_TUPLE_1 47 -#define CALL_NO_KW_TYPE_1 48 -#define COMPARE_OP_FLOAT 56 -#define COMPARE_OP_INT 57 -#define COMPARE_OP_STR 58 -#define FOR_ITER_LIST 59 -#define FOR_ITER_TUPLE 62 -#define FOR_ITER_RANGE 63 -#define FOR_ITER_GEN 64 -#define LOAD_ATTR_CLASS 65 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 66 -#define LOAD_ATTR_INSTANCE_VALUE 67 -#define LOAD_ATTR_MODULE 70 -#define LOAD_ATTR_PROPERTY 72 -#define LOAD_ATTR_SLOT 73 -#define LOAD_ATTR_WITH_HINT 76 -#define LOAD_ATTR_METHOD_LAZY_DICT 77 -#define LOAD_ATTR_METHOD_NO_DICT 78 -#define LOAD_ATTR_METHOD_WITH_VALUES 79 -#define LOAD_CONST__LOAD_FAST 80 -#define LOAD_FAST__LOAD_CONST 81 -#define LOAD_FAST__LOAD_FAST 82 -#define LOAD_GLOBAL_BUILTIN 84 -#define LOAD_GLOBAL_MODULE 86 -#define STORE_ATTR_INSTANCE_VALUE 87 -#define STORE_ATTR_SLOT 88 -#define STORE_ATTR_WITH_HINT 111 -#define STORE_FAST__LOAD_FAST 112 -#define STORE_FAST__STORE_FAST 113 -#define STORE_SUBSCR_DICT 141 -#define STORE_SUBSCR_LIST_INT 143 -#define UNPACK_SEQUENCE_LIST 153 -#define UNPACK_SEQUENCE_TUPLE 154 -#define UNPACK_SEQUENCE_TWO_TUPLE 158 -#define SEND_GEN 159 +#define BINARY_OP_SUBTRACT_INT 18 +#define BINARY_SUBSCR_DICT 19 +#define BINARY_SUBSCR_GETITEM 20 +#define BINARY_SUBSCR_LIST_INT 21 +#define BINARY_SUBSCR_TUPLE_INT 22 +#define CALL_PY_EXACT_ARGS 23 +#define CALL_PY_WITH_DEFAULTS 24 +#define CALL_BOUND_METHOD_EXACT_ARGS 28 +#define CALL_BUILTIN_CLASS 29 +#define CALL_BUILTIN_FAST_WITH_KEYWORDS 34 +#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 38 +#define CALL_NO_KW_BUILTIN_FAST 39 +#define CALL_NO_KW_BUILTIN_O 40 +#define CALL_NO_KW_ISINSTANCE 41 +#define CALL_NO_KW_LEN 42 +#define CALL_NO_KW_LIST_APPEND 43 +#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 44 +#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 45 +#define CALL_NO_KW_METHOD_DESCRIPTOR_O 46 +#define CALL_NO_KW_STR_1 47 +#define CALL_NO_KW_TUPLE_1 48 +#define CALL_NO_KW_TYPE_1 56 +#define COMPARE_OP_FLOAT 57 +#define COMPARE_OP_INT 58 +#define COMPARE_OP_STR 59 +#define FOR_ITER_LIST 62 +#define FOR_ITER_TUPLE 63 +#define FOR_ITER_RANGE 64 +#define FOR_ITER_GEN 65 +#define LOAD_ATTR_CLASS 66 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 67 +#define LOAD_ATTR_INSTANCE_VALUE 70 +#define LOAD_ATTR_MODULE 72 +#define LOAD_ATTR_PROPERTY 73 +#define LOAD_ATTR_SLOT 76 +#define LOAD_ATTR_WITH_HINT 77 +#define LOAD_ATTR_METHOD_LAZY_DICT 78 +#define LOAD_ATTR_METHOD_NO_DICT 79 +#define LOAD_ATTR_METHOD_WITH_VALUES 80 +#define LOAD_CONST__LOAD_FAST 81 +#define LOAD_FAST__LOAD_CONST 82 +#define LOAD_FAST__LOAD_FAST 84 +#define LOAD_GLOBAL_BUILTIN 86 +#define LOAD_GLOBAL_MODULE 87 +#define STORE_ATTR_INSTANCE_VALUE 88 +#define STORE_ATTR_SLOT 111 +#define STORE_ATTR_WITH_HINT 112 +#define STORE_FAST__LOAD_FAST 113 +#define STORE_FAST__STORE_FAST 141 +#define STORE_SUBSCR_DICT 143 +#define STORE_SUBSCR_LIST_INT 153 +#define UNPACK_SEQUENCE_LIST 154 +#define UNPACK_SEQUENCE_TUPLE 158 +#define UNPACK_SEQUENCE_TWO_TUPLE 159 +#define SEND_GEN 160 #define HAS_ARG(op) ((((op) >= HAVE_ARGUMENT) && (!IS_PSEUDO_OPCODE(op)))\ || ((op) == JUMP) \ diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index f16e06c018ccf0..aa00a10474cdc5 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -454,7 +454,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3525).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3526).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c diff --git a/Lib/opcode.py b/Lib/opcode.py index 2057d451f0eb26..3ca4173e6bd83b 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -92,6 +92,8 @@ def pseudo_op(name, op, real_ops): def_op('UNARY_INVERT', 15) +def_op('RESERVED', 17) + def_op('BINARY_SUBSCR', 25) def_op('BINARY_SLICE', 26) def_op('STORE_SLICE', 27) diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 171f514b04575e..024bd7bf78daa5 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1487,9 +1487,12 @@ deopt_code(PyCodeObject *code, _Py_CODEUNIT *instructions) int opcode = _Py_GetBaseOpcode(code, i); int caches = _PyOpcode_Caches[opcode]; instructions[i].op.code = opcode; - while (caches--) { - instructions[++i].op.code = CACHE; - instructions[i].op.arg = 0; + if (caches) { + instructions[i+1].cache = adaptive_counter_warmup(); + for (int j = 2; j <= caches; j++) { + instructions[i+j].cache = 0; + } + i += caches; } } } @@ -2288,6 +2291,24 @@ _PyStaticCode_Fini(PyCodeObject *co) PyObject_ClearWeakRefs((PyObject *)co); co->co_weakreflist = NULL; } + _PyCoMonitoringData *data = co->_co_monitoring; + if (data) { + if (data->tools) { + PyMem_Free(data->tools); + } + if (data->lines) { + PyMem_Free(data->lines); + } + if (data->line_tools) { + PyMem_Free(data->line_tools); + } + if (data->per_instruction_opcodes) { + PyMem_Free(data->per_instruction_opcodes); + } + if (data->per_instruction_tools) { + PyMem_Free(data->per_instruction_tools); + } + } } int diff --git a/Programs/test_frozenmain.h b/Programs/test_frozenmain.h index 8e5055bd7bceb1..57635a398d1df0 100644 --- a/Programs/test_frozenmain.h +++ b/Programs/test_frozenmain.h @@ -3,16 +3,16 @@ unsigned char M_test_frozenmain[] = { 227,0,0,0,0,0,0,0,0,0,0,0,0,8,0,0, 0,0,0,0,0,243,182,0,0,0,151,0,100,0,100,1, 108,0,90,0,100,0,100,1,108,1,90,1,2,0,101,2, - 100,2,171,1,0,0,0,0,0,0,0,0,1,0,2,0, - 101,2,100,3,101,0,106,6,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,171,2,0,0,0,0, - 0,0,0,0,1,0,2,0,101,1,106,8,0,0,0,0, + 100,2,171,1,17,0,0,0,0,0,0,0,1,0,2,0, + 101,2,100,3,101,0,106,6,17,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,171,2,17,0,0,0, + 0,0,0,0,1,0,2,0,101,1,106,8,17,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,171,0, - 0,0,0,0,0,0,0,0,100,4,25,0,0,0,0,0, - 0,0,0,0,90,5,100,5,68,0,93,23,0,0,90,6, + 17,0,0,0,0,0,0,0,100,4,25,0,17,0,0,0, + 0,0,0,0,90,5,100,5,68,0,93,23,17,0,90,6, 2,0,101,2,100,6,101,6,155,0,100,7,101,5,101,6, - 25,0,0,0,0,0,0,0,0,0,155,0,157,4,171,1, - 0,0,0,0,0,0,0,0,1,0,140,25,4,0,121,1, + 25,0,17,0,0,0,0,0,0,0,155,0,157,4,171,1, + 17,0,0,0,0,0,0,0,1,0,140,25,4,0,121,1, 41,8,233,0,0,0,0,78,122,18,70,114,111,122,101,110, 32,72,101,108,108,111,32,87,111,114,108,100,122,8,115,121, 115,46,97,114,103,118,218,6,99,111,110,102,105,103,41,5, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index f8df0ccb030889..6768513f8d4799 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1234,6 +1234,7 @@ dummy_func( DEOPT_IF(!PyDict_CheckExact(BUILTINS()), LOAD_GLOBAL); PyDictObject *mdict = (PyDictObject *)GLOBALS(); PyDictObject *bdict = (PyDictObject *)BUILTINS(); + assert(opcode == LOAD_GLOBAL_BUILTIN); DEOPT_IF(mdict->ma_keys->dk_version != mod_version, LOAD_GLOBAL); DEOPT_IF(bdict->ma_keys->dk_version != bltn_version, LOAD_GLOBAL); assert(DK_IS_UNICODE(bdict->ma_keys)); @@ -3244,6 +3245,10 @@ dummy_func( Py_UNREACHABLE(); } + inst(RESERVED, (--)) { + assert(0 && "Executing RESERVED instruction."); + } + // END BYTECODES // diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 08a294e4803e9e..1c9f94ef40a2e1 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1760,6 +1760,7 @@ DEOPT_IF(!PyDict_CheckExact(BUILTINS()), LOAD_GLOBAL); PyDictObject *mdict = (PyDictObject *)GLOBALS(); PyDictObject *bdict = (PyDictObject *)BUILTINS(); + assert(opcode == LOAD_GLOBAL_BUILTIN); DEOPT_IF(mdict->ma_keys->dk_version != mod_version, LOAD_GLOBAL); DEOPT_IF(bdict->ma_keys->dk_version != bltn_version, LOAD_GLOBAL); assert(DK_IS_UNICODE(bdict->ma_keys)); @@ -1769,7 +1770,7 @@ Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; - #line 1773 "Python/generated_cases.c.h" + #line 1774 "Python/generated_cases.c.h" STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -1779,16 +1780,16 @@ } TARGET(DELETE_FAST) { - #line 1249 "Python/bytecodes.c" + #line 1250 "Python/bytecodes.c" PyObject *v = GETLOCAL(oparg); if (v == NULL) goto unbound_local_error; SETLOCAL(oparg, NULL); - #line 1787 "Python/generated_cases.c.h" + #line 1788 "Python/generated_cases.c.h" DISPATCH(); } TARGET(MAKE_CELL) { - #line 1255 "Python/bytecodes.c" + #line 1256 "Python/bytecodes.c" // "initial" is probably NULL but not if it's an arg (or set // via PyFrame_LocalsToFast() before MAKE_CELL has run). PyObject *initial = GETLOCAL(oparg); @@ -1797,12 +1798,12 @@ goto resume_with_error; } SETLOCAL(oparg, cell); - #line 1801 "Python/generated_cases.c.h" + #line 1802 "Python/generated_cases.c.h" DISPATCH(); } TARGET(DELETE_DEREF) { - #line 1266 "Python/bytecodes.c" + #line 1267 "Python/bytecodes.c" PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); // Can't use ERROR_IF here. @@ -1813,13 +1814,13 @@ } PyCell_SET(cell, NULL); Py_DECREF(oldobj); - #line 1817 "Python/generated_cases.c.h" + #line 1818 "Python/generated_cases.c.h" DISPATCH(); } TARGET(LOAD_CLASSDEREF) { PyObject *value; - #line 1279 "Python/bytecodes.c" + #line 1280 "Python/bytecodes.c" PyObject *name, *locals = LOCALS(); assert(locals); assert(oparg >= 0 && oparg < frame->f_code->co_nlocalsplus); @@ -1851,7 +1852,7 @@ } Py_INCREF(value); } - #line 1855 "Python/generated_cases.c.h" + #line 1856 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -1859,7 +1860,7 @@ TARGET(LOAD_DEREF) { PyObject *value; - #line 1313 "Python/bytecodes.c" + #line 1314 "Python/bytecodes.c" PyObject *cell = GETLOCAL(oparg); value = PyCell_GET(cell); if (value == NULL) { @@ -1867,7 +1868,7 @@ if (true) goto error; } Py_INCREF(value); - #line 1871 "Python/generated_cases.c.h" + #line 1872 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -1875,18 +1876,18 @@ TARGET(STORE_DEREF) { PyObject *v = stack_pointer[-1]; - #line 1323 "Python/bytecodes.c" + #line 1324 "Python/bytecodes.c" PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); PyCell_SET(cell, v); Py_XDECREF(oldobj); - #line 1884 "Python/generated_cases.c.h" + #line 1885 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(COPY_FREE_VARS) { - #line 1330 "Python/bytecodes.c" + #line 1331 "Python/bytecodes.c" /* Copy closure variables to free variables */ PyCodeObject *co = frame->f_code; assert(PyFunction_Check(frame->f_funcobj)); @@ -1897,22 +1898,22 @@ PyObject *o = PyTuple_GET_ITEM(closure, i); frame->localsplus[offset + i] = Py_NewRef(o); } - #line 1901 "Python/generated_cases.c.h" + #line 1902 "Python/generated_cases.c.h" DISPATCH(); } TARGET(BUILD_STRING) { PyObject **pieces = (stack_pointer - oparg); PyObject *str; - #line 1343 "Python/bytecodes.c" + #line 1344 "Python/bytecodes.c" str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); - #line 1910 "Python/generated_cases.c.h" + #line 1911 "Python/generated_cases.c.h" for (int _i = oparg; --_i >= 0;) { Py_DECREF(pieces[_i]); } - #line 1345 "Python/bytecodes.c" + #line 1346 "Python/bytecodes.c" if (str == NULL) { STACK_SHRINK(oparg); goto error; } - #line 1916 "Python/generated_cases.c.h" + #line 1917 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = str; @@ -1922,10 +1923,10 @@ TARGET(BUILD_TUPLE) { PyObject **values = (stack_pointer - oparg); PyObject *tup; - #line 1349 "Python/bytecodes.c" + #line 1350 "Python/bytecodes.c" tup = _PyTuple_FromArraySteal(values, oparg); if (tup == NULL) { STACK_SHRINK(oparg); goto error; } - #line 1929 "Python/generated_cases.c.h" + #line 1930 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = tup; @@ -1935,10 +1936,10 @@ TARGET(BUILD_LIST) { PyObject **values = (stack_pointer - oparg); PyObject *list; - #line 1354 "Python/bytecodes.c" + #line 1355 "Python/bytecodes.c" list = _PyList_FromArraySteal(values, oparg); if (list == NULL) { STACK_SHRINK(oparg); goto error; } - #line 1942 "Python/generated_cases.c.h" + #line 1943 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = list; @@ -1948,7 +1949,7 @@ TARGET(LIST_EXTEND) { PyObject *iterable = stack_pointer[-1]; PyObject *list = stack_pointer[-(2 + (oparg-1))]; - #line 1359 "Python/bytecodes.c" + #line 1360 "Python/bytecodes.c" PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); if (none_val == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && @@ -1959,13 +1960,13 @@ "Value after * must be an iterable, not %.200s", Py_TYPE(iterable)->tp_name); } - #line 1963 "Python/generated_cases.c.h" + #line 1964 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 1370 "Python/bytecodes.c" + #line 1371 "Python/bytecodes.c" if (true) goto pop_1_error; } Py_DECREF(none_val); - #line 1969 "Python/generated_cases.c.h" + #line 1970 "Python/generated_cases.c.h" Py_DECREF(iterable); STACK_SHRINK(1); DISPATCH(); @@ -1974,13 +1975,13 @@ TARGET(SET_UPDATE) { PyObject *iterable = stack_pointer[-1]; PyObject *set = stack_pointer[-(2 + (oparg-1))]; - #line 1377 "Python/bytecodes.c" + #line 1378 "Python/bytecodes.c" int err = _PySet_Update(set, iterable); - #line 1980 "Python/generated_cases.c.h" + #line 1981 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 1379 "Python/bytecodes.c" + #line 1380 "Python/bytecodes.c" if (err < 0) goto pop_1_error; - #line 1984 "Python/generated_cases.c.h" + #line 1985 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -1988,7 +1989,7 @@ TARGET(BUILD_SET) { PyObject **values = (stack_pointer - oparg); PyObject *set; - #line 1383 "Python/bytecodes.c" + #line 1384 "Python/bytecodes.c" set = PySet_New(NULL); if (set == NULL) goto error; @@ -2003,7 +2004,7 @@ Py_DECREF(set); if (true) { STACK_SHRINK(oparg); goto error; } } - #line 2007 "Python/generated_cases.c.h" + #line 2008 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = set; @@ -2013,7 +2014,7 @@ TARGET(BUILD_MAP) { PyObject **values = (stack_pointer - oparg*2); PyObject *map; - #line 1400 "Python/bytecodes.c" + #line 1401 "Python/bytecodes.c" map = _PyDict_FromItems( values, 2, values+1, 2, @@ -2021,13 +2022,13 @@ if (map == NULL) goto error; - #line 2025 "Python/generated_cases.c.h" + #line 2026 "Python/generated_cases.c.h" for (int _i = oparg*2; --_i >= 0;) { Py_DECREF(values[_i]); } - #line 1408 "Python/bytecodes.c" + #line 1409 "Python/bytecodes.c" if (map == NULL) { STACK_SHRINK(oparg*2); goto error; } - #line 2031 "Python/generated_cases.c.h" + #line 2032 "Python/generated_cases.c.h" STACK_SHRINK(oparg*2); STACK_GROW(1); stack_pointer[-1] = map; @@ -2035,7 +2036,7 @@ } TARGET(SETUP_ANNOTATIONS) { - #line 1412 "Python/bytecodes.c" + #line 1413 "Python/bytecodes.c" int err; PyObject *ann_dict; if (LOCALS() == NULL) { @@ -2075,7 +2076,7 @@ Py_DECREF(ann_dict); } } - #line 2079 "Python/generated_cases.c.h" + #line 2080 "Python/generated_cases.c.h" DISPATCH(); } @@ -2083,7 +2084,7 @@ PyObject *keys = stack_pointer[-1]; PyObject **values = (stack_pointer - (1 + oparg)); PyObject *map; - #line 1454 "Python/bytecodes.c" + #line 1455 "Python/bytecodes.c" if (!PyTuple_CheckExact(keys) || PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { _PyErr_SetString(tstate, PyExc_SystemError, @@ -2093,14 +2094,14 @@ map = _PyDict_FromItems( &PyTuple_GET_ITEM(keys, 0), 1, values, 1, oparg); - #line 2097 "Python/generated_cases.c.h" + #line 2098 "Python/generated_cases.c.h" for (int _i = oparg; --_i >= 0;) { Py_DECREF(values[_i]); } Py_DECREF(keys); - #line 1464 "Python/bytecodes.c" + #line 1465 "Python/bytecodes.c" if (map == NULL) { STACK_SHRINK(oparg); goto pop_1_error; } - #line 2104 "Python/generated_cases.c.h" + #line 2105 "Python/generated_cases.c.h" STACK_SHRINK(oparg); stack_pointer[-1] = map; DISPATCH(); @@ -2108,7 +2109,7 @@ TARGET(DICT_UPDATE) { PyObject *update = stack_pointer[-1]; - #line 1468 "Python/bytecodes.c" + #line 1469 "Python/bytecodes.c" PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (PyDict_Update(dict, update) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { @@ -2116,12 +2117,12 @@ "'%.200s' object is not a mapping", Py_TYPE(update)->tp_name); } - #line 2120 "Python/generated_cases.c.h" + #line 2121 "Python/generated_cases.c.h" Py_DECREF(update); - #line 1476 "Python/bytecodes.c" + #line 1477 "Python/bytecodes.c" if (true) goto pop_1_error; } - #line 2125 "Python/generated_cases.c.h" + #line 2126 "Python/generated_cases.c.h" Py_DECREF(update); STACK_SHRINK(1); DISPATCH(); @@ -2129,17 +2130,17 @@ TARGET(DICT_MERGE) { PyObject *update = stack_pointer[-1]; - #line 1482 "Python/bytecodes.c" + #line 1483 "Python/bytecodes.c" PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (_PyDict_MergeEx(dict, update, 2) < 0) { format_kwargs_error(tstate, PEEK(3 + oparg), update); - #line 2138 "Python/generated_cases.c.h" + #line 2139 "Python/generated_cases.c.h" Py_DECREF(update); - #line 1487 "Python/bytecodes.c" + #line 1488 "Python/bytecodes.c" if (true) goto pop_1_error; } - #line 2143 "Python/generated_cases.c.h" + #line 2144 "Python/generated_cases.c.h" Py_DECREF(update); STACK_SHRINK(1); PREDICT(CALL_FUNCTION_EX); @@ -2149,13 +2150,13 @@ TARGET(MAP_ADD) { PyObject *value = stack_pointer[-1]; PyObject *key = stack_pointer[-2]; - #line 1494 "Python/bytecodes.c" + #line 1495 "Python/bytecodes.c" PyObject *dict = PEEK(oparg + 2); // key, value are still on the stack assert(PyDict_CheckExact(dict)); /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; - #line 2159 "Python/generated_cases.c.h" + #line 2160 "Python/generated_cases.c.h" STACK_SHRINK(2); PREDICT(JUMP_BACKWARD); DISPATCH(); @@ -2167,7 +2168,7 @@ PyObject *owner = stack_pointer[-1]; PyObject *res2 = NULL; PyObject *res; - #line 1517 "Python/bytecodes.c" + #line 1518 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -2201,9 +2202,9 @@ NULL | meth | arg1 | ... | argN */ - #line 2205 "Python/generated_cases.c.h" + #line 2206 "Python/generated_cases.c.h" Py_DECREF(owner); - #line 1551 "Python/bytecodes.c" + #line 1552 "Python/bytecodes.c" if (meth == NULL) goto pop_1_error; res2 = NULL; res = meth; @@ -2212,12 +2213,12 @@ else { /* Classic, pushes one value. */ res = PyObject_GetAttr(owner, name); - #line 2216 "Python/generated_cases.c.h" + #line 2217 "Python/generated_cases.c.h" Py_DECREF(owner); - #line 1560 "Python/bytecodes.c" + #line 1561 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; } - #line 2221 "Python/generated_cases.c.h" + #line 2222 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -2231,7 +2232,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1565 "Python/bytecodes.c" + #line 1566 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2244,7 +2245,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2248 "Python/generated_cases.c.h" + #line 2249 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2259,7 +2260,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1581 "Python/bytecodes.c" + #line 1582 "Python/bytecodes.c" DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict != NULL); @@ -2272,7 +2273,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2276 "Python/generated_cases.c.h" + #line 2277 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2287,7 +2288,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1597 "Python/bytecodes.c" + #line 1598 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2314,7 +2315,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2318 "Python/generated_cases.c.h" + #line 2319 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2329,7 +2330,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1627 "Python/bytecodes.c" + #line 1628 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2339,7 +2340,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2343 "Python/generated_cases.c.h" + #line 2344 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2354,7 +2355,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 1640 "Python/bytecodes.c" + #line 1641 "Python/bytecodes.c" DEOPT_IF(!PyType_Check(cls), LOAD_ATTR); DEOPT_IF(((PyTypeObject *)cls)->tp_version_tag != type_version, @@ -2366,7 +2367,7 @@ res = descr; assert(res != NULL); Py_INCREF(res); - #line 2370 "Python/generated_cases.c.h" + #line 2371 "Python/generated_cases.c.h" Py_DECREF(cls); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2380,7 +2381,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t func_version = read_u32(&next_instr[3].cache); PyObject *fget = read_obj(&next_instr[5].cache); - #line 1655 "Python/bytecodes.c" + #line 1656 "Python/bytecodes.c" DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); @@ -2403,7 +2404,7 @@ new_frame->localsplus[0] = owner; JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); DISPATCH_INLINED(new_frame); - #line 2407 "Python/generated_cases.c.h" + #line 2408 "Python/generated_cases.c.h" } TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { @@ -2411,7 +2412,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t func_version = read_u32(&next_instr[3].cache); PyObject *getattribute = read_obj(&next_instr[5].cache); - #line 1680 "Python/bytecodes.c" + #line 1681 "Python/bytecodes.c" DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); @@ -2436,7 +2437,7 @@ new_frame->localsplus[1] = Py_NewRef(name); JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); DISPATCH_INLINED(new_frame); - #line 2440 "Python/generated_cases.c.h" + #line 2441 "Python/generated_cases.c.h" } TARGET(STORE_ATTR_INSTANCE_VALUE) { @@ -2444,7 +2445,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1707 "Python/bytecodes.c" + #line 1708 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2462,7 +2463,7 @@ Py_DECREF(old_value); } Py_DECREF(owner); - #line 2466 "Python/generated_cases.c.h" + #line 2467 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2473,7 +2474,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t hint = read_u16(&next_instr[3].cache); - #line 1727 "Python/bytecodes.c" + #line 1728 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2512,7 +2513,7 @@ /* PEP 509 */ dict->ma_version_tag = new_version; Py_DECREF(owner); - #line 2516 "Python/generated_cases.c.h" + #line 2517 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2523,7 +2524,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1768 "Python/bytecodes.c" + #line 1769 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2533,7 +2534,7 @@ *(PyObject **)addr = value; Py_XDECREF(old_value); Py_DECREF(owner); - #line 2537 "Python/generated_cases.c.h" + #line 2538 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2545,7 +2546,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1787 "Python/bytecodes.c" + #line 1788 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -2558,12 +2559,12 @@ #endif /* ENABLE_SPECIALIZATION */ assert((oparg >> 4) <= Py_GE); res = PyObject_RichCompare(left, right, oparg>>4); - #line 2562 "Python/generated_cases.c.h" + #line 2563 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 1800 "Python/bytecodes.c" + #line 1801 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 2567 "Python/generated_cases.c.h" + #line 2568 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2574,7 +2575,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1804 "Python/bytecodes.c" + #line 1805 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); @@ -2586,7 +2587,7 @@ _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); res = (sign_ish & oparg) ? Py_True : Py_False; Py_INCREF(res); - #line 2590 "Python/generated_cases.c.h" + #line 2591 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2597,7 +2598,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1819 "Python/bytecodes.c" + #line 1820 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); @@ -2613,7 +2614,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); res = (sign_ish & oparg) ? Py_True : Py_False; Py_INCREF(res); - #line 2617 "Python/generated_cases.c.h" + #line 2618 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2624,7 +2625,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1838 "Python/bytecodes.c" + #line 1839 "Python/bytecodes.c" DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); @@ -2637,7 +2638,7 @@ assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; Py_INCREF(res); - #line 2641 "Python/generated_cases.c.h" + #line 2642 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2648,14 +2649,14 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 1853 "Python/bytecodes.c" + #line 1854 "Python/bytecodes.c" int res = Py_Is(left, right) ^ oparg; - #line 2654 "Python/generated_cases.c.h" + #line 2655 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 1855 "Python/bytecodes.c" + #line 1856 "Python/bytecodes.c" b = Py_NewRef(res ? Py_True : Py_False); - #line 2659 "Python/generated_cases.c.h" + #line 2660 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = b; DISPATCH(); @@ -2665,15 +2666,15 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 1859 "Python/bytecodes.c" + #line 1860 "Python/bytecodes.c" int res = PySequence_Contains(right, left); - #line 2671 "Python/generated_cases.c.h" + #line 2672 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 1861 "Python/bytecodes.c" + #line 1862 "Python/bytecodes.c" if (res < 0) goto pop_2_error; b = Py_NewRef((res^oparg) ? Py_True : Py_False); - #line 2677 "Python/generated_cases.c.h" + #line 2678 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = b; DISPATCH(); @@ -2684,12 +2685,12 @@ PyObject *exc_value = stack_pointer[-2]; PyObject *rest; PyObject *match; - #line 1866 "Python/bytecodes.c" + #line 1867 "Python/bytecodes.c" if (check_except_star_type_valid(tstate, match_type) < 0) { - #line 2690 "Python/generated_cases.c.h" + #line 2691 "Python/generated_cases.c.h" Py_DECREF(exc_value); Py_DECREF(match_type); - #line 1868 "Python/bytecodes.c" + #line 1869 "Python/bytecodes.c" if (true) goto pop_2_error; } @@ -2697,10 +2698,10 @@ rest = NULL; int res = exception_group_match(exc_value, match_type, &match, &rest); - #line 2701 "Python/generated_cases.c.h" + #line 2702 "Python/generated_cases.c.h" Py_DECREF(exc_value); Py_DECREF(match_type); - #line 1876 "Python/bytecodes.c" + #line 1877 "Python/bytecodes.c" if (res < 0) goto pop_2_error; assert((match == NULL) == (rest == NULL)); @@ -2709,7 +2710,7 @@ if (!Py_IsNone(match)) { PyErr_SetExcInfo(NULL, Py_NewRef(match), NULL); } - #line 2713 "Python/generated_cases.c.h" + #line 2714 "Python/generated_cases.c.h" stack_pointer[-1] = match; stack_pointer[-2] = rest; DISPATCH(); @@ -2719,21 +2720,21 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 1887 "Python/bytecodes.c" + #line 1888 "Python/bytecodes.c" assert(PyExceptionInstance_Check(left)); if (check_except_type_valid(tstate, right) < 0) { - #line 2726 "Python/generated_cases.c.h" + #line 2727 "Python/generated_cases.c.h" Py_DECREF(right); - #line 1890 "Python/bytecodes.c" + #line 1891 "Python/bytecodes.c" if (true) goto pop_1_error; } int res = PyErr_GivenExceptionMatches(left, right); - #line 2733 "Python/generated_cases.c.h" + #line 2734 "Python/generated_cases.c.h" Py_DECREF(right); - #line 1895 "Python/bytecodes.c" + #line 1896 "Python/bytecodes.c" b = Py_NewRef(res ? Py_True : Py_False); - #line 2737 "Python/generated_cases.c.h" + #line 2738 "Python/generated_cases.c.h" stack_pointer[-1] = b; DISPATCH(); } @@ -2742,15 +2743,15 @@ PyObject *fromlist = stack_pointer[-1]; PyObject *level = stack_pointer[-2]; PyObject *res; - #line 1899 "Python/bytecodes.c" + #line 1900 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_name(tstate, frame, name, fromlist, level); - #line 2749 "Python/generated_cases.c.h" + #line 2750 "Python/generated_cases.c.h" Py_DECREF(level); Py_DECREF(fromlist); - #line 1902 "Python/bytecodes.c" + #line 1903 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 2754 "Python/generated_cases.c.h" + #line 2755 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); @@ -2759,29 +2760,29 @@ TARGET(IMPORT_FROM) { PyObject *from = stack_pointer[-1]; PyObject *res; - #line 1906 "Python/bytecodes.c" + #line 1907 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_from(tstate, from, name); if (res == NULL) goto error; - #line 2767 "Python/generated_cases.c.h" + #line 2768 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); } TARGET(JUMP_FORWARD) { - #line 1912 "Python/bytecodes.c" + #line 1913 "Python/bytecodes.c" JUMPBY(oparg); - #line 2776 "Python/generated_cases.c.h" + #line 2777 "Python/generated_cases.c.h" DISPATCH(); } TARGET(JUMP_BACKWARD) { PREDICTED(JUMP_BACKWARD); - #line 1916 "Python/bytecodes.c" + #line 1917 "Python/bytecodes.c" assert(oparg < INSTR_OFFSET()); JUMPBY(-oparg); - #line 2785 "Python/generated_cases.c.h" + #line 2786 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -2789,7 +2790,7 @@ TARGET(POP_JUMP_IF_FALSE) { PREDICTED(POP_JUMP_IF_FALSE); PyObject *cond = stack_pointer[-1]; - #line 1922 "Python/bytecodes.c" + #line 1923 "Python/bytecodes.c" if (Py_IsTrue(cond)) { _Py_DECREF_NO_DEALLOC(cond); } @@ -2799,9 +2800,9 @@ } else { int err = PyObject_IsTrue(cond); - #line 2803 "Python/generated_cases.c.h" + #line 2804 "Python/generated_cases.c.h" Py_DECREF(cond); - #line 1932 "Python/bytecodes.c" + #line 1933 "Python/bytecodes.c" if (err == 0) { JUMPBY(oparg); } @@ -2809,14 +2810,14 @@ if (err < 0) goto pop_1_error; } } - #line 2813 "Python/generated_cases.c.h" + #line 2814 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_TRUE) { PyObject *cond = stack_pointer[-1]; - #line 1942 "Python/bytecodes.c" + #line 1943 "Python/bytecodes.c" if (Py_IsFalse(cond)) { _Py_DECREF_NO_DEALLOC(cond); } @@ -2826,9 +2827,9 @@ } else { int err = PyObject_IsTrue(cond); - #line 2830 "Python/generated_cases.c.h" + #line 2831 "Python/generated_cases.c.h" Py_DECREF(cond); - #line 1952 "Python/bytecodes.c" + #line 1953 "Python/bytecodes.c" if (err > 0) { JUMPBY(oparg); } @@ -2836,67 +2837,67 @@ if (err < 0) goto pop_1_error; } } - #line 2840 "Python/generated_cases.c.h" + #line 2841 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_NOT_NONE) { PyObject *value = stack_pointer[-1]; - #line 1962 "Python/bytecodes.c" + #line 1963 "Python/bytecodes.c" if (!Py_IsNone(value)) { - #line 2849 "Python/generated_cases.c.h" + #line 2850 "Python/generated_cases.c.h" Py_DECREF(value); - #line 1964 "Python/bytecodes.c" + #line 1965 "Python/bytecodes.c" JUMPBY(oparg); } else { _Py_DECREF_NO_DEALLOC(value); } - #line 2857 "Python/generated_cases.c.h" + #line 2858 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_NONE) { PyObject *value = stack_pointer[-1]; - #line 1972 "Python/bytecodes.c" + #line 1973 "Python/bytecodes.c" if (Py_IsNone(value)) { _Py_DECREF_NO_DEALLOC(value); JUMPBY(oparg); } else { - #line 2870 "Python/generated_cases.c.h" + #line 2871 "Python/generated_cases.c.h" Py_DECREF(value); - #line 1978 "Python/bytecodes.c" + #line 1979 "Python/bytecodes.c" } - #line 2874 "Python/generated_cases.c.h" + #line 2875 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(JUMP_BACKWARD_NO_INTERRUPT) { - #line 1982 "Python/bytecodes.c" + #line 1983 "Python/bytecodes.c" /* This bytecode is used in the `yield from` or `await` loop. * If there is an interrupt, we want it handled in the innermost * generator or coroutine, so we deliberately do not check it here. * (see bpo-30039). */ JUMPBY(-oparg); - #line 2887 "Python/generated_cases.c.h" + #line 2888 "Python/generated_cases.c.h" DISPATCH(); } TARGET(GET_LEN) { PyObject *obj = stack_pointer[-1]; PyObject *len_o; - #line 1991 "Python/bytecodes.c" + #line 1992 "Python/bytecodes.c" // PUSH(len(TOS)) Py_ssize_t len_i = PyObject_Length(obj); if (len_i < 0) goto error; len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) goto error; - #line 2900 "Python/generated_cases.c.h" + #line 2901 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = len_o; DISPATCH(); @@ -2907,16 +2908,16 @@ PyObject *type = stack_pointer[-2]; PyObject *subject = stack_pointer[-3]; PyObject *attrs; - #line 1999 "Python/bytecodes.c" + #line 2000 "Python/bytecodes.c" // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. assert(PyTuple_CheckExact(names)); attrs = match_class(tstate, subject, type, oparg, names); - #line 2916 "Python/generated_cases.c.h" + #line 2917 "Python/generated_cases.c.h" Py_DECREF(subject); Py_DECREF(type); Py_DECREF(names); - #line 2004 "Python/bytecodes.c" + #line 2005 "Python/bytecodes.c" if (attrs) { assert(PyTuple_CheckExact(attrs)); // Success! } @@ -2924,7 +2925,7 @@ if (_PyErr_Occurred(tstate)) goto pop_3_error; attrs = Py_NewRef(Py_None); // Failure! } - #line 2928 "Python/generated_cases.c.h" + #line 2929 "Python/generated_cases.c.h" STACK_SHRINK(2); stack_pointer[-1] = attrs; DISPATCH(); @@ -2933,10 +2934,10 @@ TARGET(MATCH_MAPPING) { PyObject *subject = stack_pointer[-1]; PyObject *res; - #line 2014 "Python/bytecodes.c" + #line 2015 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; res = Py_NewRef(match ? Py_True : Py_False); - #line 2940 "Python/generated_cases.c.h" + #line 2941 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; PREDICT(POP_JUMP_IF_FALSE); @@ -2946,10 +2947,10 @@ TARGET(MATCH_SEQUENCE) { PyObject *subject = stack_pointer[-1]; PyObject *res; - #line 2020 "Python/bytecodes.c" + #line 2021 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; res = Py_NewRef(match ? Py_True : Py_False); - #line 2953 "Python/generated_cases.c.h" + #line 2954 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; PREDICT(POP_JUMP_IF_FALSE); @@ -2960,11 +2961,11 @@ PyObject *keys = stack_pointer[-1]; PyObject *subject = stack_pointer[-2]; PyObject *values_or_none; - #line 2026 "Python/bytecodes.c" + #line 2027 "Python/bytecodes.c" // On successful match, PUSH(values). Otherwise, PUSH(None). values_or_none = match_keys(tstate, subject, keys); if (values_or_none == NULL) goto error; - #line 2968 "Python/generated_cases.c.h" + #line 2969 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = values_or_none; DISPATCH(); @@ -2973,14 +2974,14 @@ TARGET(GET_ITER) { PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 2032 "Python/bytecodes.c" + #line 2033 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ iter = PyObject_GetIter(iterable); - #line 2980 "Python/generated_cases.c.h" + #line 2981 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 2035 "Python/bytecodes.c" + #line 2036 "Python/bytecodes.c" if (iter == NULL) goto pop_1_error; - #line 2984 "Python/generated_cases.c.h" + #line 2985 "Python/generated_cases.c.h" stack_pointer[-1] = iter; DISPATCH(); } @@ -2988,7 +2989,7 @@ TARGET(GET_YIELD_FROM_ITER) { PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 2039 "Python/bytecodes.c" + #line 2040 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ if (PyCoro_CheckExact(iterable)) { /* `iterable` is a coroutine */ @@ -3011,11 +3012,11 @@ if (iter == NULL) { goto error; } - #line 3015 "Python/generated_cases.c.h" + #line 3016 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 2062 "Python/bytecodes.c" + #line 2063 "Python/bytecodes.c" } - #line 3019 "Python/generated_cases.c.h" + #line 3020 "Python/generated_cases.c.h" stack_pointer[-1] = iter; PREDICT(LOAD_CONST); DISPATCH(); @@ -3026,7 +3027,7 @@ static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2081 "Python/bytecodes.c" + #line 2082 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyForIterCache *cache = (_PyForIterCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -3057,7 +3058,7 @@ DISPATCH(); } // Common case: no jump, leave it to the code generator - #line 3061 "Python/generated_cases.c.h" + #line 3062 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3065,7 +3066,7 @@ } TARGET(INSTRUMENTED_FOR_ITER) { - #line 2114 "Python/bytecodes.c" + #line 2115 "Python/bytecodes.c" _Py_CODEUNIT *here = next_instr-1; _Py_CODEUNIT *target; PyObject *iter = TOP(); @@ -3091,14 +3092,14 @@ target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1; } INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); - #line 3095 "Python/generated_cases.c.h" + #line 3096 "Python/generated_cases.c.h" DISPATCH(); } TARGET(FOR_ITER_LIST) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2142 "Python/bytecodes.c" + #line 2143 "Python/bytecodes.c" DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); _PyListIterObject *it = (_PyListIterObject *)iter; STAT_INC(FOR_ITER, hit); @@ -3118,7 +3119,7 @@ DISPATCH(); end_for_iter_list: // Common case: no jump, leave it to the code generator - #line 3122 "Python/generated_cases.c.h" + #line 3123 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3128,7 +3129,7 @@ TARGET(FOR_ITER_TUPLE) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2164 "Python/bytecodes.c" + #line 2165 "Python/bytecodes.c" _PyTupleIterObject *it = (_PyTupleIterObject *)iter; DEOPT_IF(Py_TYPE(it) != &PyTupleIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -3148,7 +3149,7 @@ DISPATCH(); end_for_iter_tuple: // Common case: no jump, leave it to the code generator - #line 3152 "Python/generated_cases.c.h" + #line 3153 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3158,7 +3159,7 @@ TARGET(FOR_ITER_RANGE) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2186 "Python/bytecodes.c" + #line 2187 "Python/bytecodes.c" _PyRangeIterObject *r = (_PyRangeIterObject *)iter; DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -3176,7 +3177,7 @@ if (next == NULL) { goto error; } - #line 3180 "Python/generated_cases.c.h" + #line 3181 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3185,7 +3186,7 @@ TARGET(FOR_ITER_GEN) { PyObject *iter = stack_pointer[-1]; - #line 2206 "Python/bytecodes.c" + #line 2207 "Python/bytecodes.c" PyGenObject *gen = (PyGenObject *)iter; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); @@ -3200,14 +3201,14 @@ assert(next_instr->op.code == END_FOR || next_instr->op.code == INSTRUMENTED_END_FOR); DISPATCH_INLINED(gen_frame); - #line 3204 "Python/generated_cases.c.h" + #line 3205 "Python/generated_cases.c.h" } TARGET(BEFORE_ASYNC_WITH) { PyObject *mgr = stack_pointer[-1]; PyObject *exit; PyObject *res; - #line 2223 "Python/bytecodes.c" + #line 2224 "Python/bytecodes.c" PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); if (enter == NULL) { if (!_PyErr_Occurred(tstate)) { @@ -3230,16 +3231,16 @@ Py_DECREF(enter); goto error; } - #line 3234 "Python/generated_cases.c.h" + #line 3235 "Python/generated_cases.c.h" Py_DECREF(mgr); - #line 2246 "Python/bytecodes.c" + #line 2247 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); if (true) goto pop_1_error; } - #line 3243 "Python/generated_cases.c.h" + #line 3244 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -3251,7 +3252,7 @@ PyObject *mgr = stack_pointer[-1]; PyObject *exit; PyObject *res; - #line 2256 "Python/bytecodes.c" + #line 2257 "Python/bytecodes.c" /* pop the context manager, push its __exit__ and the * value returned from calling its __enter__ */ @@ -3277,16 +3278,16 @@ Py_DECREF(enter); goto error; } - #line 3281 "Python/generated_cases.c.h" + #line 3282 "Python/generated_cases.c.h" Py_DECREF(mgr); - #line 2282 "Python/bytecodes.c" + #line 2283 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); if (true) goto pop_1_error; } - #line 3290 "Python/generated_cases.c.h" + #line 3291 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -3298,7 +3299,7 @@ PyObject *lasti = stack_pointer[-3]; PyObject *exit_func = stack_pointer[-4]; PyObject *res; - #line 2291 "Python/bytecodes.c" + #line 2292 "Python/bytecodes.c" /* At the top of the stack are 4 values: - val: TOP = exc_info() - unused: SECOND = previous exception @@ -3319,7 +3320,7 @@ res = PyObject_Vectorcall(exit_func, stack + 1, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); if (res == NULL) goto error; - #line 3323 "Python/generated_cases.c.h" + #line 3324 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); @@ -3328,7 +3329,7 @@ TARGET(PUSH_EXC_INFO) { PyObject *new_exc = stack_pointer[-1]; PyObject *prev_exc; - #line 2314 "Python/bytecodes.c" + #line 2315 "Python/bytecodes.c" _PyErr_StackItem *exc_info = tstate->exc_info; if (exc_info->exc_value != NULL) { prev_exc = exc_info->exc_value; @@ -3338,7 +3339,7 @@ } assert(PyExceptionInstance_Check(new_exc)); exc_info->exc_value = Py_NewRef(new_exc); - #line 3342 "Python/generated_cases.c.h" + #line 3343 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = new_exc; stack_pointer[-2] = prev_exc; @@ -3352,7 +3353,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t keys_version = read_u32(&next_instr[3].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2326 "Python/bytecodes.c" + #line 2327 "Python/bytecodes.c" /* Cached method object */ PyTypeObject *self_cls = Py_TYPE(self); assert(type_version != 0); @@ -3369,7 +3370,7 @@ assert(_PyType_HasFeature(Py_TYPE(res2), Py_TPFLAGS_METHOD_DESCRIPTOR)); res = self; assert(oparg & 1); - #line 3373 "Python/generated_cases.c.h" + #line 3374 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3383,7 +3384,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2345 "Python/bytecodes.c" + #line 2346 "Python/bytecodes.c" PyTypeObject *self_cls = Py_TYPE(self); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); assert(self_cls->tp_dictoffset == 0); @@ -3393,7 +3394,7 @@ res2 = Py_NewRef(descr); res = self; assert(oparg & 1); - #line 3397 "Python/generated_cases.c.h" + #line 3398 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3407,7 +3408,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2357 "Python/bytecodes.c" + #line 2358 "Python/bytecodes.c" PyTypeObject *self_cls = Py_TYPE(self); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); Py_ssize_t dictoffset = self_cls->tp_dictoffset; @@ -3421,7 +3422,7 @@ res2 = Py_NewRef(descr); res = self; assert(oparg & 1); - #line 3425 "Python/generated_cases.c.h" + #line 3426 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3430,16 +3431,16 @@ } TARGET(KW_NAMES) { - #line 2373 "Python/bytecodes.c" + #line 2374 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg < PyTuple_GET_SIZE(frame->f_code->co_consts)); kwnames = GETITEM(frame->f_code->co_consts, oparg); - #line 3438 "Python/generated_cases.c.h" + #line 3439 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_CALL) { - #line 2379 "Python/bytecodes.c" + #line 2380 "Python/bytecodes.c" int is_meth = PEEK(oparg+2) != NULL; int total_args = oparg + is_meth; PyObject *function = PEEK(total_args + 1); @@ -3452,7 +3453,7 @@ _PyCallCache *cache = (_PyCallCache *)next_instr; INCREMENT_ADAPTIVE_COUNTER(cache->counter); GO_TO_INSTRUCTION(CALL); - #line 3456 "Python/generated_cases.c.h" + #line 3457 "Python/generated_cases.c.h" } TARGET(CALL) { @@ -3462,7 +3463,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2424 "Python/bytecodes.c" + #line 2425 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3543,7 +3544,7 @@ Py_DECREF(args[i]); } if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3547 "Python/generated_cases.c.h" + #line 3548 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3555,7 +3556,7 @@ TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; - #line 2511 "Python/bytecodes.c" + #line 2512 "Python/bytecodes.c" DEOPT_IF(method != NULL, CALL); DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); STAT_INC(CALL, hit); @@ -3565,7 +3566,7 @@ PEEK(oparg + 2) = Py_NewRef(meth); // method Py_DECREF(callable); GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); - #line 3569 "Python/generated_cases.c.h" + #line 3570 "Python/generated_cases.c.h" } TARGET(CALL_PY_EXACT_ARGS) { @@ -3574,7 +3575,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); - #line 2523 "Python/bytecodes.c" + #line 2524 "Python/bytecodes.c" assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; @@ -3599,7 +3600,7 @@ STACK_SHRINK(oparg + 2); JUMPBY(INLINE_CACHE_ENTRIES_CALL); DISPATCH_INLINED(new_frame); - #line 3603 "Python/generated_cases.c.h" + #line 3604 "Python/generated_cases.c.h" } TARGET(CALL_PY_WITH_DEFAULTS) { @@ -3608,7 +3609,7 @@ PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); uint16_t min_args = read_u16(&next_instr[3].cache); - #line 2550 "Python/bytecodes.c" + #line 2551 "Python/bytecodes.c" assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; @@ -3638,7 +3639,7 @@ STACK_SHRINK(oparg + 2); JUMPBY(INLINE_CACHE_ENTRIES_CALL); DISPATCH_INLINED(new_frame); - #line 3642 "Python/generated_cases.c.h" + #line 3643 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_TYPE_1) { @@ -3646,7 +3647,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2582 "Python/bytecodes.c" + #line 2583 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3656,7 +3657,7 @@ res = Py_NewRef(Py_TYPE(obj)); Py_DECREF(obj); Py_DECREF(&PyType_Type); // I.e., callable - #line 3660 "Python/generated_cases.c.h" + #line 3661 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3669,7 +3670,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2594 "Python/bytecodes.c" + #line 2595 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3680,7 +3681,7 @@ Py_DECREF(arg); Py_DECREF(&PyUnicode_Type); // I.e., callable if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3684 "Python/generated_cases.c.h" + #line 3685 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3694,7 +3695,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2608 "Python/bytecodes.c" + #line 2609 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3705,7 +3706,7 @@ Py_DECREF(arg); Py_DECREF(&PyTuple_Type); // I.e., tuple if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3709 "Python/generated_cases.c.h" + #line 3710 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3719,7 +3720,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2622 "Python/bytecodes.c" + #line 2623 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3741,7 +3742,7 @@ } Py_DECREF(tp); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3745 "Python/generated_cases.c.h" + #line 3746 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3755,7 +3756,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2647 "Python/bytecodes.c" + #line 2648 "Python/bytecodes.c" /* Builtin METH_O functions */ assert(kwnames == NULL); int is_meth = method != NULL; @@ -3783,7 +3784,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3787 "Python/generated_cases.c.h" + #line 3788 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3797,7 +3798,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2678 "Python/bytecodes.c" + #line 2679 "Python/bytecodes.c" /* Builtin METH_FASTCALL functions, without keywords */ assert(kwnames == NULL); int is_meth = method != NULL; @@ -3829,7 +3830,7 @@ 'invalid'). In those cases an exception is set, so we must handle it. */ - #line 3833 "Python/generated_cases.c.h" + #line 3834 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3843,7 +3844,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2713 "Python/bytecodes.c" + #line 2714 "Python/bytecodes.c" /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ int is_meth = method != NULL; int total_args = oparg; @@ -3875,7 +3876,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3879 "Python/generated_cases.c.h" + #line 3880 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3889,7 +3890,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2748 "Python/bytecodes.c" + #line 2749 "Python/bytecodes.c" assert(kwnames == NULL); /* len(o) */ int is_meth = method != NULL; @@ -3914,7 +3915,7 @@ Py_DECREF(callable); Py_DECREF(arg); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3918 "Python/generated_cases.c.h" + #line 3919 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3927,7 +3928,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2775 "Python/bytecodes.c" + #line 2776 "Python/bytecodes.c" assert(kwnames == NULL); /* isinstance(o, o2) */ int is_meth = method != NULL; @@ -3954,7 +3955,7 @@ Py_DECREF(cls); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3958 "Python/generated_cases.c.h" + #line 3959 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3966,7 +3967,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *self = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; - #line 2805 "Python/bytecodes.c" + #line 2806 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); assert(method != NULL); @@ -3984,14 +3985,14 @@ JUMPBY(INLINE_CACHE_ENTRIES_CALL + 1); assert(next_instr[-1].op.code == POP_TOP); DISPATCH(); - #line 3988 "Python/generated_cases.c.h" + #line 3989 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) { PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2825 "Python/bytecodes.c" + #line 2826 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -4022,7 +4023,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4026 "Python/generated_cases.c.h" + #line 4027 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4035,7 +4036,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2859 "Python/bytecodes.c" + #line 2860 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -4064,7 +4065,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4068 "Python/generated_cases.c.h" + #line 4069 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4077,7 +4078,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2891 "Python/bytecodes.c" + #line 2892 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 0 || oparg == 1); int is_meth = method != NULL; @@ -4106,7 +4107,7 @@ Py_DECREF(self); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4110 "Python/generated_cases.c.h" + #line 4111 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4119,7 +4120,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2923 "Python/bytecodes.c" + #line 2924 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -4147,7 +4148,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4151 "Python/generated_cases.c.h" + #line 4152 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4157,9 +4158,9 @@ } TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { - #line 2954 "Python/bytecodes.c" + #line 2955 "Python/bytecodes.c" GO_TO_INSTRUCTION(CALL_FUNCTION_EX); - #line 4163 "Python/generated_cases.c.h" + #line 4164 "Python/generated_cases.c.h" } TARGET(CALL_FUNCTION_EX) { @@ -4168,7 +4169,7 @@ PyObject *callargs = stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))]; PyObject *func = stack_pointer[-(2 + ((oparg & 1) ? 1 : 0))]; PyObject *result; - #line 2958 "Python/bytecodes.c" + #line 2959 "Python/bytecodes.c" // DICT_MERGE is called before this opcode if there are kwargs. // It converts all dict subtypes in kwargs into regular dicts. assert(kwargs == NULL || PyDict_CheckExact(kwargs)); @@ -4211,14 +4212,14 @@ else { result = PyObject_Call(func, callargs, kwargs); } - #line 4215 "Python/generated_cases.c.h" + #line 4216 "Python/generated_cases.c.h" Py_DECREF(func); Py_DECREF(callargs); Py_XDECREF(kwargs); - #line 3001 "Python/bytecodes.c" + #line 3002 "Python/bytecodes.c" assert(PEEK(3 + (oparg & 1)) == NULL); if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } - #line 4222 "Python/generated_cases.c.h" + #line 4223 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 1) ? 1 : 0)); STACK_SHRINK(2); stack_pointer[-1] = result; @@ -4233,7 +4234,7 @@ PyObject *kwdefaults = (oparg & 0x02) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0))] : NULL; PyObject *defaults = (oparg & 0x01) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x01) ? 1 : 0))] : NULL; PyObject *func; - #line 3011 "Python/bytecodes.c" + #line 3012 "Python/bytecodes.c" PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); @@ -4262,14 +4263,14 @@ func_obj->func_version = ((PyCodeObject *)codeobj)->co_version; func = (PyObject *)func_obj; - #line 4266 "Python/generated_cases.c.h" + #line 4267 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 0x01) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x08) ? 1 : 0)); stack_pointer[-1] = func; DISPATCH(); } TARGET(RETURN_GENERATOR) { - #line 3042 "Python/bytecodes.c" + #line 3043 "Python/bytecodes.c" assert(PyFunction_Check(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); @@ -4290,7 +4291,7 @@ frame = cframe.current_frame = prev; _PyFrame_StackPush(frame, (PyObject *)gen); goto resume_frame; - #line 4294 "Python/generated_cases.c.h" + #line 4295 "Python/generated_cases.c.h" } TARGET(BUILD_SLICE) { @@ -4298,15 +4299,15 @@ PyObject *stop = stack_pointer[-(1 + ((oparg == 3) ? 1 : 0))]; PyObject *start = stack_pointer[-(2 + ((oparg == 3) ? 1 : 0))]; PyObject *slice; - #line 3065 "Python/bytecodes.c" + #line 3066 "Python/bytecodes.c" slice = PySlice_New(start, stop, step); - #line 4304 "Python/generated_cases.c.h" + #line 4305 "Python/generated_cases.c.h" Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); - #line 3067 "Python/bytecodes.c" + #line 3068 "Python/bytecodes.c" if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } - #line 4310 "Python/generated_cases.c.h" + #line 4311 "Python/generated_cases.c.h" STACK_SHRINK(((oparg == 3) ? 1 : 0)); STACK_SHRINK(1); stack_pointer[-1] = slice; @@ -4317,7 +4318,7 @@ PyObject *fmt_spec = ((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? stack_pointer[-((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))] : NULL; PyObject *value = stack_pointer[-(1 + (((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))]; PyObject *result; - #line 3071 "Python/bytecodes.c" + #line 3072 "Python/bytecodes.c" /* Handles f-string value formatting. */ PyObject *(*conv_fn)(PyObject *); int which_conversion = oparg & FVC_MASK; @@ -4352,7 +4353,7 @@ Py_DECREF(value); Py_XDECREF(fmt_spec); if (result == NULL) { STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); goto pop_1_error; } - #line 4356 "Python/generated_cases.c.h" + #line 4357 "Python/generated_cases.c.h" STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); stack_pointer[-1] = result; DISPATCH(); @@ -4361,10 +4362,10 @@ TARGET(COPY) { PyObject *bottom = stack_pointer[-(1 + (oparg-1))]; PyObject *top; - #line 3108 "Python/bytecodes.c" + #line 3109 "Python/bytecodes.c" assert(oparg > 0); top = Py_NewRef(bottom); - #line 4368 "Python/generated_cases.c.h" + #line 4369 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = top; DISPATCH(); @@ -4376,7 +4377,7 @@ PyObject *rhs = stack_pointer[-1]; PyObject *lhs = stack_pointer[-2]; PyObject *res; - #line 3113 "Python/bytecodes.c" + #line 3114 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -4391,12 +4392,12 @@ assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); assert(binary_ops[oparg]); res = binary_ops[oparg](lhs, rhs); - #line 4395 "Python/generated_cases.c.h" + #line 4396 "Python/generated_cases.c.h" Py_DECREF(lhs); Py_DECREF(rhs); - #line 3128 "Python/bytecodes.c" + #line 3129 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 4400 "Python/generated_cases.c.h" + #line 4401 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -4406,16 +4407,16 @@ TARGET(SWAP) { PyObject *top = stack_pointer[-1]; PyObject *bottom = stack_pointer[-(2 + (oparg-2))]; - #line 3133 "Python/bytecodes.c" + #line 3134 "Python/bytecodes.c" assert(oparg >= 2); - #line 4412 "Python/generated_cases.c.h" + #line 4413 "Python/generated_cases.c.h" stack_pointer[-1] = bottom; stack_pointer[-(2 + (oparg-2))] = top; DISPATCH(); } TARGET(INSTRUMENTED_LINE) { - #line 3137 "Python/bytecodes.c" + #line 3138 "Python/bytecodes.c" _Py_CODEUNIT *here = next_instr-1; _PyFrame_SetStackPointer(frame, stack_pointer); int original_opcode = _Py_call_instrumentation_line( @@ -4435,11 +4436,11 @@ } opcode = original_opcode; DISPATCH_GOTO(); - #line 4439 "Python/generated_cases.c.h" + #line 4440 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_INSTRUCTION) { - #line 3159 "Python/bytecodes.c" + #line 3160 "Python/bytecodes.c" int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, next_instr-1); if (next_opcode < 0) goto error; @@ -4451,26 +4452,26 @@ assert(next_opcode > 0 && next_opcode < 256); opcode = next_opcode; DISPATCH_GOTO(); - #line 4455 "Python/generated_cases.c.h" + #line 4456 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_JUMP_FORWARD) { - #line 3174 "Python/bytecodes.c" + #line 3175 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); - #line 4461 "Python/generated_cases.c.h" + #line 4462 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_JUMP_BACKWARD) { - #line 3179 "Python/bytecodes.c" + #line 3180 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr-oparg, PY_MONITORING_EVENT_JUMP); - #line 4468 "Python/generated_cases.c.h" + #line 4469 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { - #line 3184 "Python/bytecodes.c" + #line 3185 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); Py_DECREF(cond); @@ -4479,12 +4480,12 @@ assert(err == 0 || err == 1); int offset = err*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4483 "Python/generated_cases.c.h" + #line 4484 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { - #line 3195 "Python/bytecodes.c" + #line 3196 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); Py_DECREF(cond); @@ -4493,12 +4494,12 @@ assert(err == 0 || err == 1); int offset = (1-err)*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4497 "Python/generated_cases.c.h" + #line 4498 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { - #line 3206 "Python/bytecodes.c" + #line 3207 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4511,12 +4512,12 @@ offset = 0; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4515 "Python/generated_cases.c.h" + #line 4516 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { - #line 3221 "Python/bytecodes.c" + #line 3222 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4529,22 +4530,29 @@ offset = oparg; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4533 "Python/generated_cases.c.h" + #line 4534 "Python/generated_cases.c.h" DISPATCH(); } TARGET(EXTENDED_ARG) { - #line 3236 "Python/bytecodes.c" + #line 3237 "Python/bytecodes.c" assert(oparg); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); - #line 4544 "Python/generated_cases.c.h" + #line 4545 "Python/generated_cases.c.h" } TARGET(CACHE) { - #line 3244 "Python/bytecodes.c" + #line 3245 "Python/bytecodes.c" Py_UNREACHABLE(); - #line 4550 "Python/generated_cases.c.h" + #line 4551 "Python/generated_cases.c.h" + } + + TARGET(RESERVED) { + #line 3249 "Python/bytecodes.c" + assert(0 && "Executing RESERVED instruction."); + #line 4557 "Python/generated_cases.c.h" + DISPATCH(); } diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 4a1fde044f10bf..59f500c14570e4 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -11,7 +11,7 @@ #include "pycore_pystate.h" /* Uncomment this to dump debugging output when assertions fail */ -// #define INSTRUMENT_DEBUG 1 +#define INSTRUMENT_DEBUG 1 static PyObject DISABLE = { @@ -144,6 +144,7 @@ static const uint8_t INSTRUMENTED_OPCODES[256] = { static inline bool is_instrumented(int opcode) { assert(opcode != 0); + assert(opcode != RESERVED); return INSTRUMENTED_OPCODES[opcode] == opcode; } @@ -271,25 +272,48 @@ compute_line(PyCodeObject *code, int offset, int8_t line_delta) * so add a couple of utilities */ int _Py_GetBaseOpcode(PyCodeObject *code, int offset) { - int opcode = _Py_OPCODE(_PyCode_CODE(code)[offset]); - if (!is_instrumented(opcode)) { - assert(_PyOpcode_Deopt[opcode] != 0); - return _PyOpcode_Deopt[opcode]; - } + int opcode = _PyCode_CODE(code)[offset].op.code; if (opcode == INSTRUMENTED_INSTRUCTION) { opcode = code->_co_monitoring->per_instruction_opcodes[offset]; } if (opcode == INSTRUMENTED_LINE) { opcode = code->_co_monitoring->lines[offset].original_opcode; } + assert(opcode != INSTRUMENTED_INSTRUCTION); + assert(opcode != INSTRUMENTED_LINE); int deinstrumented = DE_INSTRUMENT[opcode]; if (deinstrumented) { return deinstrumented; } assert(_PyOpcode_Deopt[opcode] != 0); + assert(opcode != RESERVED); return _PyOpcode_Deopt[opcode]; } +int instruction_length(PyCodeObject *code, int offset) +{ + int opcode = _PyCode_CODE(code)[offset].op.code; + assert(opcode != 0); + assert(opcode != RESERVED); + if (opcode == INSTRUMENTED_INSTRUCTION) { + opcode = code->_co_monitoring->per_instruction_opcodes[offset]; + } + if (opcode == INSTRUMENTED_LINE) { + opcode = code->_co_monitoring->lines[offset].original_opcode; + } + int deinstrumented = DE_INSTRUMENT[opcode]; + if (deinstrumented) { + opcode = deinstrumented; + } + else { + opcode = _PyOpcode_Deopt[opcode]; + } + assert(opcode != 0); + assert(!is_instrumented(opcode)); + assert(opcode == _PyOpcode_Deopt[opcode]); + return 1 + _PyOpcode_Caches[opcode]; +} + typedef struct _Instruction { uint8_t extended_args; uint8_t deinstrumented_opcode; @@ -329,9 +353,7 @@ read_instruction(PyCodeObject *code, int offset) else { result.deinstrumented_opcode = opcode; } - int base = _PyOpcode_Deopt[result.deinstrumented_opcode]; - /* TO DO -- Use instruction length, not cache count table */ - result.length += _PyOpcode_Caches[base]; + result.length = result.extended_args + instruction_length(code, offset); return result; } @@ -440,6 +462,8 @@ dump_instrumentation_data(PyCodeObject *code, int star, FILE*out) { _PyCoMonitoringData *data = code->_co_monitoring; fprintf(out, "\n"); + PyObject_Print(code->co_name, out, Py_PRINT_RAW); + fprintf(out, "\n"); if (data == NULL) { fprintf(out, "NULL\n"); return; @@ -449,7 +473,7 @@ dump_instrumentation_data(PyCodeObject *code, int star, FILE*out) dump_monitors("Active", data->active_monitors, out); int code_len = (int)Py_SIZE(code); bool starred = false; - for (int i = 0; i < code_len; i++) { + for (int i = 0; i < code_len; i += instruction_length(code, i)) { _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; int opcode = instr->op.code; if (i == star) { @@ -463,9 +487,7 @@ dump_instrumentation_data(PyCodeObject *code, int star, FILE*out) dump_instrumentation_data_per_instruction(code, data, i, out); /* TO DO -- per instruction data */ fprintf(out, "\n"); - /* TO DO -- Use instruction length, not cache count */ - int base_opcode = get_base_opcode_best_attempt(code, i); - i += _PyOpcode_Caches[base_opcode]; + ; } if (!starred && star >= 0) { fprintf(out, "Error offset not at valid instruction offset: %d\n", star); @@ -545,11 +567,6 @@ sanity_check_instrumentation(PyCodeObject *code) event = _PyCode_CODE(code)[i + inst.extended_args].op.arg; } CHECK(active_monitors.tools[event] != 0); - CHECK(inst.deinstrumented_opcode != END_FOR); - } - if (_PyOpcode_Deopt[inst.deinstrumented_opcode] == COMPARE_AND_BRANCH) { - CHECK(_PyCode_CODE(code)[i+2].op.code == POP_JUMP_IF_FALSE || - _PyCode_CODE(code)[i+2].op.code == POP_JUMP_IF_TRUE); } if (data->lines && inst.deinstrumented_opcode != END_FOR) { int line1 = compute_line(code, i, data->lines[i].line_delta); @@ -647,14 +664,13 @@ de_instrument_line(PyCodeObject *code, int i) } _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[i]; int original_opcode = lines->original_opcode; - assert(is_instrumented(original_opcode) || _PyOpcode_Deopt[original_opcode]); if (is_instrumented(original_opcode)) { /* Instrumented original */ instr->op.code = original_opcode; } else { - original_opcode = _PyOpcode_Deopt[original_opcode]; - assert(original_opcode != 0); + CHECK(original_opcode != 0); + CHECK(original_opcode == _PyOpcode_Deopt[original_opcode]); if (_PyOpcode_Caches[original_opcode]) { instr[1].cache = adaptive_counter_warmup(); } @@ -672,51 +688,53 @@ de_instrument_line(PyCodeObject *code, int i) static void -de_instrument_per_instruction(PyCodeObject *code, int offset) +de_instrument_per_instruction(PyCodeObject *code, int i) { - _Py_CODEUNIT *instr = &_PyCode_CODE(code)[offset]; - int opcode = _Py_OPCODE(*instr); + _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; + int opcode = instr->op.code; if (opcode != INSTRUMENTED_INSTRUCTION) { return; } - int original_opcode = code->_co_monitoring->per_instruction_opcodes[offset]; - assert(is_instrumented(original_opcode) || _PyOpcode_Deopt[original_opcode]); + int original_opcode = code->_co_monitoring->per_instruction_opcodes[i]; + CHECK(original_opcode != 0); if (is_instrumented(original_opcode)) { /* Instrumented original */ instr->op.code = original_opcode; } else { - int base_opcode = _PyOpcode_Deopt[original_opcode]; - assert(base_opcode != 0); - instr->op.code = base_opcode; - if (_PyOpcode_Caches[base_opcode]) { + CHECK(original_opcode == _PyOpcode_Deopt[original_opcode]); + instr->op.code = original_opcode; + if (_PyOpcode_Caches[original_opcode]) { instr[1].cache = adaptive_counter_warmup(); } } assert(instr->op.code != INSTRUMENTED_INSTRUCTION); /* Keep things clean for sanity check */ - code->_co_monitoring->per_instruction_opcodes[offset] = 0; + code->_co_monitoring->per_instruction_opcodes[i] = 0; } static void -instrument(PyCodeObject *code, int offset) +instrument(PyCodeObject *code, int i) { - /* TO DO -- handle INSTRUMENTED_INSTRUCTION */ - _Py_CODEUNIT *instr = &_PyCode_CODE(code)[offset]; + _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; int opcode = _Py_OPCODE(*instr); + /* TO DO -- handle INSTRUMENTED_INSTRUCTION */ + CHECK(opcode != INSTRUMENTED_INSTRUCTION); if (opcode == INSTRUMENTED_LINE) { - _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[offset]; + _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[i]; opcode = lines->original_opcode; - assert(!is_instrumented(opcode)); + CHECK(opcode != 0); + CHECK(!is_instrumented(opcode)); + CHECK(opcode == _PyOpcode_Deopt[opcode]); lines->original_opcode = INSTRUMENTED_OPCODES[opcode]; + CHECK(lines->original_opcode != 0); } else if (!is_instrumented(opcode)) { opcode = _PyOpcode_Deopt[opcode]; int instrumented = INSTRUMENTED_OPCODES[opcode]; assert(instrumented); instr->op.code = instrumented; - /* TO DO -- Use instruction length, not cache count */ if (_PyOpcode_Caches[opcode]) { instr[1].cache = adaptive_counter_warmup(); } @@ -729,6 +747,7 @@ instrument_line(PyCodeObject *code, int i) _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; int opcode = _Py_OPCODE(*instr); /* TO DO -- handle INSTRUMENTED_INSTRUCTION */ + CHECK(opcode != INSTRUMENTED_INSTRUCTION); if (opcode == INSTRUMENTED_LINE) { return; } @@ -739,10 +758,9 @@ instrument_line(PyCodeObject *code, int i) } else { assert(opcode != 0); - assert(_PyOpcode_Deopt[opcode] != 0); assert(_PyOpcode_Deopt[opcode] != RESUME); lines->original_opcode = _PyOpcode_Deopt[opcode]; - + CHECK(lines->original_opcode != 0); } assert(lines->original_opcode > 0); instr->op.code = INSTRUMENTED_LINE; @@ -1343,10 +1361,11 @@ initialize_lines(PyCodeObject *code) } int current_line = -1; for (int i = code->_co_firsttraceable; i < code_len; ) { - Instruction inst = read_instruction(code, i); + int opcode = _Py_GetBaseOpcode(code, i); int line = _PyCode_CheckLineNumber(i*(int)sizeof(_Py_CODEUNIT), &range); line_data[i].line_delta = compute_line_delta(code, i, line); - switch (inst.deinstrumented_opcode) { + int length = instruction_length(code, i); + switch (opcode) { case END_ASYNC_FOR: case END_FOR: case END_SEND: @@ -1367,11 +1386,11 @@ initialize_lines(PyCodeObject *code) current_line = line; } } - for (int j = 1; j < inst.length; j++) { + for (int j = 1; j < length; j++) { line_data[i+j].original_opcode = 0; line_data[i+j].line_delta = NO_LINE; } - switch (inst.deinstrumented_opcode) { + switch (opcode) { case RETURN_VALUE: case RAISE_VARARGS: case RERAISE: @@ -1379,7 +1398,7 @@ initialize_lines(PyCodeObject *code) * should be treated as different lines */ current_line = -1; } - i += inst.length; + i += length; } } @@ -1533,12 +1552,12 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) return 0; } /* Insert instrumentation */ - for (int i = 0; i < code_len; i++) { + for (int i = 0; i < code_len; i+= instruction_length(code, i)) { _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; - int opcode = _Py_OPCODE(*instr); - if (is_super_instruction(opcode)) { - instr->op.code = _PyOpcode_Deopt[opcode]; + if (is_super_instruction(instr->op.code)) { + instr->op.code = _PyOpcode_Deopt[instr->op.code]; } + CHECK(instr->op.code != 0); int base_opcode = _Py_GetBaseOpcode(code, i); if (OPCODE_HAS_EVENT[base_opcode]) { int8_t event; @@ -1558,19 +1577,12 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) add_tools(code, i, event, new_tools); } } - while (base_opcode == EXTENDED_ARG) { - i++; - base_opcode = _Py_GetBaseOpcode(code, i); - } - i += _PyOpcode_Caches[base_opcode]; - } uint8_t new_line_tools = new_events.tools[PY_MONITORING_EVENT_LINE]; uint8_t removed_line_tools = removed_events.tools[PY_MONITORING_EVENT_LINE]; if (new_line_tools | removed_line_tools) { _PyCoLineInstrumentationData *line_data = code->_co_monitoring->lines; for (int i = code->_co_firsttraceable; i < code_len;) { - Instruction inst = read_instruction(code, i); if (line_data[i].original_opcode) { if (removed_line_tools) { remove_line_tools(code, i, removed_line_tools); @@ -1579,17 +1591,16 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) add_line_tools(code, i, new_line_tools); } } - i += inst.length; + i += instruction_length(code, i); } } uint8_t new_per_instruction_tools = new_events.tools[PY_MONITORING_EVENT_INSTRUCTION]; uint8_t removed_per_instruction_tools = removed_events.tools[PY_MONITORING_EVENT_INSTRUCTION]; if (new_per_instruction_tools | removed_per_instruction_tools) { for (int i = code->_co_firsttraceable; i < code_len;) { - Instruction inst = read_instruction(code, i); - int opcode = _PyOpcode_Deopt[inst.deinstrumented_opcode]; + int opcode = _Py_GetBaseOpcode(code, i); if (opcode == RESUME || opcode == END_FOR) { - i += inst.length; + i += instruction_length(code, i); continue; } if (removed_per_instruction_tools) { @@ -1598,7 +1609,7 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) if (new_per_instruction_tools) { add_per_instruction_tools(code, i, new_per_instruction_tools); } - i += inst.length; + i += instruction_length(code, i); } } #ifdef INSTRUMENT_DEBUG diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index 8788d44f110f35..7134bbf8f7ac0f 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -379,6 +379,8 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 0; case CACHE: return 0; + case RESERVED: + return 0; default: return -1; } @@ -761,6 +763,8 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 0; case CACHE: return 0; + case RESERVED: + return 0; default: return -1; } @@ -962,5 +966,6 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[256] = { [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IB }, [EXTENDED_ARG] = { true, INSTR_FMT_IB }, [CACHE] = { true, INSTR_FMT_IX }, + [RESERVED] = { true, INSTR_FMT_IX }, }; #endif diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 81882e08ebb9d1..9d6616666f7ac1 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -16,6 +16,7 @@ static void *opcode_targets[256] = { &&TARGET_BINARY_OP_MULTIPLY_INT, &&TARGET_UNARY_INVERT, &&TARGET_BINARY_OP_SUBTRACT_FLOAT, + &&TARGET_RESERVED, &&TARGET_BINARY_OP_SUBTRACT_INT, &&TARGET_BINARY_SUBSCR_DICT, &&TARGET_BINARY_SUBSCR_GETITEM, @@ -23,20 +24,20 @@ static void *opcode_targets[256] = { &&TARGET_BINARY_SUBSCR_TUPLE_INT, &&TARGET_CALL_PY_EXACT_ARGS, &&TARGET_CALL_PY_WITH_DEFAULTS, - &&TARGET_CALL_BOUND_METHOD_EXACT_ARGS, &&TARGET_BINARY_SUBSCR, &&TARGET_BINARY_SLICE, &&TARGET_STORE_SLICE, + &&TARGET_CALL_BOUND_METHOD_EXACT_ARGS, &&TARGET_CALL_BUILTIN_CLASS, - &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS, &&TARGET_GET_LEN, &&TARGET_MATCH_MAPPING, &&TARGET_MATCH_SEQUENCE, &&TARGET_MATCH_KEYS, - &&TARGET_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, + &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS, &&TARGET_PUSH_EXC_INFO, &&TARGET_CHECK_EXC_MATCH, &&TARGET_CHECK_EG_MATCH, + &&TARGET_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, &&TARGET_CALL_NO_KW_BUILTIN_FAST, &&TARGET_CALL_NO_KW_BUILTIN_O, &&TARGET_CALL_NO_KW_ISINSTANCE, @@ -47,7 +48,6 @@ static void *opcode_targets[256] = { &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_O, &&TARGET_CALL_NO_KW_STR_1, &&TARGET_CALL_NO_KW_TUPLE_1, - &&TARGET_CALL_NO_KW_TYPE_1, &&TARGET_WITH_EXCEPT_START, &&TARGET_GET_AITER, &&TARGET_GET_ANEXT, @@ -55,39 +55,39 @@ static void *opcode_targets[256] = { &&TARGET_BEFORE_WITH, &&TARGET_END_ASYNC_FOR, &&TARGET_CLEANUP_THROW, + &&TARGET_CALL_NO_KW_TYPE_1, &&TARGET_COMPARE_OP_FLOAT, &&TARGET_COMPARE_OP_INT, &&TARGET_COMPARE_OP_STR, - &&TARGET_FOR_ITER_LIST, &&TARGET_STORE_SUBSCR, &&TARGET_DELETE_SUBSCR, + &&TARGET_FOR_ITER_LIST, &&TARGET_FOR_ITER_TUPLE, &&TARGET_FOR_ITER_RANGE, &&TARGET_FOR_ITER_GEN, &&TARGET_LOAD_ATTR_CLASS, &&TARGET_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, - &&TARGET_LOAD_ATTR_INSTANCE_VALUE, &&TARGET_GET_ITER, &&TARGET_GET_YIELD_FROM_ITER, - &&TARGET_LOAD_ATTR_MODULE, + &&TARGET_LOAD_ATTR_INSTANCE_VALUE, &&TARGET_LOAD_BUILD_CLASS, + &&TARGET_LOAD_ATTR_MODULE, &&TARGET_LOAD_ATTR_PROPERTY, - &&TARGET_LOAD_ATTR_SLOT, &&TARGET_LOAD_ASSERTION_ERROR, &&TARGET_RETURN_GENERATOR, + &&TARGET_LOAD_ATTR_SLOT, &&TARGET_LOAD_ATTR_WITH_HINT, &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, &&TARGET_LOAD_ATTR_METHOD_NO_DICT, &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, &&TARGET_LOAD_CONST__LOAD_FAST, &&TARGET_LOAD_FAST__LOAD_CONST, - &&TARGET_LOAD_FAST__LOAD_FAST, &&TARGET_RETURN_VALUE, - &&TARGET_LOAD_GLOBAL_BUILTIN, + &&TARGET_LOAD_FAST__LOAD_FAST, &&TARGET_SETUP_ANNOTATIONS, + &&TARGET_LOAD_GLOBAL_BUILTIN, &&TARGET_LOAD_GLOBAL_MODULE, &&TARGET_STORE_ATTR_INSTANCE_VALUE, - &&TARGET_STORE_ATTR_SLOT, &&TARGET_POP_EXCEPT, &&TARGET_STORE_NAME, &&TARGET_DELETE_NAME, @@ -110,9 +110,9 @@ static void *opcode_targets[256] = { &&TARGET_IMPORT_NAME, &&TARGET_IMPORT_FROM, &&TARGET_JUMP_FORWARD, + &&TARGET_STORE_ATTR_SLOT, &&TARGET_STORE_ATTR_WITH_HINT, &&TARGET_STORE_FAST__LOAD_FAST, - &&TARGET_STORE_FAST__STORE_FAST, &&TARGET_POP_JUMP_IF_FALSE, &&TARGET_POP_JUMP_IF_TRUE, &&TARGET_LOAD_GLOBAL, @@ -140,9 +140,9 @@ static void *opcode_targets[256] = { &&TARGET_STORE_DEREF, &&TARGET_DELETE_DEREF, &&TARGET_JUMP_BACKWARD, - &&TARGET_STORE_SUBSCR_DICT, + &&TARGET_STORE_FAST__STORE_FAST, &&TARGET_CALL_FUNCTION_EX, - &&TARGET_STORE_SUBSCR_LIST_INT, + &&TARGET_STORE_SUBSCR_DICT, &&TARGET_EXTENDED_ARG, &&TARGET_LIST_APPEND, &&TARGET_SET_ADD, @@ -152,15 +152,15 @@ static void *opcode_targets[256] = { &&TARGET_YIELD_VALUE, &&TARGET_RESUME, &&TARGET_MATCH_CLASS, + &&TARGET_STORE_SUBSCR_LIST_INT, &&TARGET_UNPACK_SEQUENCE_LIST, - &&TARGET_UNPACK_SEQUENCE_TUPLE, &&TARGET_FORMAT_VALUE, &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_STRING, + &&TARGET_UNPACK_SEQUENCE_TUPLE, &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, &&TARGET_SEND_GEN, &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_LIST_EXTEND, &&TARGET_SET_UPDATE, &&TARGET_DICT_MERGE, From 7165f52c4c3e759a0542190848d3f2ab2d5b5b21 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sat, 25 Mar 2023 00:43:02 +0000 Subject: [PATCH 074/116] Fix up frame.set_lineno(). --- Include/internal/pycore_opcode.h | 1 + Lib/opcode.py | 3 + Objects/frameobject.c | 94 +++++++++++++++----------------- Python/bytecodes.c | 3 +- Python/generated_cases.c.h | 10 +++- Python/opcode_metadata.h | 6 +- 6 files changed, 61 insertions(+), 56 deletions(-) diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index b8d26b663353b0..a4f1a5ef8e33ef 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -41,6 +41,7 @@ static const uint32_t _PyOpcode_Jump[9] = { }; const uint8_t _PyOpcode_Caches[256] = { + [RESERVED] = 1000000, [BINARY_SUBSCR] = 4, [STORE_SUBSCR] = 1, [UNPACK_SEQUENCE] = 1, diff --git a/Lib/opcode.py b/Lib/opcode.py index 3ca4173e6bd83b..de162cab16ba02 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -444,6 +444,9 @@ def pseudo_op(name, op, real_ops): "SEND": { "counter": 1, }, + "RESERVED": { + "force_failure": 1_000_000 + }, } _inline_cache_entries = [ diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 7ae56e47200727..86cf579710956d 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -128,26 +128,6 @@ frame_settrace_opcodes(PyFrameObject *f, PyObject* value, void *Py_UNUSED(ignore return 0; } -// Given the index of the effective opcode, scan back to construct the oparg -// with EXTENDED_ARG. This only works correctly with *unquickened* code, -// obtained via a call to _PyCode_GetCode! -static unsigned int -get_arg(const _Py_CODEUNIT *codestr, Py_ssize_t i) -{ - _Py_CODEUNIT word; - unsigned int oparg = codestr[i].op.arg; - if (i >= 1 && (word = codestr[i-1]).op.code == EXTENDED_ARG) { - oparg |= word.op.arg << 8; - if (i >= 2 && (word = codestr[i-2]).op.code == EXTENDED_ARG) { - oparg |= word.op.arg << 16; - if (i >= 3 && (word = codestr[i-3]).op.code == EXTENDED_ARG) { - oparg |= word.op.arg << 24; - } - } - } - return oparg; -} - /* Model the evaluation stack, to determine which jumps * are safe and how many values needs to be popped. * The stack is modelled by a 64 integer, treating any @@ -323,46 +303,52 @@ mark_stacks(PyCodeObject *code_obj, int len) while (todo) { todo = 0; /* Scan instructions */ - for (i = 0; i < len; i++) { + for (i = 0; i < len;) { int64_t next_stack = stacks[i]; + opcode = _Py_GetBaseOpcode(code_obj, i); + int oparg = 0; + while (opcode == EXTENDED_ARG) { + oparg = (oparg << 8) | code[i].op.arg; + i++; + opcode = _Py_GetBaseOpcode(code_obj, i); + stacks[i] = next_stack; + } + int next_i = i + _PyOpcode_Caches[opcode] + 1; if (next_stack == UNINITIALIZED) { + i = next_i; continue; } - opcode = code[i].op.code; + oparg = (oparg << 8) | code[i].op.arg; switch (opcode) { case POP_JUMP_IF_FALSE: case POP_JUMP_IF_TRUE: { int64_t target_stack; - int j = get_arg(code, i); - j += i + 1; + int j = next_i + oparg; assert(j < len); - if (stacks[j] == UNINITIALIZED && j < i) { - todo = 1; - } next_stack = pop_value(next_stack); target_stack = next_stack; assert(stacks[j] == UNINITIALIZED || stacks[j] == target_stack); stacks[j] = target_stack; - stacks[i+1] = next_stack; + stacks[next_i] = next_stack; break; } case SEND: - j = get_arg(code, i) + i + INLINE_CACHE_ENTRIES_SEND + 1; + j = oparg + i + INLINE_CACHE_ENTRIES_SEND + 1; assert(j < len); assert(stacks[j] == UNINITIALIZED || stacks[j] == next_stack); stacks[j] = next_stack; - stacks[i+1] = next_stack; + stacks[next_i] = next_stack; break; case JUMP_FORWARD: - j = get_arg(code, i) + i + 1; + j = oparg + i + 1; assert(j < len); assert(stacks[j] == UNINITIALIZED || stacks[j] == next_stack); stacks[j] = next_stack; break; case JUMP_BACKWARD: case JUMP_BACKWARD_NO_INTERRUPT: - j = i + 1 - get_arg(code, i); + j = i + 1 - oparg; assert(j >= 0); assert(j < len); if (stacks[j] == UNINITIALIZED && j < i) { @@ -374,13 +360,13 @@ mark_stacks(PyCodeObject *code_obj, int len) case GET_ITER: case GET_AITER: next_stack = push_value(pop_value(next_stack), Iterator); - stacks[i+1] = next_stack; + stacks[next_i] = next_stack; break; case FOR_ITER: { int64_t target_stack = push_value(next_stack, Object); - stacks[i+1] = target_stack; - j = get_arg(code, i) + 1 + INLINE_CACHE_ENTRIES_FOR_ITER + i; + stacks[next_i] = target_stack; + j = oparg + 1 + INLINE_CACHE_ENTRIES_FOR_ITER + i; assert(j < len); assert(stacks[j] == UNINITIALIZED || stacks[j] == target_stack); stacks[j] = target_stack; @@ -388,16 +374,16 @@ mark_stacks(PyCodeObject *code_obj, int len) } case END_ASYNC_FOR: next_stack = pop_value(pop_value(next_stack)); - stacks[i+1] = next_stack; + stacks[next_i] = next_stack; break; case PUSH_EXC_INFO: next_stack = push_value(next_stack, Except); - stacks[i+1] = next_stack; + stacks[next_i] = next_stack; break; case POP_EXCEPT: assert(top_of_stack(next_stack) == Except); next_stack = pop_value(next_stack); - stacks[i+1] = next_stack; + stacks[next_i] = next_stack; break; case RETURN_VALUE: assert(pop_value(next_stack) == EMPTY_STACK); @@ -413,57 +399,62 @@ mark_stacks(PyCodeObject *code_obj, int len) break; case PUSH_NULL: next_stack = push_value(next_stack, Null); - stacks[i+1] = next_stack; + stacks[next_i] = next_stack; break; case LOAD_GLOBAL: { - int j = get_arg(code, i); + int j = oparg; if (j & 1) { next_stack = push_value(next_stack, Null); } next_stack = push_value(next_stack, Object); - stacks[i+1] = next_stack; + stacks[next_i] = next_stack; break; } case LOAD_ATTR: { assert(top_of_stack(next_stack) == Object); - int j = get_arg(code, i); + int j = oparg; if (j & 1) { next_stack = pop_value(next_stack); next_stack = push_value(next_stack, Null); next_stack = push_value(next_stack, Object); } - stacks[i+1] = next_stack; + stacks[next_i] = next_stack; break; } case CALL: { - int args = get_arg(code, i); + int args = oparg; for (int j = 0; j < args+2; j++) { next_stack = pop_value(next_stack); } next_stack = push_value(next_stack, Object); - stacks[i+1] = next_stack; + stacks[next_i] = next_stack; break; } case SWAP: { - int n = get_arg(code, i); + int n = oparg; next_stack = stack_swap(next_stack, n); - stacks[i+1] = next_stack; + stacks[next_i] = next_stack; break; } case COPY: { - int n = get_arg(code, i); + int n = oparg; next_stack = push_value(next_stack, peek(next_stack, n)); - stacks[i+1] = next_stack; + stacks[next_i] = next_stack; break; } + case CACHE: + case RESERVED: + { + assert(0); + } default: { - int delta = PyCompile_OpcodeStackEffect(opcode, get_arg(code, i)); + int delta = PyCompile_OpcodeStackEffect(opcode, oparg); assert(delta != PY_INVALID_STACK_EFFECT); while (delta < 0) { next_stack = pop_value(next_stack); @@ -473,9 +464,10 @@ mark_stacks(PyCodeObject *code_obj, int len) next_stack = push_value(next_stack, Object); delta--; } - stacks[i+1] = next_stack; + stacks[next_i] = next_stack; } } + i = next_i; } /* Scan exception table */ unsigned char *start = (unsigned char *)PyBytes_AS_STRING(code_obj->co_exceptiontable); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 6768513f8d4799..6faa1ac3d4f14b 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3245,7 +3245,8 @@ dummy_func( Py_UNREACHABLE(); } - inst(RESERVED, (--)) { + inst(RESERVED, (unused1/4, unused2/4, unused3/4, unused4/4, unused5[1000] --)) { + /* In case we haven't crashed already */ assert(0 && "Executing RESERVED instruction."); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 1c9f94ef40a2e1..04ada5037d351c 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -4551,8 +4551,16 @@ } TARGET(RESERVED) { + PyObject **unused5 = (stack_pointer - 1000); + PyObject *unused1 = read_obj(&next_instr[0].cache); + PyObject *unused2 = read_obj(&next_instr[4].cache); + PyObject *unused3 = read_obj(&next_instr[8].cache); + PyObject *unused4 = read_obj(&next_instr[12].cache); #line 3249 "Python/bytecodes.c" + /* In case we haven't crashed already */ assert(0 && "Executing RESERVED instruction."); - #line 4557 "Python/generated_cases.c.h" + #line 4563 "Python/generated_cases.c.h" + STACK_SHRINK(1000); + next_instr += 16; DISPATCH(); } diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index 7134bbf8f7ac0f..2c26a9122ee9b1 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -380,7 +380,7 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { case CACHE: return 0; case RESERVED: - return 0; + return 1000; default: return -1; } @@ -771,7 +771,7 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { } #endif -enum InstructionFormat { INSTR_FMT_IB, INSTR_FMT_IBC, INSTR_FMT_IBC000, INSTR_FMT_IBC00000000, INSTR_FMT_IBIB, INSTR_FMT_IX, INSTR_FMT_IXC, INSTR_FMT_IXC000 }; +enum InstructionFormat { INSTR_FMT_IB, INSTR_FMT_IBC, INSTR_FMT_IBC000, INSTR_FMT_IBC00000000, INSTR_FMT_IBIB, INSTR_FMT_IX, INSTR_FMT_IXC, INSTR_FMT_IXC000, INSTR_FMT_IXC000000000000000 }; struct opcode_metadata { bool valid_entry; enum InstructionFormat instr_format; @@ -966,6 +966,6 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[256] = { [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IB }, [EXTENDED_ARG] = { true, INSTR_FMT_IB }, [CACHE] = { true, INSTR_FMT_IX }, - [RESERVED] = { true, INSTR_FMT_IX }, + [RESERVED] = { true, INSTR_FMT_IXC000000000000000 }, }; #endif From 65c548e8776261f05ec46661da312490ae7dbbd1 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sat, 25 Mar 2023 15:02:59 +0000 Subject: [PATCH 075/116] Make line instrumentation able to handle multiple inits. --- Objects/codeobject.c | 9 +-- Python/instrumentation.c | 146 +++++++++++++-------------------------- Python/specialize.c | 3 +- 3 files changed, 54 insertions(+), 104 deletions(-) diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 024bd7bf78daa5..6288334eba11da 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1487,13 +1487,10 @@ deopt_code(PyCodeObject *code, _Py_CODEUNIT *instructions) int opcode = _Py_GetBaseOpcode(code, i); int caches = _PyOpcode_Caches[opcode]; instructions[i].op.code = opcode; - if (caches) { - instructions[i+1].cache = adaptive_counter_warmup(); - for (int j = 2; j <= caches; j++) { - instructions[i+j].cache = 0; - } - i += caches; + for (int j = 1; j <= caches; j++) { + instructions[i+j].cache = 0; } + i += caches; } } diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 59f500c14570e4..9168df3e3c3d2a 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -268,28 +268,6 @@ compute_line(PyCodeObject *code, int offset, int8_t line_delta) } } -/* We need to parse instructions from code objects a lot, - * so add a couple of utilities */ -int _Py_GetBaseOpcode(PyCodeObject *code, int offset) -{ - int opcode = _PyCode_CODE(code)[offset].op.code; - if (opcode == INSTRUMENTED_INSTRUCTION) { - opcode = code->_co_monitoring->per_instruction_opcodes[offset]; - } - if (opcode == INSTRUMENTED_LINE) { - opcode = code->_co_monitoring->lines[offset].original_opcode; - } - assert(opcode != INSTRUMENTED_INSTRUCTION); - assert(opcode != INSTRUMENTED_LINE); - int deinstrumented = DE_INSTRUMENT[opcode]; - if (deinstrumented) { - return deinstrumented; - } - assert(_PyOpcode_Deopt[opcode] != 0); - assert(opcode != RESERVED); - return _PyOpcode_Deopt[opcode]; -} - int instruction_length(PyCodeObject *code, int offset) { int opcode = _PyCode_CODE(code)[offset].op.code; @@ -325,38 +303,6 @@ typedef struct _Instruction { bool is_specialized_instrumented; } Instruction; -static Instruction -read_instruction(PyCodeObject *code, int offset) -{ - Instruction result = (Instruction){0}; - int opcode = result.actual_opcode = _PyCode_CODE(code)[offset].op.code; - if (opcode == INSTRUMENTED_INSTRUCTION) { - opcode = code->_co_monitoring->per_instruction_opcodes[offset]; - result.is_prefix_instrumented = true; - } - - if (opcode == INSTRUMENTED_LINE) { - opcode = code->_co_monitoring->lines[offset].original_opcode; - result.is_prefix_instrumented = true; - } - while (opcode == EXTENDED_ARG) { - result.extended_args += 1; - offset++; - opcode = _PyCode_CODE(code)[offset].op.code; - } - result.length = result.extended_args + 1; - int deinstrumented = DE_INSTRUMENT[opcode]; - if (deinstrumented) { - result.deinstrumented_opcode = deinstrumented; - result.is_specialized_instrumented = true; - } - else { - result.deinstrumented_opcode = opcode; - } - result.length = result.extended_args + instruction_length(code, offset); - return result; -} - #ifdef INSTRUMENT_DEBUG static void @@ -379,9 +325,6 @@ dump_instrumentation_data_lines(PyCodeObject *code, _PyCoLineInstrumentationData else if (lines[i].original_opcode == 0) { fprintf(out, ", lines = {original_opcode = No LINE (0), line_delta = %d)", lines[i].line_delta); } - else if (lines[i].original_opcode == 255) { - fprintf(out, ", lines = {original_opcode = LINE_MARKER, line_delta = %d)", lines[i].line_delta); - } else { fprintf(out, ", lines = {original_opcode = %s, line_delta = %d)", _PyOpcode_OpName[lines[i].original_opcode], lines[i].line_delta); } @@ -508,7 +451,9 @@ dump_instrumentation_data(PyCodeObject *code, int star, FILE*out) } while (0) bool valid_opcode(int opcode) { - if (opcode > 0 && opcode < 255 && + if (opcode > 0 && + opcode != RESERVED && + opcode < 255 && _PyOpcode_OpName[opcode] && _PyOpcode_OpName[opcode][0] != '<' ) { @@ -535,8 +480,8 @@ sanity_check_instrumentation(PyCodeObject *code) ); int code_len = (int)Py_SIZE(code); for (int i = 0; i < code_len;) { - Instruction inst = read_instruction(code, i); - int opcode = inst.actual_opcode; + int opcode = _PyCode_CODE(code)[i].op.code; + int base_opcode = _Py_GetBaseOpcode(code, i); if (opcode == INSTRUMENTED_INSTRUCTION) { opcode = data->per_instruction_opcodes[i]; if (!is_instrumented(opcode)) { @@ -547,7 +492,7 @@ sanity_check_instrumentation(PyCodeObject *code) if (opcode == INSTRUMENTED_LINE) { CHECK(data->lines); CHECK(valid_opcode(data->lines[i].original_opcode)); - int opcode = data->lines[i].original_opcode; + opcode = data->lines[i].original_opcode; CHECK(opcode != END_FOR); CHECK(opcode != RESUME); CHECK(opcode != INSTRUMENTED_RESUME); @@ -556,38 +501,32 @@ sanity_check_instrumentation(PyCodeObject *code) } CHECK(opcode != INSTRUMENTED_LINE); } - else if (data->lines) { + else if (data->lines && !is_instrumented(opcode)) { CHECK(data->lines[i].original_opcode == 0 || - data->lines[i].original_opcode == 255); + data->lines[i].original_opcode == base_opcode); } - if (inst.is_specialized_instrumented) { - int event = EVENT_FOR_OPCODE[inst.deinstrumented_opcode]; + if (is_instrumented(opcode)) { + CHECK(DE_INSTRUMENT[opcode] == base_opcode); + int event = EVENT_FOR_OPCODE[DE_INSTRUMENT[opcode]]; if (event < 0) { /* RESUME fixup */ - event = _PyCode_CODE(code)[i + inst.extended_args].op.arg; + event = _PyCode_CODE(code)[i].op.arg; } CHECK(active_monitors.tools[event] != 0); } - if (data->lines && inst.deinstrumented_opcode != END_FOR) { + if (data->lines && base_opcode != END_FOR) { int line1 = compute_line(code, i, data->lines[i].line_delta); int line2 = PyCode_Addr2Line(code, i*sizeof(_Py_CODEUNIT)); CHECK(line1 == line2); } - CHECK(valid_opcode(inst.deinstrumented_opcode)); - if (inst.is_prefix_instrumented) { - CHECK(inst.extended_args || _PyOpcode_Deopt[inst.deinstrumented_opcode] == inst.deinstrumented_opcode); - } - if (inst.is_specialized_instrumented) { - CHECK(_PyOpcode_Deopt[inst.deinstrumented_opcode] == inst.deinstrumented_opcode); - } + CHECK(valid_opcode(opcode)); if (data->tools) { - uint8_t local_tools = data->tools[i + inst.extended_args]; - int deopt = _PyOpcode_Deopt[inst.deinstrumented_opcode]; - if (OPCODE_HAS_EVENT[deopt]) { - int event = EVENT_FOR_OPCODE[deopt]; + uint8_t local_tools = data->tools[i]; + if (OPCODE_HAS_EVENT[base_opcode]) { + int event = EVENT_FOR_OPCODE[base_opcode]; if (event == -1) { /* RESUME fixup */ - event = _PyCode_CODE(code)[i + inst.extended_args].op.arg; + event = _PyCode_CODE(code)[i].op.arg; } CHECK((active_monitors.tools[event] & local_tools) == local_tools); } @@ -595,7 +534,7 @@ sanity_check_instrumentation(PyCodeObject *code) CHECK(local_tools == 0xff); } } - i += inst.length; + i += instruction_length(code, i); assert(i <= code_len); } } @@ -605,6 +544,27 @@ sanity_check_instrumentation(PyCodeObject *code) #endif +/* Get the underlying opcode, stripping instrumentation */ +int _Py_GetBaseOpcode(PyCodeObject *code, int i) +{ + int opcode = _PyCode_CODE(code)[i].op.code; + if (opcode == INSTRUMENTED_INSTRUCTION) { + opcode = code->_co_monitoring->per_instruction_opcodes[i]; + } + if (opcode == INSTRUMENTED_LINE) { + opcode = code->_co_monitoring->lines[i].original_opcode; + } + CHECK(opcode != INSTRUMENTED_INSTRUCTION); + CHECK(opcode != INSTRUMENTED_LINE); + int deinstrumented = DE_INSTRUMENT[opcode]; + if (deinstrumented) { + return deinstrumented; + } + CHECK(_PyOpcode_Deopt[opcode] != 0); + CHECK(opcode != RESERVED); + return _PyOpcode_Deopt[opcode]; +} + static void de_instrument(PyCodeObject *code, int offset, int event) { @@ -682,8 +642,6 @@ de_instrument_line(PyCodeObject *code, int i) instr->op.code = original_opcode; } assert(instr->op.code != INSTRUMENTED_LINE); - /* Mark instruction as candidate for line instrumentation */ - lines->original_opcode = 255; } @@ -752,19 +710,15 @@ instrument_line(PyCodeObject *code, int i) return; } _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[i]; - CHECK(lines->original_opcode == 255); - if (is_instrumented(opcode)) { - lines->original_opcode = opcode; - } - else { - assert(opcode != 0); - assert(_PyOpcode_Deopt[opcode] != RESUME); - lines->original_opcode = _PyOpcode_Deopt[opcode]; - CHECK(lines->original_opcode != 0); - } - assert(lines->original_opcode > 0); + CHECK(_PyOpcode_Deopt[opcode] == lines->original_opcode || + ( + is_instrumented(opcode) && + lines->original_opcode == DE_INSTRUMENT[opcode] + ) + ); + lines->original_opcode = _PyOpcode_Deopt[opcode]; + CHECK(lines->original_opcode > 0); instr->op.code = INSTRUMENTED_LINE; - CHECK(code->_co_monitoring->lines[i].original_opcode != 0); } static void @@ -1343,8 +1297,6 @@ initialize_tools(PyCodeObject *code) } } -#define LINE_MARKER 255 - #define NO_LINE -128 static void @@ -1377,7 +1329,7 @@ initialize_lines(PyCodeObject *code) break; default: if (line != current_line && line >= 0) { - line_data[i].original_opcode = LINE_MARKER; + line_data[i].original_opcode = opcode; } else { line_data[i].original_opcode = 0; diff --git a/Python/specialize.c b/Python/specialize.c index 38392579be08b5..77e636d3ac2d2b 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -273,7 +273,8 @@ _PyCode_Quicken(PyCodeObject *code) _Py_CODEUNIT *instructions = _PyCode_CODE(code); for (int i = 0; i < Py_SIZE(code); i++) { int previous_opcode = opcode; - opcode = _PyOpcode_Deopt[instructions[i].op.code]; + opcode = _Py_GetBaseOpcode(code, i); + assert(opcode < 230); int caches = _PyOpcode_Caches[opcode]; if (caches) { instructions[i + 1].cache = adaptive_counter_warmup(); From 575f7d15c8377691448de359bb8e790ce43950a8 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sat, 25 Mar 2023 15:14:13 +0000 Subject: [PATCH 076/116] Add brief comments on instrumentation data structures. --- Include/cpython/code.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Include/cpython/code.h b/Include/cpython/code.h index 39ec413c9d4980..c61926b6fd19b2 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -64,11 +64,17 @@ typedef struct { PyObject *_co_freevars; } _PyCoCached; +/* Ancilliary data structure used for instrumentation. + Line instrumentation creates an array of + these. One entry per code unit.*/ typedef struct { uint8_t original_opcode; int8_t line_delta; } _PyCoLineInstrumentationData; +/* Main data structure used for instrumentation. + * This is allocated when needed for instrumentation + */ typedef struct { _Py_Monitors local_monitors; _Py_Monitors active_monitors; From 415741d30fddb7b1c99b6b082c4f60eb81c959d4 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sat, 25 Mar 2023 15:23:35 +0000 Subject: [PATCH 077/116] Add symbol constant for min instrumented opcode. --- Include/cpython/code.h | 1 + Include/opcode.h | 1 + Lib/opcode.py | 1 + Lib/test/test__opcode.py | 6 ++---- Programs/test_frozenmain.h | 16 ++++++++-------- Tools/build/generate_opcode_h.py | 3 +++ 6 files changed, 16 insertions(+), 12 deletions(-) diff --git a/Include/cpython/code.h b/Include/cpython/code.h index c61926b6fd19b2..ea2d9a3d72f1a5 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -11,6 +11,7 @@ extern "C" { #define PY_MONITORING_EVENTS 16 #define PY_MONITORING_UNGROUPED_EVENTS 14 +/* Table of which tools are active for each monitored event. */ typedef struct _Py_Monitors { uint8_t tools[PY_MONITORING_UNGROUPED_EVENTS]; } _Py_Monitors; diff --git a/Include/opcode.h b/Include/opcode.h index 701d7f81837e5e..aa8716ef5b4030 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -116,6 +116,7 @@ extern "C" { #define KW_NAMES 172 #define CALL_INTRINSIC_1 173 #define CALL_INTRINSIC_2 174 +#define MIN_INSTRUMENTED_OPCODE 238 #define INSTRUMENTED_POP_JUMP_IF_NONE 238 #define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 239 #define INSTRUMENTED_RESUME 240 diff --git a/Lib/opcode.py b/Lib/opcode.py index de162cab16ba02..f471dd9c65f0d2 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -225,6 +225,7 @@ def pseudo_op(name, op, real_ops): def_op('CALL_INTRINSIC_2', 174) # Instrumented instructions +MIN_INSTRUMENTED_OPCODE = 238 def_op('INSTRUMENTED_POP_JUMP_IF_NONE', 238) def_op('INSTRUMENTED_POP_JUMP_IF_NOT_NONE', 239) diff --git a/Lib/test/test__opcode.py b/Lib/test/test__opcode.py index eb8666cc41ce8b..7640c6fb57d4f3 100644 --- a/Lib/test/test__opcode.py +++ b/Lib/test/test__opcode.py @@ -20,8 +20,7 @@ def test_stack_effect(self): # All defined opcodes has_arg = dis.hasarg for name, code in filter(lambda item: item[0] not in dis.deoptmap, dis.opmap.items()): - if code >= 230: - # Opcodes 230 and up are internal for instrumentation + if code >= opcode.MIN_INSTRUMENTED_OPCODE: continue with self.subTest(opname=name): if code not in has_arg: @@ -50,8 +49,7 @@ def test_stack_effect_jump(self): has_exc = dis.hasexc has_jump = dis.hasjabs + dis.hasjrel for name, code in filter(lambda item: item[0] not in dis.deoptmap, dis.opmap.items()): - if code >= 230: - # Opcodes 230 and up are internal for instrumentation + if code >= opcode.MIN_INSTRUMENTED_OPCODE: continue with self.subTest(opname=name): if code not in has_arg: diff --git a/Programs/test_frozenmain.h b/Programs/test_frozenmain.h index 57635a398d1df0..8e5055bd7bceb1 100644 --- a/Programs/test_frozenmain.h +++ b/Programs/test_frozenmain.h @@ -3,16 +3,16 @@ unsigned char M_test_frozenmain[] = { 227,0,0,0,0,0,0,0,0,0,0,0,0,8,0,0, 0,0,0,0,0,243,182,0,0,0,151,0,100,0,100,1, 108,0,90,0,100,0,100,1,108,1,90,1,2,0,101,2, - 100,2,171,1,17,0,0,0,0,0,0,0,1,0,2,0, - 101,2,100,3,101,0,106,6,17,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,171,2,17,0,0,0, - 0,0,0,0,1,0,2,0,101,1,106,8,17,0,0,0, + 100,2,171,1,0,0,0,0,0,0,0,0,1,0,2,0, + 101,2,100,3,101,0,106,6,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,171,2,0,0,0,0, + 0,0,0,0,1,0,2,0,101,1,106,8,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,171,0, - 17,0,0,0,0,0,0,0,100,4,25,0,17,0,0,0, - 0,0,0,0,90,5,100,5,68,0,93,23,17,0,90,6, + 0,0,0,0,0,0,0,0,100,4,25,0,0,0,0,0, + 0,0,0,0,90,5,100,5,68,0,93,23,0,0,90,6, 2,0,101,2,100,6,101,6,155,0,100,7,101,5,101,6, - 25,0,17,0,0,0,0,0,0,0,155,0,157,4,171,1, - 17,0,0,0,0,0,0,0,1,0,140,25,4,0,121,1, + 25,0,0,0,0,0,0,0,0,0,155,0,157,4,171,1, + 0,0,0,0,0,0,0,0,1,0,140,25,4,0,121,1, 41,8,233,0,0,0,0,78,122,18,70,114,111,122,101,110, 32,72,101,108,108,111,32,87,111,114,108,100,122,8,115,121, 115,46,97,114,103,118,218,6,99,111,110,102,105,103,41,5, diff --git a/Tools/build/generate_opcode_h.py b/Tools/build/generate_opcode_h.py index 7aaef6b16f3443..53cf11ddeb0093 100644 --- a/Tools/build/generate_opcode_h.py +++ b/Tools/build/generate_opcode_h.py @@ -89,6 +89,7 @@ def main(opcode_py, outfile='Include/opcode.h', internaloutfile='Include/interna HAVE_ARGUMENT = opcode["HAVE_ARGUMENT"] MIN_PSEUDO_OPCODE = opcode["MIN_PSEUDO_OPCODE"] MAX_PSEUDO_OPCODE = opcode["MAX_PSEUDO_OPCODE"] + MIN_INSTRUMENTED_OPCODE = opcode["MIN_INSTRUMENTED_OPCODE"] NUM_OPCODES = len(opname) used = [ False ] * len(opname) @@ -117,6 +118,8 @@ def main(opcode_py, outfile='Include/opcode.h', internaloutfile='Include/interna fobj.write(DEFINE.format("HAVE_ARGUMENT", HAVE_ARGUMENT)) if op == MIN_PSEUDO_OPCODE: fobj.write(DEFINE.format("MIN_PSEUDO_OPCODE", MIN_PSEUDO_OPCODE)) + if op == MIN_INSTRUMENTED_OPCODE: + fobj.write(DEFINE.format("MIN_INSTRUMENTED_OPCODE", MIN_INSTRUMENTED_OPCODE)) fobj.write(DEFINE.format(name, op)) From d70a1a417d3a8bb44a15f04e03493e9148305527 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sat, 25 Mar 2023 15:44:26 +0000 Subject: [PATCH 078/116] Code cleanup from review. --- Python/instrumentation.c | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 9168df3e3c3d2a..63dbb2c7908ec9 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -145,7 +145,7 @@ static inline bool is_instrumented(int opcode) { assert(opcode != 0); assert(opcode != RESERVED); - return INSTRUMENTED_OPCODES[opcode] == opcode; + return opcode >= MIN_INSTRUMENTED_OPCODE; } static inline bool @@ -190,7 +190,7 @@ monitors_or(_Py_Monitors a, _Py_Monitors b) } static inline bool -monitors_empty(_Py_Monitors m) +monitors_are_empty(_Py_Monitors m) { for (int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { if (m.tools[i]) { @@ -200,7 +200,7 @@ monitors_empty(_Py_Monitors m) return true; } -static inline int +static inline bool multiple_tools(_Py_Monitors *m) { for (int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { @@ -227,7 +227,7 @@ get_events(_Py_Monitors *m, int tool_id) * 8 bit value. * if line_delta == -128: * line = None # represented as -1 - * elif line == -127: + * elif line_delta == -127: * line = PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT)); * else: * line = first_line + (offset >> OFFSET_SHIFT) + line_delta; @@ -241,9 +241,6 @@ compute_line_delta(PyCodeObject *code, int offset, int line) if (line < 0) { return -128; } - // assert(line >= code->co_firstlineno); - // assert(offset >= code->_co_firsttraceable); - // assert(offset < Py_SIZE(code)); int delta = line - code->co_firstlineno - (offset >> OFFSET_SHIFT); if (delta < 128 && delta > -128) { return delta; @@ -255,7 +252,6 @@ static int compute_line(PyCodeObject *code, int offset, int8_t line_delta) { if (line_delta > -127) { - // assert((offset >> OFFSET_SHIFT) + line_delta >= 0); return code->co_firstlineno + (offset >> OFFSET_SHIFT) + line_delta; } if (line_delta == -128) { @@ -482,6 +478,8 @@ sanity_check_instrumentation(PyCodeObject *code) for (int i = 0; i < code_len;) { int opcode = _PyCode_CODE(code)[i].op.code; int base_opcode = _Py_GetBaseOpcode(code, i); + CHECK(valid_opcode(opcode)); + CHECK(valid_opcode(base_opcode)); if (opcode == INSTRUMENTED_INSTRUCTION) { opcode = data->per_instruction_opcodes[i]; if (!is_instrumented(opcode)) { @@ -503,7 +501,8 @@ sanity_check_instrumentation(PyCodeObject *code) } else if (data->lines && !is_instrumented(opcode)) { CHECK(data->lines[i].original_opcode == 0 || - data->lines[i].original_opcode == base_opcode); + data->lines[i].original_opcode == base_opcode || + DE_INSTRUMENT[data->lines[i].original_opcode] == base_opcode); } if (is_instrumented(opcode)) { CHECK(DE_INSTRUMENT[opcode] == base_opcode); @@ -1493,11 +1492,11 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) else { removed_events = monitors_sub(code->_co_monitoring->active_monitors, active_events); new_events = monitors_sub(active_events, code->_co_monitoring->active_monitors); - assert(monitors_empty(monitors_and(new_events, removed_events))); + assert(monitors_are_empty(monitors_and(new_events, removed_events))); } code->_co_monitoring->active_monitors = active_events; code->_co_instrumentation_version = interp->monitoring_version; - if (monitors_empty(new_events) && monitors_empty(removed_events)) { + if (monitors_are_empty(new_events) && monitors_are_empty(removed_events)) { #ifdef INSTRUMENT_DEBUG sanity_check_instrumentation(code); #endif From d580de6338dcea14374b74a16ad72b43f34c5e29 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sat, 25 Mar 2023 16:06:35 +0000 Subject: [PATCH 079/116] A bit more cleanup. --- Python/bytecodes.c | 5 ++-- Python/instrumentation.c | 49 ++++++++-------------------------------- 2 files changed, 13 insertions(+), 41 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 6faa1ac3d4f14b..7426c1d8cfa1b4 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3242,12 +3242,13 @@ dummy_func( } inst(CACHE, (--)) { + assert(0 && "Executing a cache."); Py_UNREACHABLE(); } - inst(RESERVED, (unused1/4, unused2/4, unused3/4, unused4/4, unused5[1000] --)) { - /* In case we haven't crashed already */ + inst(RESERVED, (--)) { assert(0 && "Executing RESERVED instruction."); + Py_UNREACHABLE(); } diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 63dbb2c7908ec9..6a02c9319385f6 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -11,7 +11,7 @@ #include "pycore_pystate.h" /* Uncomment this to dump debugging output when assertions fail */ -#define INSTRUMENT_DEBUG 1 +// #define INSTRUMENT_DEBUG 1 static PyObject DISABLE = { @@ -57,38 +57,6 @@ static const int8_t EVENT_FOR_OPCODE[256] = { [INSTRUMENTED_END_SEND] = PY_MONITORING_EVENT_STOP_ITERATION, }; -static const bool OPCODE_HAS_EVENT[256] = { - [RETURN_CONST] = true, - [INSTRUMENTED_RETURN_CONST] = true, - [RETURN_VALUE] = true, - [INSTRUMENTED_RETURN_VALUE] = true, - [CALL] = true, - [INSTRUMENTED_CALL] = true, - [CALL_FUNCTION_EX] = true, - [INSTRUMENTED_CALL_FUNCTION_EX] = true, - [RESUME] = true, - [INSTRUMENTED_RESUME] = true, - [YIELD_VALUE] = true, - [INSTRUMENTED_YIELD_VALUE] = true, - [JUMP_FORWARD] = true, - [JUMP_BACKWARD] = true, - [POP_JUMP_IF_FALSE] = true, - [POP_JUMP_IF_TRUE] = true, - [POP_JUMP_IF_NONE] = true, - [POP_JUMP_IF_NOT_NONE] = true, - [INSTRUMENTED_JUMP_FORWARD] = true, - [INSTRUMENTED_JUMP_BACKWARD] = true, - [INSTRUMENTED_POP_JUMP_IF_FALSE] = true, - [INSTRUMENTED_POP_JUMP_IF_TRUE] = true, - [INSTRUMENTED_POP_JUMP_IF_NONE] = true, - [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = true, - [INSTRUMENTED_FOR_ITER] = true, - [END_FOR] = true, - [INSTRUMENTED_END_FOR] = true, - [END_SEND] = true, - [INSTRUMENTED_END_SEND] = true, -}; - static const uint8_t DE_INSTRUMENT[256] = { [INSTRUMENTED_RESUME] = RESUME, [INSTRUMENTED_RETURN_VALUE] = RETURN_VALUE, @@ -141,6 +109,12 @@ static const uint8_t INSTRUMENTED_OPCODES[256] = { [INSTRUMENTED_INSTRUCTION] = INSTRUMENTED_INSTRUCTION, }; +static inline bool +opcode_has_event(int opcode) { + return opcode < INSTRUMENTED_LINE && + INSTRUMENTED_OPCODES[opcode] > 0; +} + static inline bool is_instrumented(int opcode) { assert(opcode != 0); @@ -559,8 +533,6 @@ int _Py_GetBaseOpcode(PyCodeObject *code, int i) if (deinstrumented) { return deinstrumented; } - CHECK(_PyOpcode_Deopt[opcode] != 0); - CHECK(opcode != RESERVED); return _PyOpcode_Deopt[opcode]; } @@ -754,8 +726,7 @@ instruction_has_event(PyCodeObject *code, int offset) if (opcode == INSTRUMENTED_LINE) { opcode = code->_co_monitoring->lines[offset].original_opcode; } - assert(DE_INSTRUMENT[opcode] == 0 || OPCODE_HAS_EVENT[DE_INSTRUMENT[opcode]] == OPCODE_HAS_EVENT[opcode]); - return OPCODE_HAS_EVENT[opcode]; + return opcode_has_event(opcode); } #endif @@ -1264,7 +1235,7 @@ initialize_tools(PyCodeObject *code) assert(opcode != 0); } opcode = _PyOpcode_Deopt[opcode]; - if (OPCODE_HAS_EVENT[opcode]) { + if (opcode_has_event(opcode)) { if (instrumented) { int8_t event; if (opcode == RESUME) { @@ -1510,7 +1481,7 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) } CHECK(instr->op.code != 0); int base_opcode = _Py_GetBaseOpcode(code, i); - if (OPCODE_HAS_EVENT[base_opcode]) { + if (opcode_has_event(base_opcode)) { int8_t event; if (base_opcode == RESUME) { event = instr->op.arg > 0; From 2076d5fe143c89368e0f4fe21a6f27e4f3560b92 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sat, 25 Mar 2023 17:01:21 +0000 Subject: [PATCH 080/116] Add a few more comments. --- Include/cpython/code.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Include/cpython/code.h b/Include/cpython/code.h index ea2d9a3d72f1a5..bf7a3b666b2d39 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -77,12 +77,20 @@ typedef struct { * This is allocated when needed for instrumentation */ typedef struct { + /* Monitoring specific to this code object */ _Py_Monitors local_monitors; + /* Monitoring that is active on this code object */ _Py_Monitors active_monitors; + /* The tools that are to be notified for events for the matching code unit */ uint8_t *tools; + /* Information to support line events */ _PyCoLineInstrumentationData *lines; + /* The tools that are to be notified for line events for the matching code unit */ uint8_t *line_tools; + /* Information to support instruction events */ + /* The underlying instructions, which can themselves be instrumented */ uint8_t *per_instruction_opcodes; + /* The tools that are to be notified for instruction events for the matching code unit */ uint8_t *per_instruction_tools; } _PyCoMonitoringData; From 7b32d7974559f8d6188c43d413e30b78a5427335 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sat, 25 Mar 2023 17:04:10 +0000 Subject: [PATCH 081/116] Remove unused code. --- Include/internal/pycore_instruments.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index f34bd074e6a92f..e3327c29389dd2 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -43,14 +43,10 @@ extern "C" { #define PY_MONITORING_EVENT_C_RAISE 15 -/* #define INSTRUMENT_EVENT_BRANCH_NOT_TAKEN xxx -- If we can afford this */ - - typedef uint32_t _PyMonitoringEventSet; /* Reserved IDs */ -#define PY_INSTRUMENT_PEP_523 5 #define PY_INSTRUMENT_SYS_PROFILE 6 #define PY_INSTRUMENT_SYS_TRACE 7 From 662c16c6911793b2413f1e8ce03425a3af31fb64 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sat, 25 Mar 2023 17:17:46 +0000 Subject: [PATCH 082/116] Fix leak of monitoring blocks. --- Objects/codeobject.c | 62 +++++++++++++++++++------------------------- 1 file changed, 26 insertions(+), 36 deletions(-) diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 6288334eba11da..34e1edc6c2678b 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1642,6 +1642,30 @@ code_new_impl(PyTypeObject *type, int argcount, int posonlyargcount, return co; } +static void +free_monitoring_data(_PyCoMonitoringData *data) +{ + if (data == NULL) { + return; + } + if (data->tools) { + PyMem_Free(data->tools); + } + if (data->lines) { + PyMem_Free(data->lines); + } + if (data->line_tools) { + PyMem_Free(data->line_tools); + } + if (data->per_instruction_opcodes) { + PyMem_Free(data->per_instruction_opcodes); + } + if (data->per_instruction_tools) { + PyMem_Free(data->per_instruction_tools); + } + PyMem_Free(data); +} + static void code_dealloc(PyCodeObject *co) { @@ -1688,24 +1712,7 @@ code_dealloc(PyCodeObject *co) if (co->co_weakreflist != NULL) { PyObject_ClearWeakRefs((PyObject*)co); } - _PyCoMonitoringData *data = co->_co_monitoring; - if (data) { - if (data->tools) { - PyMem_Free(data->tools); - } - if (data->lines) { - PyMem_Free(data->lines); - } - if (data->line_tools) { - PyMem_Free(data->line_tools); - } - if (data->per_instruction_opcodes) { - PyMem_Free(data->per_instruction_opcodes); - } - if (data->per_instruction_tools) { - PyMem_Free(data->per_instruction_tools); - } - } + free_monitoring_data(co->_co_monitoring); PyObject_Free(co); } @@ -2288,24 +2295,7 @@ _PyStaticCode_Fini(PyCodeObject *co) PyObject_ClearWeakRefs((PyObject *)co); co->co_weakreflist = NULL; } - _PyCoMonitoringData *data = co->_co_monitoring; - if (data) { - if (data->tools) { - PyMem_Free(data->tools); - } - if (data->lines) { - PyMem_Free(data->lines); - } - if (data->line_tools) { - PyMem_Free(data->line_tools); - } - if (data->per_instruction_opcodes) { - PyMem_Free(data->per_instruction_opcodes); - } - if (data->per_instruction_tools) { - PyMem_Free(data->per_instruction_tools); - } - } + free_monitoring_data(co->_co_monitoring); } int From 51a93e7ff64c4541acd71c8331a2b26e46cbcd2c Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sat, 25 Mar 2023 19:02:39 +0000 Subject: [PATCH 083/116] Make function static. --- Python/instrumentation.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 6a02c9319385f6..869b4445344151 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -238,7 +238,8 @@ compute_line(PyCodeObject *code, int offset, int8_t line_delta) } } -int instruction_length(PyCodeObject *code, int offset) +static int +instruction_length(PyCodeObject *code, int offset) { int opcode = _PyCode_CODE(code)[offset].op.code; assert(opcode != 0); From 64bf37ff351046b2ab5ed4d528d917ac9b0fb194 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 26 Mar 2023 07:47:22 +0100 Subject: [PATCH 084/116] Use symbolic constants in line computations. --- Python/instrumentation.c | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 869b4445344151..4736c3ed1e9b6d 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -207,35 +207,37 @@ get_events(_Py_Monitors *m, int tool_id) * line = first_line + (offset >> OFFSET_SHIFT) + line_delta; */ +#define NO_LINE -128 +#define COMPUTED_LINE -127 + #define OFFSET_SHIFT 4 static int8_t compute_line_delta(PyCodeObject *code, int offset, int line) { if (line < 0) { - return -128; + return NO_LINE; } int delta = line - code->co_firstlineno - (offset >> OFFSET_SHIFT); - if (delta < 128 && delta > -128) { + if (delta <= INT8_MAX && delta > COMPUTED_LINE) { return delta; } - return -127; + return COMPUTED_LINE; } static int compute_line(PyCodeObject *code, int offset, int8_t line_delta) { - if (line_delta > -127) { + if (line_delta > COMPUTED_LINE) { return code->co_firstlineno + (offset >> OFFSET_SHIFT) + line_delta; } - if (line_delta == -128) { + if (line_delta == NO_LINE) { + return -1; } - else { - assert(line_delta == -127); - /* Look it up */ - return PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT)); - } + assert(line_delta == COMPUTED_LINE); + /* Look it up */ + return PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT)); } static int From 5faec77e8e8e5407e1202019dfa7835d5da49d30 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 26 Mar 2023 07:53:05 +0100 Subject: [PATCH 085/116] Make data static. --- Python/instrumentation.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 4736c3ed1e9b6d..bd68901a72ba23 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1908,7 +1908,7 @@ add_power2_constant(PyObject *obj, const char *name, int i) return err; } -const char *const event_names [] = { +static const char *const event_names [] = { [PY_MONITORING_EVENT_PY_START] = "PY_START", [PY_MONITORING_EVENT_PY_RESUME] = "PY_RESUME", [PY_MONITORING_EVENT_PY_RETURN] = "PY_RETURN", From 0be156286aa08b30205c24dbaf067e6c29355087 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 26 Mar 2023 08:12:15 +0100 Subject: [PATCH 086/116] Add extra check. --- Python/instrumentation.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index bd68901a72ba23..66f485b2e65723 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -11,7 +11,7 @@ #include "pycore_pystate.h" /* Uncomment this to dump debugging output when assertions fail */ -// #define INSTRUMENT_DEBUG 1 +#define INSTRUMENT_DEBUG 1 static PyObject DISABLE = { @@ -401,7 +401,6 @@ dump_instrumentation_data(PyCodeObject *code, int star, FILE*out) dump_instrumentation_data_lines(code, data->lines, i, out); dump_instrumentation_data_line_tools(code, data->line_tools, i, out); dump_instrumentation_data_per_instruction(code, data, i, out); - /* TO DO -- per instruction data */ fprintf(out, "\n"); ; } @@ -462,7 +461,10 @@ sanity_check_instrumentation(PyCodeObject *code) if (!is_instrumented(opcode)) { CHECK(_PyOpcode_Deopt[opcode] == opcode); } - /* TO DO -- check tools */ + if (data->per_instruction_tools) { + uint8_t tools = active_monitors.tools[PY_MONITORING_EVENT_INSTRUCTION]; + CHECK((tools & data->per_instruction_tools[i]) == data->per_instruction_tools[i]); + } } if (opcode == INSTRUMENTED_LINE) { CHECK(data->lines); @@ -498,7 +500,7 @@ sanity_check_instrumentation(PyCodeObject *code) CHECK(valid_opcode(opcode)); if (data->tools) { uint8_t local_tools = data->tools[i]; - if (OPCODE_HAS_EVENT[base_opcode]) { + if (opcode_has_event(base_opcode)) { int event = EVENT_FOR_OPCODE[base_opcode]; if (event == -1) { /* RESUME fixup */ From b32a075094f175ad30fef0939ac48016ec34f3fd Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 26 Mar 2023 10:38:44 +0100 Subject: [PATCH 087/116] Make sure that instrumentation of line and instructions at the same time is supported. --- Lib/test/test_monitoring.py | 162 ++++++++++++++++++++++++++++++++++++ Python/instrumentation.c | 38 +++++---- 2 files changed, 183 insertions(+), 17 deletions(-) diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 9278a474ed66fe..0b1e171ac3a474 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -767,6 +767,168 @@ def func3(): ('line', 'check_events', 11), ('call', 'set_events', 2)]) +class InstructionRecorder: + + event_type = E.INSTRUCTION + + def __init__(self, events): + self.events = events + + def __call__(self, code, offset): + # Filter out instructions in check_events to lower noise + if code.co_name != "check_events": + self.events.append(("instruction", code.co_name, offset)) + + +LINE_AND_INSTRUCTION_RECORDERS = InstructionRecorder, LineRecorder + +class TestLineAndInstructionEvents(CheckEvents): + maxDiff = None + + def test_simple(self): + + def func1(): + line1 = 1 + line2 = 2 + line3 = 3 + + self.check_events(func1, recorders = LINE_AND_INSTRUCTION_RECORDERS, expected = [ + ('line', 'check_events', 10), + ('instruction', 'func1', 1), + ('line', 'func1', 1), + ('instruction', 'func1', 2), + ('instruction', 'func1', 3), + ('line', 'func1', 2), + ('instruction', 'func1', 4), + ('instruction', 'func1', 5), + ('line', 'func1', 3), + ('instruction', 'func1', 6), + ('instruction', 'func1', 7), + ('line', 'check_events', 11)]) + + def test_c_call(self): + + def func2(): + line1 = 1 + [].append(2) + line3 = 3 + + self.check_events(func2, recorders = LINE_AND_INSTRUCTION_RECORDERS, expected = [ + ('line', 'check_events', 10), + ('instruction', 'func2', 1), + ('line', 'func2', 1), + ('instruction', 'func2', 2), + ('instruction', 'func2', 3), + ('line', 'func2', 2), + ('instruction', 'func2', 4), + ('instruction', 'func2', 14), + ('instruction', 'func2', 15), + ('instruction', 'func2', 20), + ('instruction', 'func2', 21), + ('line', 'func2', 3), + ('instruction', 'func2', 22), + ('instruction', 'func2', 23), + ('line', 'check_events', 11)]) + + def test_try_except(self): + + def func3(): + try: + line = 2 + raise KeyError + except: + line = 5 + line = 6 + + self.check_events(func3, recorders = LINE_AND_INSTRUCTION_RECORDERS, expected = [ + ('line', 'check_events', 10), + ('instruction', 'func3', 1), + ('line', 'func3', 1), + ('instruction', 'func3', 2), + ('line', 'func3', 2), + ('instruction', 'func3', 3), + ('instruction', 'func3', 4), + ('line', 'func3', 3), + ('instruction', 'func3', 9), + ('instruction', 'func3', 10), + ('instruction', 'func3', 11), + ('line', 'func3', 4), + ('instruction', 'func3', 12), + ('line', 'func3', 5), + ('instruction', 'func3', 13), + ('instruction', 'func3', 14), + ('instruction', 'func3', 15), + ('line', 'func3', 6), + ('instruction', 'func3', 16), + ('instruction', 'func3', 17), + ('line', 'check_events', 11)]) + +class TestInstallIncrementallly(unittest.TestCase): + + def check_events(self, func, must_include, tool=TEST_TOOL, recorders=(ExceptionRecorder,)): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + event_list = [] + all_events = 0 + for recorder in recorders: + all_events |= recorder.event_type + sys.monitoring.set_events(tool, all_events) + for recorder in recorders: + sys.monitoring.register_callback(tool, recorder.event_type, recorder(event_list)) + func() + sys.monitoring.set_events(tool, 0) + for recorder in recorders: + sys.monitoring.register_callback(tool, recorder.event_type, None) + for line in must_include: + self.assertIn(line, event_list) + finally: + sys.monitoring.set_events(tool, 0) + for recorder in recorders: + sys.monitoring.register_callback(tool, recorder.event_type, None) + + @staticmethod + def func1(): + line1 = 1 + + MUST_INCLUDE_LI = [ + ('instruction', 'func1', 1), + ('line', 'func1', 1), + ('instruction', 'func1', 2), + ('instruction', 'func1', 3)] + + def test_line_then_instruction(self): + recorders = [ LineRecorder, InstructionRecorder ] + self.check_events(self.func1, + recorders = recorders, must_include = self.EXPECTED_LI) + + def test_instruction_then_line(self): + recorders = [ InstructionRecorder, LineRecorderLowNoise ] + self.check_events(self.func1, + recorders = recorders, must_include = self.EXPECTED_LI) + + @staticmethod + def func2(): + len(()) + + MUST_INCLUDE_CI = [ + ('instruction', 'func2', 1), + ('call', 'func2', sys.monitoring.MISSING), + ('call', 'len', ()), + ('instruction', 'func2', 6), + ('instruction', 'func2', 7)] + + + + def test_line_then_instruction(self): + recorders = [ CallRecorder, InstructionRecorder ] + self.check_events(self.func2, + recorders = recorders, must_include = self.MUST_INCLUDE_CI) + + def test_instruction_then_line(self): + recorders = [ InstructionRecorder, CallRecorder ] + self.check_events(self.func2, + recorders = recorders, must_include = self.MUST_INCLUDE_CI) + class TestLocalEvents(unittest.TestCase): def check_events(self, func, expected, tool=TEST_TOOL, recorders=(ExceptionRecorder,)): diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 66f485b2e65723..217a2d62a62af5 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -652,24 +652,26 @@ static void instrument(PyCodeObject *code, int i) { _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; - int opcode = _Py_OPCODE(*instr); - /* TO DO -- handle INSTRUMENTED_INSTRUCTION */ - CHECK(opcode != INSTRUMENTED_INSTRUCTION); + uint8_t *opcode_ptr = &instr->op.code; + int opcode =*opcode_ptr; + if (opcode == INSTRUMENTED_INSTRUCTION) { + opcode_ptr = &code->_co_monitoring->per_instruction_opcodes[i]; + opcode = *opcode_ptr; + } if (opcode == INSTRUMENTED_LINE) { _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[i]; - opcode = lines->original_opcode; - CHECK(opcode != 0); + opcode_ptr = &lines->original_opcode; + opcode = *opcode_ptr; CHECK(!is_instrumented(opcode)); CHECK(opcode == _PyOpcode_Deopt[opcode]); - lines->original_opcode = INSTRUMENTED_OPCODES[opcode]; - CHECK(lines->original_opcode != 0); } - else if (!is_instrumented(opcode)) { - opcode = _PyOpcode_Deopt[opcode]; - int instrumented = INSTRUMENTED_OPCODES[opcode]; + CHECK(opcode != 0); + if (!is_instrumented(opcode)) { + int deopt = _PyOpcode_Deopt[opcode]; + int instrumented = INSTRUMENTED_OPCODES[deopt]; assert(instrumented); - instr->op.code = instrumented; - if (_PyOpcode_Caches[opcode]) { + *opcode_ptr = instrumented; + if (_PyOpcode_Caches[deopt]) { instr[1].cache = adaptive_counter_warmup(); } } @@ -678,10 +680,12 @@ instrument(PyCodeObject *code, int i) static void instrument_line(PyCodeObject *code, int i) { - _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; - int opcode = _Py_OPCODE(*instr); - /* TO DO -- handle INSTRUMENTED_INSTRUCTION */ - CHECK(opcode != INSTRUMENTED_INSTRUCTION); + uint8_t *opcode_ptr = &_PyCode_CODE(code)[i].op.code; + int opcode =*opcode_ptr; + if (opcode == INSTRUMENTED_INSTRUCTION) { + opcode_ptr = &code->_co_monitoring->per_instruction_opcodes[i]; + opcode = *opcode_ptr; + } if (opcode == INSTRUMENTED_LINE) { return; } @@ -694,7 +698,7 @@ instrument_line(PyCodeObject *code, int i) ); lines->original_opcode = _PyOpcode_Deopt[opcode]; CHECK(lines->original_opcode > 0); - instr->op.code = INSTRUMENTED_LINE; + *opcode_ptr = INSTRUMENTED_LINE; } static void From 85d6923432491583bd70628c6340be8d11852acb Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 31 Mar 2023 15:31:48 +0100 Subject: [PATCH 088/116] Remove incorrect assertion. --- Python/instrumentation.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 217a2d62a62af5..9ca745d2f65c85 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -690,12 +690,6 @@ instrument_line(PyCodeObject *code, int i) return; } _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[i]; - CHECK(_PyOpcode_Deopt[opcode] == lines->original_opcode || - ( - is_instrumented(opcode) && - lines->original_opcode == DE_INSTRUMENT[opcode] - ) - ); lines->original_opcode = _PyOpcode_Deopt[opcode]; CHECK(lines->original_opcode > 0); *opcode_ptr = INSTRUMENTED_LINE; From ebcc42f134d1f6a44c2891cd48bf25d64d5c184f Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 31 Mar 2023 16:03:27 +0100 Subject: [PATCH 089/116] Turn off instrumentation debugging. --- Python/instrumentation.c | 98 ++++++++++++++-------------------------- 1 file changed, 33 insertions(+), 65 deletions(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 9ca745d2f65c85..2ff4f35218fb7e 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -11,7 +11,7 @@ #include "pycore_pystate.h" /* Uncomment this to dump debugging output when assertions fail */ -#define INSTRUMENT_DEBUG 1 +// #define INSTRUMENT_DEBUG 1 static PyObject DISABLE = { @@ -542,47 +542,29 @@ int _Py_GetBaseOpcode(PyCodeObject *code, int i) } static void -de_instrument(PyCodeObject *code, int offset, int event) +de_instrument(PyCodeObject *code, int i, int event) { assert(event != PY_MONITORING_EVENT_INSTRUCTION); assert(event != PY_MONITORING_EVENT_LINE); - _Py_CODEUNIT *instr = &_PyCode_CODE(code)[offset]; - int opcode = _Py_OPCODE(*instr); - if (!is_instrumented(opcode)) { - return; + + _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; + uint8_t *opcode_ptr = &instr->op.code; + int opcode = *opcode_ptr; + if (opcode == INSTRUMENTED_INSTRUCTION) { + opcode_ptr = &code->_co_monitoring->per_instruction_opcodes[i]; + opcode = *opcode_ptr; } - switch(opcode) { - case INSTRUMENTED_INSTRUCTION: - opcode = code->_co_monitoring->per_instruction_opcodes[offset]; - if (opcode != INSTRUMENTED_LINE) { - int deinstrumented = DE_INSTRUMENT[opcode]; - if (deinstrumented) { - code->_co_monitoring->per_instruction_opcodes[offset] = opcode = deinstrumented; - } - break; - } - /* Intentional fall through */ - case INSTRUMENTED_LINE: - { - _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[offset]; - int original_opcode = lines->original_opcode; - int deinstrumented = DE_INSTRUMENT[original_opcode]; - if (deinstrumented) { - lines->original_opcode = opcode = deinstrumented; - } - break; - } - default: - { - int deinstrumented = DE_INSTRUMENT[opcode]; - assert(deinstrumented); - instr->op.code = opcode = deinstrumented; - } + if (opcode == INSTRUMENTED_LINE) { + opcode_ptr = &code->_co_monitoring->lines[i].original_opcode; + opcode = *opcode_ptr; } - assert(!is_instrumented(opcode)); - int base_opcode = _PyOpcode_Deopt[opcode]; - assert(base_opcode != 0); - if (_PyOpcode_Caches[base_opcode]) { + int deinstrumented = DE_INSTRUMENT[opcode]; + if (deinstrumented == 0) { + return; + } + CHECK(_PyOpcode_Deopt[deinstrumented] == deinstrumented); + *opcode_ptr = deinstrumented; + if (_PyOpcode_Caches[deinstrumented]) { instr[1].cache = adaptive_counter_warmup(); } } @@ -591,32 +573,24 @@ static void de_instrument_line(PyCodeObject *code, int i) { _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; - int opcode = _Py_OPCODE(*instr); + uint8_t *opcode_ptr = &instr->op.code; + int opcode =*opcode_ptr; if (opcode == INSTRUMENTED_INSTRUCTION) { - opcode = code->_co_monitoring->per_instruction_opcodes[i]; + opcode_ptr = &code->_co_monitoring->per_instruction_opcodes[i]; + opcode = *opcode_ptr; } if (opcode != INSTRUMENTED_LINE) { return; } _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[i]; int original_opcode = lines->original_opcode; - if (is_instrumented(original_opcode)) { - /* Instrumented original */ - instr->op.code = original_opcode; - } - else { - CHECK(original_opcode != 0); - CHECK(original_opcode == _PyOpcode_Deopt[original_opcode]); - if (_PyOpcode_Caches[original_opcode]) { - instr[1].cache = adaptive_counter_warmup(); - } - } - if (instr->op.code == INSTRUMENTED_INSTRUCTION) { - code->_co_monitoring->per_instruction_opcodes[i] = original_opcode; - } - else { - instr->op.code = original_opcode; + CHECK(original_opcode != 0); + CHECK(original_opcode == _PyOpcode_Deopt[original_opcode]); + *opcode_ptr = instr->op.code = original_opcode; + if (_PyOpcode_Caches[original_opcode]) { + instr[1].cache = adaptive_counter_warmup(); } + assert(*opcode_ptr != INSTRUMENTED_LINE); assert(instr->op.code != INSTRUMENTED_LINE); } @@ -631,16 +605,10 @@ de_instrument_per_instruction(PyCodeObject *code, int i) } int original_opcode = code->_co_monitoring->per_instruction_opcodes[i]; CHECK(original_opcode != 0); - if (is_instrumented(original_opcode)) { - /* Instrumented original */ - instr->op.code = original_opcode; - } - else { - CHECK(original_opcode == _PyOpcode_Deopt[original_opcode]); - instr->op.code = original_opcode; - if (_PyOpcode_Caches[original_opcode]) { - instr[1].cache = adaptive_counter_warmup(); - } + CHECK(original_opcode == _PyOpcode_Deopt[original_opcode]); + instr->op.code = original_opcode; + if (_PyOpcode_Caches[original_opcode]) { + instr[1].cache = adaptive_counter_warmup(); } assert(instr->op.code != INSTRUMENTED_INSTRUCTION); /* Keep things clean for sanity check */ From 7cfbc7ee4ff581727549868f80d4e9fcc52f458e Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 31 Mar 2023 16:35:04 +0100 Subject: [PATCH 090/116] Fix a couple of errors. --- Lib/test/test_monitoring.py | 8 ++++---- Objects/codeobject.c | 1 + Python/instrumentation.c | 6 +++++- Python/pystate.c | 3 +++ 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 0b1e171ac3a474..8cc2fa8792f4fa 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -480,7 +480,7 @@ def test_two_with_disable(self): self.assertEqual(sys.monitoring._all_events(), {}) sys.monitoring.restart_events() -class LineMontoringTest(unittest.TestCase): +class LineMonitoringTest(unittest.TestCase): def test_lines_single(self): try: @@ -492,7 +492,7 @@ def test_lines_single(self): f1() sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) - start = LineMontoringTest.test_lines_single.__code__.co_firstlineno + start = LineMonitoringTest.test_lines_single.__code__.co_firstlineno self.assertEqual(events, [start+7, 14, start+8]) finally: sys.monitoring.set_events(TEST_TOOL, 0) @@ -510,7 +510,7 @@ def test_lines_loop(self): floop() sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) - start = LineMontoringTest.test_lines_loop.__code__.co_firstlineno + start = LineMonitoringTest.test_lines_loop.__code__.co_firstlineno self.assertEqual(events, [start+7, 21, 22, 22, 21, start+8]) finally: sys.monitoring.set_events(TEST_TOOL, 0) @@ -532,7 +532,7 @@ def test_lines_two(self): sys.monitoring.set_events(TEST_TOOL, 0); sys.monitoring.set_events(TEST_TOOL2, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) sys.monitoring.register_callback(TEST_TOOL2, E.LINE, None) - start = LineMontoringTest.test_lines_two.__code__.co_firstlineno + start = LineMonitoringTest.test_lines_two.__code__.co_firstlineno expected = [start+10, 14, start+11] self.assertEqual(events, expected) self.assertEqual(events2, expected) diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 34e1edc6c2678b..693ab710c22200 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -2296,6 +2296,7 @@ _PyStaticCode_Fini(PyCodeObject *co) co->co_weakreflist = NULL; } free_monitoring_data(co->_co_monitoring); + co->_co_monitoring = NULL; } int diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 2ff4f35218fb7e..0255fb6b9f0875 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1705,6 +1705,7 @@ monitoring_register_callback_impl(PyObject *module, int tool_id, int event, } if (_Py_popcount32(event) != 1) { PyErr_SetString(PyExc_ValueError, "The callback can only be set for one event at a time"); + return NULL; } int event_id = _Py_bit_length(event)-1; if (event_id < 0 || event_id >= PY_MONITORING_EVENTS) { @@ -1913,7 +1914,10 @@ monitoring__all_events_impl(PyObject *module) if (tools == 0) { continue; } - int err = PyDict_SetItemString(res, event_names[e], PyLong_FromLong(tools)); + PyObject *tools_obj = PyLong_FromLong(tools); + assert(tools_obj != NULL); + int err = PyDict_SetItemString(res, event_names[e], tools_obj); + Py_DECREF(tools_obj); if (err < 0) { Py_DECREF(res); return NULL; diff --git a/Python/pystate.c b/Python/pystate.c index 36adfacb3f82c5..822627d8bb3b34 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -815,6 +815,9 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) Py_CLEAR(interp->monitoring_callables[t][e]); } } + for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { + Py_CLEAR(interp->monitoring_tool_names[t]); + } PyConfig_Clear(&interp->config); Py_CLEAR(interp->codec_search_path); From d366364642405ccc030c0e20780b81f566eea9e6 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 31 Mar 2023 17:24:15 +0100 Subject: [PATCH 091/116] Add minimal news entry. --- .../2023-03-31-17-24-03.gh-issue-103082.isRUcV.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-31-17-24-03.gh-issue-103082.isRUcV.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-31-17-24-03.gh-issue-103082.isRUcV.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-31-17-24-03.gh-issue-103082.isRUcV.rst new file mode 100644 index 00000000000000..4fd6fa29bc3648 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-03-31-17-24-03.gh-issue-103082.isRUcV.rst @@ -0,0 +1 @@ +Adds low overhead monitoring support. See PEP 669 for full details. From fdb486026511eda234537e9d3d21a25f6007547b Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 31 Mar 2023 17:33:22 +0100 Subject: [PATCH 092/116] Pacify the global object checker. --- Python/instrumentation.c | 14 +++++++------- Tools/c-analyzer/cpython/globals-to-fix.tsv | 4 ++++ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 0255fb6b9f0875..07546417c15aca 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1391,18 +1391,18 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) return 0; } -static const bool super_instructions[256] = { - [LOAD_FAST__LOAD_FAST] = true, - [LOAD_FAST__LOAD_CONST] = true, - [STORE_FAST__LOAD_FAST] = true, - [STORE_FAST__STORE_FAST] = true, - [LOAD_CONST__LOAD_FAST] = true, +static const uint8_t super_instructions[256] = { + [LOAD_FAST__LOAD_FAST] = 1, + [LOAD_FAST__LOAD_CONST] = 1, + [STORE_FAST__LOAD_FAST] = 1, + [STORE_FAST__STORE_FAST] = 1, + [LOAD_CONST__LOAD_FAST] = 1, }; /* Should use instruction metadata for this */ static bool is_super_instruction(int opcode) { - return super_instructions[opcode]; + return super_instructions[opcode] != 0; } int diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index e0e45265209fff..f2f7b9b7020bc7 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -135,6 +135,8 @@ Objects/stringlib/unicode_format.h - PyFieldNameIter_Type - Objects/unicodeobject.c - EncodingMapType - #Objects/unicodeobject.c - PyFieldNameIter_Type - #Objects/unicodeobject.c - PyFormatterIter_Type - +Python/legacy_tracing.c - _PyLegacyEventHandler_Type - + ##----------------------- ## static builtin structseq @@ -297,6 +299,8 @@ Objects/object.c - _Py_NotImplementedStruct - Objects/setobject.c - _dummy_struct - Objects/setobject.c - _PySet_Dummy - Objects/sliceobject.c - _Py_EllipsisObject - +Python/instrumentation.c - DISABLE - +Python/instrumentation.c - _PyInstrumentation_MISSING - ################################## From aee722f0a1a469057e76f3ac7c9044959530c322 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 3 Apr 2023 10:15:04 +0100 Subject: [PATCH 093/116] Minor fixups. --- Python/instrumentation.c | 3 ++- Python/pystate.c | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 07546417c15aca..305cd178c81bc5 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1626,7 +1626,8 @@ monitoring_use_tool_impl(PyObject *module, int tool_id, PyObject *name) return NULL; } if (!PyUnicode_Check(name)) { - PyErr_SetString(PyExc_ValueError, "tool name must be a str"); + PyErr_SetString(PyExc_ValueError, "tool name must be a str"); + return NULL; } PyInterpreterState *interp = _PyInterpreterState_Get(); if (interp->monitoring_tool_names[tool_id] != NULL) { diff --git a/Python/pystate.c b/Python/pystate.c index 822627d8bb3b34..5b2dc5a97a79a8 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -670,6 +670,8 @@ init_interpreter(PyInterpreterState *interp, } } + interp->sys_profile_initialized = false; + interp->sys_profile_initialized = false; if (interp != &runtime->_main_interpreter) { /* Fix the self-referential, statically initialized fields. */ interp->dtoa = (struct _dtoa_state)_dtoa_state_INIT(interp); @@ -815,6 +817,8 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) Py_CLEAR(interp->monitoring_callables[t][e]); } } + interp->sys_profile_initialized = false; + interp->sys_profile_initialized = false; for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { Py_CLEAR(interp->monitoring_tool_names[t]); } From bba53b8198491d161ce061da6fe953484b2286db Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 3 Apr 2023 10:39:07 +0100 Subject: [PATCH 094/116] Fix refleak when error in RETURN instrumentation. --- Include/internal/pycore_instruments.h | 2 -- Python/bytecodes.c | 6 +++--- Python/generated_cases.c.h | 4 ++-- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index e3327c29389dd2..e4c262c9e213a9 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -51,8 +51,6 @@ typedef uint32_t _PyMonitoringEventSet; #define PY_INSTRUMENT_SYS_TRACE 7 -/* API functions */ - PyObject *_PyMonitoring_RegisterCallback(int tool_id, int event_id, PyObject *obj); void _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index be98fa22b86222..4bb51bd51fb912 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -597,7 +597,7 @@ dummy_func( int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, next_instr-1, retval); - ERROR_IF(err, error); + if (err) goto error; STACK_SHRINK(1); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -628,11 +628,11 @@ dummy_func( inst(INSTRUMENTED_RETURN_CONST, (--)) { PyObject *retval = GETITEM(frame->f_code->co_consts, oparg); - Py_INCREF(retval); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, next_instr-1, retval); - ERROR_IF(err, error); + if (err) goto error; + Py_INCREF(retval); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); _Py_LeaveRecursiveCallPy(tstate); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 89d865b50e1617..8f8b6ae50f3c0b 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -931,7 +931,7 @@ int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, next_instr-1, retval); - if (err) goto pop_1_error; + if (err) goto error; STACK_SHRINK(1); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -966,11 +966,11 @@ TARGET(INSTRUMENTED_RETURN_CONST) { #line 630 "Python/bytecodes.c" PyObject *retval = GETITEM(frame->f_code->co_consts, oparg); - Py_INCREF(retval); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, next_instr-1, retval); if (err) goto error; + Py_INCREF(retval); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); _Py_LeaveRecursiveCallPy(tstate); From 97ec1c5498b54e4aaaf09de5f407af443271f14c Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 3 Apr 2023 12:20:55 +0100 Subject: [PATCH 095/116] Fix another refleak in error handling in instrumented instruction. --- Python/bytecodes.c | 2 +- Python/generated_cases.c.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index fd528db85f723b..8a5649503137cd 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -877,7 +877,7 @@ dummy_func( int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_YIELD, frame, next_instr-1, retval); - ERROR_IF(err, error); + if (err) goto error; tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 81eb702554e4c6..054efa24d21082 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1206,7 +1206,7 @@ int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_YIELD, frame, next_instr-1, retval); - if (err) goto pop_1_error; + if (err) goto error; tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); From edc6709d6f687f418876a7fe1e2b15acc8218fdf Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 3 Apr 2023 14:18:18 +0100 Subject: [PATCH 096/116] Remove debug code ready for final review. --- Include/internal/pycore_opcode.h | 1 - Lib/importlib/_bootstrap_external.py | 5 ++--- Lib/opcode.py | 3 --- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index 65588bc01fd739..a79afda558dc92 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -45,7 +45,6 @@ const uint32_t _PyOpcode_Jump[9] = { }; const uint8_t _PyOpcode_Caches[256] = { - [RESERVED] = 1000000, [BINARY_SUBSCR] = 1, [STORE_SUBSCR] = 1, [UNPACK_SEQUENCE] = 1, diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 8cf7d374d1a420..8a52946de690b4 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -438,8 +438,7 @@ def _write_atomic(path, data, mode=0o666): # Python 3.12a7 3522 (Removed JUMP_IF_FALSE_OR_POP/JUMP_IF_TRUE_OR_POP) # Python 3.12a7 3523 (Convert COMPARE_AND_BRANCH back to COMPARE_OP) # Python 3.12a7 3524 (Shrink the BINARY_SUBSCR caches) - -# Python 3.12a? 3526 (Add instrumentation support) +# Python 3.12a7 3525 (Add instrumentation support) # Python 3.13 will start with 3550 @@ -456,7 +455,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3526).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3525).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c diff --git a/Lib/opcode.py b/Lib/opcode.py index d69bc7091450f7..5aa32d3cb77613 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -443,9 +443,6 @@ def pseudo_op(name, op, real_ops): "SEND": { "counter": 1, }, - "RESERVED": { - "force_failure": 1_000_000 - }, } _inline_cache_entries = [ From e40a68f367c73685247fe066a3d1b76c602c8b00 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 3 Apr 2023 15:27:58 +0100 Subject: [PATCH 097/116] Fix another (and hopefully final) refleak when handling an error in an instrumented bytecode. --- Python/bytecodes.c | 2 +- Python/generated_cases.c.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 8a5649503137cd..39f5add0019989 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3027,7 +3027,7 @@ dummy_func( int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, next_instr-1, func, arg); - ERROR_IF(err, error); + if (err) goto error; result = PyObject_Call(func, callargs, kwargs); if (result == NULL) { _Py_call_instrumentation_exc2( diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 054efa24d21082..cc108f01a2062d 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -4193,7 +4193,7 @@ int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, next_instr-1, func, arg); - if (err) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } + if (err) goto error; result = PyObject_Call(func, callargs, kwargs); if (result == NULL) { _Py_call_instrumentation_exc2( From d9f8192a6549ec60a25e2858257925f23614ffcc Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 4 Apr 2023 12:15:46 +0100 Subject: [PATCH 098/116] Comment and rename #defines to better reflect names in PEP. --- Include/internal/pycore_frame.h | 2 +- Include/internal/pycore_instruments.h | 14 +++++----- Python/bytecodes.c | 4 +-- Python/ceval.c | 4 +-- Python/generated_cases.c.h | 4 +-- Python/instrumentation.c | 2 +- Python/legacy_tracing.c | 40 +++++++++++++-------------- 7 files changed, 35 insertions(+), 35 deletions(-) diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index 93b37e6fee64b3..acdfe17ee7cd99 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -143,7 +143,7 @@ _PyFrame_GetLocalsArray(_PyInterpreterFrame *frame) values are not visible to the cycle GC. We choose -1 rather than 0 to assist debugging. */ static inline PyObject** -_PyFrame_FetchStackPointer(_PyInterpreterFrame *frame) +_PyFrame_GetStackPointer(_PyInterpreterFrame *frame) { PyObject **sp = frame->localsplus+frame->stacktop; frame->stacktop = -1; diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index e4c262c9e213a9..9615efcaf0a35b 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -14,7 +14,8 @@ extern "C" { #define PY_MONITORING_TOOL_IDS 8 -/* Require bytecode instrumentation */ +/* Local events. + * These require bytecode instrumentation */ #define PY_MONITORING_EVENT_PY_START 0 #define PY_MONITORING_EVENT_PY_RESUME 1 @@ -29,7 +30,7 @@ extern "C" { #define PY_MONITORING_INSTRUMENTED_EVENTS 10 -/* Exceptional events */ +/* Other events, mainly exceptions */ #define PY_MONITORING_EVENT_RAISE 10 #define PY_MONITORING_EVENT_EXCEPTION_HANDLED 11 @@ -37,7 +38,7 @@ extern "C" { #define PY_MONITORING_EVENT_PY_THROW 13 -/* Grouped events */ +/* Ancilliary events */ #define PY_MONITORING_EVENT_C_RETURN 14 #define PY_MONITORING_EVENT_C_RAISE 15 @@ -45,10 +46,9 @@ extern "C" { typedef uint32_t _PyMonitoringEventSet; -/* Reserved IDs */ - -#define PY_INSTRUMENT_SYS_PROFILE 6 -#define PY_INSTRUMENT_SYS_TRACE 7 +/* Tool IDs */ +#define PY_MONITORING_SYS_PROFILE_ID 6 +#define PY_MONITORING_SYS_TRACE_ID 7 PyObject *_PyMonitoring_RegisterCallback(int tool_id, int event_id, PyObject *obj); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 39f5add0019989..322c8c82d0a89c 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -145,7 +145,7 @@ dummy_func( _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation( tstate, oparg > 0, frame, next_instr-1); - stack_pointer = _PyFrame_FetchStackPointer(frame); + stack_pointer = _PyFrame_GetStackPointer(frame); ERROR_IF(err, error); if (frame->prev_instr != next_instr-1) { /* Instrumentation has jumped */ @@ -3187,7 +3187,7 @@ dummy_func( _PyFrame_SetStackPointer(frame, stack_pointer); int original_opcode = _Py_call_instrumentation_line( tstate, frame, here); - stack_pointer = _PyFrame_FetchStackPointer(frame); + stack_pointer = _PyFrame_GetStackPointer(frame); if (original_opcode < 0) { next_instr = here+1; goto error; diff --git a/Python/ceval.c b/Python/ceval.c index c2b0dd5ea95776..19315c87c62f1b 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -617,7 +617,7 @@ static inline void _Py_LeaveRecursiveCallPy(PyThreadState *tstate) { do { \ _PyFrame_SetStackPointer(frame, stack_pointer); \ int err = _Py_call_instrumentation_jump(tstate, event, frame, src, dest); \ - stack_pointer = _PyFrame_FetchStackPointer(frame); \ + stack_pointer = _PyFrame_GetStackPointer(frame); \ if (err) { \ next_instr = (dest)+1; \ goto error; \ @@ -723,7 +723,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int assert(_PyInterpreterFrame_LASTI(frame) >= -1); \ /* Jump back to the last instruction executed... */ \ next_instr = frame->prev_instr + 1; \ - stack_pointer = _PyFrame_FetchStackPointer(frame); + stack_pointer = _PyFrame_GetStackPointer(frame); start_frame: if (_Py_EnterRecursivePy(tstate)) { diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index cc108f01a2062d..e37d122986a27b 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -21,7 +21,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation( tstate, oparg > 0, frame, next_instr-1); - stack_pointer = _PyFrame_FetchStackPointer(frame); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err) goto error; if (frame->prev_instr != next_instr-1) { /* Instrumentation has jumped */ @@ -4421,7 +4421,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int original_opcode = _Py_call_instrumentation_line( tstate, frame, here); - stack_pointer = _PyFrame_FetchStackPointer(frame); + stack_pointer = _PyFrame_GetStackPointer(frame); if (original_opcode < 0) { next_instr = here+1; goto error; diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 305cd178c81bc5..ef729362b9b1ee 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1600,7 +1600,7 @@ module monitoring static int check_valid_tool(int tool_id) { - if (tool_id < 0 || tool_id >= PY_INSTRUMENT_SYS_PROFILE) { + if (tool_id < 0 || tool_id >= PY_MONITORING_SYS_PROFILE_ID) { PyErr_Format(PyExc_ValueError, "invalid tool %d (must be between 0 and 5)", tool_id); return -1; } diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index a0a47fa6e6abea..677910e9de8c8e 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -388,32 +388,32 @@ _PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) /* Setup PEP 669 monitoring callbacks and events. */ if (!tstate->interp->sys_profile_initialized) { tstate->interp->sys_profile_initialized = true; - if (set_callbacks(PY_INSTRUMENT_SYS_PROFILE, + if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, (vectorcallfunc)sys_profile_func2, PyTrace_CALL, PY_MONITORING_EVENT_PY_START, PY_MONITORING_EVENT_PY_RESUME)) { return -1; } - if (set_callbacks(PY_INSTRUMENT_SYS_PROFILE, + if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, (vectorcallfunc)sys_profile_func3, PyTrace_RETURN, PY_MONITORING_EVENT_PY_RETURN, PY_MONITORING_EVENT_PY_YIELD)) { return -1; } - if (set_callbacks(PY_INSTRUMENT_SYS_PROFILE, + if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, (vectorcallfunc)sys_profile_func2, PyTrace_RETURN, PY_MONITORING_EVENT_PY_UNWIND, -1)) { return -1; } - if (set_callbacks(PY_INSTRUMENT_SYS_PROFILE, + if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, (vectorcallfunc)sys_profile_call_or_return, PyTrace_C_CALL, PY_MONITORING_EVENT_CALL, -1)) { return -1; } - if (set_callbacks(PY_INSTRUMENT_SYS_PROFILE, + if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, (vectorcallfunc)sys_profile_call_or_return, PyTrace_C_RETURN, PY_MONITORING_EVENT_C_RETURN, -1)) { return -1; } - if (set_callbacks(PY_INSTRUMENT_SYS_PROFILE, + if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, (vectorcallfunc)sys_profile_call_or_return, PyTrace_C_EXCEPTION, PY_MONITORING_EVENT_C_RAISE, -1)) { return -1; @@ -433,10 +433,10 @@ _PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) (1 << PY_MONITORING_EVENT_PY_START) | (1 << PY_MONITORING_EVENT_PY_RESUME) | (1 << PY_MONITORING_EVENT_PY_RETURN) | (1 << PY_MONITORING_EVENT_PY_YIELD) | (1 << PY_MONITORING_EVENT_CALL) | (1 << PY_MONITORING_EVENT_PY_UNWIND); - _PyMonitoring_SetEvents(PY_INSTRUMENT_SYS_PROFILE, events); + _PyMonitoring_SetEvents(PY_MONITORING_SYS_PROFILE_ID, events); } else { - _PyMonitoring_SetEvents(PY_INSTRUMENT_SYS_PROFILE, 0); + _PyMonitoring_SetEvents(PY_MONITORING_SYS_PROFILE_ID, 0); } return 0; } @@ -459,52 +459,52 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) /* Setup PEP 669 monitoring callbacks and events. */ if (!tstate->interp->sys_trace_initialized) { tstate->interp->sys_trace_initialized = true; - if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, (vectorcallfunc)sys_trace_func2, PyTrace_CALL, PY_MONITORING_EVENT_PY_START, PY_MONITORING_EVENT_PY_RESUME)) { return -1; } - if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, (vectorcallfunc)sys_trace_func2, PyTrace_CALL, PY_MONITORING_EVENT_PY_THROW, -1)) { return -1; } - if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, (vectorcallfunc)sys_trace_return, PyTrace_RETURN, PY_MONITORING_EVENT_PY_RETURN, -1)) { return -1; } - if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, (vectorcallfunc)sys_trace_yield, PyTrace_RETURN, PY_MONITORING_EVENT_PY_YIELD, -1)) { return -1; } - if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, (vectorcallfunc)sys_trace_exception_func, PyTrace_EXCEPTION, PY_MONITORING_EVENT_RAISE, PY_MONITORING_EVENT_STOP_ITERATION)) { return -1; } - if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, (vectorcallfunc)sys_trace_line_func, PyTrace_LINE, PY_MONITORING_EVENT_LINE, -1)) { return -1; } - if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, (vectorcallfunc)sys_trace_func2, PyTrace_RETURN, PY_MONITORING_EVENT_PY_UNWIND, -1)) { return -1; } - if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, (vectorcallfunc)sys_trace_jump_func, PyTrace_LINE, PY_MONITORING_EVENT_JUMP, PY_MONITORING_EVENT_BRANCH)) { return -1; } - if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, (vectorcallfunc)sys_trace_instruction_func, PyTrace_OPCODE, PY_MONITORING_EVENT_INSTRUCTION, -1)) { return -1; } - if (set_callbacks(PY_INSTRUMENT_SYS_TRACE, + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, (vectorcallfunc)sys_trace_exception_handled, PyTrace_LINE, PY_MONITORING_EVENT_EXCEPTION_HANDLED, -1)) { return -1; @@ -531,10 +531,10 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) if (tstate->interp->f_opcode_trace_set) { events |= (1 << PY_MONITORING_EVENT_INSTRUCTION); } - _PyMonitoring_SetEvents(PY_INSTRUMENT_SYS_TRACE, events); + _PyMonitoring_SetEvents(PY_MONITORING_SYS_TRACE_ID, events); } else { - _PyMonitoring_SetEvents(PY_INSTRUMENT_SYS_TRACE, 0); + _PyMonitoring_SetEvents(PY_MONITORING_SYS_TRACE_ID, 0); } return 0; From 2d9a3803b4dbff955e68a037169d98c79c910478 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 4 Apr 2023 12:22:01 +0100 Subject: [PATCH 099/116] Add various IDs as defined in the PEP. --- Include/internal/pycore_instruments.h | 8 ++++++++ Python/instrumentation.c | 16 ++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index 9615efcaf0a35b..fa5eedcefb8f4f 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -47,6 +47,14 @@ extern "C" { typedef uint32_t _PyMonitoringEventSet; /* Tool IDs */ + +/* These are defined in PEP 669 for convenience to avoid clashes */ +#define PY_MONITORING_DEBUGGER_ID 0 +#define PY_MONITORING_COVERAGE_ID 1 +#define PY_MONITORING_PROFILER_ID 2 +#define PY_MONITORING_OPTIMIZER_ID 5 + +/* Internal IDs used to suuport sys.setprofile() and sys.settrace() */ #define PY_MONITORING_SYS_PROFILE_ID 6 #define PY_MONITORING_SYS_TRACE_ID 7 diff --git a/Python/instrumentation.c b/Python/instrumentation.c index ef729362b9b1ee..016c5c47e5a02e 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1979,6 +1979,22 @@ PyObject *_Py_CreateMonitoringObject(void) goto error; } } + PyObject *val = PyLong_FromLong(PY_MONITORING_DEBUGGER_ID); + err = PyObject_SetAttrString(mod, "DEBUGGER_ID", val); + Py_DECREF(val); + if (err) goto error; + val = PyLong_FromLong(PY_MONITORING_COVERAGE_ID); + err = PyObject_SetAttrString(mod, "COVERAGE_ID", val); + Py_DECREF(val); + if (err) goto error; + val = PyLong_FromLong(PY_MONITORING_PROFILER_ID); + err = PyObject_SetAttrString(mod, "PROFILER_ID", val); + Py_DECREF(val); + if (err) goto error; + val = PyLong_FromLong(PY_MONITORING_OPTIMIZER_ID); + err = PyObject_SetAttrString(mod, "OPTIMIZER_ID", val); + Py_DECREF(val); + if (err) goto error; return mod; error: Py_DECREF(mod); From 29f41e7a3263f284474ae31cb38254d55540290a Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 4 Apr 2023 12:25:21 +0100 Subject: [PATCH 100/116] Remove _PyThreadState_UpdateTracingState. --- Include/internal/pycore_pystate.h | 6 ------ Python/ceval.c | 1 - 2 files changed, 7 deletions(-) diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index f9b5077f51fe1f..6e5f2289cb6b95 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -133,12 +133,6 @@ extern void _PyThreadState_BindDetached(PyThreadState *); extern void _PyThreadState_UnbindDetached(PyThreadState *); -static inline void -_PyThreadState_UpdateTracingState(PyThreadState *tstate) -{ -} - - /* Other */ PyAPI_FUNC(PyThreadState *) _PyThreadState_Swap( diff --git a/Python/ceval.c b/Python/ceval.c index 19315c87c62f1b..72f1b47e3a303d 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1967,7 +1967,6 @@ PyThreadState_LeaveTracing(PyThreadState *tstate) { assert(tstate->tracing > 0); tstate->tracing--; - _PyThreadState_UpdateTracingState(tstate); } From 38b7f4350a7695cfc2e5e1a7b6b4c327fe891f0c Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 4 Apr 2023 12:48:24 +0100 Subject: [PATCH 101/116] Formatting fixes, and check error codes. --- Include/internal/pycore_instruments.h | 2 +- Objects/frameobject.c | 4 +-- Python/ceval.c | 31 +++++++++---------- Python/instrumentation.c | 44 +++++++++++++-------------- 4 files changed, 38 insertions(+), 43 deletions(-) diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index fa5eedcefb8f4f..e94d8755546efd 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -61,7 +61,7 @@ typedef uint32_t _PyMonitoringEventSet; PyObject *_PyMonitoring_RegisterCallback(int tool_id, int event_id, PyObject *obj); -void _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events); +int _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events); extern int _Py_call_instrumentation(PyThreadState *tstate, int event, diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 86cf579710956d..2facffaca66c0b 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -668,13 +668,13 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore "f_lineno can only be set in a trace function"); return -1; } - switch(what_event) { + switch (what_event) { case PY_MONITORING_EVENT_PY_RESUME: case PY_MONITORING_EVENT_JUMP: case PY_MONITORING_EVENT_BRANCH: case PY_MONITORING_EVENT_LINE: case PY_MONITORING_EVENT_PY_YIELD: - /* OK */ + /* Setting f_lineno is allowed for the above events */ break; case PY_MONITORING_EVENT_PY_START: PyErr_Format(PyExc_ValueError, diff --git a/Python/ceval.c b/Python/ceval.c index 72f1b47e3a303d..8d2fc2c4d0e9e8 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1865,15 +1865,13 @@ unpack_iterable(PyThreadState *tstate, PyObject *v, } static int -do_monitor_exc( - PyThreadState *tstate, _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr, int event -) { +do_monitor_exc(PyThreadState *tstate, _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr, int event) +{ assert(event < PY_MONITORING_UNGROUPED_EVENTS); PyObject *exc = PyErr_GetRaisedException(); assert(exc != NULL); - int err; - err = _Py_call_instrumentation_arg(tstate, event, frame, instr, exc); + int err = _Py_call_instrumentation_arg(tstate, event, frame, instr, exc); if (err == 0) { PyErr_SetRaisedException(exc); } @@ -1901,8 +1899,8 @@ no_tools_for_event(PyThreadState *tstate, _PyInterpreterFrame *frame, int event) } static void -monitor_raise( - PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) +monitor_raise(PyThreadState *tstate, _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr) { if (no_tools_for_event(tstate, frame, PY_MONITORING_EVENT_RAISE)) { return; @@ -1911,8 +1909,8 @@ monitor_raise( } static int -monitor_stop_iteration( - PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) +monitor_stop_iteration(PyThreadState *tstate, _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr) { if (no_tools_for_event(tstate, frame, PY_MONITORING_EVENT_STOP_ITERATION)) { return 0; @@ -1922,10 +1920,9 @@ monitor_stop_iteration( static void monitor_unwind(PyThreadState *tstate, - _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr) + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr) { - if (no_tools_for_event(tstate, frame, PY_MONITORING_EVENT_PY_UNWIND)) { return; } @@ -1935,8 +1932,8 @@ monitor_unwind(PyThreadState *tstate, static void monitor_handled(PyThreadState *tstate, - _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr, PyObject *exc) + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr, PyObject *exc) { if (no_tools_for_event(tstate, frame, PY_MONITORING_EVENT_EXCEPTION_HANDLED)) { return; @@ -1946,8 +1943,8 @@ monitor_handled(PyThreadState *tstate, static void monitor_throw(PyThreadState *tstate, - _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr) + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr) { if (no_tools_for_event(tstate, frame, PY_MONITORING_EVENT_PY_THROW)) { return; diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 016c5c47e5a02e..6588eff344cb8f 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -265,17 +265,6 @@ instruction_length(PyCodeObject *code, int offset) return 1 + _PyOpcode_Caches[opcode]; } -typedef struct _Instruction { - uint8_t extended_args; - uint8_t deinstrumented_opcode; - uint8_t actual_opcode; - uint8_t length; - /* Whether a prefix instrument like INSTRUMENTED_INSTRUCTION is attached */ - bool is_prefix_instrumented; - /* Whether a specialized instruction like INSTRUMENTED_RETURN_VALUE is used */ - bool is_specialized_instrumented; -} Instruction; - #ifdef INSTRUMENT_DEBUG static void @@ -628,7 +617,7 @@ instrument(PyCodeObject *code, int i) } if (opcode == INSTRUMENTED_LINE) { _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[i]; - opcode_ptr = &lines->original_opcode; + opcode_ptr = &lines->original_opcode; opcode = *opcode_ptr; CHECK(!is_instrumented(opcode)); CHECK(opcode == _PyOpcode_Deopt[opcode]); @@ -1099,7 +1088,7 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, PyObject *args[3] = { NULL, (PyObject *)code, line_obj }; while (tools) { int tool = most_significant_bit(tools); - assert(tool < 8); + assert(tool >= 0 && tool < 8); assert(tools & (1 << tool)); tools &= ~(1 << tool); int res = call_one_instrument(interp, tstate, &args[1], @@ -1152,7 +1141,7 @@ _Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame* PyObject *args[3] = { NULL, (PyObject *)code, offset_obj }; while (tools) { int tool = most_significant_bit(tools); - assert(tool < 8); + assert(tool >= 0 && tool < 8); assert(tools & (1 << tool)); tools &= ~(1 << tool); int res = call_one_instrument(interp, tstate, &args[1], @@ -1265,7 +1254,7 @@ initialize_lines(PyCodeObject *code) case RESUME: /* END_FOR cannot start a line, as it is skipped by FOR_ITER * END_SEND cannot start a line, as it is skipped by SEND - * RESUME must not be instrumented with INSTRUMENT_LINE */ + * RESUME must not be instrumented with INSTRUMENT_LINE */ line_data[i].original_opcode = 0; break; default: @@ -1519,7 +1508,7 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) (C_RETURN_EVENTS | (1 << PY_MONITORING_EVENT_CALL)) -static void +static int instrument_all_executing_code_objects(PyInterpreterState *interp) { _PyRuntimeState *runtime = &_PyRuntime; HEAD_LOCK(runtime); @@ -1529,7 +1518,9 @@ instrument_all_executing_code_objects(PyInterpreterState *interp) { _PyInterpreterFrame *frame = ts->cframe->current_frame; while (frame) { if (frame->owner != FRAME_OWNED_BY_CSTACK) { - _Py_Instrument(frame->f_code, interp); + if (_Py_Instrument(frame->f_code, interp)) { + return -1; + } } frame = frame->previous; } @@ -1537,6 +1528,7 @@ instrument_all_executing_code_objects(PyInterpreterState *interp) { ts = PyThreadState_Next(ts); HEAD_UNLOCK(runtime); } + return 0; } static void @@ -1551,7 +1543,7 @@ set_events(_Py_Monitors *m, int tool_id, _PyMonitoringEventSet events) } } -void +int _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) { assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); @@ -1559,11 +1551,11 @@ _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) assert(events < (1 << PY_MONITORING_UNGROUPED_EVENTS)); uint32_t existing_events = get_events(&interp->monitors, tool_id); if (existing_events == events) { - return; + return 0; } set_events(&interp->monitors, tool_id, events); interp->monitoring_version++; - instrument_all_executing_code_objects(interp); + return instrument_all_executing_code_objects(interp); } int @@ -1585,7 +1577,9 @@ _PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEvent /* Force instrumentation update */ code->_co_instrumentation_version = UINT64_MAX; } - _Py_Instrument(code, interp); + if (_Py_Instrument(code, interp)) { + return -1; + } return 0; } @@ -1768,7 +1762,9 @@ monitoring_set_events_impl(PyObject *module, int tool_id, int event_set) return NULL; } event_set &= ~C_RETURN_EVENTS; - _PyMonitoring_SetEvents(tool_id, event_set); + if (_PyMonitoring_SetEvents(tool_id, event_set)) { + return NULL; + } Py_RETURN_NONE; } @@ -1862,7 +1858,9 @@ monitoring_restart_events_impl(PyObject *module) PyInterpreterState *interp = _PyInterpreterState_Get(); interp->last_restart_version = interp->monitoring_version + 1; interp->monitoring_version = interp->last_restart_version + 1; - instrument_all_executing_code_objects(interp); + if (instrument_all_executing_code_objects(interp)) { + return NULL; + } Py_RETURN_NONE; } From 9a40dad178c067fa6a977943823107b4d9c0a0cc Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 4 Apr 2023 13:03:24 +0100 Subject: [PATCH 102/116] Make sure tool API function names match PEP. --- Lib/test/test_monitoring.py | 8 ++++---- Python/clinic/instrumentation.c.h | 32 +++++++++++++++---------------- Python/instrumentation.c | 20 ++++++++----------- 3 files changed, 28 insertions(+), 32 deletions(-) diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 8cc2fa8792f4fa..162466cf78c091 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -38,8 +38,8 @@ class MonitoringBaseTest(unittest.TestCase): def test_has_objects(self): m = sys.monitoring m.events - m.use_tool - m.free_tool + m.use_tool_id + m.free_tool_id m.get_tool m.get_events m.set_events @@ -51,9 +51,9 @@ def test_has_objects(self): m.MISSING def test_tool(self): - sys.monitoring.use_tool(TEST_TOOL, "MonitoringTest.Tool") + sys.monitoring.use_tool_id(TEST_TOOL, "MonitoringTest.Tool") self.assertEqual(sys.monitoring.get_tool(TEST_TOOL), "MonitoringTest.Tool") - sys.monitoring.free_tool(TEST_TOOL) + sys.monitoring.free_tool_id(TEST_TOOL) self.assertEqual(sys.monitoring.get_tool(TEST_TOOL), None) sys.monitoring.set_events(TEST_TOOL, 15) self.assertEqual(sys.monitoring.get_events(TEST_TOOL), 15) diff --git a/Python/clinic/instrumentation.c.h b/Python/clinic/instrumentation.c.h index 57021191e29df6..deaef2e21902da 100644 --- a/Python/clinic/instrumentation.c.h +++ b/Python/clinic/instrumentation.c.h @@ -8,25 +8,25 @@ preserve #endif -PyDoc_STRVAR(monitoring_use_tool__doc__, -"use_tool($module, tool_id, name, /)\n" +PyDoc_STRVAR(monitoring_use_tool_id__doc__, +"use_tool_id($module, tool_id, name, /)\n" "--\n" "\n"); -#define MONITORING_USE_TOOL_METHODDEF \ - {"use_tool", _PyCFunction_CAST(monitoring_use_tool), METH_FASTCALL, monitoring_use_tool__doc__}, +#define MONITORING_USE_TOOL_ID_METHODDEF \ + {"use_tool_id", _PyCFunction_CAST(monitoring_use_tool_id), METH_FASTCALL, monitoring_use_tool_id__doc__}, static PyObject * -monitoring_use_tool_impl(PyObject *module, int tool_id, PyObject *name); +monitoring_use_tool_id_impl(PyObject *module, int tool_id, PyObject *name); static PyObject * -monitoring_use_tool(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +monitoring_use_tool_id(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; int tool_id; PyObject *name; - if (!_PyArg_CheckPositional("use_tool", nargs, 2, 2)) { + if (!_PyArg_CheckPositional("use_tool_id", nargs, 2, 2)) { goto exit; } tool_id = _PyLong_AsInt(args[0]); @@ -34,25 +34,25 @@ monitoring_use_tool(PyObject *module, PyObject *const *args, Py_ssize_t nargs) goto exit; } name = args[1]; - return_value = monitoring_use_tool_impl(module, tool_id, name); + return_value = monitoring_use_tool_id_impl(module, tool_id, name); exit: return return_value; } -PyDoc_STRVAR(monitoring_free_tool__doc__, -"free_tool($module, tool_id, /)\n" +PyDoc_STRVAR(monitoring_free_tool_id__doc__, +"free_tool_id($module, tool_id, /)\n" "--\n" "\n"); -#define MONITORING_FREE_TOOL_METHODDEF \ - {"free_tool", (PyCFunction)monitoring_free_tool, METH_O, monitoring_free_tool__doc__}, +#define MONITORING_FREE_TOOL_ID_METHODDEF \ + {"free_tool_id", (PyCFunction)monitoring_free_tool_id, METH_O, monitoring_free_tool_id__doc__}, static PyObject * -monitoring_free_tool_impl(PyObject *module, int tool_id); +monitoring_free_tool_id_impl(PyObject *module, int tool_id); static PyObject * -monitoring_free_tool(PyObject *module, PyObject *arg) +monitoring_free_tool_id(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; int tool_id; @@ -61,7 +61,7 @@ monitoring_free_tool(PyObject *module, PyObject *arg) if (tool_id == -1 && PyErr_Occurred()) { goto exit; } - return_value = monitoring_free_tool_impl(module, tool_id); + return_value = monitoring_free_tool_id_impl(module, tool_id); exit: return return_value; @@ -298,4 +298,4 @@ monitoring__all_events(PyObject *module, PyObject *Py_UNUSED(ignored)) { return monitoring__all_events_impl(module); } -/*[clinic end generated code: output=9e5f270f3ce1e945 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f5d03f9aab1fa692 input=a9049054013a1b77]*/ diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 6588eff344cb8f..e8695637e56606 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1602,7 +1602,7 @@ check_valid_tool(int tool_id) } /*[clinic input] -monitoring.use_tool +monitoring.use_tool_id tool_id: int name: object @@ -1611,10 +1611,8 @@ monitoring.use_tool [clinic start generated code]*/ static PyObject * -monitoring_use_tool_impl(PyObject *module, int tool_id, PyObject *name) -/*[clinic end generated code: output=d00b74d147bab1e3 input=506e604e1ea75567]*/ - -/*[clinic end generated code]*/ +monitoring_use_tool_id_impl(PyObject *module, int tool_id, PyObject *name) +/*[clinic end generated code: output=30d76dc92b7cd653 input=ebc453761c621be1]*/ { if (check_valid_tool(tool_id)) { return NULL; @@ -1633,7 +1631,7 @@ monitoring_use_tool_impl(PyObject *module, int tool_id, PyObject *name) } /*[clinic input] -monitoring.free_tool +monitoring.free_tool_id tool_id: int / @@ -1641,10 +1639,8 @@ monitoring.free_tool [clinic start generated code]*/ static PyObject * -monitoring_free_tool_impl(PyObject *module, int tool_id) -/*[clinic end generated code: output=7893bfdad26f51fa input=919fecb6f63130ed]*/ - -/*[clinic end generated code]*/ +monitoring_free_tool_id_impl(PyObject *module, int tool_id) +/*[clinic end generated code: output=86c2d2a1219a8591 input=a23fb6be3a8618e9]*/ { if (check_valid_tool(tool_id)) { return NULL; @@ -1926,8 +1922,8 @@ monitoring__all_events_impl(PyObject *module) } static PyMethodDef methods[] = { - MONITORING_USE_TOOL_METHODDEF - MONITORING_FREE_TOOL_METHODDEF + MONITORING_USE_TOOL_ID_METHODDEF + MONITORING_FREE_TOOL_ID_METHODDEF MONITORING_GET_TOOL_METHODDEF MONITORING_REGISTER_CALLBACK_METHODDEF MONITORING_GET_EVENTS_METHODDEF From a551d65f4dcd1eb5877a203e36aa058967951e8c Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 4 Apr 2023 13:41:05 +0100 Subject: [PATCH 103/116] A bit more cleanup. --- .../2023-03-31-17-24-03.gh-issue-103082.isRUcV.rst | 2 +- Python/instrumentation.c | 11 +++-------- Python/pystate.c | 5 +++-- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-31-17-24-03.gh-issue-103082.isRUcV.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-31-17-24-03.gh-issue-103082.isRUcV.rst index 4fd6fa29bc3648..631ef4c7890450 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2023-03-31-17-24-03.gh-issue-103082.isRUcV.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2023-03-31-17-24-03.gh-issue-103082.isRUcV.rst @@ -1 +1 @@ -Adds low overhead monitoring support. See PEP 669 for full details. +Implement :pep:`669` Low Impact Monitoring for CPython. diff --git a/Python/instrumentation.c b/Python/instrumentation.c index e8695637e56606..7d4633da5149dc 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1937,14 +1937,9 @@ static PyMethodDef methods[] = { static struct PyModuleDef monitoring_module = { PyModuleDef_HEAD_INIT, - "sys.monitoring", - NULL, - -1, /* multiple "initialization" just copies the module dict. */ - methods, - NULL, - NULL, - NULL, - NULL + .m_name = "sys.monitoring", + .m_size = -1, /* multiple "initialization" just copies the module dict. */ + .m_methods = methods, }; PyObject *_Py_CreateMonitoringObject(void) diff --git a/Python/pystate.c b/Python/pystate.c index 5b2dc5a97a79a8..3818434c485194 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -671,7 +671,7 @@ init_interpreter(PyInterpreterState *interp, } } interp->sys_profile_initialized = false; - interp->sys_profile_initialized = false; + interp->sys_trace_initialized = false; if (interp != &runtime->_main_interpreter) { /* Fix the self-referential, statically initialized fields. */ interp->dtoa = (struct _dtoa_state)_dtoa_state_INIT(interp); @@ -818,7 +818,7 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) } } interp->sys_profile_initialized = false; - interp->sys_profile_initialized = false; + interp->sys_trace_initialized = false; for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { Py_CLEAR(interp->monitoring_tool_names[t]); } @@ -880,6 +880,7 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) interp->code_watchers[i] = NULL; } interp->active_code_watchers = 0; + interp->f_opcode_trace_set = false; // XXX Once we have one allocator per interpreter (i.e. // per-interpreter GC) we must ensure that all of the interpreter's // objects have been cleaned up at the point. From 495130672b7aa49aac10e6af9bf8dcc9f5af3d5a Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 4 Apr 2023 13:44:17 +0100 Subject: [PATCH 104/116] Tidy up imports. --- Lib/test/test_monitoring.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 162466cf78c091..685e6e8f16d310 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -1,12 +1,12 @@ """Test suite for the sys.monitoring.""" -import sys -import unittest -import enum -import operator +import collections import functools +import operator +import sys import types -import collections +import unittest + PAIR = (0,1) From 44a031e255678acbde5e7801443b9366081bfe94 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 4 Apr 2023 13:53:35 +0100 Subject: [PATCH 105/116] Use clinic return converters. --- Python/clinic/instrumentation.c.h | 20 +++++++++++++++----- Python/instrumentation.c | 22 +++++++++++----------- 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/Python/clinic/instrumentation.c.h b/Python/clinic/instrumentation.c.h index deaef2e21902da..98ec460071f399 100644 --- a/Python/clinic/instrumentation.c.h +++ b/Python/clinic/instrumentation.c.h @@ -140,7 +140,7 @@ PyDoc_STRVAR(monitoring_get_events__doc__, #define MONITORING_GET_EVENTS_METHODDEF \ {"get_events", (PyCFunction)monitoring_get_events, METH_O, monitoring_get_events__doc__}, -static PyObject * +static int monitoring_get_events_impl(PyObject *module, int tool_id); static PyObject * @@ -148,12 +148,17 @@ monitoring_get_events(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; int tool_id; + int _return_value; tool_id = _PyLong_AsInt(arg); if (tool_id == -1 && PyErr_Occurred()) { goto exit; } - return_value = monitoring_get_events_impl(module, tool_id); + _return_value = monitoring_get_events_impl(module, tool_id); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromLong((long)_return_value); exit: return return_value; @@ -202,7 +207,7 @@ PyDoc_STRVAR(monitoring_get_local_events__doc__, #define MONITORING_GET_LOCAL_EVENTS_METHODDEF \ {"get_local_events", _PyCFunction_CAST(monitoring_get_local_events), METH_FASTCALL, monitoring_get_local_events__doc__}, -static PyObject * +static int monitoring_get_local_events_impl(PyObject *module, PyObject *code, int tool_id); @@ -212,6 +217,7 @@ monitoring_get_local_events(PyObject *module, PyObject *const *args, Py_ssize_t PyObject *return_value = NULL; PyObject *code; int tool_id; + int _return_value; if (!_PyArg_CheckPositional("get_local_events", nargs, 2, 2)) { goto exit; @@ -221,7 +227,11 @@ monitoring_get_local_events(PyObject *module, PyObject *const *args, Py_ssize_t if (tool_id == -1 && PyErr_Occurred()) { goto exit; } - return_value = monitoring_get_local_events_impl(module, code, tool_id); + _return_value = monitoring_get_local_events_impl(module, code, tool_id); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromLong((long)_return_value); exit: return return_value; @@ -298,4 +308,4 @@ monitoring__all_events(PyObject *module, PyObject *Py_UNUSED(ignored)) { return monitoring__all_events_impl(module); } -/*[clinic end generated code: output=f5d03f9aab1fa692 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=aa896325abcb6ca9 input=a9049054013a1b77]*/ diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 7d4633da5149dc..97215f8167bde4 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1714,23 +1714,23 @@ monitoring_register_callback_impl(PyObject *module, int tool_id, int event, } /*[clinic input] -monitoring.get_events +monitoring.get_events -> int tool_id: int / [clinic start generated code]*/ -static PyObject * +static int monitoring_get_events_impl(PyObject *module, int tool_id) -/*[clinic end generated code: output=d8b92576efaa12f9 input=49b77c12cc517025]*/ +/*[clinic end generated code: output=4450cc13f826c8c0 input=a64b238f76c4b2f7]*/ { if (check_valid_tool(tool_id)) { - return NULL; + return -1; } _Py_Monitors *m = &_PyInterpreterState_Get()->monitors; _PyMonitoringEventSet event_set = get_events(m, tool_id); - return PyLong_FromUnsignedLong(event_set); + return event_set; } /*[clinic input] @@ -1765,7 +1765,7 @@ monitoring_set_events_impl(PyObject *module, int tool_id, int event_set) } /*[clinic input] -monitoring.get_local_events +monitoring.get_local_events -> int code: object tool_id: int @@ -1773,20 +1773,20 @@ monitoring.get_local_events [clinic start generated code]*/ -static PyObject * +static int monitoring_get_local_events_impl(PyObject *module, PyObject *code, int tool_id) -/*[clinic end generated code: output=22fad50d2e6404a7 input=c14650d27de48264]*/ +/*[clinic end generated code: output=d37536a48b3b2332 input=e33227382525a36d]*/ { if (!PyCode_Check(code)) { PyErr_Format( PyExc_TypeError, "code must be a code object" ); - return NULL; + return -1; } if (check_valid_tool(tool_id)) { - return NULL; + return -1; } _PyMonitoringEventSet event_set = 0; for (int e = 0; e < PY_MONITORING_UNGROUPED_EVENTS; e++) { @@ -1795,7 +1795,7 @@ monitoring_get_local_events_impl(PyObject *module, PyObject *code, event_set |= (1 << e); } } - return PyLong_FromUnsignedLong(event_set); + return event_set; } /*[clinic input] From c2155b70c99047f5c1098432aa25d4081547416f Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 4 Apr 2023 23:38:00 +0100 Subject: [PATCH 106/116] Address code review. --- Include/cpython/code.h | 5 ++++- Include/internal/pycore_frame.h | 2 +- Include/internal/pycore_interp.h | 1 - Lib/opcode.py | 2 ++ Lib/test/test_monitoring.py | 30 +++++++++++++++++++++++++++--- Python/clinic/instrumentation.c.h | 30 +++++++++++++++--------------- Python/instrumentation.c | 16 ++++++++-------- Python/specialize.c | 2 +- 8 files changed, 58 insertions(+), 30 deletions(-) diff --git a/Include/cpython/code.h b/Include/cpython/code.h index bf7a3b666b2d39..6bead361c79245 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -8,8 +8,11 @@ extern "C" { #endif -#define PY_MONITORING_EVENTS 16 + +/* Count of all "real" monitoring events (not derived from other events) */ #define PY_MONITORING_UNGROUPED_EVENTS 14 +/* Count of all monitoring events */ +#define PY_MONITORING_EVENTS 16 /* Table of which tools are active for each monitored event. */ typedef struct _Py_Monitors { diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index acdfe17ee7cd99..856297aaea8ab4 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -145,7 +145,7 @@ _PyFrame_GetLocalsArray(_PyInterpreterFrame *frame) static inline PyObject** _PyFrame_GetStackPointer(_PyInterpreterFrame *frame) { - PyObject **sp = frame->localsplus+frame->stacktop; + PyObject **sp = frame->localsplus + frame->stacktop; frame->stacktop = -1; return sp; } diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 1ef416ce19dada..a07510a4b26c46 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -219,7 +219,6 @@ PyAPI_FUNC(int) _PyInterpreterState_IDInitref(PyInterpreterState *); PyAPI_FUNC(int) _PyInterpreterState_IDIncref(PyInterpreterState *); PyAPI_FUNC(void) _PyInterpreterState_IDDecref(PyInterpreterState *); - #ifdef __cplusplus } #endif diff --git a/Lib/opcode.py b/Lib/opcode.py index 5aa32d3cb77613..78ae36f34da7e9 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -92,6 +92,8 @@ def pseudo_op(name, op, real_ops): def_op('UNARY_INVERT', 15) +# We reserve 17 as it is the initial value for the specializing counter +# This helps us catch cases where we attempt to execute a cache. def_op('RESERVED', 17) def_op('BINARY_SUBSCR', 25) diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 685e6e8f16d310..cdc5ca19516a69 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -940,14 +940,14 @@ def check_events(self, func, expected, tool=TEST_TOOL, recorders=(ExceptionRecor ev = recorder.event_type sys.monitoring.register_callback(tool, ev, recorder(event_list)) all_events |= ev - sys.monitoring.set_local_events(func.__code__, tool, all_events) + sys.monitoring.set_local_events(tool, func.__code__, all_events) func() - sys.monitoring.set_local_events(func.__code__, tool, 0) + sys.monitoring.set_local_events(tool, func.__code__, 0) for recorder in recorders: sys.monitoring.register_callback(tool, recorder.event_type, None) self.assertEqual(event_list, expected) finally: - sys.monitoring.set_local_events(func.__code__, tool, 0) + sys.monitoring.set_local_events(tool, func.__code__, 0) for recorder in recorders: sys.monitoring.register_callback(tool, recorder.event_type, None) @@ -996,3 +996,27 @@ def func3(): ('line', 'func3', 4), ('line', 'func3', 5), ('line', 'func3', 6)]) + + +class TestSetGetEvents(unittest.TestCase, MonitoringTestBase): + + def test_global(self): + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL), E.PY_START) + sys.monitoring.set_events(TEST_TOOL2, E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL2), E.PY_START) + sys.monitoring.set_events(TEST_TOOL, 0) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL), 0) + sys.monitoring.set_events(TEST_TOOL2,0) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL2), 0) + + def test_local(self): + code = f1.__code__ + sys.monitoring.set_local_events(TEST_TOOL, code, E.PY_START) + self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL, code), E.PY_START) + sys.monitoring.set_local_events(TEST_TOOL2, code, E.PY_START) + self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL2, code), E.PY_START) + sys.monitoring.set_local_events(TEST_TOOL, code, 0) + self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL, code), 0) + sys.monitoring.set_local_events(TEST_TOOL2, code, 0) + self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL2, code), 0) diff --git a/Python/clinic/instrumentation.c.h b/Python/clinic/instrumentation.c.h index 98ec460071f399..cf3984ca24bbfe 100644 --- a/Python/clinic/instrumentation.c.h +++ b/Python/clinic/instrumentation.c.h @@ -200,7 +200,7 @@ monitoring_set_events(PyObject *module, PyObject *const *args, Py_ssize_t nargs) } PyDoc_STRVAR(monitoring_get_local_events__doc__, -"get_local_events($module, code, tool_id, /)\n" +"get_local_events($module, tool_id, code, /)\n" "--\n" "\n"); @@ -208,26 +208,26 @@ PyDoc_STRVAR(monitoring_get_local_events__doc__, {"get_local_events", _PyCFunction_CAST(monitoring_get_local_events), METH_FASTCALL, monitoring_get_local_events__doc__}, static int -monitoring_get_local_events_impl(PyObject *module, PyObject *code, - int tool_id); +monitoring_get_local_events_impl(PyObject *module, int tool_id, + PyObject *code); static PyObject * monitoring_get_local_events(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - PyObject *code; int tool_id; + PyObject *code; int _return_value; if (!_PyArg_CheckPositional("get_local_events", nargs, 2, 2)) { goto exit; } - code = args[0]; - tool_id = _PyLong_AsInt(args[1]); + tool_id = _PyLong_AsInt(args[0]); if (tool_id == -1 && PyErr_Occurred()) { goto exit; } - _return_value = monitoring_get_local_events_impl(module, code, tool_id); + code = args[1]; + _return_value = monitoring_get_local_events_impl(module, tool_id, code); if ((_return_value == -1) && PyErr_Occurred()) { goto exit; } @@ -238,7 +238,7 @@ monitoring_get_local_events(PyObject *module, PyObject *const *args, Py_ssize_t } PyDoc_STRVAR(monitoring_set_local_events__doc__, -"set_local_events($module, code, tool_id, event_set, /)\n" +"set_local_events($module, tool_id, code, event_set, /)\n" "--\n" "\n"); @@ -246,30 +246,30 @@ PyDoc_STRVAR(monitoring_set_local_events__doc__, {"set_local_events", _PyCFunction_CAST(monitoring_set_local_events), METH_FASTCALL, monitoring_set_local_events__doc__}, static PyObject * -monitoring_set_local_events_impl(PyObject *module, PyObject *code, - int tool_id, int event_set); +monitoring_set_local_events_impl(PyObject *module, int tool_id, + PyObject *code, int event_set); static PyObject * monitoring_set_local_events(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - PyObject *code; int tool_id; + PyObject *code; int event_set; if (!_PyArg_CheckPositional("set_local_events", nargs, 3, 3)) { goto exit; } - code = args[0]; - tool_id = _PyLong_AsInt(args[1]); + tool_id = _PyLong_AsInt(args[0]); if (tool_id == -1 && PyErr_Occurred()) { goto exit; } + code = args[1]; event_set = _PyLong_AsInt(args[2]); if (event_set == -1 && PyErr_Occurred()) { goto exit; } - return_value = monitoring_set_local_events_impl(module, code, tool_id, event_set); + return_value = monitoring_set_local_events_impl(module, tool_id, code, event_set); exit: return return_value; @@ -308,4 +308,4 @@ monitoring__all_events(PyObject *module, PyObject *Py_UNUSED(ignored)) { return monitoring__all_events_impl(module); } -/*[clinic end generated code: output=aa896325abcb6ca9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=11cc0803875b3ffa input=a9049054013a1b77]*/ diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 97215f8167bde4..688381049aaa30 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1767,16 +1767,16 @@ monitoring_set_events_impl(PyObject *module, int tool_id, int event_set) /*[clinic input] monitoring.get_local_events -> int - code: object tool_id: int + code: object / [clinic start generated code]*/ static int -monitoring_get_local_events_impl(PyObject *module, PyObject *code, - int tool_id) -/*[clinic end generated code: output=d37536a48b3b2332 input=e33227382525a36d]*/ +monitoring_get_local_events_impl(PyObject *module, int tool_id, + PyObject *code) +/*[clinic end generated code: output=d3e92c1c9c1de8f9 input=bb0f927530386a94]*/ { if (!PyCode_Check(code)) { PyErr_Format( @@ -1801,17 +1801,17 @@ monitoring_get_local_events_impl(PyObject *module, PyObject *code, /*[clinic input] monitoring.set_local_events - code: object tool_id: int + code: object event_set: int / [clinic start generated code]*/ static PyObject * -monitoring_set_local_events_impl(PyObject *module, PyObject *code, - int tool_id, int event_set) -/*[clinic end generated code: output=65d616f95cbb76d8 input=2706fbfe062404bf]*/ +monitoring_set_local_events_impl(PyObject *module, int tool_id, + PyObject *code, int event_set) +/*[clinic end generated code: output=68cc755a65dfea99 input=5655ecd78d937a29]*/ { if (!PyCode_Check(code)) { PyErr_Format( diff --git a/Python/specialize.c b/Python/specialize.c index 74fa7a4f9bc24a..7853d3c94424e9 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -274,7 +274,7 @@ _PyCode_Quicken(PyCodeObject *code) for (int i = 0; i < Py_SIZE(code); i++) { int previous_opcode = opcode; opcode = _Py_GetBaseOpcode(code, i); - assert(opcode < 230); + assert(opcode < MIN_INSTRUMENTED_OPCODE); int caches = _PyOpcode_Caches[opcode]; if (caches) { instructions[i + 1].cache = adaptive_counter_warmup(); From b218428a3e41e75eb99d5884f3b061fb74fd90eb Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 5 Apr 2023 00:24:20 +0100 Subject: [PATCH 107/116] Address further review comments. --- Python/legacy_tracing.c | 38 ++++++++++++++++---------------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index 677910e9de8c8e..09ee5cf1840c28 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -14,17 +14,11 @@ typedef struct _PyLegacyEventHandler { int event; } _PyLegacyEventHandler; -static void -dealloc(_PyLegacyEventHandler *self) -{ - PyObject_Free(self); -} - - /* The Py_tracefunc function expects the following arguments: - * frame: FrameObject - * kind: c-int - * arg: The arg + * obj: the trace object (PyObject *) + * frame: the current frame (PyFrameObject *) + * kind: the kind of event, see PyTrace_XXX #defines (int) + * arg: The arg (a PyObject *) */ static PyObject * @@ -34,7 +28,7 @@ call_profile_func(_PyLegacyEventHandler *self, PyObject *arg) if (tstate->c_profilefunc == NULL) { Py_RETURN_NONE; } - PyFrameObject* frame = PyEval_GetFrame(); + PyFrameObject *frame = PyEval_GetFrame(); if (frame == NULL) { PyErr_SetString(PyExc_SystemError, "Missing frame when calling profile function."); @@ -101,8 +95,6 @@ sys_profile_call_or_return( Py_RETURN_NONE; } - - static PyObject * call_trace_func(_PyLegacyEventHandler *self, PyObject *arg) { @@ -110,7 +102,7 @@ call_trace_func(_PyLegacyEventHandler *self, PyObject *arg) if (tstate->c_tracefunc == NULL) { Py_RETURN_NONE; } - PyFrameObject* frame = PyEval_GetFrame(); + PyFrameObject *frame = PyEval_GetFrame(); if (frame == NULL) { PyErr_SetString(PyExc_SystemError, "Missing frame when calling trace function."); @@ -139,7 +131,7 @@ sys_trace_exception_func( if (tb == NULL) { tb = Py_NewRef(Py_None); } - PyObject * tuple = PyTuple_Pack(3, type, exc, tb); + PyObject *tuple = PyTuple_Pack(3, type, exc, tb); Py_DECREF(tb); if (tuple == NULL) { return NULL; @@ -190,7 +182,7 @@ sys_trace_instruction_func( ) { assert(kwnames == NULL); assert(PyVectorcall_NARGS(nargsf) == 2); - PyFrameObject* frame = PyEval_GetFrame(); + PyFrameObject *frame = PyEval_GetFrame(); if (frame == NULL) { PyErr_SetString(PyExc_SystemError, "Missing frame when calling trace function."); @@ -213,7 +205,7 @@ sys_trace_instruction_func( static PyObject * trace_line( PyThreadState *tstate, _PyLegacyEventHandler *self, - PyFrameObject* frame, int line + PyFrameObject *frame, int line ) { if (!frame->f_trace_lines) { Py_RETURN_NONE; @@ -246,7 +238,7 @@ sys_trace_line_func( assert(PyVectorcall_NARGS(nargsf) == 2); int line = _PyLong_AsInt(args[1]); assert(line >= 0); - PyFrameObject* frame = PyEval_GetFrame(); + PyFrameObject *frame = PyEval_GetFrame(); if (frame == NULL) { PyErr_SetString(PyExc_SystemError, "Missing frame when calling trace function."); @@ -276,7 +268,7 @@ sys_trace_jump_func( assert(from >= 0); int to = _PyLong_AsInt(args[2]); assert(to >= 0); - PyFrameObject* frame = PyEval_GetFrame(); + PyFrameObject *frame = PyEval_GetFrame(); if (frame == NULL) { PyErr_SetString(PyExc_SystemError, "Missing frame when calling trace function."); @@ -314,10 +306,11 @@ sys_trace_exception_handled( Py_RETURN_NONE; } assert(PyVectorcall_NARGS(nargsf) == 3); - PyFrameObject* frame = PyEval_GetFrame(); + PyFrameObject *frame = PyEval_GetFrame(); PyCodeObject *code = (PyCodeObject *)args[0]; assert(PyCode_Check(code)); assert(code == frame->f_frame->f_code); + assert(PyLong_Check(args[1])); int offset = _PyLong_AsInt(args[1]); /* We can call _Py_Instrumentation_GetLine because we always set * line events for tracing */ @@ -334,9 +327,10 @@ PyTypeObject _PyLegacyEventHandler_Type = { _PyVarObject_IMMORTAL_INIT(&PyType_Type, 0), "sys.legacy_event_handler", sizeof(_PyLegacyEventHandler), - .tp_dealloc = (destructor)dealloc, + .tp_dealloc = (destructor)PyObject_Free, .tp_vectorcall_offset = offsetof(_PyLegacyEventHandler, vectorcall), - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_VECTORCALL, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_HAVE_VECTORCALL | Py_TPFLAGS_DISALLOW_INSTANTIATION, .tp_call = PyVectorcall_Call, }; From 9c0429e6ab32f0554341b9ab56700a74e609d9a6 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 5 Apr 2023 04:55:55 +0100 Subject: [PATCH 108/116] Use byte offset, to conform to the convention used in other APIs taking code offsets. --- Lib/test/test_monitoring.py | 72 ++++++++++++++++++------------------- Python/instrumentation.c | 18 ++++++---- Python/legacy_tracing.c | 6 ++-- 3 files changed, 50 insertions(+), 46 deletions(-) diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index cdc5ca19516a69..c889854b7250be 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -794,16 +794,16 @@ def func1(): self.check_events(func1, recorders = LINE_AND_INSTRUCTION_RECORDERS, expected = [ ('line', 'check_events', 10), - ('instruction', 'func1', 1), - ('line', 'func1', 1), ('instruction', 'func1', 2), - ('instruction', 'func1', 3), - ('line', 'func1', 2), + ('line', 'func1', 1), ('instruction', 'func1', 4), - ('instruction', 'func1', 5), - ('line', 'func1', 3), ('instruction', 'func1', 6), - ('instruction', 'func1', 7), + ('line', 'func1', 2), + ('instruction', 'func1', 8), + ('instruction', 'func1', 10), + ('line', 'func1', 3), + ('instruction', 'func1', 12), + ('instruction', 'func1', 14), ('line', 'check_events', 11)]) def test_c_call(self): @@ -815,19 +815,19 @@ def func2(): self.check_events(func2, recorders = LINE_AND_INSTRUCTION_RECORDERS, expected = [ ('line', 'check_events', 10), - ('instruction', 'func2', 1), - ('line', 'func2', 1), ('instruction', 'func2', 2), - ('instruction', 'func2', 3), - ('line', 'func2', 2), + ('line', 'func2', 1), ('instruction', 'func2', 4), - ('instruction', 'func2', 14), - ('instruction', 'func2', 15), - ('instruction', 'func2', 20), - ('instruction', 'func2', 21), + ('instruction', 'func2', 6), + ('line', 'func2', 2), + ('instruction', 'func2', 8), + ('instruction', 'func2', 28), + ('instruction', 'func2', 30), + ('instruction', 'func2', 40), + ('instruction', 'func2', 42), ('line', 'func2', 3), - ('instruction', 'func2', 22), - ('instruction', 'func2', 23), + ('instruction', 'func2', 44), + ('instruction', 'func2', 46), ('line', 'check_events', 11)]) def test_try_except(self): @@ -842,25 +842,25 @@ def func3(): self.check_events(func3, recorders = LINE_AND_INSTRUCTION_RECORDERS, expected = [ ('line', 'check_events', 10), - ('instruction', 'func3', 1), - ('line', 'func3', 1), ('instruction', 'func3', 2), - ('line', 'func3', 2), - ('instruction', 'func3', 3), + ('line', 'func3', 1), ('instruction', 'func3', 4), + ('line', 'func3', 2), + ('instruction', 'func3', 6), + ('instruction', 'func3', 8), ('line', 'func3', 3), - ('instruction', 'func3', 9), - ('instruction', 'func3', 10), - ('instruction', 'func3', 11), + ('instruction', 'func3', 18), + ('instruction', 'func3', 20), + ('instruction', 'func3', 22), ('line', 'func3', 4), - ('instruction', 'func3', 12), + ('instruction', 'func3', 24), ('line', 'func3', 5), - ('instruction', 'func3', 13), - ('instruction', 'func3', 14), - ('instruction', 'func3', 15), + ('instruction', 'func3', 26), + ('instruction', 'func3', 28), + ('instruction', 'func3', 30), ('line', 'func3', 6), - ('instruction', 'func3', 16), - ('instruction', 'func3', 17), + ('instruction', 'func3', 32), + ('instruction', 'func3', 34), ('line', 'check_events', 11)]) class TestInstallIncrementallly(unittest.TestCase): @@ -891,10 +891,10 @@ def func1(): line1 = 1 MUST_INCLUDE_LI = [ - ('instruction', 'func1', 1), - ('line', 'func1', 1), ('instruction', 'func1', 2), - ('instruction', 'func1', 3)] + ('line', 'func1', 1), + ('instruction', 'func1', 4), + ('instruction', 'func1', 6)] def test_line_then_instruction(self): recorders = [ LineRecorder, InstructionRecorder ] @@ -911,11 +911,11 @@ def func2(): len(()) MUST_INCLUDE_CI = [ - ('instruction', 'func2', 1), + ('instruction', 'func2', 2), ('call', 'func2', sys.monitoring.MISSING), ('call', 'len', ()), - ('instruction', 'func2', 6), - ('instruction', 'func2', 7)] + ('instruction', 'func2', 12), + ('instruction', 'func2', 14)] diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 688381049aaa30..436c2b48d178ab 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -924,8 +924,11 @@ call_instrumentation_vector( assert(instrumentation_cross_checks(tstate->interp, code)); assert(args[1] == NULL); args[1] = (PyObject *)code; - int offset = instr - _PyCode_CODE(code); - PyObject *offset_obj = PyLong_FromSsize_t(offset); + int offset = (int)(instr - _PyCode_CODE(code)); + /* Offset visible to user should be the offset in bytes, as that is the + * convention for APIs involving code offsets. */ + int bytes_offset = offset * (int)sizeof(_Py_CODEUNIT); + PyObject *offset_obj = PyLong_FromSsize_t(bytes_offset); if (offset_obj == NULL) { return -1; } @@ -996,8 +999,8 @@ _Py_call_instrumentation_jump( assert(frame->prev_instr == instr); frame->prev_instr = target; PyCodeObject *code = frame->f_code; - int to = target - _PyCode_CODE(code); - PyObject *to_obj = PyLong_FromLong(to); + int to = (int)(target - _PyCode_CODE(code)); + PyObject *to_obj = PyLong_FromLong(to * (int)sizeof(_Py_CODEUNIT)); if (to_obj == NULL) { return -1; } @@ -1066,7 +1069,7 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, PyCodeObject *code = frame->f_code; assert(is_version_up_to_date(code, tstate->interp)); assert(instrumentation_cross_checks(tstate->interp, code)); - int i = instr - _PyCode_CODE(code); + int i = (int)(instr - _PyCode_CODE(code)); _PyCoMonitoringData *monitoring = code->_co_monitoring; _PyCoLineInstrumentationData *line_data = &monitoring->lines[i]; uint8_t original_opcode = line_data->original_opcode; @@ -1121,7 +1124,7 @@ _Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame* PyCodeObject *code = frame->f_code; assert(is_version_up_to_date(code, tstate->interp)); assert(instrumentation_cross_checks(tstate->interp, code)); - int offset = instr - _PyCode_CODE(code); + int offset = (int)(instr - _PyCode_CODE(code)); _PyCoMonitoringData *instrumentation_data = code->_co_monitoring; assert(instrumentation_data->per_instruction_opcodes); int next_opcode = instrumentation_data->per_instruction_opcodes[offset]; @@ -1134,7 +1137,8 @@ _Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame* (interp->monitors.tools[PY_MONITORING_EVENT_INSTRUCTION] | code->_co_monitoring->local_monitors.tools[PY_MONITORING_EVENT_INSTRUCTION] ); - PyObject *offset_obj = PyLong_FromSsize_t(offset); + int bytes_offset = offset * (int)sizeof(_Py_CODEUNIT); + PyObject *offset_obj = PyLong_FromSsize_t(bytes_offset); if (offset_obj == NULL) { return -1; } diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index 09ee5cf1840c28..6e6deea7b818ce 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -264,9 +264,9 @@ sys_trace_jump_func( Py_RETURN_NONE; } assert(PyVectorcall_NARGS(nargsf) == 3); - int from = _PyLong_AsInt(args[1]); + int from = _PyLong_AsInt(args[1])/sizeof(_Py_CODEUNIT); assert(from >= 0); - int to = _PyLong_AsInt(args[2]); + int to = _PyLong_AsInt(args[2])/sizeof(_Py_CODEUNIT); assert(to >= 0); PyFrameObject *frame = PyEval_GetFrame(); if (frame == NULL) { @@ -311,7 +311,7 @@ sys_trace_exception_handled( assert(PyCode_Check(code)); assert(code == frame->f_frame->f_code); assert(PyLong_Check(args[1])); - int offset = _PyLong_AsInt(args[1]); + int offset = _PyLong_AsInt(args[1])/sizeof(_Py_CODEUNIT); /* We can call _Py_Instrumentation_GetLine because we always set * line events for tracing */ int line = _Py_Instrumentation_GetLine(code, offset); From c6bf38e29d126a6b812693fc89f59736c05363c9 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 5 Apr 2023 18:29:16 +0100 Subject: [PATCH 109/116] Address code review. --- Objects/object.c | 2 + Python/bytecodes.c | 40 +- Python/ceval.c | 15 - Python/ceval_macros.h | 15 + Python/generated_cases.c.h | 986 +++++++++++++++++++------------------ Python/opcode_metadata.h | 18 +- 6 files changed, 540 insertions(+), 536 deletions(-) diff --git a/Objects/object.c b/Objects/object.c index 9dd5eb998217f6..fcd6fa59af8b65 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1978,6 +1978,7 @@ extern PyTypeObject _Py_GenericAliasIterType; extern PyTypeObject _PyMemoryIter_Type; extern PyTypeObject _PyLineIterator; extern PyTypeObject _PyPositionsIterator; +extern PyTypeObject _PyLegacyEventHandler_Type; static PyTypeObject* static_types[] = { // The two most important base types: must be initialized first and @@ -2075,6 +2076,7 @@ static PyTypeObject* static_types[] = { &_PyHamt_BitmapNode_Type, &_PyHamt_CollisionNode_Type, &_PyHamt_Type, + &_PyLegacyEventHandler_Type, &_PyInterpreterID_Type, &_PyLineIterator, &_PyManagedBuffer_Type, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 322c8c82d0a89c..bf26d13c5884ca 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -132,9 +132,25 @@ dummy_func( inst(NOP, (--)) { } - inst(INSTRUMENTED_RESUME, (--)) { - /* Check monitoring *before* calling instrument */ + inst(RESUME, (--)) { + assert(tstate->cframe == &cframe); + assert(frame == cframe.current_frame); /* Possibly combine this with eval breaker */ + if (frame->f_code->_co_instrumentation_version != tstate->interp->monitoring_version) { + int err = _Py_Instrument(frame->f_code, tstate->interp); + ERROR_IF(err, error); + next_instr--; + } + else if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { + goto handle_eval_breaker; + } + } + + inst(INSTRUMENTED_RESUME, (--)) { + /* Possible performance enhancement: + * We need to check the eval breaker anyway, can we + * combine the instrument verison check and the eval breaker test? + */ if (frame->f_code->_co_instrumentation_version != tstate->interp->monitoring_version) { if (_Py_Instrument(frame->f_code, tstate->interp)) { goto error; @@ -158,20 +174,6 @@ dummy_func( } } - inst(RESUME, (--)) { - assert(tstate->cframe == &cframe); - assert(frame == cframe.current_frame); - /* Possibly combine this with eval breaker */ - if (frame->f_code->_co_instrumentation_version != tstate->interp->monitoring_version) { - int err = _Py_Instrument(frame->f_code, tstate->interp); - ERROR_IF(err, error); - next_instr--; - } - else if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { - goto handle_eval_breaker; - } - } - inst(LOAD_CLOSURE, (-- value)) { /* We keep LOAD_CLOSURE so that the bytecode stays more readable. */ value = GETLOCAL(oparg); @@ -2159,7 +2161,7 @@ dummy_func( // Common case: no jump, leave it to the code generator } - inst(INSTRUMENTED_FOR_ITER) { + inst(INSTRUMENTED_FOR_ITER, ( -- )) { _Py_CODEUNIT *here = next_instr-1; _Py_CODEUNIT *target; PyObject *iter = TOP(); @@ -2999,7 +3001,7 @@ dummy_func( CHECK_EVAL_BREAKER(); } - inst(INSTRUMENTED_CALL_FUNCTION_EX) { + inst(INSTRUMENTED_CALL_FUNCTION_EX, ( -- )) { GO_TO_INSTRUCTION(CALL_FUNCTION_EX); } @@ -3218,12 +3220,10 @@ dummy_func( DISPATCH_GOTO(); } - // stack effect: ( -- ) inst(INSTRUMENTED_JUMP_FORWARD, ( -- )) { INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); } - // stack effect: ( -- ) inst(INSTRUMENTED_JUMP_BACKWARD, ( -- )) { INSTRUMENTED_JUMP(next_instr-1, next_instr-oparg, PY_MONITORING_EVENT_JUMP); CHECK_EVAL_BREAKER(); diff --git a/Python/ceval.c b/Python/ceval.c index 8d2fc2c4d0e9e8..a38c9ec9ad5f9c 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -610,21 +610,6 @@ static inline void _Py_LeaveRecursiveCallPy(PyThreadState *tstate) { tstate->py_recursion_remaining++; } -// If a trace function sets a new f_lineno and -// *then* raises, we use the destination when searching -// for an exception handler, displaying the traceback, and so on -#define INSTRUMENTED_JUMP(src, dest, event) \ -do { \ - _PyFrame_SetStackPointer(frame, stack_pointer); \ - int err = _Py_call_instrumentation_jump(tstate, event, frame, src, dest); \ - stack_pointer = _PyFrame_GetStackPointer(frame); \ - if (err) { \ - next_instr = (dest)+1; \ - goto error; \ - } \ - next_instr = frame->prev_instr; \ -} while (0); - /* Disable unused label warnings. They are handy for debugging, even if computed gotos aren't used. */ diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 2fe740991497b0..485771ac65a767 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -327,3 +327,18 @@ do { \ _Py_DECREF_NO_DEALLOC(right); \ } \ } while (0) + +// If a trace function sets a new f_lineno and +// *then* raises, we use the destination when searching +// for an exception handler, displaying the traceback, and so on +#define INSTRUMENTED_JUMP(src, dest, event) \ +do { \ + _PyFrame_SetStackPointer(frame, stack_pointer); \ + int err = _Py_call_instrumentation_jump(tstate, event, frame, src, dest); \ + stack_pointer = _PyFrame_GetStackPointer(frame); \ + if (err) { \ + next_instr = (dest)+1; \ + goto error; \ + } \ + next_instr = frame->prev_instr; \ +} while (0); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index e37d122986a27b..a2877cb86b63b8 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -7,10 +7,29 @@ DISPATCH(); } - TARGET(INSTRUMENTED_RESUME) { + TARGET(RESUME) { #line 136 "Python/bytecodes.c" - /* Check monitoring *before* calling instrument */ + assert(tstate->cframe == &cframe); + assert(frame == cframe.current_frame); /* Possibly combine this with eval breaker */ + if (frame->f_code->_co_instrumentation_version != tstate->interp->monitoring_version) { + int err = _Py_Instrument(frame->f_code, tstate->interp); + if (err) goto error; + next_instr--; + } + else if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { + goto handle_eval_breaker; + } + #line 24 "Python/generated_cases.c.h" + DISPATCH(); + } + + TARGET(INSTRUMENTED_RESUME) { + #line 150 "Python/bytecodes.c" + /* Possible performance enhancement: + * We need to check the eval breaker anyway, can we + * combine the instrument verison check and the eval breaker test? + */ if (frame->f_code->_co_instrumentation_version != tstate->interp->monitoring_version) { if (_Py_Instrument(frame->f_code, tstate->interp)) { goto error; @@ -32,35 +51,18 @@ goto handle_eval_breaker; } } - #line 36 "Python/generated_cases.c.h" - DISPATCH(); - } - - TARGET(RESUME) { - #line 162 "Python/bytecodes.c" - assert(tstate->cframe == &cframe); - assert(frame == cframe.current_frame); - /* Possibly combine this with eval breaker */ - if (frame->f_code->_co_instrumentation_version != tstate->interp->monitoring_version) { - int err = _Py_Instrument(frame->f_code, tstate->interp); - if (err) goto error; - next_instr--; - } - else if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) { - goto handle_eval_breaker; - } - #line 53 "Python/generated_cases.c.h" + #line 55 "Python/generated_cases.c.h" DISPATCH(); } TARGET(LOAD_CLOSURE) { PyObject *value; - #line 176 "Python/bytecodes.c" + #line 178 "Python/bytecodes.c" /* We keep LOAD_CLOSURE so that the bytecode stays more readable. */ value = GETLOCAL(oparg); if (value == NULL) goto unbound_local_error; Py_INCREF(value); - #line 64 "Python/generated_cases.c.h" + #line 66 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -68,11 +70,11 @@ TARGET(LOAD_FAST_CHECK) { PyObject *value; - #line 183 "Python/bytecodes.c" + #line 185 "Python/bytecodes.c" value = GETLOCAL(oparg); if (value == NULL) goto unbound_local_error; Py_INCREF(value); - #line 76 "Python/generated_cases.c.h" + #line 78 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -80,11 +82,11 @@ TARGET(LOAD_FAST) { PyObject *value; - #line 189 "Python/bytecodes.c" + #line 191 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - #line 88 "Python/generated_cases.c.h" + #line 90 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -93,10 +95,10 @@ TARGET(LOAD_CONST) { PREDICTED(LOAD_CONST); PyObject *value; - #line 195 "Python/bytecodes.c" + #line 197 "Python/bytecodes.c" value = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(value); - #line 100 "Python/generated_cases.c.h" + #line 102 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -104,9 +106,9 @@ TARGET(STORE_FAST) { PyObject *value = stack_pointer[-1]; - #line 200 "Python/bytecodes.c" + #line 202 "Python/bytecodes.c" SETLOCAL(oparg, value); - #line 110 "Python/generated_cases.c.h" + #line 112 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -116,21 +118,21 @@ PyObject *_tmp_2; { PyObject *value; - #line 189 "Python/bytecodes.c" + #line 191 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - #line 124 "Python/generated_cases.c.h" + #line 126 "Python/generated_cases.c.h" _tmp_2 = value; } oparg = (next_instr++)->op.arg; { PyObject *value; - #line 189 "Python/bytecodes.c" + #line 191 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - #line 134 "Python/generated_cases.c.h" + #line 136 "Python/generated_cases.c.h" _tmp_1 = value; } STACK_GROW(2); @@ -144,20 +146,20 @@ PyObject *_tmp_2; { PyObject *value; - #line 189 "Python/bytecodes.c" + #line 191 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - #line 152 "Python/generated_cases.c.h" + #line 154 "Python/generated_cases.c.h" _tmp_2 = value; } oparg = (next_instr++)->op.arg; { PyObject *value; - #line 195 "Python/bytecodes.c" + #line 197 "Python/bytecodes.c" value = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(value); - #line 161 "Python/generated_cases.c.h" + #line 163 "Python/generated_cases.c.h" _tmp_1 = value; } STACK_GROW(2); @@ -170,18 +172,18 @@ PyObject *_tmp_1 = stack_pointer[-1]; { PyObject *value = _tmp_1; - #line 200 "Python/bytecodes.c" + #line 202 "Python/bytecodes.c" SETLOCAL(oparg, value); - #line 176 "Python/generated_cases.c.h" + #line 178 "Python/generated_cases.c.h" } oparg = (next_instr++)->op.arg; { PyObject *value; - #line 189 "Python/bytecodes.c" + #line 191 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - #line 185 "Python/generated_cases.c.h" + #line 187 "Python/generated_cases.c.h" _tmp_1 = value; } stack_pointer[-1] = _tmp_1; @@ -193,16 +195,16 @@ PyObject *_tmp_2 = stack_pointer[-2]; { PyObject *value = _tmp_1; - #line 200 "Python/bytecodes.c" + #line 202 "Python/bytecodes.c" SETLOCAL(oparg, value); - #line 199 "Python/generated_cases.c.h" + #line 201 "Python/generated_cases.c.h" } oparg = (next_instr++)->op.arg; { PyObject *value = _tmp_2; - #line 200 "Python/bytecodes.c" + #line 202 "Python/bytecodes.c" SETLOCAL(oparg, value); - #line 206 "Python/generated_cases.c.h" + #line 208 "Python/generated_cases.c.h" } STACK_SHRINK(2); DISPATCH(); @@ -213,20 +215,20 @@ PyObject *_tmp_2; { PyObject *value; - #line 195 "Python/bytecodes.c" + #line 197 "Python/bytecodes.c" value = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(value); - #line 220 "Python/generated_cases.c.h" + #line 222 "Python/generated_cases.c.h" _tmp_2 = value; } oparg = (next_instr++)->op.arg; { PyObject *value; - #line 189 "Python/bytecodes.c" + #line 191 "Python/bytecodes.c" value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - #line 230 "Python/generated_cases.c.h" + #line 232 "Python/generated_cases.c.h" _tmp_1 = value; } STACK_GROW(2); @@ -237,8 +239,8 @@ TARGET(POP_TOP) { PyObject *value = stack_pointer[-1]; - #line 210 "Python/bytecodes.c" - #line 242 "Python/generated_cases.c.h" + #line 212 "Python/bytecodes.c" + #line 244 "Python/generated_cases.c.h" Py_DECREF(value); STACK_SHRINK(1); DISPATCH(); @@ -246,9 +248,9 @@ TARGET(PUSH_NULL) { PyObject *res; - #line 214 "Python/bytecodes.c" + #line 216 "Python/bytecodes.c" res = NULL; - #line 252 "Python/generated_cases.c.h" + #line 254 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); @@ -259,14 +261,14 @@ PyObject *_tmp_2 = stack_pointer[-2]; { PyObject *value = _tmp_1; - #line 210 "Python/bytecodes.c" - #line 264 "Python/generated_cases.c.h" + #line 212 "Python/bytecodes.c" + #line 266 "Python/generated_cases.c.h" Py_DECREF(value); } { PyObject *value = _tmp_2; - #line 210 "Python/bytecodes.c" - #line 270 "Python/generated_cases.c.h" + #line 212 "Python/bytecodes.c" + #line 272 "Python/generated_cases.c.h" Py_DECREF(value); } STACK_SHRINK(2); @@ -276,7 +278,7 @@ TARGET(INSTRUMENTED_END_FOR) { PyObject *value = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; - #line 220 "Python/bytecodes.c" + #line 222 "Python/bytecodes.c" /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ if (PyGen_Check(receiver)) { @@ -286,7 +288,7 @@ } PyErr_SetRaisedException(NULL); } - #line 290 "Python/generated_cases.c.h" + #line 292 "Python/generated_cases.c.h" Py_DECREF(receiver); Py_DECREF(value); STACK_SHRINK(2); @@ -296,9 +298,9 @@ TARGET(END_SEND) { PyObject *value = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; - #line 233 "Python/bytecodes.c" + #line 235 "Python/bytecodes.c" Py_DECREF(receiver); - #line 302 "Python/generated_cases.c.h" + #line 304 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = value; DISPATCH(); @@ -307,7 +309,7 @@ TARGET(INSTRUMENTED_END_SEND) { PyObject *value = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; - #line 237 "Python/bytecodes.c" + #line 239 "Python/bytecodes.c" if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); if (monitor_stop_iteration(tstate, frame, next_instr-1)) { @@ -316,7 +318,7 @@ PyErr_SetRaisedException(NULL); } Py_DECREF(receiver); - #line 320 "Python/generated_cases.c.h" + #line 322 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = value; DISPATCH(); @@ -325,13 +327,13 @@ TARGET(UNARY_NEGATIVE) { PyObject *value = stack_pointer[-1]; PyObject *res; - #line 248 "Python/bytecodes.c" + #line 250 "Python/bytecodes.c" res = PyNumber_Negative(value); - #line 331 "Python/generated_cases.c.h" + #line 333 "Python/generated_cases.c.h" Py_DECREF(value); - #line 250 "Python/bytecodes.c" + #line 252 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; - #line 335 "Python/generated_cases.c.h" + #line 337 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -339,11 +341,11 @@ TARGET(UNARY_NOT) { PyObject *value = stack_pointer[-1]; PyObject *res; - #line 254 "Python/bytecodes.c" + #line 256 "Python/bytecodes.c" int err = PyObject_IsTrue(value); - #line 345 "Python/generated_cases.c.h" + #line 347 "Python/generated_cases.c.h" Py_DECREF(value); - #line 256 "Python/bytecodes.c" + #line 258 "Python/bytecodes.c" if (err < 0) goto pop_1_error; if (err == 0) { res = Py_True; @@ -352,7 +354,7 @@ res = Py_False; } Py_INCREF(res); - #line 356 "Python/generated_cases.c.h" + #line 358 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -360,13 +362,13 @@ TARGET(UNARY_INVERT) { PyObject *value = stack_pointer[-1]; PyObject *res; - #line 267 "Python/bytecodes.c" + #line 269 "Python/bytecodes.c" res = PyNumber_Invert(value); - #line 366 "Python/generated_cases.c.h" + #line 368 "Python/generated_cases.c.h" Py_DECREF(value); - #line 269 "Python/bytecodes.c" + #line 271 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; - #line 370 "Python/generated_cases.c.h" + #line 372 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -375,7 +377,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *prod; - #line 286 "Python/bytecodes.c" + #line 288 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -383,7 +385,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (prod == NULL) goto pop_2_error; - #line 387 "Python/generated_cases.c.h" + #line 389 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = prod; next_instr += 1; @@ -394,14 +396,14 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *prod; - #line 296 "Python/bytecodes.c" + #line 298 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); double dprod = ((PyFloatObject *)left)->ob_fval * ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dprod, prod); - #line 405 "Python/generated_cases.c.h" + #line 407 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = prod; next_instr += 1; @@ -412,7 +414,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *sub; - #line 305 "Python/bytecodes.c" + #line 307 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -420,7 +422,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (sub == NULL) goto pop_2_error; - #line 424 "Python/generated_cases.c.h" + #line 426 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = sub; next_instr += 1; @@ -431,13 +433,13 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *sub; - #line 315 "Python/bytecodes.c" + #line 317 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); STAT_INC(BINARY_OP, hit); double dsub = ((PyFloatObject *)left)->ob_fval - ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dsub, sub); - #line 441 "Python/generated_cases.c.h" + #line 443 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = sub; next_instr += 1; @@ -448,7 +450,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 323 "Python/bytecodes.c" + #line 325 "Python/bytecodes.c" DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -456,7 +458,7 @@ _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); if (res == NULL) goto pop_2_error; - #line 460 "Python/generated_cases.c.h" + #line 462 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -466,7 +468,7 @@ TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; - #line 339 "Python/bytecodes.c" + #line 341 "Python/bytecodes.c" DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); _Py_CODEUNIT true_next = next_instr[INLINE_CACHE_ENTRIES_BINARY_OP]; @@ -493,7 +495,7 @@ if (*target_local == NULL) goto pop_2_error; // The STORE_FAST is already done. JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP + 1); - #line 497 "Python/generated_cases.c.h" + #line 499 "Python/generated_cases.c.h" STACK_SHRINK(2); DISPATCH(); } @@ -502,14 +504,14 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *sum; - #line 368 "Python/bytecodes.c" + #line 370 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); double dsum = ((PyFloatObject *)left)->ob_fval + ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dsum, sum); - #line 513 "Python/generated_cases.c.h" + #line 515 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = sum; next_instr += 1; @@ -520,7 +522,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *sum; - #line 377 "Python/bytecodes.c" + #line 379 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); STAT_INC(BINARY_OP, hit); @@ -528,7 +530,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (sum == NULL) goto pop_2_error; - #line 532 "Python/generated_cases.c.h" + #line 534 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = sum; next_instr += 1; @@ -541,7 +543,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *container = stack_pointer[-2]; PyObject *res; - #line 395 "Python/bytecodes.c" + #line 397 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -553,12 +555,12 @@ DECREMENT_ADAPTIVE_COUNTER(cache->counter); #endif /* ENABLE_SPECIALIZATION */ res = PyObject_GetItem(container, sub); - #line 557 "Python/generated_cases.c.h" + #line 559 "Python/generated_cases.c.h" Py_DECREF(container); Py_DECREF(sub); - #line 407 "Python/bytecodes.c" + #line 409 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 562 "Python/generated_cases.c.h" + #line 564 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -570,7 +572,7 @@ PyObject *start = stack_pointer[-2]; PyObject *container = stack_pointer[-3]; PyObject *res; - #line 411 "Python/bytecodes.c" + #line 413 "Python/bytecodes.c" PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); // Can't use ERROR_IF() here, because we haven't // DECREF'ed container yet, and we still own slice. @@ -583,7 +585,7 @@ } Py_DECREF(container); if (res == NULL) goto pop_3_error; - #line 587 "Python/generated_cases.c.h" + #line 589 "Python/generated_cases.c.h" STACK_SHRINK(2); stack_pointer[-1] = res; DISPATCH(); @@ -594,7 +596,7 @@ PyObject *start = stack_pointer[-2]; PyObject *container = stack_pointer[-3]; PyObject *v = stack_pointer[-4]; - #line 426 "Python/bytecodes.c" + #line 428 "Python/bytecodes.c" PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); int err; if (slice == NULL) { @@ -607,7 +609,7 @@ Py_DECREF(v); Py_DECREF(container); if (err) goto pop_4_error; - #line 611 "Python/generated_cases.c.h" + #line 613 "Python/generated_cases.c.h" STACK_SHRINK(4); DISPATCH(); } @@ -616,7 +618,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *list = stack_pointer[-2]; PyObject *res; - #line 441 "Python/bytecodes.c" + #line 443 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); @@ -630,7 +632,7 @@ Py_INCREF(res); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(list); - #line 634 "Python/generated_cases.c.h" + #line 636 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -641,7 +643,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *tuple = stack_pointer[-2]; PyObject *res; - #line 457 "Python/bytecodes.c" + #line 459 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); @@ -655,7 +657,7 @@ Py_INCREF(res); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(tuple); - #line 659 "Python/generated_cases.c.h" + #line 661 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -666,7 +668,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *dict = stack_pointer[-2]; PyObject *res; - #line 473 "Python/bytecodes.c" + #line 475 "Python/bytecodes.c" DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); res = PyDict_GetItemWithError(dict, sub); @@ -674,14 +676,14 @@ if (!_PyErr_Occurred(tstate)) { _PyErr_SetKeyError(sub); } - #line 678 "Python/generated_cases.c.h" + #line 680 "Python/generated_cases.c.h" Py_DECREF(dict); Py_DECREF(sub); - #line 481 "Python/bytecodes.c" + #line 483 "Python/bytecodes.c" if (true) goto pop_2_error; } Py_INCREF(res); // Do this before DECREF'ing dict, sub - #line 685 "Python/generated_cases.c.h" + #line 687 "Python/generated_cases.c.h" Py_DECREF(dict); Py_DECREF(sub); STACK_SHRINK(1); @@ -693,7 +695,7 @@ TARGET(BINARY_SUBSCR_GETITEM) { PyObject *sub = stack_pointer[-1]; PyObject *container = stack_pointer[-2]; - #line 488 "Python/bytecodes.c" + #line 490 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(container); DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; @@ -714,15 +716,15 @@ new_frame->localsplus[1] = sub; JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR); DISPATCH_INLINED(new_frame); - #line 718 "Python/generated_cases.c.h" + #line 720 "Python/generated_cases.c.h" } TARGET(LIST_APPEND) { PyObject *v = stack_pointer[-1]; PyObject *list = stack_pointer[-(2 + (oparg-1))]; - #line 511 "Python/bytecodes.c" + #line 513 "Python/bytecodes.c" if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; - #line 726 "Python/generated_cases.c.h" + #line 728 "Python/generated_cases.c.h" STACK_SHRINK(1); PREDICT(JUMP_BACKWARD); DISPATCH(); @@ -731,13 +733,13 @@ TARGET(SET_ADD) { PyObject *v = stack_pointer[-1]; PyObject *set = stack_pointer[-(2 + (oparg-1))]; - #line 516 "Python/bytecodes.c" + #line 518 "Python/bytecodes.c" int err = PySet_Add(set, v); - #line 737 "Python/generated_cases.c.h" + #line 739 "Python/generated_cases.c.h" Py_DECREF(v); - #line 518 "Python/bytecodes.c" + #line 520 "Python/bytecodes.c" if (err) goto pop_1_error; - #line 741 "Python/generated_cases.c.h" + #line 743 "Python/generated_cases.c.h" STACK_SHRINK(1); PREDICT(JUMP_BACKWARD); DISPATCH(); @@ -750,7 +752,7 @@ PyObject *container = stack_pointer[-2]; PyObject *v = stack_pointer[-3]; uint16_t counter = read_u16(&next_instr[0].cache); - #line 529 "Python/bytecodes.c" + #line 531 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr--; @@ -765,13 +767,13 @@ #endif /* ENABLE_SPECIALIZATION */ /* container[sub] = v */ int err = PyObject_SetItem(container, sub, v); - #line 769 "Python/generated_cases.c.h" + #line 771 "Python/generated_cases.c.h" Py_DECREF(v); Py_DECREF(container); Py_DECREF(sub); - #line 544 "Python/bytecodes.c" + #line 546 "Python/bytecodes.c" if (err) goto pop_3_error; - #line 775 "Python/generated_cases.c.h" + #line 777 "Python/generated_cases.c.h" STACK_SHRINK(3); next_instr += 1; DISPATCH(); @@ -781,7 +783,7 @@ PyObject *sub = stack_pointer[-1]; PyObject *list = stack_pointer[-2]; PyObject *value = stack_pointer[-3]; - #line 548 "Python/bytecodes.c" + #line 550 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); @@ -798,7 +800,7 @@ Py_DECREF(old_value); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(list); - #line 802 "Python/generated_cases.c.h" + #line 804 "Python/generated_cases.c.h" STACK_SHRINK(3); next_instr += 1; DISPATCH(); @@ -808,13 +810,13 @@ PyObject *sub = stack_pointer[-1]; PyObject *dict = stack_pointer[-2]; PyObject *value = stack_pointer[-3]; - #line 567 "Python/bytecodes.c" + #line 569 "Python/bytecodes.c" DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); Py_DECREF(dict); if (err) goto pop_3_error; - #line 818 "Python/generated_cases.c.h" + #line 820 "Python/generated_cases.c.h" STACK_SHRINK(3); next_instr += 1; DISPATCH(); @@ -823,15 +825,15 @@ TARGET(DELETE_SUBSCR) { PyObject *sub = stack_pointer[-1]; PyObject *container = stack_pointer[-2]; - #line 575 "Python/bytecodes.c" + #line 577 "Python/bytecodes.c" /* del container[sub] */ int err = PyObject_DelItem(container, sub); - #line 830 "Python/generated_cases.c.h" + #line 832 "Python/generated_cases.c.h" Py_DECREF(container); Py_DECREF(sub); - #line 578 "Python/bytecodes.c" + #line 580 "Python/bytecodes.c" if (err) goto pop_2_error; - #line 835 "Python/generated_cases.c.h" + #line 837 "Python/generated_cases.c.h" STACK_SHRINK(2); DISPATCH(); } @@ -839,14 +841,14 @@ TARGET(CALL_INTRINSIC_1) { PyObject *value = stack_pointer[-1]; PyObject *res; - #line 582 "Python/bytecodes.c" + #line 584 "Python/bytecodes.c" assert(oparg <= MAX_INTRINSIC_1); res = _PyIntrinsics_UnaryFunctions[oparg](tstate, value); - #line 846 "Python/generated_cases.c.h" + #line 848 "Python/generated_cases.c.h" Py_DECREF(value); - #line 585 "Python/bytecodes.c" + #line 587 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; - #line 850 "Python/generated_cases.c.h" + #line 852 "Python/generated_cases.c.h" stack_pointer[-1] = res; DISPATCH(); } @@ -855,15 +857,15 @@ PyObject *value1 = stack_pointer[-1]; PyObject *value2 = stack_pointer[-2]; PyObject *res; - #line 589 "Python/bytecodes.c" + #line 591 "Python/bytecodes.c" assert(oparg <= MAX_INTRINSIC_2); res = _PyIntrinsics_BinaryFunctions[oparg](tstate, value2, value1); - #line 862 "Python/generated_cases.c.h" + #line 864 "Python/generated_cases.c.h" Py_DECREF(value2); Py_DECREF(value1); - #line 592 "Python/bytecodes.c" + #line 594 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 867 "Python/generated_cases.c.h" + #line 869 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); @@ -871,7 +873,7 @@ TARGET(RAISE_VARARGS) { PyObject **args = (stack_pointer - oparg); - #line 596 "Python/bytecodes.c" + #line 598 "Python/bytecodes.c" PyObject *cause = NULL, *exc = NULL; switch (oparg) { case 2: @@ -889,12 +891,12 @@ break; } if (true) { STACK_SHRINK(oparg); goto error; } - #line 893 "Python/generated_cases.c.h" + #line 895 "Python/generated_cases.c.h" } TARGET(INTERPRETER_EXIT) { PyObject *retval = stack_pointer[-1]; - #line 616 "Python/bytecodes.c" + #line 618 "Python/bytecodes.c" assert(frame == &entry_frame); assert(_PyFrame_IsIncomplete(frame)); STACK_SHRINK(1); // Since we're not going to DISPATCH() @@ -905,12 +907,12 @@ assert(!_PyErr_Occurred(tstate)); _Py_LeaveRecursiveCallTstate(tstate); return retval; - #line 909 "Python/generated_cases.c.h" + #line 911 "Python/generated_cases.c.h" } TARGET(RETURN_VALUE) { PyObject *retval = stack_pointer[-1]; - #line 629 "Python/bytecodes.c" + #line 631 "Python/bytecodes.c" STACK_SHRINK(1); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -922,12 +924,12 @@ _PyEvalFrameClearAndPop(tstate, dying); _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 926 "Python/generated_cases.c.h" + #line 928 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_RETURN_VALUE) { PyObject *retval = stack_pointer[-1]; - #line 643 "Python/bytecodes.c" + #line 645 "Python/bytecodes.c" int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, next_instr-1, retval); @@ -943,11 +945,11 @@ _PyEvalFrameClearAndPop(tstate, dying); _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 947 "Python/generated_cases.c.h" + #line 949 "Python/generated_cases.c.h" } TARGET(RETURN_CONST) { - #line 661 "Python/bytecodes.c" + #line 663 "Python/bytecodes.c" PyObject *retval = GETITEM(frame->f_code->co_consts, oparg); Py_INCREF(retval); assert(EMPTY()); @@ -960,11 +962,11 @@ _PyEvalFrameClearAndPop(tstate, dying); _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 964 "Python/generated_cases.c.h" + #line 966 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_RETURN_CONST) { - #line 676 "Python/bytecodes.c" + #line 678 "Python/bytecodes.c" PyObject *retval = GETITEM(frame->f_code->co_consts, oparg); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, @@ -981,13 +983,13 @@ _PyEvalFrameClearAndPop(tstate, dying); _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 985 "Python/generated_cases.c.h" + #line 987 "Python/generated_cases.c.h" } TARGET(GET_AITER) { PyObject *obj = stack_pointer[-1]; PyObject *iter; - #line 695 "Python/bytecodes.c" + #line 697 "Python/bytecodes.c" unaryfunc getter = NULL; PyTypeObject *type = Py_TYPE(obj); @@ -1000,16 +1002,16 @@ "'async for' requires an object with " "__aiter__ method, got %.100s", type->tp_name); - #line 1004 "Python/generated_cases.c.h" + #line 1006 "Python/generated_cases.c.h" Py_DECREF(obj); - #line 708 "Python/bytecodes.c" + #line 710 "Python/bytecodes.c" if (true) goto pop_1_error; } iter = (*getter)(obj); - #line 1011 "Python/generated_cases.c.h" + #line 1013 "Python/generated_cases.c.h" Py_DECREF(obj); - #line 713 "Python/bytecodes.c" + #line 715 "Python/bytecodes.c" if (iter == NULL) goto pop_1_error; if (Py_TYPE(iter)->tp_as_async == NULL || @@ -1022,7 +1024,7 @@ Py_DECREF(iter); if (true) goto pop_1_error; } - #line 1026 "Python/generated_cases.c.h" + #line 1028 "Python/generated_cases.c.h" stack_pointer[-1] = iter; DISPATCH(); } @@ -1030,7 +1032,7 @@ TARGET(GET_ANEXT) { PyObject *aiter = stack_pointer[-1]; PyObject *awaitable; - #line 728 "Python/bytecodes.c" + #line 730 "Python/bytecodes.c" unaryfunc getter = NULL; PyObject *next_iter = NULL; PyTypeObject *type = Py_TYPE(aiter); @@ -1074,7 +1076,7 @@ } } - #line 1078 "Python/generated_cases.c.h" + #line 1080 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = awaitable; PREDICT(LOAD_CONST); @@ -1085,16 +1087,16 @@ PREDICTED(GET_AWAITABLE); PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 775 "Python/bytecodes.c" + #line 777 "Python/bytecodes.c" iter = _PyCoro_GetAwaitableIter(iterable); if (iter == NULL) { format_awaitable_error(tstate, Py_TYPE(iterable), oparg); } - #line 1096 "Python/generated_cases.c.h" + #line 1098 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 782 "Python/bytecodes.c" + #line 784 "Python/bytecodes.c" if (iter != NULL && PyCoro_CheckExact(iter)) { PyObject *yf = _PyGen_yf((PyGenObject*)iter); @@ -1112,7 +1114,7 @@ if (iter == NULL) goto pop_1_error; - #line 1116 "Python/generated_cases.c.h" + #line 1118 "Python/generated_cases.c.h" stack_pointer[-1] = iter; PREDICT(LOAD_CONST); DISPATCH(); @@ -1123,7 +1125,7 @@ PyObject *v = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; PyObject *retval; - #line 808 "Python/bytecodes.c" + #line 810 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PySendCache *cache = (_PySendCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1169,7 +1171,7 @@ } } Py_DECREF(v); - #line 1173 "Python/generated_cases.c.h" + #line 1175 "Python/generated_cases.c.h" stack_pointer[-1] = retval; next_instr += 1; DISPATCH(); @@ -1178,7 +1180,7 @@ TARGET(SEND_GEN) { PyObject *v = stack_pointer[-1]; PyObject *receiver = stack_pointer[-2]; - #line 856 "Python/bytecodes.c" + #line 858 "Python/bytecodes.c" PyGenObject *gen = (PyGenObject *)receiver; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); @@ -1193,12 +1195,12 @@ tstate->exc_info = &gen->gi_exc_state; JUMPBY(INLINE_CACHE_ENTRIES_SEND + oparg); DISPATCH_INLINED(gen_frame); - #line 1197 "Python/generated_cases.c.h" + #line 1199 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_YIELD_VALUE) { PyObject *retval = stack_pointer[-1]; - #line 873 "Python/bytecodes.c" + #line 875 "Python/bytecodes.c" assert(frame != &entry_frame); PyGenObject *gen = _PyFrame_GetGenerator(frame); gen->gi_frame_state = FRAME_SUSPENDED; @@ -1216,12 +1218,12 @@ frame->prev_instr -= frame->yield_offset; _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 1220 "Python/generated_cases.c.h" + #line 1222 "Python/generated_cases.c.h" } TARGET(YIELD_VALUE) { PyObject *retval = stack_pointer[-1]; - #line 893 "Python/bytecodes.c" + #line 895 "Python/bytecodes.c" // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. @@ -1238,15 +1240,15 @@ frame->prev_instr -= frame->yield_offset; _PyFrame_StackPush(frame, retval); goto resume_frame; - #line 1242 "Python/generated_cases.c.h" + #line 1244 "Python/generated_cases.c.h" } TARGET(POP_EXCEPT) { PyObject *exc_value = stack_pointer[-1]; - #line 912 "Python/bytecodes.c" + #line 914 "Python/bytecodes.c" _PyErr_StackItem *exc_info = tstate->exc_info; Py_XSETREF(exc_info->exc_value, exc_value); - #line 1250 "Python/generated_cases.c.h" + #line 1252 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -1254,7 +1256,7 @@ TARGET(RERAISE) { PyObject *exc = stack_pointer[-1]; PyObject **values = (stack_pointer - (1 + oparg)); - #line 917 "Python/bytecodes.c" + #line 919 "Python/bytecodes.c" assert(oparg >= 0 && oparg <= 2); if (oparg) { PyObject *lasti = values[0]; @@ -1272,26 +1274,26 @@ Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); goto exception_unwind; - #line 1276 "Python/generated_cases.c.h" + #line 1278 "Python/generated_cases.c.h" } TARGET(END_ASYNC_FOR) { PyObject *exc = stack_pointer[-1]; PyObject *awaitable = stack_pointer[-2]; - #line 937 "Python/bytecodes.c" + #line 939 "Python/bytecodes.c" assert(exc && PyExceptionInstance_Check(exc)); if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { - #line 1285 "Python/generated_cases.c.h" + #line 1287 "Python/generated_cases.c.h" Py_DECREF(awaitable); Py_DECREF(exc); - #line 940 "Python/bytecodes.c" + #line 942 "Python/bytecodes.c" } else { Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); goto exception_unwind; } - #line 1295 "Python/generated_cases.c.h" + #line 1297 "Python/generated_cases.c.h" STACK_SHRINK(2); DISPATCH(); } @@ -1302,23 +1304,23 @@ PyObject *sub_iter = stack_pointer[-3]; PyObject *none; PyObject *value; - #line 949 "Python/bytecodes.c" + #line 951 "Python/bytecodes.c" assert(throwflag); assert(exc_value && PyExceptionInstance_Check(exc_value)); if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { value = Py_NewRef(((PyStopIterationObject *)exc_value)->value); - #line 1311 "Python/generated_cases.c.h" + #line 1313 "Python/generated_cases.c.h" Py_DECREF(sub_iter); Py_DECREF(last_sent_val); Py_DECREF(exc_value); - #line 954 "Python/bytecodes.c" + #line 956 "Python/bytecodes.c" none = Py_NewRef(Py_None); } else { _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); goto exception_unwind; } - #line 1322 "Python/generated_cases.c.h" + #line 1324 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = value; stack_pointer[-2] = none; @@ -1327,9 +1329,9 @@ TARGET(LOAD_ASSERTION_ERROR) { PyObject *value; - #line 963 "Python/bytecodes.c" + #line 965 "Python/bytecodes.c" value = Py_NewRef(PyExc_AssertionError); - #line 1333 "Python/generated_cases.c.h" + #line 1335 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -1337,7 +1339,7 @@ TARGET(LOAD_BUILD_CLASS) { PyObject *bc; - #line 967 "Python/bytecodes.c" + #line 969 "Python/bytecodes.c" if (PyDict_CheckExact(BUILTINS())) { bc = _PyDict_GetItemWithError(BUILTINS(), &_Py_ID(__build_class__)); @@ -1359,7 +1361,7 @@ if (true) goto error; } } - #line 1363 "Python/generated_cases.c.h" + #line 1365 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = bc; DISPATCH(); @@ -1367,33 +1369,33 @@ TARGET(STORE_NAME) { PyObject *v = stack_pointer[-1]; - #line 991 "Python/bytecodes.c" + #line 993 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); PyObject *ns = LOCALS(); int err; if (ns == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals found when storing %R", name); - #line 1378 "Python/generated_cases.c.h" + #line 1380 "Python/generated_cases.c.h" Py_DECREF(v); - #line 998 "Python/bytecodes.c" + #line 1000 "Python/bytecodes.c" if (true) goto pop_1_error; } if (PyDict_CheckExact(ns)) err = PyDict_SetItem(ns, name, v); else err = PyObject_SetItem(ns, name, v); - #line 1387 "Python/generated_cases.c.h" + #line 1389 "Python/generated_cases.c.h" Py_DECREF(v); - #line 1005 "Python/bytecodes.c" + #line 1007 "Python/bytecodes.c" if (err) goto pop_1_error; - #line 1391 "Python/generated_cases.c.h" + #line 1393 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(DELETE_NAME) { - #line 1009 "Python/bytecodes.c" + #line 1011 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); PyObject *ns = LOCALS(); int err; @@ -1410,7 +1412,7 @@ name); goto error; } - #line 1414 "Python/generated_cases.c.h" + #line 1416 "Python/generated_cases.c.h" DISPATCH(); } @@ -1418,7 +1420,7 @@ PREDICTED(UNPACK_SEQUENCE); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); PyObject *seq = stack_pointer[-1]; - #line 1035 "Python/bytecodes.c" + #line 1037 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1431,11 +1433,11 @@ #endif /* ENABLE_SPECIALIZATION */ PyObject **top = stack_pointer + oparg - 1; int res = unpack_iterable(tstate, seq, oparg, -1, top); - #line 1435 "Python/generated_cases.c.h" + #line 1437 "Python/generated_cases.c.h" Py_DECREF(seq); - #line 1048 "Python/bytecodes.c" + #line 1050 "Python/bytecodes.c" if (res == 0) goto pop_1_error; - #line 1439 "Python/generated_cases.c.h" + #line 1441 "Python/generated_cases.c.h" STACK_SHRINK(1); STACK_GROW(oparg); next_instr += 1; @@ -1445,14 +1447,14 @@ TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { PyObject *seq = stack_pointer[-1]; PyObject **values = stack_pointer - (1); - #line 1052 "Python/bytecodes.c" + #line 1054 "Python/bytecodes.c" DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); assert(oparg == 2); STAT_INC(UNPACK_SEQUENCE, hit); values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); - #line 1456 "Python/generated_cases.c.h" + #line 1458 "Python/generated_cases.c.h" Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); @@ -1463,7 +1465,7 @@ TARGET(UNPACK_SEQUENCE_TUPLE) { PyObject *seq = stack_pointer[-1]; PyObject **values = stack_pointer - (1); - #line 1062 "Python/bytecodes.c" + #line 1064 "Python/bytecodes.c" DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); @@ -1471,7 +1473,7 @@ for (int i = oparg; --i >= 0; ) { *values++ = Py_NewRef(items[i]); } - #line 1475 "Python/generated_cases.c.h" + #line 1477 "Python/generated_cases.c.h" Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); @@ -1482,7 +1484,7 @@ TARGET(UNPACK_SEQUENCE_LIST) { PyObject *seq = stack_pointer[-1]; PyObject **values = stack_pointer - (1); - #line 1073 "Python/bytecodes.c" + #line 1075 "Python/bytecodes.c" DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); @@ -1490,7 +1492,7 @@ for (int i = oparg; --i >= 0; ) { *values++ = Py_NewRef(items[i]); } - #line 1494 "Python/generated_cases.c.h" + #line 1496 "Python/generated_cases.c.h" Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); @@ -1500,15 +1502,15 @@ TARGET(UNPACK_EX) { PyObject *seq = stack_pointer[-1]; - #line 1084 "Python/bytecodes.c" + #line 1086 "Python/bytecodes.c" int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); PyObject **top = stack_pointer + totalargs - 1; int res = unpack_iterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); - #line 1508 "Python/generated_cases.c.h" + #line 1510 "Python/generated_cases.c.h" Py_DECREF(seq); - #line 1088 "Python/bytecodes.c" + #line 1090 "Python/bytecodes.c" if (res == 0) goto pop_1_error; - #line 1512 "Python/generated_cases.c.h" + #line 1514 "Python/generated_cases.c.h" STACK_GROW((oparg & 0xFF) + (oparg >> 8)); DISPATCH(); } @@ -1519,7 +1521,7 @@ PyObject *owner = stack_pointer[-1]; PyObject *v = stack_pointer[-2]; uint16_t counter = read_u16(&next_instr[0].cache); - #line 1099 "Python/bytecodes.c" + #line 1101 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { PyObject *name = GETITEM(frame->f_code->co_names, oparg); @@ -1535,12 +1537,12 @@ #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyObject_SetAttr(owner, name, v); - #line 1539 "Python/generated_cases.c.h" + #line 1541 "Python/generated_cases.c.h" Py_DECREF(v); Py_DECREF(owner); - #line 1115 "Python/bytecodes.c" + #line 1117 "Python/bytecodes.c" if (err) goto pop_2_error; - #line 1544 "Python/generated_cases.c.h" + #line 1546 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -1548,34 +1550,34 @@ TARGET(DELETE_ATTR) { PyObject *owner = stack_pointer[-1]; - #line 1119 "Python/bytecodes.c" + #line 1121 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyObject_SetAttr(owner, name, (PyObject *)NULL); - #line 1555 "Python/generated_cases.c.h" + #line 1557 "Python/generated_cases.c.h" Py_DECREF(owner); - #line 1122 "Python/bytecodes.c" + #line 1124 "Python/bytecodes.c" if (err) goto pop_1_error; - #line 1559 "Python/generated_cases.c.h" + #line 1561 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(STORE_GLOBAL) { PyObject *v = stack_pointer[-1]; - #line 1126 "Python/bytecodes.c" + #line 1128 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err = PyDict_SetItem(GLOBALS(), name, v); - #line 1569 "Python/generated_cases.c.h" + #line 1571 "Python/generated_cases.c.h" Py_DECREF(v); - #line 1129 "Python/bytecodes.c" + #line 1131 "Python/bytecodes.c" if (err) goto pop_1_error; - #line 1573 "Python/generated_cases.c.h" + #line 1575 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(DELETE_GLOBAL) { - #line 1133 "Python/bytecodes.c" + #line 1135 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); int err; err = PyDict_DelItem(GLOBALS(), name); @@ -1587,13 +1589,13 @@ } goto error; } - #line 1591 "Python/generated_cases.c.h" + #line 1593 "Python/generated_cases.c.h" DISPATCH(); } TARGET(LOAD_NAME) { PyObject *v; - #line 1147 "Python/bytecodes.c" + #line 1149 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); PyObject *locals = LOCALS(); if (locals == NULL) { @@ -1652,7 +1654,7 @@ } } } - #line 1656 "Python/generated_cases.c.h" + #line 1658 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = v; DISPATCH(); @@ -1663,7 +1665,7 @@ static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyObject *null = NULL; PyObject *v; - #line 1214 "Python/bytecodes.c" + #line 1216 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1715,7 +1717,7 @@ } } null = NULL; - #line 1719 "Python/generated_cases.c.h" + #line 1721 "Python/generated_cases.c.h" STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = v; @@ -1729,7 +1731,7 @@ PyObject *res; uint16_t index = read_u16(&next_instr[1].cache); uint16_t version = read_u16(&next_instr[2].cache); - #line 1268 "Python/bytecodes.c" + #line 1270 "Python/bytecodes.c" DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); PyDictObject *dict = (PyDictObject *)GLOBALS(); DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); @@ -1740,7 +1742,7 @@ Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; - #line 1744 "Python/generated_cases.c.h" + #line 1746 "Python/generated_cases.c.h" STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -1755,7 +1757,7 @@ uint16_t index = read_u16(&next_instr[1].cache); uint16_t mod_version = read_u16(&next_instr[2].cache); uint16_t bltn_version = read_u16(&next_instr[3].cache); - #line 1281 "Python/bytecodes.c" + #line 1283 "Python/bytecodes.c" DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL); DEOPT_IF(!PyDict_CheckExact(BUILTINS()), LOAD_GLOBAL); PyDictObject *mdict = (PyDictObject *)GLOBALS(); @@ -1770,7 +1772,7 @@ Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; - #line 1774 "Python/generated_cases.c.h" + #line 1776 "Python/generated_cases.c.h" STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -1780,16 +1782,16 @@ } TARGET(DELETE_FAST) { - #line 1298 "Python/bytecodes.c" + #line 1300 "Python/bytecodes.c" PyObject *v = GETLOCAL(oparg); if (v == NULL) goto unbound_local_error; SETLOCAL(oparg, NULL); - #line 1788 "Python/generated_cases.c.h" + #line 1790 "Python/generated_cases.c.h" DISPATCH(); } TARGET(MAKE_CELL) { - #line 1304 "Python/bytecodes.c" + #line 1306 "Python/bytecodes.c" // "initial" is probably NULL but not if it's an arg (or set // via PyFrame_LocalsToFast() before MAKE_CELL has run). PyObject *initial = GETLOCAL(oparg); @@ -1798,12 +1800,12 @@ goto resume_with_error; } SETLOCAL(oparg, cell); - #line 1802 "Python/generated_cases.c.h" + #line 1804 "Python/generated_cases.c.h" DISPATCH(); } TARGET(DELETE_DEREF) { - #line 1315 "Python/bytecodes.c" + #line 1317 "Python/bytecodes.c" PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); // Can't use ERROR_IF here. @@ -1814,13 +1816,13 @@ } PyCell_SET(cell, NULL); Py_DECREF(oldobj); - #line 1818 "Python/generated_cases.c.h" + #line 1820 "Python/generated_cases.c.h" DISPATCH(); } TARGET(LOAD_CLASSDEREF) { PyObject *value; - #line 1328 "Python/bytecodes.c" + #line 1330 "Python/bytecodes.c" PyObject *name, *locals = LOCALS(); assert(locals); assert(oparg >= 0 && oparg < frame->f_code->co_nlocalsplus); @@ -1852,7 +1854,7 @@ } Py_INCREF(value); } - #line 1856 "Python/generated_cases.c.h" + #line 1858 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -1860,7 +1862,7 @@ TARGET(LOAD_DEREF) { PyObject *value; - #line 1362 "Python/bytecodes.c" + #line 1364 "Python/bytecodes.c" PyObject *cell = GETLOCAL(oparg); value = PyCell_GET(cell); if (value == NULL) { @@ -1868,7 +1870,7 @@ if (true) goto error; } Py_INCREF(value); - #line 1872 "Python/generated_cases.c.h" + #line 1874 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -1876,18 +1878,18 @@ TARGET(STORE_DEREF) { PyObject *v = stack_pointer[-1]; - #line 1372 "Python/bytecodes.c" + #line 1374 "Python/bytecodes.c" PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); PyCell_SET(cell, v); Py_XDECREF(oldobj); - #line 1885 "Python/generated_cases.c.h" + #line 1887 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(COPY_FREE_VARS) { - #line 1379 "Python/bytecodes.c" + #line 1381 "Python/bytecodes.c" /* Copy closure variables to free variables */ PyCodeObject *co = frame->f_code; assert(PyFunction_Check(frame->f_funcobj)); @@ -1898,22 +1900,22 @@ PyObject *o = PyTuple_GET_ITEM(closure, i); frame->localsplus[offset + i] = Py_NewRef(o); } - #line 1902 "Python/generated_cases.c.h" + #line 1904 "Python/generated_cases.c.h" DISPATCH(); } TARGET(BUILD_STRING) { PyObject **pieces = (stack_pointer - oparg); PyObject *str; - #line 1392 "Python/bytecodes.c" + #line 1394 "Python/bytecodes.c" str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); - #line 1911 "Python/generated_cases.c.h" + #line 1913 "Python/generated_cases.c.h" for (int _i = oparg; --_i >= 0;) { Py_DECREF(pieces[_i]); } - #line 1394 "Python/bytecodes.c" + #line 1396 "Python/bytecodes.c" if (str == NULL) { STACK_SHRINK(oparg); goto error; } - #line 1917 "Python/generated_cases.c.h" + #line 1919 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = str; @@ -1923,10 +1925,10 @@ TARGET(BUILD_TUPLE) { PyObject **values = (stack_pointer - oparg); PyObject *tup; - #line 1398 "Python/bytecodes.c" + #line 1400 "Python/bytecodes.c" tup = _PyTuple_FromArraySteal(values, oparg); if (tup == NULL) { STACK_SHRINK(oparg); goto error; } - #line 1930 "Python/generated_cases.c.h" + #line 1932 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = tup; @@ -1936,10 +1938,10 @@ TARGET(BUILD_LIST) { PyObject **values = (stack_pointer - oparg); PyObject *list; - #line 1403 "Python/bytecodes.c" + #line 1405 "Python/bytecodes.c" list = _PyList_FromArraySteal(values, oparg); if (list == NULL) { STACK_SHRINK(oparg); goto error; } - #line 1943 "Python/generated_cases.c.h" + #line 1945 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = list; @@ -1949,7 +1951,7 @@ TARGET(LIST_EXTEND) { PyObject *iterable = stack_pointer[-1]; PyObject *list = stack_pointer[-(2 + (oparg-1))]; - #line 1408 "Python/bytecodes.c" + #line 1410 "Python/bytecodes.c" PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); if (none_val == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && @@ -1960,13 +1962,13 @@ "Value after * must be an iterable, not %.200s", Py_TYPE(iterable)->tp_name); } - #line 1964 "Python/generated_cases.c.h" + #line 1966 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 1419 "Python/bytecodes.c" + #line 1421 "Python/bytecodes.c" if (true) goto pop_1_error; } Py_DECREF(none_val); - #line 1970 "Python/generated_cases.c.h" + #line 1972 "Python/generated_cases.c.h" Py_DECREF(iterable); STACK_SHRINK(1); DISPATCH(); @@ -1975,13 +1977,13 @@ TARGET(SET_UPDATE) { PyObject *iterable = stack_pointer[-1]; PyObject *set = stack_pointer[-(2 + (oparg-1))]; - #line 1426 "Python/bytecodes.c" + #line 1428 "Python/bytecodes.c" int err = _PySet_Update(set, iterable); - #line 1981 "Python/generated_cases.c.h" + #line 1983 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 1428 "Python/bytecodes.c" + #line 1430 "Python/bytecodes.c" if (err < 0) goto pop_1_error; - #line 1985 "Python/generated_cases.c.h" + #line 1987 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -1989,7 +1991,7 @@ TARGET(BUILD_SET) { PyObject **values = (stack_pointer - oparg); PyObject *set; - #line 1432 "Python/bytecodes.c" + #line 1434 "Python/bytecodes.c" set = PySet_New(NULL); if (set == NULL) goto error; @@ -2004,7 +2006,7 @@ Py_DECREF(set); if (true) { STACK_SHRINK(oparg); goto error; } } - #line 2008 "Python/generated_cases.c.h" + #line 2010 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = set; @@ -2014,7 +2016,7 @@ TARGET(BUILD_MAP) { PyObject **values = (stack_pointer - oparg*2); PyObject *map; - #line 1449 "Python/bytecodes.c" + #line 1451 "Python/bytecodes.c" map = _PyDict_FromItems( values, 2, values+1, 2, @@ -2022,13 +2024,13 @@ if (map == NULL) goto error; - #line 2026 "Python/generated_cases.c.h" + #line 2028 "Python/generated_cases.c.h" for (int _i = oparg*2; --_i >= 0;) { Py_DECREF(values[_i]); } - #line 1457 "Python/bytecodes.c" + #line 1459 "Python/bytecodes.c" if (map == NULL) { STACK_SHRINK(oparg*2); goto error; } - #line 2032 "Python/generated_cases.c.h" + #line 2034 "Python/generated_cases.c.h" STACK_SHRINK(oparg*2); STACK_GROW(1); stack_pointer[-1] = map; @@ -2036,7 +2038,7 @@ } TARGET(SETUP_ANNOTATIONS) { - #line 1461 "Python/bytecodes.c" + #line 1463 "Python/bytecodes.c" int err; PyObject *ann_dict; if (LOCALS() == NULL) { @@ -2076,7 +2078,7 @@ Py_DECREF(ann_dict); } } - #line 2080 "Python/generated_cases.c.h" + #line 2082 "Python/generated_cases.c.h" DISPATCH(); } @@ -2084,7 +2086,7 @@ PyObject *keys = stack_pointer[-1]; PyObject **values = (stack_pointer - (1 + oparg)); PyObject *map; - #line 1503 "Python/bytecodes.c" + #line 1505 "Python/bytecodes.c" if (!PyTuple_CheckExact(keys) || PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { _PyErr_SetString(tstate, PyExc_SystemError, @@ -2094,14 +2096,14 @@ map = _PyDict_FromItems( &PyTuple_GET_ITEM(keys, 0), 1, values, 1, oparg); - #line 2098 "Python/generated_cases.c.h" + #line 2100 "Python/generated_cases.c.h" for (int _i = oparg; --_i >= 0;) { Py_DECREF(values[_i]); } Py_DECREF(keys); - #line 1513 "Python/bytecodes.c" + #line 1515 "Python/bytecodes.c" if (map == NULL) { STACK_SHRINK(oparg); goto pop_1_error; } - #line 2105 "Python/generated_cases.c.h" + #line 2107 "Python/generated_cases.c.h" STACK_SHRINK(oparg); stack_pointer[-1] = map; DISPATCH(); @@ -2109,7 +2111,7 @@ TARGET(DICT_UPDATE) { PyObject *update = stack_pointer[-1]; - #line 1517 "Python/bytecodes.c" + #line 1519 "Python/bytecodes.c" PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (PyDict_Update(dict, update) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { @@ -2117,12 +2119,12 @@ "'%.200s' object is not a mapping", Py_TYPE(update)->tp_name); } - #line 2121 "Python/generated_cases.c.h" + #line 2123 "Python/generated_cases.c.h" Py_DECREF(update); - #line 1525 "Python/bytecodes.c" + #line 1527 "Python/bytecodes.c" if (true) goto pop_1_error; } - #line 2126 "Python/generated_cases.c.h" + #line 2128 "Python/generated_cases.c.h" Py_DECREF(update); STACK_SHRINK(1); DISPATCH(); @@ -2130,17 +2132,17 @@ TARGET(DICT_MERGE) { PyObject *update = stack_pointer[-1]; - #line 1531 "Python/bytecodes.c" + #line 1533 "Python/bytecodes.c" PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (_PyDict_MergeEx(dict, update, 2) < 0) { format_kwargs_error(tstate, PEEK(3 + oparg), update); - #line 2139 "Python/generated_cases.c.h" + #line 2141 "Python/generated_cases.c.h" Py_DECREF(update); - #line 1536 "Python/bytecodes.c" + #line 1538 "Python/bytecodes.c" if (true) goto pop_1_error; } - #line 2144 "Python/generated_cases.c.h" + #line 2146 "Python/generated_cases.c.h" Py_DECREF(update); STACK_SHRINK(1); PREDICT(CALL_FUNCTION_EX); @@ -2150,13 +2152,13 @@ TARGET(MAP_ADD) { PyObject *value = stack_pointer[-1]; PyObject *key = stack_pointer[-2]; - #line 1543 "Python/bytecodes.c" + #line 1545 "Python/bytecodes.c" PyObject *dict = PEEK(oparg + 2); // key, value are still on the stack assert(PyDict_CheckExact(dict)); /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; - #line 2160 "Python/generated_cases.c.h" + #line 2162 "Python/generated_cases.c.h" STACK_SHRINK(2); PREDICT(JUMP_BACKWARD); DISPATCH(); @@ -2168,7 +2170,7 @@ PyObject *owner = stack_pointer[-1]; PyObject *res2 = NULL; PyObject *res; - #line 1566 "Python/bytecodes.c" + #line 1568 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -2202,9 +2204,9 @@ NULL | meth | arg1 | ... | argN */ - #line 2206 "Python/generated_cases.c.h" + #line 2208 "Python/generated_cases.c.h" Py_DECREF(owner); - #line 1600 "Python/bytecodes.c" + #line 1602 "Python/bytecodes.c" if (meth == NULL) goto pop_1_error; res2 = NULL; res = meth; @@ -2213,12 +2215,12 @@ else { /* Classic, pushes one value. */ res = PyObject_GetAttr(owner, name); - #line 2217 "Python/generated_cases.c.h" + #line 2219 "Python/generated_cases.c.h" Py_DECREF(owner); - #line 1609 "Python/bytecodes.c" + #line 1611 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; } - #line 2222 "Python/generated_cases.c.h" + #line 2224 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -2232,7 +2234,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1614 "Python/bytecodes.c" + #line 1616 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2245,7 +2247,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2249 "Python/generated_cases.c.h" + #line 2251 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2260,7 +2262,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1630 "Python/bytecodes.c" + #line 1632 "Python/bytecodes.c" DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict != NULL); @@ -2273,7 +2275,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2277 "Python/generated_cases.c.h" + #line 2279 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2288,7 +2290,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1646 "Python/bytecodes.c" + #line 1648 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2315,7 +2317,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2319 "Python/generated_cases.c.h" + #line 2321 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2330,7 +2332,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1676 "Python/bytecodes.c" + #line 1678 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2340,7 +2342,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2344 "Python/generated_cases.c.h" + #line 2346 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2355,7 +2357,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 1689 "Python/bytecodes.c" + #line 1691 "Python/bytecodes.c" DEOPT_IF(!PyType_Check(cls), LOAD_ATTR); DEOPT_IF(((PyTypeObject *)cls)->tp_version_tag != type_version, @@ -2367,7 +2369,7 @@ res = descr; assert(res != NULL); Py_INCREF(res); - #line 2371 "Python/generated_cases.c.h" + #line 2373 "Python/generated_cases.c.h" Py_DECREF(cls); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2381,7 +2383,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t func_version = read_u32(&next_instr[3].cache); PyObject *fget = read_obj(&next_instr[5].cache); - #line 1704 "Python/bytecodes.c" + #line 1706 "Python/bytecodes.c" DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); @@ -2404,7 +2406,7 @@ new_frame->localsplus[0] = owner; JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); DISPATCH_INLINED(new_frame); - #line 2408 "Python/generated_cases.c.h" + #line 2410 "Python/generated_cases.c.h" } TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { @@ -2412,7 +2414,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t func_version = read_u32(&next_instr[3].cache); PyObject *getattribute = read_obj(&next_instr[5].cache); - #line 1729 "Python/bytecodes.c" + #line 1731 "Python/bytecodes.c" DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); @@ -2437,7 +2439,7 @@ new_frame->localsplus[1] = Py_NewRef(name); JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); DISPATCH_INLINED(new_frame); - #line 2441 "Python/generated_cases.c.h" + #line 2443 "Python/generated_cases.c.h" } TARGET(STORE_ATTR_INSTANCE_VALUE) { @@ -2445,7 +2447,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1756 "Python/bytecodes.c" + #line 1758 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2463,7 +2465,7 @@ Py_DECREF(old_value); } Py_DECREF(owner); - #line 2467 "Python/generated_cases.c.h" + #line 2469 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2474,7 +2476,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t hint = read_u16(&next_instr[3].cache); - #line 1776 "Python/bytecodes.c" + #line 1778 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2513,7 +2515,7 @@ /* PEP 509 */ dict->ma_version_tag = new_version; Py_DECREF(owner); - #line 2517 "Python/generated_cases.c.h" + #line 2519 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2524,7 +2526,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1817 "Python/bytecodes.c" + #line 1819 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2534,7 +2536,7 @@ *(PyObject **)addr = value; Py_XDECREF(old_value); Py_DECREF(owner); - #line 2538 "Python/generated_cases.c.h" + #line 2540 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2546,7 +2548,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1836 "Python/bytecodes.c" + #line 1838 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -2559,12 +2561,12 @@ #endif /* ENABLE_SPECIALIZATION */ assert((oparg >> 4) <= Py_GE); res = PyObject_RichCompare(left, right, oparg>>4); - #line 2563 "Python/generated_cases.c.h" + #line 2565 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 1849 "Python/bytecodes.c" + #line 1851 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 2568 "Python/generated_cases.c.h" + #line 2570 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2575,7 +2577,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1853 "Python/bytecodes.c" + #line 1855 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); @@ -2587,7 +2589,7 @@ _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); res = (sign_ish & oparg) ? Py_True : Py_False; Py_INCREF(res); - #line 2591 "Python/generated_cases.c.h" + #line 2593 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2598,7 +2600,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1868 "Python/bytecodes.c" + #line 1870 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); @@ -2614,7 +2616,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); res = (sign_ish & oparg) ? Py_True : Py_False; Py_INCREF(res); - #line 2618 "Python/generated_cases.c.h" + #line 2620 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2625,7 +2627,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 1887 "Python/bytecodes.c" + #line 1889 "Python/bytecodes.c" DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); @@ -2638,7 +2640,7 @@ assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; Py_INCREF(res); - #line 2642 "Python/generated_cases.c.h" + #line 2644 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2649,14 +2651,14 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 1902 "Python/bytecodes.c" + #line 1904 "Python/bytecodes.c" int res = Py_Is(left, right) ^ oparg; - #line 2655 "Python/generated_cases.c.h" + #line 2657 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 1904 "Python/bytecodes.c" + #line 1906 "Python/bytecodes.c" b = Py_NewRef(res ? Py_True : Py_False); - #line 2660 "Python/generated_cases.c.h" + #line 2662 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = b; DISPATCH(); @@ -2666,15 +2668,15 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 1908 "Python/bytecodes.c" + #line 1910 "Python/bytecodes.c" int res = PySequence_Contains(right, left); - #line 2672 "Python/generated_cases.c.h" + #line 2674 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 1910 "Python/bytecodes.c" + #line 1912 "Python/bytecodes.c" if (res < 0) goto pop_2_error; b = Py_NewRef((res^oparg) ? Py_True : Py_False); - #line 2678 "Python/generated_cases.c.h" + #line 2680 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = b; DISPATCH(); @@ -2685,12 +2687,12 @@ PyObject *exc_value = stack_pointer[-2]; PyObject *rest; PyObject *match; - #line 1915 "Python/bytecodes.c" + #line 1917 "Python/bytecodes.c" if (check_except_star_type_valid(tstate, match_type) < 0) { - #line 2691 "Python/generated_cases.c.h" + #line 2693 "Python/generated_cases.c.h" Py_DECREF(exc_value); Py_DECREF(match_type); - #line 1917 "Python/bytecodes.c" + #line 1919 "Python/bytecodes.c" if (true) goto pop_2_error; } @@ -2698,10 +2700,10 @@ rest = NULL; int res = exception_group_match(exc_value, match_type, &match, &rest); - #line 2702 "Python/generated_cases.c.h" + #line 2704 "Python/generated_cases.c.h" Py_DECREF(exc_value); Py_DECREF(match_type); - #line 1925 "Python/bytecodes.c" + #line 1927 "Python/bytecodes.c" if (res < 0) goto pop_2_error; assert((match == NULL) == (rest == NULL)); @@ -2710,7 +2712,7 @@ if (!Py_IsNone(match)) { PyErr_SetHandledException(match); } - #line 2714 "Python/generated_cases.c.h" + #line 2716 "Python/generated_cases.c.h" stack_pointer[-1] = match; stack_pointer[-2] = rest; DISPATCH(); @@ -2720,21 +2722,21 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 1936 "Python/bytecodes.c" + #line 1938 "Python/bytecodes.c" assert(PyExceptionInstance_Check(left)); if (check_except_type_valid(tstate, right) < 0) { - #line 2727 "Python/generated_cases.c.h" + #line 2729 "Python/generated_cases.c.h" Py_DECREF(right); - #line 1939 "Python/bytecodes.c" + #line 1941 "Python/bytecodes.c" if (true) goto pop_1_error; } int res = PyErr_GivenExceptionMatches(left, right); - #line 2734 "Python/generated_cases.c.h" + #line 2736 "Python/generated_cases.c.h" Py_DECREF(right); - #line 1944 "Python/bytecodes.c" + #line 1946 "Python/bytecodes.c" b = Py_NewRef(res ? Py_True : Py_False); - #line 2738 "Python/generated_cases.c.h" + #line 2740 "Python/generated_cases.c.h" stack_pointer[-1] = b; DISPATCH(); } @@ -2743,15 +2745,15 @@ PyObject *fromlist = stack_pointer[-1]; PyObject *level = stack_pointer[-2]; PyObject *res; - #line 1948 "Python/bytecodes.c" + #line 1950 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_name(tstate, frame, name, fromlist, level); - #line 2750 "Python/generated_cases.c.h" + #line 2752 "Python/generated_cases.c.h" Py_DECREF(level); Py_DECREF(fromlist); - #line 1951 "Python/bytecodes.c" + #line 1953 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 2755 "Python/generated_cases.c.h" + #line 2757 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); @@ -2760,29 +2762,29 @@ TARGET(IMPORT_FROM) { PyObject *from = stack_pointer[-1]; PyObject *res; - #line 1955 "Python/bytecodes.c" + #line 1957 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_from(tstate, from, name); if (res == NULL) goto error; - #line 2768 "Python/generated_cases.c.h" + #line 2770 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); } TARGET(JUMP_FORWARD) { - #line 1961 "Python/bytecodes.c" + #line 1963 "Python/bytecodes.c" JUMPBY(oparg); - #line 2777 "Python/generated_cases.c.h" + #line 2779 "Python/generated_cases.c.h" DISPATCH(); } TARGET(JUMP_BACKWARD) { PREDICTED(JUMP_BACKWARD); - #line 1965 "Python/bytecodes.c" + #line 1967 "Python/bytecodes.c" assert(oparg < INSTR_OFFSET()); JUMPBY(-oparg); - #line 2786 "Python/generated_cases.c.h" + #line 2788 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -2790,7 +2792,7 @@ TARGET(POP_JUMP_IF_FALSE) { PREDICTED(POP_JUMP_IF_FALSE); PyObject *cond = stack_pointer[-1]; - #line 1971 "Python/bytecodes.c" + #line 1973 "Python/bytecodes.c" if (Py_IsTrue(cond)) { _Py_DECREF_NO_DEALLOC(cond); } @@ -2800,9 +2802,9 @@ } else { int err = PyObject_IsTrue(cond); - #line 2804 "Python/generated_cases.c.h" + #line 2806 "Python/generated_cases.c.h" Py_DECREF(cond); - #line 1981 "Python/bytecodes.c" + #line 1983 "Python/bytecodes.c" if (err == 0) { JUMPBY(oparg); } @@ -2810,14 +2812,14 @@ if (err < 0) goto pop_1_error; } } - #line 2814 "Python/generated_cases.c.h" + #line 2816 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_TRUE) { PyObject *cond = stack_pointer[-1]; - #line 1991 "Python/bytecodes.c" + #line 1993 "Python/bytecodes.c" if (Py_IsFalse(cond)) { _Py_DECREF_NO_DEALLOC(cond); } @@ -2827,9 +2829,9 @@ } else { int err = PyObject_IsTrue(cond); - #line 2831 "Python/generated_cases.c.h" + #line 2833 "Python/generated_cases.c.h" Py_DECREF(cond); - #line 2001 "Python/bytecodes.c" + #line 2003 "Python/bytecodes.c" if (err > 0) { JUMPBY(oparg); } @@ -2837,67 +2839,67 @@ if (err < 0) goto pop_1_error; } } - #line 2841 "Python/generated_cases.c.h" + #line 2843 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_NOT_NONE) { PyObject *value = stack_pointer[-1]; - #line 2011 "Python/bytecodes.c" + #line 2013 "Python/bytecodes.c" if (!Py_IsNone(value)) { - #line 2850 "Python/generated_cases.c.h" + #line 2852 "Python/generated_cases.c.h" Py_DECREF(value); - #line 2013 "Python/bytecodes.c" + #line 2015 "Python/bytecodes.c" JUMPBY(oparg); } else { _Py_DECREF_NO_DEALLOC(value); } - #line 2858 "Python/generated_cases.c.h" + #line 2860 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_NONE) { PyObject *value = stack_pointer[-1]; - #line 2021 "Python/bytecodes.c" + #line 2023 "Python/bytecodes.c" if (Py_IsNone(value)) { _Py_DECREF_NO_DEALLOC(value); JUMPBY(oparg); } else { - #line 2871 "Python/generated_cases.c.h" + #line 2873 "Python/generated_cases.c.h" Py_DECREF(value); - #line 2027 "Python/bytecodes.c" + #line 2029 "Python/bytecodes.c" } - #line 2875 "Python/generated_cases.c.h" + #line 2877 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(JUMP_BACKWARD_NO_INTERRUPT) { - #line 2031 "Python/bytecodes.c" + #line 2033 "Python/bytecodes.c" /* This bytecode is used in the `yield from` or `await` loop. * If there is an interrupt, we want it handled in the innermost * generator or coroutine, so we deliberately do not check it here. * (see bpo-30039). */ JUMPBY(-oparg); - #line 2888 "Python/generated_cases.c.h" + #line 2890 "Python/generated_cases.c.h" DISPATCH(); } TARGET(GET_LEN) { PyObject *obj = stack_pointer[-1]; PyObject *len_o; - #line 2040 "Python/bytecodes.c" + #line 2042 "Python/bytecodes.c" // PUSH(len(TOS)) Py_ssize_t len_i = PyObject_Length(obj); if (len_i < 0) goto error; len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) goto error; - #line 2901 "Python/generated_cases.c.h" + #line 2903 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = len_o; DISPATCH(); @@ -2908,16 +2910,16 @@ PyObject *type = stack_pointer[-2]; PyObject *subject = stack_pointer[-3]; PyObject *attrs; - #line 2048 "Python/bytecodes.c" + #line 2050 "Python/bytecodes.c" // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. assert(PyTuple_CheckExact(names)); attrs = match_class(tstate, subject, type, oparg, names); - #line 2917 "Python/generated_cases.c.h" + #line 2919 "Python/generated_cases.c.h" Py_DECREF(subject); Py_DECREF(type); Py_DECREF(names); - #line 2053 "Python/bytecodes.c" + #line 2055 "Python/bytecodes.c" if (attrs) { assert(PyTuple_CheckExact(attrs)); // Success! } @@ -2925,7 +2927,7 @@ if (_PyErr_Occurred(tstate)) goto pop_3_error; attrs = Py_NewRef(Py_None); // Failure! } - #line 2929 "Python/generated_cases.c.h" + #line 2931 "Python/generated_cases.c.h" STACK_SHRINK(2); stack_pointer[-1] = attrs; DISPATCH(); @@ -2934,10 +2936,10 @@ TARGET(MATCH_MAPPING) { PyObject *subject = stack_pointer[-1]; PyObject *res; - #line 2063 "Python/bytecodes.c" + #line 2065 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; res = Py_NewRef(match ? Py_True : Py_False); - #line 2941 "Python/generated_cases.c.h" + #line 2943 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; PREDICT(POP_JUMP_IF_FALSE); @@ -2947,10 +2949,10 @@ TARGET(MATCH_SEQUENCE) { PyObject *subject = stack_pointer[-1]; PyObject *res; - #line 2069 "Python/bytecodes.c" + #line 2071 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; res = Py_NewRef(match ? Py_True : Py_False); - #line 2954 "Python/generated_cases.c.h" + #line 2956 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; PREDICT(POP_JUMP_IF_FALSE); @@ -2961,11 +2963,11 @@ PyObject *keys = stack_pointer[-1]; PyObject *subject = stack_pointer[-2]; PyObject *values_or_none; - #line 2075 "Python/bytecodes.c" + #line 2077 "Python/bytecodes.c" // On successful match, PUSH(values). Otherwise, PUSH(None). values_or_none = match_keys(tstate, subject, keys); if (values_or_none == NULL) goto error; - #line 2969 "Python/generated_cases.c.h" + #line 2971 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = values_or_none; DISPATCH(); @@ -2974,14 +2976,14 @@ TARGET(GET_ITER) { PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 2081 "Python/bytecodes.c" + #line 2083 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ iter = PyObject_GetIter(iterable); - #line 2981 "Python/generated_cases.c.h" + #line 2983 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 2084 "Python/bytecodes.c" + #line 2086 "Python/bytecodes.c" if (iter == NULL) goto pop_1_error; - #line 2985 "Python/generated_cases.c.h" + #line 2987 "Python/generated_cases.c.h" stack_pointer[-1] = iter; DISPATCH(); } @@ -2989,7 +2991,7 @@ TARGET(GET_YIELD_FROM_ITER) { PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 2088 "Python/bytecodes.c" + #line 2090 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ if (PyCoro_CheckExact(iterable)) { /* `iterable` is a coroutine */ @@ -3012,11 +3014,11 @@ if (iter == NULL) { goto error; } - #line 3016 "Python/generated_cases.c.h" + #line 3018 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 2111 "Python/bytecodes.c" + #line 2113 "Python/bytecodes.c" } - #line 3020 "Python/generated_cases.c.h" + #line 3022 "Python/generated_cases.c.h" stack_pointer[-1] = iter; PREDICT(LOAD_CONST); DISPATCH(); @@ -3027,7 +3029,7 @@ static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2130 "Python/bytecodes.c" + #line 2132 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyForIterCache *cache = (_PyForIterCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -3058,7 +3060,7 @@ DISPATCH(); } // Common case: no jump, leave it to the code generator - #line 3062 "Python/generated_cases.c.h" + #line 3064 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3066,7 +3068,7 @@ } TARGET(INSTRUMENTED_FOR_ITER) { - #line 2163 "Python/bytecodes.c" + #line 2165 "Python/bytecodes.c" _Py_CODEUNIT *here = next_instr-1; _Py_CODEUNIT *target; PyObject *iter = TOP(); @@ -3092,14 +3094,14 @@ target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1; } INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); - #line 3096 "Python/generated_cases.c.h" + #line 3098 "Python/generated_cases.c.h" DISPATCH(); } TARGET(FOR_ITER_LIST) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2191 "Python/bytecodes.c" + #line 2193 "Python/bytecodes.c" DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); _PyListIterObject *it = (_PyListIterObject *)iter; STAT_INC(FOR_ITER, hit); @@ -3119,7 +3121,7 @@ DISPATCH(); end_for_iter_list: // Common case: no jump, leave it to the code generator - #line 3123 "Python/generated_cases.c.h" + #line 3125 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3129,7 +3131,7 @@ TARGET(FOR_ITER_TUPLE) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2213 "Python/bytecodes.c" + #line 2215 "Python/bytecodes.c" _PyTupleIterObject *it = (_PyTupleIterObject *)iter; DEOPT_IF(Py_TYPE(it) != &PyTupleIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -3149,7 +3151,7 @@ DISPATCH(); end_for_iter_tuple: // Common case: no jump, leave it to the code generator - #line 3153 "Python/generated_cases.c.h" + #line 3155 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3159,7 +3161,7 @@ TARGET(FOR_ITER_RANGE) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2235 "Python/bytecodes.c" + #line 2237 "Python/bytecodes.c" _PyRangeIterObject *r = (_PyRangeIterObject *)iter; DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -3177,7 +3179,7 @@ if (next == NULL) { goto error; } - #line 3181 "Python/generated_cases.c.h" + #line 3183 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3186,7 +3188,7 @@ TARGET(FOR_ITER_GEN) { PyObject *iter = stack_pointer[-1]; - #line 2255 "Python/bytecodes.c" + #line 2257 "Python/bytecodes.c" PyGenObject *gen = (PyGenObject *)iter; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); @@ -3201,14 +3203,14 @@ assert(next_instr->op.code == END_FOR || next_instr->op.code == INSTRUMENTED_END_FOR); DISPATCH_INLINED(gen_frame); - #line 3205 "Python/generated_cases.c.h" + #line 3207 "Python/generated_cases.c.h" } TARGET(BEFORE_ASYNC_WITH) { PyObject *mgr = stack_pointer[-1]; PyObject *exit; PyObject *res; - #line 2272 "Python/bytecodes.c" + #line 2274 "Python/bytecodes.c" PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); if (enter == NULL) { if (!_PyErr_Occurred(tstate)) { @@ -3231,16 +3233,16 @@ Py_DECREF(enter); goto error; } - #line 3235 "Python/generated_cases.c.h" + #line 3237 "Python/generated_cases.c.h" Py_DECREF(mgr); - #line 2295 "Python/bytecodes.c" + #line 2297 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); if (true) goto pop_1_error; } - #line 3244 "Python/generated_cases.c.h" + #line 3246 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -3252,7 +3254,7 @@ PyObject *mgr = stack_pointer[-1]; PyObject *exit; PyObject *res; - #line 2305 "Python/bytecodes.c" + #line 2307 "Python/bytecodes.c" /* pop the context manager, push its __exit__ and the * value returned from calling its __enter__ */ @@ -3278,16 +3280,16 @@ Py_DECREF(enter); goto error; } - #line 3282 "Python/generated_cases.c.h" + #line 3284 "Python/generated_cases.c.h" Py_DECREF(mgr); - #line 2331 "Python/bytecodes.c" + #line 2333 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); if (true) goto pop_1_error; } - #line 3291 "Python/generated_cases.c.h" + #line 3293 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -3299,7 +3301,7 @@ PyObject *lasti = stack_pointer[-3]; PyObject *exit_func = stack_pointer[-4]; PyObject *res; - #line 2340 "Python/bytecodes.c" + #line 2342 "Python/bytecodes.c" /* At the top of the stack are 4 values: - val: TOP = exc_info() - unused: SECOND = previous exception @@ -3320,7 +3322,7 @@ res = PyObject_Vectorcall(exit_func, stack + 1, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); if (res == NULL) goto error; - #line 3324 "Python/generated_cases.c.h" + #line 3326 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); @@ -3329,7 +3331,7 @@ TARGET(PUSH_EXC_INFO) { PyObject *new_exc = stack_pointer[-1]; PyObject *prev_exc; - #line 2363 "Python/bytecodes.c" + #line 2365 "Python/bytecodes.c" _PyErr_StackItem *exc_info = tstate->exc_info; if (exc_info->exc_value != NULL) { prev_exc = exc_info->exc_value; @@ -3339,7 +3341,7 @@ } assert(PyExceptionInstance_Check(new_exc)); exc_info->exc_value = Py_NewRef(new_exc); - #line 3343 "Python/generated_cases.c.h" + #line 3345 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = new_exc; stack_pointer[-2] = prev_exc; @@ -3353,7 +3355,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t keys_version = read_u32(&next_instr[3].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2375 "Python/bytecodes.c" + #line 2377 "Python/bytecodes.c" /* Cached method object */ PyTypeObject *self_cls = Py_TYPE(self); assert(type_version != 0); @@ -3370,7 +3372,7 @@ assert(_PyType_HasFeature(Py_TYPE(res2), Py_TPFLAGS_METHOD_DESCRIPTOR)); res = self; assert(oparg & 1); - #line 3374 "Python/generated_cases.c.h" + #line 3376 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3384,7 +3386,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2394 "Python/bytecodes.c" + #line 2396 "Python/bytecodes.c" PyTypeObject *self_cls = Py_TYPE(self); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); assert(self_cls->tp_dictoffset == 0); @@ -3394,7 +3396,7 @@ res2 = Py_NewRef(descr); res = self; assert(oparg & 1); - #line 3398 "Python/generated_cases.c.h" + #line 3400 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3408,7 +3410,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2406 "Python/bytecodes.c" + #line 2408 "Python/bytecodes.c" PyTypeObject *self_cls = Py_TYPE(self); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); Py_ssize_t dictoffset = self_cls->tp_dictoffset; @@ -3422,7 +3424,7 @@ res2 = Py_NewRef(descr); res = self; assert(oparg & 1); - #line 3426 "Python/generated_cases.c.h" + #line 3428 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3431,16 +3433,16 @@ } TARGET(KW_NAMES) { - #line 2422 "Python/bytecodes.c" + #line 2424 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg < PyTuple_GET_SIZE(frame->f_code->co_consts)); kwnames = GETITEM(frame->f_code->co_consts, oparg); - #line 3439 "Python/generated_cases.c.h" + #line 3441 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_CALL) { - #line 2428 "Python/bytecodes.c" + #line 2430 "Python/bytecodes.c" int is_meth = PEEK(oparg+2) != NULL; int total_args = oparg + is_meth; PyObject *function = PEEK(total_args + 1); @@ -3453,7 +3455,7 @@ _PyCallCache *cache = (_PyCallCache *)next_instr; INCREMENT_ADAPTIVE_COUNTER(cache->counter); GO_TO_INSTRUCTION(CALL); - #line 3457 "Python/generated_cases.c.h" + #line 3459 "Python/generated_cases.c.h" } TARGET(CALL) { @@ -3463,7 +3465,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2473 "Python/bytecodes.c" + #line 2475 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3544,7 +3546,7 @@ Py_DECREF(args[i]); } if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3548 "Python/generated_cases.c.h" + #line 3550 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3556,7 +3558,7 @@ TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; - #line 2560 "Python/bytecodes.c" + #line 2562 "Python/bytecodes.c" DEOPT_IF(method != NULL, CALL); DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); STAT_INC(CALL, hit); @@ -3566,7 +3568,7 @@ PEEK(oparg + 2) = Py_NewRef(meth); // method Py_DECREF(callable); GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); - #line 3570 "Python/generated_cases.c.h" + #line 3572 "Python/generated_cases.c.h" } TARGET(CALL_PY_EXACT_ARGS) { @@ -3575,7 +3577,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); - #line 2572 "Python/bytecodes.c" + #line 2574 "Python/bytecodes.c" assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; @@ -3600,7 +3602,7 @@ STACK_SHRINK(oparg + 2); JUMPBY(INLINE_CACHE_ENTRIES_CALL); DISPATCH_INLINED(new_frame); - #line 3604 "Python/generated_cases.c.h" + #line 3606 "Python/generated_cases.c.h" } TARGET(CALL_PY_WITH_DEFAULTS) { @@ -3609,7 +3611,7 @@ PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); uint16_t min_args = read_u16(&next_instr[3].cache); - #line 2599 "Python/bytecodes.c" + #line 2601 "Python/bytecodes.c" assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; @@ -3639,7 +3641,7 @@ STACK_SHRINK(oparg + 2); JUMPBY(INLINE_CACHE_ENTRIES_CALL); DISPATCH_INLINED(new_frame); - #line 3643 "Python/generated_cases.c.h" + #line 3645 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_TYPE_1) { @@ -3647,7 +3649,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2631 "Python/bytecodes.c" + #line 2633 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3657,7 +3659,7 @@ res = Py_NewRef(Py_TYPE(obj)); Py_DECREF(obj); Py_DECREF(&PyType_Type); // I.e., callable - #line 3661 "Python/generated_cases.c.h" + #line 3663 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3670,7 +3672,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2643 "Python/bytecodes.c" + #line 2645 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3681,7 +3683,7 @@ Py_DECREF(arg); Py_DECREF(&PyUnicode_Type); // I.e., callable if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3685 "Python/generated_cases.c.h" + #line 3687 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3695,7 +3697,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2657 "Python/bytecodes.c" + #line 2659 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3706,7 +3708,7 @@ Py_DECREF(arg); Py_DECREF(&PyTuple_Type); // I.e., tuple if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3710 "Python/generated_cases.c.h" + #line 3712 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3720,7 +3722,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2671 "Python/bytecodes.c" + #line 2673 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3742,7 +3744,7 @@ } Py_DECREF(tp); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3746 "Python/generated_cases.c.h" + #line 3748 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3756,7 +3758,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2696 "Python/bytecodes.c" + #line 2698 "Python/bytecodes.c" /* Builtin METH_O functions */ assert(kwnames == NULL); int is_meth = method != NULL; @@ -3784,7 +3786,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3788 "Python/generated_cases.c.h" + #line 3790 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3798,7 +3800,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2727 "Python/bytecodes.c" + #line 2729 "Python/bytecodes.c" /* Builtin METH_FASTCALL functions, without keywords */ assert(kwnames == NULL); int is_meth = method != NULL; @@ -3830,7 +3832,7 @@ 'invalid'). In those cases an exception is set, so we must handle it. */ - #line 3834 "Python/generated_cases.c.h" + #line 3836 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3844,7 +3846,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2762 "Python/bytecodes.c" + #line 2764 "Python/bytecodes.c" /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ int is_meth = method != NULL; int total_args = oparg; @@ -3876,7 +3878,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3880 "Python/generated_cases.c.h" + #line 3882 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3890,7 +3892,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2797 "Python/bytecodes.c" + #line 2799 "Python/bytecodes.c" assert(kwnames == NULL); /* len(o) */ int is_meth = method != NULL; @@ -3915,7 +3917,7 @@ Py_DECREF(callable); Py_DECREF(arg); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3919 "Python/generated_cases.c.h" + #line 3921 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3928,7 +3930,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2824 "Python/bytecodes.c" + #line 2826 "Python/bytecodes.c" assert(kwnames == NULL); /* isinstance(o, o2) */ int is_meth = method != NULL; @@ -3955,7 +3957,7 @@ Py_DECREF(cls); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3959 "Python/generated_cases.c.h" + #line 3961 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3967,7 +3969,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *self = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; - #line 2854 "Python/bytecodes.c" + #line 2856 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); assert(method != NULL); @@ -3985,14 +3987,14 @@ JUMPBY(INLINE_CACHE_ENTRIES_CALL + 1); assert(next_instr[-1].op.code == POP_TOP); DISPATCH(); - #line 3989 "Python/generated_cases.c.h" + #line 3991 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) { PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2874 "Python/bytecodes.c" + #line 2876 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -4023,7 +4025,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4027 "Python/generated_cases.c.h" + #line 4029 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4036,7 +4038,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2908 "Python/bytecodes.c" + #line 2910 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -4065,7 +4067,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4069 "Python/generated_cases.c.h" + #line 4071 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4078,7 +4080,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2940 "Python/bytecodes.c" + #line 2942 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 0 || oparg == 1); int is_meth = method != NULL; @@ -4107,7 +4109,7 @@ Py_DECREF(self); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4111 "Python/generated_cases.c.h" + #line 4113 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4120,7 +4122,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2972 "Python/bytecodes.c" + #line 2974 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -4148,7 +4150,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4152 "Python/generated_cases.c.h" + #line 4154 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4158,9 +4160,9 @@ } TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { - #line 3003 "Python/bytecodes.c" + #line 3005 "Python/bytecodes.c" GO_TO_INSTRUCTION(CALL_FUNCTION_EX); - #line 4164 "Python/generated_cases.c.h" + #line 4166 "Python/generated_cases.c.h" } TARGET(CALL_FUNCTION_EX) { @@ -4169,7 +4171,7 @@ PyObject *callargs = stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))]; PyObject *func = stack_pointer[-(2 + ((oparg & 1) ? 1 : 0))]; PyObject *result; - #line 3007 "Python/bytecodes.c" + #line 3009 "Python/bytecodes.c" // DICT_MERGE is called before this opcode if there are kwargs. // It converts all dict subtypes in kwargs into regular dicts. assert(kwargs == NULL || PyDict_CheckExact(kwargs)); @@ -4212,14 +4214,14 @@ else { result = PyObject_Call(func, callargs, kwargs); } - #line 4216 "Python/generated_cases.c.h" + #line 4218 "Python/generated_cases.c.h" Py_DECREF(func); Py_DECREF(callargs); Py_XDECREF(kwargs); - #line 3050 "Python/bytecodes.c" + #line 3052 "Python/bytecodes.c" assert(PEEK(3 + (oparg & 1)) == NULL); if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } - #line 4223 "Python/generated_cases.c.h" + #line 4225 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 1) ? 1 : 0)); STACK_SHRINK(2); stack_pointer[-1] = result; @@ -4234,7 +4236,7 @@ PyObject *kwdefaults = (oparg & 0x02) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0))] : NULL; PyObject *defaults = (oparg & 0x01) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x01) ? 1 : 0))] : NULL; PyObject *func; - #line 3060 "Python/bytecodes.c" + #line 3062 "Python/bytecodes.c" PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); @@ -4263,14 +4265,14 @@ func_obj->func_version = ((PyCodeObject *)codeobj)->co_version; func = (PyObject *)func_obj; - #line 4267 "Python/generated_cases.c.h" + #line 4269 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 0x01) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x08) ? 1 : 0)); stack_pointer[-1] = func; DISPATCH(); } TARGET(RETURN_GENERATOR) { - #line 3091 "Python/bytecodes.c" + #line 3093 "Python/bytecodes.c" assert(PyFunction_Check(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); @@ -4291,7 +4293,7 @@ frame = cframe.current_frame = prev; _PyFrame_StackPush(frame, (PyObject *)gen); goto resume_frame; - #line 4295 "Python/generated_cases.c.h" + #line 4297 "Python/generated_cases.c.h" } TARGET(BUILD_SLICE) { @@ -4299,15 +4301,15 @@ PyObject *stop = stack_pointer[-(1 + ((oparg == 3) ? 1 : 0))]; PyObject *start = stack_pointer[-(2 + ((oparg == 3) ? 1 : 0))]; PyObject *slice; - #line 3114 "Python/bytecodes.c" + #line 3116 "Python/bytecodes.c" slice = PySlice_New(start, stop, step); - #line 4305 "Python/generated_cases.c.h" + #line 4307 "Python/generated_cases.c.h" Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); - #line 3116 "Python/bytecodes.c" + #line 3118 "Python/bytecodes.c" if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } - #line 4311 "Python/generated_cases.c.h" + #line 4313 "Python/generated_cases.c.h" STACK_SHRINK(((oparg == 3) ? 1 : 0)); STACK_SHRINK(1); stack_pointer[-1] = slice; @@ -4318,7 +4320,7 @@ PyObject *fmt_spec = ((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? stack_pointer[-((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))] : NULL; PyObject *value = stack_pointer[-(1 + (((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))]; PyObject *result; - #line 3120 "Python/bytecodes.c" + #line 3122 "Python/bytecodes.c" /* Handles f-string value formatting. */ PyObject *(*conv_fn)(PyObject *); int which_conversion = oparg & FVC_MASK; @@ -4353,7 +4355,7 @@ Py_DECREF(value); Py_XDECREF(fmt_spec); if (result == NULL) { STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); goto pop_1_error; } - #line 4357 "Python/generated_cases.c.h" + #line 4359 "Python/generated_cases.c.h" STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); stack_pointer[-1] = result; DISPATCH(); @@ -4362,10 +4364,10 @@ TARGET(COPY) { PyObject *bottom = stack_pointer[-(1 + (oparg-1))]; PyObject *top; - #line 3157 "Python/bytecodes.c" + #line 3159 "Python/bytecodes.c" assert(oparg > 0); top = Py_NewRef(bottom); - #line 4369 "Python/generated_cases.c.h" + #line 4371 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = top; DISPATCH(); @@ -4377,7 +4379,7 @@ PyObject *rhs = stack_pointer[-1]; PyObject *lhs = stack_pointer[-2]; PyObject *res; - #line 3162 "Python/bytecodes.c" + #line 3164 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -4392,12 +4394,12 @@ assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); assert(binary_ops[oparg]); res = binary_ops[oparg](lhs, rhs); - #line 4396 "Python/generated_cases.c.h" + #line 4398 "Python/generated_cases.c.h" Py_DECREF(lhs); Py_DECREF(rhs); - #line 3177 "Python/bytecodes.c" + #line 3179 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 4401 "Python/generated_cases.c.h" + #line 4403 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -4407,16 +4409,16 @@ TARGET(SWAP) { PyObject *top = stack_pointer[-1]; PyObject *bottom = stack_pointer[-(2 + (oparg-2))]; - #line 3182 "Python/bytecodes.c" + #line 3184 "Python/bytecodes.c" assert(oparg >= 2); - #line 4413 "Python/generated_cases.c.h" + #line 4415 "Python/generated_cases.c.h" stack_pointer[-1] = bottom; stack_pointer[-(2 + (oparg-2))] = top; DISPATCH(); } TARGET(INSTRUMENTED_LINE) { - #line 3186 "Python/bytecodes.c" + #line 3188 "Python/bytecodes.c" _Py_CODEUNIT *here = next_instr-1; _PyFrame_SetStackPointer(frame, stack_pointer); int original_opcode = _Py_call_instrumentation_line( @@ -4436,11 +4438,11 @@ } opcode = original_opcode; DISPATCH_GOTO(); - #line 4440 "Python/generated_cases.c.h" + #line 4442 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_INSTRUCTION) { - #line 3208 "Python/bytecodes.c" + #line 3210 "Python/bytecodes.c" int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, next_instr-1); if (next_opcode < 0) goto error; @@ -4452,20 +4454,20 @@ assert(next_opcode > 0 && next_opcode < 256); opcode = next_opcode; DISPATCH_GOTO(); - #line 4456 "Python/generated_cases.c.h" + #line 4458 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_JUMP_FORWARD) { - #line 3223 "Python/bytecodes.c" + #line 3224 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); - #line 4462 "Python/generated_cases.c.h" + #line 4464 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_JUMP_BACKWARD) { #line 3228 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr-oparg, PY_MONITORING_EVENT_JUMP); - #line 4469 "Python/generated_cases.c.h" + #line 4471 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -4480,7 +4482,7 @@ assert(err == 0 || err == 1); int offset = err*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4484 "Python/generated_cases.c.h" + #line 4486 "Python/generated_cases.c.h" DISPATCH(); } @@ -4494,7 +4496,7 @@ assert(err == 0 || err == 1); int offset = (1-err)*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4498 "Python/generated_cases.c.h" + #line 4500 "Python/generated_cases.c.h" DISPATCH(); } @@ -4512,7 +4514,7 @@ offset = 0; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4516 "Python/generated_cases.c.h" + #line 4518 "Python/generated_cases.c.h" DISPATCH(); } @@ -4530,7 +4532,7 @@ offset = oparg; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4534 "Python/generated_cases.c.h" + #line 4536 "Python/generated_cases.c.h" DISPATCH(); } @@ -4541,19 +4543,19 @@ oparg = oparg << 8 | next_instr->op.arg; PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); - #line 4545 "Python/generated_cases.c.h" + #line 4547 "Python/generated_cases.c.h" } TARGET(CACHE) { #line 3293 "Python/bytecodes.c" assert(0 && "Executing a cache."); Py_UNREACHABLE(); - #line 4552 "Python/generated_cases.c.h" + #line 4554 "Python/generated_cases.c.h" } TARGET(RESERVED) { #line 3298 "Python/bytecodes.c" assert(0 && "Executing RESERVED instruction."); Py_UNREACHABLE(); - #line 4559 "Python/generated_cases.c.h" + #line 4561 "Python/generated_cases.c.h" } diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index d1ec45511d841e..20ebdd8bf52ff7 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -11,10 +11,10 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { switch(opcode) { case NOP: return 0; - case INSTRUMENTED_RESUME: - return 0; case RESUME: return 0; + case INSTRUMENTED_RESUME: + return 0; case LOAD_CLOSURE: return 0; case LOAD_FAST_CHECK: @@ -278,7 +278,7 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { case FOR_ITER: return 1; case INSTRUMENTED_FOR_ITER: - return -1; + return 0; case FOR_ITER_LIST: return 1; case FOR_ITER_TUPLE: @@ -342,7 +342,7 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { case CALL_NO_KW_METHOD_DESCRIPTOR_FAST: return oparg + 2; case INSTRUMENTED_CALL_FUNCTION_EX: - return -1; + return 0; case CALL_FUNCTION_EX: return ((oparg & 1) ? 1 : 0) + 3; case MAKE_FUNCTION: @@ -395,10 +395,10 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { switch(opcode) { case NOP: return 0; - case INSTRUMENTED_RESUME: - return 0; case RESUME: return 0; + case INSTRUMENTED_RESUME: + return 0; case LOAD_CLOSURE: return 1; case LOAD_FAST_CHECK: @@ -662,7 +662,7 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { case FOR_ITER: return 2; case INSTRUMENTED_FOR_ITER: - return -1; + return 0; case FOR_ITER_LIST: return 2; case FOR_ITER_TUPLE: @@ -726,7 +726,7 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { case CALL_NO_KW_METHOD_DESCRIPTOR_FAST: return 1; case INSTRUMENTED_CALL_FUNCTION_EX: - return -1; + return 0; case CALL_FUNCTION_EX: return 1; case MAKE_FUNCTION: @@ -782,8 +782,8 @@ extern const struct opcode_metadata _PyOpcode_opcode_metadata[256]; #else const struct opcode_metadata _PyOpcode_opcode_metadata[256] = { [NOP] = { true, INSTR_FMT_IX }, - [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB }, [RESUME] = { true, INSTR_FMT_IB }, + [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB }, [LOAD_CLOSURE] = { true, INSTR_FMT_IB }, [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB }, [LOAD_FAST] = { true, INSTR_FMT_IB }, From 43618a9d8f766f64f6a35e4c5a21d55aa9bc7715 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 5 Apr 2023 19:56:45 +0100 Subject: [PATCH 110/116] Keep check-c-globals happy. --- Tools/c-analyzer/cpython/globals-to-fix.tsv | 1 + 1 file changed, 1 insertion(+) diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index f7e6843c12969f..8745f8abc224fe 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -136,6 +136,7 @@ Objects/unicodeobject.c - EncodingMapType - #Objects/unicodeobject.c - PyFieldNameIter_Type - #Objects/unicodeobject.c - PyFormatterIter_Type - Python/legacy_tracing.c - _PyLegacyEventHandler_Type - +Objects/object.c - _PyLegacyEventHandler_Type - ##----------------------- From 821ae52abcd44eccae583a647bd4c50c3b7125a7 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 9 Apr 2023 02:43:40 +0100 Subject: [PATCH 111/116] Don't crash if locals events aren't set. --- Lib/test/test_monitoring.py | 9 +++++++++ Python/instrumentation.c | 10 ++++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index fe93046c8321f9..0e88efa644ba8a 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -1020,3 +1020,12 @@ def test_local(self): self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL, code), 0) sys.monitoring.set_local_events(TEST_TOOL2, code, 0) self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL2, code), 0) + +class TestUninitialized(unittest.TestCase, MonitoringTestBase): + + @staticmethod + def f(): + pass + + def test_get_local_events_uninitialized(self): + self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL, self.f.__code__), 0) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 436c2b48d178ab..dcb9dc81b19dc0 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1793,10 +1793,12 @@ monitoring_get_local_events_impl(PyObject *module, int tool_id, return -1; } _PyMonitoringEventSet event_set = 0; - for (int e = 0; e < PY_MONITORING_UNGROUPED_EVENTS; e++) { - _Py_Monitors *m = &((PyCodeObject *)code)->_co_monitoring->local_monitors; - if ((m->tools[e] >> tool_id) & 1) { - event_set |= (1 << e); + _PyCoMonitoringData *data = ((PyCodeObject *)code)->_co_monitoring; + if (data != NULL) { + for (int e = 0; e < PY_MONITORING_UNGROUPED_EVENTS; e++) { + if ((data->local_monitors.tools[e] >> tool_id) & 1) { + event_set |= (1 << e); + } } } return event_set; From 505a08dfa740ef3296e3779e5b469df96c087850 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 9 Apr 2023 03:00:00 +0100 Subject: [PATCH 112/116] Add NO_EVENTS. --- Lib/test/test_monitoring.py | 1 + Python/instrumentation.c | 3 +++ 2 files changed, 4 insertions(+) diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 0e88efa644ba8a..8679bca3f234ec 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -49,6 +49,7 @@ def test_has_objects(self): m.restart_events m.DISABLE m.MISSING + m.events.NO_EVENTS def test_tool(self): sys.monitoring.use_tool_id(TEST_TOOL, "MonitoringTest.Tool") diff --git a/Python/instrumentation.c b/Python/instrumentation.c index dcb9dc81b19dc0..05783ece83b0d0 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -4,6 +4,7 @@ #include "pycore_call.h" #include "pycore_frame.h" #include "pycore_interp.h" +#include "pycore_long.h" #include "pycore_namespace.h" #include "pycore_object.h" #include "pycore_opcode.h" @@ -1974,6 +1975,8 @@ PyObject *_Py_CreateMonitoringObject(void) goto error; } } + err = PyObject_SetAttrString(events, "NO_EVENTS", _PyLong_GetZero()); + if (err) goto error; PyObject *val = PyLong_FromLong(PY_MONITORING_DEBUGGER_ID); err = PyObject_SetAttrString(mod, "DEBUGGER_ID", val); Py_DECREF(val); From f63da91d90a9b3e6bf4b6023a68b541c6f11e19b Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 9 Apr 2023 05:00:54 +0100 Subject: [PATCH 113/116] Flip order of LINE and INSTRUCTION events. --- Lib/test/test_monitoring.py | 84 +++++++++++++++++++++---------------- Python/instrumentation.c | 61 ++++++++++++++------------- Python/legacy_tracing.c | 19 +++------ 3 files changed, 85 insertions(+), 79 deletions(-) diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 8679bca3f234ec..73406f771f5975 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -33,7 +33,7 @@ def g1(): TEST_TOOL2 = 3 TEST_TOOL3 = 4 -class MonitoringBaseTest(unittest.TestCase): +class MonitoringBasicTest(unittest.TestCase): def test_has_objects(self): m = sys.monitoring @@ -54,8 +54,6 @@ def test_has_objects(self): def test_tool(self): sys.monitoring.use_tool_id(TEST_TOOL, "MonitoringTest.Tool") self.assertEqual(sys.monitoring.get_tool(TEST_TOOL), "MonitoringTest.Tool") - sys.monitoring.free_tool_id(TEST_TOOL) - self.assertEqual(sys.monitoring.get_tool(TEST_TOOL), None) sys.monitoring.set_events(TEST_TOOL, 15) self.assertEqual(sys.monitoring.get_events(TEST_TOOL), 15) sys.monitoring.set_events(TEST_TOOL, 0) @@ -63,9 +61,33 @@ def test_tool(self): sys.monitoring.set_events(TEST_TOOL, sys.monitoring.events.C_RETURN) with self.assertRaises(ValueError): sys.monitoring.set_events(TEST_TOOL, sys.monitoring.events.C_RAISE) + sys.monitoring.free_tool_id(TEST_TOOL) + self.assertEqual(sys.monitoring.get_tool(TEST_TOOL), None) -class MonitoringCountTest(unittest.TestCase): +class MonitoringTestBase: + + def setUp(self): + # Check that a previous test hasn't left monitoring on. + for tool in range(6): + self.assertEqual(sys.monitoring.get_events(tool), 0) + self.assertIs(sys.monitoring.get_tool(TEST_TOOL), None) + self.assertIs(sys.monitoring.get_tool(TEST_TOOL2), None) + self.assertIs(sys.monitoring.get_tool(TEST_TOOL3), None) + sys.monitoring.use_tool_id(TEST_TOOL, "test " + self.__class__.__name__) + sys.monitoring.use_tool_id(TEST_TOOL2, "test2 " + self.__class__.__name__) + sys.monitoring.use_tool_id(TEST_TOOL3, "test3 " + self.__class__.__name__) + + def tearDown(self): + # Check that test hasn't left monitoring on. + for tool in range(6): + self.assertEqual(sys.monitoring.get_events(tool), 0) + sys.monitoring.free_tool_id(TEST_TOOL) + sys.monitoring.free_tool_id(TEST_TOOL2) + sys.monitoring.free_tool_id(TEST_TOOL3) + + +class MonitoringCountTest(MonitoringTestBase, unittest.TestCase): def check_event_count(self, func, event, expected): @@ -186,18 +208,6 @@ def nested_call(): PY_CALLABLES = (types.FunctionType, types.MethodType) -class MonitoringTestBase: - - def setUp(self): - # Check that a previous test hasn't left monitoring on. - for tool in range(6): - self.assertEqual(sys.monitoring.get_events(tool), 0) - - def tearDown(self): - # Check that test hasn't left monitoring on. - for tool in range(6): - self.assertEqual(sys.monitoring.get_events(tool), 0) - class MonitoringEventsBase(MonitoringTestBase): def gather_events(self, func): @@ -230,7 +240,7 @@ def check_events(self, func, expected=None): self.assertEqual(events, expected) -class MonitoringEventsTest(unittest.TestCase, MonitoringEventsBase): +class MonitoringEventsTest(MonitoringEventsBase, unittest.TestCase): def test_just_pass(self): self.check_events(just_pass) @@ -256,7 +266,7 @@ def test_nested_call(self): from test.profilee import testfunc -class SimulateProfileTest(unittest.TestCase, MonitoringEventsBase): +class SimulateProfileTest(MonitoringEventsBase, unittest.TestCase): def test_balanced(self): events = self.gather_events(testfunc) @@ -322,7 +332,7 @@ def __call__(self, code, event): return sys.monitoring.DISABLE -class MontoringDisableAndRestartTest(unittest.TestCase, MonitoringTestBase): +class MontoringDisableAndRestartTest(MonitoringTestBase, unittest.TestCase): def test_disable(self): try: @@ -363,7 +373,7 @@ def test_restart(self): sys.monitoring.restart_events() -class MultipleMonitorsTest(unittest.TestCase, MonitoringTestBase): +class MultipleMonitorsTest(MonitoringTestBase, unittest.TestCase): def test_two_same(self): try: @@ -481,7 +491,7 @@ def test_two_with_disable(self): self.assertEqual(sys.monitoring._all_events(), {}) sys.monitoring.restart_events() -class LineMonitoringTest(unittest.TestCase): +class LineMonitoringTest(MonitoringTestBase, unittest.TestCase): def test_lines_single(self): try: @@ -616,7 +626,7 @@ def __init__(self, events): def __call__(self, code, offset, exc): self.events.append(("raise", type(exc))) -class CheckEvents(unittest.TestCase): +class CheckEvents(MonitoringTestBase, unittest.TestCase): def check_events(self, func, expected, tool=TEST_TOOL, recorders=(ExceptionRecorder,)): try: @@ -795,14 +805,14 @@ def func1(): self.check_events(func1, recorders = LINE_AND_INSTRUCTION_RECORDERS, expected = [ ('line', 'check_events', 10), - ('instruction', 'func1', 2), ('line', 'func1', 1), + ('instruction', 'func1', 2), ('instruction', 'func1', 4), - ('instruction', 'func1', 6), ('line', 'func1', 2), + ('instruction', 'func1', 6), ('instruction', 'func1', 8), - ('instruction', 'func1', 10), ('line', 'func1', 3), + ('instruction', 'func1', 10), ('instruction', 'func1', 12), ('instruction', 'func1', 14), ('line', 'check_events', 11)]) @@ -816,17 +826,17 @@ def func2(): self.check_events(func2, recorders = LINE_AND_INSTRUCTION_RECORDERS, expected = [ ('line', 'check_events', 10), - ('instruction', 'func2', 2), ('line', 'func2', 1), + ('instruction', 'func2', 2), ('instruction', 'func2', 4), - ('instruction', 'func2', 6), ('line', 'func2', 2), + ('instruction', 'func2', 6), ('instruction', 'func2', 8), ('instruction', 'func2', 28), ('instruction', 'func2', 30), ('instruction', 'func2', 38), - ('instruction', 'func2', 40), ('line', 'func2', 3), + ('instruction', 'func2', 40), ('instruction', 'func2', 42), ('instruction', 'func2', 44), ('line', 'check_events', 11)]) @@ -843,28 +853,28 @@ def func3(): self.check_events(func3, recorders = LINE_AND_INSTRUCTION_RECORDERS, expected = [ ('line', 'check_events', 10), - ('instruction', 'func3', 2), ('line', 'func3', 1), - ('instruction', 'func3', 4), + ('instruction', 'func3', 2), ('line', 'func3', 2), + ('instruction', 'func3', 4), ('instruction', 'func3', 6), - ('instruction', 'func3', 8), ('line', 'func3', 3), + ('instruction', 'func3', 8), ('instruction', 'func3', 18), ('instruction', 'func3', 20), - ('instruction', 'func3', 22), ('line', 'func3', 4), - ('instruction', 'func3', 24), + ('instruction', 'func3', 22), ('line', 'func3', 5), + ('instruction', 'func3', 24), ('instruction', 'func3', 26), ('instruction', 'func3', 28), - ('instruction', 'func3', 30), ('line', 'func3', 6), + ('instruction', 'func3', 30), ('instruction', 'func3', 32), ('instruction', 'func3', 34), ('line', 'check_events', 11)]) -class TestInstallIncrementallly(unittest.TestCase): +class TestInstallIncrementallly(MonitoringTestBase, unittest.TestCase): def check_events(self, func, must_include, tool=TEST_TOOL, recorders=(ExceptionRecorder,)): try: @@ -930,7 +940,7 @@ def test_instruction_then_line(self): self.check_events(self.func2, recorders = recorders, must_include = self.MUST_INCLUDE_CI) -class TestLocalEvents(unittest.TestCase): +class TestLocalEvents(MonitoringTestBase, unittest.TestCase): def check_events(self, func, expected, tool=TEST_TOOL, recorders=(ExceptionRecorder,)): try: @@ -999,7 +1009,7 @@ def func3(): ('line', 'func3', 6)]) -class TestSetGetEvents(unittest.TestCase, MonitoringTestBase): +class TestSetGetEvents(MonitoringTestBase, unittest.TestCase): def test_global(self): sys.monitoring.set_events(TEST_TOOL, E.PY_START) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 05783ece83b0d0..f672456439fabb 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -247,12 +247,12 @@ instruction_length(PyCodeObject *code, int offset) int opcode = _PyCode_CODE(code)[offset].op.code; assert(opcode != 0); assert(opcode != RESERVED); - if (opcode == INSTRUMENTED_INSTRUCTION) { - opcode = code->_co_monitoring->per_instruction_opcodes[offset]; - } if (opcode == INSTRUMENTED_LINE) { opcode = code->_co_monitoring->lines[offset].original_opcode; } + if (opcode == INSTRUMENTED_INSTRUCTION) { + opcode = code->_co_monitoring->per_instruction_opcodes[offset]; + } int deinstrumented = DE_INSTRUMENT[opcode]; if (deinstrumented) { opcode = deinstrumented; @@ -516,12 +516,12 @@ sanity_check_instrumentation(PyCodeObject *code) int _Py_GetBaseOpcode(PyCodeObject *code, int i) { int opcode = _PyCode_CODE(code)[i].op.code; - if (opcode == INSTRUMENTED_INSTRUCTION) { - opcode = code->_co_monitoring->per_instruction_opcodes[i]; - } if (opcode == INSTRUMENTED_LINE) { opcode = code->_co_monitoring->lines[i].original_opcode; } + if (opcode == INSTRUMENTED_INSTRUCTION) { + opcode = code->_co_monitoring->per_instruction_opcodes[i]; + } CHECK(opcode != INSTRUMENTED_INSTRUCTION); CHECK(opcode != INSTRUMENTED_LINE); int deinstrumented = DE_INSTRUMENT[opcode]; @@ -540,15 +540,15 @@ de_instrument(PyCodeObject *code, int i, int event) _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; uint8_t *opcode_ptr = &instr->op.code; int opcode = *opcode_ptr; - if (opcode == INSTRUMENTED_INSTRUCTION) { - opcode_ptr = &code->_co_monitoring->per_instruction_opcodes[i]; - opcode = *opcode_ptr; - } if (opcode == INSTRUMENTED_LINE) { opcode_ptr = &code->_co_monitoring->lines[i].original_opcode; opcode = *opcode_ptr; } - int deinstrumented = DE_INSTRUMENT[opcode]; + if (opcode == INSTRUMENTED_INSTRUCTION) { + opcode_ptr = &code->_co_monitoring->per_instruction_opcodes[i]; + opcode = *opcode_ptr; + } + int deinstrumented = DE_INSTRUMENT[opcode]; if (deinstrumented == 0) { return; } @@ -565,10 +565,6 @@ de_instrument_line(PyCodeObject *code, int i) _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; uint8_t *opcode_ptr = &instr->op.code; int opcode =*opcode_ptr; - if (opcode == INSTRUMENTED_INSTRUCTION) { - opcode_ptr = &code->_co_monitoring->per_instruction_opcodes[i]; - opcode = *opcode_ptr; - } if (opcode != INSTRUMENTED_LINE) { return; } @@ -589,7 +585,12 @@ static void de_instrument_per_instruction(PyCodeObject *code, int i) { _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; - int opcode = instr->op.code; + uint8_t *opcode_ptr = &instr->op.code; + int opcode =*opcode_ptr; + if (opcode == INSTRUMENTED_LINE) { + opcode_ptr = &code->_co_monitoring->lines[i].original_opcode; + opcode = *opcode_ptr; + } if (opcode != INSTRUMENTED_INSTRUCTION) { return; } @@ -612,14 +613,14 @@ instrument(PyCodeObject *code, int i) _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; uint8_t *opcode_ptr = &instr->op.code; int opcode =*opcode_ptr; - if (opcode == INSTRUMENTED_INSTRUCTION) { - opcode_ptr = &code->_co_monitoring->per_instruction_opcodes[i]; - opcode = *opcode_ptr; - } if (opcode == INSTRUMENTED_LINE) { _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[i]; opcode_ptr = &lines->original_opcode; opcode = *opcode_ptr; + } + if (opcode == INSTRUMENTED_INSTRUCTION) { + opcode_ptr = &code->_co_monitoring->per_instruction_opcodes[i]; + opcode = *opcode_ptr; CHECK(!is_instrumented(opcode)); CHECK(opcode == _PyOpcode_Deopt[opcode]); } @@ -640,10 +641,6 @@ instrument_line(PyCodeObject *code, int i) { uint8_t *opcode_ptr = &_PyCode_CODE(code)[i].op.code; int opcode =*opcode_ptr; - if (opcode == INSTRUMENTED_INSTRUCTION) { - opcode_ptr = &code->_co_monitoring->per_instruction_opcodes[i]; - opcode = *opcode_ptr; - } if (opcode == INSTRUMENTED_LINE) { return; } @@ -657,7 +654,13 @@ static void instrument_per_instruction(PyCodeObject *code, int i) { _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; - int opcode = instr->op.code; + uint8_t *opcode_ptr = &instr->op.code; + int opcode =*opcode_ptr; + if (opcode == INSTRUMENTED_LINE) { + _PyCoLineInstrumentationData *lines = &code->_co_monitoring->lines[i]; + opcode_ptr = &lines->original_opcode; + opcode = *opcode_ptr; + } if (opcode == INSTRUMENTED_INSTRUCTION) { return; } @@ -672,7 +675,7 @@ instrument_per_instruction(PyCodeObject *code, int i) code->_co_monitoring->per_instruction_opcodes[i] = _PyOpcode_Deopt[opcode]; } assert(code->_co_monitoring->per_instruction_opcodes[i] > 0); - instr->op.code = INSTRUMENTED_INSTRUCTION; + *opcode_ptr = INSTRUMENTED_INSTRUCTION; } #ifndef NDEBUG @@ -681,12 +684,12 @@ instruction_has_event(PyCodeObject *code, int offset) { _Py_CODEUNIT instr = _PyCode_CODE(code)[offset]; int opcode = instr.op.code; - if (opcode == INSTRUMENTED_INSTRUCTION) { - opcode = code->_co_monitoring->per_instruction_opcodes[offset]; - } if (opcode == INSTRUMENTED_LINE) { opcode = code->_co_monitoring->lines[offset].original_opcode; } + if (opcode == INSTRUMENTED_INSTRUCTION) { + opcode = code->_co_monitoring->per_instruction_opcodes[offset]; + } return opcode_has_event(opcode); } #endif diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index 6e6deea7b818ce..cf345bddda79b0 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -422,17 +422,14 @@ _PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) tstate->interp->sys_profiling_threads += delta; assert(tstate->interp->sys_profiling_threads >= 0); + uint32_t events = 0; if (tstate->interp->sys_profiling_threads) { - uint32_t events = + events = (1 << PY_MONITORING_EVENT_PY_START) | (1 << PY_MONITORING_EVENT_PY_RESUME) | (1 << PY_MONITORING_EVENT_PY_RETURN) | (1 << PY_MONITORING_EVENT_PY_YIELD) | (1 << PY_MONITORING_EVENT_CALL) | (1 << PY_MONITORING_EVENT_PY_UNWIND); - _PyMonitoring_SetEvents(PY_MONITORING_SYS_PROFILE_ID, events); } - else { - _PyMonitoring_SetEvents(PY_MONITORING_SYS_PROFILE_ID, 0); - } - return 0; + return _PyMonitoring_SetEvents(PY_MONITORING_SYS_PROFILE_ID, events); } int @@ -513,8 +510,9 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) tstate->interp->sys_tracing_threads += delta; assert(tstate->interp->sys_tracing_threads >= 0); + uint32_t events = 0; if (tstate->interp->sys_tracing_threads) { - uint32_t events = + events = (1 << PY_MONITORING_EVENT_PY_START) | (1 << PY_MONITORING_EVENT_PY_RESUME) | (1 << PY_MONITORING_EVENT_PY_RETURN) | (1 << PY_MONITORING_EVENT_PY_YIELD) | (1 << PY_MONITORING_EVENT_RAISE) | (1 << PY_MONITORING_EVENT_LINE) | @@ -525,11 +523,6 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) if (tstate->interp->f_opcode_trace_set) { events |= (1 << PY_MONITORING_EVENT_INSTRUCTION); } - _PyMonitoring_SetEvents(PY_MONITORING_SYS_TRACE_ID, events); - } - else { - _PyMonitoring_SetEvents(PY_MONITORING_SYS_TRACE_ID, 0); } - - return 0; + return _PyMonitoring_SetEvents(PY_MONITORING_SYS_TRACE_ID, events); } From c32434403726a4d6ca63198bfa31210a06a0c5d7 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 9 Apr 2023 05:31:48 +0100 Subject: [PATCH 114/116] Check tool is in use when setting events. --- Lib/test/test_monitoring.py | 2 ++ Python/instrumentation.c | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 73406f771f5975..4aad3da61b6bee 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -63,6 +63,8 @@ def test_tool(self): sys.monitoring.set_events(TEST_TOOL, sys.monitoring.events.C_RAISE) sys.monitoring.free_tool_id(TEST_TOOL) self.assertEqual(sys.monitoring.get_tool(TEST_TOOL), None) + with self.assertRaises(ValueError): + sys.monitoring.set_events(TEST_TOOL, sys.monitoring.events.CALL) class MonitoringTestBase: diff --git a/Python/instrumentation.c b/Python/instrumentation.c index f672456439fabb..39a7eaa3bd2d42 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1551,12 +1551,27 @@ set_events(_Py_Monitors *m, int tool_id, _PyMonitoringEventSet events) } } +static int +check_tool(PyInterpreterState *interp, int tool_id) +{ + if (tool_id < PY_MONITORING_SYS_PROFILE_ID && + interp->monitoring_tool_names[tool_id] == NULL + ) { + PyErr_Format(PyExc_ValueError, "tool %d is not in use", tool_id); + return -1; + } + return 0; +} + int _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) { assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); PyInterpreterState *interp = _PyInterpreterState_Get(); assert(events < (1 << PY_MONITORING_UNGROUPED_EVENTS)); + if (check_tool(interp, tool_id)) { + return -1; + } uint32_t existing_events = get_events(&interp->monitors, tool_id); if (existing_events == events) { return 0; @@ -1572,6 +1587,9 @@ _PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEvent assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); PyInterpreterState *interp = _PyInterpreterState_Get(); assert(events < (1 << PY_MONITORING_UNGROUPED_EVENTS)); + if (check_tool(interp, tool_id)) { + return -1; + } if (allocate_instrumentation_data(code)) { return -1; } From 168b34ab6f94ccdcc88f0ab7fc6d2b6e0182b74c Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 12 Apr 2023 11:02:56 +0100 Subject: [PATCH 115/116] Reset last traced line number when setting frame.f_trace only if set to new value. --- Lib/test/test_sys_settrace.py | 42 +++++++++++++++++++++++++++++++++++ Objects/frameobject.c | 6 +++-- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 103f41cc36440c..1cb0afc99b978d 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -2825,5 +2825,47 @@ def test_events(self): (6, 'line')]) +class TestSetLocalTrace(TraceTestCase): + + def test_with_branches(self): + events = [] + + def tracefunc(frame, event, arg): + if frame.f_code.co_name == "func": + frame.f_trace = tracefunc + line = frame.f_lineno - frame.f_code.co_firstlineno + events.append((line, event)) + return tracefunc + def func(arg = 1): + N = 1 + if arg >= 2: # step 1 + not_reached = 3 + else: + reached = 5 + if arg >= 3: # step 3 + not_reached = 7 + else: + reached = 9 + the_end = 10 + + EXPECTED_EVENTS = [ + (0, 'call'), + (1, 'line'), + (2, 'line'), + (5, 'line'), + (6, 'line'), + (9, 'line'), + (10, 'line'), + (10, 'return'), + ] + + sys.settrace(tracefunc) + sys._getframe().f_trace = tracefunc + func() + self.assertEqual(events, EXPECTED_EVENTS) + sys.settrace(None) + + + if __name__ == "__main__": unittest.main() diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 2facffaca66c0b..ef0070199ab2c0 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -852,8 +852,10 @@ frame_settrace(PyFrameObject *f, PyObject* v, void *closure) if (v == Py_None) { v = NULL; } - Py_XSETREF(f->f_trace, Py_XNewRef(v)); - f->f_last_traced_line = -1; + if (v != f->f_trace) { + Py_XSETREF(f->f_trace, Py_XNewRef(v)); + f->f_last_traced_line = -1; + } return 0; } From f07a080b07034730dab295dd431f1851d23fa358 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 12 Apr 2023 11:19:31 +0100 Subject: [PATCH 116/116] Tidy up test case. --- Lib/test/test_sys_settrace.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 1cb0afc99b978d..980321e169b9e5 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -2828,7 +2828,6 @@ def test_events(self): class TestSetLocalTrace(TraceTestCase): def test_with_branches(self): - events = [] def tracefunc(frame, event, arg): if frame.f_code.co_name == "func": @@ -2836,13 +2835,14 @@ def tracefunc(frame, event, arg): line = frame.f_lineno - frame.f_code.co_firstlineno events.append((line, event)) return tracefunc + def func(arg = 1): N = 1 - if arg >= 2: # step 1 + if arg >= 2: not_reached = 3 else: reached = 5 - if arg >= 3: # step 3 + if arg >= 3: not_reached = 7 else: reached = 9 @@ -2859,6 +2859,7 @@ def func(arg = 1): (10, 'return'), ] + events = [] sys.settrace(tracefunc) sys._getframe().f_trace = tracefunc func() pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy