From 20825373364ceec32fd3a115e3814f3d08420b16 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 21 Jan 2025 08:54:50 +0800 Subject: [PATCH 01/14] Move labels in ceval.c to bytecodes.c --- Lib/test/test_generated_cases.py | 106 +++++++++++++ Makefile.pre.in | 9 +- PCbuild/regen.targets | 2 + Python/bytecodes.c | 120 +++++++++++++++ Python/ceval.c | 106 +------------ Python/generated_labels.c.h | 130 ++++++++++++++++ Tools/cases_generator/analyzer.py | 21 ++- Tools/cases_generator/labels_generator.py | 172 ++++++++++++++++++++++ Tools/cases_generator/lexer.py | 3 + Tools/cases_generator/parser.py | 1 + Tools/cases_generator/parsing.py | 19 ++- 11 files changed, 581 insertions(+), 108 deletions(-) create mode 100644 Python/generated_labels.c.h create mode 100644 Tools/cases_generator/labels_generator.py diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 7a50a29bb0126c..e0ae87bc859675 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -36,6 +36,7 @@ def skip_if_different_mount_drives(): import parser from stack import Local, Stack import tier1_generator + import labels_generator import opcode_metadata_generator import optimizer_generator @@ -1757,6 +1758,111 @@ def test_kill_in_wrong_order(self): self.run_cases_test(input, "") +class TestGeneratedLabels(unittest.TestCase): + def setUp(self) -> None: + super().setUp() + self.maxDiff = None + + self.temp_dir = tempfile.gettempdir() + self.temp_input_filename = os.path.join(self.temp_dir, "input.txt") + self.temp_output_filename = os.path.join(self.temp_dir, "output.txt") + self.temp_metadata_filename = os.path.join(self.temp_dir, "metadata.txt") + self.temp_pymetadata_filename = os.path.join(self.temp_dir, "pymetadata.txt") + self.temp_executor_filename = os.path.join(self.temp_dir, "executor.txt") + + def tearDown(self) -> None: + for filename in [ + self.temp_input_filename, + self.temp_output_filename, + self.temp_metadata_filename, + self.temp_pymetadata_filename, + self.temp_executor_filename, + ]: + try: + os.remove(filename) + except Exception: + pass + super().tearDown() + + def run_cases_test(self, input: str, expected: str): + with open(self.temp_input_filename, "w+") as temp_input: + temp_input.write(parser.BEGIN_MARKER) + temp_input.write(input) + temp_input.write(parser.END_MARKER) + temp_input.flush() + + with handle_stderr(): + labels_generator.generate_labels_from_files( + [self.temp_input_filename], self.temp_output_filename + ) + + with open(self.temp_output_filename) as temp_output: + lines = temp_output.readlines() + while lines and lines[0].startswith(("// ", "#", " #", "\n")): + lines.pop(0) + while lines and lines[-1].startswith(("#", "\n")): + lines.pop(-1) + actual = "".join(lines) + + self.assertEqual(actual.strip(), expected.strip()) + + def test_complex_label(self): + input = """ + label(my_label) { + // Comment + do_thing() + if (complex) { + goto other_label; + } + goto other_label2; + } + """ + + output = """ + my_label: + { + // Comment + do_thing() + if (complex) { + goto other_label; + } + goto other_label2; + } + """ + self.run_cases_test(input, output) + + def test_multiple_labels(self): + input = """ + label(my_label_1) { + // Comment + do_thing1(); + goto my_label_2; + } + + label(my_label_2) { + // Comment + do_thing2(); + goto my_label_3; + } + """ + + output = """ + my_label_1: + { + // Comment + do_thing1(); + goto my_label_2; + } + + my_label_2: + { + // Comment + do_thing2(); + goto my_label_3; + } + """ + self.run_cases_test(input, output) + class TestGeneratedAbstractCases(unittest.TestCase): def setUp(self) -> None: super().setUp() diff --git a/Makefile.pre.in b/Makefile.pre.in index 67acf0fc520087..a13c25f74b9243 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1985,7 +1985,7 @@ Objects/mimalloc/page.o: $(srcdir)/Objects/mimalloc/page-queue.c .PHONY: regen-cases regen-cases: \ regen-opcode-ids regen-opcode-targets regen-uop-ids regen-opcode-metadata-py \ - regen-generated-cases regen-executor-cases regen-optimizer-cases \ + regen-generated-cases regen-generated-labels regen-executor-cases regen-optimizer-cases \ regen-opcode-metadata regen-uop-metadata .PHONY: regen-opcode-ids @@ -2018,6 +2018,13 @@ regen-generated-cases: -o $(srcdir)/Python/generated_cases.c.h.new $(srcdir)/Python/bytecodes.c $(UPDATE_FILE) $(srcdir)/Python/generated_cases.c.h $(srcdir)/Python/generated_cases.c.h.new +.PHONY: regen-generated-labels +regen-generated-labels: + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/labels_generator.py \ + -o $(srcdir)/Python/generated_labels.c.h.new $(srcdir)/Python/bytecodes.c + $(UPDATE_FILE) $(srcdir)/Python/generated_labels.c.h $(srcdir)/Python/generated_labels.c.h.new + + .PHONY: regen-executor-cases regen-executor-cases: $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/tier2_generator.py \ diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index 416241d9d0df10..cc91479675d478 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -100,6 +100,8 @@ WorkingDirectory="$(PySourcePath)" /> + tracing || eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version)); } + label(pop_4_error) { + STACK_SHRINK(1); + goto pop_3_error; + } + + label(pop_3_error) { + STACK_SHRINK(1); + goto pop_2_error; + } + + label(pop_2_error) { + STACK_SHRINK(1); + goto pop_1_error; + } + + label(pop_1_error) { + STACK_SHRINK(1); + goto error; + } + + label(error) { + /* Double-check exception status. */ +#ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } +#else + assert(_PyErr_Occurred(tstate)); +#endif + + /* Log traceback info. */ + assert(frame != &entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); + goto exception_unwind; + } + + label(exception_unwind) { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } +#endif + DISPATCH(); + } + + label(exit_unwind) { + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != &entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == &entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + goto resume_with_error; + } + + label(resume_with_error) { + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + } // END BYTECODES // } diff --git a/Python/ceval.c b/Python/ceval.c index 28b0b4c6de39a7..13e40cb2419e46 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -895,7 +895,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int DISPATCH(); - { /* Start instructions */ #if !USE_COMPUTED_GOTOS dispatch_opcode: @@ -927,110 +926,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int or goto error. */ Py_UNREACHABLE(); -pop_4_error: - STACK_SHRINK(1); -pop_3_error: - STACK_SHRINK(1); -pop_2_error: - STACK_SHRINK(1); -pop_1_error: - STACK_SHRINK(1); -error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - assert(frame != &entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); -exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ -#ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } -#endif - DISPATCH(); - } - } - -exit_unwind: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == &entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - -resume_with_error: - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - +#include "generated_labels.c.h" #ifdef _Py_TIER2 diff --git a/Python/generated_labels.c.h b/Python/generated_labels.c.h new file mode 100644 index 00000000000000..9d0b959353a5b8 --- /dev/null +++ b/Python/generated_labels.c.h @@ -0,0 +1,130 @@ +// This file is generated by Tools/cases_generator/labels_generator.py +// from: +// Python/bytecodes.c +// Do not edit! + + pop_4_error: + { + STACK_SHRINK(1); + goto pop_3_error; + } + + pop_3_error: + { + STACK_SHRINK(1); + goto pop_2_error; + } + + pop_2_error: + { + STACK_SHRINK(1); + goto pop_1_error; + } + + pop_1_error: + { + STACK_SHRINK(1); + goto error; + } + + error: + { + /* Double-check exception status. */ + #ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } + #else + assert(_PyErr_Occurred(tstate)); + #endif + + /* Log traceback info. */ + assert(frame != &entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); + goto exception_unwind; + } + + exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ + #ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } + #endif + DISPATCH(); + } + + exit_unwind: + { + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != &entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == &entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + goto resume_with_error; + } + + resume_with_error: + { + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + } + + diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 4013b503502df6..6f22d5b6e71357 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -258,6 +258,12 @@ def is_super(self) -> bool: return False +@dataclass +class Label: + name: str + body: list[lexer.Token] + + @dataclass class PseudoInstruction: name: str @@ -291,6 +297,7 @@ class Analysis: uops: dict[str, Uop] families: dict[str, Family] pseudos: dict[str, PseudoInstruction] + labels: dict[str, Label] opmap: dict[str, int] have_arg: int min_instrumented: int @@ -1015,6 +1022,13 @@ def add_pseudo( ) +def add_label( + label: parser.LabelDef, + labels: dict[str, Label], +) -> None: + labels[label.name] = Label(label.name, label.block.tokens) + + def assign_opcodes( instructions: dict[str, Instruction], families: dict[str, Family], @@ -1133,6 +1147,7 @@ def analyze_forest(forest: list[parser.AstNode]) -> Analysis: uops: dict[str, Uop] = {} families: dict[str, Family] = {} pseudos: dict[str, PseudoInstruction] = {} + labels: dict[str, Label] = {} for node in forest: match node: case parser.InstDef(name): @@ -1147,6 +1162,8 @@ def analyze_forest(forest: list[parser.AstNode]) -> Analysis: pass case parser.Pseudo(): pass + case parser.LabelDef(): + pass case _: assert False for node in forest: @@ -1158,6 +1175,8 @@ def analyze_forest(forest: list[parser.AstNode]) -> Analysis: add_family(node, instructions, families) case parser.Pseudo(): add_pseudo(node, instructions, pseudos) + case parser.LabelDef(): + add_label(node, labels) case _: pass for uop in uops.values(): @@ -1183,7 +1202,7 @@ def analyze_forest(forest: list[parser.AstNode]) -> Analysis: families["BINARY_OP"].members.append(inst) opmap, first_arg, min_instrumented = assign_opcodes(instructions, families, pseudos) return Analysis( - instructions, uops, families, pseudos, opmap, first_arg, min_instrumented + instructions, uops, families, pseudos, labels, opmap, first_arg, min_instrumented ) diff --git a/Tools/cases_generator/labels_generator.py b/Tools/cases_generator/labels_generator.py new file mode 100644 index 00000000000000..79e10fe0babcb8 --- /dev/null +++ b/Tools/cases_generator/labels_generator.py @@ -0,0 +1,172 @@ +"""Generate the main interpreter labels. +Reads the label definitions from bytecodes.c. +Writes the labels to generated_labels.c.h, which is #included in ceval.c. +""" + +import argparse + +from analyzer import ( + Analysis, + Instruction, + Uop, + Part, + analyze_files, + Skip, + Flush, + analysis_error, + StackItem, +) +from generators_common import ( + DEFAULT_INPUT, + ROOT, + write_header, + type_and_null, + Emitter, +) +from cwriter import CWriter +from typing import TextIO +from stack import Local, Stack, StackError, get_stack_effect, Storage + + +DEFAULT_OUTPUT = ROOT / "Python/generated_labels.c.h" + + + +def declare_variable(var: StackItem, out: CWriter) -> None: + type, null = type_and_null(var) + space = " " if type[-1].isalnum() else "" + if var.condition: + out.emit(f"{type}{space}{var.name} = {null};\n") + else: + out.emit(f"{type}{space}{var.name};\n") + + +def declare_variables(inst: Instruction, out: CWriter) -> None: + try: + stack = get_stack_effect(inst) + except StackError as ex: + raise analysis_error(ex.args[0], inst.where) from None + required = set(stack.defined) + required.discard("unused") + for part in inst.parts: + if not isinstance(part, Uop): + continue + for var in part.stack.inputs: + if var.name in required: + required.remove(var.name) + declare_variable(var, out) + for var in part.stack.outputs: + if var.name in required: + required.remove(var.name) + declare_variable(var, out) + + +def write_uop( + uop: Part, + emitter: Emitter, + offset: int, + stack: Stack, + inst: Instruction, + braces: bool, +) -> tuple[int, Stack]: + # out.emit(stack.as_comment() + "\n") + if isinstance(uop, Skip): + entries = "entries" if uop.size > 1 else "entry" + emitter.emit(f"/* Skip {uop.size} cache {entries} */\n") + return (offset + uop.size), stack + if isinstance(uop, Flush): + emitter.emit(f"// flush\n") + stack.flush(emitter.out) + return offset, stack + try: + locals: dict[str, Local] = {} + emitter.out.start_line() + if braces: + emitter.out.emit(f"// {uop.name}\n") + emitter.emit("{\n") + code_list, storage = Storage.for_uop(stack, uop) + emitter._print_storage(storage) + for code in code_list: + emitter.emit(code) + + for cache in uop.caches: + if cache.name != "unused": + if cache.size == 4: + type = "PyObject *" + reader = "read_obj" + else: + type = f"uint{cache.size*16}_t " + reader = f"read_u{cache.size*16}" + emitter.emit( + f"{type}{cache.name} = {reader}(&this_instr[{offset}].cache);\n" + ) + if inst.family is None: + emitter.emit(f"(void){cache.name};\n") + offset += cache.size + + storage = emitter.emit_tokens(uop, storage, inst) + if braces: + emitter.out.start_line() + emitter.emit("}\n") + # emitter.emit(stack.as_comment() + "\n") + return offset, storage.stack + except StackError as ex: + raise analysis_error(ex.args[0], uop.body[0]) + + +def uses_this(inst: Instruction) -> bool: + if inst.properties.needs_this: + return True + for uop in inst.parts: + if not isinstance(uop, Uop): + continue + for cache in uop.caches: + if cache.name != "unused": + return True + return False + + +def generate_labels( + filenames: list[str], analysis: Analysis, outfile: TextIO +) -> None: + write_header(__file__, filenames, outfile) + out = CWriter(outfile, 2, False) + out.emit("\n") + for name, label in analysis.labels.items(): + out.emit(f"{name}:\n") + for tkn in label.body: + out.emit(tkn) + out.emit("\n") + out.emit("\n") + out.emit("\n") + + +arg_parser = argparse.ArgumentParser( + description="Generate the code for the interpreter labels.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, +) + +arg_parser.add_argument( + "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT +) + +arg_parser.add_argument( + "input", nargs=argparse.REMAINDER, help="Instruction definition file(s)" +) + + +def generate_labels_from_files( + filenames: list[str], outfilename: str +) -> None: + data = analyze_files(filenames) + with open(outfilename, "w") as outfile: + generate_labels(filenames, data, outfile) + + +if __name__ == "__main__": + args = arg_parser.parse_args() + if len(args.input) == 0: + args.input.append(DEFAULT_INPUT) + data = analyze_files(args.input) + with open(args.output, "w") as outfile: + generate_labels(args.input, data, outfile) diff --git a/Tools/cases_generator/lexer.py b/Tools/cases_generator/lexer.py index bee2a185745f4d..cf3c39762f29cb 100644 --- a/Tools/cases_generator/lexer.py +++ b/Tools/cases_generator/lexer.py @@ -213,6 +213,9 @@ def choice(*opts: str) -> str: # A macro in the DSL MACRO = "MACRO" kwds.append(MACRO) +# A label in the DSL +LABEL = "LABEL" +kwds.append(LABEL) keywords = {name.lower(): name for name in kwds} ANNOTATION = "ANNOTATION" diff --git a/Tools/cases_generator/parser.py b/Tools/cases_generator/parser.py index db672ad5501f15..68bbb88719e682 100644 --- a/Tools/cases_generator/parser.py +++ b/Tools/cases_generator/parser.py @@ -3,6 +3,7 @@ Macro, Pseudo, Family, + LabelDef, Parser, Context, CacheEffect, diff --git a/Tools/cases_generator/parsing.py b/Tools/cases_generator/parsing.py index 41b36b6a546360..eb8c8a7ecd32e8 100644 --- a/Tools/cases_generator/parsing.py +++ b/Tools/cases_generator/parsing.py @@ -150,8 +150,13 @@ class Pseudo(Node): targets: list[str] # opcodes this can be replaced by as_sequence: bool +@dataclass +class LabelDef(Node): + name: str + block: Block -AstNode = InstDef | Macro | Pseudo | Family + +AstNode = InstDef | Macro | Pseudo | Family | LabelDef class Parser(PLexer): @@ -165,6 +170,18 @@ def definition(self) -> AstNode | None: return pseudo if inst := self.inst_def(): return inst + if label := self.label_def(): + return label + return None + + @contextual + def label_def(self) -> LabelDef | None: + if self.expect(lx.LABEL): + if self.expect(lx.LPAREN): + if tkn := self.expect(lx.IDENTIFIER): + if self.expect(lx.RPAREN): + if block := self.block(): + return LabelDef(tkn.text, block) return None @contextual From 5cccb986ba393cf38643b7ccb0b2d5cc28567188 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 21 Jan 2025 08:58:53 +0800 Subject: [PATCH 02/14] Fix mypy --- Tools/cases_generator/labels_generator.py | 105 ---------------------- 1 file changed, 105 deletions(-) diff --git a/Tools/cases_generator/labels_generator.py b/Tools/cases_generator/labels_generator.py index 79e10fe0babcb8..04a00ae35af89c 100644 --- a/Tools/cases_generator/labels_generator.py +++ b/Tools/cases_generator/labels_generator.py @@ -7,125 +7,20 @@ from analyzer import ( Analysis, - Instruction, - Uop, - Part, analyze_files, - Skip, - Flush, - analysis_error, - StackItem, ) from generators_common import ( DEFAULT_INPUT, ROOT, write_header, - type_and_null, - Emitter, ) from cwriter import CWriter from typing import TextIO -from stack import Local, Stack, StackError, get_stack_effect, Storage DEFAULT_OUTPUT = ROOT / "Python/generated_labels.c.h" - -def declare_variable(var: StackItem, out: CWriter) -> None: - type, null = type_and_null(var) - space = " " if type[-1].isalnum() else "" - if var.condition: - out.emit(f"{type}{space}{var.name} = {null};\n") - else: - out.emit(f"{type}{space}{var.name};\n") - - -def declare_variables(inst: Instruction, out: CWriter) -> None: - try: - stack = get_stack_effect(inst) - except StackError as ex: - raise analysis_error(ex.args[0], inst.where) from None - required = set(stack.defined) - required.discard("unused") - for part in inst.parts: - if not isinstance(part, Uop): - continue - for var in part.stack.inputs: - if var.name in required: - required.remove(var.name) - declare_variable(var, out) - for var in part.stack.outputs: - if var.name in required: - required.remove(var.name) - declare_variable(var, out) - - -def write_uop( - uop: Part, - emitter: Emitter, - offset: int, - stack: Stack, - inst: Instruction, - braces: bool, -) -> tuple[int, Stack]: - # out.emit(stack.as_comment() + "\n") - if isinstance(uop, Skip): - entries = "entries" if uop.size > 1 else "entry" - emitter.emit(f"/* Skip {uop.size} cache {entries} */\n") - return (offset + uop.size), stack - if isinstance(uop, Flush): - emitter.emit(f"// flush\n") - stack.flush(emitter.out) - return offset, stack - try: - locals: dict[str, Local] = {} - emitter.out.start_line() - if braces: - emitter.out.emit(f"// {uop.name}\n") - emitter.emit("{\n") - code_list, storage = Storage.for_uop(stack, uop) - emitter._print_storage(storage) - for code in code_list: - emitter.emit(code) - - for cache in uop.caches: - if cache.name != "unused": - if cache.size == 4: - type = "PyObject *" - reader = "read_obj" - else: - type = f"uint{cache.size*16}_t " - reader = f"read_u{cache.size*16}" - emitter.emit( - f"{type}{cache.name} = {reader}(&this_instr[{offset}].cache);\n" - ) - if inst.family is None: - emitter.emit(f"(void){cache.name};\n") - offset += cache.size - - storage = emitter.emit_tokens(uop, storage, inst) - if braces: - emitter.out.start_line() - emitter.emit("}\n") - # emitter.emit(stack.as_comment() + "\n") - return offset, storage.stack - except StackError as ex: - raise analysis_error(ex.args[0], uop.body[0]) - - -def uses_this(inst: Instruction) -> bool: - if inst.properties.needs_this: - return True - for uop in inst.parts: - if not isinstance(uop, Uop): - continue - for cache in uop.caches: - if cache.name != "unused": - return True - return False - - def generate_labels( filenames: list[str], analysis: Analysis, outfile: TextIO ) -> None: From 3ae88a5b8863063b84fcf0de2cc131b10822d415 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 21 Jan 2025 09:01:39 +0800 Subject: [PATCH 03/14] lint --- Lib/test/test_generated_cases.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index e0ae87bc859675..2f543829c0596a 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -1838,12 +1838,12 @@ def test_multiple_labels(self): do_thing1(); goto my_label_2; } - + label(my_label_2) { // Comment do_thing2(); goto my_label_3; - } + } """ output = """ From af4bf1a34d389bff47761eec7d2dd7f61973b421 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 21 Jan 2025 15:52:53 +0800 Subject: [PATCH 04/14] Add to generated files --- .gitattributes | 1 + Tools/c-analyzer/cpython/_parser.py | 1 + 2 files changed, 2 insertions(+) diff --git a/.gitattributes b/.gitattributes index 2f5a030981fb94..67b5a56eed564f 100644 --- a/.gitattributes +++ b/.gitattributes @@ -95,6 +95,7 @@ Programs/test_frozenmain.h generated Python/Python-ast.c generated Python/executor_cases.c.h generated Python/generated_cases.c.h generated +Python/generated_labels.c.h generated Python/optimizer_cases.c.h generated Python/opcode_targets.h generated Python/stdlib_module_names.h generated diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index a08b32fa45db3e..396d840dc43f12 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -80,6 +80,7 @@ def clean_lines(text): Python/deepfreeze/*.c Python/frozen_modules/*.h Python/generated_cases.c.h +Python/generated_labels.c.h Python/executor_cases.c.h Python/optimizer_cases.c.h From b911bb1bb60577e9708d3a92e9f17298d5f7d11b Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 21 Jan 2025 20:11:27 +0800 Subject: [PATCH 05/14] Move labels into tier 1 generator --- .gitattributes | 1 - Lib/test/test_generated_cases.py | 51 -------- Makefile.pre.in | 9 +- PCbuild/regen.targets | 2 - Python/ceval.c | 30 ----- Python/generated_cases.c.h | 153 ++++++++++++++++++++++ Tools/c-analyzer/cpython/_parser.py | 1 - Tools/cases_generator/labels_generator.py | 67 ---------- Tools/cases_generator/tier1_generator.py | 53 +++++++- 9 files changed, 205 insertions(+), 162 deletions(-) delete mode 100644 Tools/cases_generator/labels_generator.py diff --git a/.gitattributes b/.gitattributes index 67b5a56eed564f..2f5a030981fb94 100644 --- a/.gitattributes +++ b/.gitattributes @@ -95,7 +95,6 @@ Programs/test_frozenmain.h generated Python/Python-ast.c generated Python/executor_cases.c.h generated Python/generated_cases.c.h generated -Python/generated_labels.c.h generated Python/optimizer_cases.c.h generated Python/opcode_targets.h generated Python/stdlib_module_names.h generated diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 2f543829c0596a..a60eb4bdc3773e 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -36,7 +36,6 @@ def skip_if_different_mount_drives(): import parser from stack import Local, Stack import tier1_generator - import labels_generator import opcode_metadata_generator import optimizer_generator @@ -1757,55 +1756,6 @@ def test_kill_in_wrong_order(self): with self.assertRaises(SyntaxError): self.run_cases_test(input, "") - -class TestGeneratedLabels(unittest.TestCase): - def setUp(self) -> None: - super().setUp() - self.maxDiff = None - - self.temp_dir = tempfile.gettempdir() - self.temp_input_filename = os.path.join(self.temp_dir, "input.txt") - self.temp_output_filename = os.path.join(self.temp_dir, "output.txt") - self.temp_metadata_filename = os.path.join(self.temp_dir, "metadata.txt") - self.temp_pymetadata_filename = os.path.join(self.temp_dir, "pymetadata.txt") - self.temp_executor_filename = os.path.join(self.temp_dir, "executor.txt") - - def tearDown(self) -> None: - for filename in [ - self.temp_input_filename, - self.temp_output_filename, - self.temp_metadata_filename, - self.temp_pymetadata_filename, - self.temp_executor_filename, - ]: - try: - os.remove(filename) - except Exception: - pass - super().tearDown() - - def run_cases_test(self, input: str, expected: str): - with open(self.temp_input_filename, "w+") as temp_input: - temp_input.write(parser.BEGIN_MARKER) - temp_input.write(input) - temp_input.write(parser.END_MARKER) - temp_input.flush() - - with handle_stderr(): - labels_generator.generate_labels_from_files( - [self.temp_input_filename], self.temp_output_filename - ) - - with open(self.temp_output_filename) as temp_output: - lines = temp_output.readlines() - while lines and lines[0].startswith(("// ", "#", " #", "\n")): - lines.pop(0) - while lines and lines[-1].startswith(("#", "\n")): - lines.pop(-1) - actual = "".join(lines) - - self.assertEqual(actual.strip(), expected.strip()) - def test_complex_label(self): input = """ label(my_label) { @@ -1861,7 +1811,6 @@ def test_multiple_labels(self): goto my_label_3; } """ - self.run_cases_test(input, output) class TestGeneratedAbstractCases(unittest.TestCase): def setUp(self) -> None: diff --git a/Makefile.pre.in b/Makefile.pre.in index a13c25f74b9243..67acf0fc520087 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1985,7 +1985,7 @@ Objects/mimalloc/page.o: $(srcdir)/Objects/mimalloc/page-queue.c .PHONY: regen-cases regen-cases: \ regen-opcode-ids regen-opcode-targets regen-uop-ids regen-opcode-metadata-py \ - regen-generated-cases regen-generated-labels regen-executor-cases regen-optimizer-cases \ + regen-generated-cases regen-executor-cases regen-optimizer-cases \ regen-opcode-metadata regen-uop-metadata .PHONY: regen-opcode-ids @@ -2018,13 +2018,6 @@ regen-generated-cases: -o $(srcdir)/Python/generated_cases.c.h.new $(srcdir)/Python/bytecodes.c $(UPDATE_FILE) $(srcdir)/Python/generated_cases.c.h $(srcdir)/Python/generated_cases.c.h.new -.PHONY: regen-generated-labels -regen-generated-labels: - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/labels_generator.py \ - -o $(srcdir)/Python/generated_labels.c.h.new $(srcdir)/Python/bytecodes.c - $(UPDATE_FILE) $(srcdir)/Python/generated_labels.c.h $(srcdir)/Python/generated_labels.c.h.new - - .PHONY: regen-executor-cases regen-executor-cases: $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/tier2_generator.py \ diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index cc91479675d478..416241d9d0df10 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -100,8 +100,6 @@ WorkingDirectory="$(PySourcePath)" /> - op.code; - _PyErr_Format(tstate, PyExc_SystemError, - "%U:%d: unknown opcode %d", - _PyFrame_GetCode(frame)->co_filename, - PyUnstable_InterpreterFrame_GetLine(frame), - opcode); - goto error; - - } /* End instructions */ - - /* This should never be reached. Every opcode should end with DISPATCH() - or goto error. */ - Py_UNREACHABLE(); - -#include "generated_labels.c.h" - #ifdef _Py_TIER2 // Tier 2 is also here! diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index dc90f75f2645e1..2fc065b119b301 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -8,6 +8,13 @@ #endif #define TIER_ONE 1 +/* Start instructions */ +#if !USE_COMPUTED_GOTOS + dispatch_opcode: + switch (opcode) +#endif + { + TARGET(BINARY_OP) { frame->instr_ptr = next_instr; @@ -8431,4 +8438,150 @@ assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } + +#if USE_COMPUTED_GOTOS + _unknown_opcode: +#else + EXTRA_CASES // From pycore_opcode_metadata.h, a 'case' for each unused opcode +#endif + /* Tell C compilers not to hold the opcode variable in the loop. + next_instr points the current instruction without TARGET(). */ + opcode = next_instr->op.code; + _PyErr_Format(tstate, PyExc_SystemError, + "%U:%d: unknown opcode %d", + _PyFrame_GetCode(frame)->co_filename, + PyUnstable_InterpreterFrame_GetLine(frame), + opcode); + goto error; + + } /* End instructions */ + + /* This should never be reached. Every opcode should end with DISPATCH() + or goto error. */ + Py_UNREACHABLE(); + + pop_4_error: + { + STACK_SHRINK(1); + goto pop_3_error; + } + + pop_3_error: + { + STACK_SHRINK(1); + goto pop_2_error; + } + + pop_2_error: + { + STACK_SHRINK(1); + goto pop_1_error; + } + + pop_1_error: + { + STACK_SHRINK(1); + goto error; + } + + error: + { + /* Double-check exception status. */ + #ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + } + #else + assert(_PyErr_Occurred(tstate)); + #endif + + /* Log traceback info. */ + assert(frame != &entry_frame); + if (!_PyFrame_IsIncomplete(frame)) { + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + if (f != NULL) { + PyTraceBack_Here(f); + } + } + _PyEval_MonitorRaise(tstate, frame, next_instr-1); + goto exception_unwind; + } + + exception_unwind: + { + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); + /* Pop remaining stack entries. */ + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyStackRef_XCLOSE(POP()); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_unwind(tstate, frame, next_instr-1); + goto exit_unwind; + } + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyStackRef_XCLOSE(POP()); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + PyObject *lasti = PyLong_FromLong(frame_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(PyStackRef_FromPyObjectSteal(lasti)); + } + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + PyObject *exc = _PyErr_GetRaisedException(tstate); + PUSH(PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + goto exception_unwind; + } + /* Resume normal execution */ + #ifdef LLTRACE + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } + #endif + DISPATCH(); + } + + exit_unwind: + { + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame != &entry_frame); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame == &entry_frame) { + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; + return NULL; + } + goto resume_with_error; + } + + resume_with_error: + { + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + goto error; + } + #undef TIER_ONE diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index 396d840dc43f12..a08b32fa45db3e 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -80,7 +80,6 @@ def clean_lines(text): Python/deepfreeze/*.c Python/frozen_modules/*.h Python/generated_cases.c.h -Python/generated_labels.c.h Python/executor_cases.c.h Python/optimizer_cases.c.h diff --git a/Tools/cases_generator/labels_generator.py b/Tools/cases_generator/labels_generator.py deleted file mode 100644 index 04a00ae35af89c..00000000000000 --- a/Tools/cases_generator/labels_generator.py +++ /dev/null @@ -1,67 +0,0 @@ -"""Generate the main interpreter labels. -Reads the label definitions from bytecodes.c. -Writes the labels to generated_labels.c.h, which is #included in ceval.c. -""" - -import argparse - -from analyzer import ( - Analysis, - analyze_files, -) -from generators_common import ( - DEFAULT_INPUT, - ROOT, - write_header, -) -from cwriter import CWriter -from typing import TextIO - - -DEFAULT_OUTPUT = ROOT / "Python/generated_labels.c.h" - - -def generate_labels( - filenames: list[str], analysis: Analysis, outfile: TextIO -) -> None: - write_header(__file__, filenames, outfile) - out = CWriter(outfile, 2, False) - out.emit("\n") - for name, label in analysis.labels.items(): - out.emit(f"{name}:\n") - for tkn in label.body: - out.emit(tkn) - out.emit("\n") - out.emit("\n") - out.emit("\n") - - -arg_parser = argparse.ArgumentParser( - description="Generate the code for the interpreter labels.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, -) - -arg_parser.add_argument( - "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT -) - -arg_parser.add_argument( - "input", nargs=argparse.REMAINDER, help="Instruction definition file(s)" -) - - -def generate_labels_from_files( - filenames: list[str], outfilename: str -) -> None: - data = analyze_files(filenames) - with open(outfilename, "w") as outfile: - generate_labels(filenames, data, outfile) - - -if __name__ == "__main__": - args = arg_parser.parse_args() - if len(args.input) == 0: - args.input.append(DEFAULT_INPUT) - data = analyze_files(args.input) - with open(args.output, "w") as outfile: - generate_labels(args.input, data, outfile) diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 40562da99b20ea..0c964a477a8d3d 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -138,8 +138,56 @@ def generate_tier1( #error "This file is for Tier 1 only" #endif #define TIER_ONE 1 + +/* Start instructions */ +#if !USE_COMPUTED_GOTOS + dispatch_opcode: + switch (opcode) +#endif + { """ ) + generate_tier1_cases(analysis, outfile, lines) + outfile.write(""" +#if USE_COMPUTED_GOTOS + _unknown_opcode: +#else + EXTRA_CASES // From pycore_opcode_metadata.h, a 'case' for each unused opcode +#endif + /* Tell C compilers not to hold the opcode variable in the loop. + next_instr points the current instruction without TARGET(). */ + opcode = next_instr->op.code; + _PyErr_Format(tstate, PyExc_SystemError, + "%U:%d: unknown opcode %d", + _PyFrame_GetCode(frame)->co_filename, + PyUnstable_InterpreterFrame_GetLine(frame), + opcode); + goto error; + + } /* End instructions */ + + /* This should never be reached. Every opcode should end with DISPATCH() + or goto error. */ + Py_UNREACHABLE(); +""") + generate_tier1_labels(analysis, outfile, lines) + outfile.write(FOOTER) + +def generate_tier1_labels( + analysis: Analysis, outfile: TextIO, lines: bool +) -> None: + out = CWriter(outfile, 2, lines) + out.emit("\n") + for name, label in analysis.labels.items(): + out.emit(f"{name}:\n") + for tkn in label.body: + out.emit(tkn) + out.emit("\n") + out.emit("\n") + +def generate_tier1_cases( + analysis: Analysis, outfile: TextIO, lines: bool +) -> None: out = CWriter(outfile, 2, lines) emitter = Emitter(out) out.emit("\n") @@ -185,7 +233,7 @@ def generate_tier1( out.start_line() out.emit("}") out.emit("\n") - outfile.write(FOOTER) + arg_parser = argparse.ArgumentParser( @@ -211,7 +259,8 @@ def generate_tier1_from_files( ) -> None: data = analyze_files(filenames) with open(outfilename, "w") as outfile: - generate_tier1(filenames, data, outfile, lines) + generate_tier1_cases(data, outfile, lines) + generate_tier1_labels(data, outfile, lines) if __name__ == "__main__": From 63a88ab5677d70f47392b3868d148c2765e85c37 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 21 Jan 2025 20:13:12 +0800 Subject: [PATCH 06/14] remove outdated file --- Python/generated_labels.c.h | 130 ------------------------------------ 1 file changed, 130 deletions(-) delete mode 100644 Python/generated_labels.c.h diff --git a/Python/generated_labels.c.h b/Python/generated_labels.c.h deleted file mode 100644 index 9d0b959353a5b8..00000000000000 --- a/Python/generated_labels.c.h +++ /dev/null @@ -1,130 +0,0 @@ -// This file is generated by Tools/cases_generator/labels_generator.py -// from: -// Python/bytecodes.c -// Do not edit! - - pop_4_error: - { - STACK_SHRINK(1); - goto pop_3_error; - } - - pop_3_error: - { - STACK_SHRINK(1); - goto pop_2_error; - } - - pop_2_error: - { - STACK_SHRINK(1); - goto pop_1_error; - } - - pop_1_error: - { - STACK_SHRINK(1); - goto error; - } - - error: - { - /* Double-check exception status. */ - #ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, - "error return without exception set"); - } - #else - assert(_PyErr_Occurred(tstate)); - #endif - - /* Log traceback info. */ - assert(frame != &entry_frame); - if (!_PyFrame_IsIncomplete(frame)) { - PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyTraceBack_Here(f); - } - } - _PyEval_MonitorRaise(tstate, frame, next_instr-1); - goto exception_unwind; - } - - exception_unwind: - { - /* We can't use frame->instr_ptr here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); - /* Pop remaining stack entries. */ - _PyStackRef *stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyStackRef_XCLOSE(POP()); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_unwind(tstate, frame, next_instr-1); - goto exit_unwind; - } - assert(STACK_LEVEL() >= level); - _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyStackRef_XCLOSE(POP()); - } - if (lasti) { - int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - PUSH(PyStackRef_FromPyObjectSteal(lasti)); - } - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - PyObject *exc = _PyErr_GetRaisedException(tstate); - PUSH(PyStackRef_FromPyObjectSteal(exc)); - next_instr = _PyFrame_GetBytecode(frame) + handler; - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { - goto exception_unwind; - } - /* Resume normal execution */ - #ifdef LLTRACE - if (lltrace >= 5) { - lltrace_resume_frame(frame); - } - #endif - DISPATCH(); - } - - exit_unwind: - { - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - if (frame == &entry_frame) { - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; - return NULL; - } - goto resume_with_error; - } - - resume_with_error: - { - next_instr = frame->instr_ptr; - stack_pointer = _PyFrame_GetStackPointer(frame); - goto error; - } - - From c6df7a1285e76c0c92fdefb88478f24b2c00c90a Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 21 Jan 2025 20:14:10 +0800 Subject: [PATCH 07/14] cleanup diff --- Tools/cases_generator/tier1_generator.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 0c964a477a8d3d..66791a36944e37 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -235,7 +235,6 @@ def generate_tier1_cases( out.emit("\n") - arg_parser = argparse.ArgumentParser( description="Generate the code for the interpreter switch.", formatter_class=argparse.ArgumentDefaultsHelpFormatter, From 1ec803345165dc6b917e4ec66b90da364e9efa87 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 21 Jan 2025 20:16:42 +0800 Subject: [PATCH 08/14] Lint --- Python/generated_cases.c.h | 2 +- Tools/cases_generator/tier1_generator.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 2fc065b119b301..52d742c3f76583 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -8458,7 +8458,7 @@ /* This should never be reached. Every opcode should end with DISPATCH() or goto error. */ - Py_UNREACHABLE(); + Py_UNREACHABLE(); pop_4_error: { diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 66791a36944e37..3afedec450ef5f 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -168,7 +168,7 @@ def generate_tier1( /* This should never be reached. Every opcode should end with DISPATCH() or goto error. */ - Py_UNREACHABLE(); + Py_UNREACHABLE(); """) generate_tier1_labels(analysis, outfile, lines) outfile.write(FOOTER) From 15ca6dd3927e599f884c7c4528a7bf280cea632f Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 22 Jan 2025 09:27:37 +0800 Subject: [PATCH 09/14] fix upstream changes --- Python/bytecodes.c | 2 +- Python/generated_cases.c.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 133a0a653ec168..b03c50b30f0641 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5282,7 +5282,7 @@ dummy_func( } /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { + if (frame->lltrace >= 5) { lltrace_resume_frame(frame); } #endif diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 0891a3324e80b9..fab8aad7808144 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -8636,7 +8636,7 @@ } /* Resume normal execution */ #ifdef LLTRACE - if (lltrace >= 5) { + if (frame->lltrace >= 5) { lltrace_resume_frame(frame); } #endif From e1f94758cdec9d071e3951c5dad1b711e9bdd02e Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 22 Jan 2025 09:39:54 +0800 Subject: [PATCH 10/14] Remove entry_frame --- Python/bytecodes.c | 5 ++--- Python/generated_cases.c.h | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index b03c50b30f0641..2fd876cc79a6bf 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -104,7 +104,6 @@ dummy_func( PyObject *codeobj; PyObject *cond; PyObject *descr; - _PyInterpreterFrame entry_frame; PyObject *exc; PyObject *exit; PyObject *fget; @@ -5225,7 +5224,7 @@ dummy_func( #endif /* Log traceback info. */ - assert(frame != &entry_frame); + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); if (!_PyFrame_IsIncomplete(frame)) { PyFrameObject *f = _PyFrame_GetFrameObject(frame); if (f != NULL) { @@ -5292,7 +5291,7 @@ dummy_func( label(exit_unwind) { assert(_PyErr_Occurred(tstate)); _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index fab8aad7808144..86557066899817 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -8582,7 +8582,7 @@ #endif /* Log traceback info. */ - assert(frame != &entry_frame); + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); if (!_PyFrame_IsIncomplete(frame)) { PyFrameObject *f = _PyFrame_GetFrameObject(frame); if (f != NULL) { @@ -8647,7 +8647,7 @@ { assert(_PyErr_Occurred(tstate)); _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; From 5d561305ee0db2c72e581b95751e80b2bd23c2bb Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 23 Jan 2025 02:00:44 +0800 Subject: [PATCH 11/14] Address review by removing in test cases generator --- Lib/test/test_generated_cases.py | 17 +++++++++++++++++ Tools/cases_generator/tier1_generator.py | 20 +++++++++++++------- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 99c9bb1c5d7d7f..c1cd714da953ef 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -253,6 +253,23 @@ def run_cases_test(self, input: str, expected: str): lines.pop(0) while lines and lines[-1].startswith(("#", "\n")): lines.pop(-1) + while lines and tier1_generator.INSTRUCTION_START_MARKER not in lines[0]: + lines.pop(0) + lines.pop(0) + for instruction_end_marker_index, line in enumerate(lines): + if tier1_generator.INSTRUCTION_END_MARKER in line: + break + else: + assert False, "No instruction end marker found." + for label_start_marker_index, line in enumerate(lines): + if tier1_generator.LABEL_START_MARKER in line: + break + else: + assert False, "No label start marker found." + del lines[instruction_end_marker_index:label_start_marker_index+1] + # Pop the label markers themselves + lines.pop(0) + lines.pop(-1) actual = "".join(lines) # if actual.strip() != expected.strip(): # print("Actual:") diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index ab46164caeeb87..349f324b6900c9 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -32,6 +32,10 @@ FOOTER = "#undef TIER_ONE\n" +INSTRUCTION_START_MARKER = "/* BEGIN INSTRUCTIONS */" +INSTRUCTION_END_MARKER = "/* END INSTRUCTIONS */" +LABEL_START_MARKER = "/* BEGIN LABEL */" +LABEL_END_MARKER = "/* END LABEL */" def declare_variable(var: StackItem, out: CWriter) -> None: @@ -130,22 +134,23 @@ def generate_tier1( ) -> None: write_header(__file__, filenames, outfile) outfile.write( - """ + f""" #ifdef TIER_TWO #error "This file is for Tier 1 only" #endif #define TIER_ONE 1 -/* Start instructions */ #if !USE_COMPUTED_GOTOS dispatch_opcode: switch (opcode) #endif - { + {{ + {INSTRUCTION_START_MARKER} """ ) generate_tier1_cases(analysis, outfile, lines) - outfile.write(""" + outfile.write(f""" + {INSTRUCTION_END_MARKER} #if USE_COMPUTED_GOTOS _unknown_opcode: #else @@ -161,13 +166,15 @@ def generate_tier1( opcode); goto error; - } /* End instructions */ + }} /* This should never be reached. Every opcode should end with DISPATCH() or goto error. */ Py_UNREACHABLE(); + {LABEL_START_MARKER} """) generate_tier1_labels(analysis, outfile, lines) + outfile.write(f"{LABEL_END_MARKER}\n") outfile.write(FOOTER) def generate_tier1_labels( @@ -255,8 +262,7 @@ def generate_tier1_from_files( ) -> None: data = analyze_files(filenames) with open(outfilename, "w") as outfile: - generate_tier1_cases(data, outfile, lines) - generate_tier1_labels(data, outfile, lines) + generate_tier1(filenames, data, outfile, lines) if __name__ == "__main__": From e4a9de79140b7f5df66db581a9579616dbe942d5 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 23 Jan 2025 02:05:13 +0800 Subject: [PATCH 12/14] Regen files --- Python/generated_cases.c.h | 7 +++++-- Tools/cases_generator/tier1_generator.py | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 86557066899817..bdf4e74cfaf3c2 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -8,12 +8,12 @@ #endif #define TIER_ONE 1 -/* Start instructions */ #if !USE_COMPUTED_GOTOS dispatch_opcode: switch (opcode) #endif { + /* BEGIN INSTRUCTIONS */ TARGET(BINARY_OP) { @@ -8524,6 +8524,7 @@ DISPATCH(); } + /* END INSTRUCTIONS */ #if USE_COMPUTED_GOTOS _unknown_opcode: #else @@ -8539,11 +8540,12 @@ opcode); goto error; - } /* End instructions */ + } /* This should never be reached. Every opcode should end with DISPATCH() or goto error. */ Py_UNREACHABLE(); + /* BEGIN LABELS */ pop_4_error: { @@ -8669,4 +8671,5 @@ goto error; } +/* END LABELS */ #undef TIER_ONE diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 349f324b6900c9..dee0f805b5540a 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -34,8 +34,8 @@ FOOTER = "#undef TIER_ONE\n" INSTRUCTION_START_MARKER = "/* BEGIN INSTRUCTIONS */" INSTRUCTION_END_MARKER = "/* END INSTRUCTIONS */" -LABEL_START_MARKER = "/* BEGIN LABEL */" -LABEL_END_MARKER = "/* END LABEL */" +LABEL_START_MARKER = "/* BEGIN LABELS */" +LABEL_END_MARKER = "/* END LABELS */" def declare_variable(var: StackItem, out: CWriter) -> None: From f89f147940645823bda29669b7870adb67530cdc Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 25 Jan 2025 11:18:43 +0800 Subject: [PATCH 13/14] Address review --- Lib/test/test_generated_cases.py | 29 ++++++----------------------- 1 file changed, 6 insertions(+), 23 deletions(-) diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index c1cd714da953ef..a4bea0b107ecad 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -248,29 +248,12 @@ def run_cases_test(self, input: str, expected: str): ) with open(self.temp_output_filename) as temp_output: - lines = temp_output.readlines() - while lines and lines[0].startswith(("// ", "#", " #", "\n")): - lines.pop(0) - while lines and lines[-1].startswith(("#", "\n")): - lines.pop(-1) - while lines and tier1_generator.INSTRUCTION_START_MARKER not in lines[0]: - lines.pop(0) - lines.pop(0) - for instruction_end_marker_index, line in enumerate(lines): - if tier1_generator.INSTRUCTION_END_MARKER in line: - break - else: - assert False, "No instruction end marker found." - for label_start_marker_index, line in enumerate(lines): - if tier1_generator.LABEL_START_MARKER in line: - break - else: - assert False, "No label start marker found." - del lines[instruction_end_marker_index:label_start_marker_index+1] - # Pop the label markers themselves - lines.pop(0) - lines.pop(-1) - actual = "".join(lines) + lines = temp_output.read() + _, rest = lines.split(tier1_generator.INSTRUCTION_START_MARKER) + instructions, labels_with_prelude_and_postlude = rest.split(tier1_generator.INSTRUCTION_END_MARKER) + _, labels_with_postlude = labels_with_prelude_and_postlude.split(tier1_generator.LABEL_START_MARKER) + labels, _ = labels_with_postlude.split(tier1_generator.LABEL_END_MARKER) + actual = instructions + labels # if actual.strip() != expected.strip(): # print("Actual:") # print(actual) From 56413ae4d6c44c4cc8e81b48c99ad5b5e1c91d9b Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 25 Jan 2025 15:34:38 +0800 Subject: [PATCH 14/14] fixup --- Python/bytecodes.c | 2 +- Python/generated_cases.c.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 2fd876cc79a6bf..61d0f1d8f09cbb 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5297,7 +5297,7 @@ dummy_func( frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); frame->return_offset = 0; - if (frame == &entry_frame) { + if (frame->owner == FRAME_OWNED_BY_INTERPRETER) { /* Restore previous frame and exit */ tstate->current_frame = frame->previous; tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index bdf4e74cfaf3c2..3fab4c77b3d050 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -8655,7 +8655,7 @@ frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); frame->return_offset = 0; - if (frame == &entry_frame) { + if (frame->owner == FRAME_OWNED_BY_INTERPRETER) { /* Restore previous frame and exit */ tstate->current_frame = frame->previous; tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy