From 714c3f9b798a413183b2bdf232499d1091f9a4b8 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Thu, 13 Feb 2025 16:37:21 +0000 Subject: [PATCH 01/18] gh-130080: implement PEP 765 --- Include/internal/pycore_compile.h | 1 + Lib/test/test_except_star.py | 6 +- Lib/test/test_syntax.py | 159 +++++++++++++++++++++++------- Python/ast_opt.c | 135 +++++++++++++++++++++++-- Python/compile.c | 4 +- 5 files changed, 258 insertions(+), 47 deletions(-) diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index 9f0ca33892a43b..6dfd5598604cd3 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -36,6 +36,7 @@ extern int _PyCompile_AstOptimize( extern int _PyAST_Optimize( struct _mod *, struct _arena *arena, + PyObject *filename, int optimize, int ff_features); diff --git a/Lib/test/test_except_star.py b/Lib/test/test_except_star.py index 284907f61213f8..47006c6d3a0c36 100644 --- a/Lib/test/test_except_star.py +++ b/Lib/test/test_except_star.py @@ -84,7 +84,8 @@ def test_break_in_except_star(self): if i == 2: break finally: - return 0 + pass + return 0 """) @@ -117,7 +118,8 @@ def test_continue_in_except_star_block_invalid(self): if i == 2: continue finally: - return 0 + pass + return 0 """) def test_return_in_except_star_block_invalid(self): diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index 6d06e6f6dcb566..24e30c7ce56e4a 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -846,7 +846,7 @@ SyntaxError: 'function call' is an illegal expression for augmented assignment -Test continue in finally in weird combinations. +Test control flow in finally continue in for loop under finally should be ok. @@ -860,51 +860,63 @@ >>> test() 9 -continue in a finally should be ok. +break in for loop under finally should be ok. >>> def test(): - ... for abc in range(10): - ... try: - ... pass - ... finally: - ... continue - ... print(abc) + ... try: + ... pass + ... finally: + ... for abc in range(10): + ... break + ... print(abc) >>> test() - 9 + 0 + +return in function under finally should be ok. >>> def test(): - ... for abc in range(10): - ... try: - ... pass - ... finally: - ... try: - ... continue - ... except: - ... pass - ... print(abc) + ... try: + ... pass + ... finally: + ... def f(): + ... return 42 + ... print(f()) >>> test() - 9 + 42 + +combine for loop and function def + +return in function under finally should be ok. >>> def test(): - ... for abc in range(10): - ... try: - ... pass - ... finally: - ... try: - ... pass - ... except: - ... continue - ... print(abc) + ... try: + ... pass + ... finally: + ... for i in range(10): + ... def f(): + ... return 42 + ... print(f()) >>> test() - 9 + 42 + + >>> def test(): + ... try: + ... pass + ... finally: + ... def f(): + ... for i in range(10): + ... return 42 + ... print(f()) + >>> test() + 42 A continue outside loop should not be allowed. >>> def foo(): ... try: - ... pass - ... finally: ... continue + ... finally: + ... pass Traceback (most recent call last): ... SyntaxError: 'continue' not properly in loop @@ -2381,7 +2393,88 @@ def f(x: *b) from test import support -class SyntaxTestCase(unittest.TestCase): +class SyntaxWarningTest(unittest.TestCase): + def check_warning(self, code, errtext, filename="", mode="exec"): + """Check that compiling code raises SyntaxWarning with errtext. + + errtest is a regular expression that must be present in the + text of the warning raised. + """ + with self.assertWarnsRegex(SyntaxWarning, errtext): + compile(code, filename, mode) + + def test_return_in_finally(self): + source = textwrap.dedent(""" + def f(): + try: + pass + finally: + return 42 + """) + self.check_warning(source, "'return' in a 'finally' block") + + source = textwrap.dedent(""" + def f(): + try: + pass + finally: + try: + return 42 + except: + pass + """) + self.check_warning(source, "'return' in a 'finally' block") + + source = textwrap.dedent(""" + def f(): + try: + pass + finally: + try: + pass + except: + return 42 + """) + self.check_warning(source, "'return' in a 'finally' block") + + def test_break_and_continue_in_finally(self): + for kw in ('break', 'continue'): + + source = textwrap.dedent(f""" + for abc in range(10): + try: + pass + finally: + {kw} + """) + self.check_warning(source, f"'{kw}' in a 'finally' block") + + source = textwrap.dedent(f""" + for abc in range(10): + try: + pass + finally: + try: + {kw} + except: + pass + """) + self.check_warning(source, f"'{kw}' in a 'finally' block") + + source = textwrap.dedent(f""" + for abc in range(10): + try: + pass + finally: + try: + pass + except: + {kw} + """) + self.check_warning(source, f"'{kw}' in a 'finally' block") + + +class SyntaxErrorTestCase(unittest.TestCase): def _check_error(self, code, errtext, filename="", mode="exec", subclass=None, @@ -2389,7 +2482,7 @@ def _check_error(self, code, errtext, """Check that compiling code raises SyntaxError with errtext. errtest is a regular expression that must be present in the - test of the exception raised. If subclass is specified it + text of the exception raised. If subclass is specified it is the expected subclass of SyntaxError (e.g. IndentationError). """ try: diff --git a/Python/ast_opt.c b/Python/ast_opt.c index 0b58e8cd2a2ced..ecf6d58785b93b 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -7,12 +7,22 @@ #include "pycore_setobject.h" // _PySet_NextEntry() +/* See PEP 765 */ typedef struct { + bool in_finally; + bool in_funcdef; + bool in_loop; +} ControlFlowInFinallyState; + +typedef struct { + PyObject *filename; int optimize; int ff_features; int recursion_depth; /* current recursion depth */ int recursion_limit; /* recursion limit */ + + ControlFlowInFinallyState cf_finally; } _PyASTOptimizeState; #define ENTER_RECURSIVE(ST) \ @@ -29,6 +39,82 @@ typedef struct { --(ST)->recursion_depth; \ } while(0) + +static ControlFlowInFinallyState +overwrite_state(_PyASTOptimizeState *state, bool finally, bool funcdef, bool loop) +{ + ControlFlowInFinallyState saved = state->cf_finally; + state->cf_finally.in_finally = finally; + state->cf_finally.in_funcdef = funcdef; + state->cf_finally.in_loop = loop; + return saved; +} + +static int +restore_state(_PyASTOptimizeState *state, ControlFlowInFinallyState *saved) +{ + state->cf_finally = *saved; + return 1; +} + +static int +control_flow_in_finally_warning(const char *kw, stmt_ty n, _PyASTOptimizeState *state) +{ + PyObject *msg = PyUnicode_FromFormat("'%s' in a 'finally' block", kw); + if (msg == NULL) { + return 0; + } + int ret = _PyErr_EmitSyntaxWarning(msg, state->filename, n->lineno, + n->col_offset + 1, n->end_lineno, + n->end_col_offset + 1); + Py_DECREF(msg); + return ret < 0 ? 0 : 1; +} + +static int +before_return(_PyASTOptimizeState *state, stmt_ty node_) +{ + if (state->cf_finally.in_finally && ! state->cf_finally.in_funcdef) { + if (!control_flow_in_finally_warning("return", node_, state)) { + return 0; + } + } + return 1; +} + +static int +before_loop_exit(_PyASTOptimizeState *state, stmt_ty node_, const char *kw) +{ + if (state->cf_finally.in_finally && ! state->cf_finally.in_loop) { + if (!control_flow_in_finally_warning(kw, node_, state)) { + return 0; + } + } + return 1; +} + +#define RESTORE_STATE_CHECKED(S, CFS) \ + if (!restore_state((S), (CFS))) { \ + return 0; \ + } + +#define BEFORE_FINALLY(S) overwrite_state((S), true, false, false) +#define AFTER_FINALLY(S, CFS) RESTORE_STATE_CHECKED((S), (CFS)) +#define BEFORE_FUNC_BODY(S) overwrite_state((S), false, true, false) +#define AFTER_FUNC_BODY(S, CFS) RESTORE_STATE_CHECKED((S), (CFS)) +#define BEFORE_LOOP_BODY(S) overwrite_state((S), false, false, true) +#define AFTER_LOOP_BODY(S, CFS) RESTORE_STATE_CHECKED((S), (CFS)) + +#define BEFORE_RETURN(S, N) \ + if (!before_return((S), (N))) { \ + return 0; \ + } + +#define BEFORE_LOOP_EXIT(S, N, KW) \ + if (!before_loop_exit((S), (N), (KW))) { \ + return 0; \ + } + static int make_const(expr_ty node, PyObject *val, PyArena *arena) { @@ -825,24 +911,30 @@ astfold_stmt(stmt_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) { ENTER_RECURSIVE(state); switch (node_->kind) { - case FunctionDef_kind: + case FunctionDef_kind: { CALL_SEQ(astfold_type_param, type_param, node_->v.FunctionDef.type_params); CALL(astfold_arguments, arguments_ty, node_->v.FunctionDef.args); + ControlFlowInFinallyState saved_state = BEFORE_FUNC_BODY(state); CALL(astfold_body, asdl_seq, node_->v.FunctionDef.body); + AFTER_FUNC_BODY(state, &saved_state); CALL_SEQ(astfold_expr, expr, node_->v.FunctionDef.decorator_list); if (!(state->ff_features & CO_FUTURE_ANNOTATIONS)) { CALL_OPT(astfold_expr, expr_ty, node_->v.FunctionDef.returns); } break; - case AsyncFunctionDef_kind: + } + case AsyncFunctionDef_kind: { CALL_SEQ(astfold_type_param, type_param, node_->v.AsyncFunctionDef.type_params); CALL(astfold_arguments, arguments_ty, node_->v.AsyncFunctionDef.args); + ControlFlowInFinallyState saved_state = BEFORE_FUNC_BODY(state); CALL(astfold_body, asdl_seq, node_->v.AsyncFunctionDef.body); + AFTER_FUNC_BODY(state, &saved_state); CALL_SEQ(astfold_expr, expr, node_->v.AsyncFunctionDef.decorator_list); if (!(state->ff_features & CO_FUTURE_ANNOTATIONS)) { CALL_OPT(astfold_expr, expr_ty, node_->v.AsyncFunctionDef.returns); } break; + } case ClassDef_kind: CALL_SEQ(astfold_type_param, type_param, node_->v.ClassDef.type_params); CALL_SEQ(astfold_expr, expr, node_->v.ClassDef.bases); @@ -851,6 +943,7 @@ astfold_stmt(stmt_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) CALL_SEQ(astfold_expr, expr, node_->v.ClassDef.decorator_list); break; case Return_kind: + BEFORE_RETURN(state, node_); CALL_OPT(astfold_expr, expr_ty, node_->v.Return.value); break; case Delete_kind: @@ -876,23 +969,32 @@ astfold_stmt(stmt_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) CALL_SEQ(astfold_type_param, type_param, node_->v.TypeAlias.type_params); CALL(astfold_expr, expr_ty, node_->v.TypeAlias.value); break; - case For_kind: + case For_kind: { CALL(astfold_expr, expr_ty, node_->v.For.target); CALL(astfold_expr, expr_ty, node_->v.For.iter); + ControlFlowInFinallyState saved_state = BEFORE_LOOP_BODY(state); CALL_SEQ(astfold_stmt, stmt, node_->v.For.body); + AFTER_LOOP_BODY(state, &saved_state); CALL_SEQ(astfold_stmt, stmt, node_->v.For.orelse); break; - case AsyncFor_kind: + } + case AsyncFor_kind: { CALL(astfold_expr, expr_ty, node_->v.AsyncFor.target); CALL(astfold_expr, expr_ty, node_->v.AsyncFor.iter); + ControlFlowInFinallyState saved_state = BEFORE_LOOP_BODY(state); CALL_SEQ(astfold_stmt, stmt, node_->v.AsyncFor.body); + AFTER_LOOP_BODY(state, &saved_state); CALL_SEQ(astfold_stmt, stmt, node_->v.AsyncFor.orelse); break; - case While_kind: + } + case While_kind: { CALL(astfold_expr, expr_ty, node_->v.While.test); + ControlFlowInFinallyState saved_state = BEFORE_LOOP_BODY(state); CALL_SEQ(astfold_stmt, stmt, node_->v.While.body); + AFTER_LOOP_BODY(state, &saved_state); CALL_SEQ(astfold_stmt, stmt, node_->v.While.orelse); break; + } case If_kind: CALL(astfold_expr, expr_ty, node_->v.If.test); CALL_SEQ(astfold_stmt, stmt, node_->v.If.body); @@ -910,18 +1012,24 @@ astfold_stmt(stmt_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) CALL_OPT(astfold_expr, expr_ty, node_->v.Raise.exc); CALL_OPT(astfold_expr, expr_ty, node_->v.Raise.cause); break; - case Try_kind: + case Try_kind: { CALL_SEQ(astfold_stmt, stmt, node_->v.Try.body); CALL_SEQ(astfold_excepthandler, excepthandler, node_->v.Try.handlers); CALL_SEQ(astfold_stmt, stmt, node_->v.Try.orelse); + ControlFlowInFinallyState saved_state = BEFORE_FINALLY(state); CALL_SEQ(astfold_stmt, stmt, node_->v.Try.finalbody); + AFTER_FINALLY(state, &saved_state); break; - case TryStar_kind: + } + case TryStar_kind: { CALL_SEQ(astfold_stmt, stmt, node_->v.TryStar.body); CALL_SEQ(astfold_excepthandler, excepthandler, node_->v.TryStar.handlers); CALL_SEQ(astfold_stmt, stmt, node_->v.TryStar.orelse); + ControlFlowInFinallyState saved_state = BEFORE_FINALLY(state); CALL_SEQ(astfold_stmt, stmt, node_->v.TryStar.finalbody); + AFTER_FINALLY(state, &saved_state); break; + } case Assert_kind: CALL(astfold_expr, expr_ty, node_->v.Assert.test); CALL_OPT(astfold_expr, expr_ty, node_->v.Assert.msg); @@ -933,14 +1041,18 @@ astfold_stmt(stmt_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) CALL(astfold_expr, expr_ty, node_->v.Match.subject); CALL_SEQ(astfold_match_case, match_case, node_->v.Match.cases); break; + case Break_kind: + BEFORE_LOOP_EXIT(state, node_, "break"); + break; + case Continue_kind: + BEFORE_LOOP_EXIT(state, node_, "continue"); + break; // The following statements don't contain any subexpressions to be folded case Import_kind: case ImportFrom_kind: case Global_kind: case Nonlocal_kind: case Pass_kind: - case Break_kind: - case Continue_kind: break; // No default case, so the compiler will emit a warning if new statement // kinds are added without being handled here @@ -1045,12 +1157,15 @@ astfold_type_param(type_param_ty node_, PyArena *ctx_, _PyASTOptimizeState *stat #undef CALL_SEQ int -_PyAST_Optimize(mod_ty mod, PyArena *arena, int optimize, int ff_features) +_PyAST_Optimize(mod_ty mod, PyArena *arena, PyObject *filename, int optimize, + int ff_features) { PyThreadState *tstate; int starting_recursion_depth; _PyASTOptimizeState state; + memset(&state, 0, sizeof(_PyASTOptimizeState)); + state.filename = filename; state.optimize = optimize; state.ff_features = ff_features; diff --git a/Python/compile.c b/Python/compile.c index b58c12d4b881ac..18332ada307958 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -126,7 +126,7 @@ compiler_setup(compiler *c, mod_ty mod, PyObject *filename, c->c_optimize = (optimize == -1) ? _Py_GetConfig()->optimization_level : optimize; c->c_save_nested_seqs = false; - if (!_PyAST_Optimize(mod, arena, c->c_optimize, merged)) { + if (!_PyAST_Optimize(mod, arena, filename, c->c_optimize, merged)) { return ERROR; } c->c_st = _PySymtable_Build(mod, filename, &c->c_future); @@ -1397,7 +1397,7 @@ _PyCompile_AstOptimize(mod_ty mod, PyObject *filename, PyCompilerFlags *cf, if (optimize == -1) { optimize = _Py_GetConfig()->optimization_level; } - if (!_PyAST_Optimize(mod, arena, optimize, flags)) { + if (!_PyAST_Optimize(mod, arena, filename, optimize, flags)) { return -1; } return 0; From 830fcfd63c70f0f0b9e390dbe15d15cab29f0355 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Thu, 13 Feb 2025 19:08:08 +0000 Subject: [PATCH 02/18] add docs --- Doc/reference/compound_stmts.rst | 7 ++++++- Doc/tutorial/errors.rst | 8 ++++++-- Doc/whatsnew/3.14.rst | 10 ++++++++++ .../2025-02-13-19-07-54.gh-issue-130080.IoJpuy.rst | 1 + 4 files changed, 23 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-02-13-19-07-54.gh-issue-130080.IoJpuy.rst diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 71cc0c83de567e..30bf28e0d1af74 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -420,7 +420,7 @@ is executed. If there is a saved exception it is re-raised at the end of the :keyword:`!finally` clause. If the :keyword:`!finally` clause raises another exception, the saved exception is set as the context of the new exception. If the :keyword:`!finally` clause executes a :keyword:`return`, :keyword:`break` -or :keyword:`continue` statement, the saved exception is discarded:: +or :keyword:`continue` statement, the saved exception is discarded. >>> def f(): ... try: @@ -461,6 +461,11 @@ always be the last one executed:: Prior to Python 3.8, a :keyword:`continue` statement was illegal in the :keyword:`!finally` clause due to a problem with the implementation. +.. versionchanged:: 3.14 + The compiler emits a :exc:`SyntaxWarning` when a :keyword:`return`, + :keyword:`break` or :keyword:`continue` appears in a :keyword:`!finally` + block (see :pep:`765`). + .. _with: .. _as: diff --git a/Doc/tutorial/errors.rst b/Doc/tutorial/errors.rst index c01cb8c14a0360..28990f3a65a6c1 100644 --- a/Doc/tutorial/errors.rst +++ b/Doc/tutorial/errors.rst @@ -417,7 +417,9 @@ points discuss more complex cases when an exception occurs: * If the :keyword:`!finally` clause executes a :keyword:`break`, :keyword:`continue` or :keyword:`return` statement, exceptions are not - re-raised. + re-raised. This can be confusing and is therefore discouraged. From + version 3.14 the compiler emits a :exc:`SyntaxWarning` for it + (see :pep:`765`). * If the :keyword:`!try` statement reaches a :keyword:`break`, :keyword:`continue` or :keyword:`return` statement, the @@ -429,7 +431,9 @@ points discuss more complex cases when an exception occurs: statement, the returned value will be the one from the :keyword:`!finally` clause's :keyword:`!return` statement, not the value from the :keyword:`!try` clause's :keyword:`!return` - statement. + statement. This can be confusing and is therefore discouraged. From + version 3.14 the compiler emits a :exc:`SyntaxWarning` for it + (see :pep:`765`). For example:: diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index ac0ae8cf0133e6..022ee5091f3325 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -68,6 +68,7 @@ Summary -- release highlights * :ref:`PEP 741: Python Configuration C API ` * :ref:`PEP 761: Discontinuation of PGP signatures ` * :ref:`A new type of interpreter ` +* :ref:`PEP 765: Disallow return/break/continue that exit a finally block ` New features @@ -307,6 +308,15 @@ Other language changes The testbed can also be used to run the test suite of projects other than CPython itself. (Contributed by Russell Keith-Magee in :gh:`127592`.) +.. _whatsnew314-pep765: + +PEP 765: Disallow return/break/continue that exit a finally block +----------------------------------------------------------------- + +The compiler emits a ``SyntaxWarning`` when a ``return``, ``break`` or +``continue`` statements appears where it exits a ``finally`` block. +This change in is specified in :pep:`765`. + New modules =========== diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-02-13-19-07-54.gh-issue-130080.IoJpuy.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-02-13-19-07-54.gh-issue-130080.IoJpuy.rst new file mode 100644 index 00000000000000..7c9f30a9f973f6 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-02-13-19-07-54.gh-issue-130080.IoJpuy.rst @@ -0,0 +1 @@ +Implement PEP 765: Disallow return/break/continue that exit a finally block. From 3a8d59ea2306315a68faf2b09944d974a6ec1d82 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Thu, 13 Feb 2025 19:27:22 +0000 Subject: [PATCH 03/18] doctest doesn't like it --- Doc/reference/compound_stmts.rst | 33 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 30bf28e0d1af74..d4efd36c2db3b3 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -420,16 +420,15 @@ is executed. If there is a saved exception it is re-raised at the end of the :keyword:`!finally` clause. If the :keyword:`!finally` clause raises another exception, the saved exception is set as the context of the new exception. If the :keyword:`!finally` clause executes a :keyword:`return`, :keyword:`break` -or :keyword:`continue` statement, the saved exception is discarded. +or :keyword:`continue` statement, the saved exception is discarded. For example, +this function returns 42. - >>> def f(): - ... try: - ... 1/0 - ... finally: - ... return 42 - ... - >>> f() - 42 +.. code:: + def f(): + try: + 1/0 + finally: + return 42 The exception information is not available to the program during execution of the :keyword:`!finally` clause. @@ -446,16 +445,14 @@ statement, the :keyword:`!finally` clause is also executed 'on the way out.' The return value of a function is determined by the last :keyword:`return` statement executed. Since the :keyword:`!finally` clause always executes, a :keyword:`!return` statement executed in the :keyword:`!finally` clause will -always be the last one executed:: +always be the last one executed. The following function returns 'finally'. - >>> def foo(): - ... try: - ... return 'try' - ... finally: - ... return 'finally' - ... - >>> foo() - 'finally' +.. code:: + def foo(): + try: + return 'try' + finally: + return 'finally' .. versionchanged:: 3.8 Prior to Python 3.8, a :keyword:`continue` statement was illegal in the From 51cdf5a20b926c356a4b80259086e8aa604bcbe5 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Thu, 13 Feb 2025 22:19:37 +0000 Subject: [PATCH 04/18] formatting --- Doc/reference/compound_stmts.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index d4efd36c2db3b3..5392c2358a6a43 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -424,6 +424,7 @@ or :keyword:`continue` statement, the saved exception is discarded. For example, this function returns 42. .. code:: + def f(): try: 1/0 @@ -448,6 +449,7 @@ statement executed. Since the :keyword:`!finally` clause always executes, a always be the last one executed. The following function returns 'finally'. .. code:: + def foo(): try: return 'try' From 480ac8efc0a3fafe57c6e3350ed73bdd76300cb7 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Sun, 16 Feb 2025 13:47:58 +0000 Subject: [PATCH 05/18] always run ast optimizer --- Include/internal/pycore_compile.h | 6 ++-- Lib/test/test___all__.py | 1 + Python/ast_opt.c | 16 ++++++++- Python/bltinmodule.c | 54 +++++++++++++------------------ Python/compile.c | 6 ++-- Python/pythonrun.c | 9 +++--- 6 files changed, 49 insertions(+), 43 deletions(-) diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index 6dfd5598604cd3..63628af1a8f765 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -31,14 +31,16 @@ extern int _PyCompile_AstOptimize( PyObject *filename, PyCompilerFlags *flags, int optimize, - struct _arena *arena); + struct _arena *arena, + int syntax_check_only); extern int _PyAST_Optimize( struct _mod *, struct _arena *arena, PyObject *filename, int optimize, - int ff_features); + int ff_features, + int syntax_check_only); typedef struct { diff --git a/Lib/test/test___all__.py b/Lib/test/test___all__.py index e405056c8ffcb5..fd156a5dfa8572 100644 --- a/Lib/test/test___all__.py +++ b/Lib/test/test___all__.py @@ -37,6 +37,7 @@ def check_all(self, modname): (".* (module|package)", DeprecationWarning), (".* (module|package)", PendingDeprecationWarning), ("", ResourceWarning), + ("", SyntaxWarning), quiet=True): try: exec("import %s" % modname, names) diff --git a/Python/ast_opt.c b/Python/ast_opt.c index ecf6d58785b93b..57ada204bfb097 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -18,6 +18,7 @@ typedef struct { PyObject *filename; int optimize; int ff_features; + int syntax_check_only; int recursion_depth; /* current recursion depth */ int recursion_limit; /* recursion limit */ @@ -165,6 +166,9 @@ unary_not(PyObject *v) static int fold_unaryop(expr_ty node, PyArena *arena, _PyASTOptimizeState *state) { + if (state->syntax_check_only) { + return 1; + } expr_ty arg = node->v.UnaryOp.operand; if (arg->kind != Constant_kind) { @@ -548,6 +552,9 @@ optimize_format(expr_ty node, PyObject *fmt, asdl_expr_seq *elts, PyArena *arena static int fold_binop(expr_ty node, PyArena *arena, _PyASTOptimizeState *state) { + if (state->syntax_check_only) { + return 1; + } expr_ty lhs, rhs; lhs = node->v.BinOp.left; rhs = node->v.BinOp.right; @@ -644,6 +651,9 @@ make_const_tuple(asdl_expr_seq *elts) static int fold_tuple(expr_ty node, PyArena *arena, _PyASTOptimizeState *state) { + if (state->syntax_check_only) { + return 1; + } PyObject *newval; if (node->v.Tuple.ctx != Load) @@ -849,6 +859,9 @@ astfold_expr(expr_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) CALL(fold_tuple, expr_ty, node_); break; case Name_kind: + if (state->syntax_check_only) { + break; + } if (node_->v.Name.ctx == Load && _PyUnicode_EqualToASCIIString(node_->v.Name.id, "__debug__")) { LEAVE_RECURSIVE(state); @@ -1158,7 +1171,7 @@ astfold_type_param(type_param_ty node_, PyArena *ctx_, _PyASTOptimizeState *stat int _PyAST_Optimize(mod_ty mod, PyArena *arena, PyObject *filename, int optimize, - int ff_features) + int ff_features, int syntax_check_only) { PyThreadState *tstate; int starting_recursion_depth; @@ -1168,6 +1181,7 @@ _PyAST_Optimize(mod_ty mod, PyArena *arena, PyObject *filename, int optimize, state.filename = filename; state.optimize = optimize; state.ff_features = ff_features; + state.syntax_check_only = syntax_check_only; /* Setup recursion depth check counters */ tstate = _PyThreadState_GET(); diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index a7243baa64c2a8..162fe11b3cb987 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -834,45 +834,35 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, if (is_ast == -1) goto error; if (is_ast) { - if ((flags & PyCF_OPTIMIZED_AST) == PyCF_ONLY_AST) { - if (PyAst_CheckMode(source, compile_mode) < 0) { - goto error; - } - // return an un-optimized AST - result = Py_NewRef(source); + PyArena *arena = _PyArena_New(); + if (arena == NULL) { + goto error; } - else { - // Return an optimized AST or code object - PyArena *arena = _PyArena_New(); - if (arena == NULL) { + if (flags & PyCF_ONLY_AST) { + mod_ty mod = PyAST_obj2mod(source, arena, compile_mode); + if (mod == NULL || !_PyAST_Validate(mod)) { + _PyArena_Free(arena); goto error; } - - if (flags & PyCF_ONLY_AST) { - mod_ty mod = PyAST_obj2mod(source, arena, compile_mode); - if (mod == NULL || !_PyAST_Validate(mod)) { - _PyArena_Free(arena); - goto error; - } - if (_PyCompile_AstOptimize(mod, filename, &cf, optimize, - arena) < 0) { - _PyArena_Free(arena); - goto error; - } - result = PyAST_mod2obj(mod); + int syntax_check_only = ((flags & PyCF_OPTIMIZED_AST) == PyCF_ONLY_AST); /* unoptiomized AST */ + if (_PyCompile_AstOptimize(mod, filename, &cf, optimize, + arena, syntax_check_only) < 0) { + _PyArena_Free(arena); + goto error; } - else { - mod_ty mod = PyAST_obj2mod(source, arena, compile_mode); - if (mod == NULL || !_PyAST_Validate(mod)) { - _PyArena_Free(arena); - goto error; - } - result = (PyObject*)_PyAST_Compile(mod, filename, - &cf, optimize, arena); + result = PyAST_mod2obj(mod); + } + else { + mod_ty mod = PyAST_obj2mod(source, arena, compile_mode); + if (mod == NULL || !_PyAST_Validate(mod)) { + _PyArena_Free(arena); + goto error; } - _PyArena_Free(arena); + result = (PyObject*)_PyAST_Compile(mod, filename, + &cf, optimize, arena); } + _PyArena_Free(arena); goto finally; } diff --git a/Python/compile.c b/Python/compile.c index 18332ada307958..6d0a48e2a7fb9b 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -126,7 +126,7 @@ compiler_setup(compiler *c, mod_ty mod, PyObject *filename, c->c_optimize = (optimize == -1) ? _Py_GetConfig()->optimization_level : optimize; c->c_save_nested_seqs = false; - if (!_PyAST_Optimize(mod, arena, filename, c->c_optimize, merged)) { + if (!_PyAST_Optimize(mod, arena, filename, c->c_optimize, merged, 0)) { return ERROR; } c->c_st = _PySymtable_Build(mod, filename, &c->c_future); @@ -1387,7 +1387,7 @@ _PyAST_Compile(mod_ty mod, PyObject *filename, PyCompilerFlags *pflags, int _PyCompile_AstOptimize(mod_ty mod, PyObject *filename, PyCompilerFlags *cf, - int optimize, PyArena *arena) + int optimize, PyArena *arena, int no_const_folding) { _PyFutureFeatures future; if (!_PyFuture_FromAST(mod, filename, &future)) { @@ -1397,7 +1397,7 @@ _PyCompile_AstOptimize(mod_ty mod, PyObject *filename, PyCompilerFlags *cf, if (optimize == -1) { optimize = _Py_GetConfig()->optimization_level; } - if (!_PyAST_Optimize(mod, arena, filename, optimize, flags)) { + if (!_PyAST_Optimize(mod, arena, filename, optimize, flags, no_const_folding)) { return -1; } return 0; diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 945e267ef72c6f..84150577f7649b 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1450,11 +1450,10 @@ Py_CompileStringObject(const char *str, PyObject *filename, int start, return NULL; } if (flags && (flags->cf_flags & PyCF_ONLY_AST)) { - if ((flags->cf_flags & PyCF_OPTIMIZED_AST) == PyCF_OPTIMIZED_AST) { - if (_PyCompile_AstOptimize(mod, filename, flags, optimize, arena) < 0) { - _PyArena_Free(arena); - return NULL; - } + int syntax_check_only = ((flags->cf_flags & PyCF_OPTIMIZED_AST) == PyCF_ONLY_AST); /* unoptiomized AST */ + if (_PyCompile_AstOptimize(mod, filename, flags, optimize, arena, syntax_check_only) < 0) { + _PyArena_Free(arena); + return NULL; } PyObject *result = PyAST_mod2obj(mod); _PyArena_Free(arena); From 17a554f3c892b133f4b55a56c4866c9d3af97f76 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 18 Feb 2025 08:54:47 +0000 Subject: [PATCH 06/18] fix test_unparse --- Lib/test/test_unparse.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_unparse.py b/Lib/test/test_unparse.py index f45a651c7ccb5d..9efea1e037f447 100644 --- a/Lib/test/test_unparse.py +++ b/Lib/test/test_unparse.py @@ -422,9 +422,11 @@ def test_docstrings(self): self.check_ast_roundtrip(f"'''{docstring}'''") def test_constant_tuples(self): - self.check_src_roundtrip(ast.Module([ast.Constant(value=(1,))]), "(1,)") + locs = ast.fix_missing_locations self.check_src_roundtrip( - ast.Module([ast.Constant(value=(1, 2, 3))]), "(1, 2, 3)" + locs(ast.Module([ast.Expr(ast.Constant(value=(1,)))])), "(1,)") + self.check_src_roundtrip( + locs(ast.Module([ast.Expr(ast.Constant(value=(1, 2, 3)))])), "(1, 2, 3)" ) def test_function_type(self): From d344fba3c8b5e504f0cf1f2aca9f720b7e9ef048 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 18 Feb 2025 10:31:48 +0000 Subject: [PATCH 07/18] add tests for ast.parse() --- Lib/test/test_ast/test_ast.py | 55 +++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index 42dbb6e79c33b4..de0287743ccc6b 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -820,6 +820,61 @@ def test_repr_large_input_crash(self): r"Exceeds the limit \(\d+ digits\)"): repr(ast.Constant(value=eval(source))) + def test_pep_765_warnings(self): + srcs = [ + textwrap.dedent(""" + def f(): + try: + pass + finally: + return 42 + """), + textwrap.dedent(""" + for x in y: + try: + pass + finally: + break + """), + textwrap.dedent(""" + for x in y: + try: + pass + finally: + continue + """), + ] + for src in srcs: + with self.assertWarnsRegex(SyntaxWarning, 'finally'): + ast.parse(src) + + def test_pep_765_no_warnings(self): + srcs = [ + textwrap.dedent(""" + try: + pass + finally: + def f(): + return 42 + """), + textwrap.dedent(""" + try: + pass + finally: + for x in y: + break + """), + textwrap.dedent(""" + try: + pass + finally: + for x in y: + continue + """), + ] + for src in srcs: + ast.parse(src) + class CopyTests(unittest.TestCase): """Test copying and pickling AST nodes.""" From 0994bf95351968c9f45822930c8fe9a0f8298cc7 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 18 Feb 2025 15:57:19 +0000 Subject: [PATCH 08/18] update test___all__ --- Lib/test/test___all__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test___all__.py b/Lib/test/test___all__.py index fd156a5dfa8572..f35b1194308262 100644 --- a/Lib/test/test___all__.py +++ b/Lib/test/test___all__.py @@ -53,6 +53,7 @@ def check_all(self, modname): with warnings_helper.check_warnings( ("", DeprecationWarning), ("", ResourceWarning), + ("", SyntaxWarning), quiet=True): try: exec("from %s import *" % modname, names) From df8f210b4a32b9e594938adf0eadba0994a5d551 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 4 Mar 2025 16:52:56 +0000 Subject: [PATCH 09/18] rename State->Context. no return value --- Python/ast_opt.c | 52 ++++++++++++++++++++++-------------------------- 1 file changed, 24 insertions(+), 28 deletions(-) diff --git a/Python/ast_opt.c b/Python/ast_opt.c index 57ada204bfb097..eee5cd7815ae75 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -12,7 +12,7 @@ typedef struct { bool in_finally; bool in_funcdef; bool in_loop; -} ControlFlowInFinallyState; +} ControlFlowInFinallyContext; typedef struct { PyObject *filename; @@ -23,7 +23,7 @@ typedef struct { int recursion_depth; /* current recursion depth */ int recursion_limit; /* recursion limit */ - ControlFlowInFinallyState cf_finally; + ControlFlowInFinallyContext cf_finally; } _PyASTOptimizeState; #define ENTER_RECURSIVE(ST) \ @@ -41,21 +41,20 @@ typedef struct { } while(0) -static ControlFlowInFinallyState +static ControlFlowInFinallyContext overwrite_state(_PyASTOptimizeState *state, bool finally, bool funcdef, bool loop) { - ControlFlowInFinallyState saved = state->cf_finally; + ControlFlowInFinallyContext saved = state->cf_finally; state->cf_finally.in_finally = finally; state->cf_finally.in_funcdef = funcdef; state->cf_finally.in_loop = loop; return saved; } -static int -restore_state(_PyASTOptimizeState *state, ControlFlowInFinallyState *saved) +static void +restore_state(_PyASTOptimizeState *state, ControlFlowInFinallyContext *saved) { state->cf_finally = *saved; - return 1; } static int @@ -94,17 +93,14 @@ before_loop_exit(_PyASTOptimizeState *state, stmt_ty node_, const char *kw) return 1; } -#define RESTORE_STATE_CHECKED(S, CFS) \ - if (!restore_state((S), (CFS))) { \ - return 0; \ - } +#define RESTORE_STATE(S, CFS) restore_state((S), (CFS)) #define BEFORE_FINALLY(S) overwrite_state((S), true, false, false) -#define AFTER_FINALLY(S, CFS) RESTORE_STATE_CHECKED((S), (CFS)) +#define AFTER_FINALLY(S, CFS) RESTORE_STATE((S), (CFS)) #define BEFORE_FUNC_BODY(S) overwrite_state((S), false, true, false) -#define AFTER_FUNC_BODY(S, CFS) RESTORE_STATE_CHECKED((S), (CFS)) +#define AFTER_FUNC_BODY(S, CFS) RESTORE_STATE((S), (CFS)) #define BEFORE_LOOP_BODY(S) overwrite_state((S), false, false, true) -#define AFTER_LOOP_BODY(S, CFS) RESTORE_STATE_CHECKED((S), (CFS)) +#define AFTER_LOOP_BODY(S, CFS) RESTORE_STATE((S), (CFS)) #define BEFORE_RETURN(S, N) \ if (!before_return((S), (N))) { \ @@ -927,9 +923,9 @@ astfold_stmt(stmt_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) case FunctionDef_kind: { CALL_SEQ(astfold_type_param, type_param, node_->v.FunctionDef.type_params); CALL(astfold_arguments, arguments_ty, node_->v.FunctionDef.args); - ControlFlowInFinallyState saved_state = BEFORE_FUNC_BODY(state); + ControlFlowInFinallyContext saved_context = BEFORE_FUNC_BODY(state); CALL(astfold_body, asdl_seq, node_->v.FunctionDef.body); - AFTER_FUNC_BODY(state, &saved_state); + AFTER_FUNC_BODY(state, &saved_context); CALL_SEQ(astfold_expr, expr, node_->v.FunctionDef.decorator_list); if (!(state->ff_features & CO_FUTURE_ANNOTATIONS)) { CALL_OPT(astfold_expr, expr_ty, node_->v.FunctionDef.returns); @@ -939,9 +935,9 @@ astfold_stmt(stmt_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) case AsyncFunctionDef_kind: { CALL_SEQ(astfold_type_param, type_param, node_->v.AsyncFunctionDef.type_params); CALL(astfold_arguments, arguments_ty, node_->v.AsyncFunctionDef.args); - ControlFlowInFinallyState saved_state = BEFORE_FUNC_BODY(state); + ControlFlowInFinallyContext saved_context = BEFORE_FUNC_BODY(state); CALL(astfold_body, asdl_seq, node_->v.AsyncFunctionDef.body); - AFTER_FUNC_BODY(state, &saved_state); + AFTER_FUNC_BODY(state, &saved_context); CALL_SEQ(astfold_expr, expr, node_->v.AsyncFunctionDef.decorator_list); if (!(state->ff_features & CO_FUTURE_ANNOTATIONS)) { CALL_OPT(astfold_expr, expr_ty, node_->v.AsyncFunctionDef.returns); @@ -985,26 +981,26 @@ astfold_stmt(stmt_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) case For_kind: { CALL(astfold_expr, expr_ty, node_->v.For.target); CALL(astfold_expr, expr_ty, node_->v.For.iter); - ControlFlowInFinallyState saved_state = BEFORE_LOOP_BODY(state); + ControlFlowInFinallyContext saved_context = BEFORE_LOOP_BODY(state); CALL_SEQ(astfold_stmt, stmt, node_->v.For.body); - AFTER_LOOP_BODY(state, &saved_state); + AFTER_LOOP_BODY(state, &saved_context); CALL_SEQ(astfold_stmt, stmt, node_->v.For.orelse); break; } case AsyncFor_kind: { CALL(astfold_expr, expr_ty, node_->v.AsyncFor.target); CALL(astfold_expr, expr_ty, node_->v.AsyncFor.iter); - ControlFlowInFinallyState saved_state = BEFORE_LOOP_BODY(state); + ControlFlowInFinallyContext saved_context = BEFORE_LOOP_BODY(state); CALL_SEQ(astfold_stmt, stmt, node_->v.AsyncFor.body); - AFTER_LOOP_BODY(state, &saved_state); + AFTER_LOOP_BODY(state, &saved_context); CALL_SEQ(astfold_stmt, stmt, node_->v.AsyncFor.orelse); break; } case While_kind: { CALL(astfold_expr, expr_ty, node_->v.While.test); - ControlFlowInFinallyState saved_state = BEFORE_LOOP_BODY(state); + ControlFlowInFinallyContext saved_context = BEFORE_LOOP_BODY(state); CALL_SEQ(astfold_stmt, stmt, node_->v.While.body); - AFTER_LOOP_BODY(state, &saved_state); + AFTER_LOOP_BODY(state, &saved_context); CALL_SEQ(astfold_stmt, stmt, node_->v.While.orelse); break; } @@ -1029,18 +1025,18 @@ astfold_stmt(stmt_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) CALL_SEQ(astfold_stmt, stmt, node_->v.Try.body); CALL_SEQ(astfold_excepthandler, excepthandler, node_->v.Try.handlers); CALL_SEQ(astfold_stmt, stmt, node_->v.Try.orelse); - ControlFlowInFinallyState saved_state = BEFORE_FINALLY(state); + ControlFlowInFinallyContext saved_context = BEFORE_FINALLY(state); CALL_SEQ(astfold_stmt, stmt, node_->v.Try.finalbody); - AFTER_FINALLY(state, &saved_state); + AFTER_FINALLY(state, &saved_context); break; } case TryStar_kind: { CALL_SEQ(astfold_stmt, stmt, node_->v.TryStar.body); CALL_SEQ(astfold_excepthandler, excepthandler, node_->v.TryStar.handlers); CALL_SEQ(astfold_stmt, stmt, node_->v.TryStar.orelse); - ControlFlowInFinallyState saved_state = BEFORE_FINALLY(state); + ControlFlowInFinallyContext saved_context = BEFORE_FINALLY(state); CALL_SEQ(astfold_stmt, stmt, node_->v.TryStar.finalbody); - AFTER_FINALLY(state, &saved_state); + AFTER_FINALLY(state, &saved_context); break; } case Assert_kind: From ed2f4e7acf60df779420e6b8e4056cf2092df8b2 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Wed, 5 Mar 2025 15:10:11 +0000 Subject: [PATCH 10/18] use stack for context --- Python/ast_opt.c | 94 +++++++++++++++++++++++++++++------------------- 1 file changed, 57 insertions(+), 37 deletions(-) diff --git a/Python/ast_opt.c b/Python/ast_opt.c index eee5cd7815ae75..ca30c6fe223b6c 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -5,6 +5,7 @@ #include "pycore_long.h" // _PyLong #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_setobject.h" // _PySet_NextEntry() +#include "cpython/code.h" // CO_MAXBLOCKS /* See PEP 765 */ @@ -23,7 +24,8 @@ typedef struct { int recursion_depth; /* current recursion depth */ int recursion_limit; /* recursion limit */ - ControlFlowInFinallyContext cf_finally; + int cf_finally_next; + ControlFlowInFinallyContext cf_finally[CO_MAXBLOCKS]; } _PyASTOptimizeState; #define ENTER_RECURSIVE(ST) \ @@ -41,20 +43,27 @@ typedef struct { } while(0) -static ControlFlowInFinallyContext -overwrite_state(_PyASTOptimizeState *state, bool finally, bool funcdef, bool loop) +static int +push_cf_context(_PyASTOptimizeState *state, stmt_ty node, bool finally, bool funcdef, bool loop) { - ControlFlowInFinallyContext saved = state->cf_finally; - state->cf_finally.in_finally = finally; - state->cf_finally.in_funcdef = funcdef; - state->cf_finally.in_loop = loop; - return saved; + if (state->cf_finally_next == CO_MAXBLOCKS) { + PyErr_SetString(PyExc_SyntaxError, "too many statically nested blocks"); + PyErr_RangedSyntaxLocationObject(state->filename, node->lineno, node->col_offset + 1, + node->end_lineno, node->end_col_offset + 1); + return 0; + } + ControlFlowInFinallyContext *ctx = &state->cf_finally[state->cf_finally_next++]; + ctx->in_finally = finally; + ctx->in_funcdef = funcdef; + ctx->in_loop = loop; + return 1; } static void -restore_state(_PyASTOptimizeState *state, ControlFlowInFinallyContext *saved) +pop_cf_context(_PyASTOptimizeState *state) { - state->cf_finally = *saved; + assert(state->cf_finally_next > 0); + state->cf_finally_next--; } static int @@ -74,9 +83,12 @@ control_flow_in_finally_warning(const char *kw, stmt_ty n, _PyASTOptimizeState * static int before_return(_PyASTOptimizeState *state, stmt_ty node_) { - if (state->cf_finally.in_finally && ! state->cf_finally.in_funcdef) { - if (!control_flow_in_finally_warning("return", node_, state)) { - return 0; + if (state->cf_finally_next > 0) { + ControlFlowInFinallyContext *ctx = &state->cf_finally[state->cf_finally_next - 1]; + if (ctx->in_finally && ! ctx->in_funcdef) { + if (!control_flow_in_finally_warning("return", node_, state)) { + return 0; + } } } return 1; @@ -85,22 +97,30 @@ before_return(_PyASTOptimizeState *state, stmt_ty node_) static int before_loop_exit(_PyASTOptimizeState *state, stmt_ty node_, const char *kw) { - if (state->cf_finally.in_finally && ! state->cf_finally.in_loop) { - if (!control_flow_in_finally_warning(kw, node_, state)) { - return 0; + if (state->cf_finally_next > 0) { + ControlFlowInFinallyContext *ctx = &state->cf_finally[state->cf_finally_next - 1]; + if (ctx->in_finally && ! ctx->in_loop) { + if (!control_flow_in_finally_warning(kw, node_, state)) { + return 0; + } } } return 1; } -#define RESTORE_STATE(S, CFS) restore_state((S), (CFS)) +#define PUSH_CONTEXT(S, N, FINALLY, FUNCDEF, LOOP) \ + if (!push_cf_context((S), (N), (FINALLY), (FUNCDEF), (LOOP))) { \ + return 0; \ + } + +#define POP_CONTEXT(S) pop_cf_context(S) -#define BEFORE_FINALLY(S) overwrite_state((S), true, false, false) -#define AFTER_FINALLY(S, CFS) RESTORE_STATE((S), (CFS)) -#define BEFORE_FUNC_BODY(S) overwrite_state((S), false, true, false) -#define AFTER_FUNC_BODY(S, CFS) RESTORE_STATE((S), (CFS)) -#define BEFORE_LOOP_BODY(S) overwrite_state((S), false, false, true) -#define AFTER_LOOP_BODY(S, CFS) RESTORE_STATE((S), (CFS)) +#define BEFORE_FINALLY(S, N) PUSH_CONTEXT((S), (N), true, false, false) +#define AFTER_FINALLY(S) POP_CONTEXT(S) +#define BEFORE_FUNC_BODY(S, N) PUSH_CONTEXT((S), (N), false, true, false) +#define AFTER_FUNC_BODY(S) POP_CONTEXT(S) +#define BEFORE_LOOP_BODY(S, N) PUSH_CONTEXT((S), (N), false, false, true) +#define AFTER_LOOP_BODY(S) POP_CONTEXT(S) #define BEFORE_RETURN(S, N) \ if (!before_return((S), (N))) { \ @@ -923,9 +943,9 @@ astfold_stmt(stmt_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) case FunctionDef_kind: { CALL_SEQ(astfold_type_param, type_param, node_->v.FunctionDef.type_params); CALL(astfold_arguments, arguments_ty, node_->v.FunctionDef.args); - ControlFlowInFinallyContext saved_context = BEFORE_FUNC_BODY(state); + BEFORE_FUNC_BODY(state, node_); CALL(astfold_body, asdl_seq, node_->v.FunctionDef.body); - AFTER_FUNC_BODY(state, &saved_context); + AFTER_FUNC_BODY(state); CALL_SEQ(astfold_expr, expr, node_->v.FunctionDef.decorator_list); if (!(state->ff_features & CO_FUTURE_ANNOTATIONS)) { CALL_OPT(astfold_expr, expr_ty, node_->v.FunctionDef.returns); @@ -935,9 +955,9 @@ astfold_stmt(stmt_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) case AsyncFunctionDef_kind: { CALL_SEQ(astfold_type_param, type_param, node_->v.AsyncFunctionDef.type_params); CALL(astfold_arguments, arguments_ty, node_->v.AsyncFunctionDef.args); - ControlFlowInFinallyContext saved_context = BEFORE_FUNC_BODY(state); + BEFORE_FUNC_BODY(state, node_); CALL(astfold_body, asdl_seq, node_->v.AsyncFunctionDef.body); - AFTER_FUNC_BODY(state, &saved_context); + AFTER_FUNC_BODY(state); CALL_SEQ(astfold_expr, expr, node_->v.AsyncFunctionDef.decorator_list); if (!(state->ff_features & CO_FUTURE_ANNOTATIONS)) { CALL_OPT(astfold_expr, expr_ty, node_->v.AsyncFunctionDef.returns); @@ -981,26 +1001,26 @@ astfold_stmt(stmt_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) case For_kind: { CALL(astfold_expr, expr_ty, node_->v.For.target); CALL(astfold_expr, expr_ty, node_->v.For.iter); - ControlFlowInFinallyContext saved_context = BEFORE_LOOP_BODY(state); + BEFORE_LOOP_BODY(state, node_); CALL_SEQ(astfold_stmt, stmt, node_->v.For.body); - AFTER_LOOP_BODY(state, &saved_context); + AFTER_LOOP_BODY(state); CALL_SEQ(astfold_stmt, stmt, node_->v.For.orelse); break; } case AsyncFor_kind: { CALL(astfold_expr, expr_ty, node_->v.AsyncFor.target); CALL(astfold_expr, expr_ty, node_->v.AsyncFor.iter); - ControlFlowInFinallyContext saved_context = BEFORE_LOOP_BODY(state); + BEFORE_LOOP_BODY(state, node_); CALL_SEQ(astfold_stmt, stmt, node_->v.AsyncFor.body); - AFTER_LOOP_BODY(state, &saved_context); + AFTER_LOOP_BODY(state); CALL_SEQ(astfold_stmt, stmt, node_->v.AsyncFor.orelse); break; } case While_kind: { CALL(astfold_expr, expr_ty, node_->v.While.test); - ControlFlowInFinallyContext saved_context = BEFORE_LOOP_BODY(state); + BEFORE_LOOP_BODY(state, node_); CALL_SEQ(astfold_stmt, stmt, node_->v.While.body); - AFTER_LOOP_BODY(state, &saved_context); + AFTER_LOOP_BODY(state); CALL_SEQ(astfold_stmt, stmt, node_->v.While.orelse); break; } @@ -1025,18 +1045,18 @@ astfold_stmt(stmt_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) CALL_SEQ(astfold_stmt, stmt, node_->v.Try.body); CALL_SEQ(astfold_excepthandler, excepthandler, node_->v.Try.handlers); CALL_SEQ(astfold_stmt, stmt, node_->v.Try.orelse); - ControlFlowInFinallyContext saved_context = BEFORE_FINALLY(state); + BEFORE_FINALLY(state, node_); CALL_SEQ(astfold_stmt, stmt, node_->v.Try.finalbody); - AFTER_FINALLY(state, &saved_context); + AFTER_FINALLY(state); break; } case TryStar_kind: { CALL_SEQ(astfold_stmt, stmt, node_->v.TryStar.body); CALL_SEQ(astfold_excepthandler, excepthandler, node_->v.TryStar.handlers); CALL_SEQ(astfold_stmt, stmt, node_->v.TryStar.orelse); - ControlFlowInFinallyContext saved_context = BEFORE_FINALLY(state); + BEFORE_FINALLY(state, node_); CALL_SEQ(astfold_stmt, stmt, node_->v.TryStar.finalbody); - AFTER_FINALLY(state, &saved_context); + AFTER_FINALLY(state); break; } case Assert_kind: From 09e7fb403e08db50ff0def37a1309942ecce16bd Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Fri, 14 Mar 2025 10:58:07 +0000 Subject: [PATCH 11/18] use c_array instead of bounded array for context --- Python/ast_opt.c | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/Python/ast_opt.c b/Python/ast_opt.c index 51d08fecf0a14b..f44266e481824b 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -1,11 +1,11 @@ /* AST Optimizer */ #include "Python.h" #include "pycore_ast.h" // _PyAST_GetDocString() +#include "pycore_c_array.h" // _Py_CArray_EnsureCapacity() #include "pycore_format.h" // F_LJUST #include "pycore_long.h" // _PyLong #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_setobject.h" // _PySet_NextEntry() -#include "cpython/code.h" // CO_MAXBLOCKS /* See PEP 765 */ @@ -24,8 +24,8 @@ typedef struct { int recursion_depth; /* current recursion depth */ int recursion_limit; /* recursion limit */ - int cf_finally_next; - ControlFlowInFinallyContext cf_finally[CO_MAXBLOCKS]; + _Py_c_array_t cf_finally; /* context for PEP 678 check */ + int cf_finally_used; } _PyASTOptimizeState; #define ENTER_RECURSIVE() \ @@ -35,17 +35,23 @@ if (Py_EnterRecursiveCall(" during compilation")) { \ #define LEAVE_RECURSIVE() Py_LeaveRecursiveCall(); +static ControlFlowInFinallyContext* +get_cf_finally_top(_PyASTOptimizeState *state) +{ + int idx = state->cf_finally_used+1; + return state->cf_finally.array + idx * sizeof(ControlFlowInFinallyContext); +} static int push_cf_context(_PyASTOptimizeState *state, stmt_ty node, bool finally, bool funcdef, bool loop) { - if (state->cf_finally_next == CO_MAXBLOCKS) { - PyErr_SetString(PyExc_SyntaxError, "too many statically nested blocks"); - PyErr_RangedSyntaxLocationObject(state->filename, node->lineno, node->col_offset + 1, - node->end_lineno, node->end_col_offset + 1); + if (_Py_CArray_EnsureCapacity(&state->cf_finally, state->cf_finally_used+1) < 0) { return 0; } - ControlFlowInFinallyContext *ctx = &state->cf_finally[state->cf_finally_next++]; + + state->cf_finally_used++; + ControlFlowInFinallyContext *ctx = get_cf_finally_top(state); + ctx->in_finally = finally; ctx->in_funcdef = funcdef; ctx->in_loop = loop; @@ -55,8 +61,8 @@ push_cf_context(_PyASTOptimizeState *state, stmt_ty node, bool finally, bool fun static void pop_cf_context(_PyASTOptimizeState *state) { - assert(state->cf_finally_next > 0); - state->cf_finally_next--; + assert(state->cf_finally_used > 0); + state->cf_finally_used--; } static int @@ -76,8 +82,8 @@ control_flow_in_finally_warning(const char *kw, stmt_ty n, _PyASTOptimizeState * static int before_return(_PyASTOptimizeState *state, stmt_ty node_) { - if (state->cf_finally_next > 0) { - ControlFlowInFinallyContext *ctx = &state->cf_finally[state->cf_finally_next - 1]; + if (state->cf_finally_used > 0) { + ControlFlowInFinallyContext *ctx = get_cf_finally_top(state); if (ctx->in_finally && ! ctx->in_funcdef) { if (!control_flow_in_finally_warning("return", node_, state)) { return 0; @@ -90,8 +96,8 @@ before_return(_PyASTOptimizeState *state, stmt_ty node_) static int before_loop_exit(_PyASTOptimizeState *state, stmt_ty node_, const char *kw) { - if (state->cf_finally_next > 0) { - ControlFlowInFinallyContext *ctx = &state->cf_finally[state->cf_finally_next - 1]; + if (state->cf_finally_used > 0) { + ControlFlowInFinallyContext *ctx = get_cf_finally_top(state); if (ctx->in_finally && ! ctx->in_loop) { if (!control_flow_in_finally_warning(kw, node_, state)) { return 0; @@ -978,9 +984,13 @@ _PyAST_Optimize(mod_ty mod, PyArena *arena, PyObject *filename, int optimize, state.optimize = optimize; state.ff_features = ff_features; state.syntax_check_only = syntax_check_only; + if (_Py_CArray_Init(&state.cf_finally, sizeof(ControlFlowInFinallyContext), 20) < 0) { + return -1; + } int ret = astfold_mod(mod, arena, &state); assert(ret || PyErr_Occurred()); + _Py_CArray_Fini(&state.cf_finally); return ret; } From 1f5323c61cf8b350bd9404ed4b6d18a80cb153dd Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Sat, 15 Mar 2025 18:27:25 +0000 Subject: [PATCH 12/18] fix crash --- Python/ast_opt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/ast_opt.c b/Python/ast_opt.c index f44266e481824b..982e85a4ed33f7 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -38,7 +38,7 @@ if (Py_EnterRecursiveCall(" during compilation")) { \ static ControlFlowInFinallyContext* get_cf_finally_top(_PyASTOptimizeState *state) { - int idx = state->cf_finally_used+1; + int idx = state->cf_finally_used; return state->cf_finally.array + idx * sizeof(ControlFlowInFinallyContext); } From 7a9a3e1b70cb03ca9d6b7c6a52f6f2322433eb3c Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Sat, 15 Mar 2025 18:40:37 +0000 Subject: [PATCH 13/18] Apply suggestions from code review Co-authored-by: Tomas R. --- Doc/whatsnew/3.14.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 9971bf19ff2eca..fe2023db2fc1fe 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -376,9 +376,9 @@ Other language changes PEP 765: Disallow return/break/continue that exit a finally block ----------------------------------------------------------------- -The compiler emits a ``SyntaxWarning`` when a ``return``, ``break`` or -``continue`` statements appears where it exits a ``finally`` block. -This change in is specified in :pep:`765`. +The compiler emits a `:exc:`SyntaxWarning` when a :keyword:`return`, :keyword:`break` or +:keyword:`continue` statements appears where it exits a :keyword:`!finally` block. +This change is specified in :pep:`765`. New modules =========== From 6a37d813f3ba67d551016642b0f92601c01ea231 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Sat, 15 Mar 2025 19:09:42 +0000 Subject: [PATCH 14/18] typo --- Doc/whatsnew/3.14.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index fe2023db2fc1fe..e962fdbef629cd 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -376,7 +376,7 @@ Other language changes PEP 765: Disallow return/break/continue that exit a finally block ----------------------------------------------------------------- -The compiler emits a `:exc:`SyntaxWarning` when a :keyword:`return`, :keyword:`break` or +The compiler emits a :exc:`SyntaxWarning` when a :keyword:`return`, :keyword:`break` or :keyword:`continue` statements appears where it exits a :keyword:`!finally` block. This change is specified in :pep:`765`. From 0b257c3970c735e91beccd073da916e4c0a436a4 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Sat, 15 Mar 2025 20:11:29 +0000 Subject: [PATCH 15/18] fix windows issue --- Python/ast_opt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/ast_opt.c b/Python/ast_opt.c index 982e85a4ed33f7..182424c50583a0 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -39,7 +39,7 @@ static ControlFlowInFinallyContext* get_cf_finally_top(_PyASTOptimizeState *state) { int idx = state->cf_finally_used; - return state->cf_finally.array + idx * sizeof(ControlFlowInFinallyContext); + return ((ControlFlowInFinallyContext*)state->cf_finally.array) + idx; } static int From a4b65f725d6ceba4909ee8ee14d3cebe34c1e368 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 17 Mar 2025 17:14:36 +0000 Subject: [PATCH 16/18] removed unused --- Python/ast_opt.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/Python/ast_opt.c b/Python/ast_opt.c index 4819adb60ea164..4a191e919e412c 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -21,9 +21,6 @@ typedef struct { int ff_features; int syntax_check_only; - int recursion_depth; /* current recursion depth */ - int recursion_limit; /* recursion limit */ - _Py_c_array_t cf_finally; /* context for PEP 678 check */ int cf_finally_used; } _PyASTOptimizeState; From a00f297c51dcd4871f7f2ca70a9e266925cf5db0 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Mon, 17 Mar 2025 20:12:08 +0000 Subject: [PATCH 17/18] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Doc/reference/compound_stmts.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 5392c2358a6a43..949cdf3be8b7e3 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -423,7 +423,7 @@ If the :keyword:`!finally` clause executes a :keyword:`return`, :keyword:`break` or :keyword:`continue` statement, the saved exception is discarded. For example, this function returns 42. -.. code:: +.. code-block:: def f(): try: @@ -448,7 +448,7 @@ statement executed. Since the :keyword:`!finally` clause always executes, a :keyword:`!return` statement executed in the :keyword:`!finally` clause will always be the last one executed. The following function returns 'finally'. -.. code:: +.. code-block:: def foo(): try: @@ -460,7 +460,7 @@ always be the last one executed. The following function returns 'finally'. Prior to Python 3.8, a :keyword:`continue` statement was illegal in the :keyword:`!finally` clause due to a problem with the implementation. -.. versionchanged:: 3.14 +.. versionchanged:: next The compiler emits a :exc:`SyntaxWarning` when a :keyword:`return`, :keyword:`break` or :keyword:`continue` appears in a :keyword:`!finally` block (see :pep:`765`). From 459ccf0b9ad03c39436beff9f6f82633d5d74116 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Mon, 17 Mar 2025 20:20:00 +0000 Subject: [PATCH 18/18] Update Doc/whatsnew/3.14.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Doc/whatsnew/3.14.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index f1916eee4a1003..789156974cb0d1 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -377,7 +377,7 @@ PEP 765: Disallow return/break/continue that exit a finally block ----------------------------------------------------------------- The compiler emits a :exc:`SyntaxWarning` when a :keyword:`return`, :keyword:`break` or -:keyword:`continue` statements appears where it exits a :keyword:`!finally` block. +:keyword:`continue` statements appears where it exits a :keyword:`finally` block. This change is specified in :pep:`765`. New modules pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy