From 8d2a921a094a94e191373c69460dac3a060ca003 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Mon, 5 Aug 2019 14:11:14 +1000 Subject: [PATCH 01/10] Bug #37757: Raise TargetScopeError as specified in PEP 572 - comprehension iteration variables are explicitly local, but named expression targets in comprehensions are nonlocal or global. Raise TargetScopeError as specified in PEP 572 - named expression targets in the outermost iterable of a comprehension have an ambiguous target scope. Avoid resolving that question now by raising TargetScopeError. PEP 572 explicitly requires this for cases where the bound name conflicts with the iteration variable in the comprehension, but CPython can't easily restrict the exception to that case (as it doesn't know the target variable names when visiting the outermost iterator expression) --- Include/symtable.h | 3 + Lib/test/test_named_expressions.py | 79 ++++++++++++--------- Python/symtable.c | 109 +++++++++++++++++++++++------ 3 files changed, 134 insertions(+), 57 deletions(-) diff --git a/Include/symtable.h b/Include/symtable.h index 9392e64387794f..5dcfa7e2c2bb68 100644 --- a/Include/symtable.h +++ b/Include/symtable.h @@ -58,6 +58,8 @@ typedef struct _symtable_entry { unsigned ste_needs_class_closure : 1; /* for class scopes, true if a closure over __class__ should be created */ + unsigned ste_comp_iter_target : 1; /* true if visiting comprehension target */ + int ste_comp_iter_expr; /* non-zero if visiting a comprehension range expression */ int ste_lineno; /* first line of block */ int ste_col_offset; /* offset of first line of block */ int ste_opt_lineno; /* lineno of last exec or import * */ @@ -94,6 +96,7 @@ PyAPI_FUNC(void) PySymtable_Free(struct symtable *); #define DEF_FREE_CLASS 2<<5 /* free variable from class's method */ #define DEF_IMPORT 2<<6 /* assignment occurred via import */ #define DEF_ANNOT 2<<7 /* this name is annotated */ +#define DEF_COMP_ITER 2<<8 /* this name is a comprehension iteration variable */ #define DEF_BOUND (DEF_LOCAL | DEF_PARAM | DEF_IMPORT) diff --git a/Lib/test/test_named_expressions.py b/Lib/test/test_named_expressions.py index f73e6fee70ce0b..ea85b6280e9f3b 100644 --- a/Lib/test/test_named_expressions.py +++ b/Lib/test/test_named_expressions.py @@ -104,7 +104,7 @@ def test_named_expression_invalid_17(self): with self.assertRaisesRegex(SyntaxError, "invalid syntax"): exec(code, {}, {}) - def test_named_expression_invalid_18(self): + def test_named_expression_invalid_in_class_body(self): code = """class Foo(): [(42, 1 + ((( j := i )))) for i in range(5)] """ @@ -113,6 +113,50 @@ def test_named_expression_invalid_18(self): "named expression within a comprehension cannot be used in a class body"): exec(code, {}, {}) + def test_named_expression_invalid_rebinding_comprehension_iteration_variable(self): + cases = [ + ("Local reuse", "[i := 0 for i in range(5)]"), + ("Nested reuse", "[[(j := 0) for i in range(5)] for j in range(5)]"), + ("Reuse inner loop target", "[(j := 0) for i in range(5) for j in range(5)]"), + ("Unpacking reuse", "[i := 0 for i, j in [(0, 1)]]"), + ("Reuse in loop condition", "[i+1 for i in range(5) if (i := 0)]"), + ("Unreachable reuse", "[False or (i:=0) for i in range(5)]"), + ("Unreachable nested reuse", + "[(i, j) for i in range(5) for j in range(5) if True or (i:=10)]"), + ("Nested comprehension condition", "[i for i in [j for j in range(5) if (j := True)]]"), + ("Nested comprehension body", "[i for i in [(j := True) for j in range(5)]]"), + ] + for case, code in cases: + with self.subTest(case=case): + with self.assertRaisesRegex(TargetScopeError, + "named expression cannot rebind comprehension iteration variable"): + exec(code, {}, {}) + + def test_named_expression_invalid_rebinding_comprehension_inner_loop(self): + cases = [ + ("Inner reuse", "[i for i in range(5) if (j := 0) for j in range(5)]"), + ("Inner unpacking reuse", "[i for i in range(5) if (j := 0) for j, k in [(0, 1)]]"), + ] + for case, code in cases: + with self.subTest(case=case): + with self.assertRaisesRegex(TargetScopeError, + "comprehension inner loop cannot rebind named expression target"): + exec(code, {}, {}) + + def test_named_expression_invalid_comprehension_iterator_expression(self): + cases = [ + ("Top level", "[i for i in (i := range(5))]"), + ("Inside container", "[i for i in (2, 3, i := range(5))]"), + ("Different name", "[i for i in (j := range(5))]"), + ("Inner loop", "[i for i in range(5) for j in (i := range(5))]"), + ("Nested comprehension", "[i for i in [j for j in (k := range(5))]]"), + ] + for case, code in cases: + with self.subTest(case=case): + with self.assertRaisesRegex(TargetScopeError, + "named expression cannot be used in comprehension iterator expression"): + exec(code, {}, {}) + class NamedExpressionAssignmentTest(unittest.TestCase): @@ -306,39 +350,6 @@ def test_named_expression_scope_11(self): self.assertEqual(res, [0, 1, 2, 3, 4]) self.assertEqual(j, 4) - def test_named_expression_scope_12(self): - res = [i := i for i in range(5)] - - self.assertEqual(res, [0, 1, 2, 3, 4]) - self.assertEqual(i, 4) - - def test_named_expression_scope_13(self): - res = [i := 0 for i, j in [(1, 2), (3, 4)]] - - self.assertEqual(res, [0, 0]) - self.assertEqual(i, 0) - - def test_named_expression_scope_14(self): - res = [(i := 0, j := 1) for i, j in [(1, 2), (3, 4)]] - - self.assertEqual(res, [(0, 1), (0, 1)]) - self.assertEqual(i, 0) - self.assertEqual(j, 1) - - def test_named_expression_scope_15(self): - res = [(i := i, j := j) for i, j in [(1, 2), (3, 4)]] - - self.assertEqual(res, [(1, 2), (3, 4)]) - self.assertEqual(i, 3) - self.assertEqual(j, 4) - - def test_named_expression_scope_16(self): - res = [(i := j, j := i) for i, j in [(1, 2), (3, 4)]] - - self.assertEqual(res, [(2, 2), (4, 4)]) - self.assertEqual(i, 4) - self.assertEqual(j, 4) - def test_named_expression_scope_17(self): b = 0 res = [b := i + b for i in range(5)] diff --git a/Python/symtable.c b/Python/symtable.c index 668cc21b2df987..cc758072f6b37d 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -34,6 +34,15 @@ #define NAMED_EXPR_COMP_IN_CLASS \ "named expression within a comprehension cannot be used in a class body" +#define NAMED_EXPR_COMP_CONFLICT \ +"named expression cannot rebind comprehension iteration variable" + +#define NAMED_EXPR_COMP_INNER_LOOP_CONFLICT \ +"comprehension inner loop cannot rebind named expression target" + +#define NAMED_EXPR_COMP_ITER_EXPR \ +"named expression cannot be used in comprehension iterator expression" + static PySTEntryObject * ste_new(struct symtable *st, identifier name, _Py_block_ty block, void *key, int lineno, int col_offset) @@ -81,6 +90,8 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block, ste->ste_comprehension = 0; ste->ste_returns_value = 0; ste->ste_needs_class_closure = 0; + ste->ste_comp_iter_target = 0; + ste->ste_comp_iter_expr = 0; ste->ste_symbols = PyDict_New(); ste->ste_varnames = PyList_New(0); @@ -373,14 +384,21 @@ PySymtable_Lookup(struct symtable *st, void *key) return (PySTEntryObject *)v; } -int -PyST_GetScope(PySTEntryObject *ste, PyObject *name) +static long +_PyST_GetSymbol(PySTEntryObject *ste, PyObject *name) { PyObject *v = PyDict_GetItem(ste->ste_symbols, name); if (!v) return 0; assert(PyLong_Check(v)); - return (PyLong_AS_LONG(v) >> SCOPE_OFFSET) & SCOPE_MASK; + return PyLong_AS_LONG(v); +} + +int +PyST_GetScope(PySTEntryObject *ste, PyObject *name) +{ + long symbol = _PyST_GetSymbol(ste, name); + return (symbol >> SCOPE_OFFSET) & SCOPE_MASK; } static int @@ -971,15 +989,10 @@ symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block, static long symtable_lookup(struct symtable *st, PyObject *name) { - PyObject *o; PyObject *mangled = _Py_Mangle(st->st_private, name); if (!mangled) return 0; - o = PyDict_GetItem(st->st_cur->ste_symbols, mangled); - Py_DECREF(mangled); - if (!o) - return 0; - return PyLong_AsLong(o); + return _PyST_GetSymbol(st->st_cur, mangled); } static int @@ -1012,6 +1025,17 @@ symtable_add_def_helper(struct symtable *st, PyObject *name, int flag, struct _s else { val = flag; } + if (ste->ste_comp_iter_target) { + if (val & (DEF_GLOBAL | DEF_NONLOCAL)) { + PyErr_Format(PyExc_TargetScopeError, + NAMED_EXPR_COMP_INNER_LOOP_CONFLICT, name); + PyErr_SyntaxLocationObject(st->st_filename, + ste->ste_lineno, + ste->ste_col_offset + 1); + goto error; + } + val |= DEF_COMP_ITER; + } o = PyLong_FromLong(val); if (o == NULL) goto error; @@ -1392,7 +1416,9 @@ static int symtable_extend_namedexpr_scope(struct symtable *st, expr_ty e) { assert(st->st_stack); + assert(e->kind == Name_kind); + PyObject *target_name = e->v.Name.id; Py_ssize_t i, size; struct _symtable_entry *ste; size = PyList_GET_SIZE(st->st_stack); @@ -1402,32 +1428,42 @@ symtable_extend_namedexpr_scope(struct symtable *st, expr_ty e) for (i = size - 1; i >= 0; i--) { ste = (struct _symtable_entry *) PyList_GET_ITEM(st->st_stack, i); - /* If our current entry is a comprehension, skip it */ + /* If we find a comprehension scope, check for a target + * binding conflict with iteration variables, otherwise skip it + */ if (ste->ste_comprehension) { + long target_in_scope = _PyST_GetSymbol(ste, target_name); + if (target_in_scope & DEF_COMP_ITER) { + PyErr_Format(PyExc_TargetScopeError, NAMED_EXPR_COMP_CONFLICT, target_name); + PyErr_SyntaxLocationObject(st->st_filename, + e->lineno, + e->col_offset); + VISIT_QUIT(st, 0); + } continue; } /* If we find a FunctionBlock entry, add as NONLOCAL/LOCAL */ if (ste->ste_type == FunctionBlock) { - if (!symtable_add_def(st, e->v.Name.id, DEF_NONLOCAL)) + if (!symtable_add_def(st, target_name, DEF_NONLOCAL)) VISIT_QUIT(st, 0); - if (!symtable_record_directive(st, e->v.Name.id, e->lineno, e->col_offset)) + if (!symtable_record_directive(st, target_name, e->lineno, e->col_offset)) VISIT_QUIT(st, 0); - return symtable_add_def_helper(st, e->v.Name.id, DEF_LOCAL, ste); + return symtable_add_def_helper(st, target_name, DEF_LOCAL, ste); } /* If we find a ModuleBlock entry, add as GLOBAL */ if (ste->ste_type == ModuleBlock) { - if (!symtable_add_def(st, e->v.Name.id, DEF_GLOBAL)) + if (!symtable_add_def(st, target_name, DEF_GLOBAL)) VISIT_QUIT(st, 0); - if (!symtable_record_directive(st, e->v.Name.id, e->lineno, e->col_offset)) + if (!symtable_record_directive(st, target_name, e->lineno, e->col_offset)) VISIT_QUIT(st, 0); - return symtable_add_def_helper(st, e->v.Name.id, DEF_GLOBAL, ste); + return symtable_add_def_helper(st, target_name, DEF_GLOBAL, ste); } /* Disallow usage in ClassBlock */ if (ste->ste_type == ClassBlock) { - PyErr_Format(PyExc_TargetScopeError, NAMED_EXPR_COMP_IN_CLASS, e->v.Name.id); + PyErr_Format(PyExc_TargetScopeError, NAMED_EXPR_COMP_IN_CLASS, target_name); PyErr_SyntaxLocationObject(st->st_filename, e->lineno, e->col_offset); @@ -1442,6 +1478,28 @@ symtable_extend_namedexpr_scope(struct symtable *st, expr_ty e) return 0; } +static int +symtable_handle_namedexpr(struct symtable *st, expr_ty e) +{ + if (st->st_cur->ste_comp_iter_expr > 0) { + /* Evaluating the outermost range expression for a comprehension */ + assert(e->v.NamedExpr.target->kind == Name_kind); + PyObject *target_name = e->v.NamedExpr.target->v.Name.id; + PyErr_Format(PyExc_TargetScopeError, NAMED_EXPR_COMP_ITER_EXPR, target_name); + PyErr_SyntaxLocationObject(st->st_filename, + e->lineno, + e->col_offset); + VISIT_QUIT(st, 0); + } + if (st->st_cur->ste_comprehension) { + /* Already inside a comprehension body */ + if (!symtable_extend_namedexpr_scope(st, e->v.NamedExpr.target)) + VISIT_QUIT(st, 0); + } + VISIT(st, expr, e->v.NamedExpr.value); + VISIT(st, expr, e->v.NamedExpr.target); +} + static int symtable_visit_expr(struct symtable *st, expr_ty e) { @@ -1452,12 +1510,7 @@ symtable_visit_expr(struct symtable *st, expr_ty e) } switch (e->kind) { case NamedExpr_kind: - if (st->st_cur->ste_comprehension) { - if (!symtable_extend_namedexpr_scope(st, e->v.NamedExpr.target)) - VISIT_QUIT(st, 0); - } - VISIT(st, expr, e->v.NamedExpr.value); - VISIT(st, expr, e->v.NamedExpr.target); + symtable_handle_namedexpr(st, e); break; case BoolOp_kind: VISIT_SEQ(st, expr, e->v.BoolOp.values); @@ -1739,8 +1792,12 @@ symtable_visit_alias(struct symtable *st, alias_ty a) static int symtable_visit_comprehension(struct symtable *st, comprehension_ty lc) { + st->st_cur->ste_comp_iter_target = 1; VISIT(st, expr, lc->target); + st->st_cur->ste_comp_iter_target = 0; + st->st_cur->ste_comp_iter_expr++; VISIT(st, expr, lc->iter); + st->st_cur->ste_comp_iter_expr--; VISIT_SEQ(st, expr, lc->ifs); if (lc->is_async) { st->st_cur->ste_coroutine = 1; @@ -1788,7 +1845,9 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e, comprehension_ty outermost = ((comprehension_ty) asdl_seq_GET(generators, 0)); /* Outermost iterator is evaluated in current scope */ + st->st_cur->ste_comp_iter_expr++; VISIT(st, expr, outermost->iter); + st->st_cur->ste_comp_iter_expr--; /* Create comprehension scope for the rest */ if (!scope_name || !symtable_enter_block(st, scope_name, FunctionBlock, (void *)e, @@ -1805,7 +1864,11 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e, symtable_exit_block(st, (void *)e); return 0; } + /* Visit iteration variable target, and mark them as such */ + st->st_cur->ste_comp_iter_target = 1; VISIT(st, expr, outermost->target); + st->st_cur->ste_comp_iter_target = 0; + /* Visit the rest of the comprehension body */ VISIT_SEQ(st, expr, outermost->ifs); VISIT_SEQ_TAIL(st, comprehension, generators, 1); if (value) From cf0c6c72840469a1c2cf223ed002a0981ba264ba Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Mon, 5 Aug 2019 14:23:24 +1000 Subject: [PATCH 02/10] Add NEWS entry --- .../Core and Builtins/2019-08-05-14-22-59.bpo-37757.lRv5HX.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2019-08-05-14-22-59.bpo-37757.lRv5HX.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-08-05-14-22-59.bpo-37757.lRv5HX.rst b/Misc/NEWS.d/next/Core and Builtins/2019-08-05-14-22-59.bpo-37757.lRv5HX.rst new file mode 100644 index 00000000000000..f3477a189cfec4 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2019-08-05-14-22-59.bpo-37757.lRv5HX.rst @@ -0,0 +1,3 @@ +PEP 572: As described in the PEP, assignment expressions now raised +TargetScopeError when their interaction with comprehension scoping results +in an ambiguous target scope. From c0f2b5bfa7f3b0318493a5d663ac59cf3f705df0 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Mon, 5 Aug 2019 16:18:36 +1000 Subject: [PATCH 03/10] Close nested comprehension and lambda expression loopholes --- Lib/test/test_named_expressions.py | 5 +++-- Python/symtable.c | 7 +++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_named_expressions.py b/Lib/test/test_named_expressions.py index ea85b6280e9f3b..769dcaa7507fd7 100644 --- a/Lib/test/test_named_expressions.py +++ b/Lib/test/test_named_expressions.py @@ -123,8 +123,6 @@ def test_named_expression_invalid_rebinding_comprehension_iteration_variable(sel ("Unreachable reuse", "[False or (i:=0) for i in range(5)]"), ("Unreachable nested reuse", "[(i, j) for i in range(5) for j in range(5) if True or (i:=10)]"), - ("Nested comprehension condition", "[i for i in [j for j in range(5) if (j := True)]]"), - ("Nested comprehension body", "[i for i in [(j := True) for j in range(5)]]"), ] for case, code in cases: with self.subTest(case=case): @@ -148,8 +146,11 @@ def test_named_expression_invalid_comprehension_iterator_expression(self): ("Top level", "[i for i in (i := range(5))]"), ("Inside container", "[i for i in (2, 3, i := range(5))]"), ("Different name", "[i for i in (j := range(5))]"), + ("Lambda expression", "[i for i in (lambda:(j := range(5)))()]"), ("Inner loop", "[i for i in range(5) for j in (i := range(5))]"), ("Nested comprehension", "[i for i in [j for j in (k := range(5))]]"), + ("Nested comprehension condition", "[i for i in [j for j in range(5) if (j := True)]]"), + ("Nested comprehension body", "[i for i in [(j := True) for j in range(5)]]"), ] for case, code in cases: with self.subTest(case=case): diff --git a/Python/symtable.c b/Python/symtable.c index cc758072f6b37d..ae059cdd4706a8 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -973,6 +973,13 @@ symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block, return 0; } prev = st->st_cur; + /* bpo-37757: For now, disallow *all* assignment expressions in the + * outermost iterator expression of a comprehension, even those inside + * a nested comprehension or a lambda expression. + */ + if (prev) { + ste->ste_comp_iter_expr = prev->ste_comp_iter_expr; + } /* The entry is owned by the stack. Borrow it for st_cur. */ Py_DECREF(ste); st->st_cur = ste; From b3375276376814d12e03e4376d20d74e27a850f9 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Fri, 9 Aug 2019 23:50:33 +1000 Subject: [PATCH 04/10] Iterable, not iterator --- Lib/test/test_named_expressions.py | 2 +- Python/symtable.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_named_expressions.py b/Lib/test/test_named_expressions.py index 769dcaa7507fd7..1d3889dfae2f41 100644 --- a/Lib/test/test_named_expressions.py +++ b/Lib/test/test_named_expressions.py @@ -155,7 +155,7 @@ def test_named_expression_invalid_comprehension_iterator_expression(self): for case, code in cases: with self.subTest(case=case): with self.assertRaisesRegex(TargetScopeError, - "named expression cannot be used in comprehension iterator expression"): + "named expression cannot be used in comprehension iterable expression"): exec(code, {}, {}) diff --git a/Python/symtable.c b/Python/symtable.c index ae059cdd4706a8..d8373d5bd30097 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -41,7 +41,7 @@ "comprehension inner loop cannot rebind named expression target" #define NAMED_EXPR_COMP_ITER_EXPR \ -"named expression cannot be used in comprehension iterator expression" +"named expression cannot be used in comprehension iterable expression" static PySTEntryObject * ste_new(struct symtable *st, identifier name, _Py_block_ty block, From 34f0c495c77fb9f7c847b422da66ec09115e92a0 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Fri, 9 Aug 2019 23:54:56 +1000 Subject: [PATCH 05/10] Drop TargetScopeError in favour of SyntaxError --- Doc/c-api/exceptions.rst | 3 --- Include/pyerrors.h | 1 - Lib/_compat_pickle.py | 1 - Lib/test/exception_hierarchy.txt | 1 - Lib/test/test_named_expressions.py | 8 ++++---- .../2019-08-05-14-22-59.bpo-37757.lRv5HX.rst | 8 ++++++-- Objects/exceptions.c | 9 --------- PC/python3.def | 1 - Python/symtable.c | 8 ++++---- 9 files changed, 14 insertions(+), 26 deletions(-) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index d8173935596bb8..25bb657dca0e16 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -809,7 +809,6 @@ the variables: single: PyExc_SystemError single: PyExc_SystemExit single: PyExc_TabError - single: PyExc_TargetScopeError single: PyExc_TimeoutError single: PyExc_TypeError single: PyExc_UnboundLocalError @@ -911,8 +910,6 @@ the variables: +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_TabError` | :exc:`TabError` | | +-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_TargetScopeError` | :exc:`TargetScopeError` | | -+-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_TimeoutError` | :exc:`TimeoutError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_TypeError` | :exc:`TypeError` | | diff --git a/Include/pyerrors.h b/Include/pyerrors.h index 94af3cb3420ec6..5125a51ec1aa17 100644 --- a/Include/pyerrors.h +++ b/Include/pyerrors.h @@ -97,7 +97,6 @@ PyAPI_DATA(PyObject *) PyExc_NotImplementedError; PyAPI_DATA(PyObject *) PyExc_SyntaxError; PyAPI_DATA(PyObject *) PyExc_IndentationError; PyAPI_DATA(PyObject *) PyExc_TabError; -PyAPI_DATA(PyObject *) PyExc_TargetScopeError; PyAPI_DATA(PyObject *) PyExc_ReferenceError; PyAPI_DATA(PyObject *) PyExc_SystemError; PyAPI_DATA(PyObject *) PyExc_SystemExit; diff --git a/Lib/_compat_pickle.py b/Lib/_compat_pickle.py index 8bb1cf80afa559..f68496ae639f5f 100644 --- a/Lib/_compat_pickle.py +++ b/Lib/_compat_pickle.py @@ -128,7 +128,6 @@ "SystemError", "SystemExit", "TabError", - "TargetScopeError", "TypeError", "UnboundLocalError", "UnicodeDecodeError", diff --git a/Lib/test/exception_hierarchy.txt b/Lib/test/exception_hierarchy.txt index 15f4491cf237d2..763a6c899b48eb 100644 --- a/Lib/test/exception_hierarchy.txt +++ b/Lib/test/exception_hierarchy.txt @@ -42,7 +42,6 @@ BaseException | +-- NotImplementedError | +-- RecursionError +-- SyntaxError - | +-- TargetScopeError | +-- IndentationError | +-- TabError +-- SystemError diff --git a/Lib/test/test_named_expressions.py b/Lib/test/test_named_expressions.py index 1d3889dfae2f41..4a864c5ba33979 100644 --- a/Lib/test/test_named_expressions.py +++ b/Lib/test/test_named_expressions.py @@ -109,7 +109,7 @@ def test_named_expression_invalid_in_class_body(self): [(42, 1 + ((( j := i )))) for i in range(5)] """ - with self.assertRaisesRegex(TargetScopeError, + with self.assertRaisesRegex(SyntaxError, "named expression within a comprehension cannot be used in a class body"): exec(code, {}, {}) @@ -126,7 +126,7 @@ def test_named_expression_invalid_rebinding_comprehension_iteration_variable(sel ] for case, code in cases: with self.subTest(case=case): - with self.assertRaisesRegex(TargetScopeError, + with self.assertRaisesRegex(SyntaxError, "named expression cannot rebind comprehension iteration variable"): exec(code, {}, {}) @@ -137,7 +137,7 @@ def test_named_expression_invalid_rebinding_comprehension_inner_loop(self): ] for case, code in cases: with self.subTest(case=case): - with self.assertRaisesRegex(TargetScopeError, + with self.assertRaisesRegex(SyntaxError, "comprehension inner loop cannot rebind named expression target"): exec(code, {}, {}) @@ -154,7 +154,7 @@ def test_named_expression_invalid_comprehension_iterator_expression(self): ] for case, code in cases: with self.subTest(case=case): - with self.assertRaisesRegex(TargetScopeError, + with self.assertRaisesRegex(SyntaxError, "named expression cannot be used in comprehension iterable expression"): exec(code, {}, {}) diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-08-05-14-22-59.bpo-37757.lRv5HX.rst b/Misc/NEWS.d/next/Core and Builtins/2019-08-05-14-22-59.bpo-37757.lRv5HX.rst index f3477a189cfec4..258df0dc09bdcf 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2019-08-05-14-22-59.bpo-37757.lRv5HX.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2019-08-05-14-22-59.bpo-37757.lRv5HX.rst @@ -1,3 +1,7 @@ -PEP 572: As described in the PEP, assignment expressions now raised -TargetScopeError when their interaction with comprehension scoping results +:pep:`572`: As described in the PEP, assignment expressions now raise +:exc:`SyntaxError` when their interaction with comprehension scoping results in an ambiguous target scope. + +The ``TargetScopeError`` subclass originally proposed by the PEP has been +removed in favour of just raising regular syntax errors for the disallowed +cases. diff --git a/Objects/exceptions.c b/Objects/exceptions.c index ef9dd512dc9c50..631f5375f73829 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -1521,13 +1521,6 @@ MiddlingExtendsException(PyExc_SyntaxError, IndentationError, SyntaxError, "Improper indentation."); -/* - * TargetScopeError extends SyntaxError - */ -MiddlingExtendsException(PyExc_SyntaxError, TargetScopeError, SyntaxError, - "Improper scope target."); - - /* * TabError extends IndentationError */ @@ -2539,7 +2532,6 @@ _PyExc_Init(void) PRE_INIT(AttributeError); PRE_INIT(SyntaxError); PRE_INIT(IndentationError); - PRE_INIT(TargetScopeError); PRE_INIT(TabError); PRE_INIT(LookupError); PRE_INIT(IndexError); @@ -2680,7 +2672,6 @@ _PyBuiltins_AddExceptions(PyObject *bltinmod) POST_INIT(AttributeError); POST_INIT(SyntaxError); POST_INIT(IndentationError); - POST_INIT(TargetScopeError); POST_INIT(TabError); POST_INIT(LookupError); POST_INIT(IndexError); diff --git a/PC/python3.def b/PC/python3.def index 6844cf1f3b8e14..1f355bffc9bdd0 100644 --- a/PC/python3.def +++ b/PC/python3.def @@ -235,7 +235,6 @@ EXPORTS PyExc_SystemError=python39.PyExc_SystemError DATA PyExc_SystemExit=python39.PyExc_SystemExit DATA PyExc_TabError=python39.PyExc_TabError DATA - PyExc_TargetScopeError=python39.PyExc_TargetScopeError DATA PyExc_TimeoutError=python39.PyExc_TimeoutError DATA PyExc_TypeError=python39.PyExc_TypeError DATA PyExc_UnboundLocalError=python39.PyExc_UnboundLocalError DATA diff --git a/Python/symtable.c b/Python/symtable.c index d8373d5bd30097..69bf12249d849c 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -1034,7 +1034,7 @@ symtable_add_def_helper(struct symtable *st, PyObject *name, int flag, struct _s } if (ste->ste_comp_iter_target) { if (val & (DEF_GLOBAL | DEF_NONLOCAL)) { - PyErr_Format(PyExc_TargetScopeError, + PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_INNER_LOOP_CONFLICT, name); PyErr_SyntaxLocationObject(st->st_filename, ste->ste_lineno, @@ -1441,7 +1441,7 @@ symtable_extend_namedexpr_scope(struct symtable *st, expr_ty e) if (ste->ste_comprehension) { long target_in_scope = _PyST_GetSymbol(ste, target_name); if (target_in_scope & DEF_COMP_ITER) { - PyErr_Format(PyExc_TargetScopeError, NAMED_EXPR_COMP_CONFLICT, target_name); + PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_CONFLICT, target_name); PyErr_SyntaxLocationObject(st->st_filename, e->lineno, e->col_offset); @@ -1470,7 +1470,7 @@ symtable_extend_namedexpr_scope(struct symtable *st, expr_ty e) } /* Disallow usage in ClassBlock */ if (ste->ste_type == ClassBlock) { - PyErr_Format(PyExc_TargetScopeError, NAMED_EXPR_COMP_IN_CLASS, target_name); + PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_IN_CLASS, target_name); PyErr_SyntaxLocationObject(st->st_filename, e->lineno, e->col_offset); @@ -1492,7 +1492,7 @@ symtable_handle_namedexpr(struct symtable *st, expr_ty e) /* Evaluating the outermost range expression for a comprehension */ assert(e->v.NamedExpr.target->kind == Name_kind); PyObject *target_name = e->v.NamedExpr.target->v.Name.id; - PyErr_Format(PyExc_TargetScopeError, NAMED_EXPR_COMP_ITER_EXPR, target_name); + PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_ITER_EXPR, target_name); PyErr_SyntaxLocationObject(st->st_filename, e->lineno, e->col_offset); From b1c109e5a05075ef375e4a7864b7b725bc8608e1 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sat, 10 Aug 2019 01:12:40 +1000 Subject: [PATCH 06/10] Tweak details of exception messages - use the public "assignment expression" name rather than the internal "named expression" one - mention the variable name when the conflict is related to the exact name rather than the code structure - don't pass the variable name when the exception message doesn't need it --- Lib/test/test_named_expressions.py | 36 +++++++++++++++--------------- Python/symtable.c | 14 +++++------- 2 files changed, 24 insertions(+), 26 deletions(-) diff --git a/Lib/test/test_named_expressions.py b/Lib/test/test_named_expressions.py index 4a864c5ba33979..dfe5c22a663cd0 100644 --- a/Lib/test/test_named_expressions.py +++ b/Lib/test/test_named_expressions.py @@ -110,35 +110,35 @@ def test_named_expression_invalid_in_class_body(self): """ with self.assertRaisesRegex(SyntaxError, - "named expression within a comprehension cannot be used in a class body"): + "assignment expression within a comprehension cannot be used in a class body"): exec(code, {}, {}) def test_named_expression_invalid_rebinding_comprehension_iteration_variable(self): cases = [ - ("Local reuse", "[i := 0 for i in range(5)]"), - ("Nested reuse", "[[(j := 0) for i in range(5)] for j in range(5)]"), - ("Reuse inner loop target", "[(j := 0) for i in range(5) for j in range(5)]"), - ("Unpacking reuse", "[i := 0 for i, j in [(0, 1)]]"), - ("Reuse in loop condition", "[i+1 for i in range(5) if (i := 0)]"), - ("Unreachable reuse", "[False or (i:=0) for i in range(5)]"), - ("Unreachable nested reuse", + ("Local reuse", 'i', "[i := 0 for i in range(5)]"), + ("Nested reuse", 'j', "[[(j := 0) for i in range(5)] for j in range(5)]"), + ("Reuse inner loop target", 'j', "[(j := 0) for i in range(5) for j in range(5)]"), + ("Unpacking reuse", 'i', "[i := 0 for i, j in [(0, 1)]]"), + ("Reuse in loop condition", 'i', "[i+1 for i in range(5) if (i := 0)]"), + ("Unreachable reuse", 'i', "[False or (i:=0) for i in range(5)]"), + ("Unreachable nested reuse", 'i', "[(i, j) for i in range(5) for j in range(5) if True or (i:=10)]"), ] - for case, code in cases: + for case, target, code in cases: + msg = f"assignment expression cannot rebind comprehension iteration variable '{target}'" with self.subTest(case=case): - with self.assertRaisesRegex(SyntaxError, - "named expression cannot rebind comprehension iteration variable"): + with self.assertRaisesRegex(SyntaxError, msg): exec(code, {}, {}) def test_named_expression_invalid_rebinding_comprehension_inner_loop(self): cases = [ - ("Inner reuse", "[i for i in range(5) if (j := 0) for j in range(5)]"), - ("Inner unpacking reuse", "[i for i in range(5) if (j := 0) for j, k in [(0, 1)]]"), + ("Inner reuse", 'j', "[i for i in range(5) if (j := 0) for j in range(5)]"), + ("Inner unpacking reuse", 'j', "[i for i in range(5) if (j := 0) for j, k in [(0, 1)]]"), ] - for case, code in cases: + for case, target, code in cases: + msg = f"comprehension inner loop cannot rebind assignment expression target '{target}'" with self.subTest(case=case): - with self.assertRaisesRegex(SyntaxError, - "comprehension inner loop cannot rebind named expression target"): + with self.assertRaisesRegex(SyntaxError, msg): exec(code, {}, {}) def test_named_expression_invalid_comprehension_iterator_expression(self): @@ -152,10 +152,10 @@ def test_named_expression_invalid_comprehension_iterator_expression(self): ("Nested comprehension condition", "[i for i in [j for j in range(5) if (j := True)]]"), ("Nested comprehension body", "[i for i in [(j := True) for j in range(5)]]"), ] + msg = "assignment expression cannot be used in a comprehension iterable expression" for case, code in cases: with self.subTest(case=case): - with self.assertRaisesRegex(SyntaxError, - "named expression cannot be used in comprehension iterable expression"): + with self.assertRaisesRegex(SyntaxError, msg): exec(code, {}, {}) diff --git a/Python/symtable.c b/Python/symtable.c index 69bf12249d849c..6cfdc53880eb6a 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -32,16 +32,16 @@ #define IMPORT_STAR_WARNING "import * only allowed at module level" #define NAMED_EXPR_COMP_IN_CLASS \ -"named expression within a comprehension cannot be used in a class body" +"assignment expression within a comprehension cannot be used in a class body" #define NAMED_EXPR_COMP_CONFLICT \ -"named expression cannot rebind comprehension iteration variable" +"assignment expression cannot rebind comprehension iteration variable '%U'" #define NAMED_EXPR_COMP_INNER_LOOP_CONFLICT \ -"comprehension inner loop cannot rebind named expression target" +"comprehension inner loop cannot rebind assignment expression target '%U'" #define NAMED_EXPR_COMP_ITER_EXPR \ -"named expression cannot be used in comprehension iterable expression" +"assignment expression cannot be used in a comprehension iterable expression" static PySTEntryObject * ste_new(struct symtable *st, identifier name, _Py_block_ty block, @@ -1470,7 +1470,7 @@ symtable_extend_namedexpr_scope(struct symtable *st, expr_ty e) } /* Disallow usage in ClassBlock */ if (ste->ste_type == ClassBlock) { - PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_IN_CLASS, target_name); + PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_IN_CLASS); PyErr_SyntaxLocationObject(st->st_filename, e->lineno, e->col_offset); @@ -1490,9 +1490,7 @@ symtable_handle_namedexpr(struct symtable *st, expr_ty e) { if (st->st_cur->ste_comp_iter_expr > 0) { /* Evaluating the outermost range expression for a comprehension */ - assert(e->v.NamedExpr.target->kind == Name_kind); - PyObject *target_name = e->v.NamedExpr.target->v.Name.id; - PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_ITER_EXPR, target_name); + PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_ITER_EXPR); PyErr_SyntaxLocationObject(st->st_filename, e->lineno, e->col_offset); From 1cf1ae8e5d87d84ce845f16a40252a8f8cc33244 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Wed, 14 Aug 2019 13:16:29 +1000 Subject: [PATCH 07/10] Fix/clarify comments --- Python/symtable.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/symtable.c b/Python/symtable.c index 6cfdc53880eb6a..946db539f46d79 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -1489,7 +1489,7 @@ static int symtable_handle_namedexpr(struct symtable *st, expr_ty e) { if (st->st_cur->ste_comp_iter_expr > 0) { - /* Evaluating the outermost range expression for a comprehension */ + /* Assignment isn't allowed in a comprehension iterable expression */ PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_ITER_EXPR); PyErr_SyntaxLocationObject(st->st_filename, e->lineno, @@ -1497,7 +1497,7 @@ symtable_handle_namedexpr(struct symtable *st, expr_ty e) VISIT_QUIT(st, 0); } if (st->st_cur->ste_comprehension) { - /* Already inside a comprehension body */ + /* Inside a comprehension body, so find the right target scope */ if (!symtable_extend_namedexpr_scope(st, e->v.NamedExpr.target)) VISIT_QUIT(st, 0); } From 40307d59f51b2623befe68b1a7d051537364fa75 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Wed, 14 Aug 2019 13:23:49 +1000 Subject: [PATCH 08/10] Add comment for iteration variable marking --- Python/symtable.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Python/symtable.c b/Python/symtable.c index 946db539f46d79..e48baa808d41a6 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -1033,6 +1033,11 @@ symtable_add_def_helper(struct symtable *st, PyObject *name, int flag, struct _s val = flag; } if (ste->ste_comp_iter_target) { + /* This name is an iteration variable in a comprehension, + * so check for a binding conflict with any named expressions. + * Otherwise, mark it as an iteration variable so subsequent + * named expressions can check for conflicts. + */ if (val & (DEF_GLOBAL | DEF_NONLOCAL)) { PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_INNER_LOOP_CONFLICT, name); From 532cf125f4d50f23fc4eb8dbda46ff5d0d7910c9 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Wed, 14 Aug 2019 22:26:37 +1000 Subject: [PATCH 09/10] Cover more potentially different cases in new tests --- Lib/test/test_named_expressions.py | 45 +++++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_named_expressions.py b/Lib/test/test_named_expressions.py index dfe5c22a663cd0..37f2c06daa57cf 100644 --- a/Lib/test/test_named_expressions.py +++ b/Lib/test/test_named_expressions.py @@ -139,12 +139,17 @@ def test_named_expression_invalid_rebinding_comprehension_inner_loop(self): msg = f"comprehension inner loop cannot rebind assignment expression target '{target}'" with self.subTest(case=case): with self.assertRaisesRegex(SyntaxError, msg): - exec(code, {}, {}) + exec(code, {}) # Module scope + with self.assertRaisesRegex(SyntaxError, msg): + exec(code, {}, {}) # Class scope + with self.assertRaisesRegex(SyntaxError, msg): + exec(f"lambda: {code}", {}) # Function scope - def test_named_expression_invalid_comprehension_iterator_expression(self): + def test_named_expression_invalid_comprehension_iterable_expression(self): cases = [ ("Top level", "[i for i in (i := range(5))]"), - ("Inside container", "[i for i in (2, 3, i := range(5))]"), + ("Inside tuple", "[i for i in (2, 3, i := range(5))]"), + ("Inside list", "[i for i in [2, 3, i := range(5)]]"), ("Different name", "[i for i in (j := range(5))]"), ("Lambda expression", "[i for i in (lambda:(j := range(5)))()]"), ("Inner loop", "[i for i in range(5) for j in (i := range(5))]"), @@ -156,7 +161,11 @@ def test_named_expression_invalid_comprehension_iterator_expression(self): for case, code in cases: with self.subTest(case=case): with self.assertRaisesRegex(SyntaxError, msg): - exec(code, {}, {}) + exec(code, {}) # Module scope + with self.assertRaisesRegex(SyntaxError, msg): + exec(code, {}, {}) # Class scope + with self.assertRaisesRegex(SyntaxError, msg): + exec(f"lambda: {code}", {}) # Function scope class NamedExpressionAssignmentTest(unittest.TestCase): @@ -433,6 +442,34 @@ def spam(): self.assertEqual(ns["a"], 20) + def test_named_expression_variable_reuse_in_comprehensions(self): + # The compiler is expected to raise syntax error for comprehension + # iteration variables, but should be fine with rebinding of other + # names (e.g. globals, nonlocals, other assignment expressions) + + # The cases are all defined to produce the same expected result + # Each comprehension is checked at both function scope and module scope + rebinding = "[x := i for i in range(3) if (x := i) or not x]" + filter_ref = "[x := i for i in range(3) if x or not x]" + body_ref = "[x for i in range(3) if (x := i) or not x]" + nested_ref = "[j for i in range(3) if x or not x for j in range(3) if (x := i)][:-3]" + cases = [ + ("Rebind global", f"x = 1; result = {rebinding}"), + ("Rebind nonlocal", f"result, x = (lambda x=1: ({rebinding}, x))()"), + ("Filter global", f"x = 1; result = {filter_ref}"), + ("Filter nonlocal", f"result, x = (lambda x=1: ({filter_ref}, x))()"), + ("Body global", f"x = 1; result = {body_ref}"), + ("Body nonlocal", f"result, x = (lambda x=1: ({body_ref}, x))()"), + ("Nested global", f"x = 1; result = {nested_ref}"), + ("Nested nonlocal", f"result, x = (lambda x=1: ({nested_ref}, x))()"), + ] + msg = "assignment expression cannot be used in a comprehension iterable expression" + for case, code in cases: + with self.subTest(case=case): + ns = {} + exec(code, ns) + self.assertEqual(ns["x"], 2) + self.assertEqual(ns["result"], [0, 1, 2]) if __name__ == "__main__": unittest.main() From 4ee6cc0f76d9a1bb06a2fffd6e5104596c6602b0 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sun, 25 Aug 2019 22:32:12 +1000 Subject: [PATCH 10/10] Remove unused variable from test case --- Lib/test/test_named_expressions.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_named_expressions.py b/Lib/test/test_named_expressions.py index 37f2c06daa57cf..b1027ce78006de 100644 --- a/Lib/test/test_named_expressions.py +++ b/Lib/test/test_named_expressions.py @@ -463,7 +463,6 @@ def test_named_expression_variable_reuse_in_comprehensions(self): ("Nested global", f"x = 1; result = {nested_ref}"), ("Nested nonlocal", f"result, x = (lambda x=1: ({nested_ref}, x))()"), ] - msg = "assignment expression cannot be used in a comprehension iterable expression" for case, code in cases: with self.subTest(case=case): ns = {} pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy