From 9d452fb174388ed861a0939f6099eefa7c6ad29d Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Mon, 15 May 2023 17:33:43 -0700 Subject: [PATCH 1/2] gh-104374: don't inline comprehensions in non-function scopes --- Include/internal/pycore_code.h | 1 - Include/internal/pycore_compile.h | 3 -- Lib/test/test_compile.py | 12 ++++++++ Lib/test/test_compiler_assemble.py | 5 +--- Lib/test/test_listcomps.py | 40 ++++++++++++++++++++++++-- Modules/_testinternalcapi.c | 2 -- Objects/frameobject.c | 3 -- Python/assemble.c | 3 -- Python/compile.c | 46 +----------------------------- Python/symtable.c | 1 + 10 files changed, 53 insertions(+), 63 deletions(-) diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 75a23f3f5af560..2d0c49dfa0c790 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -128,7 +128,6 @@ struct callable_cache { // Note that these all fit within a byte, as do combinations. // Later, we will use the smaller numbers to differentiate the different // kinds of locals (e.g. pos-only arg, varkwargs, local-only). -#define CO_FAST_HIDDEN 0x10 #define CO_FAST_LOCAL 0x20 #define CO_FAST_CELL 0x40 #define CO_FAST_FREE 0x80 diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index 499f55f3e276be..d2b12c91fe7a00 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -70,9 +70,6 @@ typedef struct { PyObject *u_varnames; /* local variables */ PyObject *u_cellvars; /* cell variables */ PyObject *u_freevars; /* free variables */ - PyObject *u_fasthidden; /* dict; keys are names that are fast-locals only - temporarily within an inlined comprehension. When - value is True, treat as fast-local. */ Py_ssize_t u_argcount; /* number of arguments for block */ Py_ssize_t u_posonlyargcount; /* number of positional only arguments for block */ diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index c68b9ce388466e..304edc9bb12b83 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1352,11 +1352,14 @@ def test_multiline_list_comprehension(self): and x != 50)] """) compiled_code, _ = self.check_positions_against_ast(snippet) + compiled_code = compiled_code.co_consts[0] self.assertIsInstance(compiled_code, types.CodeType) self.assertOpcodeSourcePositionIs(compiled_code, 'LIST_APPEND', line=1, end_line=2, column=1, end_column=8, occurrence=1) self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', line=1, end_line=2, column=1, end_column=8, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE', + line=1, end_line=6, column=0, end_column=32, occurrence=1) def test_multiline_async_list_comprehension(self): snippet = textwrap.dedent("""\ @@ -1390,11 +1393,14 @@ def test_multiline_set_comprehension(self): and x != 50)} """) compiled_code, _ = self.check_positions_against_ast(snippet) + compiled_code = compiled_code.co_consts[0] self.assertIsInstance(compiled_code, types.CodeType) self.assertOpcodeSourcePositionIs(compiled_code, 'SET_ADD', line=1, end_line=2, column=1, end_column=8, occurrence=1) self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', line=1, end_line=2, column=1, end_column=8, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE', + line=1, end_line=6, column=0, end_column=32, occurrence=1) def test_multiline_async_set_comprehension(self): snippet = textwrap.dedent("""\ @@ -1428,11 +1434,14 @@ def test_multiline_dict_comprehension(self): and x != 50)} """) compiled_code, _ = self.check_positions_against_ast(snippet) + compiled_code = compiled_code.co_consts[0] self.assertIsInstance(compiled_code, types.CodeType) self.assertOpcodeSourcePositionIs(compiled_code, 'MAP_ADD', line=1, end_line=2, column=1, end_column=7, occurrence=1) self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', line=1, end_line=2, column=1, end_column=7, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE', + line=1, end_line=6, column=0, end_column=32, occurrence=1) def test_multiline_async_dict_comprehension(self): snippet = textwrap.dedent("""\ @@ -1702,6 +1711,9 @@ def test_column_offset_deduplication(self): for source in [ "lambda: a", "(a for b in c)", + "[a for b in c]", + "{a for b in c}", + "{a: b for c in d}", ]: with self.subTest(source): code = compile(f"{source}, {source}", "", "eval") diff --git a/Lib/test/test_compiler_assemble.py b/Lib/test/test_compiler_assemble.py index 3e2a127de728cd..0bd7a09b001c1d 100644 --- a/Lib/test/test_compiler_assemble.py +++ b/Lib/test/test_compiler_assemble.py @@ -16,7 +16,7 @@ def complete_metadata(self, metadata, filename="myfile.py"): metadata.setdefault(key, key) for key in ['consts']: metadata.setdefault(key, []) - for key in ['names', 'varnames', 'cellvars', 'freevars', 'fasthidden']: + for key in ['names', 'varnames', 'cellvars', 'freevars']: metadata.setdefault(key, {}) for key in ['argcount', 'posonlyargcount', 'kwonlyargcount']: metadata.setdefault(key, 0) @@ -33,9 +33,6 @@ def assemble_test(self, insts, metadata, expected): expected_metadata = {} for key, value in metadata.items(): - if key == "fasthidden": - # not exposed on code object - continue if isinstance(value, list): expected_metadata[key] = tuple(value) elif isinstance(value, dict): diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index 23e1b8c1ce3193..7e29c784f698a2 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -200,7 +200,7 @@ def f(): y = [g for x in [1]] """ outputs = {"y": [2]} - self._check_in_scopes(code, outputs) + self._check_in_scopes(code, outputs, scopes=["module", "function"]) def test_inner_cell_shadows_outer_redefined(self): code = """ @@ -328,7 +328,7 @@ def test_nested_2(self): y = [x for [x ** x for x in range(x)][x - 1] in l] """ outputs = {"y": [3, 3, 3]} - self._check_in_scopes(code, outputs) + self._check_in_scopes(code, outputs, scopes=["module", "function"]) def test_nested_3(self): code = """ @@ -379,6 +379,42 @@ def f(): with self.assertRaises(UnboundLocalError): f() + def test_name_error_in_class_scope(self): + code = """ + y = 1 + [x + y for x in range(2)] + """ + self._check_in_scopes(code, raises=NameError, scopes=["class"]) + + def test_global_in_class_scope(self): + code = """ + y = 2 + vals = [(x, y) for x in range(2)] + """ + outputs = {"vals": [(0, 1), (1, 1)]} + self._check_in_scopes(code, outputs, ns={"y": 1}, scopes=["class"]) + + def test_in_class_scope_inside_function_1(self): + code = """ + class C: + y = 2 + vals = [(x, y) for x in range(2)] + vals = C.vals + """ + outputs = {"vals": [(0, 1), (1, 1)]} + self._check_in_scopes(code, outputs, ns={"y": 1}, scopes=["function"]) + + def test_in_class_scope_inside_function_2(self): + code = """ + y = 1 + class C: + y = 2 + vals = [(x, y) for x in range(2)] + vals = C.vals + """ + outputs = {"vals": [(0, 1), (1, 1)]} + self._check_in_scopes(code, outputs, scopes=["function"]) + __test__ = {'doctests' : doctests} diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index ea9b6e72b3c924..8009dca3d0b746 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -669,14 +669,12 @@ _testinternalcapi_assemble_code_object_impl(PyObject *module, umd.u_varnames = PyDict_GetItemString(metadata, "varnames"); umd.u_cellvars = PyDict_GetItemString(metadata, "cellvars"); umd.u_freevars = PyDict_GetItemString(metadata, "freevars"); - umd.u_fasthidden = PyDict_GetItemString(metadata, "fasthidden"); assert(PyDict_Check(umd.u_consts)); assert(PyDict_Check(umd.u_names)); assert(PyDict_Check(umd.u_varnames)); assert(PyDict_Check(umd.u_cellvars)); assert(PyDict_Check(umd.u_freevars)); - assert(PyDict_Check(umd.u_fasthidden)); umd.u_argcount = get_nonnegative_int_from_dict(metadata, "argcount"); umd.u_posonlyargcount = get_nonnegative_int_from_dict(metadata, "posonlyargcount"); diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 2c90a6b71311ca..df0910d58e116c 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1222,9 +1222,6 @@ _PyFrame_FastToLocalsWithError(_PyInterpreterFrame *frame) PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); - if (kind & CO_FAST_HIDDEN) { - continue; - } if (value == NULL) { if (PyObject_DelItem(locals, name) != 0) { if (PyErr_ExceptionMatches(PyExc_KeyError)) { diff --git a/Python/assemble.c b/Python/assemble.c index 8789d8ef978c22..aefff3cd9d8402 100644 --- a/Python/assemble.c +++ b/Python/assemble.c @@ -457,9 +457,6 @@ compute_localsplus_info(_PyCompile_CodeUnitMetadata *umd, int nlocalsplus, assert(offset < nlocalsplus); // For now we do not distinguish arg kinds. _PyLocals_Kind kind = CO_FAST_LOCAL; - if (PyDict_Contains(umd->u_fasthidden, k)) { - kind |= CO_FAST_HIDDEN; - } if (PyDict_GetItem(umd->u_cellvars, k) != NULL) { kind |= CO_FAST_CELL; } diff --git a/Python/compile.c b/Python/compile.c index bf5e4a52482a4a..8991000ad7d415 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -691,7 +691,6 @@ compiler_unit_free(struct compiler_unit *u) Py_CLEAR(u->u_metadata.u_varnames); Py_CLEAR(u->u_metadata.u_freevars); Py_CLEAR(u->u_metadata.u_cellvars); - Py_CLEAR(u->u_metadata.u_fasthidden); Py_CLEAR(u->u_private); PyObject_Free(u); } @@ -1260,12 +1259,6 @@ compiler_enter_scope(struct compiler *c, identifier name, return ERROR; } - u->u_metadata.u_fasthidden = PyDict_New(); - if (!u->u_metadata.u_fasthidden) { - compiler_unit_free(u); - return ERROR; - } - u->u_nfblocks = 0; u->u_metadata.u_firstlineno = lineno; u->u_metadata.u_consts = PyDict_New(); @@ -3725,8 +3718,7 @@ compiler_nameop(struct compiler *c, location loc, optype = OP_DEREF; break; case LOCAL: - if (c->u->u_ste->ste_type == FunctionBlock || - (PyDict_GetItem(c->u->u_metadata.u_fasthidden, mangled) == Py_True)) + if (c->u->u_ste->ste_type == FunctionBlock) optype = OP_FAST; break; case GLOBAL_IMPLICIT: @@ -4988,7 +4980,6 @@ compiler_async_comprehension_generator(struct compiler *c, location loc, typedef struct { PyObject *pushed_locals; PyObject *temp_symbols; - PyObject *fast_hidden; } inlined_comprehension_state; static int @@ -5008,24 +4999,6 @@ push_inlined_comprehension_state(struct compiler *c, location loc, // assignment expression to a nonlocal in the comprehension, these don't // need handling here since they shouldn't be isolated if (symbol & DEF_LOCAL && !(symbol & DEF_NONLOCAL)) { - if (c->u->u_ste->ste_type != FunctionBlock) { - // non-function scope: override this name to use fast locals - PyObject *orig = PyDict_GetItem(c->u->u_metadata.u_fasthidden, k); - if (orig != Py_True) { - if (PyDict_SetItem(c->u->u_metadata.u_fasthidden, k, Py_True) < 0) { - return ERROR; - } - if (state->fast_hidden == NULL) { - state->fast_hidden = PySet_New(NULL); - if (state->fast_hidden == NULL) { - return ERROR; - } - } - if (PySet_Add(state->fast_hidden, k) < 0) { - return ERROR; - } - } - } long scope = (symbol >> SCOPE_OFFSET) & SCOPE_MASK; PyObject *outv = PyDict_GetItemWithError(c->u->u_ste->ste_symbols, k); if (outv == NULL) { @@ -5131,22 +5104,6 @@ pop_inlined_comprehension_state(struct compiler *c, location loc, } Py_CLEAR(state.pushed_locals); } - if (state.fast_hidden) { - while (PySet_Size(state.fast_hidden) > 0) { - PyObject *k = PySet_Pop(state.fast_hidden); - if (k == NULL) { - return ERROR; - } - // we set to False instead of clearing, so we can track which names - // were temporarily fast-locals and should use CO_FAST_HIDDEN - if (PyDict_SetItem(c->u->u_metadata.u_fasthidden, k, Py_False)) { - Py_DECREF(k); - return ERROR; - } - Py_DECREF(k); - } - Py_CLEAR(state.fast_hidden); - } return SUCCESS; } @@ -5293,7 +5250,6 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, Py_XDECREF(entry); Py_XDECREF(inline_state.pushed_locals); Py_XDECREF(inline_state.temp_symbols); - Py_XDECREF(inline_state.fast_hidden); return ERROR; } diff --git a/Python/symtable.c b/Python/symtable.c index 2c29f608413501..c51c21988ec7f1 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -912,6 +912,7 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, // we inline all non-generator-expression comprehensions int inline_comp = + ste->ste_type == FunctionBlock && entry->ste_comprehension && !entry->ste_generator; From 2c47a7551dd67263cce189ef4017463c914f19e9 Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Mon, 15 May 2023 17:51:30 -0700 Subject: [PATCH 2/2] update NEWS and What's New --- Doc/whatsnew/3.12.rst | 7 ++++--- .../2023-01-30-15-40-29.gh-issue-97933.nUlp3r.rst | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 3e55b3fa0f4734..abd6ed4f038d0d 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -158,9 +158,10 @@ New Features PEP 709: Comprehension inlining ------------------------------- -Dictionary, list, and set comprehensions are now inlined, rather than creating a -new single-use function object for each execution of the comprehension. This -speeds up execution of a comprehension by up to 2x. +Dictionary, list, and set comprehensions occurring inside functions are now +inlined, rather than creating a new single-use function object for each +execution of the comprehension. This speeds up execution of a comprehension by +up to 2x. Comprehension iteration variables remain isolated; they don't overwrite a variable of the same name in the outer scope, nor are they visible after the diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-01-30-15-40-29.gh-issue-97933.nUlp3r.rst b/Misc/NEWS.d/next/Core and Builtins/2023-01-30-15-40-29.gh-issue-97933.nUlp3r.rst index 2eec05cb3ace5c..8d55567a9a6d36 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2023-01-30-15-40-29.gh-issue-97933.nUlp3r.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2023-01-30-15-40-29.gh-issue-97933.nUlp3r.rst @@ -1,2 +1,2 @@ -:pep:`709`: inline list, dict and set comprehensions to improve performance -and reduce bytecode size. +:pep:`709`: inline list, dict and set comprehensions in functions to improve +performance and reduce bytecode size. pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy