diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index b186408931c92e..30851dc2dbec44 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -559,6 +559,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(dot)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(dot_locals)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(empty)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(format)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(generic_base)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(json_decoder)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(kwdefaults)); @@ -745,7 +746,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_abstract_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_active)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_align_)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_annotation)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_anonymous_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_argtypes_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_as_parameter_)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index e1808c85acfb2d..009802c441685c 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -45,6 +45,7 @@ struct _Py_global_strings { STRUCT_FOR_STR(dot, ".") STRUCT_FOR_STR(dot_locals, ".") STRUCT_FOR_STR(empty, "") + STRUCT_FOR_STR(format, ".format") STRUCT_FOR_STR(generic_base, ".generic_base") STRUCT_FOR_STR(json_decoder, "json.decoder") STRUCT_FOR_STR(kwdefaults, ".kwdefaults") @@ -234,7 +235,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(_abstract_) STRUCT_FOR_ID(_active) STRUCT_FOR_ID(_align_) - STRUCT_FOR_ID(_annotation) STRUCT_FOR_ID(_anonymous_) STRUCT_FOR_ID(_argtypes_) STRUCT_FOR_ID(_as_parameter_) diff --git a/Include/internal/pycore_opcode_utils.h b/Include/internal/pycore_opcode_utils.h index b06e469dd5bd91..e76f4840a66891 100644 --- a/Include/internal/pycore_opcode_utils.h +++ b/Include/internal/pycore_opcode_utils.h @@ -57,6 +57,7 @@ extern "C" { #define MAKE_FUNCTION_KWDEFAULTS 0x02 #define MAKE_FUNCTION_ANNOTATIONS 0x04 #define MAKE_FUNCTION_CLOSURE 0x08 +#define MAKE_FUNCTION_ANNOTATE 0x10 /* Values used as the oparg for LOAD_COMMON_CONSTANT */ #define CONSTANT_ASSERTIONERROR 0 diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 2dde6febc2cae4..ff5b6ee8e0f006 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -554,6 +554,7 @@ extern "C" { INIT_STR(dot, "."), \ INIT_STR(dot_locals, "."), \ INIT_STR(empty, ""), \ + INIT_STR(format, ".format"), \ INIT_STR(generic_base, ".generic_base"), \ INIT_STR(json_decoder, "json.decoder"), \ INIT_STR(kwdefaults, ".kwdefaults"), \ @@ -743,7 +744,6 @@ extern "C" { INIT_ID(_abstract_), \ INIT_ID(_active), \ INIT_ID(_align_), \ - INIT_ID(_annotation), \ INIT_ID(_anonymous_), \ INIT_ID(_argtypes_), \ INIT_ID(_as_parameter_), \ diff --git a/Include/internal/pycore_symtable.h b/Include/internal/pycore_symtable.h index ac6c499c08264e..5d544765237df5 100644 --- a/Include/internal/pycore_symtable.h +++ b/Include/internal/pycore_symtable.h @@ -12,8 +12,9 @@ struct _mod; // Type defined in pycore_ast.h typedef enum _block_type { FunctionBlock, ClassBlock, ModuleBlock, - // Used for annotations if 'from __future__ import annotations' is active. - // Annotation blocks cannot bind names and are not evaluated. + // Used for annotations. If 'from __future__ import annotations' is active, + // annotation blocks cannot bind names and are not evaluated. Otherwise, they + // are lazily evaluated (see PEP 649). AnnotationBlock, // Used for generics and type aliases. These work mostly like functions // (see PEP 695 for details). The three different blocks function identically; @@ -89,6 +90,7 @@ typedef struct _symtable_entry { including free refs to globals */ unsigned ste_generator : 1; /* true if namespace is a generator */ unsigned ste_coroutine : 1; /* true if namespace is a coroutine */ + unsigned ste_annotations_used : 1; /* true if there are any annotations in this scope */ _Py_comprehension_ty ste_comprehension; /* Kind of comprehension (if any) */ unsigned ste_varargs : 1; /* true if block has varargs */ unsigned ste_varkeywords : 1; /* true if block has varkeywords */ @@ -110,6 +112,7 @@ typedef struct _symtable_entry { int ste_end_col_offset; /* end offset of first line of block */ int ste_opt_lineno; /* lineno of last exec or import * */ int ste_opt_col_offset; /* offset of last exec or import * */ + struct _symtable_entry *ste_annotation_block; /* symbol table entry for this entry's annotations */ struct symtable *ste_table; } PySTEntryObject; @@ -126,6 +129,7 @@ extern struct symtable* _PySymtable_Build( PyObject *filename, _PyFutureFeatures *future); extern PySTEntryObject* _PySymtable_Lookup(struct symtable *, void *); +extern int _PySymtable_LookupOptional(struct symtable *, void *, PySTEntryObject **); extern void _PySymtable_Free(struct symtable *); diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index b00119a1bad7ff..69d93a9610a2e5 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -543,9 +543,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(_align_); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(_annotation); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(_anonymous_); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Lib/inspect.py b/Lib/inspect.py index 2b7f8bec482f8e..5570a43ebfea19 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -220,13 +220,7 @@ def get_annotations(obj, *, globals=None, locals=None, eval_str=False): """ if isinstance(obj, type): # class - obj_dict = getattr(obj, '__dict__', None) - if obj_dict and hasattr(obj_dict, 'get'): - ann = obj_dict.get('__annotations__', None) - if isinstance(ann, types.GetSetDescriptorType): - ann = None - else: - ann = None + ann = obj.__annotations__ obj_globals = None module_name = getattr(obj, '__module__', None) diff --git a/Lib/symtable.py b/Lib/symtable.py index ba2f0dafcd0063..af65e93e68eda4 100644 --- a/Lib/symtable.py +++ b/Lib/symtable.py @@ -222,6 +222,8 @@ def get_methods(self): if self.__methods is None: d = {} for st in self._table.children: + if st.type == _symtable.TYPE_ANNOTATION: + continue d[st.name] = 1 self.__methods = tuple(d) return self.__methods diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index b1a1b77c53e8cb..b0ae1289224070 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -352,32 +352,21 @@ def wrap_func_w_kwargs(): dis_annot_stmt_str = """\ 0 RESUME 0 - 2 SETUP_ANNOTATIONS - LOAD_CONST 0 (1) + 2 LOAD_CONST 0 (1) STORE_NAME 0 (x) - LOAD_NAME 1 (int) - LOAD_NAME 2 (__annotations__) - LOAD_CONST 1 ('x') - STORE_SUBSCR - - 3 LOAD_NAME 3 (fun) - PUSH_NULL - LOAD_CONST 0 (1) - CALL 1 - LOAD_NAME 2 (__annotations__) - LOAD_CONST 2 ('y') - STORE_SUBSCR 4 LOAD_CONST 0 (1) - LOAD_NAME 4 (lst) - LOAD_NAME 3 (fun) + LOAD_NAME 1 (lst) + LOAD_NAME 2 (fun) PUSH_NULL - LOAD_CONST 3 (0) + LOAD_CONST 1 (0) CALL 1 STORE_SUBSCR - LOAD_NAME 1 (int) - POP_TOP - RETURN_CONST 4 (None) + + 2 LOAD_CONST 2 (", line 2>) + MAKE_FUNCTION + STORE_NAME 3 (__annotate__) + RETURN_CONST 3 (None) """ compound_stmt_str = """\ diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index c72f4387108ca8..5b7a639c025a0f 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -306,16 +306,6 @@ def test_eof_error(self): var_annot_global: int # a global annotated is necessary for test_var_annot -# custom namespace for testing __annotations__ - -class CNS: - def __init__(self): - self._dct = {} - def __setitem__(self, item, value): - self._dct[item.lower()] = value - def __getitem__(self, item): - return self._dct[item] - class GrammarTests(unittest.TestCase): @@ -446,22 +436,12 @@ class F(C, A): self.assertEqual(E.__annotations__, {}) self.assertEqual(F.__annotations__, {}) - - def test_var_annot_metaclass_semantics(self): - class CMeta(type): - @classmethod - def __prepare__(metacls, name, bases, **kwds): - return {'__annotations__': CNS()} - class CC(metaclass=CMeta): - XX: 'ANNOT' - self.assertEqual(CC.__annotations__['xx'], 'ANNOT') - def test_var_annot_module_semantics(self): self.assertEqual(test.__annotations__, {}) self.assertEqual(ann_module.__annotations__, - {1: 2, 'x': int, 'y': str, 'f': typing.Tuple[int, int], 'u': int | float}) + {'x': int, 'y': str, 'f': typing.Tuple[int, int], 'u': int | float}) self.assertEqual(ann_module.M.__annotations__, - {'123': 123, 'o': type}) + {'o': type}) self.assertEqual(ann_module2.__annotations__, {}) def test_var_annot_in_module(self): @@ -476,51 +456,12 @@ def test_var_annot_in_module(self): ann_module3.D_bad_ann(5) def test_var_annot_simple_exec(self): - gns = {}; lns= {} + gns = {}; lns = {} exec("'docstring'\n" - "__annotations__[1] = 2\n" "x: int = 5\n", gns, lns) - self.assertEqual(lns["__annotations__"], {1: 2, 'x': int}) - with self.assertRaises(KeyError): - gns['__annotations__'] - - def test_var_annot_custom_maps(self): - # tests with custom locals() and __annotations__ - ns = {'__annotations__': CNS()} - exec('X: int; Z: str = "Z"; (w): complex = 1j', ns) - self.assertEqual(ns['__annotations__']['x'], int) - self.assertEqual(ns['__annotations__']['z'], str) + self.assertEqual(lns["__annotate__"](1), {'x': int}) with self.assertRaises(KeyError): - ns['__annotations__']['w'] - nonloc_ns = {} - class CNS2: - def __init__(self): - self._dct = {} - def __setitem__(self, item, value): - nonlocal nonloc_ns - self._dct[item] = value - nonloc_ns[item] = value - def __getitem__(self, item): - return self._dct[item] - exec('x: int = 1', {}, CNS2()) - self.assertEqual(nonloc_ns['__annotations__']['x'], int) - - def test_var_annot_refleak(self): - # complex case: custom locals plus custom __annotations__ - # this was causing refleak - cns = CNS() - nonloc_ns = {'__annotations__': cns} - class CNS2: - def __init__(self): - self._dct = {'__annotations__': cns} - def __setitem__(self, item, value): - nonlocal nonloc_ns - self._dct[item] = value - nonloc_ns[item] = value - def __getitem__(self, item): - return self._dct[item] - exec('X: str', {}, CNS2()) - self.assertEqual(nonloc_ns['__annotations__']['x'], str) + gns['__annotate__'] def test_var_annot_rhs(self): ns = {} diff --git a/Lib/test/test_module/__init__.py b/Lib/test/test_module/__init__.py index 952ba43f72504d..56edd0c637f376 100644 --- a/Lib/test/test_module/__init__.py +++ b/Lib/test/test_module/__init__.py @@ -360,6 +360,8 @@ def test_annotations_are_created_correctly(self): ann_module4 = import_helper.import_fresh_module( 'test.typinganndata.ann_module4', ) + self.assertFalse("__annotations__" in ann_module4.__dict__) + self.assertEqual(ann_module4.__annotations__, {"a": int, "b": str}) self.assertTrue("__annotations__" in ann_module4.__dict__) del ann_module4.__annotations__ self.assertFalse("__annotations__" in ann_module4.__dict__) diff --git a/Lib/test/test_opcodes.py b/Lib/test/test_opcodes.py index 72488b2bb6b4ff..f7cc8331b8d844 100644 --- a/Lib/test/test_opcodes.py +++ b/Lib/test/test_opcodes.py @@ -39,16 +39,19 @@ class C: pass def test_use_existing_annotations(self): ns = {'__annotations__': {1: 2}} exec('x: int', ns) - self.assertEqual(ns['__annotations__'], {'x': int, 1: 2}) + self.assertEqual(ns['__annotations__'], {1: 2}) def test_do_not_recreate_annotations(self): # Don't rely on the existence of the '__annotations__' global. with support.swap_item(globals(), '__annotations__', {}): - del globals()['__annotations__'] + globals().pop('__annotations__', None) class C: - del __annotations__ - with self.assertRaises(NameError): - x: int + try: + del __annotations__ + except NameError: + pass + x: int + self.assertEqual(C.__annotations__, {"x": int}) def test_raise_class_exceptions(self): diff --git a/Lib/test/test_positional_only_arg.py b/Lib/test/test_positional_only_arg.py index 1a193814d7535d..eea0625012da6d 100644 --- a/Lib/test/test_positional_only_arg.py +++ b/Lib/test/test_positional_only_arg.py @@ -2,6 +2,7 @@ import dis import pickle +import types import unittest from test.support import check_syntax_error @@ -440,7 +441,9 @@ def f(x: not (int is int), /): ... # without constant folding we end up with # COMPARE_OP(is), IS_OP (0) # with constant folding we should expect a IS_OP (1) - codes = [(i.opname, i.argval) for i in dis.get_instructions(g)] + code_obj = next(const for const in g.__code__.co_consts + if isinstance(const, types.CodeType) and const.co_name == "__annotate__") + codes = [(i.opname, i.argval) for i in dis.get_instructions(code_obj)] self.assertNotIn(('UNARY_NOT', None), codes) self.assertIn(('IS_OP', 1), codes) diff --git a/Lib/test/test_pyclbr.py b/Lib/test/test_pyclbr.py index 46206accbafc36..0c12a3085b12af 100644 --- a/Lib/test/test_pyclbr.py +++ b/Lib/test/test_pyclbr.py @@ -109,6 +109,8 @@ def ismethod(oclass, obj, name): actualMethods = [] for m in py_item.__dict__.keys(): + if m == "__annotate__": + continue if ismethod(py_item, getattr(py_item, m), m): actualMethods.append(m) foundMethods = [] diff --git a/Lib/test/test_pydoc/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py index 57e5b8e8abddfa..a17c16cc73cf0e 100644 --- a/Lib/test/test_pydoc/test_pydoc.py +++ b/Lib/test/test_pydoc/test_pydoc.py @@ -77,6 +77,11 @@ class A(builtins.object) | __weakref__%s class B(builtins.object) + | Methods defined here: + | + | __annotate__(...) + | + | ---------------------------------------------------------------------- | Data descriptors defined here: | | __dict__%s @@ -87,8 +92,6 @@ class B(builtins.object) | Data and other attributes defined here: | | NO_MEANING = 'eggs' - | - | __annotations__ = {'NO_MEANING': } class C(builtins.object) | Methods defined here: @@ -176,6 +179,9 @@ class A(builtins.object) list of weak references to the object class B(builtins.object) + Methods defined here: + __annotate__(...) + ---------------------------------------------------------------------- Data descriptors defined here: __dict__ dictionary for instance variables @@ -184,7 +190,6 @@ class B(builtins.object) ---------------------------------------------------------------------- Data and other attributes defined here: NO_MEANING = 'eggs' - __annotations__ = {'NO_MEANING': } class C(builtins.object) diff --git a/Lib/test/test_pyrepl/test_interact.py b/Lib/test/test_pyrepl/test_interact.py index df97b1354a168e..31f08cdb25e078 100644 --- a/Lib/test/test_pyrepl/test_interact.py +++ b/Lib/test/test_pyrepl/test_interact.py @@ -105,7 +105,7 @@ def test_runsource_shows_syntax_error_for_failed_compilation(self): def test_no_active_future(self): console = InteractiveColoredConsole() - source = "x: int = 1; print(__annotations__)" + source = "x: int = 1; print(__annotate__(1))" f = io.StringIO() with contextlib.redirect_stdout(f): result = console.runsource(source) diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py index ef2a228b15ed4e..a4b111e865c86e 100644 --- a/Lib/test/test_symtable.py +++ b/Lib/test/test_symtable.py @@ -205,12 +205,14 @@ def test_assigned(self): def test_annotated(self): st1 = symtable.symtable('def f():\n x: int\n', 'test', 'exec') - st2 = st1.get_children()[0] + st2 = st1.get_children()[1] + self.assertEqual(st2.get_type(), "function") self.assertTrue(st2.lookup('x').is_local()) self.assertTrue(st2.lookup('x').is_annotated()) self.assertFalse(st2.lookup('x').is_global()) st3 = symtable.symtable('def f():\n x = 1\n', 'test', 'exec') - st4 = st3.get_children()[0] + st4 = st3.get_children()[1] + self.assertEqual(st4.get_type(), "function") self.assertTrue(st4.lookup('x').is_local()) self.assertFalse(st4.lookup('x').is_annotated()) diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 5035de114b5e9d..1895c88d23b70d 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -622,6 +622,7 @@ def test_caret_in_type_annotation(self): def f_with_type(): def foo(a: THIS_DOES_NOT_EXIST ) -> int: return 0 + foo.__annotations__ lineno_f = f_with_type.__code__.co_firstlineno expected_f = ( @@ -629,7 +630,9 @@ def foo(a: THIS_DOES_NOT_EXIST ) -> int: f' File "{__file__}", line {self.callable_line}, in get_exception\n' ' callable()\n' ' ~~~~~~~~^^\n' - f' File "{__file__}", line {lineno_f+1}, in f_with_type\n' + f' File "{__file__}", line {lineno_f+3}, in f_with_type\n' + ' foo.__annotations__\n' + f' File "{__file__}", line {lineno_f+1}, in __annotate__\n' ' def foo(a: THIS_DOES_NOT_EXIST ) -> int:\n' ' ^^^^^^^^^^^^^^^^^^^\n' ) diff --git a/Lib/test/test_type_annotations.py b/Lib/test/test_type_annotations.py index 5e3c3347a41571..a9be1f5aa84681 100644 --- a/Lib/test/test_type_annotations.py +++ b/Lib/test/test_type_annotations.py @@ -1,7 +1,12 @@ import textwrap import types import unittest -from test.support import run_code +from test.support import run_code, check_syntax_error + +VALUE = 1 +FORWARDREF = 2 +SOURCE = 3 + class TypeAnnotationTests(unittest.TestCase): @@ -49,6 +54,7 @@ def test_annotations_are_created_correctly(self): class C: a:int=3 b:str=4 + self.assertEqual(C.__annotations__, {"a": int, "b": str}) self.assertTrue("__annotations__" in C.__dict__) del C.__annotations__ self.assertFalse("__annotations__" in C.__dict__) @@ -106,6 +112,13 @@ class D(metaclass=C): self.assertEqual(D.__annotations__, {}) +def build_module(code: str, name: str = "top") -> types.ModuleType: + ns = run_code(code) + mod = types.ModuleType(name) + mod.__dict__.update(ns) + return mod + + class TestSetupAnnotations(unittest.TestCase): def check(self, code: str): code = textwrap.dedent(code) @@ -113,11 +126,10 @@ def check(self, code: str): with self.subTest(scope=scope): if scope == "class": code = f"class C:\n{textwrap.indent(code, ' ')}" - ns = run_code(code) - if scope == "class": + ns = run_code(code) annotations = ns["C"].__annotations__ else: - annotations = ns["__annotations__"] + annotations = build_module(code).__annotations__ self.assertEqual(annotations, {"x": int}) def test_top_level(self): @@ -256,3 +268,146 @@ def check_annotations(self, f): # Setting f.__annotations__ also clears __annotate__ f.__annotations__ = {"z": 43} self.assertIs(f.__annotate__, None) + + +class DeferredEvaluationTests(unittest.TestCase): + def test_function(self): + def func(x: undefined, /, y: undefined, *args: undefined, z: undefined, **kwargs: undefined) -> undefined: + pass + + with self.assertRaises(NameError): + func.__annotations__ + + undefined = 1 + self.assertEqual(func.__annotations__, { + "x": 1, + "y": 1, + "args": 1, + "z": 1, + "kwargs": 1, + "return": 1, + }) + + def test_async_function(self): + async def func(x: undefined, /, y: undefined, *args: undefined, z: undefined, **kwargs: undefined) -> undefined: + pass + + with self.assertRaises(NameError): + func.__annotations__ + + undefined = 1 + self.assertEqual(func.__annotations__, { + "x": 1, + "y": 1, + "args": 1, + "z": 1, + "kwargs": 1, + "return": 1, + }) + + def test_class(self): + class X: + a: undefined + + with self.assertRaises(NameError): + X.__annotations__ + + undefined = 1 + self.assertEqual(X.__annotations__, {"a": 1}) + + def test_module(self): + ns = run_code("x: undefined = 1") + anno = ns["__annotate__"] + with self.assertRaises(NotImplementedError): + anno(2) + + with self.assertRaises(NameError): + anno(1) + + ns["undefined"] = 1 + self.assertEqual(anno(1), {"x": 1}) + + def test_class_scoping(self): + class Outer: + def meth(self, x: Nested): ... + x: Nested + class Nested: ... + + self.assertEqual(Outer.meth.__annotations__, {"x": Outer.Nested}) + self.assertEqual(Outer.__annotations__, {"x": Outer.Nested}) + + def test_no_exotic_expressions(self): + check_syntax_error(self, "def func(x: (yield)): ...", "yield expression cannot be used within an annotation") + check_syntax_error(self, "def func(x: (yield from x)): ...", "yield expression cannot be used within an annotation") + check_syntax_error(self, "def func(x: (y := 3)): ...", "named expression cannot be used within an annotation") + check_syntax_error(self, "def func(x: (await 42)): ...", "await expression cannot be used within an annotation") + + def test_no_exotic_expressions_in_unevaluated_annotations(self): + preludes = [ + "", + "class X: ", + "def f(): ", + "async def f(): ", + ] + for prelude in preludes: + with self.subTest(prelude=prelude): + check_syntax_error(self, prelude + "(x): (yield)", "yield expression cannot be used within an annotation") + check_syntax_error(self, prelude + "(x): (yield from x)", "yield expression cannot be used within an annotation") + check_syntax_error(self, prelude + "(x): (y := 3)", "named expression cannot be used within an annotation") + check_syntax_error(self, prelude + "(x): (await 42)", "await expression cannot be used within an annotation") + + def test_ignore_non_simple_annotations(self): + ns = run_code("class X: (y): int") + self.assertEqual(ns["X"].__annotations__, {}) + ns = run_code("class X: int.b: int") + self.assertEqual(ns["X"].__annotations__, {}) + ns = run_code("class X: int[str]: int") + self.assertEqual(ns["X"].__annotations__, {}) + + def test_generated_annotate(self): + def func(x: int): + pass + class X: + x: int + mod = build_module("x: int") + for obj in (func, X, mod): + with self.subTest(obj=obj): + annotate = obj.__annotate__ + self.assertIsInstance(annotate, types.FunctionType) + self.assertEqual(annotate.__name__, "__annotate__") + with self.assertRaises(NotImplementedError): + annotate(FORWARDREF) + with self.assertRaises(NotImplementedError): + annotate(SOURCE) + with self.assertRaises(NotImplementedError): + annotate(None) + self.assertEqual(annotate(VALUE), {"x": int}) + + def test_comprehension_in_annotation(self): + # This crashed in an earlier version of the code + ns = run_code("x: [y for y in range(10)]") + self.assertEqual(ns["__annotate__"](1), {"x": list(range(10))}) + + def test_future_annotations(self): + code = """ + from __future__ import annotations + + def f(x: int) -> int: pass + """ + ns = run_code(code) + f = ns["f"] + self.assertIsInstance(f.__annotate__, types.FunctionType) + annos = {"x": "int", "return": "int"} + self.assertEqual(f.__annotate__(VALUE), annos) + self.assertEqual(f.__annotations__, annos) + + def test_name_clash_with_format(self): + # this test would fail if __annotate__'s parameter was called "format" + code = """ + class format: pass + + def f(x: format): pass + """ + ns = run_code(code) + f = ns["f"] + self.assertEqual(f.__annotations__, {"x": ns["format"]}) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index dac55ceb9e99e0..9800b3b6a7da29 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -6634,7 +6634,7 @@ def test_get_type_hints_from_various_objects(self): gth(None) def test_get_type_hints_modules(self): - ann_module_type_hints = {1: 2, 'f': Tuple[int, int], 'x': int, 'y': str, 'u': int | float} + ann_module_type_hints = {'f': Tuple[int, int], 'x': int, 'y': str, 'u': int | float} self.assertEqual(gth(ann_module), ann_module_type_hints) self.assertEqual(gth(ann_module2), {}) self.assertEqual(gth(ann_module3), {}) @@ -6652,7 +6652,7 @@ def test_get_type_hints_classes(self): self.assertEqual(gth(ann_module.C), # gth will find the right globalns {'y': Optional[ann_module.C]}) self.assertIsInstance(gth(ann_module.j_class), dict) - self.assertEqual(gth(ann_module.M), {'123': 123, 'o': type}) + self.assertEqual(gth(ann_module.M), {'o': type}) self.assertEqual(gth(ann_module.D), {'j': str, 'k': str, 'y': Optional[ann_module.C]}) self.assertEqual(gth(ann_module.Y), {'z': int}) diff --git a/Lib/test/typinganndata/ann_module.py b/Lib/test/typinganndata/ann_module.py index 5081e6b58345a9..e1a1792cb4a867 100644 --- a/Lib/test/typinganndata/ann_module.py +++ b/Lib/test/typinganndata/ann_module.py @@ -8,8 +8,6 @@ from typing import Optional from functools import wraps -__annotations__[1] = 2 - class C: x = 5; y: Optional['C'] = None @@ -18,8 +16,6 @@ class C: x: int = 5; y: str = x; f: Tuple[int, int] class M(type): - - __annotations__['123'] = 123 o: type = object (pars): bool = True diff --git a/Lib/typing.py b/Lib/typing.py index be49aa63464f05..7a9149d3f3c2c1 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -2412,7 +2412,7 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False): base_globals = getattr(sys.modules.get(base.__module__, None), '__dict__', {}) else: base_globals = globalns - ann = base.__dict__.get('__annotations__', {}) + ann = getattr(base, '__annotations__', {}) if isinstance(ann, types.GetSetDescriptorType): ann = {} base_locals = dict(vars(base)) if localns is None else localns @@ -2970,7 +2970,12 @@ def __new__(cls, typename, bases, ns): raise TypeError( 'can only inherit from a NamedTuple type and Generic') bases = tuple(tuple if base is _NamedTuple else base for base in bases) - types = ns.get('__annotations__', {}) + if "__annotations__" in ns: + types = ns["__annotations__"] + elif "__annotate__" in ns: + types = ns["__annotate__"](1) # VALUE + else: + types = {} default_names = [] for field_name in types: if field_name in ns: @@ -3131,7 +3136,12 @@ def __new__(cls, name, bases, ns, total=True): tp_dict.__orig_bases__ = bases annotations = {} - own_annotations = ns.get('__annotations__', {}) + if "__annotations__" in ns: + own_annotations = ns["__annotations__"] + elif "__annotate__" in ns: + own_annotations = ns["__annotate__"](1) # VALUE + else: + own_annotations = {} msg = "TypedDict('Name', {f0: t0, f1: t1, ...}); each t must be a type" own_annotations = { n: _type_check(tp, msg, module=tp_dict.__module__) @@ -3143,7 +3153,12 @@ def __new__(cls, name, bases, ns, total=True): mutable_keys = set() for base in bases: - annotations.update(base.__dict__.get('__annotations__', {})) + # TODO: Avoid eagerly evaluating annotations in VALUE format. + # Instead, evaluate in FORWARDREF format to figure out which + # keys have Required/NotRequired/ReadOnly qualifiers, and create + # a new __annotate__ function for the resulting TypedDict that + # combines the annotations from this class and its parents. + annotations.update(base.__annotations__) base_required = base.__dict__.get('__required_keys__', set()) required_keys |= base_required diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-22-06-22-47.gh-issue-119180.vZMiXm.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-22-06-22-47.gh-issue-119180.vZMiXm.rst new file mode 100644 index 00000000000000..265ffb32e6a1f9 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-22-06-22-47.gh-issue-119180.vZMiXm.rst @@ -0,0 +1 @@ +Evaluation of annotations is now deferred. See :pep:`649` for details. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 413ad1105f9428..05c17ac334b69f 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3975,6 +3975,11 @@ dummy_func( assert(func_obj->func_defaults == NULL); func_obj->func_defaults = attr; break; + case MAKE_FUNCTION_ANNOTATE: + assert(PyCallable_Check(attr)); + assert(func_obj->func_annotate == NULL); + func_obj->func_annotate = attr; + break; default: Py_UNREACHABLE(); } diff --git a/Python/compile.c b/Python/compile.c index cb724154206b7e..c3372766d0bd50 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -132,7 +132,7 @@ enum { COMPILER_SCOPE_ASYNC_FUNCTION, COMPILER_SCOPE_LAMBDA, COMPILER_SCOPE_COMPREHENSION, - COMPILER_SCOPE_TYPEPARAMS, + COMPILER_SCOPE_ANNOTATIONS, }; @@ -142,6 +142,15 @@ typedef _PyInstructionSequence instr_sequence; #define INITIAL_INSTR_SEQUENCE_SIZE 100 #define INITIAL_INSTR_SEQUENCE_LABELS_MAP_SIZE 10 +static const int compare_masks[] = { + [Py_LT] = COMPARISON_LESS_THAN, + [Py_LE] = COMPARISON_LESS_THAN | COMPARISON_EQUALS, + [Py_EQ] = COMPARISON_EQUALS, + [Py_NE] = COMPARISON_NOT_EQUALS, + [Py_GT] = COMPARISON_GREATER_THAN, + [Py_GE] = COMPARISON_GREATER_THAN | COMPARISON_EQUALS, +}; + /* * Resize the array if index is out of range. * @@ -208,6 +217,7 @@ struct compiler_unit { PyObject *u_private; /* for private name mangling */ PyObject *u_static_attributes; /* for class: attributes accessed via self.X */ + PyObject *u_deferred_annotations; /* AnnAssign nodes deferred to the end of compilation */ instr_sequence *u_instr_sequence; /* codegen output */ @@ -330,6 +340,8 @@ static int compiler_pattern(struct compiler *, pattern_ty, pattern_context *); static int compiler_match(struct compiler *, stmt_ty); static int compiler_pattern_subpattern(struct compiler *, pattern_ty, pattern_context *); +static int compiler_make_closure(struct compiler *c, location loc, + PyCodeObject *co, Py_ssize_t flags); static PyCodeObject *optimize_and_assemble(struct compiler *, int addNone); @@ -545,6 +557,7 @@ compiler_unit_free(struct compiler_unit *u) Py_CLEAR(u->u_metadata.u_fasthidden); Py_CLEAR(u->u_private); Py_CLEAR(u->u_static_attributes); + Py_CLEAR(u->u_deferred_annotations); PyMem_Free(u); } @@ -582,8 +595,8 @@ compiler_set_qualname(struct compiler *c) capsule = PyList_GET_ITEM(c->c_stack, stack_size - 1); parent = (struct compiler_unit *)PyCapsule_GetPointer(capsule, CAPSULE_NAME); assert(parent); - if (parent->u_scope_type == COMPILER_SCOPE_TYPEPARAMS) { - /* The parent is a type parameter scope, so we need to + if (parent->u_scope_type == COMPILER_SCOPE_ANNOTATIONS) { + /* The parent is an annotation scope, so we need to look at the grandparent. */ if (stack_size == 2) { // If we're immediately within the module, we can skip @@ -1128,6 +1141,7 @@ compiler_enter_scope(struct compiler *c, identifier name, } u->u_private = NULL; + u->u_deferred_annotations = NULL; if (scope_type == COMPILER_SCOPE_CLASS) { u->u_static_attributes = PySet_New(0); if (!u->u_static_attributes) { @@ -1209,85 +1223,6 @@ compiler_exit_scope(struct compiler *c) PyErr_SetRaisedException(exc); } -/* Search if variable annotations are present statically in a block. */ - -static bool -find_ann(asdl_stmt_seq *stmts) -{ - int i, j, res = 0; - stmt_ty st; - - for (i = 0; i < asdl_seq_LEN(stmts); i++) { - st = (stmt_ty)asdl_seq_GET(stmts, i); - switch (st->kind) { - case AnnAssign_kind: - return true; - case For_kind: - res = find_ann(st->v.For.body) || - find_ann(st->v.For.orelse); - break; - case AsyncFor_kind: - res = find_ann(st->v.AsyncFor.body) || - find_ann(st->v.AsyncFor.orelse); - break; - case While_kind: - res = find_ann(st->v.While.body) || - find_ann(st->v.While.orelse); - break; - case If_kind: - res = find_ann(st->v.If.body) || - find_ann(st->v.If.orelse); - break; - case With_kind: - res = find_ann(st->v.With.body); - break; - case AsyncWith_kind: - res = find_ann(st->v.AsyncWith.body); - break; - case Try_kind: - for (j = 0; j < asdl_seq_LEN(st->v.Try.handlers); j++) { - excepthandler_ty handler = (excepthandler_ty)asdl_seq_GET( - st->v.Try.handlers, j); - if (find_ann(handler->v.ExceptHandler.body)) { - return true; - } - } - res = find_ann(st->v.Try.body) || - find_ann(st->v.Try.finalbody) || - find_ann(st->v.Try.orelse); - break; - case TryStar_kind: - for (j = 0; j < asdl_seq_LEN(st->v.TryStar.handlers); j++) { - excepthandler_ty handler = (excepthandler_ty)asdl_seq_GET( - st->v.TryStar.handlers, j); - if (find_ann(handler->v.ExceptHandler.body)) { - return true; - } - } - res = find_ann(st->v.TryStar.body) || - find_ann(st->v.TryStar.finalbody) || - find_ann(st->v.TryStar.orelse); - break; - case Match_kind: - for (j = 0; j < asdl_seq_LEN(st->v.Match.cases); j++) { - match_case_ty match_case = (match_case_ty)asdl_seq_GET( - st->v.Match.cases, j); - if (find_ann(match_case->body)) { - return true; - } - } - break; - default: - res = false; - break; - } - if (res) { - break; - } - } - return res; -} - /* * Frame block handling functions */ @@ -1502,6 +1437,47 @@ compiler_unwind_fblock_stack(struct compiler *c, location *ploc, return SUCCESS; } +static int +compiler_setup_annotations_scope(struct compiler *c, location loc, + void *key, PyObject *name) +{ + if (compiler_enter_scope(c, name, COMPILER_SCOPE_ANNOTATIONS, + key, loc.lineno) == -1) { + return ERROR; + } + c->u->u_metadata.u_posonlyargcount = 1; + // if .format != 1: raise NotImplementedError + _Py_DECLARE_STR(format, ".format"); + ADDOP_I(c, loc, LOAD_FAST, 0); + ADDOP_LOAD_CONST(c, loc, _PyLong_GetOne()); + ADDOP_I(c, loc, COMPARE_OP, (Py_NE << 5) | compare_masks[Py_NE]); + NEW_JUMP_TARGET_LABEL(c, body); + ADDOP_JUMP(c, loc, POP_JUMP_IF_FALSE, body); + ADDOP_I(c, loc, LOAD_COMMON_CONSTANT, CONSTANT_NOTIMPLEMENTEDERROR); + ADDOP_I(c, loc, RAISE_VARARGS, 1); + USE_LABEL(c, body); + return 0; +} + +static int +compiler_leave_annotations_scope(struct compiler *c, location loc, + Py_ssize_t annotations_len) +{ + ADDOP_I(c, loc, BUILD_MAP, annotations_len); + ADDOP_IN_SCOPE(c, loc, RETURN_VALUE); + PyCodeObject *co = optimize_and_assemble(c, 1); + compiler_exit_scope(c); + if (co == NULL) { + return ERROR; + } + if (compiler_make_closure(c, loc, co, 0) < 0) { + Py_DECREF(co); + return ERROR; + } + Py_DECREF(co); + return 0; +} + /* Compile a sequence of statements, checking for a docstring and for annotations. */ @@ -1517,34 +1493,79 @@ compiler_body(struct compiler *c, location loc, asdl_stmt_seq *stmts) stmt_ty st = (stmt_ty)asdl_seq_GET(stmts, 0); loc = LOC(st); } - /* Every annotated class and module should have __annotations__. */ - if (find_ann(stmts)) { + /* If from __future__ import annotations is active, + * every annotated class and module should have __annotations__. + * Else __annotate__ is created when necessary. */ + if ((c->c_future.ff_features & CO_FUTURE_ANNOTATIONS) && c->u->u_ste->ste_annotations_used) { ADDOP(c, loc, SETUP_ANNOTATIONS); } if (!asdl_seq_LEN(stmts)) { return SUCCESS; } Py_ssize_t first_instr = 0; - PyObject *docstring = _PyAST_GetDocString(stmts); - if (docstring) { - first_instr = 1; - /* if not -OO mode, set docstring */ - if (c->c_optimize < 2) { - PyObject *cleandoc = _PyCompile_CleanDoc(docstring); - if (cleandoc == NULL) { - return ERROR; + if (!c->c_interactive) { + PyObject *docstring = _PyAST_GetDocString(stmts); + if (docstring) { + first_instr = 1; + /* if not -OO mode, set docstring */ + if (c->c_optimize < 2) { + PyObject *cleandoc = _PyCompile_CleanDoc(docstring); + if (cleandoc == NULL) { + return ERROR; + } + stmt_ty st = (stmt_ty)asdl_seq_GET(stmts, 0); + assert(st->kind == Expr_kind); + location loc = LOC(st->v.Expr.value); + ADDOP_LOAD_CONST(c, loc, cleandoc); + Py_DECREF(cleandoc); + RETURN_IF_ERROR(compiler_nameop(c, NO_LOCATION, &_Py_ID(__doc__), Store)); } - stmt_ty st = (stmt_ty)asdl_seq_GET(stmts, 0); - assert(st->kind == Expr_kind); - location loc = LOC(st->v.Expr.value); - ADDOP_LOAD_CONST(c, loc, cleandoc); - Py_DECREF(cleandoc); - RETURN_IF_ERROR(compiler_nameop(c, NO_LOCATION, &_Py_ID(__doc__), Store)); } } for (Py_ssize_t i = first_instr; i < asdl_seq_LEN(stmts); i++) { VISIT(c, stmt, (stmt_ty)asdl_seq_GET(stmts, i)); } + // If there are annotations and the future import is not on, we + // collect the annotations in a separate pass and generate an + // __annotate__ function. See PEP 649. + if (!(c->c_future.ff_features & CO_FUTURE_ANNOTATIONS) && + c->u->u_deferred_annotations != NULL) { + + // It's possible that ste_annotations_block is set but + // u_deferred_annotations is not, because the former is still + // set if there are only non-simple annotations (i.e., annotations + // for attributes, subscripts, or parenthesized names). However, the + // reverse should not be possible. + assert(c->u->u_ste->ste_annotation_block != NULL); + PyObject *deferred_anno = Py_NewRef(c->u->u_deferred_annotations); + void *key = (void *)((uintptr_t)c->u->u_ste->ste_id + 1); + if (compiler_setup_annotations_scope(c, loc, key, + c->u->u_ste->ste_annotation_block->ste_name) == -1) { + Py_DECREF(deferred_anno); + return ERROR; + } + Py_ssize_t annotations_len = PyList_Size(deferred_anno); + for (Py_ssize_t i = 0; i < annotations_len; i++) { + PyObject *ptr = PyList_GET_ITEM(deferred_anno, i); + stmt_ty st = (stmt_ty)PyLong_AsVoidPtr(ptr); + if (st == NULL) { + compiler_exit_scope(c); + Py_DECREF(deferred_anno); + return ERROR; + } + PyObject *mangled = _Py_Mangle(c->u->u_private, st->v.AnnAssign.target->v.Name.id); + ADDOP_LOAD_CONST_NEW(c, LOC(st), mangled); + VISIT(c, expr, st->v.AnnAssign.annotation); + } + Py_DECREF(deferred_anno); + + RETURN_IF_ERROR( + compiler_leave_annotations_scope(c, loc, annotations_len) + ); + RETURN_IF_ERROR( + compiler_nameop(c, loc, &_Py_ID(__annotate__), Store) + ); + } return SUCCESS; } @@ -1559,11 +1580,10 @@ compiler_codegen(struct compiler *c, mod_ty mod) } break; case Interactive_kind: - if (find_ann(mod->v.Interactive.body)) { - ADDOP(c, loc, SETUP_ANNOTATIONS); - } c->c_interactive = 1; - VISIT_SEQ(c, stmt, mod->v.Interactive.body); + if (compiler_body(c, loc, mod->v.Interactive.body) < 0) { + return ERROR; + } break; case Expression_kind: VISIT(c, expr, mod->v.Expression.body); @@ -1702,6 +1722,9 @@ compiler_make_closure(struct compiler *c, location loc, if (flags & MAKE_FUNCTION_ANNOTATIONS) { ADDOP_I(c, loc, SET_FUNCTION_ATTRIBUTE, MAKE_FUNCTION_ANNOTATIONS); } + if (flags & MAKE_FUNCTION_ANNOTATE) { + ADDOP_I(c, loc, SET_FUNCTION_ATTRIBUTE, MAKE_FUNCTION_ANNOTATE); + } if (flags & MAKE_FUNCTION_KWDEFAULTS) { ADDOP_I(c, loc, SET_FUNCTION_ATTRIBUTE, MAKE_FUNCTION_KWDEFAULTS); } @@ -1833,7 +1856,7 @@ compiler_visit_argannotation(struct compiler *c, identifier id, VISIT(c, expr, annotation); } } - *annotations_len += 2; + *annotations_len += 1; return SUCCESS; } @@ -1856,43 +1879,76 @@ compiler_visit_argannotations(struct compiler *c, asdl_arg_seq* args, } static int -compiler_visit_annotations(struct compiler *c, location loc, - arguments_ty args, expr_ty returns) +compiler_visit_annotations_in_scope(struct compiler *c, location loc, + arguments_ty args, expr_ty returns, + Py_ssize_t *annotations_len) { - /* Push arg annotation names and values. - The expressions are evaluated out-of-order wrt the source code. - - Return -1 on error, 0 if no annotations pushed, 1 if a annotations is pushed. - */ - Py_ssize_t annotations_len = 0; - RETURN_IF_ERROR( - compiler_visit_argannotations(c, args->args, &annotations_len, loc)); + compiler_visit_argannotations(c, args->args, annotations_len, loc)); RETURN_IF_ERROR( - compiler_visit_argannotations(c, args->posonlyargs, &annotations_len, loc)); + compiler_visit_argannotations(c, args->posonlyargs, annotations_len, loc)); if (args->vararg && args->vararg->annotation) { RETURN_IF_ERROR( compiler_visit_argannotation(c, args->vararg->arg, - args->vararg->annotation, &annotations_len, loc)); + args->vararg->annotation, annotations_len, loc)); } RETURN_IF_ERROR( - compiler_visit_argannotations(c, args->kwonlyargs, &annotations_len, loc)); + compiler_visit_argannotations(c, args->kwonlyargs, annotations_len, loc)); if (args->kwarg && args->kwarg->annotation) { RETURN_IF_ERROR( compiler_visit_argannotation(c, args->kwarg->arg, - args->kwarg->annotation, &annotations_len, loc)); + args->kwarg->annotation, annotations_len, loc)); } RETURN_IF_ERROR( - compiler_visit_argannotation(c, &_Py_ID(return), returns, &annotations_len, loc)); + compiler_visit_argannotation(c, &_Py_ID(return), returns, annotations_len, loc)); - if (annotations_len) { - ADDOP_I(c, loc, BUILD_TUPLE, annotations_len); - return 1; + return 0; +} + +static int +compiler_visit_annotations(struct compiler *c, location loc, + arguments_ty args, expr_ty returns) +{ + /* Push arg annotation names and values. + The expressions are evaluated separately from the rest of the source code. + + Return -1 on error, or a combination of flags to add to the function. + */ + Py_ssize_t annotations_len = 0; + + PySTEntryObject *ste; + if (_PySymtable_LookupOptional(c->c_st, args, &ste) < 0) { + return ERROR; + } + assert(ste != NULL); + bool annotations_used = ste->ste_annotations_used; + + if (annotations_used) { + if (compiler_setup_annotations_scope(c, loc, (void *)args, + ste->ste_name) < 0) { + Py_DECREF(ste); + return ERROR; + } + } + Py_DECREF(ste); + + if (compiler_visit_annotations_in_scope(c, loc, args, returns, &annotations_len) < 0) { + if (annotations_used) { + compiler_exit_scope(c); + } + return ERROR; + } + + if (annotations_used) { + RETURN_IF_ERROR( + compiler_leave_annotations_scope(c, loc, annotations_len) + ); + return MAKE_FUNCTION_ANNOTATE; } return 0; @@ -2001,7 +2057,7 @@ compiler_type_param_bound_or_default(struct compiler *c, expr_ty e, identifier name, void *key, bool allow_starred) { - if (compiler_enter_scope(c, name, COMPILER_SCOPE_TYPEPARAMS, + if (compiler_enter_scope(c, name, COMPILER_SCOPE_ANNOTATIONS, key, e->lineno) == -1) { return ERROR; } @@ -2220,7 +2276,6 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) asdl_expr_seq *decos; asdl_type_param_seq *type_params; Py_ssize_t funcflags; - int annotations; int firstlineno; if (is_async) { @@ -2274,7 +2329,7 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) if (!type_params_name) { return ERROR; } - if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_TYPEPARAMS, + if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_ANNOTATIONS, (void *)type_params, firstlineno) == -1) { Py_DECREF(type_params_name); return ERROR; @@ -2286,16 +2341,14 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) } } - annotations = compiler_visit_annotations(c, loc, args, returns); - if (annotations < 0) { + int annotations_flag = compiler_visit_annotations(c, loc, args, returns); + if (annotations_flag < 0) { if (is_generic) { compiler_exit_scope(c); } return ERROR; } - if (annotations > 0) { - funcflags |= MAKE_FUNCTION_ANNOTATIONS; - } + funcflags |= annotations_flag; if (compiler_function_body(c, s, is_async, funcflags, firstlineno) < 0) { if (is_generic) { @@ -2510,7 +2563,7 @@ compiler_class(struct compiler *c, stmt_ty s) if (!type_params_name) { return ERROR; } - if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_TYPEPARAMS, + if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_ANNOTATIONS, (void *)type_params, firstlineno) == -1) { Py_DECREF(type_params_name); return ERROR; @@ -2630,7 +2683,7 @@ compiler_typealias(struct compiler *c, stmt_ty s) if (!type_params_name) { return ERROR; } - if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_TYPEPARAMS, + if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_ANNOTATIONS, (void *)type_params, loc.lineno) == -1) { Py_DECREF(type_params_name); return ERROR; @@ -2719,15 +2772,6 @@ check_compare(struct compiler *c, expr_ty e) return SUCCESS; } -static const int compare_masks[] = { - [Py_LT] = COMPARISON_LESS_THAN, - [Py_LE] = COMPARISON_LESS_THAN | COMPARISON_EQUALS, - [Py_EQ] = COMPARISON_EQUALS, - [Py_NE] = COMPARISON_NOT_EQUALS, - [Py_GT] = COMPARISON_GREATER_THAN, - [Py_GE] = COMPARISON_GREATER_THAN | COMPARISON_EQUALS, -}; - static int compiler_addcompare(struct compiler *c, location loc, cmpop_ty op) { @@ -6366,7 +6410,8 @@ compiler_annassign(struct compiler *c, stmt_ty s) { location loc = LOC(s); expr_ty targ = s->v.AnnAssign.target; - PyObject* mangled; + bool future_annotations = c->c_future.ff_features & CO_FUTURE_ANNOTATIONS; + PyObject *mangled; assert(s->kind == AnnAssign_kind); @@ -6384,16 +6429,30 @@ compiler_annassign(struct compiler *c, stmt_ty s) if (s->v.AnnAssign.simple && (c->u->u_scope_type == COMPILER_SCOPE_MODULE || c->u->u_scope_type == COMPILER_SCOPE_CLASS)) { - if (c->c_future.ff_features & CO_FUTURE_ANNOTATIONS) { - VISIT(c, annexpr, s->v.AnnAssign.annotation) + if (future_annotations) { + VISIT(c, annexpr, s->v.AnnAssign.annotation); + ADDOP_NAME(c, loc, LOAD_NAME, &_Py_ID(__annotations__), names); + mangled = _Py_MaybeMangle(c->u->u_private, c->u->u_ste, targ->v.Name.id); + ADDOP_LOAD_CONST_NEW(c, loc, mangled); + ADDOP(c, loc, STORE_SUBSCR); } else { - VISIT(c, expr, s->v.AnnAssign.annotation); + if (c->u->u_deferred_annotations == NULL) { + c->u->u_deferred_annotations = PyList_New(0); + if (c->u->u_deferred_annotations == NULL) { + return ERROR; + } + } + PyObject *ptr = PyLong_FromVoidPtr((void *)s); + if (ptr == NULL) { + return ERROR; + } + if (PyList_Append(c->u->u_deferred_annotations, ptr) < 0) { + Py_DECREF(ptr); + return ERROR; + } + Py_DECREF(ptr); } - ADDOP_NAME(c, loc, LOAD_NAME, &_Py_ID(__annotations__), names); - mangled = _Py_MaybeMangle(c->u->u_private, c->u->u_ste, targ->v.Name.id); - ADDOP_LOAD_CONST_NEW(c, loc, mangled); - ADDOP(c, loc, STORE_SUBSCR); } break; case Attribute_kind: @@ -6419,7 +6478,7 @@ compiler_annassign(struct compiler *c, stmt_ty s) return ERROR; } /* Annotation is evaluated last. */ - if (!s->v.AnnAssign.simple && check_annotation(c, s) < 0) { + if (future_annotations && !s->v.AnnAssign.simple && check_annotation(c, s) < 0) { return ERROR; } return SUCCESS; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index bab629684c53f6..470c82d938ab7c 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4061,6 +4061,11 @@ assert(func_obj->func_defaults == NULL); func_obj->func_defaults = attr; break; + case MAKE_FUNCTION_ANNOTATE: + assert(PyCallable_Check(attr)); + assert(func_obj->func_annotate == NULL); + func_obj->func_annotate = attr; + break; default: Py_UNREACHABLE(); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 355be966cbb84a..0274f8b7a48c3c 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5450,6 +5450,11 @@ assert(func_obj->func_defaults == NULL); func_obj->func_defaults = attr; break; + case MAKE_FUNCTION_ANNOTATE: + assert(PyCallable_Check(attr)); + assert(func_obj->func_annotate == NULL); + func_obj->func_annotate = attr; + break; default: Py_UNREACHABLE(); } diff --git a/Python/symtable.c b/Python/symtable.c index 7e452cdb13badf..287bc2bd58107d 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -112,6 +112,7 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block, ste->ste_varkeywords = 0; ste->ste_opt_lineno = 0; ste->ste_opt_col_offset = 0; + ste->ste_annotations_used = 0; ste->ste_lineno = lineno; ste->ste_col_offset = col_offset; ste->ste_end_lineno = end_lineno; @@ -132,6 +133,7 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block, ste->ste_can_see_class_scope = 0; ste->ste_comp_iter_expr = 0; ste->ste_needs_classdict = 0; + ste->ste_annotation_block = NULL; ste->ste_symbols = PyDict_New(); ste->ste_varnames = PyList_New(0); @@ -167,6 +169,7 @@ ste_dealloc(PySTEntryObject *ste) Py_XDECREF(ste->ste_varnames); Py_XDECREF(ste->ste_children); Py_XDECREF(ste->ste_directives); + Py_XDECREF(ste->ste_annotation_block); Py_XDECREF(ste->ste_mangled_names); PyObject_Free(ste); } @@ -245,10 +248,11 @@ static int symtable_visit_alias(struct symtable *st, alias_ty); static int symtable_visit_comprehension(struct symtable *st, comprehension_ty); static int symtable_visit_keyword(struct symtable *st, keyword_ty); static int symtable_visit_params(struct symtable *st, asdl_arg_seq *args); -static int symtable_visit_annotation(struct symtable *st, expr_ty annotation); +static int symtable_visit_annotation(struct symtable *st, expr_ty annotation, void *key); static int symtable_visit_argannotations(struct symtable *st, asdl_arg_seq *args); static int symtable_implicit_arg(struct symtable *st, int pos); -static int symtable_visit_annotations(struct symtable *st, stmt_ty, arguments_ty, expr_ty); +static int symtable_visit_annotations(struct symtable *st, stmt_ty, arguments_ty, expr_ty, + struct _symtable_entry *parent_ste); static int symtable_visit_withitem(struct symtable *st, withitem_ty item); static int symtable_visit_match_case(struct symtable *st, match_case_ty m); static int symtable_visit_pattern(struct symtable *st, pattern_ty s); @@ -504,6 +508,21 @@ _PySymtable_Lookup(struct symtable *st, void *key) return (PySTEntryObject *)v; } +int +_PySymtable_LookupOptional(struct symtable *st, void *key, + PySTEntryObject **out) +{ + PyObject *k = PyLong_FromVoidPtr(key); + if (k == NULL) { + *out = NULL; + return -1; + } + int result = PyDict_GetItemRef(st->st_blocks, k, (PyObject **)out); + Py_DECREF(k); + assert(*out == NULL || PySTEntry_Check(*out)); + return result; +} + long _PyST_GetSymbol(PySTEntryObject *ste, PyObject *name) { @@ -525,6 +544,7 @@ int _PyST_IsFunctionLike(PySTEntryObject *ste) { return ste->ste_type == FunctionBlock + || ste->ste_type == AnnotationBlock || ste->ste_type == TypeVarBoundBlock || ste->ste_type == TypeAliasBlock || ste->ste_type == TypeParamBlock; @@ -1317,20 +1337,12 @@ symtable_exit_block(struct symtable *st) } static int -symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block, - void *ast, int lineno, int col_offset, - int end_lineno, int end_col_offset) +symtable_enter_existing_block(struct symtable *st, PySTEntryObject* ste) { - PySTEntryObject *prev = NULL, *ste; - - ste = ste_new(st, name, block, ast, lineno, col_offset, end_lineno, end_col_offset); - if (ste == NULL) - return 0; if (PyList_Append(st->st_stack, (PyObject *)ste) < 0) { - Py_DECREF(ste); return 0; } - prev = st->st_cur; + PySTEntryObject *prev = st->st_cur; /* bpo-37757: For now, disallow *all* assignment expressions in the * outermost iterator expression of a comprehension, even those inside * a nested comprehension or a lambda expression. @@ -1340,21 +1352,20 @@ symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block, } /* No need to inherit ste_mangled_names in classes, where all names * are mangled. */ - if (prev && prev->ste_mangled_names != NULL && block != ClassBlock) { + if (prev && prev->ste_mangled_names != NULL && ste->ste_type != ClassBlock) { ste->ste_mangled_names = Py_NewRef(prev->ste_mangled_names); } /* The entry is owned by the stack. Borrow it for st_cur. */ - Py_DECREF(ste); st->st_cur = ste; - /* Annotation blocks shouldn't have any affect on the symbol table since in - * the compilation stage, they will all be transformed to strings. They are - * only created if future 'annotations' feature is activated. */ - if (block == AnnotationBlock) { + /* If "from __future__ import annotations" is active, + * annotation blocks shouldn't have any affect on the symbol table since in + * the compilation stage, they will all be transformed to strings. */ + if (st->st_future->ff_features & CO_FUTURE_ANNOTATIONS && ste->ste_type == AnnotationBlock) { return 1; } - if (block == ModuleBlock) + if (ste->ste_type == ModuleBlock) st->st_global = st->st_cur->ste_symbols; if (prev) { @@ -1365,6 +1376,20 @@ symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block, return 1; } +static int +symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block, + void *ast, int lineno, int col_offset, + int end_lineno, int end_col_offset) +{ + PySTEntryObject *ste = ste_new(st, name, block, ast, + lineno, col_offset, end_lineno, end_col_offset); + if (ste == NULL) + return 0; + int result = symtable_enter_existing_block(st, ste); + Py_DECREF(ste); + return result; +} + static long symtable_lookup_entry(struct symtable *st, PySTEntryObject *ste, PyObject *name) { @@ -1643,7 +1668,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT_QUIT(st, 0); } switch (s->kind) { - case FunctionDef_kind: + case FunctionDef_kind: { if (!symtable_add_def(st, s->v.FunctionDef.name, DEF_LOCAL, LOCATION(s))) VISIT_QUIT(st, 0); if (s->v.FunctionDef.args->defaults) @@ -1665,13 +1690,22 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) } VISIT_SEQ(st, type_param, s->v.FunctionDef.type_params); } + PySTEntryObject *new_ste = ste_new(st, s->v.FunctionDef.name, FunctionBlock, (void *)s, + LOCATION(s)); + if (!new_ste) { + VISIT_QUIT(st, 0); + } + if (!symtable_visit_annotations(st, s, s->v.FunctionDef.args, - s->v.FunctionDef.returns)) + s->v.FunctionDef.returns, new_ste)) { + Py_DECREF(new_ste); VISIT_QUIT(st, 0); - if (!symtable_enter_block(st, s->v.FunctionDef.name, - FunctionBlock, (void *)s, - LOCATION(s))) + } + if (!symtable_enter_existing_block(st, new_ste)) { + Py_DECREF(new_ste); VISIT_QUIT(st, 0); + } + Py_DECREF(new_ste); VISIT(st, arguments, s->v.FunctionDef.args); VISIT_SEQ(st, stmt, s->v.FunctionDef.body); if (!symtable_exit_block(st)) @@ -1681,6 +1715,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT_QUIT(st, 0); } break; + } case ClassDef_kind: { PyObject *tmp; if (!symtable_add_def(st, s->v.ClassDef.name, DEF_LOCAL, LOCATION(s))) @@ -1776,6 +1811,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT(st, expr, s->v.Assign.value); break; case AnnAssign_kind: + st->st_cur->ste_annotations_used = 1; if (s->v.AnnAssign.target->kind == Name_kind) { expr_ty e_name = s->v.AnnAssign.target; long cur = symtable_lookup(st, e_name->v.Name.id); @@ -1810,7 +1846,8 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) else { VISIT(st, expr, s->v.AnnAssign.target); } - if (!symtable_visit_annotation(st, s->v.AnnAssign.annotation)) { + if (!symtable_visit_annotation(st, s->v.AnnAssign.annotation, + (void *)((uintptr_t)st->st_cur->ste_id + 1))) { VISIT_QUIT(st, 0); } @@ -1960,7 +1997,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT_SEQ(st, withitem, s->v.With.items); VISIT_SEQ(st, stmt, s->v.With.body); break; - case AsyncFunctionDef_kind: + case AsyncFunctionDef_kind: { if (!symtable_add_def(st, s->v.AsyncFunctionDef.name, DEF_LOCAL, LOCATION(s))) VISIT_QUIT(st, 0); if (s->v.AsyncFunctionDef.args->defaults) @@ -1983,14 +2020,21 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) } VISIT_SEQ(st, type_param, s->v.AsyncFunctionDef.type_params); } + PySTEntryObject *new_ste = ste_new(st, s->v.FunctionDef.name, FunctionBlock, (void *)s, + LOCATION(s)); + if (!new_ste) { + VISIT_QUIT(st, 0); + } + if (!symtable_visit_annotations(st, s, s->v.AsyncFunctionDef.args, - s->v.AsyncFunctionDef.returns)) + s->v.AsyncFunctionDef.returns, new_ste)) VISIT_QUIT(st, 0); - if (!symtable_enter_block(st, s->v.AsyncFunctionDef.name, - FunctionBlock, (void *)s, - s->lineno, s->col_offset, - s->end_lineno, s->end_col_offset)) + if (!symtable_enter_existing_block(st, new_ste)) { + Py_DECREF(new_ste); VISIT_QUIT(st, 0); + } + Py_DECREF(new_ste); + st->st_cur->ste_coroutine = 1; VISIT(st, arguments, s->v.AsyncFunctionDef.args); VISIT_SEQ(st, stmt, s->v.AsyncFunctionDef.body); @@ -2001,6 +2045,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT_QUIT(st, 0); } break; + } case AsyncWith_kind: VISIT_SEQ(st, withitem, s->v.AsyncWith.items); VISIT_SEQ(st, stmt, s->v.AsyncWith.body); @@ -2444,18 +2489,44 @@ symtable_visit_params(struct symtable *st, asdl_arg_seq *args) } static int -symtable_visit_annotation(struct symtable *st, expr_ty annotation) +symtable_visit_annotation(struct symtable *st, expr_ty annotation, void *key) { - int future_annotations = st->st_future->ff_features & CO_FUTURE_ANNOTATIONS; - if (future_annotations && - !symtable_enter_block(st, &_Py_ID(_annotation), AnnotationBlock, - (void *)annotation, annotation->lineno, - annotation->col_offset, annotation->end_lineno, - annotation->end_col_offset)) { - VISIT_QUIT(st, 0); + struct _symtable_entry *parent_ste = st->st_cur; + if (parent_ste->ste_annotation_block == NULL) { + _Py_block_ty current_type = parent_ste->ste_type; + if (!symtable_enter_block(st, &_Py_ID(__annotate__), AnnotationBlock, + key, LOCATION(annotation))) { + VISIT_QUIT(st, 0); + } + parent_ste->ste_annotation_block = + (struct _symtable_entry *)Py_NewRef(st->st_cur); + int future_annotations = st->st_future->ff_features & CO_FUTURE_ANNOTATIONS; + if (current_type == ClassBlock && !future_annotations) { + st->st_cur->ste_can_see_class_scope = 1; + if (!symtable_add_def(st, &_Py_ID(__classdict__), USE, LOCATION(annotation))) { + return 0; + } + } + + _Py_DECLARE_STR(format, ".format"); + // The generated __annotate__ function takes a single parameter with the + // internal name ".format". + if (!symtable_add_def(st, &_Py_STR(format), DEF_PARAM, + LOCATION(annotation))) { + return 0; + } + if (!symtable_add_def(st, &_Py_STR(format), USE, + LOCATION(annotation))) { + return 0; + } + } + else { + if (!symtable_enter_existing_block(st, parent_ste->ste_annotation_block)) { + VISIT_QUIT(st, 0); + } } VISIT(st, expr, annotation); - if (future_annotations && !symtable_exit_block(st)) { + if (!symtable_exit_block(st)) { VISIT_QUIT(st, 0); } return 1; @@ -2471,37 +2542,58 @@ symtable_visit_argannotations(struct symtable *st, asdl_arg_seq *args) for (i = 0; i < asdl_seq_LEN(args); i++) { arg_ty arg = (arg_ty)asdl_seq_GET(args, i); - if (arg->annotation) + if (arg->annotation) { + st->st_cur->ste_annotations_used = 1; VISIT(st, expr, arg->annotation); + } } return 1; } static int -symtable_visit_annotations(struct symtable *st, stmt_ty o, arguments_ty a, expr_ty returns) +symtable_visit_annotations(struct symtable *st, stmt_ty o, arguments_ty a, expr_ty returns, + struct _symtable_entry *function_ste) { - int future_annotations = st->st_future->ff_features & CO_FUTURE_ANNOTATIONS; - if (future_annotations && - !symtable_enter_block(st, &_Py_ID(_annotation), AnnotationBlock, - (void *)o, o->lineno, o->col_offset, o->end_lineno, - o->end_col_offset)) { + int is_in_class = st->st_cur->ste_can_see_class_scope; + _Py_block_ty current_type = st->st_cur->ste_type; + if (!symtable_enter_block(st, &_Py_ID(__annotate__), AnnotationBlock, + (void *)a, LOCATION(o))) { VISIT_QUIT(st, 0); } + if (is_in_class || current_type == ClassBlock) { + st->st_cur->ste_can_see_class_scope = 1; + if (!symtable_add_def(st, &_Py_ID(__classdict__), USE, LOCATION(o))) { + return 0; + } + } + _Py_DECLARE_STR(format, ".format"); + // We need to insert code that reads this "parameter" to the function. + if (!symtable_add_def(st, &_Py_STR(format), DEF_PARAM, LOCATION(o))) { + return 0; + } + if (!symtable_add_def(st, &_Py_STR(format), USE, LOCATION(o))) { + return 0; + } if (a->posonlyargs && !symtable_visit_argannotations(st, a->posonlyargs)) return 0; if (a->args && !symtable_visit_argannotations(st, a->args)) return 0; - if (a->vararg && a->vararg->annotation) + if (a->vararg && a->vararg->annotation) { + st->st_cur->ste_annotations_used = 1; VISIT(st, expr, a->vararg->annotation); - if (a->kwarg && a->kwarg->annotation) + } + if (a->kwarg && a->kwarg->annotation) { + st->st_cur->ste_annotations_used = 1; VISIT(st, expr, a->kwarg->annotation); + } if (a->kwonlyargs && !symtable_visit_argannotations(st, a->kwonlyargs)) return 0; - if (future_annotations && !symtable_exit_block(st)) { - VISIT_QUIT(st, 0); + if (returns) { + st->st_cur->ste_annotations_used = 1; + VISIT(st, expr, returns); } - if (returns && !symtable_visit_annotation(st, returns)) { + if (!symtable_exit_block(st)) { VISIT_QUIT(st, 0); } return 1; @@ -2733,7 +2825,7 @@ symtable_visit_dictcomp(struct symtable *st, expr_ty e) static int symtable_raise_if_annotation_block(struct symtable *st, const char *name, expr_ty e) { - enum _block_type type = st->st_cur->ste_type; + _Py_block_ty type = st->st_cur->ste_type; if (type == AnnotationBlock) PyErr_Format(PyExc_SyntaxError, ANNOTATION_NOT_ALLOWED, name); else if (type == TypeVarBoundBlock) pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy