From fba5efa2a04871b845403443f85b18a20d303e3a Mon Sep 17 00:00:00 2001 From: David C Ellis Date: Tue, 22 Apr 2025 16:53:28 +0100 Subject: [PATCH 01/12] Add a test for union forward references --- Lib/test/test_annotationlib.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index 0890be529a7e52..e5dd8dc20e26b0 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -936,6 +936,28 @@ def __call__(self): annotationlib.get_annotations(obj, format=format), {} ) + def test_union_forwardref(self): + # Test unions with '|' syntax equal unions with typing.Union[] with forwardrefs + class UnionForwardrefs: + pipe: str | undefined + union: Union[str, undefined] + + annos = get_annotations(UnionForwardrefs, format=Format.FORWARDREF) + + match = ( + str, + support.EqualToForwardRef("undefined", is_class=True, owner=UnionForwardrefs) + ) + + self.assertEqual( + typing.get_args(annos["pipe"]), + typing.get_args(annos["union"]) + ) + + self.assertEqual(typing.get_args(annos["pipe"]), match) + self.assertEqual(typing.get_args(annos["union"]), match) + + def test_pep695_generic_class_with_future_annotations(self): ann_module695 = inspect_stringized_annotations_pep695 A_annotations = annotationlib.get_annotations(ann_module695.A, eval_str=True) From d8109538837278622b4f3809d5ae51af942ad9ac Mon Sep 17 00:00:00 2001 From: David C Ellis Date: Tue, 22 Apr 2025 16:54:52 +0100 Subject: [PATCH 02/12] Make stringifiers create unions if create_unions is True --- Lib/annotationlib.py | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index 971f636f9714d7..c095efd6f3626a 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -392,12 +392,19 @@ def binop(self, other): __mod__ = _make_binop(ast.Mod()) __lshift__ = _make_binop(ast.LShift()) __rshift__ = _make_binop(ast.RShift()) - __or__ = _make_binop(ast.BitOr()) __xor__ = _make_binop(ast.BitXor()) __and__ = _make_binop(ast.BitAnd()) __floordiv__ = _make_binop(ast.FloorDiv()) __pow__ = _make_binop(ast.Pow()) + def __or__(self, other): + if self.__stringifier_dict__.create_unions: + return types.UnionType[self, other] + + return self.__make_new( + ast.BinOp(self.__get_ast(), ast.BitOr(), self.__convert_to_ast(other)) + ) + del _make_binop def _make_rbinop(op: ast.AST): @@ -416,12 +423,19 @@ def rbinop(self, other): __rmod__ = _make_rbinop(ast.Mod()) __rlshift__ = _make_rbinop(ast.LShift()) __rrshift__ = _make_rbinop(ast.RShift()) - __ror__ = _make_rbinop(ast.BitOr()) __rxor__ = _make_rbinop(ast.BitXor()) __rand__ = _make_rbinop(ast.BitAnd()) __rfloordiv__ = _make_rbinop(ast.FloorDiv()) __rpow__ = _make_rbinop(ast.Pow()) + def __ror__(self, other): + if self.__stringifier_dict__.create_unions: + return types.UnionType[other, self] + + return self.__make_new( + ast.BinOp(self.__convert_to_ast(other), ast.BitOr(), self.__get_ast()) + ) + del _make_rbinop def _make_compare(op): @@ -459,12 +473,13 @@ def unary_op(self): class _StringifierDict(dict): - def __init__(self, namespace, globals=None, owner=None, is_class=False): + def __init__(self, namespace, globals=None, owner=None, is_class=False, create_unions=False): super().__init__(namespace) self.namespace = namespace self.globals = globals self.owner = owner self.is_class = is_class + self.create_unions = create_unions self.stringifiers = [] def __missing__(self, key): @@ -569,7 +584,13 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): # that returns a bool and an defined set of attributes. namespace = {**annotate.__builtins__, **annotate.__globals__} is_class = isinstance(owner, type) - globals = _StringifierDict(namespace, annotate.__globals__, owner, is_class) + globals = _StringifierDict( + namespace, + annotate.__globals__, + owner, + is_class, + create_unions=True + ) if annotate.__closure__: freevars = annotate.__code__.co_freevars new_closure = [] From 1908a4a6ca0a70e45128237a07f0e307ff283eef Mon Sep 17 00:00:00 2001 From: David C Ellis Date: Tue, 22 Apr 2025 16:59:03 +0100 Subject: [PATCH 03/12] modify broken test, move test to forwardref format group --- Lib/test/test_annotationlib.py | 50 ++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index e5dd8dc20e26b0..fba1f11197c87d 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -115,8 +115,11 @@ def f( self.assertEqual(z_anno, support.EqualToForwardRef("some(module)", owner=f)) alpha_anno = anno["alpha"] - self.assertIsInstance(alpha_anno, ForwardRef) - self.assertEqual(alpha_anno, support.EqualToForwardRef("some | obj", owner=f)) + self.assertIsInstance(alpha_anno, Union) + self.assertEqual( + typing.get_args(alpha_anno), + (support.EqualToForwardRef("some", owner=f), support.EqualToForwardRef("obj", owner=f)) + ) beta_anno = anno["beta"] self.assertIsInstance(beta_anno, ForwardRef) @@ -126,6 +129,27 @@ def f( self.assertIsInstance(gamma_anno, ForwardRef) self.assertEqual(gamma_anno, support.EqualToForwardRef("some < obj", owner=f)) + def test_partially_nonexistent_union(self): + # Test unions with '|' syntax equal unions with typing.Union[] with some forwardrefs + class UnionForwardrefs: + pipe: str | undefined + union: Union[str, undefined] + + annos = get_annotations(UnionForwardrefs, format=Format.FORWARDREF) + + match = ( + str, + support.EqualToForwardRef("undefined", is_class=True, owner=UnionForwardrefs) + ) + + self.assertEqual( + typing.get_args(annos["pipe"]), + typing.get_args(annos["union"]) + ) + + self.assertEqual(typing.get_args(annos["pipe"]), match) + self.assertEqual(typing.get_args(annos["union"]), match) + class TestSourceFormat(unittest.TestCase): def test_closure(self): @@ -936,28 +960,6 @@ def __call__(self): annotationlib.get_annotations(obj, format=format), {} ) - def test_union_forwardref(self): - # Test unions with '|' syntax equal unions with typing.Union[] with forwardrefs - class UnionForwardrefs: - pipe: str | undefined - union: Union[str, undefined] - - annos = get_annotations(UnionForwardrefs, format=Format.FORWARDREF) - - match = ( - str, - support.EqualToForwardRef("undefined", is_class=True, owner=UnionForwardrefs) - ) - - self.assertEqual( - typing.get_args(annos["pipe"]), - typing.get_args(annos["union"]) - ) - - self.assertEqual(typing.get_args(annos["pipe"]), match) - self.assertEqual(typing.get_args(annos["union"]), match) - - def test_pep695_generic_class_with_future_annotations(self): ann_module695 = inspect_stringized_annotations_pep695 A_annotations = annotationlib.get_annotations(ann_module695.A, eval_str=True) From 97abdc7f44f17d84d248e4f762d5499e5091cbb7 Mon Sep 17 00:00:00 2001 From: David C Ellis Date: Tue, 22 Apr 2025 18:21:20 +0100 Subject: [PATCH 04/12] Apparently trim trailing whitespace was turned off --- Lib/annotationlib.py | 14 +++++++------- Lib/test/test_annotationlib.py | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index c095efd6f3626a..b4cecb41d13c46 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -400,7 +400,7 @@ def binop(self, other): def __or__(self, other): if self.__stringifier_dict__.create_unions: return types.UnionType[self, other] - + return self.__make_new( ast.BinOp(self.__get_ast(), ast.BitOr(), self.__convert_to_ast(other)) ) @@ -431,11 +431,11 @@ def rbinop(self, other): def __ror__(self, other): if self.__stringifier_dict__.create_unions: return types.UnionType[other, self] - + return self.__make_new( ast.BinOp(self.__convert_to_ast(other), ast.BitOr(), self.__get_ast()) ) - + del _make_rbinop def _make_compare(op): @@ -585,10 +585,10 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): namespace = {**annotate.__builtins__, **annotate.__globals__} is_class = isinstance(owner, type) globals = _StringifierDict( - namespace, - annotate.__globals__, - owner, - is_class, + namespace, + annotate.__globals__, + owner, + is_class, create_unions=True ) if annotate.__closure__: diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index fba1f11197c87d..6f17c85659c34e 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -117,7 +117,7 @@ def f( alpha_anno = anno["alpha"] self.assertIsInstance(alpha_anno, Union) self.assertEqual( - typing.get_args(alpha_anno), + typing.get_args(alpha_anno), (support.EqualToForwardRef("some", owner=f), support.EqualToForwardRef("obj", owner=f)) ) From dd36a625070c267715265dea5488a4d72df54846 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 22 Apr 2025 13:37:00 -0700 Subject: [PATCH 05/12] Alternative approach --- Lib/annotationlib.py | 147 ++++++++++++++++++++------------- Lib/ast.py | 1 + Lib/test/test_annotationlib.py | 84 +++++++++++++++---- 3 files changed, 161 insertions(+), 71 deletions(-) diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index b4cecb41d13c46..310c6a1801c970 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -38,6 +38,7 @@ class Format(enum.IntEnum): "__weakref__", "__arg__", "__globals__", + "__extra_names__", "__code__", "__ast_node__", "__cell__", @@ -82,6 +83,7 @@ def __init__( # is created through __class__ assignment on a _Stringifier object. self.__globals__ = None self.__cell__ = None + self.__extra_names__ = None # These are initially None but serve as a cache and may be set to a non-None # value later. self.__code__ = None @@ -151,6 +153,8 @@ def evaluate(self, *, globals=None, locals=None, type_params=None, owner=None): if not self.__forward_is_class__ or param_name not in globals: globals[param_name] = param locals.pop(param_name, None) + if self.__extra_names__: + locals = {**locals, **self.__extra_names__} arg = self.__forward_arg__ if arg.isidentifier() and not keyword.iskeyword(arg): @@ -274,6 +278,7 @@ def __init__( cell=None, *, stringifier_dict, + extra_names=None, ): # Either an AST node or a simple str (for the common case where a ForwardRef # represent a single name). @@ -285,6 +290,7 @@ def __init__( self.__code__ = None self.__ast_node__ = node self.__globals__ = globals + self.__extra_names__ = extra_names self.__cell__ = cell self.__owner__ = owner self.__stringifier_dict__ = stringifier_dict @@ -292,28 +298,33 @@ def __init__( def __convert_to_ast(self, other): if isinstance(other, _Stringifier): if isinstance(other.__ast_node__, str): - return ast.Name(id=other.__ast_node__) - return other.__ast_node__ - elif isinstance(other, slice): + return ast.Name(id=other.__ast_node__), other.__extra_names__ + return other.__ast_node__, other.__extra_names__ + elif other is None or type(other) in (str, int, float, bool, complex): + return ast.Constant(value=other), None + else: + name = self.__stringifier_dict__.create_unique_name() + return ast.Name(id=name), {name: other} + + def __convert_to_ast_getitem(self, other): + if isinstance(other, slice): + extra_names = {} + + def conv(obj): + if obj is None: + return None + new_obj, new_extra_names = self.__convert_to_ast(obj) + if new_extra_names is not None: + extra_names.update(new_extra_names) + return new_obj + return ast.Slice( - lower=( - self.__convert_to_ast(other.start) - if other.start is not None - else None - ), - upper=( - self.__convert_to_ast(other.stop) - if other.stop is not None - else None - ), - step=( - self.__convert_to_ast(other.step) - if other.step is not None - else None - ), - ) + lower=conv(other.start), + upper=conv(other.stop), + step=conv(other.step), + ), extra_names else: - return ast.Constant(value=other) + return self.__convert_to_ast(other) def __get_ast(self): node = self.__ast_node__ @@ -321,13 +332,19 @@ def __get_ast(self): return ast.Name(id=node) return node - def __make_new(self, node): + def __make_new(self, node, extra_names=None): + new_extra_names = {} + if self.__extra_names__ is not None: + new_extra_names.update(self.__extra_names__) + if extra_names is not None: + new_extra_names.update(extra_names) stringifier = _Stringifier( node, self.__globals__, self.__owner__, self.__forward_is_class__, stringifier_dict=self.__stringifier_dict__, + extra_names=new_extra_names, ) self.__stringifier_dict__.stringifiers.append(stringifier) return stringifier @@ -343,27 +360,37 @@ def __getitem__(self, other): if self.__ast_node__ == "__classdict__": raise KeyError if isinstance(other, tuple): - elts = [self.__convert_to_ast(elt) for elt in other] + extra_names = {} + elts = [] + for elt in other: + new_elt, new_extra_names = self.__convert_to_ast_getitem(elt) + if new_extra_names is not None: + extra_names.update(new_extra_names) + elts.append(new_elt) other = ast.Tuple(elts) else: - other = self.__convert_to_ast(other) + other, extra_names = self.__convert_to_ast_getitem(other) assert isinstance(other, ast.AST), repr(other) - return self.__make_new(ast.Subscript(self.__get_ast(), other)) + return self.__make_new(ast.Subscript(self.__get_ast(), other), extra_names) def __getattr__(self, attr): return self.__make_new(ast.Attribute(self.__get_ast(), attr)) def __call__(self, *args, **kwargs): - return self.__make_new( - ast.Call( - self.__get_ast(), - [self.__convert_to_ast(arg) for arg in args], - [ - ast.keyword(key, self.__convert_to_ast(value)) - for key, value in kwargs.items() - ], - ) - ) + extra_names = {} + ast_args = [] + for arg in args: + new_arg, new_extra_names = self.__convert_to_ast(arg) + if new_extra_names is not None: + extra_names.update(new_extra_names) + ast_args.append(new_arg) + ast_kwargs = [] + for key, value in kwargs.items(): + new_value, new_extra_names = self.__convert_to_ast(value) + if new_extra_names is not None: + extra_names.update(new_extra_names) + ast_kwargs.append(ast.keyword(key, new_value)) + return self.__make_new(ast.Call(self.__get_ast(), ast_args, ast_kwargs), extra_names) def __iter__(self): yield self.__make_new(ast.Starred(self.__get_ast())) @@ -378,8 +405,9 @@ def __format__(self, format_spec): def _make_binop(op: ast.AST): def binop(self, other): + rhs, extra_names = self.__convert_to_ast(other) return self.__make_new( - ast.BinOp(self.__get_ast(), op, self.__convert_to_ast(other)) + ast.BinOp(self.__get_ast(), op, rhs), extra_names ) return binop @@ -392,25 +420,19 @@ def binop(self, other): __mod__ = _make_binop(ast.Mod()) __lshift__ = _make_binop(ast.LShift()) __rshift__ = _make_binop(ast.RShift()) + __or__ = _make_binop(ast.BitOr()) __xor__ = _make_binop(ast.BitXor()) __and__ = _make_binop(ast.BitAnd()) __floordiv__ = _make_binop(ast.FloorDiv()) __pow__ = _make_binop(ast.Pow()) - def __or__(self, other): - if self.__stringifier_dict__.create_unions: - return types.UnionType[self, other] - - return self.__make_new( - ast.BinOp(self.__get_ast(), ast.BitOr(), self.__convert_to_ast(other)) - ) - del _make_binop def _make_rbinop(op: ast.AST): def rbinop(self, other): + new_other, extra_names = self.__convert_to_ast(other) return self.__make_new( - ast.BinOp(self.__convert_to_ast(other), op, self.__get_ast()) + ast.BinOp(new_other, op, self.__get_ast()), extra_names ) return rbinop @@ -423,29 +445,24 @@ def rbinop(self, other): __rmod__ = _make_rbinop(ast.Mod()) __rlshift__ = _make_rbinop(ast.LShift()) __rrshift__ = _make_rbinop(ast.RShift()) + __ror__ = _make_rbinop(ast.BitOr()) __rxor__ = _make_rbinop(ast.BitXor()) __rand__ = _make_rbinop(ast.BitAnd()) __rfloordiv__ = _make_rbinop(ast.FloorDiv()) __rpow__ = _make_rbinop(ast.Pow()) - def __ror__(self, other): - if self.__stringifier_dict__.create_unions: - return types.UnionType[other, self] - - return self.__make_new( - ast.BinOp(self.__convert_to_ast(other), ast.BitOr(), self.__get_ast()) - ) - del _make_rbinop def _make_compare(op): def compare(self, other): + rhs, extra_names = self.__convert_to_ast(other) return self.__make_new( ast.Compare( left=self.__get_ast(), ops=[op], - comparators=[self.__convert_to_ast(other)], - ) + comparators=[rhs], + ), + extra_names, ) return compare @@ -479,8 +496,9 @@ def __init__(self, namespace, globals=None, owner=None, is_class=False, create_u self.globals = globals self.owner = owner self.is_class = is_class - self.create_unions = create_unions + self.create_unions = False self.stringifiers = [] + self.next_id = 1 def __missing__(self, key): fwdref = _Stringifier( @@ -493,6 +511,11 @@ def __missing__(self, key): self.stringifiers.append(fwdref) return fwdref + def create_unique_name(self): + name = f"__annotationlib_name_{self.next_id}__" + self.next_id += 1 + return name + def call_evaluate_function(evaluate, format, *, owner=None): """Call an evaluate function. Evaluate functions are normally generated for @@ -559,9 +582,9 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): ) annos = func(Format.VALUE_WITH_FAKE_GLOBALS) if _is_evaluate: - return annos if isinstance(annos, str) else repr(annos) + return _stringify_single(annos) return { - key: val if isinstance(val, str) else repr(val) + key: _stringify_single(val) for key, val in annos.items() } elif format == Format.FORWARDREF: @@ -640,6 +663,16 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): raise ValueError(f"Invalid format: {format!r}") +def _stringify_single(anno): + if anno is ...: + return "..." + # We have to handle str specially to support PEP 563 stringified annotations. + elif isinstance(anno, str): + return anno + else: + return repr(anno) + + def get_annotate_function(obj): """Get the __annotate__ function for an object. diff --git a/Lib/ast.py b/Lib/ast.py index 507fec5f2d3890..91624d2a7d6246 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -276,6 +276,7 @@ def iter_fields(node): Yield a tuple of ``(fieldname, value)`` for each field in ``node._fields`` that is present on *node*. """ + print(node) for field in node._fields: try: yield field, getattr(node, field) diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index 6f17c85659c34e..2351880840575e 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -115,11 +115,8 @@ def f( self.assertEqual(z_anno, support.EqualToForwardRef("some(module)", owner=f)) alpha_anno = anno["alpha"] - self.assertIsInstance(alpha_anno, Union) - self.assertEqual( - typing.get_args(alpha_anno), - (support.EqualToForwardRef("some", owner=f), support.EqualToForwardRef("obj", owner=f)) - ) + self.assertIsInstance(alpha_anno, ForwardRef) + self.assertEqual(alpha_anno, support.EqualToForwardRef("some | obj", owner=f)) beta_anno = anno["beta"] self.assertIsInstance(beta_anno, ForwardRef) @@ -137,19 +134,20 @@ class UnionForwardrefs: annos = get_annotations(UnionForwardrefs, format=Format.FORWARDREF) - match = ( - str, - support.EqualToForwardRef("undefined", is_class=True, owner=UnionForwardrefs) + pipe = annos["pipe"] + self.assertIsInstance(pipe, ForwardRef) + self.assertEqual( + pipe.evaluate(globals={"undefined": int}), + str | int, ) - + union = annos["union"] + self.assertIsInstance(union, Union) + arg1, arg2 = typing.get_args(union) + self.assertIs(arg1, str) self.assertEqual( - typing.get_args(annos["pipe"]), - typing.get_args(annos["union"]) + arg2, support.EqualToForwardRef("undefined", is_class=True, owner=UnionForwardrefs) ) - self.assertEqual(typing.get_args(annos["pipe"]), match) - self.assertEqual(typing.get_args(annos["union"]), match) - class TestSourceFormat(unittest.TestCase): def test_closure(self): @@ -280,6 +278,64 @@ def f( }, ) + def test_getitem(self): + def f(x: undef1[str, undef2]): + pass + anno = annotationlib.get_annotations(f, format=Format.STRING) + self.assertEqual(anno, {"x": "undef1[str, undef2]"}) + + anno = annotationlib.get_annotations(f, format=Format.FORWARDREF) + fwdref = anno["x"] + self.assertIsInstance(fwdref, ForwardRef) + self.assertEqual( + fwdref.evaluate(globals={"undef1": dict, "undef2": float}), dict[str, float] + ) + + def test_slice(self): + def f(x: a[b:c]): + pass + anno = annotationlib.get_annotations(f, format=Format.STRING) + self.assertEqual(anno, {"x": "a[b:c]"}) + + def f(x: a[b:c, d:e]): + pass + anno = annotationlib.get_annotations(f, format=Format.STRING) + self.assertEqual(anno, {"x": "a[b:c, d:e]"}) + + obj = slice(1, 1, 1) + def f(x: obj): + pass + anno = annotationlib.get_annotations(f, format=Format.STRING) + self.assertEqual(anno, {"x": "obj"}) + + def test_literals(self): + def f( + a: 1, + b: 1.0, + c: "hello", + d: b"hello", + e: True, + f: None, + g: ..., + h: 1j, + ): + pass + + anno = annotationlib.get_annotations(f, format=Format.STRING) + self.assertEqual( + anno, + { + "a": "1", + "b": "1.0", + "c": 'hello', + "d": "b'hello'", + "e": "True", + "f": "None", + "g": "...", + "h": "1j", + }, + ) + def test_nested_expressions(self): def f( nested: list[Annotated[set[int], "set of ints", 4j]], From 8266a5e358818e686b789c166093a3677bf1259e Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 22 Apr 2025 13:41:27 -0700 Subject: [PATCH 06/12] fixes --- Lib/annotationlib.py | 5 +++++ Lib/ast.py | 1 - 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index 310c6a1801c970..18894d5985d101 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -235,6 +235,10 @@ def __eq__(self, other): and self.__forward_is_class__ == other.__forward_is_class__ and self.__cell__ == other.__cell__ and self.__owner__ == other.__owner__ + and ( + (tuple(sorted(self.__extra_names__.items())) if self.__extra_names__ else None) == + (tuple(sorted(other.__extra_names__.items())) if other.__extra_names__ else None) + ) ) def __hash__(self): @@ -245,6 +249,7 @@ def __hash__(self): self.__forward_is_class__, self.__cell__, self.__owner__, + tuple(sorted(self.__extra_names__.items())) if self.__extra_names__ else None, )) def __or__(self, other): diff --git a/Lib/ast.py b/Lib/ast.py index 91624d2a7d6246..507fec5f2d3890 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -276,7 +276,6 @@ def iter_fields(node): Yield a tuple of ``(fieldname, value)`` for each field in ``node._fields`` that is present on *node*. """ - print(node) for field in node._fields: try: yield field, getattr(node, field) From 1c629dc17366ea9dc243328520115ae1318407c4 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 22 Apr 2025 13:42:15 -0700 Subject: [PATCH 07/12] blurb --- .../next/Library/2025-04-22-13-42-12.gh-issue-132805.r-dhmJ.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2025-04-22-13-42-12.gh-issue-132805.r-dhmJ.rst diff --git a/Misc/NEWS.d/next/Library/2025-04-22-13-42-12.gh-issue-132805.r-dhmJ.rst b/Misc/NEWS.d/next/Library/2025-04-22-13-42-12.gh-issue-132805.r-dhmJ.rst new file mode 100644 index 00000000000000..d62b95775a67c2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-04-22-13-42-12.gh-issue-132805.r-dhmJ.rst @@ -0,0 +1,2 @@ +Fix incorrect handling of nested non-constant values in the FORWARDREF +format in :mod:`annotationlib`. From 3d7fec8d3efac475f988df409d2cf4c899d385f6 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 22 Apr 2025 14:07:22 -0700 Subject: [PATCH 08/12] Handle displays --- Lib/annotationlib.py | 24 ++++++++++++++++++++++++ Lib/test/test_annotationlib.py | 25 +++++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index 18894d5985d101..330313fe285d56 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -307,6 +307,30 @@ def __convert_to_ast(self, other): return other.__ast_node__, other.__extra_names__ elif other is None or type(other) in (str, int, float, bool, complex): return ast.Constant(value=other), None + elif type(other) is dict: + extra_names = {} + keys = [] + values = [] + for key, value in other.items(): + new_key, new_extra_names = self.__convert_to_ast(key) + if new_extra_names is not None: + extra_names.update(new_extra_names) + keys.append(new_key) + new_value, new_extra_names = self.__convert_to_ast(value) + if new_extra_names is not None: + extra_names.update(new_extra_names) + values.append(new_value) + return ast.Dict(keys, values), extra_names + elif type(other) in (list, tuple, set): + extra_names = {} + elts = [] + for elt in other: + new_elt, new_extra_names = self.__convert_to_ast(elt) + if new_extra_names is not None: + extra_names.update(new_extra_names) + elts.append(new_elt) + ast_class = {list: ast.List, tuple: ast.Tuple, set: ast.Set}[type(other)] + return ast_class(elts), extra_names else: name = self.__stringifier_dict__.create_unique_name() return ast.Name(id=name), {name: other} diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index 2351880840575e..28ab5e653499ac 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -336,6 +336,31 @@ def f( }, ) + def test_displays(self): + # Simple case first + def f(x: a[[int, str], float]): + pass + anno = annotationlib.get_annotations(f, format=Format.STRING) + self.assertEqual(anno, {"x": "a[[int, str], float]"}) + + def g( + w: a[[int, str], float], + x: a[{int, str}, 3], + y: a[{int: str}, 4], + z: a[(int, str), 5], + ): + pass + anno = annotationlib.get_annotations(g, format=Format.STRING) + self.assertEqual( + anno, + { + "w": "a[[int, str], float]", + "x": "a[{int, str}, 3]", + "y": "a[{int: str}, 4]", + "z": "a[(int, str), 5]", + }, + ) + def test_nested_expressions(self): def f( nested: list[Annotated[set[int], "set of ints", 4j]], From ec4dd0e1fbc619a34f8ace9602c0bafab2eab9f1 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 22 Apr 2025 14:08:14 -0700 Subject: [PATCH 09/12] remove create_unions --- Lib/annotationlib.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index 330313fe285d56..92f8b773576142 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -519,13 +519,12 @@ def unary_op(self): class _StringifierDict(dict): - def __init__(self, namespace, globals=None, owner=None, is_class=False, create_unions=False): + def __init__(self, namespace, globals=None, owner=None, is_class=False): super().__init__(namespace) self.namespace = namespace self.globals = globals self.owner = owner self.is_class = is_class - self.create_unions = False self.stringifiers = [] self.next_id = 1 @@ -641,7 +640,6 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): annotate.__globals__, owner, is_class, - create_unions=True ) if annotate.__closure__: freevars = annotate.__code__.co_freevars From 6e31c69e4f33fb3951cf16eb0000b28b1e2bb7a3 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 22 Apr 2025 16:48:31 -0700 Subject: [PATCH 10/12] prefer empty dict --- Lib/annotationlib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index 92f8b773576142..01c48de679cc2c 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -373,7 +373,7 @@ def __make_new(self, node, extra_names=None): self.__owner__, self.__forward_is_class__, stringifier_dict=self.__stringifier_dict__, - extra_names=new_extra_names, + extra_names=new_extra_names or None, ) self.__stringifier_dict__.stringifiers.append(stringifier) return stringifier From 525a4e499c68177b418f3dabe983dc91ab7e4028 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 23 Apr 2025 08:13:54 -0700 Subject: [PATCH 11/12] Do not use special names in STRING format --- Lib/annotationlib.py | 20 ++++++++++++++------ Lib/test/test_annotationlib.py | 11 +++++++++++ 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index 01c48de679cc2c..f6318507ff50de 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -305,7 +305,13 @@ def __convert_to_ast(self, other): if isinstance(other.__ast_node__, str): return ast.Name(id=other.__ast_node__), other.__extra_names__ return other.__ast_node__, other.__extra_names__ - elif other is None or type(other) in (str, int, float, bool, complex): + elif ( + # In STRING format we don't bother with the create_unique_name() dance; + # it's better to emit the repr() of the object instead of an opaque name. + self.__stringifier_dict__.format == Format.STRING + or other is None + or type(other) in (str, int, float, bool, complex) + ): return ast.Constant(value=other), None elif type(other) is dict: extra_names = {} @@ -519,7 +525,7 @@ def unary_op(self): class _StringifierDict(dict): - def __init__(self, namespace, globals=None, owner=None, is_class=False): + def __init__(self, namespace, *, globals=None, owner=None, is_class=False, format): super().__init__(namespace) self.namespace = namespace self.globals = globals @@ -527,6 +533,7 @@ def __init__(self, namespace, globals=None, owner=None, is_class=False): self.is_class = is_class self.stringifiers = [] self.next_id = 1 + self.format = format def __missing__(self, key): fwdref = _Stringifier( @@ -587,7 +594,7 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): # possibly constants if the annotate function uses them directly). We then # convert each of those into a string to get an approximation of the # original source. - globals = _StringifierDict({}) + globals = _StringifierDict({}, format=format) if annotate.__closure__: freevars = annotate.__code__.co_freevars new_closure = [] @@ -637,9 +644,10 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): is_class = isinstance(owner, type) globals = _StringifierDict( namespace, - annotate.__globals__, - owner, - is_class, + globals=annotate.__globals__, + owner=owner, + is_class=is_class, + format=format, ) if annotate.__closure__: freevars = annotate.__code__.co_freevars diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index 28ab5e653499ac..1c95ae2ef3d7e2 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -406,6 +406,17 @@ def f(fstring_format: f"{a:02d}"): with self.assertRaisesRegex(TypeError, format_msg): annotationlib.get_annotations(f, format=Format.STRING) + def test_shenanigans(self): + # In cases like this we can't reconstruct the source; test that we do something + # halfway reasonable. + def f(x: x | (1).__class__, y: (1).__class__): + pass + + self.assertEqual( + annotationlib.get_annotations(f, format=Format.STRING), + {"x": "x | ", "y": ""}, + ) + class TestForwardRefClass(unittest.TestCase): def test_special_attrs(self): From 22a9691884f9db71bbac7741ce449a0111d793ae Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sun, 4 May 2025 07:36:48 -0700 Subject: [PATCH 12/12] fix bad merge --- Lib/test/test_annotationlib.py | 106 --------------------------------- 1 file changed, 106 deletions(-) diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index ec63d5dbc05f6d..d9000b6392277e 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -413,112 +413,6 @@ def f(x: x | (1).__class__, y: (1).__class__): ) -class TestForwardRefClass(unittest.TestCase): - def test_special_attrs(self): - # Forward refs provide a different introspection API. __name__ and - # __qualname__ make little sense for forward refs as they can store - # complex typing expressions. - fr = ForwardRef("set[Any]") - self.assertFalse(hasattr(fr, "__name__")) - self.assertFalse(hasattr(fr, "__qualname__")) - self.assertEqual(fr.__module__, "annotationlib") - # Forward refs are currently unpicklable once they contain a code object. - fr.__forward_code__ # fill the cache - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - with self.assertRaises(TypeError): - pickle.dumps(fr, proto) - - def test_evaluate_with_type_params(self): - class Gen[T]: - alias = int - - with self.assertRaises(NameError): - ForwardRef("T").evaluate() - with self.assertRaises(NameError): - ForwardRef("T").evaluate(type_params=()) - with self.assertRaises(NameError): - ForwardRef("T").evaluate(owner=int) - - (T,) = Gen.__type_params__ - self.assertIs(ForwardRef("T").evaluate(type_params=Gen.__type_params__), T) - self.assertIs(ForwardRef("T").evaluate(owner=Gen), T) - - with self.assertRaises(NameError): - ForwardRef("alias").evaluate(type_params=Gen.__type_params__) - self.assertIs(ForwardRef("alias").evaluate(owner=Gen), int) - # If you pass custom locals, we don't look at the owner's locals - with self.assertRaises(NameError): - ForwardRef("alias").evaluate(owner=Gen, locals={}) - # But if the name exists in the locals, it works - self.assertIs( - ForwardRef("alias").evaluate(owner=Gen, locals={"alias": str}), str - ) - - def test_fwdref_with_module(self): - self.assertIs(ForwardRef("Format", module="annotationlib").evaluate(), Format) - self.assertIs( - ForwardRef("Counter", module="collections").evaluate(), collections.Counter - ) - self.assertEqual( - ForwardRef("Counter[int]", module="collections").evaluate(), - collections.Counter[int], - ) - - with self.assertRaises(NameError): - # If globals are passed explicitly, we don't look at the module dict - ForwardRef("Format", module="annotationlib").evaluate(globals={}) - - def test_fwdref_to_builtin(self): - self.assertIs(ForwardRef("int").evaluate(), int) - self.assertIs(ForwardRef("int", module="collections").evaluate(), int) - self.assertIs(ForwardRef("int", owner=str).evaluate(), int) - - # builtins are still searched with explicit globals - self.assertIs(ForwardRef("int").evaluate(globals={}), int) - - # explicit values in globals have precedence - obj = object() - self.assertIs(ForwardRef("int").evaluate(globals={"int": obj}), obj) - - def test_fwdref_value_is_not_cached(self): - fr = ForwardRef("hello") - with self.assertRaises(NameError): - fr.evaluate() - self.assertIs(fr.evaluate(globals={"hello": str}), str) - with self.assertRaises(NameError): - fr.evaluate() - - def test_fwdref_with_owner(self): - self.assertEqual( - ForwardRef("Counter[int]", owner=collections).evaluate(), - collections.Counter[int], - ) - - def test_name_lookup_without_eval(self): - # test the codepath where we look up simple names directly in the - # namespaces without going through eval() - self.assertIs(ForwardRef("int").evaluate(), int) - self.assertIs(ForwardRef("int").evaluate(locals={"int": str}), str) - self.assertIs( - ForwardRef("int").evaluate(locals={"int": float}, globals={"int": str}), - float, - ) - self.assertIs(ForwardRef("int").evaluate(globals={"int": str}), str) - with support.swap_attr(builtins, "int", dict): - self.assertIs(ForwardRef("int").evaluate(), dict) - - with self.assertRaises(NameError): - ForwardRef("doesntexist").evaluate() - - def test_fwdref_invalid_syntax(self): - fr = ForwardRef("if") - with self.assertRaises(SyntaxError): - fr.evaluate() - fr = ForwardRef("1+") - with self.assertRaises(SyntaxError): - fr.evaluate() - - class TestGetAnnotations(unittest.TestCase): def test_builtin_type(self): self.assertEqual(get_annotations(int), {}) pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy