Skip to content

Commit 5d52eac

Browse files
ilevkivskyiilevkivskyi
authored
New semantic analyzer: Make assignment analysis logic more straightforward (python#6527)
Fixes python#6412 Fixes python#6453 In this PR: * Wait until we can classify r.h.s. of an assignment as normal, type alias, named tuple etc. * Choose the corresponding branch instead of performing all of them (as it used to be before) * Never create (or rely on creation of) temporary `Var`s * Add every symbol to symbol table only once (as it is done for classes) * Don't "patch" existing symbols (with one exception for type variables, see below) * Add and update docstrings and comments Notes: * I allow placeholders for type alias targets and NewTypes targets, they essentially behave as base classes. In general I make logic for type aliases and NewTypes much more similar to logic of class definitions (especially NewTypes), this allows to support a whole bunch of tricky forward references * I don't store types with placeholders in annotations, and instead if an alias target got updated (e.g. placeholders are resolved), I set `progress=True` Follow-ups: * There is still a small hack to keep compatibility with old analyzer in type checker, but now it is basically "don't use binder for l.h.s. of special forms" * Type variable declarations still have a bug that duplicate type variable definition is not reported as error, second one silently wins, this will be fixed in a separate PR * This PR doesn't aim to fix aspects of assignment analysis related to python#6422, that will be also a separate PR Test updates: * Add and enable more tests for type aliases and NewTypes * Add test for cyclic definition error messages * Change order of error messages on the same line in few tests * Enable one test file where previously only some tests were opted in (I think it was not enabled by mistake in one of the previous PRs, I enable it here since order of errors in one test in that file is affected by this PR.)
1 parent 69a5a02 commit 5d52eac

15 files changed

+570
-237
lines changed

‎mypy/checker.py

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1715,6 +1715,11 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None:
17151715
with self.enter_final_context(s.is_final_def):
17161716
self.check_assignment(s.lvalues[-1], s.rvalue, s.type is None, s.new_syntax)
17171717

1718+
if s.is_alias_def:
1719+
# We do this mostly for compatibility with old semantic analyzer.
1720+
# TODO: should we get rid of this?
1721+
self.store_type(s.lvalues[-1], self.expr_checker.accept(s.rvalue))
1722+
17181723
if (s.type is not None and
17191724
self.options.disallow_any_unimported and
17201725
has_any_from_unimported_type(s.type)):
@@ -1824,7 +1829,9 @@ def check_assignment(self, lvalue: Lvalue, rvalue: Expression, infer_lvalue_type
18241829
self.msg.concrete_only_assign(lvalue_type, rvalue)
18251830
return
18261831
if rvalue_type and infer_lvalue_type and not isinstance(lvalue_type, PartialType):
1827-
self.binder.assign_type(lvalue, rvalue_type, lvalue_type, False)
1832+
# Don't use type binder for definitions of special forms, like named tuples.
1833+
if not (isinstance(lvalue, NameExpr) and lvalue.is_special_form):
1834+
self.binder.assign_type(lvalue, rvalue_type, lvalue_type, False)
18281835

18291836
elif index_lvalue:
18301837
self.check_indexed_assignment(index_lvalue, rvalue, lvalue)

‎mypy/newsemanal/semanal.py

Lines changed: 275 additions & 108 deletions
Large diffs are not rendered by default.

‎mypy/newsemanal/semanal_enum.py

Lines changed: 25 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -19,20 +19,22 @@ def __init__(self, options: Options, api: SemanticAnalyzerInterface) -> None:
1919
self.options = options
2020
self.api = api
2121

22-
def process_enum_call(self, s: AssignmentStmt, is_func_scope: bool) -> None:
23-
"""Check if s defines an Enum; if yes, store the definition in symbol table."""
22+
def process_enum_call(self, s: AssignmentStmt, is_func_scope: bool) -> bool:
23+
"""Check if s defines an Enum; if yes, store the definition in symbol table.
24+
25+
Return True if this looks like an Enum definition (but maybe with errors),
26+
otherwise return False.
27+
"""
2428
if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr):
25-
return
29+
return False
2630
lvalue = s.lvalues[0]
2731
name = lvalue.name
2832
enum_call = self.check_enum_call(s.rvalue, name, is_func_scope)
2933
if enum_call is None:
30-
return
34+
return False
3135
# Yes, it's a valid Enum definition. Add it to the symbol table.
32-
node = self.api.lookup(name, s)
33-
if node:
34-
node.kind = GDEF # TODO locally defined Enum
35-
node.node = enum_call
36+
self.api.add_symbol(name, enum_call, s)
37+
return True
3638

3739
def check_enum_call(self,
3840
node: Expression,
@@ -62,18 +64,19 @@ class A(enum.Enum):
6264
items, values, ok = self.parse_enum_call_args(call, fullname.split('.')[-1])
6365
if not ok:
6466
# Error. Construct dummy return value.
65-
return self.build_enum_call_typeinfo(var_name, [], fullname)
66-
name = cast(Union[StrExpr, UnicodeExpr], call.args[0]).value
67-
if name != var_name or is_func_scope:
68-
# Give it a unique name derived from the line number.
69-
name += '@' + str(call.line)
70-
info = self.build_enum_call_typeinfo(name, items, fullname)
71-
# Store it as a global just in case it would remain anonymous.
72-
# (Or in the nearest class if there is one.)
73-
stnode = SymbolTableNode(GDEF, info)
74-
self.api.add_symbol_table_node(name, stnode)
67+
info = self.build_enum_call_typeinfo(var_name, [], fullname)
68+
else:
69+
name = cast(Union[StrExpr, UnicodeExpr], call.args[0]).value
70+
if name != var_name or is_func_scope:
71+
# Give it a unique name derived from the line number.
72+
name += '@' + str(call.line)
73+
info = self.build_enum_call_typeinfo(name, items, fullname)
74+
# Store generated TypeInfo under both names, see semanal_namedtuple for more details.
75+
if name != var_name or is_func_scope:
76+
self.api.add_symbol_skip_local(name, info)
7577
call.analyzed = EnumCallExpr(info, items, values)
7678
call.analyzed.set_line(call.line, call.column)
79+
info.line = node.line
7780
return info
7881

7982
def build_enum_call_typeinfo(self, name: str, items: List[str], fullname: str) -> TypeInfo:
@@ -93,6 +96,10 @@ def build_enum_call_typeinfo(self, name: str, items: List[str], fullname: str) -
9396
def parse_enum_call_args(self, call: CallExpr,
9497
class_name: str) -> Tuple[List[str],
9598
List[Optional[Expression]], bool]:
99+
"""Parse arguments of an Enum call.
100+
101+
Return a tuple of fields, values, was there an error.
102+
"""
96103
args = call.args
97104
if len(args) < 2:
98105
return self.fail_enum_call_arg("Too few arguments for %s()" % class_name, call)

‎mypy/newsemanal/semanal_newtype.py

Lines changed: 75 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,14 @@
55

66
from typing import Tuple, Optional
77

8-
from mypy.types import Type, Instance, CallableType, NoneTyp, TupleType, AnyType, TypeOfAny
8+
from mypy.types import (
9+
Type, Instance, CallableType, NoneTyp, TupleType, AnyType, PlaceholderType,
10+
TypeOfAny
11+
)
912
from mypy.nodes import (
1013
AssignmentStmt, NewTypeExpr, CallExpr, NameExpr, RefExpr, Context, StrExpr, BytesExpr,
11-
UnicodeExpr, Block, FuncDef, Argument, TypeInfo, Var, SymbolTableNode, GDEF, MDEF, ARG_POS
14+
UnicodeExpr, Block, FuncDef, Argument, TypeInfo, Var, SymbolTableNode, MDEF, ARG_POS,
15+
PlaceholderNode
1216
)
1317
from mypy.newsemanal.semanal_shared import SemanticAnalyzerInterface
1418
from mypy.options import Options
@@ -26,17 +30,36 @@ def __init__(self,
2630
self.api = api
2731
self.msg = msg
2832

29-
def process_newtype_declaration(self, s: AssignmentStmt) -> None:
30-
"""Check if s declares a NewType; if yes, store it in symbol table."""
31-
# Extract and check all information from newtype declaration
33+
def process_newtype_declaration(self, s: AssignmentStmt) -> bool:
34+
"""Check if s declares a NewType; if yes, store it in symbol table.
35+
36+
Return True if it's a NewType declaration. The current target may be
37+
deferred as a side effect if the base type is not ready, even if
38+
the return value is True.
39+
40+
The logic in this function mostly copies the logic for visit_class_def()
41+
with a single (non-Generic) base.
42+
"""
3243
name, call = self.analyze_newtype_declaration(s)
3344
if name is None or call is None:
34-
return
35-
36-
old_type = self.check_newtype_args(name, call, s)
37-
call.analyzed = NewTypeExpr(name, old_type, line=call.line)
45+
return False
46+
# OK, now we know this is a NewType. But the base type may be not ready yet,
47+
# add placeholder as we do for ClassDef.
48+
49+
fullname = self.api.qualified_name(name)
50+
if (not call.analyzed or
51+
isinstance(call.analyzed, NewTypeExpr) and not call.analyzed.info):
52+
# Start from labeling this as a future class, as we do for normal ClassDefs.
53+
self.api.add_symbol(name, PlaceholderNode(fullname, s, becomes_typeinfo=True), s)
54+
55+
old_type, should_defer = self.check_newtype_args(name, call, s)
56+
if not call.analyzed:
57+
call.analyzed = NewTypeExpr(name, old_type, line=call.line)
3858
if old_type is None:
39-
return
59+
if should_defer:
60+
# Base type is not ready.
61+
self.api.defer()
62+
return True
4063

4164
# Create the corresponding class definition if the aliased type is subtypeable
4265
if isinstance(old_type, TupleType):
@@ -48,9 +71,14 @@ def process_newtype_declaration(self, s: AssignmentStmt) -> None:
4871
self.fail("NewType cannot be used with protocol classes", s)
4972
newtype_class_info = self.build_newtype_typeinfo(name, old_type, old_type)
5073
else:
51-
message = "Argument 2 to NewType(...) must be subclassable (got {})"
52-
self.fail(message.format(self.msg.format(old_type)), s)
53-
return
74+
if old_type is not None:
75+
message = "Argument 2 to NewType(...) must be subclassable (got {})"
76+
self.fail(message.format(self.msg.format(old_type)), s)
77+
# Otherwise the error was already reported.
78+
old_type = AnyType(TypeOfAny.from_error)
79+
object_type = self.api.named_type('__builtins__.object')
80+
newtype_class_info = self.build_newtype_typeinfo(name, old_type, object_type)
81+
newtype_class_info.fallback_to_any = True
5482

5583
check_for_explicit_any(old_type, self.options, self.api.is_typeshed_stub_file, self.msg,
5684
context=s)
@@ -59,13 +87,16 @@ def process_newtype_declaration(self, s: AssignmentStmt) -> None:
5987
self.msg.unimported_type_becomes_any("Argument 2 to NewType(...)", old_type, s)
6088

6189
# If so, add it to the symbol table.
62-
node = self.api.lookup(name, s)
63-
if node is None:
64-
self.fail("Could not find {} in current namespace".format(name), s)
65-
return
66-
# TODO: why does NewType work in local scopes despite always being of kind GDEF?
67-
node.kind = GDEF
68-
call.analyzed.info = node.node = newtype_class_info
90+
assert isinstance(call.analyzed, NewTypeExpr)
91+
# As we do for normal classes, create the TypeInfo only once, then just
92+
# update base classes on next iterations (to get rid of placeholders there).
93+
if not call.analyzed.info:
94+
call.analyzed.info = newtype_class_info
95+
else:
96+
call.analyzed.info.bases = newtype_class_info.bases
97+
self.api.add_symbol(name, call.analyzed.info, s)
98+
newtype_class_info.line = s.line
99+
return True
69100

70101
def analyze_newtype_declaration(self,
71102
s: AssignmentStmt) -> Tuple[Optional[str], Optional[CallExpr]]:
@@ -76,13 +107,18 @@ def analyze_newtype_declaration(self,
76107
and isinstance(s.rvalue, CallExpr)
77108
and isinstance(s.rvalue.callee, RefExpr)
78109
and s.rvalue.callee.fullname == 'typing.NewType'):
79-
lvalue = s.lvalues[0]
80110
name = s.lvalues[0].name
81-
if not lvalue.is_inferred_def:
82-
if s.type:
83-
self.fail("Cannot declare the type of a NewType declaration", s)
84-
else:
85-
self.fail("Cannot redefine '%s' as a NewType" % name, s)
111+
112+
if s.type:
113+
self.fail("Cannot declare the type of a NewType declaration", s)
114+
115+
names = self.api.current_symbol_table()
116+
existing = names.get(name)
117+
# Give a better error message than generic "Name already defined",
118+
# like the old semantic analyzer does.
119+
if (existing and
120+
not isinstance(existing.node, PlaceholderNode) and not s.rvalue.analyzed):
121+
self.fail("Cannot redefine '%s' as a NewType" % name, s)
86122

87123
# This dummy NewTypeExpr marks the call as sufficiently analyzed; it will be
88124
# overwritten later with a fully complete NewTypeExpr if there are no other
@@ -91,12 +127,17 @@ def analyze_newtype_declaration(self,
91127

92128
return name, call
93129

94-
def check_newtype_args(self, name: str, call: CallExpr, context: Context) -> Optional[Type]:
130+
def check_newtype_args(self, name: str, call: CallExpr,
131+
context: Context) -> Tuple[Optional[Type], bool]:
132+
"""Ananlyze base type in NewType call.
133+
134+
Return a tuple (type, should defer).
135+
"""
95136
has_failed = False
96137
args, arg_kinds = call.args, call.arg_kinds
97138
if len(args) != 2 or arg_kinds[0] != ARG_POS or arg_kinds[1] != ARG_POS:
98139
self.fail("NewType(...) expects exactly two positional arguments", context)
99-
return None
140+
return None, False
100141

101142
# Check first argument
102143
if not isinstance(args[0], (StrExpr, BytesExpr, UnicodeExpr)):
@@ -113,19 +154,22 @@ def check_newtype_args(self, name: str, call: CallExpr, context: Context) -> Opt
113154
unanalyzed_type = expr_to_unanalyzed_type(args[1])
114155
except TypeTranslationError:
115156
self.fail(msg, context)
116-
return None
157+
return None, False
117158

118159
# We want to use our custom error message (see above), so we suppress
119160
# the default error message for invalid types here.
120161
old_type = self.api.anal_type(unanalyzed_type, report_invalid_types=False)
162+
should_defer = False
163+
if old_type is None or isinstance(old_type, PlaceholderType):
164+
should_defer = True
121165

122166
# The caller of this function assumes that if we return a Type, it's always
123167
# a valid one. So, we translate AnyTypes created from errors into None.
124168
if isinstance(old_type, AnyType) and old_type.is_from_error:
125169
self.fail(msg, context)
126-
return None
170+
return None, False
127171

128-
return None if has_failed else old_type
172+
return None if has_failed else old_type, should_defer
129173

130174
def build_newtype_typeinfo(self, name: str, old_type: Type, base_type: Instance) -> TypeInfo:
131175
info = self.api.basic_new_typeinfo(name, base_type)

‎mypy/newsemanal/semanal_shared.py

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
from mypy.nodes import (
88
Context, SymbolTableNode, MypyFile, ImportedName, FuncDef, Node, TypeInfo, Expression, GDEF,
9-
SymbolNode
9+
SymbolNode, SymbolTable
1010
)
1111
from mypy.util import correct_relative_import
1212
from mypy.types import Type, FunctionLike, Instance, TupleType
@@ -123,6 +123,14 @@ def add_symbol_table_node(self, name: str, stnode: SymbolTableNode) -> bool:
123123
"""Add node to the current symbol table."""
124124
raise NotImplementedError
125125

126+
@abstractmethod
127+
def current_symbol_table(self) -> SymbolTable:
128+
"""Get currently active symbol table.
129+
130+
May be module, class, or local namespace.
131+
"""
132+
raise NotImplementedError
133+
126134
@abstractmethod
127135
def add_symbol(self, name: str, node: SymbolNode, context: Optional[Context],
128136
module_public: bool = True, module_hidden: bool = False) -> bool:

‎mypy/newsemanal/typeanal.py

Lines changed: 3 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -75,48 +75,6 @@ def analyze_type_alias(node: Expression,
7575
full names of type aliases it depends on (directly or indirectly).
7676
Return None otherwise. 'node' must have been semantically analyzed.
7777
"""
78-
# Quickly return None if the expression doesn't look like a type. Note
79-
# that we don't support straight string literals as type aliases
80-
# (only string literals within index expressions).
81-
if isinstance(node, RefExpr):
82-
# Note that this misses the case where someone tried to use a
83-
# class-referenced type variable as a type alias. It's easier to catch
84-
# that one in checkmember.py
85-
if isinstance(node.node, TypeVarExpr):
86-
api.fail('Type variable "{}" is invalid as target for type alias'.format(
87-
node.fullname), node)
88-
return None
89-
if not (isinstance(node.node, TypeInfo) or
90-
node.fullname in ('typing.Any', 'typing.Tuple', 'typing.Callable') or
91-
isinstance(node.node, TypeAlias)):
92-
return None
93-
elif isinstance(node, IndexExpr):
94-
base = node.base
95-
if isinstance(base, RefExpr):
96-
if not (isinstance(base.node, TypeInfo) or
97-
base.fullname in type_constructors or
98-
isinstance(base.node, TypeAlias)):
99-
return None
100-
# Enums can't be generic, and without this check we may incorrectly interpret indexing
101-
# an Enum class as creating a type alias.
102-
if isinstance(base.node, TypeInfo) and base.node.is_enum:
103-
return None
104-
else:
105-
return None
106-
elif isinstance(node, CallExpr):
107-
if (isinstance(node.callee, NameExpr) and len(node.args) == 1 and
108-
isinstance(node.args[0], NameExpr)):
109-
call = api.lookup_qualified(node.callee.name, node.callee)
110-
arg = api.lookup_qualified(node.args[0].name, node.args[0])
111-
if (call is not None and call.node and call.node.fullname() == 'builtins.type' and
112-
arg is not None and arg.node and arg.node.fullname() == 'builtins.None'):
113-
return NoneTyp(), set()
114-
return None
115-
return None
116-
else:
117-
return None
118-
119-
# It's a type alias (though it may be an invalid one).
12078
try:
12179
type = expr_to_unanalyzed_type(node)
12280
except TypeTranslationError:
@@ -414,14 +372,10 @@ def analyze_unbound_type_without_type_info(self, t: UnboundType, sym: SymbolTabl
414372
(not self.tvar_scope or self.tvar_scope.get_binding(sym) is None))
415373
if self.allow_unbound_tvars and unbound_tvar and not self.third_pass:
416374
return t
375+
417376
# None of the above options worked, we give up.
418-
# NOTE: 'final_iteration' is iteration when we hit the maximum number of iterations limit.
419-
if unbound_tvar or self.api.final_iteration:
420-
# TODO: This is problematic, since we will have to wait until the maximum number
421-
# of iterations to report an invalid type.
422-
self.fail('Invalid type "{}"'.format(name), t)
423-
else:
424-
self.api.defer()
377+
self.fail('Invalid type "{}"'.format(name), t)
378+
425379
if self.third_pass and isinstance(sym.node, TypeVarExpr):
426380
self.note_func("Forward references to type variables are prohibited", t)
427381
return AnyType(TypeOfAny.from_error)

‎mypy/nodes.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1418,11 +1418,13 @@ class NameExpr(RefExpr):
14181418
This refers to a local name, global name or a module.
14191419
"""
14201420

1421-
__slots__ = ('name',)
1421+
__slots__ = ('name', 'is_special_form')
14221422

14231423
def __init__(self, name: str) -> None:
14241424
super().__init__()
14251425
self.name = name # Name referred to (may be qualified)
1426+
# Is this a l.h.s. of a special form assignment like typed dict or type variable?
1427+
self.is_special_form = False
14261428

14271429
def accept(self, visitor: ExpressionVisitor[T]) -> T:
14281430
return visitor.visit_name_expr(self)

‎mypy/strconv.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -345,7 +345,9 @@ def visit_star_expr(self, o: 'mypy.nodes.StarExpr') -> str:
345345
return self.dump([o.expr], o)
346346

347347
def visit_name_expr(self, o: 'mypy.nodes.NameExpr') -> str:
348-
pretty = self.pretty_name(o.name, o.kind, o.fullname, o.is_inferred_def, o.node)
348+
pretty = self.pretty_name(o.name, o.kind, o.fullname,
349+
o.is_inferred_def or o.is_special_form,
350+
o.node)
349351
if isinstance(o.node, mypy.nodes.Var) and o.node.is_final:
350352
pretty += ' = {}'.format(o.node.final_value)
351353
return short_type(o) + '(' + pretty + ')'

‎mypy/test/hacks.py

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,8 @@
1515
'check-incremental.test',
1616
'check-literal.test',
1717
'check-modules.test',
18-
'check-newtype.test',
1918
'check-overloading.test',
2019
'check-python2.test',
21-
'check-semanal-error.test',
2220
'check-statements.test',
2321
'check-unions.test',
2422
'check-unreachable-code.test',

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy