Don't simplify out ErasedType from unions during type inference (python#8095)

ilevkivskyi · web-flow · commit 5680f5c5a8f5 · 2019-12-05T20:05:31.000Z
Fixes python#8093 The fix is kind of simple, we really need to keep all `ErasedType`s intact to preserve the existing type inference logic, but it is also long because I need to thread the flag down like dozen function calls. The issue itself btw highlight how fragile is current type inference logic (note that `expand_type()` implicitly calls `make_simplified_union()` too, but I think that one doesn't need to be updated because it shouldn't be called with any erased components).
diff --git a/mypy/constraints.py b/mypy/constraints.py
@@ -109,10 +109,12 @@ def _infer_constraints(template: Type, actual: Type,
     actual = get_proper_type(actual)
 
     # Type inference shouldn't be affected by whether union types have been simplified.
+    # We however keep any ErasedType items, so that the caller will see it when using
+    # checkexpr.has_erased_component().
     if isinstance(template, UnionType):
-        template = mypy.typeops.make_simplified_union(template.items)
+        template = mypy.typeops.make_simplified_union(template.items, keep_erased=True)
     if isinstance(actual, UnionType):
-        actual = mypy.typeops.make_simplified_union(actual.items)
+        actual = mypy.typeops.make_simplified_union(actual.items, keep_erased=True)
 
     # Ignore Any types from the type suggestion engine to avoid them
     # causing us to infer Any in situations where a better job could
diff --git a/mypy/subtypes.py b/mypy/subtypes.py
@@ -1116,15 +1116,18 @@ def covers_at_runtime(item: Type, supertype: Type, ignore_promotions: bool) -> b
     return False
 
 
-def is_proper_subtype(left: Type, right: Type, *, ignore_promotions: bool = False,
-                      erase_instances: bool = False) -> bool:
+def is_proper_subtype(left: Type, right: Type, *,
+                      ignore_promotions: bool = False,
+                      erase_instances: bool = False,
+                      keep_erased_types: bool = False) -> bool:
     """Is left a proper subtype of right?
 
     For proper subtypes, there's no need to rely on compatibility due to
     Any types. Every usable type is a proper subtype of itself.
 
     If erase_instances is True, erase left instance *after* mapping it to supertype
-    (this is useful for runtime isinstance() checks).
+    (this is useful for runtime isinstance() checks). If keep_erased_types is True,
+    do not consider ErasedType a subtype of all types (used by type inference against unions).
     """
     if TypeState.is_assumed_proper_subtype(left, right):
         return True
@@ -1135,50 +1138,63 @@ def is_proper_subtype(left: Type, right: Type, *, ignore_promotions: bool = Fals
         with pop_on_exit(TypeState._assuming_proper, left, right):
             return _is_proper_subtype(left, right,
                                       ignore_promotions=ignore_promotions,
-                                      erase_instances=erase_instances)
+                                      erase_instances=erase_instances,
+                                      keep_erased_types=keep_erased_types)
     return _is_proper_subtype(left, right,
                               ignore_promotions=ignore_promotions,
-                              erase_instances=erase_instances)
+                              erase_instances=erase_instances,
+                              keep_erased_types=keep_erased_types)
 
 
-def _is_proper_subtype(left: Type, right: Type, *, ignore_promotions: bool = False,
-                       erase_instances: bool = False) -> bool:
+def _is_proper_subtype(left: Type, right: Type, *,
+                       ignore_promotions: bool = False,
+                       erase_instances: bool = False,
+                       keep_erased_types: bool = False) -> bool:
     orig_left = left
     orig_right = right
     left = get_proper_type(left)
     right = get_proper_type(right)
 
     if isinstance(right, UnionType) and not isinstance(left, UnionType):
-        return any([is_proper_subtype(orig_left, item, ignore_promotions=ignore_promotions,
-                                      erase_instances=erase_instances)
+        return any([is_proper_subtype(orig_left, item,
+                                      ignore_promotions=ignore_promotions,
+                                      erase_instances=erase_instances,
+                                      keep_erased_types=keep_erased_types)
                     for item in right.items])
     return left.accept(ProperSubtypeVisitor(orig_right,
                                             ignore_promotions=ignore_promotions,
-                                            erase_instances=erase_instances))
+                                            erase_instances=erase_instances,
+                                            keep_erased_types=keep_erased_types))
 
 
 class ProperSubtypeVisitor(TypeVisitor[bool]):
     def __init__(self, right: Type, *,
                  ignore_promotions: bool = False,
-                 erase_instances: bool = False) -> None:
+                 erase_instances: bool = False,
+                 keep_erased_types: bool = False) -> None:
         self.right = get_proper_type(right)
         self.orig_right = right
         self.ignore_promotions = ignore_promotions
         self.erase_instances = erase_instances
+        self.keep_erased_types = keep_erased_types
         self._subtype_kind = ProperSubtypeVisitor.build_subtype_kind(
             ignore_promotions=ignore_promotions,
             erase_instances=erase_instances,
+            keep_erased_types=keep_erased_types
         )
 
     @staticmethod
-    def build_subtype_kind(*, ignore_promotions: bool = False,
-                           erase_instances: bool = False) -> SubtypeKind:
-        return True, ignore_promotions, erase_instances
+    def build_subtype_kind(*,
+                           ignore_promotions: bool = False,
+                           erase_instances: bool = False,
+                           keep_erased_types: bool = False) -> SubtypeKind:
+        return True, ignore_promotions, erase_instances, keep_erased_types
 
     def _is_proper_subtype(self, left: Type, right: Type) -> bool:
         return is_proper_subtype(left, right,
                                  ignore_promotions=self.ignore_promotions,
-                                 erase_instances=self.erase_instances)
+                                 erase_instances=self.erase_instances,
+                                 keep_erased_types=self.keep_erased_types)
 
     def visit_unbound_type(self, left: UnboundType) -> bool:
         # This can be called if there is a bad type annotation. The result probably
@@ -1201,6 +1217,9 @@ def visit_uninhabited_type(self, left: UninhabitedType) -> bool:
     def visit_erased_type(self, left: ErasedType) -> bool:
         # This may be encountered during type inference. The result probably doesn't
         # matter much.
+        # TODO: it actually does matter, figure out more principled logic about this.
+        if self.keep_erased_types:
+            return False
         return True
 
     def visit_deleted_type(self, left: DeletedType) -> bool:
diff --git a/mypy/typeops.py b/mypy/typeops.py
@@ -312,7 +312,8 @@ def callable_corresponding_argument(typ: CallableType,
 
 
 def make_simplified_union(items: Sequence[Type],
-                          line: int = -1, column: int = -1) -> ProperType:
+                          line: int = -1, column: int = -1,
+                          *, keep_erased: bool = False) -> ProperType:
     """Build union type with redundant union items removed.
 
     If only a single item remains, this may return a non-union type.
@@ -327,6 +328,8 @@ def make_simplified_union(items: Sequence[Type],
 
     Note: This must NOT be used during semantic analysis, since TypeInfos may not
           be fully initialized.
+    The keep_erased flag is used for type inference against union types
+    containing type variables. If set to True, keep all ErasedType items.
     """
     items = get_proper_types(items)
     while any(isinstance(typ, UnionType) for typ in items):
@@ -346,7 +349,7 @@ def make_simplified_union(items: Sequence[Type],
         # Keep track of the truishness info for deleted subtypes which can be relevant
         cbt = cbf = False
         for j, tj in enumerate(items):
-            if i != j and is_proper_subtype(tj, ti):
+            if i != j and is_proper_subtype(tj, ti, keep_erased_types=keep_erased):
                 # We found a redundant item in the union.
                 removed.add(j)
                 cbt = cbt or tj.can_be_true
diff --git a/test-data/unit/check-inference.test b/test-data/unit/check-inference.test
@@ -2939,3 +2939,17 @@ X = Union[Cov[T], Inv[T]]
 def f(x: X[T]) -> T: ...
 x: Inv[int]
 reveal_type(f(x))  # N: Revealed type is 'builtins.int*'
+
+[case testOptionalTypeVarAgainstOptional]
+# flags: --strict-optional
+from typing import Optional, TypeVar, Iterable, Iterator, List
+
+_T = TypeVar('_T')
+
+def filter(__function: None, __iterable: Iterable[Optional[_T]]) -> List[_T]: ...
+
+x: Optional[str]
+
+y = filter(None, [x])
+reveal_type(y)  # N: Revealed type is 'builtins.list[builtins.str*]'
+[builtins fixtures/list.pyi]
