[suggest] Fix inference from collection literals (python#7979)

msullivan · web-flow · commit c39335a3c223 · 2019-11-20T13:06:43.000-08:00
This is another take at python#7810 which was reverted because it caused internal trouble. Do this by ignoring constraints with Any when the Any comes from the suggestion engine. The Any context causes assigned types to be Any overaggressively. Restricting it only to Anys from the suggestion engine avoids the issues seen last time. Doing it at a fine-grained level in constraints allows us to do better refining things like `List[Any]`.
diff --git a/mypy/constraints.py b/mypy/constraints.py
@@ -108,6 +108,13 @@ def _infer_constraints(template: Type, actual: Type,
     template = get_proper_type(template)
     actual = get_proper_type(actual)
 
+    # Ignore Any types from the type suggestion engine to avoid them
+    # causing us to infer Any in situations where a better job could
+    # be done otherwise. (This can produce false positives but that
+    # doesn't really matter because it is all heuristic anyway.)
+    if isinstance(actual, AnyType) and actual.type_of_any == TypeOfAny.suggestion_engine:
+        return []
+
     # If the template is simply a type variable, emit a Constraint directly.
     # We need to handle this case before handling Unions for two reasons:
     #  1. "T <: Union[U1, U2]" is not equivalent to "T <: U1 or T <: U2",
diff --git a/mypy/suggestions.py b/mypy/suggestions.py
@@ -23,14 +23,14 @@
 """
 
 from typing import (
-    List, Optional, Tuple, Dict, Callable, Union, NamedTuple, TypeVar, Iterator,
+    List, Optional, Tuple, Dict, Callable, Union, NamedTuple, TypeVar, Iterator, cast,
 )
 from typing_extensions import TypedDict
 
 from mypy.state import strict_optional_set
 from mypy.types import (
     Type, AnyType, TypeOfAny, CallableType, UnionType, NoneType, Instance, TupleType,
-    TypeVarType, FunctionLike,
+    TypeVarType, FunctionLike, UninhabitedType,
     TypeStrVisitor, TypeTranslator,
     is_optional, remove_optional, ProperType, get_proper_type,
     TypedDictType, TypeAliasType
@@ -282,17 +282,19 @@ def with_export_types(self) -> Iterator[None]:
 
     def get_trivial_type(self, fdef: FuncDef) -> CallableType:
         """Generate a trivial callable type from a func def, with all Anys"""
+        # The Anys are marked as being from the suggestion engine
+        # since they need some special treatment (specifically,
+        # constraint generation ignores them.)
         return CallableType(
-            [AnyType(TypeOfAny.unannotated) for a in fdef.arg_kinds],
+            [AnyType(TypeOfAny.suggestion_engine) for a in fdef.arg_kinds],
             fdef.arg_kinds,
             fdef.arg_names,
-            # We call this a special form so that has_any_type doesn't consider it to be a real any
-            AnyType(TypeOfAny.special_form),
+            AnyType(TypeOfAny.suggestion_engine),
             self.builtin_type('builtins.function'))
 
     def get_starting_type(self, fdef: FuncDef) -> CallableType:
         if isinstance(fdef.type, CallableType):
-            return fdef.type
+            return make_suggestion_anys(fdef.type)
         else:
             return self.get_trivial_type(fdef)
 
@@ -304,9 +306,8 @@ def get_args(self, is_method: bool,
         types = []  # type: List[List[Type]]
         for i in range(len(base.arg_kinds)):
             # Make self args Any but this will get overriden somewhere in the checker
-            # We call this a special form so that has_any_type doesn't consider it to be a real any
             if i == 0 and is_method:
-                types.append([AnyType(TypeOfAny.special_form)])
+                types.append([AnyType(TypeOfAny.suggestion_engine)])
                 continue
 
             all_arg_types = []
@@ -383,13 +384,16 @@ def get_callsites(self, func: FuncDef) -> Tuple[List[Callsite], List[str]]:
 
         return collector_plugin.mystery_hits, errors
 
-    def filter_options(self, guesses: List[CallableType], is_method: bool) -> List[CallableType]:
+    def filter_options(
+        self, guesses: List[CallableType], is_method: bool, ignore_return: bool
+    ) -> List[CallableType]:
         """Apply any configured filters to the possible guesses.
 
         Currently the only option is filtering based on Any prevalance."""
         return [
             t for t in guesses
-            if self.flex_any is None or any_score_callable(t, is_method) >= self.flex_any
+            if self.flex_any is None
+            or any_score_callable(t, is_method, ignore_return) >= self.flex_any
         ]
 
     def find_best(self, func: FuncDef, guesses: List[CallableType]) -> Tuple[CallableType, int]:
@@ -426,7 +430,7 @@ def get_suggestion(self, mod: str, node: FuncDef) -> PyAnnotateSignature:
                 callsites,
                 uses,
             )
-        guesses = self.filter_options(guesses, is_method)
+        guesses = self.filter_options(guesses, is_method, ignore_return=True)
         best, _ = self.find_best(node, guesses)
 
         # Now try to find the return type!
@@ -439,7 +443,7 @@ def get_suggestion(self, mod: str, node: FuncDef) -> PyAnnotateSignature:
                 ret_types = [NoneType()]
 
         guesses = [best.copy_modified(ret_type=refine_type(best.ret_type, t)) for t in ret_types]
-        guesses = self.filter_options(guesses, is_method)
+        guesses = self.filter_options(guesses, is_method, ignore_return=False)
         best, errors = self.find_best(node, guesses)
 
         if self.no_errors and errors:
@@ -710,7 +714,7 @@ def any_score_type(ut: Type, arg_pos: bool) -> float:
     Higher is better, 1.0 is max
     """
     t = get_proper_type(ut)
-    if isinstance(t, AnyType) and t.type_of_any != TypeOfAny.special_form:
+    if isinstance(t, AnyType) and t.type_of_any != TypeOfAny.suggestion_engine:
         return 0
     if isinstance(t, NoneType) and arg_pos:
         return 0.5
@@ -727,14 +731,14 @@ def any_score_type(ut: Type, arg_pos: bool) -> float:
     return 1.0
 
 
-def any_score_callable(t: CallableType, is_method: bool) -> float:
+def any_score_callable(t: CallableType, is_method: bool, ignore_return: bool) -> float:
     # Ignore the first argument of methods
     scores = [any_score_type(x, arg_pos=True) for x in t.arg_types[int(is_method):]]
     # Return type counts twice (since it spreads type information), unless it is
     # None in which case it does not count at all. (Though it *does* still count
     # if there are no arguments.)
     if not isinstance(get_proper_type(t.ret_type), NoneType) or not scores:
-        ret = any_score_type(t.ret_type, arg_pos=False)
+        ret = 1.0 if ignore_return else any_score_type(t.ret_type, arg_pos=False)
         scores += [ret, ret]
 
     return sum(scores) / len(scores)
@@ -796,6 +800,9 @@ def visit_tuple_type(self, t: TupleType) -> str:
         s = self.list_str(t.items)
         return 'Tuple[{}]'.format(s)
 
+    def visit_uninhabited_type(self, t: UninhabitedType) -> str:
+        return "Any"
+
     def visit_typeddict_type(self, t: TypedDictType) -> str:
         return t.fallback.accept(self)
 
@@ -837,6 +844,26 @@ def visit_instance(self, t: Instance) -> Type:
             return super().visit_instance(t)
 
 
+TType = TypeVar('TType', bound=Type)
+
+
+def make_suggestion_anys(t: TType) -> TType:
+    """Make all anys in the type as coming from the suggestion engine.
+
+    This keeps those Anys from influencing constraint generation,
+    which allows us to do better when refining types.
+    """
+    return cast(TType, t.accept(MakeSuggestionAny()))
+
+
+class MakeSuggestionAny(TypeTranslator):
+    def visit_any(self, t: AnyType) -> Type:
+        return t.copy_modified(type_of_any=TypeOfAny.suggestion_engine)
+
+    def visit_type_alias_type(self, t: TypeAliasType) -> Type:
+        return t.copy_modified(args=[a.accept(self) for a in t.args])
+
+
 def generate_type_combinations(types: List[Type]) -> List[Type]:
     """Generate possible combinations of a list of types.
 
diff --git a/mypy/types.py b/mypy/types.py
@@ -103,6 +103,10 @@ class TypeOfAny:
     from_another_any = 7  # type: Final
     # Does this Any come from an implementation limitation/bug?
     implementation_artifact = 8  # type: Final
+    # Does this Any come from use in the suggestion engine?  This is
+    # used to ignore Anys inserted by the suggestion engine when
+    # generating constraints.
+    suggestion_engine = 9  # type: Final
 
 
 def deserialize_type(data: Union[JsonDict, str]) -> 'Type':
diff --git a/test-data/unit/fine-grained-suggest.test b/test-data/unit/fine-grained-suggest.test
@@ -908,6 +908,41 @@ z = foo(f(), g())
 (foo.List[Any], UNKNOWN) -> Tuple[foo.List[Any], Any]
 ==
 
+[case testSuggestDict]
+# suggest: foo.foo
+# suggest: foo.bar
+# suggest: foo.baz
+# suggest: foo.quux
+# suggest: foo.spam
+[file foo.py]
+from typing import List, Any
+
+def foo():
+    return {'x': 5}
+
+def bar():
+    return {}
+
+def baz() -> List[Any]:
+    return [{'x': 5}]
+
+def quux() -> List[Any]:
+    return [1]
+
+def spam(x):
+    pass
+
+spam({'x': 5})
+
+[builtins fixtures/dict.pyi]
+[out]
+() -> typing.Dict[str, int]
+() -> typing.Dict[Any, Any]
+() -> foo:List[typing.Dict[str, int]]
+() -> foo.List[int]
+(typing.Dict[str, int]) -> None
+==
+
 [case testSuggestWithErrors]
 # suggest: foo.foo
 [file foo.py]
