diff --git a/mypy/moduleinspect.py b/mypy/moduleinspect.py new file mode 100644 index 000000000000..9580e9b03b18 --- /dev/null +++ b/mypy/moduleinspect.py @@ -0,0 +1,175 @@ +"""Basic introspection of modules.""" + +from typing import List, Optional, Union +from types import ModuleType +from multiprocessing import Process, Queue +import importlib +import inspect +import os +import pkgutil +import queue +import sys + + +class ModuleProperties: + def __init__(self, + name: str, + file: Optional[str], + path: Optional[List[str]], + all: Optional[List[str]], + is_c_module: bool, + subpackages: List[str]) -> None: + self.name = name # __name__ attribute + self.file = file # __file__ attribute + self.path = path # __path__ attribute + self.all = all # __all__ attribute + self.is_c_module = is_c_module + self.subpackages = subpackages + + +def is_c_module(module: ModuleType) -> bool: + if module.__dict__.get('__file__') is None: + # Could be a namespace package. These must be handled through + # introspection, since there is no source file. + return True + return os.path.splitext(module.__dict__['__file__'])[-1] in ['.so', '.pyd'] + + +class InspectError(Exception): + pass + + +def get_package_properties(package_id: str) -> ModuleProperties: + """Use runtime introspection to get information about a module/package.""" + try: + package = importlib.import_module(package_id) + except BaseException as e: + raise InspectError(str(e)) + name = getattr(package, '__name__', None) + file = getattr(package, '__file__', None) + path = getattr(package, '__path__', None) # type: Optional[List[str]] + if not isinstance(path, list): + path = None + pkg_all = getattr(package, '__all__', None) + if pkg_all is not None: + try: + pkg_all = list(pkg_all) + except Exception: + pkg_all = None + is_c = is_c_module(package) + + if path is None: + # Object has no path; this means it's either a module inside a package + # (and thus no sub-packages), or it could be a C extension package. + if is_c: + # This is a C extension module, now get the list of all sub-packages + # using the inspect module + subpackages = [package.__name__ + "." + name + for name, val in inspect.getmembers(package) + if inspect.ismodule(val) + and val.__name__ == package.__name__ + "." + name] + else: + # It's a module inside a package. There's nothing else to walk/yield. + subpackages = [] + else: + all_packages = pkgutil.walk_packages(path, prefix=package.__name__ + ".", + onerror=lambda r: None) + subpackages = [qualified_name for importer, qualified_name, ispkg in all_packages] + return ModuleProperties(name=name, + file=file, + path=path, + all=pkg_all, + is_c_module=is_c, + subpackages=subpackages) + + +def worker(tasks: 'Queue[str]', + results: 'Queue[Union[str, ModuleProperties]]', + sys_path: List[str]) -> None: + """The main loop of a worker introspection process.""" + sys.path = sys_path + while True: + mod = tasks.get() + try: + prop = get_package_properties(mod) + except InspectError as e: + results.put(str(e)) + continue + results.put(prop) + + +class ModuleInspect: + """Perform runtime introspection of modules in a separate process. + + Reuse the process for multiple modules for efficiency. However, if there is an + error, retry using a fresh process to avoid cross-contamination of state between + modules. + + We use a separate process to isolate us from many side effects. For example, the + import of a module may kill the current process, and we want to recover from that. + + Always use in a with statement for proper clean-up: + + with ModuleInspect() as m: + p = m.get_package_properties('urllib.parse') + """ + + def __init__(self) -> None: + self._start() + + def _start(self) -> None: + self.tasks = Queue() # type: Queue[str] + self.results = Queue() # type: Queue[Union[ModuleProperties, str]] + self.proc = Process(target=worker, args=(self.tasks, self.results, sys.path)) + self.proc.start() + self.counter = 0 # Number of successfull roundtrips + + def close(self) -> None: + """Free any resources used.""" + self.proc.terminate() + + def get_package_properties(self, package_id: str) -> ModuleProperties: + """Return some properties of a module/package using runtime introspection. + + Raise InspectError if the target couldn't be imported. + """ + self.tasks.put(package_id) + res = self._get_from_queue() + if res is None: + # The process died; recover and report error. + self._start() + raise InspectError('Process died when importing %r' % package_id) + if isinstance(res, str): + # Error importing module + if self.counter > 0: + # Also try with a fresh process. Maybe one of the previous imports has + # corrupted some global state. + self.close() + self._start() + return self.get_package_properties(package_id) + raise InspectError(res) + self.counter += 1 + return res + + def _get_from_queue(self) -> Union[ModuleProperties, str, None]: + """Get value from the queue. + + Return the value read from the queue, or None if the process unexpectedly died. + """ + max_iter = 100 + n = 0 + while True: + if n == max_iter: + raise RuntimeError('Timeout waiting for subprocess') + try: + return self.results.get(timeout=0.05) + except queue.Empty: + if not self.proc.is_alive(): + return None + n += 1 + + def __enter__(self) -> 'ModuleInspect': + return self + + def __exit__(self, *args: object) -> None: + self.close() diff --git a/mypy/semanal.py b/mypy/semanal.py index 31ad131335a0..c18b44394002 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -865,19 +865,26 @@ def analyze_property_with_multi_part_definition(self, defn: OverloadedFuncDef) - defn.is_property = True items = defn.items first_item = cast(Decorator, defn.items[0]) - for item in items[1:]: - if isinstance(item, Decorator) and len(item.decorators) == 1: - node = item.decorators[0] - if isinstance(node, MemberExpr): - if node.name == 'setter': - # The first item represents the entire property. - first_item.var.is_settable_property = True - # Get abstractness from the original definition. - item.func.is_abstract = first_item.func.is_abstract - else: - self.fail("Decorated property not supported", item) + deleted_items = [] + for i, item in enumerate(items[1:]): if isinstance(item, Decorator): + if len(item.decorators) == 1: + node = item.decorators[0] + if isinstance(node, MemberExpr): + if node.name == 'setter': + # The first item represents the entire property. + first_item.var.is_settable_property = True + # Get abstractness from the original definition. + item.func.is_abstract = first_item.func.is_abstract + else: + self.fail("Decorated property not supported", item) item.func.accept(self) + else: + self.fail('Unexpected definition for property "{}"'.format(first_item.func.name), + item) + deleted_items.append(i + 1) + for i in reversed(deleted_items): + del items[i] def add_function_to_symbol_table(self, func: Union[FuncDef, OverloadedFuncDef]) -> None: if self.is_class_scope(): diff --git a/mypy/stubdoc.py b/mypy/stubdoc.py index 48c678e6af31..d2fd85914009 100644 --- a/mypy/stubdoc.py +++ b/mypy/stubdoc.py @@ -5,7 +5,6 @@ """ import re import io -import sys import contextlib import tokenize @@ -18,14 +17,25 @@ Sig = Tuple[str, str] +_TYPE_RE = re.compile(r'^[a-zA-Z_][\w\[\], ]*(\.[a-zA-Z_][\w\[\], ]*)*$') # type: Final +_ARG_NAME_RE = re.compile(r'\**[A-Za-z_][A-Za-z0-9_]*$') # type: Final + + +def is_valid_type(s: str) -> bool: + """Try to determine whether a string might be a valid type annotation.""" + if s in ('True', 'False', 'retval'): + return False + if ',' in s and '[' not in s: + return False + return _TYPE_RE.match(s) is not None + + class ArgSig: """Signature info for a single argument.""" - _TYPE_RE = re.compile(r'^[a-zA-Z_][\w\[\], ]*(\.[a-zA-Z_][\w\[\], ]*)*$') # type: Final - def __init__(self, name: str, type: Optional[str] = None, default: bool = False): self.name = name - if type and not self._TYPE_RE.match(type): + if type and not is_valid_type(type): raise ValueError("Invalid type: " + type) self.type = type # Does this argument have a default value? @@ -60,7 +70,8 @@ def __eq__(self, other: Any) -> bool: class DocStringParser: - """Parse function signstures in documentation.""" + """Parse function signatures in documentation.""" + def __init__(self, function_name: str) -> None: # Only search for signatures of function with this name. self.function_name = function_name @@ -76,7 +87,7 @@ def __init__(self, function_name: str) -> None: self.signatures = [] # type: List[FunctionSig] def add_token(self, token: tokenize.TokenInfo) -> None: - """Process next token fro the token stream.""" + """Process next token from the token stream.""" if (token.type == tokenize.NAME and token.string == self.function_name and self.state[-1] == STATE_INIT): self.state.append(STATE_FUNCTION_NAME) @@ -129,6 +140,10 @@ def add_token(self, token: tokenize.TokenInfo) -> None: self.state.pop() elif self.state[-1] == STATE_ARGUMENT_LIST: self.arg_name = self.accumulator + if not _ARG_NAME_RE.match(self.arg_name): + # Invalid argument name. + self.reset() + return if token.string == ')': self.state.pop() @@ -152,6 +167,9 @@ def add_token(self, token: tokenize.TokenInfo) -> None: elif (token.type in (tokenize.NEWLINE, tokenize.ENDMARKER) and self.state[-1] in (STATE_INIT, STATE_RETURN_VALUE)): if self.state[-1] == STATE_RETURN_VALUE: + if not is_valid_type(self.accumulator): + self.reset() + return self.ret_type = self.accumulator self.accumulator = "" self.state.pop() @@ -166,6 +184,12 @@ def add_token(self, token: tokenize.TokenInfo) -> None: else: self.accumulator += token.string + def reset(self) -> None: + self.state = [STATE_INIT] + self.args = [] + self.found = False + self.accumulator = "" + def get_signatures(self) -> List[FunctionSig]: """Return sorted copy of the list of signatures found so far.""" def has_arg(name: str, signature: FunctionSig) -> bool: @@ -211,13 +235,7 @@ def is_unique_args(sig: FunctionSig) -> bool: """return true if function argument names are unique""" return len(sig.args) == len(set((arg.name for arg in sig.args))) - # Warn about invalid signatures - invalid_sigs = [sig for sig in sigs if not is_unique_args(sig)] - if invalid_sigs: - print("Warning: Invalid signatures found:", file=sys.stderr) - print("\n".join(str(sig) for sig in invalid_sigs), file=sys.stderr) - - # return only signatures, that have unique argument names. mypy fails on non-uqniue arg names + # Return only signatures that have unique argument names. Mypy fails on non-uniqnue arg names. return [sig for sig in sigs if is_unique_args(sig)] diff --git a/mypy/stubgen.py b/mypy/stubgen.py index f7edbdf2d4c4..214dc7763819 100755 --- a/mypy/stubgen.py +++ b/mypy/stubgen.py @@ -71,7 +71,7 @@ TupleExpr, ListExpr, ComparisonExpr, CallExpr, IndexExpr, EllipsisExpr, ClassDef, MypyFile, Decorator, AssignmentStmt, TypeInfo, IfStmt, ImportAll, ImportFrom, Import, FuncDef, FuncBase, TempNode, Block, - ARG_POS, ARG_STAR, ARG_STAR2, ARG_NAMED, ARG_NAMED_OPT + Statement, OverloadedFuncDef, ARG_POS, ARG_STAR, ARG_STAR2, ARG_NAMED, ARG_NAMED_OPT ) from mypy.stubgenc import generate_stub_for_c_module from mypy.stubutil import ( @@ -90,6 +90,7 @@ from mypy.build import build from mypy.errors import CompileError, Errors from mypy.traverser import has_return_statement +from mypy.moduleinspect import ModuleInspect # Common ways of naming package containing vendored modules. @@ -97,22 +98,56 @@ 'packages', 'vendor', 'vendored', -] + '_vendor', + '_vendored_packages', +] # type: Final # Avoid some file names that are unnecessary or likely to cause trouble (\n for end of path). BLACKLIST = [ '/six.py\n', # Likely vendored six; too dynamic for us to handle '/vendored/', # Vendored packages '/vendor/', # Vendored packages -] + '/_vendor/', + '/_vendored_packages/', +] # type: Final # Special-cased names that are implicitly exported from the stub (from m import y as y). EXTRA_EXPORTED = { 'pyasn1_modules.rfc2437.univ', 'pyasn1_modules.rfc2459.char', 'pyasn1_modules.rfc2459.univ', - 'elasticsearch.client.utils._make_path', -} +} # type: Final + +# These names should be omitted from generated stubs. +IGNORED_DUNDERS = { + '__all__', + '__author__', + '__version__', + '__about__', + '__copyright__', + '__email__', + '__license__', + '__summary__', + '__title__', + '__uri__', + '__str__', + '__repr__', + '__getstate__', + '__setstate__', + '__slots__', +} # type: Final + +# These methods are expected to always return a non-trivial value. +METHODS_WITH_RETURN_VALUE = { + '__ne__', + '__eq__', + '__lt__', + '__le__', + '__gt__', + '__ge__', + '__hash__', + '__iter__', +} # type: Final class Options: @@ -134,7 +169,8 @@ def __init__(self, packages: List[str], files: List[str], verbose: bool, - quiet: bool) -> None: + quiet: bool, + export_less: bool) -> None: # See parse_options for descriptions of the flags. self.pyversion = pyversion self.no_import = no_import @@ -151,6 +187,7 @@ def __init__(self, self.files = files self.verbose = verbose self.quiet = quiet + self.export_less = export_less class StubSource(BuildSource): @@ -446,8 +483,11 @@ def add_ref(self, fullname: str) -> None: class StubGenerator(mypy.traverser.TraverserVisitor): """Generate stub text from a mypy AST.""" - def __init__(self, _all_: Optional[List[str]], pyversion: Tuple[int, int], - include_private: bool = False, analyzed: bool = False) -> None: + def __init__(self, + _all_: Optional[List[str]], pyversion: Tuple[int, int], + include_private: bool = False, + analyzed: bool = False, + export_less: bool = False) -> None: # Best known value of __all__. self._all_ = _all_ self._output = [] # type: List[str] @@ -465,15 +505,21 @@ def __init__(self, _all_: Optional[List[str]], pyversion: Tuple[int, int], self.import_tracker = ImportTracker() # Was the tree semantically analysed before? self.analyzed = analyzed + # Disable implicit exports of package-internal imports? + self.export_less = export_less # Add imports that could be implicitly generated self.import_tracker.add_import_from("collections", [("namedtuple", None)]) # Names in __all__ are required for name in _all_ or (): - self.import_tracker.reexport(name) + if name not in IGNORED_DUNDERS: + self.import_tracker.reexport(name) self.defined_names = set() # type: Set[str] + # Short names of methods defined in the body of the current class + self.method_names = set() # type: Set[str] def visit_mypy_file(self, o: MypyFile) -> None: - self.module = o.fullname + self.module = o.fullname # Current module being processed + self.path = o.path self.defined_names = find_defined_names(o) self.referenced_names = find_referenced_names(o) typing_imports = ["Any", "Optional", "TypeVar"] @@ -494,17 +540,19 @@ def visit_mypy_file(self, o: MypyFile) -> None: self.add('# %s\n' % name) def visit_func_def(self, o: FuncDef, is_abstract: bool = False) -> None: - if self.is_private_name(o.name, o.fullname): - return - if self.is_not_in_all(o.name): - return - if self.is_recorded_name(o.name): + if (self.is_private_name(o.name, o.fullname) + or self.is_not_in_all(o.name) + or self.is_recorded_name(o.name)): + self.clear_decorators() return if not self._indent and self._state not in (EMPTY, FUNC) and not o.is_awaitable_coroutine: self.add('\n') if not self.is_top_level(): self_inits = find_self_initializers(o) for init, value in self_inits: + if init in self.method_names: + # Can't have both an attribute and a method/property with the same name. + continue init_code = self.get_init(init, value) if init_code: self.add(init_code) @@ -556,8 +604,9 @@ def visit_func_def(self, o: FuncDef, is_abstract: bool = False) -> None: retname = None if isinstance(o.unanalyzed_type, CallableType): retname = self.print_annotation(o.unanalyzed_type.ret_type) - elif isinstance(o, FuncDef) and o.is_abstract: - # Always assume abstract methods return Any unless explicitly annotated. + elif isinstance(o, FuncDef) and (o.is_abstract or o.name in METHODS_WITH_RETURN_VALUE): + # Always assume abstract methods return Any unless explicitly annotated. Also + # some dunder methods should not have a None return type. retname = self.typing_name('Any') self.add_typing_import("Any") elif o.name == '__init__' or not has_return_statement(o) and not is_abstract: @@ -649,6 +698,7 @@ def process_member_expr_decorator(self, expr: MemberExpr, context: Decorator) -> return is_abstract def visit_class_def(self, o: ClassDef) -> None: + self.method_names = find_method_names(o.defs.body) sep = None # type: Optional[int] if not self._indent and self._state != EMPTY: sep = len(self._output) @@ -683,6 +733,7 @@ def visit_class_def(self, o: ClassDef) -> None: self._state = EMPTY_CLASS else: self._state = CLASS + self.method_names = set() def get_base_types(self, cdef: ClassDef) -> List[str]: """Get list of base classes for a class.""" @@ -754,7 +805,6 @@ def is_namedtuple(self, expr: Expression) -> bool: (isinstance(callee, MemberExpr) and callee.name == 'namedtuple')) def process_namedtuple(self, lvalue: NameExpr, rvalue: CallExpr) -> None: - self.import_tracker.require_name('namedtuple') if self._state != EMPTY: self.add('\n') name = repr(getattr(rvalue.args[0], 'value', ERROR_MARKER)) @@ -764,8 +814,11 @@ def process_namedtuple(self, lvalue: NameExpr, rvalue: CallExpr) -> None: list_items = cast(List[StrExpr], rvalue.args[1].items) items = '[%s]' % ', '.join(repr(item.value) for item in list_items) else: - items = ERROR_MARKER - self.add('%s = namedtuple(%s, %s)\n' % (lvalue.name, name, items)) + self.add('%s%s: Any' % (self._indent, lvalue.name)) + self.import_tracker.require_name('Any') + return + self.import_tracker.require_name('namedtuple') + self.add('%s%s = namedtuple(%s, %s)\n' % (self._indent, lvalue.name, name, items)) self._state = CLASS def is_alias_expression(self, expr: Expression, top_level: bool = True) -> bool: @@ -834,7 +887,15 @@ def visit_import_all(self, o: ImportAll) -> None: def visit_import_from(self, o: ImportFrom) -> None: exported_names = set() # type: Set[str] import_names = [] - module, relative = self.translate_module_name(o.id, o.relative) + module, relative = translate_module_name(o.id, o.relative) + if self.module: + full_module, ok = mypy.util.correct_relative_import( + self.module, relative, module, self.path.endswith('.__init__.py') + ) + if not ok: + full_module = module + else: + full_module = module if module == '__future__': return # Not preserved for name, as_name in o.names: @@ -844,13 +905,29 @@ def visit_import_from(self, o: ImportFrom) -> None: continue exported = False if as_name is None and self.module and (self.module + '.' + name) in EXTRA_EXPORTED: + # Special case certain names that should be exported, against our general rules. exported = True - if (as_name is None and name not in self.referenced_names and not self._all_ + is_private = self.is_private_name(name, full_module + '.' + name) + if (as_name is None + and name not in self.referenced_names + and (not self._all_ or name in IGNORED_DUNDERS) + and not is_private and module not in ('abc', 'typing', 'asyncio')): # An imported name that is never referenced in the module is assumed to be # exported, unless there is an explicit __all__. Note that we need to special # case 'abc' since some references are deleted during semantic analysis. exported = True + top_level = full_module.split('.')[0] + if (as_name is None + and not self.export_less + and (not self._all_ or name in IGNORED_DUNDERS) + and self.module + and not is_private + and top_level in (self.module.split('.')[0], + '_' + self.module.split('.')[0])): + # Export imports from the same package, since we can't reliably tell whether they + # are part of the public API. + exported = True if exported: self.import_tracker.reexport(name) as_name = name @@ -863,25 +940,8 @@ def visit_import_from(self, o: ImportFrom) -> None: if self._all_: # Include import froms that import names defined in __all__. names = [name for name, alias in o.names - if name in self._all_ and alias is None] + if name in self._all_ and alias is None and name not in IGNORED_DUNDERS] exported_names.update(names) - else: - # Include import from targets that import from a submodule of a package. - if relative: - sub_names = [name for name, alias in o.names - if alias is None] - exported_names.update(sub_names) - if module: - for name in sub_names: - self.import_tracker.require_name(name) - - def translate_module_name(self, module: str, relative: int) -> Tuple[str, int]: - for pkg in VENDOR_PACKAGES: - for alt in 'six', 'six.moves': - if (module.endswith('.{}.{}'.format(pkg, alt)) - or (module == '{}.{}'.format(pkg, alt) and relative)): - return alt, 0 - return module, relative def visit_import(self, o: Import) -> None: for id, as_id in o.ids: @@ -981,14 +1041,7 @@ def is_private_name(self, name: str, fullname: Optional[str] = None) -> bool: if fullname in EXTRA_EXPORTED: return False return name.startswith('_') and (not name.endswith('__') - or name in ('__all__', - '__author__', - '__version__', - '__str__', - '__repr__', - '__getstate__', - '__setstate__', - '__slots__')) + or name in IGNORED_DUNDERS) def is_private_member(self, fullname: str) -> bool: parts = fullname.split('.') @@ -1041,6 +1094,20 @@ def is_recorded_name(self, name: str) -> bool: return self.is_top_level() and name in self._toplevel_names +def find_method_names(defs: List[Statement]) -> Set[str]: + # TODO: Traverse into nested definitions + result = set() + for defn in defs: + if isinstance(defn, FuncDef): + result.add(defn.name) + elif isinstance(defn, Decorator): + result.add(defn.func.name) + elif isinstance(defn, OverloadedFuncDef): + for item in defn.items: + result.update(find_method_names([item])) + return result + + class SelfTraverser(mypy.traverser.TraverserVisitor): def __init__(self) -> None: self.results = [] # type: List[Tuple[str, Expression]] @@ -1134,52 +1201,73 @@ def find_module_paths_using_imports(modules: List[str], This function uses runtime Python imports to get the information. """ - py_modules = [] # type: List[StubSource] - c_modules = [] # type: List[StubSource] - found = list(walk_packages(packages, verbose)) - modules = modules + found - modules = [mod for mod in modules if not is_test_module(mod)] # We don't want to run any tests - for mod in modules: - try: - if pyversion[0] == 2: - result = find_module_path_and_all_py2(mod, interpreter) + with ModuleInspect() as inspect: + py_modules = [] # type: List[StubSource] + c_modules = [] # type: List[StubSource] + found = list(walk_packages(inspect, packages, verbose)) + modules = modules + found + modules = [mod + for mod in modules + if not is_non_library_module(mod)] # We don't want to run any tests or scripts + for mod in modules: + try: + if pyversion[0] == 2: + result = find_module_path_and_all_py2(mod, interpreter) + else: + result = find_module_path_and_all_py3(inspect, mod, verbose) + except CantImport as e: + tb = traceback.format_exc() + if verbose: + sys.stdout.write(tb) + if not quiet: + report_missing(mod, e.message, tb) + continue + if not result: + c_modules.append(StubSource(mod)) else: - result = find_module_path_and_all_py3(mod, verbose) - except CantImport as e: - tb = traceback.format_exc() - if verbose: - sys.stdout.write(tb) - if not quiet: - report_missing(mod, e.message, tb) - continue - if not result: - c_modules.append(StubSource(mod)) - else: - path, runtime_all = result - py_modules.append(StubSource(mod, path, runtime_all)) - return py_modules, c_modules + path, runtime_all = result + py_modules.append(StubSource(mod, path, runtime_all)) + return py_modules, c_modules -def is_test_module(module: str) -> bool: - """Does module look like a test module?""" +def is_non_library_module(module: str) -> bool: + """Does module look like a test module or a script?""" if module.endswith(( '.tests', '.test', '.testing', '_tests', - '.conftest', + '_test_suite', 'test_util', 'test_utils', 'test_base', + '.__main__', + '.conftest', # Used by pytest + '.setup', # Typically an install script )): return True if module.split('.')[-1].startswith('test_'): return True - if '.tests.' in module or '.test.' in module or '.testing.' in module: + if ('.tests.' in module + or '.test.' in module + or '.testing.' in module + or '.SelfTest.' in module): return True return False +def translate_module_name(module: str, relative: int) -> Tuple[str, int]: + for pkg in VENDOR_PACKAGES: + for alt in 'six.moves', 'six': + substr = '{}.{}'.format(pkg, alt) + if (module.endswith('.' + substr) + or (module == substr and relative)): + return alt, 0 + if '.' + substr + '.' in module: + return alt + '.' + module.partition('.' + substr + '.')[2], 0 + return module, relative + + def find_module_paths_using_search(modules: List[str], packages: List[str], search_path: List[str], pyversion: Tuple[int, int]) -> List[StubSource]: @@ -1205,7 +1293,7 @@ def find_module_paths_using_search(modules: List[str], packages: List[str], sources = [StubSource(m.module, m.path) for m in p_result] result.extend(sources) - result = [m for m in result if not is_test_module(m.module)] + result = [m for m in result if not is_non_library_module(m.module)] return result @@ -1236,6 +1324,7 @@ def parse_source_file(mod: StubSource, mypy_options: MypyOptions) -> None: errors = Errors() mod.ast = mypy.parse.parse(source, fnam=mod.path, module=mod.module, errors=errors, options=mypy_options) + mod.ast._fullname = mod.module if errors.is_blockers(): # Syntax error! for m in errors.new_messages(): @@ -1244,10 +1333,14 @@ def parse_source_file(mod: StubSource, mypy_options: MypyOptions) -> None: def generate_asts_for_modules(py_modules: List[StubSource], - parse_only: bool, mypy_options: MypyOptions) -> None: + parse_only: bool, + mypy_options: MypyOptions, + verbose: bool) -> None: """Use mypy to parse (and optionally analyze) source files.""" if not py_modules: return # Nothing to do here, but there may be C modules + if verbose: + print('Processing %d files...' % len(py_modules)) if parse_only: for mod in py_modules: parse_source_file(mod, mypy_options) @@ -1269,7 +1362,8 @@ def generate_stub_from_ast(mod: StubSource, target: str, parse_only: bool = False, pyversion: Tuple[int, int] = defaults.PYTHON3_VERSION, - include_private: bool = False) -> None: + include_private: bool = False, + export_less: bool = False) -> None: """Use analysed (or just parsed) AST to generate type stub for single file. If directory for target doesn't exist it will created. Existing stub @@ -1278,7 +1372,8 @@ def generate_stub_from_ast(mod: StubSource, gen = StubGenerator(mod.runtime_all, pyversion=pyversion, include_private=include_private, - analyzed=not parse_only) + analyzed=not parse_only, + export_less=export_less) assert mod.ast is not None, "This function must be used only with analyzed modules" mod.ast.accept(gen) @@ -1319,7 +1414,7 @@ def generate_stubs(options: Options) -> None: sigs, class_sigs = collect_docs_signatures(options.doc_dir) # Use parsed sources to generate stubs for Python modules. - generate_asts_for_modules(py_modules, options.parse_only, mypy_opts) + generate_asts_for_modules(py_modules, options.parse_only, mypy_opts, options.verbose) files = [] for mod in py_modules: assert mod.path is not None, "Not found module was not skipped" @@ -1333,7 +1428,8 @@ def generate_stubs(options: Options) -> None: with generate_guarded(mod.module, target, options.ignore_errors, options.verbose): generate_stub_from_ast(mod, target, options.parse_only, options.pyversion, - options.include_private) + options.include_private, + options.export_less) # Separately analyse C modules using different logic. for mod in c_modules: @@ -1385,6 +1481,9 @@ def parse_options(args: List[str]) -> Options: parser.add_argument('--include-private', action='store_true', help="generate stubs for objects and members considered private " "(single leading underscore and no trailing underscores)") + parser.add_argument('--export-less', action='store_true', + help=("don't implicitly export all names imported from other modules " + "in the same package")) parser.add_argument('-v', '--verbose', action='store_true', help="show more verbose messages") parser.add_argument('-q', '--quiet', action='store_true', @@ -1437,7 +1536,8 @@ def parse_options(args: List[str]) -> Options: packages=ns.packages, files=ns.files, verbose=ns.verbose, - quiet=ns.quiet) + quiet=ns.quiet, + export_less=ns.export_less) def main() -> None: diff --git a/mypy/stubgenc.py b/mypy/stubgenc.py index 7de5f450aa6d..a9c87da7e95d 100755 --- a/mypy/stubgenc.py +++ b/mypy/stubgenc.py @@ -11,7 +11,7 @@ from typing import List, Dict, Tuple, Optional, Mapping, Any, Set from types import ModuleType -from mypy.stubutil import is_c_module +from mypy.moduleinspect import is_c_module from mypy.stubdoc import ( infer_sig_from_docstring, infer_prop_type_from_docstring, ArgSig, infer_arg_sig_from_docstring, FunctionSig @@ -169,6 +169,8 @@ def generate_c_function_stub(module: ModuleType, arg_def = self_var else: arg_def = arg.name + if arg_def == 'None': + arg_def = '_none' # None is not a valid argument name if arg.type: arg_def += ": " + strip_or_import(arg.type, module, imports) @@ -198,16 +200,16 @@ def strip_or_import(typ: str, module: ModuleType, imports: List[str]) -> str: module: in which this type is used imports: list of import statements (may be modified during the call) """ - arg_type = typ - if module and typ.startswith(module.__name__): - arg_type = typ[len(module.__name__) + 1:] + stripped_type = typ + if module and typ.startswith(module.__name__ + '.'): + stripped_type = typ[len(module.__name__) + 1:] elif '.' in typ: - arg_module = arg_type[:arg_type.rindex('.')] + arg_module = typ[:typ.rindex('.')] if arg_module == 'builtins': - arg_type = arg_type[len('builtins') + 1:] + stripped_type = typ[len('builtins') + 1:] else: imports.append('import %s' % (arg_module,)) - return arg_type + return stripped_type def generate_c_property_stub(name: str, obj: object, output: List[str], readonly: bool) -> None: @@ -250,11 +252,6 @@ def generate_c_type_stub(module: ModuleType, if is_c_method(value) or is_c_classmethod(value): done.add(attr) if not is_skipped_attribute(attr): - if is_c_classmethod(value): - methods.append('@classmethod') - self_var = 'cls' - else: - self_var = 'self' if attr == '__new__': # TODO: We should support __new__. if '__init__' in obj_dict: @@ -263,6 +260,11 @@ def generate_c_type_stub(module: ModuleType, # better signature than __init__() ? continue attr = '__init__' + if is_c_classmethod(value): + methods.append('@classmethod') + self_var = 'cls' + else: + self_var = 'self' generate_c_function_stub(module, attr, value, methods, imports=imports, self_var=self_var, sigs=sigs, class_name=class_name, class_sigs=class_sigs) @@ -294,7 +296,7 @@ def generate_c_type_stub(module: ModuleType, if bases: bases_str = '(%s)' % ', '.join( strip_or_import( - '%s.%s' % (base.__module__, base.__name__), + get_type_fullname(base), module, imports ) for base in bases @@ -313,6 +315,10 @@ def generate_c_type_stub(module: ModuleType, output.append(' %s' % prop) +def get_type_fullname(typ: type) -> str: + return '%s.%s' % (typ.__module__, typ.__name__) + + def method_name_sort_key(name: str) -> Tuple[int, str]: """Sort methods in classes in a typical order. diff --git a/mypy/stubutil.py b/mypy/stubutil.py index ad3c63d8f10a..bf5de6d607e2 100644 --- a/mypy/stubutil.py +++ b/mypy/stubutil.py @@ -2,23 +2,19 @@ import sys import os.path -import inspect import json -import pkgutil -import importlib import subprocess import re -from types import ModuleType from contextlib import contextmanager from typing import Optional, Tuple, List, Iterator, Union from typing_extensions import overload +from mypy.moduleinspect import ModuleInspect, InspectError -# Modules that may fail when imported, or that may have side effects. -NOT_IMPORTABLE_MODULES = { - 'tensorflow.tools.pip_package.setup', -} + +# Modules that may fail when imported, or that may have side effects (fully qualified). +NOT_IMPORTABLE_MODULES = () class CantImport(Exception): @@ -27,14 +23,6 @@ def __init__(self, module: str, message: str): self.message = message -def is_c_module(module: ModuleType) -> bool: - if module.__dict__.get('__file__') is None: - # Could be a namespace package. These must be handled through - # introspection, since there is no source file. - return True - return os.path.splitext(module.__dict__['__file__'])[-1] in ['.so', '.pyd'] - - def default_py2_interpreter() -> str: """Find a system Python 2 interpreter. @@ -52,14 +40,16 @@ def default_py2_interpreter() -> str: "please use the --python-executable option") -def walk_packages(packages: List[str], verbose: bool = False) -> Iterator[str]: +def walk_packages(inspect: ModuleInspect, + packages: List[str], + verbose: bool = False) -> Iterator[str]: """Iterates through all packages and sub-packages in the given list. - This uses runtime imports to find both Python and C modules. For Python packages - we simply pass the __path__ attribute to pkgutil.walk_packages() to get the content - of the package (all subpackages and modules). However, packages in C extensions - do not have this attribute, so we have to roll out our own logic: recursively find - all modules imported in the package that have matching names. + This uses runtime imports (in another process) to find both Python and C modules. + For Python packages we simply pass the __path__ attribute to pkgutil.walk_packages() to + get the content of the package (all subpackages and modules). However, packages in C + extensions do not have this attribute, so we have to roll out our own logic: recursively + find all modules imported in the package that have matching names. """ for package_name in packages: if package_name in NOT_IMPORTABLE_MODULES: @@ -68,36 +58,22 @@ def walk_packages(packages: List[str], verbose: bool = False) -> Iterator[str]: if verbose: print('Trying to import %r for runtime introspection' % package_name) try: - package = importlib.import_module(package_name) - except Exception: + prop = inspect.get_package_properties(package_name) + except InspectError: report_missing(package_name) continue - yield package.__name__ - # get the path of the object (needed by pkgutil) - path = getattr(package, '__path__', None) - if path is None: - # Object has no path; this means it's either a module inside a package - # (and thus no sub-packages), or it could be a C extension package. - if is_c_module(package): - # This is a C extension module, now get the list of all sub-packages - # using the inspect module - subpackages = [package.__name__ + "." + name - for name, val in inspect.getmembers(package) - if inspect.ismodule(val) - and val.__name__ == package.__name__ + "." + name] - # Recursively iterate through the subpackages - for submodule in walk_packages(subpackages, verbose): - yield submodule - # It's a module inside a package. There's nothing else to walk/yield. + yield prop.name + if prop.is_c_module: + # Recursively iterate through the subpackages + for submodule in walk_packages(inspect, prop.subpackages, verbose): + yield submodule else: - all_packages = pkgutil.walk_packages(path, prefix=package.__name__ + ".", - onerror=lambda r: None) - for importer, qualified_name, ispkg in all_packages: - yield qualified_name + for submodule in prop.subpackages: + yield submodule def find_module_path_and_all_py2(module: str, - interpreter: str) -> Optional[Tuple[str, + interpreter: str) -> Optional[Tuple[Optional[str], Optional[List[str]]]]: """Return tuple (module path, module __all__) for a Python 2 module. @@ -112,7 +88,10 @@ def find_module_path_and_all_py2(module: str, try: output_bytes = subprocess.check_output(cmd_template % code, shell=True) except subprocess.CalledProcessError as e: - raise CantImport(module, str(e)) + path = find_module_path_using_py2_sys_path(module, interpreter) + if path is None: + raise CantImport(module, str(e)) + return path, None output = output_bytes.decode('ascii').strip().splitlines() module_path = output[0] if not module_path.endswith(('.py', '.pyc', '.pyo')): @@ -124,8 +103,38 @@ def find_module_path_and_all_py2(module: str, return module_path, module_all -def find_module_path_and_all_py3(module: str, - verbose: bool) -> Optional[Tuple[str, Optional[List[str]]]]: +def find_module_path_using_py2_sys_path(module: str, + interpreter: str) -> Optional[str]: + """Try to find the path of a .py file for a module using Python 2 sys.path. + + Return None if no match was found. + """ + out = subprocess.run( + [interpreter, '-c', 'import sys; import json; print(json.dumps(sys.path))'], + check=True, + stdout=subprocess.PIPE + ).stdout + sys_path = json.loads(out.decode('utf-8')) + return find_module_path_using_sys_path(module, sys_path) + + +def find_module_path_using_sys_path(module: str, sys_path: List[str]) -> Optional[str]: + relative_candidates = ( + module.replace('.', '/') + '.py', + os.path.join(module.replace('.', '/'), '__init__.py') + ) + for base in sys_path: + for relative_path in relative_candidates: + path = os.path.join(base, relative_path) + if os.path.isfile(path): + return path + return None + + +def find_module_path_and_all_py3(inspect: ModuleInspect, + module: str, + verbose: bool) -> Optional[Tuple[Optional[str], + Optional[List[str]]]]: """Find module and determine __all__ for a Python 3 module. Return None if the module is a C module. Return (module_path, __all__) if @@ -138,15 +147,16 @@ def find_module_path_and_all_py3(module: str, if verbose: print('Trying to import %r for runtime introspection' % module) try: - mod = importlib.import_module(module) - except Exception as e: - raise CantImport(module, str(e)) - if is_c_module(mod): + mod = inspect.get_package_properties(module) + except InspectError as e: + # Fall back to finding the module using sys.path. + path = find_module_path_using_sys_path(module, sys.path) + if path is None: + raise CantImport(module, str(e)) + return path, None + if mod.is_c_module: return None - module_all = getattr(mod, '__all__', None) - if module_all is not None: - module_all = list(module_all) - return mod.__file__, module_all + return mod.file, mod.all @contextmanager diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py index 4d3eaef112c5..dd5f56ba9243 100644 --- a/mypy/test/teststubgen.py +++ b/mypy/test/teststubgen.py @@ -4,29 +4,31 @@ import sys import tempfile import re +import unittest from types import ModuleType from typing import Any, List, Tuple, Optional from mypy.test.helpers import ( - Suite, assert_equal, assert_string_arrays_equal, local_sys_path_set + assert_equal, assert_string_arrays_equal, local_sys_path_set ) from mypy.test.data import DataSuite, DataDrivenTestCase from mypy.errors import CompileError from mypy.stubgen import ( generate_stubs, parse_options, Options, collect_build_targets, - mypy_options, is_blacklisted_path, is_test_module + mypy_options, is_blacklisted_path, is_non_library_module ) from mypy.stubutil import walk_packages, remove_misplaced_type_comments, common_dir_prefix from mypy.stubgenc import generate_c_type_stub, infer_method_sig, generate_c_function_stub from mypy.stubdoc import ( parse_signature, parse_all_signatures, build_signature, find_unique_signatures, infer_sig_from_docstring, infer_prop_type_from_docstring, FunctionSig, ArgSig, - infer_arg_sig_from_docstring + infer_arg_sig_from_docstring, is_valid_type ) +from mypy.moduleinspect import ModuleInspect, InspectError -class StubgenCmdLineSuite(Suite): +class StubgenCmdLineSuite(unittest.TestCase): """Test cases for processing command-line options and finding files.""" def test_files_found(self) -> None: @@ -68,6 +70,7 @@ def test_packages_found(self) -> None: finally: os.chdir(current) + @unittest.skipIf(sys.platform == 'win32', "clean up fails on Windows") def test_module_not_found(self) -> None: current = os.getcwd() captured_output = io.StringIO() @@ -78,7 +81,7 @@ def test_module_not_found(self) -> None: self.make_file(tmp, 'mymodule.py', content='import a') opts = parse_options(['-m', 'mymodule']) py_mods, c_mods = collect_build_targets(opts, mypy_options(opts)) - self.assertRegex(captured_output.getvalue(), "No module named 'a'") + assert captured_output.getvalue() == '' finally: sys.stdout = sys.__stdout__ os.chdir(current) @@ -93,27 +96,28 @@ def run(self, result: Optional[Any] = None) -> Optional[Any]: return super().run(result) -class StubgenCliParseSuite(Suite): +class StubgenCliParseSuite(unittest.TestCase): def test_walk_packages(self) -> None: - assert_equal( - set(walk_packages(["mypy.errors"])), - {"mypy.errors"}) - - assert_equal( - set(walk_packages(["mypy.errors", "mypy.stubgen"])), - {"mypy.errors", "mypy.stubgen"}) - - all_mypy_packages = set(walk_packages(["mypy"])) - self.assertTrue(all_mypy_packages.issuperset({ - "mypy", - "mypy.errors", - "mypy.stubgen", - "mypy.test", - "mypy.test.helpers", - })) - - -class StubgenUtilSuite(Suite): + with ModuleInspect() as m: + assert_equal( + set(walk_packages(m, ["mypy.errors"])), + {"mypy.errors"}) + + assert_equal( + set(walk_packages(m, ["mypy.errors", "mypy.stubgen"])), + {"mypy.errors", "mypy.stubgen"}) + + all_mypy_packages = set(walk_packages(m, ["mypy"])) + self.assertTrue(all_mypy_packages.issuperset({ + "mypy", + "mypy.errors", + "mypy.stubgen", + "mypy.test", + "mypy.test.helpers", + })) + + +class StubgenUtilSuite(unittest.TestCase): """Unit tests for stubgen utility functions.""" def test_parse_signature(self) -> None: @@ -281,6 +285,12 @@ def test_infer_prop_type_from_docstring(self) -> None: 'Tuple[int, int]') assert_equal(infer_prop_type_from_docstring('\nstr: A string.'), None) + def test_infer_sig_from_docstring_square_brackets(self) -> None: + assert infer_sig_from_docstring( + 'fetch_row([maxrows, how]) -- Fetches stuff', + 'fetch_row', + ) == [] + def test_remove_misplaced_type_comments_1(self) -> None: good = """ \u1234 @@ -446,7 +456,7 @@ def test_common_dir_prefix(self) -> None: assert common_dir_prefix(['foo/bar/x.pyi', 'foo/bar/zar/y.pyi']) == 'foo/bar' -class StubgenHelpersSuite(Suite): +class StubgenHelpersSuite(unittest.TestCase): def test_is_blacklisted_path(self) -> None: assert not is_blacklisted_path('foo/bar.py') assert not is_blacklisted_path('foo.py') @@ -457,29 +467,34 @@ def test_is_blacklisted_path(self) -> None: assert is_blacklisted_path('foo/vendored/bar/thing.py') assert is_blacklisted_path('foo/six.py') - def test_is_test_module(self) -> None: - assert not is_test_module('foo') - assert not is_test_module('foo.bar') + def test_is_non_library_module(self) -> None: + assert not is_non_library_module('foo') + assert not is_non_library_module('foo.bar') # The following could be test modules, but we are very conservative and # don't treat them as such since they could plausibly be real modules. - assert not is_test_module('foo.bartest') - assert not is_test_module('foo.bartests') - assert not is_test_module('foo.testbar') + assert not is_non_library_module('foo.bartest') + assert not is_non_library_module('foo.bartests') + assert not is_non_library_module('foo.testbar') + + assert is_non_library_module('foo.test') + assert is_non_library_module('foo.test.foo') + assert is_non_library_module('foo.tests') + assert is_non_library_module('foo.tests.foo') + assert is_non_library_module('foo.testing.foo') + assert is_non_library_module('foo.SelfTest.foo') + + assert is_non_library_module('foo.test_bar') + assert is_non_library_module('foo.bar_tests') + assert is_non_library_module('foo.testing') + assert is_non_library_module('foo.conftest') + assert is_non_library_module('foo.bar_test_util') + assert is_non_library_module('foo.bar_test_utils') + assert is_non_library_module('foo.bar_test_base') - assert is_test_module('foo.test') - assert is_test_module('foo.test.foo') - assert is_test_module('foo.tests') - assert is_test_module('foo.tests.foo') - assert is_test_module('foo.testing.foo') + assert is_non_library_module('foo.setup') - assert is_test_module('foo.test_bar') - assert is_test_module('foo.bar_tests') - assert is_test_module('foo.testing') - assert is_test_module('foo.conftest') - assert is_test_module('foo.bar_test_util') - assert is_test_module('foo.bar_test_utils') - assert is_test_module('foo.bar_test_base') + assert is_non_library_module('foo.__main__') class StubgenPythonSuite(DataSuite): @@ -521,7 +536,8 @@ def run_case_inner(self, testcase: DataDrivenTestCase) -> None: if mod.endswith('.__init__'): mod, _, _ = mod.rpartition('.') mods.append(mod) - extra.extend(['-m', mod]) + if '-p ' not in source: + extra.extend(['-m', mod]) with open(file, 'w') as f: f.write(content) @@ -583,7 +599,7 @@ def add_file(self, path: str, result: List[str], header: bool) -> None: self_arg = ArgSig(name='self') -class StubgencSuite(Suite): +class StubgencSuite(unittest.TestCase): """Unit tests for stub generation from C modules using introspection. Note that these don't cover a lot! @@ -787,7 +803,7 @@ def __init__(self, arg0: str) -> None: assert_equal(set(imports), {'from typing import overload'}) -class ArgSigSuite(Suite): +class ArgSigSuite(unittest.TestCase): def test_repr(self) -> None: assert_equal(repr(ArgSig(name='asd"dsa')), "ArgSig(name='asd\"dsa', type=None, default=False)") @@ -799,6 +815,62 @@ def test_repr(self) -> None: "ArgSig(name='func', type='str', default=True)") +class IsValidTypeSuite(unittest.TestCase): + def test_is_valid_type(self) -> None: + assert is_valid_type('int') + assert is_valid_type('str') + assert is_valid_type('Foo_Bar234') + assert is_valid_type('foo.bar') + assert is_valid_type('List[int]') + assert is_valid_type('Dict[str, int]') + assert is_valid_type('None') + assert not is_valid_type('foo-bar') + assert not is_valid_type('x->y') + assert not is_valid_type('True') + assert not is_valid_type('False') + assert not is_valid_type('x,y') + assert not is_valid_type('x, y') + + +class ModuleInspectSuite(unittest.TestCase): + def test_python_module(self) -> None: + with ModuleInspect() as m: + p = m.get_package_properties('inspect') + assert p is not None + assert p.name == 'inspect' + assert p.file + assert p.path is None + assert p.is_c_module is False + assert p.subpackages == [] + + def test_python_package(self) -> None: + with ModuleInspect() as m: + p = m.get_package_properties('unittest') + assert p is not None + assert p.name == 'unittest' + assert p.file + assert p.path + assert p.is_c_module is False + assert p.subpackages + assert all(sub.startswith('unittest.') for sub in p.subpackages) + + def test_c_module(self) -> None: + with ModuleInspect() as m: + p = m.get_package_properties('_socket') + assert p is not None + assert p.name == '_socket' + assert p.file + assert p.path is None + assert p.is_c_module is True + assert p.subpackages == [] + + def test_non_existent(self) -> None: + with ModuleInspect() as m: + with self.assertRaises(InspectError) as e: + m.get_package_properties('foobar-non-existent') + assert str(e.exception) == "No module named 'foobar-non-existent'" + + def module_to_path(out_dir: str, module: str) -> str: fnam = os.path.join(out_dir, '{}.pyi'.format(module.replace('.', '/'))) if not os.path.exists(fnam): diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index 5d044524a05f..03592da6e18a 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -6545,3 +6545,24 @@ class C3(Generic[TypeT1]): return 0 [builtins fixtures/isinstancelist.pyi] + +[case testPropertyWithExtraMethod] +def dec(f): + return f + +class A: + @property + def x(self): ... + @x.setter + def x(self, value) -> None: ... + def x(self) -> None: ... # E: Unexpected definition for property "x" + + @property + def y(self) -> int: ... + @y.setter + def y(self, value: int) -> None: ... + @dec + def y(self) -> None: ... # TODO: This should generate an error + +reveal_type(A().y) # N: Revealed type is 'builtins.int' +[builtins fixtures/property.pyi] diff --git a/test-data/unit/stubgen.test b/test-data/unit/stubgen.test index 18837e26dc79..409a31064530 100644 --- a/test-data/unit/stubgen.test +++ b/test-data/unit/stubgen.test @@ -297,6 +297,9 @@ class A: return 1 @f.setter def f(self, x): ... + + def h(self): + self.f = 1 [out] from typing import Any @@ -305,6 +308,7 @@ class A: def f(self): ... @f.setter def f(self, x: Any) -> None: ... + def h(self) -> None: ... [case testStaticMethod] class A: @@ -378,8 +382,10 @@ class A: def __getstate__(self): ... def __setstate__(self, state): ... [out] +from typing import Any + class A: - def __eq__(self) -> None: ... + def __eq__(self) -> Any: ... -- Tests that will perform runtime imports of modules. -- Don't use `_import` suffix if there are unquoted forward references. @@ -630,6 +636,14 @@ Y = namedtuple('Y', ['a']) Z = namedtuple('Z', ['a', 'b', 'c', 'd', 'e']) +[case testDynamicNamedTuple] +from collections import namedtuple +N = namedtuple('N', ['x', 'y'] + ['z']) +[out] +from typing import Any + +N: Any + [case testArbitraryBaseClass] import x class D(x.C): ... @@ -1248,6 +1262,24 @@ class F: @t.coroutine def g(): ... +[case testCoroutineSpecialCase_import] +import asyncio + +__all__ = ['C'] + +@asyncio.coroutine +def f(): + pass + +class C: + def f(self): + pass +[out] +import asyncio + +class C: + def f(self) -> None: ... + -- Tests for stub generation from semantically analyzed trees. -- These tests are much slower, so use the `_semanal` suffix only when needed. @@ -1666,6 +1698,8 @@ from p1.vendored import six from p1.vendor.six import foobar from p1.packages.six.moves import http_client from .packages.six.moves import queue +from p1.vendored.six.moves.http_client import foo +from p1.vendored.six.moves.urllib.parse import bar class C(http_client.HTTPMessage): pass class D(six.Iterator): pass @@ -1674,6 +1708,8 @@ class D(six.Iterator): pass import six from six import foobar as foobar from six.moves import http_client, queue as queue +from six.moves.http_client import foo as foo +from six.moves.urllib.parse import bar as bar class C(http_client.HTTPMessage): ... class D(six.Iterator): ... @@ -1763,3 +1799,357 @@ def f(x, y): pass from typing import Any def f(x: Any, y: Any) -> None: ... + +[case testImportedModuleExits_import] +# modules: a b c + +[file a.py] +def g(): pass + +[file b.py] +import sys +def f(): pass +sys.exit(1) + +[file c.py] +x = 0 + +[out] +# a.pyi +def g() -> None: ... +# b.pyi +def f() -> None: ... +# c.pyi +x: int + +[case testImportedModuleHardExits_import] +# modules: a b c + +[file a.py] +def g(): pass + +[file b.py] +import os +def f(): pass +os._exit(1) # Kill process + +[file c.py] +x = 0 + +[out] +# a.pyi +def g() -> None: ... +# b.pyi +def f() -> None: ... +# c.pyi +x: int + +[case testImportedModuleHardExits2_import] +# modules: p/a p/b p/c + +[file p/__init__.py] + +[file p/a.py] +def g(): pass + +[file p/b.py] +import os +def f(): pass +os._exit(1) # Kill process + +[file p/c.py] +x = 0 + +[out] +# p/a.pyi +def g() -> None: ... +# p/b.pyi +def f() -> None: ... +# p/c.pyi +x: int + +[case testImportedModuleHardExits3_import] +# modules: p p/a + +[file p/__init__.py] +import os +def f(): pass +os._exit(1) # Kill process + +[file p/a.py] +def g(): pass + +[out] +# p/__init__.pyi +def f() -> None: ... +# p/a.pyi +def g() -> None: ... + +[case testImportedModuleHardExits4_import] +# flags: -p p +# modules: p p/a + +[file p/__init__.py] +def ff(): pass + +[file p/a.py] +import os +def gg(): pass +os._exit(1) # Kill process + +[out] +# p/__init__.pyi +def ff() -> None: ... +# p/a.pyi +def gg() -> None: ... + +[case testExportInternalImportsByDefault] +# modules: p p/a + +[file p/__init__.py] +from p.a import A, f +from m import C + +a: A +c: C +f() + +[file p/a.py] +class A: pass +def f(): pass + +[file m.py] +class C: pass + +[out] +# p/__init__.pyi +from m import C +from p.a import A as A, f as f + +a: A +c: C +# p/a.pyi +class A: ... + +def f() -> None: ... + +[case testNoExportOfInternalImportsIfAll_import] +# modules: p p/a + +[file p/__init__.py] +from p.a import A + +__all__ = ['a'] + +a = None # type: A +b = 0 # type: int + +[file p/a.py] +class A: pass + +[out] +# p/__init__.pyi +from p.a import A + +a: A +# p/a.pyi +class A: ... + +[case testExportInternalImportsByDefaultFromUnderscorePackage] +# modules: p + +[file p.py] +from _p import A +from _m import B +from _pm import C + +a: A +b: B +c: C + +[file _p.py] +class A: pass + +[file _m.py] +class B: pass + +[file _pm.py] +class C: pass + +[out] +from _m import B +from _p import A as A +from _pm import C + +a: A +b: B +c: C + +[case testDisableExportOfInternalImports] +# flags: --export-less +# modules: p p/a + +[file p/__init__.py] +from p.a import A, B +from m import C + +a: A +c: C + +[file p/a.py] +class A: pass +class B: pass + +[file m.py] +class C: pass + +[out] +# p/__init__.pyi +from m import C +from p.a import A, B as B + +a: A +c: C +# p/a.pyi +class A: ... +class B: ... + +[case testExportInternalImportsByDefaultUsingRelativeImport] +# modules: p.a + +[file p/__init__.py] + +[file p/a.py] +from .b import f +f() + +[file p/b.py] +def f(): pass + +[out] +from .b import f as f + +[case testExportInternalImportsByDefaultSkipPrivate] +# modules: p.a + +[file p/__init__.py] + +[file p/a.py] +from .b import _f, _g as _g, _i +from p.b import _h +_f() +_h() + +[file p/b.py] +def _f(): pass +def _g(): pass +def _h(): pass +def _i(): pass +x = 0 + +[out] + +[case testExportInternalImportsByDefaultIncludePrivate] +# flags: --include-private +# modules: p.a + +[file p/__init__.py] + +[file p/a.py] +from .b import _f +_f() + +[file p/b.py] +def _f(): pass + +[out] +from .b import _f as _f + +[case testHideDunderModuleAttributes] +from m import ( + __about__, + __author__, + __copyright__, + __email__, + __license__, + __summary__, + __title__, + __uri__, + __version__ +) + +class A: + __uri__ = 0 + +[file m.py] +__about__ = '' +__author__ = '' +__copyright__ = '' +__email__ = '' +__license__ = '' +__summary__ = '' +__title__ = '' +__uri__ = '' +__version__ = '' + +[out] +class A: ... + +[case testHideDunderModuleAttributesWithAll_import] +from m import ( + __about__, + __author__, + __copyright__, + __email__, + __license__, + __summary__, + __title__, + __uri__, + __version__ +) + +__all__ = ['__about__', '__author__', '__version__'] + +[file m.py] +__about__ = '' +__author__ = '' +__copyright__ = '' +__email__ = '' +__license__ = '' +__summary__ = '' +__title__ = '' +__uri__ = '' +__version__ = '' + +[out] + +[case testAttrsClass_semanal] +import attr + +@attr.s +class C: + x = attr.ib() + +[out] +from typing import Any + +class C: + x: Any = ... + def __init__(self, x: Any) -> None: ... + def __ne__(self, other: Any) -> Any: ... + def __eq__(self, other: Any) -> Any: ... + def __lt__(self, other: Any) -> Any: ... + def __le__(self, other: Any) -> Any: ... + def __gt__(self, other: Any) -> Any: ... + def __ge__(self, other: Any) -> Any: ... + +[case testNamedTupleInClass] +from collections import namedtuple + +class C: + N = namedtuple('N', ['x', 'y']) +[out] +from collections import namedtuple + +class C: + N = namedtuple('N', ['x', 'y']) pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy