Skip to content

Commit f86d08e

Browse files
authored
Various improvements to stubgen (python#7921)
This improves various things in stubgen. Here are some notable changes: * Filter out some bad type comments (or comments that look like them) since these generate parse errors, and stubgen can't recover from them. * Remove the initial generated comment from all stubs, since it gets easily out of date and seems redundant, especially for packages with hundreds of submodules. * Don't generate a blocking error for inconsistent MROs. * Make docstring parser more robust. * Tweak the level of logging. Add `--verbose` and `--quiet` flags. * Export imported names that aren't references in the module. * Fix handling of unreachable code. * Fix signatures of various C extension methods. * Skip modules that are not useful or can cause trouble, such as vendored modules and tests. * Preserve `@abstractproperty`. * Fix clashes between `typing.Any` and a class named `Any` (etc.). * Translate imports to vendored `six` to refer to normal `six`. * Add ad-hoc whitelisting of specific definitions that should be exported.
1 parent 1781b72 commit f86d08e

File tree

11 files changed

+1069
-182
lines changed

11 files changed

+1069
-182
lines changed

mypy/nodes.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2285,6 +2285,7 @@ class is generic then it will be a type constructor of higher kind.
22852285
# Used to stash the names of the mro classes temporarily between
22862286
# deserialization and fixup. See deserialize() for why.
22872287
_mro_refs = None # type: Optional[List[str]]
2288+
bad_mro = False # Could not construct full MRO
22882289

22892290
declared_metaclass = None # type: Optional[mypy.types.Instance]
22902291
metaclass_type = None # type: Optional[mypy.types.Instance]

mypy/options.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@
33
import pprint
44
import sys
55

6-
from typing import Dict, List, Mapping, Optional, Pattern, Set, Tuple
76
from typing_extensions import Final
7+
from typing import Dict, List, Mapping, Optional, Pattern, Set, Tuple, Callable, Any
88

99
from mypy import defaults
1010
from mypy.util import get_class_descriptors, replace_object_state
@@ -262,6 +262,9 @@ def __init__(self) -> None:
262262
self.cache_map = {} # type: Dict[str, Tuple[str, str]]
263263
# Don't properly free objects on exit, just kill the current process.
264264
self.fast_exit = False
265+
# Used to transform source code before parsing if not None
266+
# TODO: Make the type precise (AnyStr -> AnyStr)
267+
self.transform_source = None # type: Optional[Callable[[Any], Any]]
265268
# Print full path to each file in the report.
266269
self.show_absolute_path = False # type: bool
267270

mypy/parse.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ def parse(source: Union[str, bytes],
1818
The python_version (major, minor) option determines the Python syntax variant.
1919
"""
2020
is_stub_file = fnam.endswith('.pyi')
21+
if options.transform_source is not None:
22+
source = options.transform_source(source)
2123
if options.python_version[0] >= 3 or is_stub_file:
2224
import mypy.fastparse
2325
return mypy.fastparse.parse(source,

mypy/semanal.py

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1490,8 +1490,7 @@ def configure_base_classes(self,
14901490

14911491
# Calculate the MRO.
14921492
if not self.verify_base_classes(defn):
1493-
# Give it an MRO consisting of just the class itself and object.
1494-
defn.info.mro = [defn.info, self.object_type().type]
1493+
self.set_dummy_mro(defn.info)
14951494
return
14961495
self.calculate_class_mro(defn, self.object_type)
14971496

@@ -1519,6 +1518,11 @@ def configure_tuple_base_class(self,
15191518

15201519
return base.partial_fallback
15211520

1521+
def set_dummy_mro(self, info: TypeInfo) -> None:
1522+
# Give it an MRO consisting of just the class itself and object.
1523+
info.mro = [info, self.object_type().type]
1524+
info.bad_mro = True
1525+
15221526
def calculate_class_mro(self, defn: ClassDef,
15231527
obj_type: Optional[Callable[[], Instance]] = None) -> None:
15241528
"""Calculate method resolution order for a class.
@@ -1530,9 +1534,9 @@ def calculate_class_mro(self, defn: ClassDef,
15301534
try:
15311535
calculate_mro(defn.info, obj_type)
15321536
except MroError:
1533-
self.fail_blocker('Cannot determine consistent method resolution '
1534-
'order (MRO) for "%s"' % defn.name, defn)
1535-
defn.info.mro = []
1537+
self.fail('Cannot determine consistent method resolution '
1538+
'order (MRO) for "%s"' % defn.name, defn)
1539+
self.set_dummy_mro(defn.info)
15361540
# Allow plugins to alter the MRO to handle the fact that `def mro()`
15371541
# on metaclasses permits MRO rewriting.
15381542
if defn.fullname:
@@ -1597,12 +1601,12 @@ def update_metaclass(self, defn: ClassDef) -> None:
15971601

15981602
def verify_base_classes(self, defn: ClassDef) -> bool:
15991603
info = defn.info
1604+
cycle = False
16001605
for base in info.bases:
16011606
baseinfo = base.type
16021607
if self.is_base_class(info, baseinfo):
1603-
self.fail('Cycle in inheritance hierarchy', defn, blocker=True)
1604-
# Clear bases to forcefully get rid of the cycle.
1605-
info.bases = []
1608+
self.fail('Cycle in inheritance hierarchy', defn)
1609+
cycle = True
16061610
if baseinfo.fullname == 'builtins.bool':
16071611
self.fail("'%s' is not a valid base class" %
16081612
baseinfo.name, defn, blocker=True)
@@ -1611,7 +1615,7 @@ def verify_base_classes(self, defn: ClassDef) -> bool:
16111615
if dup:
16121616
self.fail('Duplicate base class "%s"' % dup.name, defn, blocker=True)
16131617
return False
1614-
return True
1618+
return not cycle
16151619

16161620
def is_base_class(self, t: TypeInfo, s: TypeInfo) -> bool:
16171621
"""Determine if t is a base class of s (but do not use mro)."""

mypy/stubdoc.py

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -199,8 +199,12 @@ def infer_sig_from_docstring(docstr: str, name: str) -> Optional[List[FunctionSi
199199
state = DocStringParser(name)
200200
# Return all found signatures, even if there is a parse error after some are found.
201201
with contextlib.suppress(tokenize.TokenError):
202-
for token in tokenize.tokenize(io.BytesIO(docstr.encode('utf-8')).readline):
203-
state.add_token(token)
202+
try:
203+
tokens = tokenize.tokenize(io.BytesIO(docstr.encode('utf-8')).readline)
204+
for token in tokens:
205+
state.add_token(token)
206+
except IndentationError:
207+
return None
204208
sigs = state.get_signatures()
205209

206210
def is_unique_args(sig: FunctionSig) -> bool:

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy