diff --git a/.coveragerc b/.coveragerc index 66d32472..98113f51 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,8 +1,10 @@ [run] omit = + # leading `*/` for pytest-dev/pytest-cov#456 */.tox/* tests/* prepare/* + */_itertools.py [report] show_missing = True diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..6385b573 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,15 @@ +root = true + +[*] +charset = utf-8 +indent_style = tab +indent_size = 4 +insert_final_newline = true +end_of_line = lf + +[*.py] +indent_style = space + +[*.{yml,yaml}] +indent_style = space +indent_size = 2 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1853ace5..6377cfcd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -35,7 +35,7 @@ jobs: - name: Run benchmarks run: tox env: - TOXENV: perf + TOXENV: perf{,-ref} diffcov: runs-on: ubuntu-latest diff --git a/CHANGES.rst b/CHANGES.rst index 57901f23..a9ef4870 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,163 @@ +v4.0.1 +======= + +* #306: Clearer guidance about compatibility in readme. + +v4.0.0 +======= + +* #304: ``PackageMetadata`` as returned by ``metadata()`` + and ``Distribution.metadata()`` now provides normalized + metadata honoring PEP 566: + + - If a long description is provided in the payload of the + RFC 822 value, it can be retrieved as the ``Description`` + field. + - Any multi-line values in the metadata will be returned as + such. + - For any multi-line values, line continuation characters + are removed. This backward-incompatible change means + that any projects relying on the RFC 822 line continuation + characters being present must be tolerant to them having + been removed. + - Add a ``json`` property that provides the metadata + converted to a JSON-compatible form per PEP 566. + + +v3.10.1 +======= + +* Minor tweaks from CPython. + +v3.10.0 +======= + +* #295: Internal refactoring to unify section parsing logic. + +v3.9.1 +====== + +* #296: Exclude 'prepare' package. +* #297: Fix ValueError when entry points contains comments. + +v3.9.0 +====== + +* Use of Mapping (dict) interfaces on ``SelectableGroups`` + is now flagged as deprecated. Instead, users are advised + to use the select interface for future compatibility. + + Suppress the warning with this filter: + ``ignore:SelectableGroups dict interface``. + + Or with this invocation in the Python environment: + ``warnings.filterwarnings('ignore', 'SelectableGroups dict interface')``. + + Preferably, switch to the ``select`` interface introduced + in 3.7.0. See the + `entry points documentation `_ and changelog for the 3.6 + release below for more detail. + + For some use-cases, especially those that rely on + ``importlib.metadata`` in Python 3.8 and 3.9 or + those relying on older ``importlib_metadata`` (especially + on Python 3.5 and earlier), + `backports.entry_points_selectable `_ + was created to ease the transition. Please have a look + at that project if simply relying on importlib_metadata 3.6+ + is not straightforward. Background in #298. + +* #283: Entry point parsing no longer relies on ConfigParser + and instead uses a custom, one-pass parser to load the + config, resulting in a ~20% performance improvement when + loading entry points. + +v3.8.2 +====== + +* #293: Re-enabled lazy evaluation of path lookup through + a FreezableDefaultDict. + +v3.8.1 +====== + +* #293: Workaround for error in distribution search. + +v3.8.0 +====== + +* #290: Add mtime-based caching for ``FastPath`` and its + lookups, dramatically increasing performance for repeated + distribution lookups. + +v3.7.3 +====== + +* Docs enhancements and cleanup following review in + `GH-24782 `_. + +v3.7.2 +====== + +* Cleaned up cruft in entry_points docstring. + +v3.7.1 +====== + +* Internal refactoring to facilitate ``entry_points() -> dict`` + deprecation. + +v3.7.0 +====== + +* #131: Added ``packages_distributions`` to conveniently + resolve a top-level package or module to its distribution(s). + +v3.6.0 +====== + +* #284: Introduces new ``EntryPoints`` object, a tuple of + ``EntryPoint`` objects but with convenience properties for + selecting and inspecting the results: + + - ``.select()`` accepts ``group`` or ``name`` keyword + parameters and returns a new ``EntryPoints`` tuple + with only those that match the selection. + - ``.groups`` property presents all of the group names. + - ``.names`` property presents the names of the entry points. + - Item access (e.g. ``eps[name]``) retrieves a single + entry point by name. + + ``entry_points`` now accepts "selection parameters", + same as ``EntryPoint.select()``. + + ``entry_points()`` now provides a future-compatible + ``SelectableGroups`` object that supplies the above interface + (except item access) but remains a dict for compatibility. + + In the future, ``entry_points()`` will return an + ``EntryPoints`` object for all entry points. + + If passing selection parameters to ``entry_points``, the + future behavior is invoked and an ``EntryPoints`` is the + result. + +* #284: Construction of entry points using + ``dict([EntryPoint, ...])`` is now deprecated and raises + an appropriate DeprecationWarning and will be removed in + a future version. + +* #300: ``Distribution.entry_points`` now presents as an + ``EntryPoints`` object and access by index is no longer + allowed. If access by index is required, cast to a sequence + first. + +v3.5.0 +====== + +* #280: ``entry_points`` now only returns entry points for + unique distributions (by name). + v3.4.0 ====== diff --git a/README.rst b/README.rst index 512c3cc2..986d3ca1 100644 --- a/README.rst +++ b/README.rst @@ -20,11 +20,27 @@ Library to access the metadata for a Python package. -As of Python 3.8, this functionality has been added to the -`Python standard library -`_. -This package supplies backports of that functionality including -improvements added to subsequent Python versions. +This package supplies third-party access to the functionality of +`importlib.metadata `_ +including improvements added to subsequent Python versions. + + +Compatibility +============= + +New features are introduced in this third-party library and later merged +into CPython. The following table indicates which versions of this library +were contributed to different versions in the standard library: + +.. list-table:: + :header-rows: 1 + + * - importlib_metadata + - stdlib + * - 3.10 + - 3.10 + * - 1.4 + - 3.8 Usage diff --git a/docs/using.rst b/docs/using.rst index efa40f86..17d6f590 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -67,18 +67,48 @@ This package provides the following functionality via its public API. Entry points ------------ -The ``entry_points()`` function returns a dictionary of all entry points, -keyed by group. Entry points are represented by ``EntryPoint`` instances; +The ``entry_points()`` function returns a collection of entry points. +Entry points are represented by ``EntryPoint`` instances; each ``EntryPoint`` has a ``.name``, ``.group``, and ``.value`` attributes and a ``.load()`` method to resolve the value. There are also ``.module``, ``.attr``, and ``.extras`` attributes for getting the components of the -``.value`` attribute:: +``.value`` attribute. + +Query all entry points:: >>> eps = entry_points() - >>> list(eps) + +The ``entry_points()`` function returns an ``EntryPoints`` object, +a sequence of all ``EntryPoint`` objects with ``names`` and ``groups`` +attributes for convenience:: + + >>> sorted(eps.groups) ['console_scripts', 'distutils.commands', 'distutils.setup_keywords', 'egg_info.writers', 'setuptools.installation'] - >>> scripts = eps['console_scripts'] - >>> wheel = [ep for ep in scripts if ep.name == 'wheel'][0] + +``EntryPoints`` has a ``select`` method to select entry points +matching specific properties. Select entry points in the +``console_scripts`` group:: + + >>> scripts = eps.select(group='console_scripts') + +Equivalently, since ``entry_points`` passes keyword arguments +through to select:: + + >>> scripts = entry_points(group='console_scripts') + +Pick out a specific script named "wheel" (found in the wheel project):: + + >>> 'wheel' in scripts.names + True + >>> wheel = scripts['wheel'] + +Equivalently, query for that entry point during selection:: + + >>> (wheel,) = entry_points(group='console_scripts', name='wheel') + >>> (wheel,) = entry_points().select(group='console_scripts', name='wheel') + +Inspect the resolved entry point:: + >>> wheel EntryPoint(name='wheel', value='wheel.cli:main', group='console_scripts') >>> wheel.module @@ -97,6 +127,17 @@ group. Read `the setuptools docs `_ for more information on entry points, their definition, and usage. +*Compatibility Note* + +The "selectable" entry points were introduced in ``importlib_metadata`` +3.6 and Python 3.10. Prior to those changes, ``entry_points`` accepted +no parameters and always returned a dictionary of entry points, keyed +by group. For compatibility, if no parameters are passed to entry_points, +a ``SelectableGroups`` object is returned, implementing that dict +interface. In the future, calling ``entry_points`` with no parameters +will return an ``EntryPoints`` object. Users should rely on the selection +interface to retrieve entry points by group. + .. _metadata: @@ -180,6 +221,17 @@ function:: ["pytest (>=3.0.0) ; extra == 'test'", "pytest-cov ; extra == 'test'"] +Package distributions +--------------------- + +A convience method to resolve the distribution or +distributions (in the case of a namespace package) for top-level +Python packages or modules:: + + >>> packages_distributions() + {'importlib_metadata': ['importlib-metadata'], 'yaml': ['PyYAML'], 'jaraco': ['jaraco.classes', 'jaraco.functools'], ...} + + Distributions ============= @@ -200,9 +252,9 @@ Thus, an alternative way to get the version number is through the There are all kinds of additional metadata available on the ``Distribution`` instance:: - >>> d.metadata['Requires-Python'] + >>> dist.metadata['Requires-Python'] '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*' - >>> d.metadata['License'] + >>> dist.metadata['License'] 'MIT' The full set of available metadata is not described here. See :pep:`566` diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 079cc0f9..af4d48c5 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -7,24 +7,28 @@ import email import pathlib import operator +import textwrap +import warnings import functools import itertools import posixpath import collections +from . import _adapters, _meta +from ._collections import FreezableDefaultDict, Pair from ._compat import ( NullFinder, PyPy_repr, install, - Protocol, ) +from ._functools import method_cache +from ._itertools import unique_everseen -from configparser import ConfigParser from contextlib import suppress from importlib import import_module from importlib.abc import MetaPathFinder from itertools import starmap -from typing import Any, List, Optional, TypeVar, Union +from typing import List, Mapping, Optional, Union __all__ = [ @@ -36,6 +40,7 @@ 'entry_points', 'files', 'metadata', + 'packages_distributions', 'requires', 'version', ] @@ -54,6 +59,71 @@ def name(self): return name +class Sectioned: + """ + A simple entry point config parser for performance + + >>> for item in Sectioned.read(Sectioned._sample): + ... print(item) + Pair(name='sec1', value='# comments ignored') + Pair(name='sec1', value='a = 1') + Pair(name='sec1', value='b = 2') + Pair(name='sec2', value='a = 2') + + >>> res = Sectioned.section_pairs(Sectioned._sample) + >>> item = next(res) + >>> item.name + 'sec1' + >>> item.value + Pair(name='a', value='1') + >>> item = next(res) + >>> item.value + Pair(name='b', value='2') + >>> item = next(res) + >>> item.name + 'sec2' + >>> item.value + Pair(name='a', value='2') + >>> list(res) + [] + """ + + _sample = textwrap.dedent( + """ + [sec1] + # comments ignored + a = 1 + b = 2 + + [sec2] + a = 2 + """ + ).lstrip() + + @classmethod + def section_pairs(cls, text): + return ( + section._replace(value=Pair.parse(section.value)) + for section in cls.read(text, filter_=cls.valid) + if section.name is not None + ) + + @staticmethod + def read(text, filter_=None): + lines = filter(filter_, map(str.strip, text.splitlines())) + name = None + for value in lines: + section_match = value.startswith('[') and value.endswith(']') + if section_match: + name = value.strip('[]') + continue + yield Pair(name, value) + + @staticmethod + def valid(line): + return line and not line.startswith('#') + + class EntryPoint( PyPy_repr, collections.namedtuple('EntryPointBase', 'name value group') ): @@ -112,34 +182,19 @@ def extras(self): match = self.pattern.match(self.value) return list(re.finditer(r'\w+', match.group('extras') or '')) - @classmethod - def _from_config(cls, config): - return ( - cls(name, value, group) - for group in config.sections() - for name, value in config.items(group) - ) - - @classmethod - def _from_text(cls, text): - config = ConfigParser(delimiters='=') - # case sensitive: https://stackoverflow.com/q/1611799/812183 - config.optionxform = str - config.read_string(text) - return cls._from_config(config) - - @classmethod - def _from_text_for(cls, text, dist): - return (ep._for(dist) for ep in cls._from_text(text)) - def _for(self, dist): self.dist = dist return self def __iter__(self): """ - Supply iter so one may construct dicts of EntryPoints easily. + Supply iter so one may construct dicts of EntryPoints by name. """ + msg = ( + "Construction of dict of EntryPoints is deprecated in " + "favor of EntryPoints." + ) + warnings.warn(msg, DeprecationWarning) return iter((self.name, self)) def __reduce__(self): @@ -148,6 +203,170 @@ def __reduce__(self): (self.name, self.value, self.group), ) + def matches(self, **params): + attrs = (getattr(self, param) for param in params) + return all(map(operator.eq, params.values(), attrs)) + + +class EntryPoints(tuple): + """ + An immutable collection of selectable EntryPoint objects. + """ + + __slots__ = () + + def __getitem__(self, name): # -> EntryPoint: + """ + Get the EntryPoint in self matching name. + """ + try: + return next(iter(self.select(name=name))) + except StopIteration: + raise KeyError(name) + + def select(self, **params): + """ + Select entry points from self that match the + given parameters (typically group and/or name). + """ + return EntryPoints(ep for ep in self if ep.matches(**params)) + + @property + def names(self): + """ + Return the set of all names of all entry points. + """ + return set(ep.name for ep in self) + + @property + def groups(self): + """ + Return the set of all groups of all entry points. + + For coverage while SelectableGroups is present. + >>> EntryPoints().groups + set() + """ + return set(ep.group for ep in self) + + @classmethod + def _from_text_for(cls, text, dist): + return cls(ep._for(dist) for ep in cls._from_text(text)) + + @classmethod + def _from_text(cls, text): + return itertools.starmap(EntryPoint, cls._parse_groups(text or '')) + + @staticmethod + def _parse_groups(text): + return ( + (item.value.name, item.value.value, item.name) + for item in Sectioned.section_pairs(text) + ) + + +def flake8_bypass(func): + # defer inspect import as performance optimization. + import inspect + + is_flake8 = any('flake8' in str(frame.filename) for frame in inspect.stack()[:5]) + return func if not is_flake8 else lambda: None + + +class Deprecated: + """ + Compatibility add-in for mapping to indicate that + mapping behavior is deprecated. + + >>> recwarn = getfixture('recwarn') + >>> class DeprecatedDict(Deprecated, dict): pass + >>> dd = DeprecatedDict(foo='bar') + >>> dd.get('baz', None) + >>> dd['foo'] + 'bar' + >>> list(dd) + ['foo'] + >>> list(dd.keys()) + ['foo'] + >>> 'foo' in dd + True + >>> list(dd.values()) + ['bar'] + >>> len(recwarn) + 1 + """ + + _warn = functools.partial( + warnings.warn, + "SelectableGroups dict interface is deprecated. Use select.", + DeprecationWarning, + stacklevel=2, + ) + + def __getitem__(self, name): + self._warn() + return super().__getitem__(name) + + def get(self, name, default=None): + flake8_bypass(self._warn)() + return super().get(name, default) + + def __iter__(self): + self._warn() + return super().__iter__() + + def __contains__(self, *args): + self._warn() + return super().__contains__(*args) + + def keys(self): + self._warn() + return super().keys() + + def values(self): + self._warn() + return super().values() + + +class SelectableGroups(Deprecated, dict): + """ + A backward- and forward-compatible result from + entry_points that fully implements the dict interface. + """ + + @classmethod + def load(cls, eps): + by_group = operator.attrgetter('group') + ordered = sorted(eps, key=by_group) + grouped = itertools.groupby(ordered, by_group) + return cls((group, EntryPoints(eps)) for group, eps in grouped) + + @property + def _all(self): + """ + Reconstruct a list of all entrypoints from the groups. + """ + groups = super(Deprecated, self).values() + return EntryPoints(itertools.chain.from_iterable(groups)) + + @property + def groups(self): + return self._all.groups + + @property + def names(self): + """ + for coverage: + >>> SelectableGroups().names + set() + """ + return self._all.names + + def select(self, **params): + if not params: + return self + return self._all.select(**params) + class PackagePath(pathlib.PurePosixPath): """A reference to a path in a package""" @@ -173,25 +392,6 @@ def __repr__(self): return ''.format(self.mode, self.value) -_T = TypeVar("_T") - - -class PackageMetadata(Protocol): - def __len__(self) -> int: - ... # pragma: no cover - - def __contains__(self, item: str) -> bool: - ... # pragma: no cover - - def __getitem__(self, key: str) -> str: - ... # pragma: no cover - - def get_all(self, name: str, failobj: _T = ...) -> Union[List[Any], _T]: - """ - Return all values associated with a possibly multi-valued key. - """ - - class Distribution: """A Python distribution package.""" @@ -276,7 +476,7 @@ def _local(cls, root='.'): return PathDistribution(zipp.Path(meta.build_as_zip(builder))) @property - def metadata(self) -> PackageMetadata: + def metadata(self) -> _meta.PackageMetadata: """Return the parsed metadata for this Distribution. The returned object will have keys that name the various bits of @@ -290,7 +490,7 @@ def metadata(self) -> PackageMetadata: # (which points to the egg-info file) attribute unchanged. or self.read_text('') ) - return email.message_from_string(text) + return _adapters.Message(email.message_from_string(text)) @property def name(self): @@ -304,7 +504,7 @@ def version(self): @property def entry_points(self): - return list(EntryPoint._from_text_for(self.read_text('entry_points.txt'), self)) + return EntryPoints._from_text_for(self.read_text('entry_points.txt'), self) @property def files(self): @@ -358,24 +558,7 @@ def _read_egg_info_reqs(self): @classmethod def _deps_from_requires_text(cls, source): - section_pairs = cls._read_sections(source.splitlines()) - sections = { - section: list(map(operator.itemgetter('line'), results)) - for section, results in itertools.groupby( - section_pairs, operator.itemgetter('section') - ) - } - return cls._convert_egg_info_reqs_to_simple_reqs(sections) - - @staticmethod - def _read_sections(lines): - section = None - for line in filter(None, lines): - section_match = re.match(r'\[(.*)\]$', line) - if section_match: - section = section_match.group(1) - continue - yield locals() + return cls._convert_egg_info_reqs_to_simple_reqs(Sectioned.read(source)) @staticmethod def _convert_egg_info_reqs_to_simple_reqs(sections): @@ -400,9 +583,8 @@ def parse_condition(section): conditions = list(filter(None, [markers, make_condition(extra)])) return '; ' + ' and '.join(conditions) if conditions else '' - for section, deps in sections.items(): - for dep in deps: - yield dep + parse_condition(section) + for section in sections: + yield section.value + parse_condition(section.name) class DistributionFinder(MetaPathFinder): @@ -458,9 +640,12 @@ class FastPath: children. """ + @functools.lru_cache() # type: ignore + def __new__(cls, root): + return super().__new__(cls) + def __init__(self, root): self.root = str(root) - self.base = os.path.basename(self.root).lower() def joinpath(self, child): return pathlib.Path(self.root, child) @@ -480,11 +665,53 @@ def zip_children(self): return dict.fromkeys(child.split(posixpath.sep, 1)[0] for child in names) def search(self, name): - return ( - self.joinpath(child) - for child in self.children() - if name.matches(child, self.base) + return self.lookup(self.mtime).search(name) + + @property + def mtime(self): + with suppress(OSError): + return os.stat(self.root).st_mtime + self.lookup.cache_clear() + + @method_cache + def lookup(self, mtime): + return Lookup(self) + + +class Lookup: + def __init__(self, path: FastPath): + base = os.path.basename(path.root).lower() + base_is_egg = base.endswith(".egg") + self.infos = FreezableDefaultDict(list) + self.eggs = FreezableDefaultDict(list) + + for child in path.children(): + low = child.lower() + if low.endswith((".dist-info", ".egg-info")): + # rpartition is faster than splitext and suitable for this purpose. + name = low.rpartition(".")[0].partition("-")[0] + normalized = Prepared.normalize(name) + self.infos[normalized].append(path.joinpath(child)) + elif base_is_egg and low == "egg-info": + name = base.rpartition(".")[0].partition("-")[0] + legacy_normalized = Prepared.legacy_normalize(name) + self.eggs[legacy_normalized].append(path.joinpath(child)) + + self.infos.freeze() + self.eggs.freeze() + + def search(self, prepared): + infos = ( + self.infos[prepared.normalized] + if prepared + else itertools.chain.from_iterable(self.infos.values()) + ) + eggs = ( + self.eggs[prepared.legacy_normalized] + if prepared + else itertools.chain.from_iterable(self.eggs.values()) ) + return itertools.chain(infos, eggs) class Prepared: @@ -493,22 +720,14 @@ class Prepared: """ normalized = None - suffixes = 'dist-info', 'egg-info' - exact_matches = [''][:0] - egg_prefix = '' - versionless_egg_name = '' + legacy_normalized = None def __init__(self, name): self.name = name if name is None: return self.normalized = self.normalize(name) - self.exact_matches = [ - self.normalized + '.' + suffix for suffix in self.suffixes - ] - legacy_normalized = self.legacy_normalize(self.name) - self.egg_prefix = legacy_normalized + '-' - self.versionless_egg_name = legacy_normalized + '.egg' + self.legacy_normalized = self.legacy_normalize(name) @staticmethod def normalize(name): @@ -525,26 +744,8 @@ def legacy_normalize(name): """ return name.lower().replace('-', '_') - def matches(self, cand, base): - low = cand.lower() - # rpartition is faster than splitext and suitable for this purpose. - pre, _, ext = low.rpartition('.') - name, _, rest = pre.partition('-') - return ( - low in self.exact_matches - or ext in self.suffixes - and (not self.normalized or name.replace('.', '_') == self.normalized) - # legacy case: - or self.is_egg(base) - and low == 'egg-info' - ) - - def is_egg(self, base): - return ( - base == self.versionless_egg_name - or base.startswith(self.egg_prefix) - and base.endswith('.egg') - ) + def __bool__(self): + return bool(self.name) @install @@ -575,6 +776,9 @@ def _search_paths(cls, name, paths): path.search(prepared) for path in map(FastPath, paths) ) + def invalidate_caches(cls): + FastPath.__new__.cache_clear() + class PathDistribution(Distribution): def __init__(self, path): @@ -618,7 +822,7 @@ def distributions(**kwargs): return Distribution.discover(**kwargs) -def metadata(distribution_name) -> PackageMetadata: +def metadata(distribution_name) -> _meta.PackageMetadata: """Get the metadata for the named package. :param distribution_name: The name of the distribution package to query. @@ -637,16 +841,28 @@ def version(distribution_name): return distribution(distribution_name).version -def entry_points(): +def entry_points(**params) -> Union[EntryPoints, SelectableGroups]: """Return EntryPoint objects for all installed packages. - :return: EntryPoint objects for all installed packages. + Pass selection parameters (group or name) to filter the + result to entry points matching those properties (see + EntryPoints.select()). + + For compatibility, returns ``SelectableGroups`` object unless + selection parameters are supplied. In the future, this function + will return ``EntryPoints`` instead of ``SelectableGroups`` + even when no selection parameters are supplied. + + For maximum future compatibility, pass selection parameters + or invoke ``.select`` with parameters on the result. + + :return: EntryPoints or SelectableGroups for all installed packages. """ - eps = itertools.chain.from_iterable(dist.entry_points for dist in distributions()) - by_group = operator.attrgetter('group') - ordered = sorted(eps, key=by_group) - grouped = itertools.groupby(ordered, by_group) - return {group: tuple(eps) for group, eps in grouped} + unique = functools.partial(unique_everseen, key=operator.attrgetter('name')) + eps = itertools.chain.from_iterable( + dist.entry_points for dist in unique(distributions()) + ) + return SelectableGroups.load(eps).select(**params) def files(distribution_name): @@ -666,3 +882,20 @@ def requires(distribution_name): packaging.requirement.Requirement. """ return distribution(distribution_name).requires + + +def packages_distributions() -> Mapping[str, List[str]]: + """ + Return a mapping of top-level packages to their + distributions. + + >>> import collections.abc + >>> pkgs = packages_distributions() + >>> all(isinstance(dist, collections.abc.Sequence) for dist in pkgs.values()) + True + """ + pkg_to_dist = collections.defaultdict(list) + for dist in distributions(): + for pkg in (dist.read_text('top_level.txt') or '').split(): + pkg_to_dist[pkg].append(dist.metadata['Name']) + return dict(pkg_to_dist) diff --git a/importlib_metadata/_adapters.py b/importlib_metadata/_adapters.py new file mode 100644 index 00000000..ab086180 --- /dev/null +++ b/importlib_metadata/_adapters.py @@ -0,0 +1,67 @@ +import re +import textwrap +import email.message + +from ._text import FoldedCase + + +class Message(email.message.Message): + multiple_use_keys = set( + map( + FoldedCase, + [ + 'Classifier', + 'Obsoletes-Dist', + 'Platform', + 'Project-URL', + 'Provides-Dist', + 'Provides-Extra', + 'Requires-Dist', + 'Requires-External', + 'Supported-Platform', + ], + ) + ) + """ + Keys that may be indicated multiple times per PEP 566. + """ + + def __new__(cls, orig: email.message.Message): + res = super().__new__(cls) + vars(res).update(vars(orig)) + return res + + def __init__(self, *args, **kwargs): + self._headers = self._repair_headers() + + # suppress spurious error from mypy + def __iter__(self): + return super().__iter__() + + def _repair_headers(self): + def redent(value): + "Correct for RFC822 indentation" + if not value or '\n' not in value: + return value + return textwrap.dedent(' ' * 8 + value) + + headers = [(key, redent(value)) for key, value in vars(self)['_headers']] + if self._payload: + headers.append(('Description', self.get_payload())) + return headers + + @property + def json(self): + """ + Convert PackageMetadata to a JSON-compatible format + per PEP 0566. + """ + + def transform(key): + value = self.get_all(key) if key in self.multiple_use_keys else self[key] + if key == 'Keywords': + value = re.split(r'\s+', value) + tk = key.lower().replace('-', '_') + return tk, value + + return dict(map(transform, map(FoldedCase, self))) diff --git a/importlib_metadata/_collections.py b/importlib_metadata/_collections.py new file mode 100644 index 00000000..cf0954e1 --- /dev/null +++ b/importlib_metadata/_collections.py @@ -0,0 +1,30 @@ +import collections + + +# from jaraco.collections 3.3 +class FreezableDefaultDict(collections.defaultdict): + """ + Often it is desirable to prevent the mutation of + a default dict after its initial construction, such + as to prevent mutation during iteration. + + >>> dd = FreezableDefaultDict(list) + >>> dd[0].append('1') + >>> dd.freeze() + >>> dd[1] + [] + >>> len(dd) + 1 + """ + + def __missing__(self, key): + return getattr(self, '_frozen', super().__missing__)(key) + + def freeze(self): + self._frozen = lambda key: self.default_factory() + + +class Pair(collections.namedtuple('Pair', 'name value')): + @classmethod + def parse(cls, text): + return cls(*map(str.strip, text.split("=", 1))) diff --git a/importlib_metadata/_functools.py b/importlib_metadata/_functools.py new file mode 100644 index 00000000..73f50d00 --- /dev/null +++ b/importlib_metadata/_functools.py @@ -0,0 +1,85 @@ +import types +import functools + + +# from jaraco.functools 3.3 +def method_cache(method, cache_wrapper=None): + """ + Wrap lru_cache to support storing the cache data in the object instances. + + Abstracts the common paradigm where the method explicitly saves an + underscore-prefixed protected property on first call and returns that + subsequently. + + >>> class MyClass: + ... calls = 0 + ... + ... @method_cache + ... def method(self, value): + ... self.calls += 1 + ... return value + + >>> a = MyClass() + >>> a.method(3) + 3 + >>> for x in range(75): + ... res = a.method(x) + >>> a.calls + 75 + + Note that the apparent behavior will be exactly like that of lru_cache + except that the cache is stored on each instance, so values in one + instance will not flush values from another, and when an instance is + deleted, so are the cached values for that instance. + + >>> b = MyClass() + >>> for x in range(35): + ... res = b.method(x) + >>> b.calls + 35 + >>> a.method(0) + 0 + >>> a.calls + 75 + + Note that if method had been decorated with ``functools.lru_cache()``, + a.calls would have been 76 (due to the cached value of 0 having been + flushed by the 'b' instance). + + Clear the cache with ``.cache_clear()`` + + >>> a.method.cache_clear() + + Same for a method that hasn't yet been called. + + >>> c = MyClass() + >>> c.method.cache_clear() + + Another cache wrapper may be supplied: + + >>> cache = functools.lru_cache(maxsize=2) + >>> MyClass.method2 = method_cache(lambda self: 3, cache_wrapper=cache) + >>> a = MyClass() + >>> a.method2() + 3 + + Caution - do not subsequently wrap the method with another decorator, such + as ``@property``, which changes the semantics of the function. + + See also + http://code.activestate.com/recipes/577452-a-memoize-decorator-for-instance-methods/ + for another implementation and additional justification. + """ + cache_wrapper = cache_wrapper or functools.lru_cache() + + def wrapper(self, *args, **kwargs): + # it's the first call, replace the method with a cached, bound method + bound_method = types.MethodType(method, self) + cached_method = cache_wrapper(bound_method) + setattr(self, method.__name__, cached_method) + return cached_method(*args, **kwargs) + + # Support cache clear even before cache has been created. + wrapper.cache_clear = lambda: None + + return wrapper diff --git a/importlib_metadata/_itertools.py b/importlib_metadata/_itertools.py new file mode 100644 index 00000000..dd45f2f0 --- /dev/null +++ b/importlib_metadata/_itertools.py @@ -0,0 +1,19 @@ +from itertools import filterfalse + + +def unique_everseen(iterable, key=None): + "List unique elements, preserving order. Remember all elements ever seen." + # unique_everseen('AAAABBBCCDAABBB') --> A B C D + # unique_everseen('ABBCcAD', str.lower) --> A B C D + seen = set() + seen_add = seen.add + if key is None: + for element in filterfalse(seen.__contains__, iterable): + seen_add(element) + yield element + else: + for element in iterable: + k = key(element) + if k not in seen: + seen_add(k) + yield element diff --git a/importlib_metadata/_meta.py b/importlib_metadata/_meta.py new file mode 100644 index 00000000..5cb690fe --- /dev/null +++ b/importlib_metadata/_meta.py @@ -0,0 +1,30 @@ +from ._compat import Protocol +from typing import Any, Dict, Iterator, List, TypeVar, Union + + +_T = TypeVar("_T") + + +class PackageMetadata(Protocol): + def __len__(self) -> int: + ... # pragma: no cover + + def __contains__(self, item: str) -> bool: + ... # pragma: no cover + + def __getitem__(self, key: str) -> str: + ... # pragma: no cover + + def __iter__(self) -> Iterator[str]: + ... # pragma: no cover + + def get_all(self, name: str, failobj: _T = ...) -> Union[List[Any], _T]: + """ + Return all values associated with a possibly multi-valued key. + """ + + @property + def json(self) -> Dict[str, Union[str, List[str]]]: + """ + A JSON-compatible form of the metadata. + """ diff --git a/importlib_metadata/_text.py b/importlib_metadata/_text.py new file mode 100644 index 00000000..766979d9 --- /dev/null +++ b/importlib_metadata/_text.py @@ -0,0 +1,99 @@ +import re + +from ._functools import method_cache + + +# from jaraco.text 3.5 +class FoldedCase(str): + """ + A case insensitive string class; behaves just like str + except compares equal when the only variation is case. + + >>> s = FoldedCase('hello world') + + >>> s == 'Hello World' + True + + >>> 'Hello World' == s + True + + >>> s != 'Hello World' + False + + >>> s.index('O') + 4 + + >>> s.split('O') + ['hell', ' w', 'rld'] + + >>> sorted(map(FoldedCase, ['GAMMA', 'alpha', 'Beta'])) + ['alpha', 'Beta', 'GAMMA'] + + Sequence membership is straightforward. + + >>> "Hello World" in [s] + True + >>> s in ["Hello World"] + True + + You may test for set inclusion, but candidate and elements + must both be folded. + + >>> FoldedCase("Hello World") in {s} + True + >>> s in {FoldedCase("Hello World")} + True + + String inclusion works as long as the FoldedCase object + is on the right. + + >>> "hello" in FoldedCase("Hello World") + True + + But not if the FoldedCase object is on the left: + + >>> FoldedCase('hello') in 'Hello World' + False + + In that case, use in_: + + >>> FoldedCase('hello').in_('Hello World') + True + + >>> FoldedCase('hello') > FoldedCase('Hello') + False + """ + + def __lt__(self, other): + return self.lower() < other.lower() + + def __gt__(self, other): + return self.lower() > other.lower() + + def __eq__(self, other): + return self.lower() == other.lower() + + def __ne__(self, other): + return self.lower() != other.lower() + + def __hash__(self): + return hash(self.lower()) + + def __contains__(self, other): + return super(FoldedCase, self).lower().__contains__(other.lower()) + + def in_(self, other): + "Does self appear in other?" + return self in FoldedCase(other) + + # cache lower since it's likely to be called frequently. + @method_cache + def lower(self): + return super(FoldedCase, self).lower() + + def index(self, sub): + return self.lower().index(sub.lower()) + + def split(self, splitter=' ', maxsplit=0): + pattern = re.compile(re.escape(splitter), re.I) + return pattern.split(self, maxsplit) diff --git a/pytest.ini b/pytest.ini index d7f0b115..6bf69af1 100644 --- a/pytest.ini +++ b/pytest.ini @@ -5,5 +5,3 @@ doctest_optionflags=ALLOW_UNICODE ELLIPSIS # workaround for warning pytest-dev/pytest#6178 junit_family=xunit2 filterwarnings= - # https://github.com/pytest-dev/pytest/issues/6928 - ignore:direct construction of .*Item has been deprecated:DeprecationWarning diff --git a/setup.cfg b/setup.cfg index 93f85279..6fc73125 100644 --- a/setup.cfg +++ b/setup.cfg @@ -15,7 +15,7 @@ classifiers = Programming Language :: Python :: 3 :: Only [options] -packages = find: +packages = find_namespace: include_package_data = true python_requires = >=3.6 install_requires = @@ -25,19 +25,26 @@ setup_requires = setuptools_scm[toml] >= 3.4.1 [options.packages.find] exclude = + build* + dist* + docs* tests* - docs + prepare* [options.extras_require] testing = # upstream - pytest >= 3.5, !=3.7.3 - pytest-checkdocs >= 1.2.3 + pytest >= 4.6 + pytest-checkdocs >= 2.4 pytest-flake8 - pytest-black >= 0.3.7; python_implementation != "PyPy" + # python_implementation: workaround for jaraco/skeleton#22 + # python_version: workaround for python/typed_ast#156 + pytest-black >= 0.3.7; python_implementation != "PyPy" and python_version < "3.10" pytest-cov - pytest-mypy; python_implementation != "PyPy" - pytest-enabler + # python_implementation: workaround for jaraco/skeleton#22 + # python_version: workaround for python/typed_ast#156 + pytest-mypy; python_implementation != "PyPy" and python_version < "3.10" + pytest-enabler >= 1.0.1 # local importlib_resources>=1.3; python_version < "3.9" diff --git a/tests/fixtures.py b/tests/fixtures.py index 779c1a58..8ef54f84 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -1,5 +1,6 @@ import os import sys +import copy import shutil import pathlib import tempfile @@ -86,6 +87,10 @@ class DistInfoPkg(OnSysPath, SiteDir): Version: 1.0.0 Requires-Dist: wheel >= 1.0 Requires-Dist: pytest; extra == 'test' + Keywords: sample package + + Once upon a time + There was a distinfo pkg """, "RECORD": "mod.py,sha256=abc,20\n", "entry_points.txt": """ @@ -104,6 +109,16 @@ def setUp(self): super(DistInfoPkg, self).setUp() build_files(DistInfoPkg.files, self.site_dir) + def make_uppercase(self): + """ + Rewrite metadata with everything uppercase. + """ + shutil.rmtree(self.site_dir / "distinfo_pkg-1.0.0.dist-info") + files = copy.deepcopy(DistInfoPkg.files) + info = files["distinfo_pkg-1.0.0.dist-info"] + info["METADATA"] = info["METADATA"].upper() + build_files(files, self.site_dir) + class DistInfoPkgWithDot(OnSysPath, SiteDir): files: FilesDef = { @@ -157,6 +172,9 @@ class EggInfoPkg(OnSysPath, SiteDir): Version: 1.0.0 Classifier: Intended Audience :: Developers Classifier: Topic :: Software Development :: Libraries + Keywords: sample package + Description: Once upon a time + There was an egginfo package """, "SOURCES.txt": """ mod.py @@ -250,7 +268,7 @@ def build_files(file_defs, prefix=pathlib.Path()): with full_name.open('wb') as f: f.write(contents) else: - with full_name.open('w') as f: + with full_name.open('w', encoding='utf-8') as f: f.write(DALS(contents)) diff --git a/tests/test_api.py b/tests/test_api.py index a386551f..b3c8c2f8 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,6 +1,8 @@ import re import textwrap import unittest +import warnings +import importlib from . import fixtures from importlib_metadata import ( @@ -64,18 +66,96 @@ def test_read_text(self): self.assertEqual(top_level.read_text(), 'mod\n') def test_entry_points(self): - entries = dict(entry_points()['entries']) + eps = entry_points() + assert 'entries' in eps.groups + entries = eps.select(group='entries') + assert 'main' in entries.names ep = entries['main'] self.assertEqual(ep.value, 'mod:main') self.assertEqual(ep.extras, []) def test_entry_points_distribution(self): - entries = dict(entry_points()['entries']) + entries = entry_points(group='entries') for entry in ("main", "ns:sub"): ep = entries[entry] self.assertIn(ep.dist.name, ('distinfo-pkg', 'egginfo-pkg')) self.assertEqual(ep.dist.version, "1.0.0") + def test_entry_points_unique_packages(self): + """ + Entry points should only be exposed for the first package + on sys.path with a given name. + """ + alt_site_dir = self.fixtures.enter_context(fixtures.tempdir()) + self.fixtures.enter_context(self.add_sys_path(alt_site_dir)) + alt_pkg = { + "distinfo_pkg-1.1.0.dist-info": { + "METADATA": """ + Name: distinfo-pkg + Version: 1.1.0 + """, + "entry_points.txt": """ + [entries] + main = mod:altmain + """, + }, + } + fixtures.build_files(alt_pkg, alt_site_dir) + entries = entry_points(group='entries') + assert not any( + ep.dist.name == 'distinfo-pkg' and ep.dist.version == '1.0.0' + for ep in entries + ) + # ns:sub doesn't exist in alt_pkg + assert 'ns:sub' not in entries + + def test_entry_points_missing_name(self): + with self.assertRaises(KeyError): + entry_points(group='entries')['missing'] + + def test_entry_points_missing_group(self): + assert entry_points(group='missing') == () + + def test_entry_points_dict_construction(self): + """ + Prior versions of entry_points() returned simple lists and + allowed casting those lists into maps by name using ``dict()``. + Capture this now deprecated use-case. + """ + with warnings.catch_warnings(record=True) as caught: + eps = dict(entry_points(group='entries')) + + assert 'main' in eps + assert eps['main'] == entry_points(group='entries')['main'] + + # check warning + expected = next(iter(caught)) + assert expected.category is DeprecationWarning + assert "Construction of dict of EntryPoints is deprecated" in str(expected) + + def test_entry_points_groups_getitem(self): + """ + Prior versions of entry_points() returned a dict. Ensure + that callers using '.__getitem__()' are supported but warned to + migrate. + """ + with warnings.catch_warnings(record=True): + entry_points()['entries'] == entry_points(group='entries') + + with self.assertRaises(KeyError): + entry_points()['missing'] + + def test_entry_points_groups_get(self): + """ + Prior versions of entry_points() returned a dict. Ensure + that callers using '.get()' are supported but warned to + migrate. + """ + with warnings.catch_warnings(record=True): + entry_points().get('missing', 'default') == 'default' + entry_points().get('entries', 'default') == entry_points()['entries'] + entry_points().get('missing', ()) == () + def test_metadata_for_this_package(self): md = metadata('egginfo-pkg') assert md['author'] == 'Steven Ma' @@ -166,6 +246,29 @@ def test_more_complex_deps_requires_text(self): assert deps == expected + def test_as_json(self): + md = metadata('distinfo-pkg').json + assert 'name' in md + assert md['keywords'] == ['sample', 'package'] + desc = md['description'] + assert desc.startswith('Once upon a time\nThere was') + assert len(md['requires_dist']) == 2 + + def test_as_json_egg_info(self): + md = metadata('egginfo-pkg').json + assert 'name' in md + assert md['keywords'] == ['sample', 'package'] + desc = md['description'] + assert desc.startswith('Once upon a time\nThere was') + assert len(md['classifier']) == 2 + + def test_as_json_odd_case(self): + self.make_uppercase() + md = metadata('distinfo-pkg').json + assert 'name' in md + assert len(md['requires_dist']) == 2 + assert md['keywords'] == ['SAMPLE', 'PACKAGE'] + class LegacyDots(fixtures.DistInfoPkgWithDotLegacy, unittest.TestCase): def test_name_normalization(self): @@ -196,3 +299,9 @@ def test_distribution_at_str(self): dist_info_path = self.site_dir / 'distinfo_pkg-1.0.0.dist-info' dist = Distribution.at(str(dist_info_path)) assert dist.version == '1.0.0' + + +class InvalidateCache(unittest.TestCase): + def test_invalidate_cache(self): + # No externally observable behavior, but ensures test coverage... + importlib.invalidate_caches() diff --git a/tests/test_integration.py b/tests/test_integration.py index 11835135..00e9021a 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -7,6 +7,7 @@ Distribution, MetadataPathFinder, _compat, + distributions, version, ) @@ -59,3 +60,16 @@ def test_search_dist_dirs(self): """ res = MetadataPathFinder._search_paths('any-name', []) assert list(res) == [] + + def test_interleaved_discovery(self): + """ + When the search is cached, it is + possible for searches to be interleaved, so make sure + those use-cases are safe. + + Ref #293 + """ + dists = distributions() + next(dists) + version('importlib_metadata') + next(dists) diff --git a/tests/test_main.py b/tests/test_main.py index 74979be8..f7c9c518 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -3,6 +3,7 @@ import pickle import textwrap import unittest +import warnings import importlib import importlib_metadata import pyfakefs.fake_filesystem_unittest as ffs @@ -57,13 +58,11 @@ def test_import_nonexistent_module(self): importlib.import_module('does_not_exist') def test_resolve(self): - entries = dict(entry_points()['entries']) - ep = entries['main'] + ep = entry_points(group='entries')['main'] self.assertEqual(ep.load().__name__, "main") def test_entrypoint_with_colon_in_name(self): - entries = dict(entry_points()['entries']) - ep = entries['ns:sub'] + ep = entry_points(group='entries')['ns:sub'] self.assertEqual(ep.value, 'mod:main') def test_resolve_without_attr(self): @@ -85,7 +84,7 @@ def pkg_with_dashes(site_dir): metadata_dir = site_dir / 'my_pkg.dist-info' metadata_dir.mkdir() metadata = metadata_dir / 'METADATA' - with metadata.open('w') as strm: + with metadata.open('w', encoding='utf-8') as strm: strm.write('Version: 1.0\n') return 'my-pkg' @@ -106,7 +105,7 @@ def pkg_with_mixed_case(site_dir): metadata_dir = site_dir / 'CherryPy.dist-info' metadata_dir.mkdir() metadata = metadata_dir / 'METADATA' - with metadata.open('w') as strm: + with metadata.open('w', encoding='utf-8') as strm: strm.write('Version: 1.0\n') return 'CherryPy' @@ -131,7 +130,7 @@ def pkg_with_non_ascii_description(site_dir): metadata_dir.mkdir() metadata = metadata_dir / 'METADATA' with metadata.open('w', encoding='utf-8') as fp: - fp.write('Description: pôrˈtend\n') + fp.write('Description: pôrˈtend') return 'portend' @staticmethod @@ -151,7 +150,7 @@ def pkg_with_non_ascii_description_egg_info(site_dir): pôrˈtend """ - ).lstrip() + ).strip() ) return 'portend' @@ -163,7 +162,7 @@ def test_metadata_loads(self): def test_metadata_loads_egg_info(self): pkg_name = self.pkg_with_non_ascii_description_egg_info(self.site_dir) meta = metadata(pkg_name) - assert meta.get_payload() == 'pôrˈtend\n' + assert meta['Description'] == 'pôrˈtend' class DiscoveryTests(fixtures.EggInfoPkg, fixtures.DistInfoPkg, unittest.TestCase): @@ -249,7 +248,8 @@ def test_json_dump(self): json should not expect to be able to dump an EntryPoint """ with self.assertRaises(Exception): - json.dumps(self.ep) + with warnings.catch_warnings(record=True): + json.dumps(self.ep) def test_module(self): assert self.ep.module == 'value' diff --git a/tests/test_zip.py b/tests/test_zip.py index 67311da2..4279046d 100644 --- a/tests/test_zip.py +++ b/tests/test_zip.py @@ -45,7 +45,7 @@ def test_zip_version_does_not_match(self): version('definitely-not-installed') def test_zip_entry_points(self): - scripts = dict(entry_points()['console_scripts']) + scripts = entry_points(group='console_scripts') entry_point = scripts['example'] self.assertEqual(entry_point.value, 'example:main') entry_point = scripts['Example'] diff --git a/tox.ini b/tox.ini index 11f52d7a..42d28ba3 100644 --- a/tox.ini +++ b/tox.ini @@ -33,18 +33,50 @@ commands = diff-cover coverage.xml --compare-branch=origin/main --html-report diffcov.html diff-cover coverage.xml --compare-branch=origin/main --fail-under=100 +[perf] +deps = + ipython + pyperf + path +commands = + python -c "import path; path.Path('{env:SCOPE}.json').remove_p()" + + python -m pyperf timeit --name discovery --append {env:SCOPE}.json -s 'import importlib_metadata' 'importlib_metadata.distribution("ipython")' + + python -m pyperf timeit --name 'entry_points()' --append {env:SCOPE}.json -s 'import importlib_metadata' 'importlib_metadata.entry_points()' + + python -m pyperf timeit --name 'cached distribution' --append {env:SCOPE}.json -s 'import importlib_metadata; importlib_metadata.distribution("ipython")' 'importlib_metadata.distribution("ipython")' + + python -m pyperf timeit --name 'uncached distribution' --append {env:SCOPE}.json -s 'import importlib, importlib_metadata' 'importlib.invalidate_caches(); importlib_metadata.distribution("ipython")' + [testenv:perf] use_develop = False +# change dir to avoid picking up local package. +changedir = {toxworkdir} +setenv = + SCOPE = local +deps = {[perf]deps} +commands = {[perf]commands} + +[testenv:perf-ref] +# compare perf results to the main branch +skip_install = True +# change dir to avoid picking up local package. +changedir = {toxworkdir} +setenv = + SCOPE = main deps = - ipython + {[perf]deps} + git+https://github.com/python/importlib_metadata commands = - python -m timeit -s 'import importlib_metadata' -- 'importlib_metadata.distribution("ipython")' + {[perf]commands} + python -m pyperf compare_to --verbose main.json local.json --table [testenv:release] skip_install = True deps = build - twine[keyring]>=1.13 + twine>=3 path jaraco.develop>=7.1 passenv = pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy