From bde4e157b4ac1924a953897c92ed0b2638cdb229 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 15 Jan 2021 12:34:13 -0500 Subject: [PATCH 001/105] Fix typo in docs. Ref bpo-42728. --- docs/using.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/using.rst b/docs/using.rst index efa40f86..00409867 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -200,9 +200,9 @@ Thus, an alternative way to get the version number is through the There are all kinds of additional metadata available on the ``Distribution`` instance:: - >>> d.metadata['Requires-Python'] + >>> dist.metadata['Requires-Python'] '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*' - >>> d.metadata['License'] + >>> dist.metadata['License'] 'MIT' The full set of available metadata is not described here. See :pep:`566` From 0df40810ec54590c888ae0e4073d73f731c91f4a Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 15 Jan 2021 19:16:28 -0500 Subject: [PATCH 002/105] Add support for namespace packages. Closes jaraco/skeleton#40. --- setup.cfg | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 88bc263a..106763e3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -15,12 +15,18 @@ classifiers = Programming Language :: Python :: 3 :: Only [options] -packages = find: +packages = find_namespace: include_package_data = true python_requires = >=3.6 install_requires = setup_requires = setuptools_scm[toml] >= 3.4.1 +[options.packages.find] +exclude = + build* + docs* + tests* + [options.extras_require] testing = # upstream From 9950845a8b3f35843057d3708ed75ef30dd62659 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 21 Jan 2021 09:34:23 -0500 Subject: [PATCH 003/105] Add doctest illustrating the usage of constructing dict from EntryPoints. --- importlib_metadata/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 079cc0f9..fac3063b 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -138,7 +138,11 @@ def _for(self, dist): def __iter__(self): """ - Supply iter so one may construct dicts of EntryPoints easily. + Supply iter so one may construct dicts of EntryPoints by name. + + >>> eps = [EntryPoint('a', 'b', 'c'), EntryPoint('d', 'e', 'f')] + >>> dict(eps)['a'] + EntryPoint(name='a', value='b', group='c') """ return iter((self.name, self)) From a49252b513cf1e25e3885c60f046269ea293f13e Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 22 Jan 2021 21:21:37 -0500 Subject: [PATCH 004/105] Add explicit interfaces for loaded entrypoints, resolvable first by group then by name. --- importlib_metadata/__init__.py | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index fac3063b..7ca3e938 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -153,6 +153,27 @@ def __reduce__(self): ) +class EntryPoints(tuple): + """ + A collection of EntryPoint objects, retrievable by name. + """ + + def __getitem__(self, name) -> EntryPoint: + try: + return next(ep for ep in self if ep.name == name) + except Exception: + raise KeyError(name) + + +class GroupedEntryPoints(tuple): + """ + A collection of EntryPoint objects, retrievable by group. + """ + + def __getitem__(self, group) -> EntryPoints: + return EntryPoints(ep for ep in self if ep.group == group) + + class PackagePath(pathlib.PurePosixPath): """A reference to a path in a package""" @@ -308,7 +329,8 @@ def version(self): @property def entry_points(self): - return list(EntryPoint._from_text_for(self.read_text('entry_points.txt'), self)) + eps = EntryPoint._from_text_for(self.read_text('entry_points.txt'), self) + return GroupedEntryPoints(eps) @property def files(self): @@ -647,10 +669,7 @@ def entry_points(): :return: EntryPoint objects for all installed packages. """ eps = itertools.chain.from_iterable(dist.entry_points for dist in distributions()) - by_group = operator.attrgetter('group') - ordered = sorted(eps, key=by_group) - grouped = itertools.groupby(ordered, by_group) - return {group: tuple(eps) for group, eps in grouped} + return GroupedEntryPoints(eps) def files(distribution_name): From 6596183f79a3973698c4b2b825b12682ac6e7d96 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 22 Jan 2021 21:29:26 -0500 Subject: [PATCH 005/105] Update tests to use new preferred API. --- tests/test_api.py | 12 +++++++++--- tests/test_main.py | 6 ++---- tests/test_zip.py | 2 +- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/tests/test_api.py b/tests/test_api.py index a386551f..e6a5adeb 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -64,18 +64,24 @@ def test_read_text(self): self.assertEqual(top_level.read_text(), 'mod\n') def test_entry_points(self): - entries = dict(entry_points()['entries']) - ep = entries['main'] + ep = entry_points()['entries']['main'] self.assertEqual(ep.value, 'mod:main') self.assertEqual(ep.extras, []) def test_entry_points_distribution(self): - entries = dict(entry_points()['entries']) + entries = entry_points()['entries'] for entry in ("main", "ns:sub"): ep = entries[entry] self.assertIn(ep.dist.name, ('distinfo-pkg', 'egginfo-pkg')) self.assertEqual(ep.dist.version, "1.0.0") + def test_entry_points_missing_name(self): + with self.assertRaises(KeyError): + entry_points()['entries']['missing'] + + def test_entry_points_missing_group(self): + assert entry_points()['missing'] == () + def test_metadata_for_this_package(self): md = metadata('egginfo-pkg') assert md['author'] == 'Steven Ma' diff --git a/tests/test_main.py b/tests/test_main.py index 74979be8..566262f6 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -57,13 +57,11 @@ def test_import_nonexistent_module(self): importlib.import_module('does_not_exist') def test_resolve(self): - entries = dict(entry_points()['entries']) - ep = entries['main'] + ep = entry_points()['entries']['main'] self.assertEqual(ep.load().__name__, "main") def test_entrypoint_with_colon_in_name(self): - entries = dict(entry_points()['entries']) - ep = entries['ns:sub'] + ep = entry_points()['entries']['ns:sub'] self.assertEqual(ep.value, 'mod:main') def test_resolve_without_attr(self): diff --git a/tests/test_zip.py b/tests/test_zip.py index 67311da2..5a63465f 100644 --- a/tests/test_zip.py +++ b/tests/test_zip.py @@ -45,7 +45,7 @@ def test_zip_version_does_not_match(self): version('definitely-not-installed') def test_zip_entry_points(self): - scripts = dict(entry_points()['console_scripts']) + scripts = entry_points()['console_scripts'] entry_point = scripts['example'] self.assertEqual(entry_point.value, 'example:main') entry_point = scripts['Example'] From b5081fa78358a9bb7c47eedfe084b3f96c024c63 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 22 Jan 2021 21:33:37 -0500 Subject: [PATCH 006/105] Capture the legacy expectation. --- tests/test_api.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_api.py b/tests/test_api.py index e6a5adeb..c3e8b532 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -82,6 +82,16 @@ def test_entry_points_missing_name(self): def test_entry_points_missing_group(self): assert entry_points()['missing'] == () + def test_entry_points_dict_construction(self): + """ + Prior versions of entry_points() returned simple lists and + allowed casting those lists into maps by name using ``dict()``. + Capture this now deprecated use-case. + """ + eps = dict(entry_points()['entries']) + assert 'main' in eps + assert eps['main'] == entry_points()['entries']['main'] + def test_metadata_for_this_package(self): md = metadata('egginfo-pkg') assert md['author'] == 'Steven Ma' From 99dd2242ab9c8c0b4a082e135f6bbda11c19540a Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 22 Jan 2021 21:50:02 -0500 Subject: [PATCH 007/105] Deprecate dict construction from EntryPoint items. --- importlib_metadata/__init__.py | 10 ++++++---- tests/test_api.py | 10 +++++++++- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 7ca3e938..681743dc 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -7,6 +7,7 @@ import email import pathlib import operator +import warnings import functools import itertools import posixpath @@ -139,11 +140,12 @@ def _for(self, dist): def __iter__(self): """ Supply iter so one may construct dicts of EntryPoints by name. - - >>> eps = [EntryPoint('a', 'b', 'c'), EntryPoint('d', 'e', 'f')] - >>> dict(eps)['a'] - EntryPoint(name='a', value='b', group='c') """ + msg = ( + "Construction of dict of EntryPoints is deprecated in " + "favor of EntryPoints." + ) + warnings.warn(msg, DeprecationWarning) return iter((self.name, self)) def __reduce__(self): diff --git a/tests/test_api.py b/tests/test_api.py index c3e8b532..8ce2e468 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,6 +1,7 @@ import re import textwrap import unittest +import warnings from . import fixtures from importlib_metadata import ( @@ -88,10 +89,17 @@ def test_entry_points_dict_construction(self): allowed casting those lists into maps by name using ``dict()``. Capture this now deprecated use-case. """ - eps = dict(entry_points()['entries']) + with warnings.catch_warnings(record=True) as caught: + eps = dict(entry_points()['entries']) + assert 'main' in eps assert eps['main'] == entry_points()['entries']['main'] + # check warning + expected = next(iter(caught)) + assert expected.category is DeprecationWarning + assert "Construction of dict of EntryPoints is deprecated" in str(expected) + def test_metadata_for_this_package(self): md = metadata('egginfo-pkg') assert md['author'] == 'Steven Ma' From 2eeb629021d7218516f5ee43de51b8d93d32828a Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 22 Jan 2021 21:54:11 -0500 Subject: [PATCH 008/105] Suppress warning in test_json_dump. --- tests/test_main.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_main.py b/tests/test_main.py index 566262f6..b778572c 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -3,6 +3,7 @@ import pickle import textwrap import unittest +import warnings import importlib import importlib_metadata import pyfakefs.fake_filesystem_unittest as ffs @@ -247,7 +248,8 @@ def test_json_dump(self): json should not expect to be able to dump an EntryPoint """ with self.assertRaises(Exception): - json.dumps(self.ep) + with warnings.catch_warnings(record=True): + json.dumps(self.ep) def test_module(self): assert self.ep.module == 'value' From eedd810b90083fd5a2b0bb398478527011c474eb Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 22 Jan 2021 22:06:29 -0500 Subject: [PATCH 009/105] Add 'groups' and 'names' to EntryPoints collections. --- importlib_metadata/__init__.py | 8 ++++++++ tests/test_api.py | 6 +++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 681743dc..60967cd4 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -166,6 +166,10 @@ def __getitem__(self, name) -> EntryPoint: except Exception: raise KeyError(name) + @property + def names(self): + return set(ep.name for ep in self) + class GroupedEntryPoints(tuple): """ @@ -175,6 +179,10 @@ class GroupedEntryPoints(tuple): def __getitem__(self, group) -> EntryPoints: return EntryPoints(ep for ep in self if ep.group == group) + @property + def groups(self): + return set(ep.group for ep in self) + class PackagePath(pathlib.PurePosixPath): """A reference to a path in a package""" diff --git a/tests/test_api.py b/tests/test_api.py index 8ce2e468..7672556a 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -65,7 +65,11 @@ def test_read_text(self): self.assertEqual(top_level.read_text(), 'mod\n') def test_entry_points(self): - ep = entry_points()['entries']['main'] + eps = entry_points() + assert 'entries' in eps.groups + entries = eps['entries'] + assert 'main' in entries.names + ep = entries['main'] self.assertEqual(ep.value, 'mod:main') self.assertEqual(ep.extras, []) From 720362fe25dd0211432784de02dd483b53ee7be8 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 22 Jan 2021 22:10:34 -0500 Subject: [PATCH 010/105] Update documentation on EntryPoints to reflect the new, preferred accessors. --- docs/using.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/using.rst b/docs/using.rst index 00409867..534c1dea 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -67,7 +67,7 @@ This package provides the following functionality via its public API. Entry points ------------ -The ``entry_points()`` function returns a dictionary of all entry points, +The ``entry_points()`` function returns a sequence of all entry points, keyed by group. Entry points are represented by ``EntryPoint`` instances; each ``EntryPoint`` has a ``.name``, ``.group``, and ``.value`` attributes and a ``.load()`` method to resolve the value. There are also ``.module``, @@ -75,10 +75,12 @@ a ``.load()`` method to resolve the value. There are also ``.module``, ``.value`` attribute:: >>> eps = entry_points() - >>> list(eps) + >>> sorted(eps.groups) ['console_scripts', 'distutils.commands', 'distutils.setup_keywords', 'egg_info.writers', 'setuptools.installation'] >>> scripts = eps['console_scripts'] - >>> wheel = [ep for ep in scripts if ep.name == 'wheel'][0] + >>> 'wheel' in scripts.names + True + >>> wheel = scripts['wheel'] >>> wheel EntryPoint(name='wheel', value='wheel.cli:main', group='console_scripts') >>> wheel.module From 28adeb8f84ac3e5052ea24c93b4fa3816e1fe4e6 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 22 Jan 2021 22:13:52 -0500 Subject: [PATCH 011/105] Update changelog. --- CHANGES.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 57901f23..02900674 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,16 @@ +v3.5.0 +====== + +* ``entry_points()`` now returns an ``GroupedEntryPoints`` + object, a tuple of all entry points but with a convenience + property ``groups`` and ``__getitem__`` accessor. Further, + accessing a group returns an ``EntryPoints`` object, + another tuple of entry points in the group, accessible by + name. Construction of entry points using + ``dict([EntryPoint, ...])`` is now deprecated and raises + an appropriate DeprecationWarning and will be removed in + a future version. + v3.4.0 ====== From 342a94ba5c373b01f3c5b827da1d4bd76ff2b04f Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 22 Jan 2021 22:54:23 -0500 Subject: [PATCH 012/105] Add deprecated .get to GroupedEntryPoints and test to capture expectation. --- importlib_metadata/__init__.py | 8 ++++++++ tests/test_api.py | 11 +++++++++++ 2 files changed, 19 insertions(+) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 60967cd4..edcf2691 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -183,6 +183,14 @@ def __getitem__(self, group) -> EntryPoints: def groups(self): return set(ep.group for ep in self) + def get(self, group, default=None): + """ + For backward compatibility, supply .get + """ + msg = "GroupedEntryPoints.get is deprecated. Just use __getitem__." + warnings.warn(msg, DeprecationWarning) + return self[group] or default + class PackagePath(pathlib.PurePosixPath): """A reference to a path in a package""" diff --git a/tests/test_api.py b/tests/test_api.py index 7672556a..dc0c7870 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -104,6 +104,17 @@ def test_entry_points_dict_construction(self): assert expected.category is DeprecationWarning assert "Construction of dict of EntryPoints is deprecated" in str(expected) + def test_entry_points_groups_get(self): + """ + Prior versions of entry_points() returned a dict. Ensure + that callers using '.get()' are supported but warned to + migrate. + """ + with warnings.catch_warnings(record=True): + entry_points().get('missing', 'default') == 'default' + entry_points().get('entries', 'default') == entry_points()['entries'] + entry_points().get('missing', ()) == entry_points()['missing'] + def test_metadata_for_this_package(self): md = metadata('egginfo-pkg') assert md['author'] == 'Steven Ma' From 4e2603f5bd0b6c86a365e98e3e891fdcdfa9bc13 Mon Sep 17 00:00:00 2001 From: Antony Lee Date: Sat, 23 Jan 2021 23:30:33 +0100 Subject: [PATCH 013/105] Separately profile cached and uncached lookup performance. ... in preparation of adding a lookup cache. --- tox.ini | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 11f52d7a..f1586632 100644 --- a/tox.ini +++ b/tox.ini @@ -38,7 +38,10 @@ use_develop = False deps = ipython commands = - python -m timeit -s 'import importlib_metadata' -- 'importlib_metadata.distribution("ipython")' + python -c 'print("Cached lookup performance")' + python -m timeit -s 'import importlib_metadata; importlib_metadata.distribution("ipython")' -- 'importlib_metadata.distribution("ipython")' + python -c 'print("Uncached lookup performance")' + python -m timeit -s 'import importlib, importlib_metadata' -- 'importlib.invalidate_caches(); importlib_metadata.distribution("ipython")' [testenv:release] skip_install = True From 35bc40379340e2b23d56a57c5c782ffe93a53396 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 26 Jan 2021 15:38:00 -0500 Subject: [PATCH 014/105] When resolving entry points globally, only expose entry points for unique distributions. Fixes #280. --- .coveragerc | 1 + importlib_metadata/__init__.py | 7 ++++++- importlib_metadata/_itertools.py | 19 +++++++++++++++++++ 3 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 importlib_metadata/_itertools.py diff --git a/.coveragerc b/.coveragerc index 66d32472..e91bbd6b 100644 --- a/.coveragerc +++ b/.coveragerc @@ -3,6 +3,7 @@ omit = */.tox/* tests/* prepare/* + */_itertools.py [report] show_missing = True diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index fac3063b..4c420a55 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -19,6 +19,8 @@ Protocol, ) +from ._itertools import unique_everseen + from configparser import ConfigParser from contextlib import suppress from importlib import import_module @@ -646,7 +648,10 @@ def entry_points(): :return: EntryPoint objects for all installed packages. """ - eps = itertools.chain.from_iterable(dist.entry_points for dist in distributions()) + unique = functools.partial(unique_everseen, key=operator.attrgetter('name')) + eps = itertools.chain.from_iterable( + dist.entry_points for dist in unique(distributions()) + ) by_group = operator.attrgetter('group') ordered = sorted(eps, key=by_group) grouped = itertools.groupby(ordered, by_group) diff --git a/importlib_metadata/_itertools.py b/importlib_metadata/_itertools.py new file mode 100644 index 00000000..dd45f2f0 --- /dev/null +++ b/importlib_metadata/_itertools.py @@ -0,0 +1,19 @@ +from itertools import filterfalse + + +def unique_everseen(iterable, key=None): + "List unique elements, preserving order. Remember all elements ever seen." + # unique_everseen('AAAABBBCCDAABBB') --> A B C D + # unique_everseen('ABBCcAD', str.lower) --> A B C D + seen = set() + seen_add = seen.add + if key is None: + for element in filterfalse(seen.__contains__, iterable): + seen_add(element) + yield element + else: + for element in iterable: + k = key(element) + if k not in seen: + seen_add(k) + yield element From 8935288354ba3b843a13cdb5577c3cdb7a672e0b Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 26 Jan 2021 18:49:51 -0500 Subject: [PATCH 015/105] Add test capturing failed expectation. Ref #280. --- tests/test_api.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/test_api.py b/tests/test_api.py index a386551f..04a5b9d3 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -76,6 +76,34 @@ def test_entry_points_distribution(self): self.assertIn(ep.dist.name, ('distinfo-pkg', 'egginfo-pkg')) self.assertEqual(ep.dist.version, "1.0.0") + def test_entry_points_unique_packages(self): + """ + Entry points should only be exposed for the first package + on sys.path with a given name. + """ + alt_site_dir = self.fixtures.enter_context(fixtures.tempdir()) + self.fixtures.enter_context(self.add_sys_path(alt_site_dir)) + alt_pkg = { + "distinfo_pkg-1.1.0.dist-info": { + "METADATA": """ + Name: distinfo-pkg + Version: 1.1.0 + """, + "entry_points.txt": """ + [entries] + main = mod:altmain + """, + }, + } + fixtures.build_files(alt_pkg, alt_site_dir) + entries = dict(entry_points()['entries']) + assert not any( + ep.dist.name == 'distinfo-pkg' and ep.dist.version == '1.0.0' + for ep in entries.values() + ) + # ns:sub doesn't exist in alt_pkg + assert 'ns:sub' not in entries + def test_metadata_for_this_package(self): md = metadata('egginfo-pkg') assert md['author'] == 'Steven Ma' From 6cb27d381c0330c65fb07ce5a68d4525dea94600 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 26 Jan 2021 20:28:36 -0500 Subject: [PATCH 016/105] Update changelog --- CHANGES.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 57901f23..2bc3a356 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,9 @@ +v3.5.0 +====== + +* #280: ``entry_points`` now only returns entry points for + unique distributions (by name). + v3.4.0 ====== From 2b64fa218dcb902baa7bcc2da42e834eec6bdab8 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 26 Jan 2021 20:33:10 -0500 Subject: [PATCH 017/105] Add performance test for entry_points --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 11f52d7a..32796a47 100644 --- a/tox.ini +++ b/tox.ini @@ -39,6 +39,7 @@ deps = ipython commands = python -m timeit -s 'import importlib_metadata' -- 'importlib_metadata.distribution("ipython")' + python -m timeit -s 'import importlib_metadata' -- 'importlib_metadata.entry_points()' [testenv:release] skip_install = True From 51298a2cc4faa7253e9fe41d7a9574cf9aac997c Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 9 Feb 2021 23:08:58 -0500 Subject: [PATCH 018/105] Normalize indentation --- setup.cfg | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index 106763e3..8df8d273 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,9 +23,9 @@ setup_requires = setuptools_scm[toml] >= 3.4.1 [options.packages.find] exclude = - build* - docs* - tests* + build* + docs* + tests* [options.extras_require] testing = From 9448e13a10648ae5a086247dea8a17efff31b816 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 15 Feb 2021 18:05:18 -0500 Subject: [PATCH 019/105] Make entry point collections (more) immutable. --- importlib_metadata/__init__.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index edcf2691..4c8188ae 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -157,9 +157,11 @@ def __reduce__(self): class EntryPoints(tuple): """ - A collection of EntryPoint objects, retrievable by name. + An immutable collection of EntryPoint objects, retrievable by name. """ + __slots__ = () + def __getitem__(self, name) -> EntryPoint: try: return next(ep for ep in self if ep.name == name) @@ -173,9 +175,11 @@ def names(self): class GroupedEntryPoints(tuple): """ - A collection of EntryPoint objects, retrievable by group. + An immutable collection of EntryPoint objects, retrievable by group. """ + __slots__ = () + def __getitem__(self, group) -> EntryPoints: return EntryPoints(ep for ep in self if ep.group == group) From 71fd4a7b6a8141becd431edf51dac590493d61c2 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 15 Feb 2021 21:28:16 -0500 Subject: [PATCH 020/105] Hide the deprecation warning from flake8 users --- importlib_metadata/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 4c8188ae..f9af7824 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -5,6 +5,7 @@ import sys import zipp import email +import inspect import pathlib import operator import warnings @@ -191,8 +192,9 @@ def get(self, group, default=None): """ For backward compatibility, supply .get """ + is_flake8 = any('flake8' in str(frame) for frame in inspect.stack()) msg = "GroupedEntryPoints.get is deprecated. Just use __getitem__." - warnings.warn(msg, DeprecationWarning) + is_flake8 or warnings.warn(msg, DeprecationWarning) return self[group] or default From 8320adef797d5f14d9fff7b58ebc2a31a2a6a437 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 16 Feb 2021 22:01:49 -0500 Subject: [PATCH 021/105] Instead of presenting separate contexts for EntryPoints, unify into a single collection that can select on 'name' or 'group' or possibly other attributes. Expose that selection in the 'entry_points' function. --- docs/using.rst | 2 +- importlib_metadata/__init__.py | 56 +++++++++++++++++++--------------- tests/test_api.py | 26 +++++++++++----- tests/test_main.py | 4 +-- tests/test_zip.py | 2 +- 5 files changed, 54 insertions(+), 36 deletions(-) diff --git a/docs/using.rst b/docs/using.rst index 534c1dea..bdfe3e82 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -77,7 +77,7 @@ a ``.load()`` method to resolve the value. There are also ``.module``, >>> eps = entry_points() >>> sorted(eps.groups) ['console_scripts', 'distutils.commands', 'distutils.setup_keywords', 'egg_info.writers', 'setuptools.installation'] - >>> scripts = eps['console_scripts'] + >>> scripts = eps.select(group='console_scripts') >>> 'wheel' in scripts.names True >>> wheel = scripts['wheel'] diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index f9af7824..77057703 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -130,10 +130,6 @@ def _from_text(cls, text): config.read_string(text) return cls._from_config(config) - @classmethod - def _from_text_for(cls, text, dist): - return (ep._for(dist) for ep in cls._from_text(text)) - def _for(self, dist): self.dist = dist return self @@ -155,35 +151,42 @@ def __reduce__(self): (self.name, self.value, self.group), ) + def matches(self, **params): + attrs = (getattr(self, param) for param in params) + return all(map(operator.eq, params.values(), attrs)) + class EntryPoints(tuple): """ - An immutable collection of EntryPoint objects, retrievable by name. + An immutable collection of selectable EntryPoint objects. """ __slots__ = () - def __getitem__(self, name) -> EntryPoint: + def __getitem__(self, name) -> Union[EntryPoint, 'EntryPoints']: try: - return next(ep for ep in self if ep.name == name) - except Exception: + match = next(iter(self.select(name=name))) + return match + except StopIteration: + if name in self.groups: + return self._group_getitem(name) raise KeyError(name) + def _group_getitem(self, name): + """ + For backward compatability, supply .__getitem__ for groups. + """ + msg = "GroupedEntryPoints.__getitem__ is deprecated for groups. Use select." + warnings.warn(msg, DeprecationWarning) + return self.select(group=name) + + def select(self, **params): + return EntryPoints(ep for ep in self if ep.matches(**params)) + @property def names(self): return set(ep.name for ep in self) - -class GroupedEntryPoints(tuple): - """ - An immutable collection of EntryPoint objects, retrievable by group. - """ - - __slots__ = () - - def __getitem__(self, group) -> EntryPoints: - return EntryPoints(ep for ep in self if ep.group == group) - @property def groups(self): return set(ep.group for ep in self) @@ -193,9 +196,13 @@ def get(self, group, default=None): For backward compatibility, supply .get """ is_flake8 = any('flake8' in str(frame) for frame in inspect.stack()) - msg = "GroupedEntryPoints.get is deprecated. Just use __getitem__." + msg = "GroupedEntryPoints.get is deprecated. Use select." is_flake8 or warnings.warn(msg, DeprecationWarning) - return self[group] or default + return self.select(group=group) or default + + @classmethod + def _from_text_for(cls, text, dist): + return cls(ep._for(dist) for ep in EntryPoint._from_text(text)) class PackagePath(pathlib.PurePosixPath): @@ -353,8 +360,7 @@ def version(self): @property def entry_points(self): - eps = EntryPoint._from_text_for(self.read_text('entry_points.txt'), self) - return GroupedEntryPoints(eps) + return EntryPoints._from_text_for(self.read_text('entry_points.txt'), self) @property def files(self): @@ -687,13 +693,13 @@ def version(distribution_name): return distribution(distribution_name).version -def entry_points(): +def entry_points(**params): """Return EntryPoint objects for all installed packages. :return: EntryPoint objects for all installed packages. """ eps = itertools.chain.from_iterable(dist.entry_points for dist in distributions()) - return GroupedEntryPoints(eps) + return EntryPoints(eps).select(**params) def files(distribution_name): diff --git a/tests/test_api.py b/tests/test_api.py index dc0c7870..a6466309 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -67,14 +67,14 @@ def test_read_text(self): def test_entry_points(self): eps = entry_points() assert 'entries' in eps.groups - entries = eps['entries'] + entries = eps.select(group='entries') assert 'main' in entries.names ep = entries['main'] self.assertEqual(ep.value, 'mod:main') self.assertEqual(ep.extras, []) def test_entry_points_distribution(self): - entries = entry_points()['entries'] + entries = entry_points(group='entries') for entry in ("main", "ns:sub"): ep = entries[entry] self.assertIn(ep.dist.name, ('distinfo-pkg', 'egginfo-pkg')) @@ -82,10 +82,10 @@ def test_entry_points_distribution(self): def test_entry_points_missing_name(self): with self.assertRaises(KeyError): - entry_points()['entries']['missing'] + entry_points(group='entries')['missing'] def test_entry_points_missing_group(self): - assert entry_points()['missing'] == () + assert entry_points(group='missing') == () def test_entry_points_dict_construction(self): """ @@ -94,16 +94,28 @@ def test_entry_points_dict_construction(self): Capture this now deprecated use-case. """ with warnings.catch_warnings(record=True) as caught: - eps = dict(entry_points()['entries']) + eps = dict(entry_points(group='entries')) assert 'main' in eps - assert eps['main'] == entry_points()['entries']['main'] + assert eps['main'] == entry_points(group='entries')['main'] # check warning expected = next(iter(caught)) assert expected.category is DeprecationWarning assert "Construction of dict of EntryPoints is deprecated" in str(expected) + def test_entry_points_groups_getitem(self): + """ + Prior versions of entry_points() returned a dict. Ensure + that callers using '.__getitem__()' are supported but warned to + migrate. + """ + with warnings.catch_warnings(record=True): + entry_points()['entries'] == entry_points(group='entries') + + with self.assertRaises(KeyError): + entry_points()['missing'] + def test_entry_points_groups_get(self): """ Prior versions of entry_points() returned a dict. Ensure @@ -113,7 +125,7 @@ def test_entry_points_groups_get(self): with warnings.catch_warnings(record=True): entry_points().get('missing', 'default') == 'default' entry_points().get('entries', 'default') == entry_points()['entries'] - entry_points().get('missing', ()) == entry_points()['missing'] + entry_points().get('missing', ()) == () def test_metadata_for_this_package(self): md = metadata('egginfo-pkg') diff --git a/tests/test_main.py b/tests/test_main.py index b778572c..e8a66c0d 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -58,11 +58,11 @@ def test_import_nonexistent_module(self): importlib.import_module('does_not_exist') def test_resolve(self): - ep = entry_points()['entries']['main'] + ep = entry_points(group='entries')['main'] self.assertEqual(ep.load().__name__, "main") def test_entrypoint_with_colon_in_name(self): - ep = entry_points()['entries']['ns:sub'] + ep = entry_points(group='entries')['ns:sub'] self.assertEqual(ep.value, 'mod:main') def test_resolve_without_attr(self): diff --git a/tests/test_zip.py b/tests/test_zip.py index 5a63465f..4279046d 100644 --- a/tests/test_zip.py +++ b/tests/test_zip.py @@ -45,7 +45,7 @@ def test_zip_version_does_not_match(self): version('definitely-not-installed') def test_zip_entry_points(self): - scripts = entry_points()['console_scripts'] + scripts = entry_points(group='console_scripts') entry_point = scripts['example'] self.assertEqual(entry_point.value, 'example:main') entry_point = scripts['Example'] From 61a265c18bc481c1e49fded0476a04ba5f75b750 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 21 Feb 2021 12:32:34 -0500 Subject: [PATCH 022/105] Fix perf tests --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index a7476d12..5466077e 100644 --- a/tox.ini +++ b/tox.ini @@ -38,9 +38,9 @@ use_develop = False deps = ipython commands = - python -m 'print("Simple discovery performance")' + python -c 'print("Simple discovery performance")' python -m timeit -s 'import importlib_metadata' -- 'importlib_metadata.distribution("ipython")' - python -m 'print("Entry point discovery performance")' + python -c 'print("Entry point discovery performance")' python -m timeit -s 'import importlib_metadata' -- 'importlib_metadata.entry_points()' python -c 'print("Cached lookup performance")' python -m timeit -s 'import importlib_metadata; importlib_metadata.distribution("ipython")' -- 'importlib_metadata.distribution("ipython")' From 743af7249d56e55a7c2c5f3111958ceee008d8ea Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 21 Feb 2021 13:04:46 -0500 Subject: [PATCH 023/105] Exclude dist from discovered packages. Fixes jaraco/skeleton#46. --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index 8df8d273..af246415 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,6 +24,7 @@ setup_requires = setuptools_scm[toml] >= 3.4.1 [options.packages.find] exclude = build* + dist* docs* tests* From 0da7579828f00c267915303481567174a4ae00be Mon Sep 17 00:00:00 2001 From: Antony Lee Date: Sun, 10 Jan 2021 23:20:15 +0100 Subject: [PATCH 024/105] More speedup via mtime-base caching. Caching based on mtime is similar to the one done on importlib's FileFinder. Locally, on a large-ish environment, this speeds up repeated calls to `distribution("pip")` ~10x. --- importlib_metadata/__init__.py | 90 +++++++++++++++++++--------------- tests/test_api.py | 7 +++ 2 files changed, 58 insertions(+), 39 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 4c420a55..fb91957c 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -464,9 +464,15 @@ class FastPath: children. """ - def __init__(self, root): + @functools.lru_cache() # type: ignore + def __new__(cls, root): + self = object().__new__(cls) self.root = str(root) self.base = os.path.basename(self.root).lower() + self.last_mtime = -1 + self.infos = {} + self.eggs = {} + return self def joinpath(self, child): return pathlib.Path(self.root, child) @@ -482,15 +488,47 @@ def zip_children(self): zip_path = zipp.Path(self.root) names = zip_path.root.namelist() self.joinpath = zip_path.joinpath - return dict.fromkeys(child.split(posixpath.sep, 1)[0] for child in names) - def search(self, name): - return ( - self.joinpath(child) - for child in self.children() - if name.matches(child, self.base) - ) + def update_cache(self): + root = self.root or "." + try: + mtime = os.stat(root).st_mtime + except OSError: + self.infos.clear() + self.eggs.clear() + self.last_mtime = -1 + return + if mtime == self.last_mtime: + return + self.infos.clear() + self.eggs.clear() + base_is_egg = self.base.endswith(".egg") + for child in self.children(): + low = child.lower() + if low.endswith((".dist-info", ".egg-info")): + # rpartition is faster than splitext and suitable for this purpose. + name = low.rpartition(".")[0].partition("-")[0] + normalized = Prepared.normalize(name) + self.infos.setdefault(normalized, []).append(child) + elif base_is_egg and low == "egg-info": + name = self.base.rpartition(".")[0].partition("-")[0] + legacy_normalized = Prepared.legacy_normalize(name) + self.eggs.setdefault(legacy_normalized, []).append(child) + self.last_mtime = mtime + + def search(self, prepared): + self.update_cache() + if prepared.name: + infos = self.infos.get(prepared.normalized, []) + yield from map(self.joinpath, infos) + eggs = self.eggs.get(prepared.legacy_normalized, []) + yield from map(self.joinpath, eggs) + else: + for infos in self.infos.values(): + yield from map(self.joinpath, infos) + for eggs in self.eggs.values(): + yield from map(self.joinpath, eggs) class Prepared: @@ -499,22 +537,14 @@ class Prepared: """ normalized = None - suffixes = 'dist-info', 'egg-info' - exact_matches = [''][:0] - egg_prefix = '' - versionless_egg_name = '' + legacy_normalized = None def __init__(self, name): self.name = name if name is None: return self.normalized = self.normalize(name) - self.exact_matches = [ - self.normalized + '.' + suffix for suffix in self.suffixes - ] - legacy_normalized = self.legacy_normalize(self.name) - self.egg_prefix = legacy_normalized + '-' - self.versionless_egg_name = legacy_normalized + '.egg' + self.legacy_normalized = self.legacy_normalize(name) @staticmethod def normalize(name): @@ -531,27 +561,6 @@ def legacy_normalize(name): """ return name.lower().replace('-', '_') - def matches(self, cand, base): - low = cand.lower() - # rpartition is faster than splitext and suitable for this purpose. - pre, _, ext = low.rpartition('.') - name, _, rest = pre.partition('-') - return ( - low in self.exact_matches - or ext in self.suffixes - and (not self.normalized or name.replace('.', '_') == self.normalized) - # legacy case: - or self.is_egg(base) - and low == 'egg-info' - ) - - def is_egg(self, base): - return ( - base == self.versionless_egg_name - or base.startswith(self.egg_prefix) - and base.endswith('.egg') - ) - @install class MetadataPathFinder(NullFinder, DistributionFinder): @@ -581,6 +590,9 @@ def _search_paths(cls, name, paths): path.search(prepared) for path in map(FastPath, paths) ) + def invalidate_caches(cls): + FastPath.__new__.cache_clear() + class PathDistribution(Distribution): def __init__(self, path): diff --git a/tests/test_api.py b/tests/test_api.py index 04a5b9d3..134acf60 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,6 +1,7 @@ import re import textwrap import unittest +import importlib from . import fixtures from importlib_metadata import ( @@ -224,3 +225,9 @@ def test_distribution_at_str(self): dist_info_path = self.site_dir / 'distinfo_pkg-1.0.0.dist-info' dist = Distribution.at(str(dist_info_path)) assert dist.version == '1.0.0' + + +class InvalidateCache(unittest.TestCase): + def test_invalidate_cache(self): + # No externally observable behavior, but ensures test coverage... + importlib.invalidate_caches() From 38fff62edb5e282f144dc77cc1bf5555367336d9 Mon Sep 17 00:00:00 2001 From: KOLANICH Date: Sat, 6 Feb 2021 23:03:13 +0300 Subject: [PATCH 025/105] Added an .editorconfig. Pull request jaraco/skeleton#43. --- .editorconfig | 15 +++++++++++++++ pytest.ini | 4 ++-- 2 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..6385b573 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,15 @@ +root = true + +[*] +charset = utf-8 +indent_style = tab +indent_size = 4 +insert_final_newline = true +end_of_line = lf + +[*.py] +indent_style = space + +[*.{yml,yaml}] +indent_style = space +indent_size = 2 diff --git a/pytest.ini b/pytest.ini index d7f0b115..016063b5 100644 --- a/pytest.ini +++ b/pytest.ini @@ -5,5 +5,5 @@ doctest_optionflags=ALLOW_UNICODE ELLIPSIS # workaround for warning pytest-dev/pytest#6178 junit_family=xunit2 filterwarnings= - # https://github.com/pytest-dev/pytest/issues/6928 - ignore:direct construction of .*Item has been deprecated:DeprecationWarning + # https://github.com/pytest-dev/pytest/issues/6928 + ignore:direct construction of .*Item has been deprecated:DeprecationWarning From 5e416793c008c5ef285c37828072fbea5ced6d08 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 21 Feb 2021 21:34:35 -0500 Subject: [PATCH 026/105] It's no longer necessary to filter this warning and it's not a warning anymore. --- pytest.ini | 2 -- 1 file changed, 2 deletions(-) diff --git a/pytest.ini b/pytest.ini index 016063b5..6bf69af1 100644 --- a/pytest.ini +++ b/pytest.ini @@ -5,5 +5,3 @@ doctest_optionflags=ALLOW_UNICODE ELLIPSIS # workaround for warning pytest-dev/pytest#6178 junit_family=xunit2 filterwarnings= - # https://github.com/pytest-dev/pytest/issues/6928 - ignore:direct construction of .*Item has been deprecated:DeprecationWarning From d9a13c77ce2a3efea70c97d219ca4335c0f03c40 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 21 Feb 2021 21:36:53 -0500 Subject: [PATCH 027/105] Bump minimum pytest --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index af246415..81f70eea 100644 --- a/setup.cfg +++ b/setup.cfg @@ -31,7 +31,7 @@ exclude = [options.extras_require] testing = # upstream - pytest >= 3.5, !=3.7.3 + pytest >= 4.6 pytest-checkdocs >= 1.2.3 pytest-flake8 pytest-black >= 0.3.7; python_implementation != "PyPy" From e3d1b935b3a2185461aadca34192b93bfdeaa9ca Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 21 Feb 2021 12:27:41 -0500 Subject: [PATCH 028/105] Update changelog. --- CHANGES.rst | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 368723d4..2f5b1ec5 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,18 +1,34 @@ -v3.5.0 +v4.0.0 ====== -* #280: ``entry_points`` now only returns entry points for - unique distributions (by name). -* ``entry_points()`` now returns an ``GroupedEntryPoints`` - object, a tuple of all entry points but with a convenience - property ``groups`` and ``__getitem__`` accessor. Further, - accessing a group returns an ``EntryPoints`` object, - another tuple of entry points in the group, accessible by - name. Construction of entry points using +* #284: Introduces new ``EntryPoints`` object, a tuple of + ``EntryPoint`` objects but with convenience properties for + selecting and inspecting the results: + + - ``.select()`` accepts ``group`` or ``name`` keyword + parameters and returns a new ``EntryPoints`` tuple + with only those that match the selection. + - ``.groups`` property presents all of the group names. + - ``.names`` property presents the names of the entry points. + - Item access (e.g. ``eps[name]``) retrieves a single + entry point by name. + + ``entry_points()`` now returns an ``EntryPoints`` + object, but provides for backward compatibility with + a ``__getitem__`` accessor by group and a ``get()`` + method. + + Construction of entry points using ``dict([EntryPoint, ...])`` is now deprecated and raises an appropriate DeprecationWarning and will be removed in a future version. +v3.5.0 +====== + +* #280: ``entry_points`` now only returns entry points for + unique distributions (by name). + v3.4.0 ====== From 9d55a331c7d77025054e85f23bc23c614fab6856 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 22 Feb 2021 08:47:11 -0500 Subject: [PATCH 029/105] Separate compatibility shim from canonical EntryPoints container. --- importlib_metadata/__init__.py | 41 ++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 313beca2..94a82ffe 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -165,23 +165,12 @@ class EntryPoints(tuple): __slots__ = () - def __getitem__(self, name) -> Union[EntryPoint, 'EntryPoints']: + def __getitem__(self, name): # -> EntryPoint: try: - match = next(iter(self.select(name=name))) - return match + return next(iter(self.select(name=name))) except StopIteration: - if name in self.groups: - return self._group_getitem(name) raise KeyError(name) - def _group_getitem(self, name): - """ - For backward compatability, supply .__getitem__ for groups. - """ - msg = "GroupedEntryPoints.__getitem__ is deprecated for groups. Use select." - warnings.warn(msg, DeprecationWarning) - return self.select(group=name) - def select(self, **params): return EntryPoints(ep for ep in self if ep.matches(**params)) @@ -193,6 +182,23 @@ def names(self): def groups(self): return set(ep.group for ep in self) + @classmethod + def _from_text_for(cls, text, dist): + return cls(ep._for(dist) for ep in EntryPoint._from_text(text)) + + +class LegacyGroupedEntryPoints(EntryPoints): + def __getitem__(self, name) -> Union[EntryPoint, 'EntryPoints']: + try: + return super().__getitem__(name) + except KeyError: + if name not in self.groups: + raise + + msg = "GroupedEntryPoints.__getitem__ is deprecated for groups. Use select." + warnings.warn(msg, DeprecationWarning) + return self.select(group=name) + def get(self, group, default=None): """ For backward compatibility, supply .get @@ -202,9 +208,10 @@ def get(self, group, default=None): is_flake8 or warnings.warn(msg, DeprecationWarning) return self.select(group=group) or default - @classmethod - def _from_text_for(cls, text, dist): - return cls(ep._for(dist) for ep in EntryPoint._from_text(text)) + def select(self, **params): + if not params: + return self + return super().select(**params) class PackagePath(pathlib.PurePosixPath): @@ -704,7 +711,7 @@ def entry_points(**params): eps = itertools.chain.from_iterable( dist.entry_points for dist in unique(distributions()) ) - return EntryPoints(eps).select(**params) + return LegacyGroupedEntryPoints(eps).select(**params) def files(distribution_name): From d6f7c201b15c79bce7c4e27784a2bd61bdc43555 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 22 Feb 2021 20:49:20 -0500 Subject: [PATCH 030/105] Add docstrings to the compatibility shim. Give primacy to group lookup in compatibility shim. --- importlib_metadata/__init__.py | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 94a82ffe..bd7d4d7e 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -188,27 +188,38 @@ def _from_text_for(cls, text, dist): class LegacyGroupedEntryPoints(EntryPoints): + """ + Compatibility wrapper around EntryPoints to provide + much of the 'dict' interface previously returned by + entry_points. + """ + def __getitem__(self, name) -> Union[EntryPoint, 'EntryPoints']: - try: - return super().__getitem__(name) - except KeyError: - if name not in self.groups: - raise + """ + When accessed by name that matches a group, return the group. + """ + group = self.select(group=name) + if group: + msg = "GroupedEntryPoints.__getitem__ is deprecated for groups. Use select." + warnings.warn(msg, DeprecationWarning, stacklevel=2) + return group - msg = "GroupedEntryPoints.__getitem__ is deprecated for groups. Use select." - warnings.warn(msg, DeprecationWarning) - return self.select(group=name) + return super().__getitem__(name) def get(self, group, default=None): """ - For backward compatibility, supply .get + For backward compatibility, supply .get. """ is_flake8 = any('flake8' in str(frame) for frame in inspect.stack()) msg = "GroupedEntryPoints.get is deprecated. Use select." - is_flake8 or warnings.warn(msg, DeprecationWarning) + is_flake8 or warnings.warn(msg, DeprecationWarning, stacklevel=2) return self.select(group=group) or default def select(self, **params): + """ + Prevent transform to EntryPoints during call to entry_points if + no selection parameters were passed. + """ if not params: return self return super().select(**params) From 4e288685ed6edee15b43629ec26897c4aeeab21f Mon Sep 17 00:00:00 2001 From: Antony Lee Date: Tue, 23 Feb 2021 10:48:20 +0100 Subject: [PATCH 031/105] Use a hand-written parser for entry points. This speeds up the `entry_points()` tox perf check by ~30%, while being both shorter and easier to follow. --- importlib_metadata/__init__.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 4c420a55..ae1d9e37 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -21,7 +21,6 @@ from ._itertools import unique_everseen -from configparser import ConfigParser from contextlib import suppress from importlib import import_module from importlib.abc import MetaPathFinder @@ -114,21 +113,18 @@ def extras(self): match = self.pattern.match(self.value) return list(re.finditer(r'\w+', match.group('extras') or '')) - @classmethod - def _from_config(cls, config): - return ( - cls(name, value, group) - for group in config.sections() - for name, value in config.items(group) - ) - @classmethod def _from_text(cls, text): - config = ConfigParser(delimiters='=') - # case sensitive: https://stackoverflow.com/q/1611799/812183 - config.optionxform = str - config.read_string(text) - return cls._from_config(config) + # A hand-rolled parser is much faster than ConfigParser. + if not text: + return + group = None + for line in filter(None, map(str.strip, text.splitlines())): + if line.startswith("["): + group = line[1:-1] + else: + name, value = map(str.strip, line.split("=", 1)) + yield cls(name, value, group) @classmethod def _from_text_for(cls, text, dist): From 2db4dada379822b4767809a5c4e2436f32908658 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 23 Feb 2021 09:58:19 -0500 Subject: [PATCH 032/105] Introduce SelectableGroups, created for the 3.x line to provide forward compatibilty to the new interfaces without sacrificing backward compatibility. --- CHANGES.rst | 21 ++++++++++---- docs/using.rst | 4 +-- importlib_metadata/__init__.py | 50 ++++++++++++++++++++++++++++++++-- 3 files changed, 65 insertions(+), 10 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 2f5b1ec5..f8df681d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,4 +1,4 @@ -v4.0.0 +v3.6.0 ====== * #284: Introduces new ``EntryPoints`` object, a tuple of @@ -13,10 +13,21 @@ v4.0.0 - Item access (e.g. ``eps[name]``) retrieves a single entry point by name. - ``entry_points()`` now returns an ``EntryPoints`` - object, but provides for backward compatibility with - a ``__getitem__`` accessor by group and a ``get()`` - method. + ``entry_points`` now accepts "selection parameters", + same as ``EntryPoint.select()``. + + ``entry_points()`` now provides a future-compatible + ``SelectableGroups`` object that supplies the above interface + but remains a dict for compatibility. + + In the future, ``entry_points()`` will return an + ``EntryPoints`` object, but provide for backward + compatibility with a deprecated ``__getitem__`` + accessor by group and a ``get()`` method. + + If passing selection parameters to ``entry_points``, the + future behavior is invoked and an ``EntryPoints`` is the + result. Construction of entry points using ``dict([EntryPoint, ...])`` is now deprecated and raises diff --git a/docs/using.rst b/docs/using.rst index bdfe3e82..97941452 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -67,8 +67,8 @@ This package provides the following functionality via its public API. Entry points ------------ -The ``entry_points()`` function returns a sequence of all entry points, -keyed by group. Entry points are represented by ``EntryPoint`` instances; +The ``entry_points()`` function returns a collection of entry points. +Entry points are represented by ``EntryPoint`` instances; each ``EntryPoint`` has a ``.name``, ``.group``, and ``.value`` attributes and a ``.load()`` method to resolve the value. There are also ``.module``, ``.attr``, and ``.extras`` attributes for getting the components of the diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index bd7d4d7e..f2dc9c07 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -187,6 +187,37 @@ def _from_text_for(cls, text, dist): return cls(ep._for(dist) for ep in EntryPoint._from_text(text)) +class SelectableGroups(dict): + """ + A backward- and forward-compatible result from + entry_points that fully implements the dict interface. + """ + + @classmethod + def load(cls, eps): + by_group = operator.attrgetter('group') + ordered = sorted(eps, key=by_group) + grouped = itertools.groupby(ordered, by_group) + return cls((group, EntryPoints(eps)) for group, eps in grouped) + + @property + def groups(self): + return self.keys() + + @property + def names(self): + return (ep.name for ep in self._all) + + @property + def _all(self): + return itertools.chain.from_iterable(self.values()) + + def select(self, **params): + if not params: + return self + return EntryPoints(self._all).select(**params) + + class LegacyGroupedEntryPoints(EntryPoints): """ Compatibility wrapper around EntryPoints to provide @@ -713,16 +744,29 @@ def version(distribution_name): return distribution(distribution_name).version -def entry_points(**params): +def entry_points(**params) -> Union[EntryPoints, SelectableGroups]: """Return EntryPoint objects for all installed packages. - :return: EntryPoint objects for all installed packages. + Pass selection parameters (group or name) to filter the + result to entry points matching those properties (see + EntryPoints.select()). + + For compatibility, returns ``SelectableGroups`` object unless + selection parameters are supplied. In the future, this function + will return ``LegacyGroupedEntryPoints`` instead of + ``SelectableGroups`` and eventually will only return + ``EntryPoints``. + + For maximum future compatibility, pass selection parameters + or invoke ``.select`` with parameters on the result. + + :return: EntryPoints or SelectableGroups for all installed packages. """ unique = functools.partial(unique_everseen, key=operator.attrgetter('name')) eps = itertools.chain.from_iterable( dist.entry_points for dist in unique(distributions()) ) - return LegacyGroupedEntryPoints(eps).select(**params) + return SelectableGroups.load(eps).select(**params) def files(distribution_name): From 2def046c694cddfbd1967575f8ce7da95680c9c3 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 23 Feb 2021 13:07:55 -0500 Subject: [PATCH 033/105] Address coverage misses, ignored for LegacyGroupedEntryPoints. --- importlib_metadata/__init__.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index f2dc9c07..5f156ae6 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -180,6 +180,11 @@ def names(self): @property def groups(self): + """ + For coverage while SelectableGroups is present. + >>> EntryPoints().groups + set() + """ return set(ep.group for ep in self) @classmethod @@ -202,11 +207,16 @@ def load(cls, eps): @property def groups(self): - return self.keys() + return set(self.keys()) @property def names(self): - return (ep.name for ep in self._all) + """ + for coverage: + >>> SelectableGroups().names + set() + """ + return set(ep.name for ep in self._all) @property def _all(self): @@ -218,7 +228,7 @@ def select(self, **params): return EntryPoints(self._all).select(**params) -class LegacyGroupedEntryPoints(EntryPoints): +class LegacyGroupedEntryPoints(EntryPoints): # pragma: nocover """ Compatibility wrapper around EntryPoints to provide much of the 'dict' interface previously returned by From dd8da47fdf97d4420cca557742f8f075da2123e4 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 23 Feb 2021 13:12:42 -0500 Subject: [PATCH 034/105] Leverage EntryPoints interfaces in SelectableGroups --- importlib_metadata/__init__.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 5f156ae6..b0b1ae0e 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -205,9 +205,13 @@ def load(cls, eps): grouped = itertools.groupby(ordered, by_group) return cls((group, EntryPoints(eps)) for group, eps in grouped) + @property + def _all(self): + return EntryPoints(itertools.chain.from_iterable(self.values())) + @property def groups(self): - return set(self.keys()) + return self._all.groups @property def names(self): @@ -216,16 +220,12 @@ def names(self): >>> SelectableGroups().names set() """ - return set(ep.name for ep in self._all) - - @property - def _all(self): - return itertools.chain.from_iterable(self.values()) + return self._all.names def select(self, **params): if not params: return self - return EntryPoints(self._all).select(**params) + return self._all.select(**params) class LegacyGroupedEntryPoints(EntryPoints): # pragma: nocover From 466cd3c8e6036cbd16584629fa0e54d6c0d6b027 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Wed, 24 Feb 2021 11:42:19 -0500 Subject: [PATCH 035/105] Add 'packages_distributions'. Fixes #131. --- CHANGES.rst | 6 ++++++ docs/using.rst | 11 +++++++++++ importlib_metadata/__init__.py | 20 ++++++++++++++++++-- 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index f8df681d..a4e468c0 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,9 @@ +v3.7.0 +====== + +* #131: Added ``packages_distributions`` to conveniently + resolve a top-level package or module to its distribution(s). + v3.6.0 ====== diff --git a/docs/using.rst b/docs/using.rst index 97941452..18aa2fda 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -182,6 +182,17 @@ function:: ["pytest (>=3.0.0) ; extra == 'test'", "pytest-cov ; extra == 'test'"] +Package distributions +--------------------- + +A convience method to resolve the distribution or +distributions (in the case of a namespace package) for top-level +Python packages or modules:: + + >>> packages_distributions() + {'importlib_metadata': ['importlib-metadata'], 'yaml': ['PyYAML'], 'jaraco': ['jaraco.classes', 'jaraco.functools'], ...} + + Distributions ============= diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index b0b1ae0e..5c19c237 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -12,7 +12,7 @@ import functools import itertools import posixpath -import collections +import collections.abc from ._compat import ( NullFinder, @@ -28,7 +28,7 @@ from importlib import import_module from importlib.abc import MetaPathFinder from itertools import starmap -from typing import Any, List, Optional, TypeVar, Union +from typing import Any, List, Mapping, Optional, TypeVar, Union __all__ = [ @@ -796,3 +796,19 @@ def requires(distribution_name): packaging.requirement.Requirement. """ return distribution(distribution_name).requires + + +def packages_distributions() -> Mapping[str, List[str]]: + """ + Return a mapping of top-level packages to their + distributions. + + >>> pkgs = packages_distributions() + >>> all(isinstance(dist, collections.abc.Sequence) for dist in pkgs.values()) + True + """ + pkg_to_dist = collections.defaultdict(list) + for dist in distributions(): + for pkg in (dist.read_text('top_level.txt') or '').split(): + pkg_to_dist[pkg].append(dist.metadata['Name']) + return dict(pkg_to_dist) From 8c4cff1a2ffea6b4fa59d4a86c6608bb19861a92 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 4 Mar 2021 04:14:02 -0500 Subject: [PATCH 036/105] Convert LegacyGroupedEntryPoints into simpler dict interface deprecation. --- importlib_metadata/__init__.py | 38 ++++++++-------------------------- 1 file changed, 9 insertions(+), 29 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 5c19c237..162656a8 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -228,42 +228,22 @@ def select(self, **params): return self._all.select(**params) -class LegacyGroupedEntryPoints(EntryPoints): # pragma: nocover +class DeprecatedDict(dict): # pragma: nocover """ - Compatibility wrapper around EntryPoints to provide - much of the 'dict' interface previously returned by - entry_points. + Compatibility wrapper around dict to indicate that + Mapping behavior is deprecated. """ - def __getitem__(self, name) -> Union[EntryPoint, 'EntryPoints']: - """ - When accessed by name that matches a group, return the group. - """ - group = self.select(group=name) - if group: - msg = "GroupedEntryPoints.__getitem__ is deprecated for groups. Use select." - warnings.warn(msg, DeprecationWarning, stacklevel=2) - return group - + def __getitem__(self, name): + msg = "SelectableGroups.__getitem__ is deprecated. Use select." + warnings.warn(msg, DeprecationWarning, stacklevel=2) return super().__getitem__(name) - def get(self, group, default=None): - """ - For backward compatibility, supply .get. - """ + def get(self, name, default=None): is_flake8 = any('flake8' in str(frame) for frame in inspect.stack()) - msg = "GroupedEntryPoints.get is deprecated. Use select." + msg = "SelectableGroups.get is deprecated. Use select." is_flake8 or warnings.warn(msg, DeprecationWarning, stacklevel=2) - return self.select(group=group) or default - - def select(self, **params): - """ - Prevent transform to EntryPoints during call to entry_points if - no selection parameters were passed. - """ - if not params: - return self - return super().select(**params) + return super().get(name, default) class PackagePath(pathlib.PurePosixPath): From b022ae991389755055cec67f5112c283b2413fe1 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 4 Mar 2021 04:20:28 -0500 Subject: [PATCH 037/105] Extract warning as a method. --- importlib_metadata/__init__.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 162656a8..5d85be62 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -234,15 +234,17 @@ class DeprecatedDict(dict): # pragma: nocover Mapping behavior is deprecated. """ + def _warn(self): + msg = "SelectableGroups dict interface is deprecated. Use select." + warnings.warn(msg, DeprecationWarning, stacklevel=3) + def __getitem__(self, name): - msg = "SelectableGroups.__getitem__ is deprecated. Use select." - warnings.warn(msg, DeprecationWarning, stacklevel=2) + self._warn() return super().__getitem__(name) def get(self, name, default=None): is_flake8 = any('flake8' in str(frame) for frame in inspect.stack()) - msg = "SelectableGroups.get is deprecated. Use select." - is_flake8 or warnings.warn(msg, DeprecationWarning, stacklevel=2) + is_flake8 or self._warn() return super().get(name, default) From 1f463549a246ec9f855bea04b20080f3236a9cdc Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 4 Mar 2021 04:39:04 -0500 Subject: [PATCH 038/105] Remove flake8 bypass and implement warning as a partial. --- importlib_metadata/__init__.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 5d85be62..26d11c2b 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -5,7 +5,6 @@ import sys import zipp import email -import inspect import pathlib import operator import warnings @@ -228,23 +227,33 @@ def select(self, **params): return self._all.select(**params) -class DeprecatedDict(dict): # pragma: nocover +class DeprecatedDict(dict): """ Compatibility wrapper around dict to indicate that Mapping behavior is deprecated. + + >>> recwarn = getfixture('recwarn') + >>> dd = DeprecatedDict(foo='bar') + >>> dd.get('baz', None) + >>> dd['foo'] + 'bar' + >>> len(recwarn) + 1 """ - def _warn(self): - msg = "SelectableGroups dict interface is deprecated. Use select." - warnings.warn(msg, DeprecationWarning, stacklevel=3) + _warn = functools.partial( + warnings.warn, + "SelectableGroups dict interface is deprecated. Use select.", + DeprecationWarning, + stacklevel=3, + ) def __getitem__(self, name): self._warn() return super().__getitem__(name) def get(self, name, default=None): - is_flake8 = any('flake8' in str(frame) for frame in inspect.stack()) - is_flake8 or self._warn() + self._warn() return super().get(name, default) From 537c55da0cf507b1e800e0d7785ae983f8b1f1fb Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 4 Mar 2021 04:50:12 -0500 Subject: [PATCH 039/105] Reimplement flake8 bypass as a decorator. --- importlib_metadata/__init__.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 26d11c2b..d100d960 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -5,12 +5,14 @@ import sys import zipp import email +import inspect import pathlib import operator import warnings import functools import itertools import posixpath +import contextlib import collections.abc from ._compat import ( @@ -227,6 +229,13 @@ def select(self, **params): return self._all.select(**params) +class Flake8Bypass(warnings.catch_warnings, contextlib.ContextDecorator): + def __enter__(self): + super().__enter__() + is_flake8 = any('flake8' in str(frame) for frame in inspect.stack()) + is_flake8 and warnings.simplefilter('ignore', DeprecationWarning) + + class DeprecatedDict(dict): """ Compatibility wrapper around dict to indicate that @@ -252,6 +261,7 @@ def __getitem__(self, name): self._warn() return super().__getitem__(name) + @Flake8Bypass() def get(self, name, default=None): self._warn() return super().get(name, default) From 1f2a89cf9a6fe71dbcdacdfd040a2abbd4ece842 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 4 Mar 2021 04:54:48 -0500 Subject: [PATCH 040/105] Also deprecate iter, contains, keys, and values --- importlib_metadata/__init__.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index d100d960..00f758a3 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -246,6 +246,14 @@ class DeprecatedDict(dict): >>> dd.get('baz', None) >>> dd['foo'] 'bar' + >>> list(dd) + ['foo'] + >>> list(dd.keys()) + ['foo'] + >>> 'foo' in dd + True + >>> list(dd.values()) + ['bar'] >>> len(recwarn) 1 """ @@ -266,6 +274,22 @@ def get(self, name, default=None): self._warn() return super().get(name, default) + def __iter__(self): + self._warn() + return super().__iter__() + + def __contains__(self, *args): + self._warn() + return super().__contains__(*args) + + def keys(self): + self._warn() + return super().keys() + + def values(self): + self._warn() + return super().values() + class PackagePath(pathlib.PurePosixPath): """A reference to a path in a package""" From 401c041d15772869a6337aa7c3ddfde77cadcfb4 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 4 Mar 2021 04:57:43 -0500 Subject: [PATCH 041/105] Move selectable groups after DeprecatedDict --- importlib_metadata/__init__.py | 72 +++++++++++++++++----------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 00f758a3..fb30e2cb 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -193,42 +193,6 @@ def _from_text_for(cls, text, dist): return cls(ep._for(dist) for ep in EntryPoint._from_text(text)) -class SelectableGroups(dict): - """ - A backward- and forward-compatible result from - entry_points that fully implements the dict interface. - """ - - @classmethod - def load(cls, eps): - by_group = operator.attrgetter('group') - ordered = sorted(eps, key=by_group) - grouped = itertools.groupby(ordered, by_group) - return cls((group, EntryPoints(eps)) for group, eps in grouped) - - @property - def _all(self): - return EntryPoints(itertools.chain.from_iterable(self.values())) - - @property - def groups(self): - return self._all.groups - - @property - def names(self): - """ - for coverage: - >>> SelectableGroups().names - set() - """ - return self._all.names - - def select(self, **params): - if not params: - return self - return self._all.select(**params) - - class Flake8Bypass(warnings.catch_warnings, contextlib.ContextDecorator): def __enter__(self): super().__enter__() @@ -291,6 +255,42 @@ def values(self): return super().values() +class SelectableGroups(dict): + """ + A backward- and forward-compatible result from + entry_points that fully implements the dict interface. + """ + + @classmethod + def load(cls, eps): + by_group = operator.attrgetter('group') + ordered = sorted(eps, key=by_group) + grouped = itertools.groupby(ordered, by_group) + return cls((group, EntryPoints(eps)) for group, eps in grouped) + + @property + def _all(self): + return EntryPoints(itertools.chain.from_iterable(self.values())) + + @property + def groups(self): + return self._all.groups + + @property + def names(self): + """ + for coverage: + >>> SelectableGroups().names + set() + """ + return self._all.names + + def select(self, **params): + if not params: + return self + return self._all.select(**params) + + class PackagePath(pathlib.PurePosixPath): """A reference to a path in a package""" From 7e7fc8c8f379df4a3d47258015de8e7ae4cd54c5 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 4 Mar 2021 05:22:38 -0500 Subject: [PATCH 042/105] Just check the filename in the frame. Otherwise, it'll match on the current line. --- importlib_metadata/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index fb30e2cb..5dc51d3e 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -196,7 +196,9 @@ def _from_text_for(cls, text, dist): class Flake8Bypass(warnings.catch_warnings, contextlib.ContextDecorator): def __enter__(self): super().__enter__() - is_flake8 = any('flake8' in str(frame) for frame in inspect.stack()) + is_flake8 = any( + 'flake8' in str(frame.filename) for frame in inspect.stack()[:5] + ) is_flake8 and warnings.simplefilter('ignore', DeprecationWarning) @@ -219,7 +221,7 @@ class DeprecatedDict(dict): >>> list(dd.values()) ['bar'] >>> len(recwarn) - 1 + 2 """ _warn = functools.partial( From ed33213268c4cda0079649a410cfbfc679a90313 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 4 Mar 2021 05:26:12 -0500 Subject: [PATCH 043/105] Wrap function rather than decorating method. Avoids varying stack depths. --- importlib_metadata/__init__.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 5dc51d3e..44160596 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -12,7 +12,6 @@ import functools import itertools import posixpath -import contextlib import collections.abc from ._compat import ( @@ -193,13 +192,9 @@ def _from_text_for(cls, text, dist): return cls(ep._for(dist) for ep in EntryPoint._from_text(text)) -class Flake8Bypass(warnings.catch_warnings, contextlib.ContextDecorator): - def __enter__(self): - super().__enter__() - is_flake8 = any( - 'flake8' in str(frame.filename) for frame in inspect.stack()[:5] - ) - is_flake8 and warnings.simplefilter('ignore', DeprecationWarning) +def flake8_bypass(func): + is_flake8 = any('flake8' in str(frame.filename) for frame in inspect.stack()[:5]) + return func if not is_flake8 else lambda: None class DeprecatedDict(dict): @@ -221,7 +216,7 @@ class DeprecatedDict(dict): >>> list(dd.values()) ['bar'] >>> len(recwarn) - 2 + 1 """ _warn = functools.partial( @@ -235,9 +230,8 @@ def __getitem__(self, name): self._warn() return super().__getitem__(name) - @Flake8Bypass() def get(self, name, default=None): - self._warn() + flake8_bypass(self._warn)() return super().get(name, default) def __iter__(self): From d940e63e562e66ab2aaf9bdf8444fc40a5637966 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 4 Mar 2021 05:28:30 -0500 Subject: [PATCH 044/105] Instead of subclassing dict, make Deprecated a mix-in. --- importlib_metadata/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 44160596..8ab38328 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -197,12 +197,13 @@ def flake8_bypass(func): return func if not is_flake8 else lambda: None -class DeprecatedDict(dict): +class Deprecated: """ - Compatibility wrapper around dict to indicate that - Mapping behavior is deprecated. + Compatibility add-in for mapping to indicate that + mapping behavior is deprecated. >>> recwarn = getfixture('recwarn') + >>> class DeprecatedDict(Deprecated, dict): pass >>> dd = DeprecatedDict(foo='bar') >>> dd.get('baz', None) >>> dd['foo'] From 0460524e44896b9e5c746a21e1f06efe9b5ed475 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 4 Mar 2021 05:35:44 -0500 Subject: [PATCH 045/105] stacklevel of 2 is the right place --- importlib_metadata/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 8ab38328..b02dc159 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -224,7 +224,7 @@ class Deprecated: warnings.warn, "SelectableGroups dict interface is deprecated. Use select.", DeprecationWarning, - stacklevel=3, + stacklevel=2, ) def __getitem__(self, name): From a54488dca687fbd4e3d35bcddadc26fba836183c Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 4 Mar 2021 05:38:51 -0500 Subject: [PATCH 046/105] Querying missing key will also be deprecated. --- tests/test_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_api.py b/tests/test_api.py index a2810acc..8c8d9abb 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -141,8 +141,8 @@ def test_entry_points_groups_getitem(self): with warnings.catch_warnings(record=True): entry_points()['entries'] == entry_points(group='entries') - with self.assertRaises(KeyError): - entry_points()['missing'] + with self.assertRaises(KeyError): + entry_points()['missing'] def test_entry_points_groups_get(self): """ From cc40cd56bfd2ced7e90616149d5450e06877dbde Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 4 Mar 2021 05:39:46 -0500 Subject: [PATCH 047/105] Update docstring --- importlib_metadata/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index b02dc159..e06c70b6 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -267,6 +267,9 @@ def load(cls, eps): @property def _all(self): + """ + Reconstruct a list of all entrypoints from the groups. + """ return EntryPoints(itertools.chain.from_iterable(self.values())) @property From 3a36cbd993d2eb1a4fd3c691d5b41fb1a46f7639 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 4 Mar 2021 05:41:09 -0500 Subject: [PATCH 048/105] Deprecate dict usage of SelectableGroups. --- importlib_metadata/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index e06c70b6..da57dede 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -252,7 +252,7 @@ def values(self): return super().values() -class SelectableGroups(dict): +class SelectableGroups(Deprecated, dict): """ A backward- and forward-compatible result from entry_points that fully implements the dict interface. @@ -270,7 +270,8 @@ def _all(self): """ Reconstruct a list of all entrypoints from the groups. """ - return EntryPoints(itertools.chain.from_iterable(self.values())) + groups = super(Deprecated, self).values() + return EntryPoints(itertools.chain.from_iterable(groups)) @property def groups(self): From ae14c73354dbbeeb77038c74a5f2d2c15bc8cd25 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 4 Mar 2021 05:48:14 -0500 Subject: [PATCH 049/105] Update changelog. --- CHANGES.rst | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index a4e468c0..1fd456a3 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,10 @@ +v3.8.0 +====== + +* Use of Mapping (dict) interfaces on ``SelectableGroups`` + is now flagged as deprecated. Instead, users are advised + to use the select interface for future compatibility. + v3.7.0 ====== @@ -24,12 +31,10 @@ v3.6.0 ``entry_points()`` now provides a future-compatible ``SelectableGroups`` object that supplies the above interface - but remains a dict for compatibility. + (except item access) but remains a dict for compatibility. In the future, ``entry_points()`` will return an - ``EntryPoints`` object, but provide for backward - compatibility with a deprecated ``__getitem__`` - accessor by group and a ``get()`` method. + ``EntryPoints`` object for all entry points. If passing selection parameters to ``entry_points``, the future behavior is invoked and an ``EntryPoints`` is the From bf9fae2c0df316dc837d56ae68880620733d5ff6 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 6 Mar 2021 09:57:43 -0500 Subject: [PATCH 050/105] Require twine 3 with keyring unconditionally required. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 249f97c2..a9a50b01 100644 --- a/tox.ini +++ b/tox.ini @@ -25,7 +25,7 @@ commands = skip_install = True deps = build - twine[keyring]>=1.13 + twine>=3 path jaraco.develop>=7.1 passenv = From 7bdab57872da46ef6a5a7f5ea9099a197bdc3131 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 7 Mar 2021 12:23:48 -0500 Subject: [PATCH 051/105] Add comments indicating why the exclusions are present --- setup.cfg | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.cfg b/setup.cfg index 81f70eea..dd215c65 100644 --- a/setup.cfg +++ b/setup.cfg @@ -34,8 +34,10 @@ testing = pytest >= 4.6 pytest-checkdocs >= 1.2.3 pytest-flake8 + # python_implementation: workaround for jaraco/skeleton#22 pytest-black >= 0.3.7; python_implementation != "PyPy" pytest-cov + # python_implementation: workaround for jaraco/skeleton#22 pytest-mypy; python_implementation != "PyPy" pytest-enabler From 14312a5bd75d3313ffd3e14fc7fbbc2a9b05cee5 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 7 Mar 2021 12:24:21 -0500 Subject: [PATCH 052/105] Exclude mypy on Python 3.10 as workaround for python/typed_ast#156. --- setup.cfg | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index dd215c65..55497f8e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,7 +38,8 @@ testing = pytest-black >= 0.3.7; python_implementation != "PyPy" pytest-cov # python_implementation: workaround for jaraco/skeleton#22 - pytest-mypy; python_implementation != "PyPy" + # python_version: workaround for python/typed_ast#156 + pytest-mypy; python_implementation != "PyPy" and python_version < "3.10" pytest-enabler # local From af5445115af0cb68e671a678538a0207389586be Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 7 Mar 2021 12:30:25 -0500 Subject: [PATCH 053/105] Bump minimums on pytest-checkdocs and pytest-enabler as found on Setuptools. --- setup.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 55497f8e..3f6610be 100644 --- a/setup.cfg +++ b/setup.cfg @@ -32,7 +32,7 @@ exclude = testing = # upstream pytest >= 4.6 - pytest-checkdocs >= 1.2.3 + pytest-checkdocs >= 2.4 pytest-flake8 # python_implementation: workaround for jaraco/skeleton#22 pytest-black >= 0.3.7; python_implementation != "PyPy" @@ -40,7 +40,7 @@ testing = # python_implementation: workaround for jaraco/skeleton#22 # python_version: workaround for python/typed_ast#156 pytest-mypy; python_implementation != "PyPy" and python_version < "3.10" - pytest-enabler + pytest-enabler >= 1.0.1 # local From 86efb884f805a9e1f64661ec758f3bd084fed515 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 7 Mar 2021 12:53:54 -0500 Subject: [PATCH 054/105] Also deny black on Python 3.10 as workaround for python/typed_ast#156. --- setup.cfg | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 3f6610be..52876d55 100644 --- a/setup.cfg +++ b/setup.cfg @@ -35,7 +35,8 @@ testing = pytest-checkdocs >= 2.4 pytest-flake8 # python_implementation: workaround for jaraco/skeleton#22 - pytest-black >= 0.3.7; python_implementation != "PyPy" + # python_version: workaround for python/typed_ast#156 + pytest-black >= 0.3.7; python_implementation != "PyPy" and python_version < "3.10" pytest-cov # python_implementation: workaround for jaraco/skeleton#22 # python_version: workaround for python/typed_ast#156 From 56d312b95217ece0191d4b587b3da81e6d9a71db Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 7 Mar 2021 17:10:53 -0500 Subject: [PATCH 055/105] Update changelog. --- CHANGES.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index a4e468c0..ba99d6d5 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,9 @@ +v3.7.1 +====== + +* Internal refactoring to facilitate ``entry_points() -> dict`` + deprecation. + v3.7.0 ====== From 1e2381fe101fd70742a0171e51c1be82aedf519b Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 7 Mar 2021 18:48:21 -0500 Subject: [PATCH 056/105] Remove latent reference to LegacyGroupedEntryPoints. --- CHANGES.rst | 5 +++++ importlib_metadata/__init__.py | 5 ++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index ba99d6d5..2792caf8 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,8 @@ +v3.7.2 +====== + +* Cleaned up cruft in entry_points docstring. + v3.7.1 ====== diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index e06c70b6..95087b55 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -788,9 +788,8 @@ def entry_points(**params) -> Union[EntryPoints, SelectableGroups]: For compatibility, returns ``SelectableGroups`` object unless selection parameters are supplied. In the future, this function - will return ``LegacyGroupedEntryPoints`` instead of - ``SelectableGroups`` and eventually will only return - ``EntryPoints``. + will return ``EntryPoints`` instead of ``SelectableGroups`` + even when no selection parameters are supplied. For maximum future compatibility, pass selection parameters or invoke ``.select`` with parameters on the result. From 12e33bb37bc8517ff89f3e7b56a1b3bd585482ed Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 7 Mar 2021 21:17:36 -0500 Subject: [PATCH 057/105] Extract Lookup class out of directory lookup behavior. Separate it from mtime caching behavior. --- importlib_metadata/__init__.py | 80 +++++++++++++++++++--------------- 1 file changed, 44 insertions(+), 36 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 67c925d4..349de2e0 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -12,6 +12,7 @@ import functools import itertools import posixpath +import contextlib import collections.abc from ._compat import ( @@ -602,13 +603,10 @@ class FastPath: @functools.lru_cache() # type: ignore def __new__(cls, root): - self = object().__new__(cls) + return super().__new__(cls) + + def __init__(self, root): self.root = str(root) - self.base = os.path.basename(self.root).lower() - self.last_mtime = -1 - self.infos = {} - self.eggs = {} - return self def joinpath(self, child): return pathlib.Path(self.root, child) @@ -624,47 +622,54 @@ def zip_children(self): zip_path = zipp.Path(self.root) names = zip_path.root.namelist() self.joinpath = zip_path.joinpath + return dict.fromkeys(child.split(posixpath.sep, 1)[0] for child in names) - def update_cache(self): - root = self.root or "." - try: - mtime = os.stat(root).st_mtime - except OSError: - self.infos.clear() - self.eggs.clear() - self.last_mtime = -1 - return - if mtime == self.last_mtime: - return - self.infos.clear() - self.eggs.clear() - base_is_egg = self.base.endswith(".egg") - for child in self.children(): + def search(self, name): + return self.lookup(self.mtime).search(name) + + @property + def mtime(self): + with contextlib.suppress(OSError): + return os.stat(self.root).st_mtime + FastPath.lookup.cache_clear() + + @functools.lru_cache() + def lookup(self, mtime): + return Lookup(self) + + +class Lookup: + def __init__(self, path: FastPath): + base = os.path.basename(path.root).lower() + base_is_egg = base.endswith(".egg") + self.infos = collections.defaultdict(list) + self.eggs = collections.defaultdict(list) + + for child in path.children(): low = child.lower() if low.endswith((".dist-info", ".egg-info")): # rpartition is faster than splitext and suitable for this purpose. name = low.rpartition(".")[0].partition("-")[0] normalized = Prepared.normalize(name) - self.infos.setdefault(normalized, []).append(child) + self.infos[normalized].append(path.joinpath(child)) elif base_is_egg and low == "egg-info": - name = self.base.rpartition(".")[0].partition("-")[0] + name = base.rpartition(".")[0].partition("-")[0] legacy_normalized = Prepared.legacy_normalize(name) - self.eggs.setdefault(legacy_normalized, []).append(child) - self.last_mtime = mtime + self.eggs[legacy_normalized].append(path.joinpath(child)) def search(self, prepared): - self.update_cache() - if prepared.name: - infos = self.infos.get(prepared.normalized, []) - yield from map(self.joinpath, infos) - eggs = self.eggs.get(prepared.legacy_normalized, []) - yield from map(self.joinpath, eggs) - else: - for infos in self.infos.values(): - yield from map(self.joinpath, infos) - for eggs in self.eggs.values(): - yield from map(self.joinpath, eggs) + infos = ( + self.infos[prepared.normalized] + if prepared + else itertools.chain.from_iterable(self.infos.values()) + ) + eggs = ( + self.eggs[prepared.legacy_normalized] + if prepared + else itertools.chain.from_iterable(self.eggs.values()) + ) + return itertools.chain(infos, eggs) class Prepared: @@ -697,6 +702,9 @@ def legacy_normalize(name): """ return name.lower().replace('-', '_') + def __bool__(self): + return bool(self.name) + @install class MetadataPathFinder(NullFinder, DistributionFinder): From 7bdeaa45e3710e33735a9632c65ab7916c06c410 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 14 Mar 2021 09:24:04 -0400 Subject: [PATCH 058/105] Add packages_distributions to __all__ --- importlib_metadata/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 95087b55..f583dc91 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -40,6 +40,7 @@ 'entry_points', 'files', 'metadata', + 'packages_distributions', 'requires', 'version', ] From bb2437066473ed10dff16564d731917345251e88 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 14 Mar 2021 09:57:53 -0400 Subject: [PATCH 059/105] Expand docs to explain more about the interfaces and to include a compatibility note. --- docs/using.rst | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/docs/using.rst b/docs/using.rst index 18aa2fda..53d83959 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -72,15 +72,43 @@ Entry points are represented by ``EntryPoint`` instances; each ``EntryPoint`` has a ``.name``, ``.group``, and ``.value`` attributes and a ``.load()`` method to resolve the value. There are also ``.module``, ``.attr``, and ``.extras`` attributes for getting the components of the -``.value`` attribute:: +``.value`` attribute. + +Query all entry points:: >>> eps = entry_points() + +The ``entry_points()`` function returns an ``EntryPoints`` object, +a sequence of all ``EntryPoint`` objects with ``names`` and ``groups`` +attributes for convenience. + >>> sorted(eps.groups) ['console_scripts', 'distutils.commands', 'distutils.setup_keywords', 'egg_info.writers', 'setuptools.installation'] + +``EntryPoints`` has a ``select`` method to select entry points +matching specific properties. Select entry points in the +``console_scripts`` group:: + >>> scripts = eps.select(group='console_scripts') + +Equivalently, since ``entry_points`` passes keyword arguments +through to select:: + + >>> scripts = entry_points(group='console_scripts') + +Pick out a specific script named "wheel" (found in the wheel project):: + >>> 'wheel' in scripts.names True >>> wheel = scripts['wheel'] + +Equivalently, query for that entry point during selection:: + + >>> (wheel,) = entry_points(group='console_scripts', name='wheel') + >>> (wheel,) = entry_points().select(group='console_scripts', name='wheel') + +Inspect the resolved entry point:: + >>> wheel EntryPoint(name='wheel', value='wheel.cli:main', group='console_scripts') >>> wheel.module @@ -99,6 +127,17 @@ group. Read `the setuptools docs `_ for more information on entry points, their definition, and usage. +*Compatibility Note* + +The "selectable" entry points were introduced in ``importlib_metadata`` +3.6 and Python 3.10. Prior to those changes, ``entry_points`` accepted +no parameters and always returned a dictionary of entry points, keyed +by group. For compatibility, if no parameters are passed to entry_points, +a ``SelectableGroups`` object is returned, implementing that dict +interface. In the future, calling ``entry_points`` with no parameters +will return an ``EntryPoints`` object. Users should rely on the selection +interface to retrieve entry points by group. + .. _metadata: From bf777ae8030fd5fe778778ed159a484129083ece Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 14 Mar 2021 10:10:20 -0400 Subject: [PATCH 060/105] Expand docs on EntryPoints --- importlib_metadata/__init__.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index f583dc91..46203448 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -167,21 +167,33 @@ class EntryPoints(tuple): __slots__ = () def __getitem__(self, name): # -> EntryPoint: + """ + Get the EntryPoint in self matching name. + """ try: return next(iter(self.select(name=name))) except StopIteration: raise KeyError(name) def select(self, **params): + """ + Select entry points from self that match the + given parameters (typically group and/or name). + """ return EntryPoints(ep for ep in self if ep.matches(**params)) @property def names(self): + """ + Return the set of all names of all entry points. + """ return set(ep.name for ep in self) @property def groups(self): """ + Return the set of all groups of all entry points. + For coverage while SelectableGroups is present. >>> EntryPoints().groups set() From 60f2791a2f0aa0007cb39c46b28ad687cef8bd8a Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 14 Mar 2021 10:13:19 -0400 Subject: [PATCH 061/105] Importing inspect is expensive. Defer it unless needed. --- importlib_metadata/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 46203448..832a7116 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -5,7 +5,6 @@ import sys import zipp import email -import inspect import pathlib import operator import warnings @@ -206,6 +205,9 @@ def _from_text_for(cls, text, dist): def flake8_bypass(func): + # defer inspect import as performance optimization. + import inspect + is_flake8 = any('flake8' in str(frame.filename) for frame in inspect.stack()[:5]) return func if not is_flake8 else lambda: None From d176331ff44cb2636a7b285cfc70371890b6d95b Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 14 Mar 2021 10:15:18 -0400 Subject: [PATCH 062/105] Only import collections.abc in doctests. --- importlib_metadata/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 832a7116..112373e5 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -11,7 +11,7 @@ import functools import itertools import posixpath -import collections.abc +import collections from ._compat import ( NullFinder, @@ -842,6 +842,7 @@ def packages_distributions() -> Mapping[str, List[str]]: Return a mapping of top-level packages to their distributions. + >>> import collections.abc >>> pkgs = packages_distributions() >>> all(isinstance(dist, collections.abc.Sequence) for dist in pkgs.values()) True From da0bc8969757cbe4d1e38aeac837450ff54816f4 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 14 Mar 2021 10:20:41 -0400 Subject: [PATCH 063/105] Update changelog. Ref python/cpython#24782. --- CHANGES.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 2792caf8..5777e56e 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,9 @@ +v3.7.3 +====== + +* Docs enhancements and cleanup following review in + `GH-24782 `_. + v3.7.2 ====== From 5c9198cd8212d56db8ee9c49ee39a0980d11f5ff Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 14 Mar 2021 10:32:55 -0400 Subject: [PATCH 064/105] Indicate code block consistently. --- docs/using.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/using.rst b/docs/using.rst index 53d83959..17d6f590 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -80,7 +80,7 @@ Query all entry points:: The ``entry_points()`` function returns an ``EntryPoints`` object, a sequence of all ``EntryPoint`` objects with ``names`` and ``groups`` -attributes for convenience. +attributes for convenience:: >>> sorted(eps.groups) ['console_scripts', 'distutils.commands', 'distutils.setup_keywords', 'egg_info.writers', 'setuptools.installation'] From 7fe4ab8294a843622d20face7f9f6ccddb2d0a14 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 15 Mar 2021 18:31:04 -0400 Subject: [PATCH 065/105] Add leading */ to coverage.run.omit. Workaround for pytest-dev/pytest-cov#456. --- .coveragerc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.coveragerc b/.coveragerc index 45823064..6a34e662 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,5 +1,7 @@ [run] -omit = .tox/* +omit = + # leading `*/` for pytest-dev/pytest-cov#456 + */.tox/* [report] show_missing = True From 6e2740ec6e1166f2b60719b47920eb66e87019ff Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 26 Mar 2021 21:10:18 -0400 Subject: [PATCH 066/105] Remove 'cache clear' operation. Unlocks caching. --- importlib_metadata/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 349de2e0..5d34bbd2 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -632,7 +632,6 @@ def search(self, name): def mtime(self): with contextlib.suppress(OSError): return os.stat(self.root).st_mtime - FastPath.lookup.cache_clear() @functools.lru_cache() def lookup(self, mtime): From db55bc647bf1778c17339d57aefd2a90f074f264 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 26 Mar 2021 21:41:32 -0400 Subject: [PATCH 067/105] Restore cache-clear behavior on a per-path basis. --- importlib_metadata/__init__.py | 4 +- importlib_metadata/_functools.py | 85 ++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 importlib_metadata/_functools.py diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 5d34bbd2..42259f6f 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -22,6 +22,7 @@ Protocol, ) +from ._functools import method_cache from ._itertools import unique_everseen from configparser import ConfigParser @@ -632,8 +633,9 @@ def search(self, name): def mtime(self): with contextlib.suppress(OSError): return os.stat(self.root).st_mtime + self.lookup.cache_clear() - @functools.lru_cache() + @method_cache def lookup(self, mtime): return Lookup(self) diff --git a/importlib_metadata/_functools.py b/importlib_metadata/_functools.py new file mode 100644 index 00000000..73f50d00 --- /dev/null +++ b/importlib_metadata/_functools.py @@ -0,0 +1,85 @@ +import types +import functools + + +# from jaraco.functools 3.3 +def method_cache(method, cache_wrapper=None): + """ + Wrap lru_cache to support storing the cache data in the object instances. + + Abstracts the common paradigm where the method explicitly saves an + underscore-prefixed protected property on first call and returns that + subsequently. + + >>> class MyClass: + ... calls = 0 + ... + ... @method_cache + ... def method(self, value): + ... self.calls += 1 + ... return value + + >>> a = MyClass() + >>> a.method(3) + 3 + >>> for x in range(75): + ... res = a.method(x) + >>> a.calls + 75 + + Note that the apparent behavior will be exactly like that of lru_cache + except that the cache is stored on each instance, so values in one + instance will not flush values from another, and when an instance is + deleted, so are the cached values for that instance. + + >>> b = MyClass() + >>> for x in range(35): + ... res = b.method(x) + >>> b.calls + 35 + >>> a.method(0) + 0 + >>> a.calls + 75 + + Note that if method had been decorated with ``functools.lru_cache()``, + a.calls would have been 76 (due to the cached value of 0 having been + flushed by the 'b' instance). + + Clear the cache with ``.cache_clear()`` + + >>> a.method.cache_clear() + + Same for a method that hasn't yet been called. + + >>> c = MyClass() + >>> c.method.cache_clear() + + Another cache wrapper may be supplied: + + >>> cache = functools.lru_cache(maxsize=2) + >>> MyClass.method2 = method_cache(lambda self: 3, cache_wrapper=cache) + >>> a = MyClass() + >>> a.method2() + 3 + + Caution - do not subsequently wrap the method with another decorator, such + as ``@property``, which changes the semantics of the function. + + See also + http://code.activestate.com/recipes/577452-a-memoize-decorator-for-instance-methods/ + for another implementation and additional justification. + """ + cache_wrapper = cache_wrapper or functools.lru_cache() + + def wrapper(self, *args, **kwargs): + # it's the first call, replace the method with a cached, bound method + bound_method = types.MethodType(method, self) + cached_method = cache_wrapper(bound_method) + setattr(self, method.__name__, cached_method) + return cached_method(*args, **kwargs) + + # Support cache clear even before cache has been created. + wrapper.cache_clear = lambda: None + + return wrapper From 5ef4f529308db4e0778e71fb20522e004d057dcb Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 26 Mar 2021 21:54:45 -0400 Subject: [PATCH 068/105] Update changelog. --- CHANGES.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 2792caf8..1bc91e5c 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,10 @@ +v3.8.0 +====== + +* #290: Add mtime-based caching for ``FastPath`` and its + lookups, dramatically increasing performance for repeated + distribution lookups. + v3.7.2 ====== From 01cf7b4591671b5a44c28cb054e07074ce0ee760 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 27 Mar 2021 10:47:14 -0400 Subject: [PATCH 069/105] Extract Sectioned for parsing entry points --- importlib_metadata/__init__.py | 74 +++++++++++++++++++++++++++++----- 1 file changed, 64 insertions(+), 10 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index ae1d9e37..e2abca6d 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -7,6 +7,7 @@ import email import pathlib import operator +import textwrap import functools import itertools import posixpath @@ -55,6 +56,60 @@ def name(self): return name +class Sectioned: + """ + A simple entry point config parser for performance + + >>> res = Sectioned.get_sections(Sectioned._sample) + >>> sec, values = next(res) + >>> sec + 'sec1' + >>> [(key, value) for key, value in values] + [('a', '1'), ('b', '2')] + >>> sec, values = next(res) + >>> sec + 'sec2' + >>> [(key, value) for key, value in values] + [('a', '2')] + >>> list(res) + [] + """ + + _sample = textwrap.dedent( + """ + [sec1] + a = 1 + b = 2 + + [sec2] + a = 2 + """ + ).lstrip() + + def __init__(self): + self.section = None + + def __call__(self, line): + if line.startswith('[') and line.endswith(']'): + # new section + self.section = line.strip('[]') + return + return self.section + + @classmethod + def get_sections(cls, text): + lines = filter(None, map(str.strip, text.splitlines())) + return ( + (section, map(cls.parse_value, values)) + for section, values in itertools.groupby(lines, cls()) + if section is not None + ) + + @staticmethod + def parse_value(line): + return map(str.strip, line.split("=", 1)) + + class EntryPoint( PyPy_repr, collections.namedtuple('EntryPointBase', 'name value group') ): @@ -115,16 +170,15 @@ def extras(self): @classmethod def _from_text(cls, text): - # A hand-rolled parser is much faster than ConfigParser. - if not text: - return - group = None - for line in filter(None, map(str.strip, text.splitlines())): - if line.startswith("["): - group = line[1:-1] - else: - name, value = map(str.strip, line.split("=", 1)) - yield cls(name, value, group) + return itertools.starmap(cls, cls._parse_groups(text or '')) + + @staticmethod + def _parse_groups(text): + return ( + (name, value, section) + for section, values in Sectioned.get_sections(text) + for name, value in values + ) @classmethod def _from_text_for(cls, text, dist): From 90355b35a2289f79e15bdaddac24e780b488bac4 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 27 Mar 2021 10:53:59 -0400 Subject: [PATCH 070/105] Move entry point parsing to EntryPoints class. --- importlib_metadata/__init__.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index cef4f85f..80d6c195 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -172,18 +172,6 @@ def extras(self): match = self.pattern.match(self.value) return list(re.finditer(r'\w+', match.group('extras') or '')) - @classmethod - def _from_text(cls, text): - return itertools.starmap(cls, cls._parse_groups(text or '')) - - @staticmethod - def _parse_groups(text): - return ( - (name, value, section) - for section, values in Sectioned.get_sections(text) - for name, value in values - ) - def _for(self, dist): self.dist = dist return self @@ -253,7 +241,19 @@ def groups(self): @classmethod def _from_text_for(cls, text, dist): - return cls(ep._for(dist) for ep in EntryPoint._from_text(text)) + return cls(ep._for(dist) for ep in cls._from_text(text)) + + @classmethod + def _from_text(cls, text): + return itertools.starmap(EntryPoint, cls._parse_groups(text or '')) + + @staticmethod + def _parse_groups(text): + return ( + (name, value, section) + for section, values in Sectioned.get_sections(text) + for name, value in values + ) def flake8_bypass(func): From a0f0ba62eb6d740b4350bf7fd4067ee4591b30fd Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 27 Mar 2021 11:03:30 -0400 Subject: [PATCH 071/105] Update changelog. --- CHANGES.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 9b7ca025..49af4a9d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -14,6 +14,11 @@ v3.9.0 Preferably, switch to the ``select`` interface introduced in 3.7.0. +* #283: Entry point parsing no longer relies on ConfigParser + and instead uses a custom, one-pass parser to load the + config, resulting in a ~20% performance improvement when + loading entry points. + v3.8.0 ====== From d91dd253f2ba2c22aab4c11a9c03f1da5cb26a62 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 27 Mar 2021 12:48:56 -0400 Subject: [PATCH 072/105] Eagerly consume infos and eggs. Workaround for #293. Still needs a test. --- CHANGES.rst | 5 +++++ importlib_metadata/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index c5743ac6..b4d3effc 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,8 @@ +v3.8.1 +====== + +* #293: Workaround for error in distribution search. + v3.8.0 ====== diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 11b9aac8..fe81c778 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -685,7 +685,7 @@ def search(self, prepared): if prepared else itertools.chain.from_iterable(self.eggs.values()) ) - return itertools.chain(infos, eggs) + return list(itertools.chain(infos, eggs)) class Prepared: From f545a8f197ea90af685bd1d3218fb273968cd008 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 27 Mar 2021 13:15:53 -0400 Subject: [PATCH 073/105] Add test capturing failure. Ref #293. --- tests/test_integration.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_integration.py b/tests/test_integration.py index 11835135..00e9021a 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -7,6 +7,7 @@ Distribution, MetadataPathFinder, _compat, + distributions, version, ) @@ -59,3 +60,16 @@ def test_search_dist_dirs(self): """ res = MetadataPathFinder._search_paths('any-name', []) assert list(res) == [] + + def test_interleaved_discovery(self): + """ + When the search is cached, it is + possible for searches to be interleaved, so make sure + those use-cases are safe. + + Ref #293 + """ + dists = distributions() + next(dists) + version('importlib_metadata') + next(dists) From 8e3e4af3d0b3462d1a143f9544ce8777209b5908 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 27 Mar 2021 13:47:10 -0400 Subject: [PATCH 074/105] Add freezable dict --- importlib_metadata/_collections.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 importlib_metadata/_collections.py diff --git a/importlib_metadata/_collections.py b/importlib_metadata/_collections.py new file mode 100644 index 00000000..f0357989 --- /dev/null +++ b/importlib_metadata/_collections.py @@ -0,0 +1,21 @@ +import collections + + +class freezable_defaultdict(collections.defaultdict): + """ + Mix-in to freeze a defaultdict. + + >>> dd = freezable_defaultdict(list) + >>> dd[0].append('1') + >>> dd.freeze() + >>> dd[1] + [] + >>> len(dd) + 1 + """ + + def __missing__(self, key): + return getattr(self, '_frozen', super().__missing__)(key) + + def freeze(self): + self._frozen = lambda key: self.default_factory() From d84930cf41ba3c1580310d93cb0748ba246b1c47 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 27 Mar 2021 13:55:10 -0400 Subject: [PATCH 075/105] Freeze the defaultdict after construction and re-enable lazy evaluation of the search results. --- importlib_metadata/__init__.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index fe81c778..0dd0ec55 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -14,13 +14,13 @@ import contextlib import collections +from ._collections import freezable_defaultdict from ._compat import ( NullFinder, + Protocol, PyPy_repr, install, - Protocol, ) - from ._functools import method_cache from ._itertools import unique_everseen @@ -659,8 +659,8 @@ class Lookup: def __init__(self, path: FastPath): base = os.path.basename(path.root).lower() base_is_egg = base.endswith(".egg") - self.infos = collections.defaultdict(list) - self.eggs = collections.defaultdict(list) + self.infos = freezable_defaultdict(list) + self.eggs = freezable_defaultdict(list) for child in path.children(): low = child.lower() @@ -674,6 +674,9 @@ def __init__(self, path: FastPath): legacy_normalized = Prepared.legacy_normalize(name) self.eggs[legacy_normalized].append(path.joinpath(child)) + self.infos.freeze() + self.eggs.freeze() + def search(self, prepared): infos = ( self.infos[prepared.normalized] @@ -685,7 +688,7 @@ def search(self, prepared): if prepared else itertools.chain.from_iterable(self.eggs.values()) ) - return list(itertools.chain(infos, eggs)) + return itertools.chain(infos, eggs) class Prepared: From a6bff7e9a637bb837d283fc5b0c9bb8d5ae73697 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 28 Mar 2021 03:28:53 -0400 Subject: [PATCH 076/105] Presented FreezableDefaultDict from jaraco.collections. Keep inline to minimize dependencies. --- importlib_metadata/__init__.py | 6 +++--- importlib_metadata/_collections.py | 9 ++++++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 0dd0ec55..d705251e 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -14,7 +14,7 @@ import contextlib import collections -from ._collections import freezable_defaultdict +from ._collections import FreezableDefaultDict from ._compat import ( NullFinder, Protocol, @@ -659,8 +659,8 @@ class Lookup: def __init__(self, path: FastPath): base = os.path.basename(path.root).lower() base_is_egg = base.endswith(".egg") - self.infos = freezable_defaultdict(list) - self.eggs = freezable_defaultdict(list) + self.infos = FreezableDefaultDict(list) + self.eggs = FreezableDefaultDict(list) for child in path.children(): low = child.lower() diff --git a/importlib_metadata/_collections.py b/importlib_metadata/_collections.py index f0357989..6aa17c84 100644 --- a/importlib_metadata/_collections.py +++ b/importlib_metadata/_collections.py @@ -1,11 +1,14 @@ import collections -class freezable_defaultdict(collections.defaultdict): +# from jaraco.collections 3.3 +class FreezableDefaultDict(collections.defaultdict): """ - Mix-in to freeze a defaultdict. + Often it is desirable to prevent the mutation of + a default dict after its initial construction, such + as to prevent mutation during iteration. - >>> dd = freezable_defaultdict(list) + >>> dd = FreezableDefaultDict(list) >>> dd[0].append('1') >>> dd.freeze() >>> dd[1] From d17d6e4ae667c086abf5189d4ffb782a46984584 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 28 Mar 2021 03:30:28 -0400 Subject: [PATCH 077/105] Update changelog. --- CHANGES.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index b4d3effc..c30df1b8 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,9 @@ +v3.8.2 +====== + +* #293: Re-enabled lazy evaluation of path lookup through + a FreezableDefaultDict. + v3.8.1 ====== From a9f8a1eef4a7c41947e902ae563c612595529187 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 28 Mar 2021 13:11:35 -0400 Subject: [PATCH 078/105] Employ pyperf and separate environments to compare performance against the main branch. Ref #292. --- .github/workflows/main.yml | 2 +- tox.ini | 43 ++++++++++++++++++++++++++++++-------- 2 files changed, 35 insertions(+), 10 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1853ace5..6377cfcd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -35,7 +35,7 @@ jobs: - name: Run benchmarks run: tox env: - TOXENV: perf + TOXENV: perf{,-ref} diffcov: runs-on: ubuntu-latest diff --git a/tox.ini b/tox.ini index 771232be..42d28ba3 100644 --- a/tox.ini +++ b/tox.ini @@ -33,19 +33,44 @@ commands = diff-cover coverage.xml --compare-branch=origin/main --html-report diffcov.html diff-cover coverage.xml --compare-branch=origin/main --fail-under=100 +[perf] +deps = + ipython + pyperf + path +commands = + python -c "import path; path.Path('{env:SCOPE}.json').remove_p()" + + python -m pyperf timeit --name discovery --append {env:SCOPE}.json -s 'import importlib_metadata' 'importlib_metadata.distribution("ipython")' + + python -m pyperf timeit --name 'entry_points()' --append {env:SCOPE}.json -s 'import importlib_metadata' 'importlib_metadata.entry_points()' + + python -m pyperf timeit --name 'cached distribution' --append {env:SCOPE}.json -s 'import importlib_metadata; importlib_metadata.distribution("ipython")' 'importlib_metadata.distribution("ipython")' + + python -m pyperf timeit --name 'uncached distribution' --append {env:SCOPE}.json -s 'import importlib, importlib_metadata' 'importlib.invalidate_caches(); importlib_metadata.distribution("ipython")' + [testenv:perf] use_develop = False +# change dir to avoid picking up local package. +changedir = {toxworkdir} +setenv = + SCOPE = local +deps = {[perf]deps} +commands = {[perf]commands} + +[testenv:perf-ref] +# compare perf results to the main branch +skip_install = True +# change dir to avoid picking up local package. +changedir = {toxworkdir} +setenv = + SCOPE = main deps = - ipython + {[perf]deps} + git+https://github.com/python/importlib_metadata commands = - python -c 'print("Simple discovery performance")' - python -m timeit -s 'import importlib_metadata' -- 'importlib_metadata.distribution("ipython")' - python -c 'print("Entry point discovery performance")' - python -m timeit -s 'import importlib_metadata' -- 'importlib_metadata.entry_points()' - python -c 'print("Cached lookup performance")' - python -m timeit -s 'import importlib_metadata; importlib_metadata.distribution("ipython")' -- 'importlib_metadata.distribution("ipython")' - python -c 'print("Uncached lookup performance")' - python -m timeit -s 'import importlib, importlib_metadata' -- 'importlib.invalidate_caches(); importlib_metadata.distribution("ipython")' + {[perf]commands} + python -m pyperf compare_to --verbose main.json local.json --table [testenv:release] skip_install = True From 1b9637ad1635922616a92ed874e142c9e2197b9d Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 29 Mar 2021 08:54:25 -0400 Subject: [PATCH 079/105] Exclude prepare package. Fixes #296. --- CHANGES.rst | 5 +++++ setup.cfg | 1 + 2 files changed, 6 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 32565111..57fe7131 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,8 @@ +v3.9.1 +====== + +* #296: Exclude 'prepare' package. + v3.9.0 ====== diff --git a/setup.cfg b/setup.cfg index 8974f885..6fc73125 100644 --- a/setup.cfg +++ b/setup.cfg @@ -29,6 +29,7 @@ exclude = dist* docs* tests* + prepare* [options.extras_require] testing = From 50a1549e5efc789b6e1cf656dec5091c2af44974 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 29 Mar 2021 09:08:42 -0400 Subject: [PATCH 080/105] Ignore comments in entry point parsing. Fixes #297. --- CHANGES.rst | 1 + importlib_metadata/__init__.py | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 57fe7131..8427a258 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -2,6 +2,7 @@ v3.9.1 ====== * #296: Exclude 'prepare' package. +* #297: Fix ValueError when entry points contains comments. v3.9.0 ====== diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 7440acc4..c83e3e9d 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -82,6 +82,7 @@ class Sectioned: _sample = textwrap.dedent( """ [sec1] + # comments ignored a = 1 b = 2 @@ -102,13 +103,17 @@ def __call__(self, line): @classmethod def get_sections(cls, text): - lines = filter(None, map(str.strip, text.splitlines())) + lines = filter(cls.valid, map(str.strip, text.splitlines())) return ( (section, map(cls.parse_value, values)) for section, values in itertools.groupby(lines, cls()) if section is not None ) + @staticmethod + def valid(line): + return line and not line.startswith('#') + @staticmethod def parse_value(line): return map(str.strip, line.split("=", 1)) From 5806de1ebe998ccba452c6aa6824d9b6ada8f4ca Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 29 Mar 2021 20:43:53 -0400 Subject: [PATCH 081/105] Unify section parser behavior. Removes reliance on regex (ref #292). Fixes #295. --- importlib_metadata/__init__.py | 72 ++++++++++++++-------------------- 1 file changed, 29 insertions(+), 43 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index c83e3e9d..b545e1c2 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -65,20 +65,25 @@ class Sectioned: A simple entry point config parser for performance >>> res = Sectioned.get_sections(Sectioned._sample) - >>> sec, values = next(res) + >>> sec, pair = next(res) >>> sec 'sec1' - >>> [(key, value) for key, value in values] - [('a', '1'), ('b', '2')] - >>> sec, values = next(res) + >>> tuple(pair) + ('a', '1') + >>> sec, pair = next(res) + >>> tuple(pair) + ('b', '2') + >>> sec, pair = next(res) >>> sec 'sec2' - >>> [(key, value) for key, value in values] - [('a', '2')] + >>> tuple(pair) + ('a', '2') >>> list(res) [] """ + Pair = collections.namedtuple('Pair', 'name value') + _sample = textwrap.dedent( """ [sec1] @@ -91,25 +96,25 @@ class Sectioned: """ ).lstrip() - def __init__(self): - self.section = None - - def __call__(self, line): - if line.startswith('[') and line.endswith(']'): - # new section - self.section = line.strip('[]') - return - return self.section - @classmethod def get_sections(cls, text): - lines = filter(cls.valid, map(str.strip, text.splitlines())) return ( - (section, map(cls.parse_value, values)) - for section, values in itertools.groupby(lines, cls()) - if section is not None + (section.name, cls.parse_value(section.value)) + for section in cls.read(text, filter_=cls.valid) + if section.name is not None ) + @staticmethod + def read(text, filter_=None): + lines = filter(filter_, map(str.strip, text.splitlines())) + name = None + for value in lines: + section_match = value.startswith('[') and value.endswith(']') + if section_match: + name = value.strip('[]') + continue + yield Sectioned.Pair(name, value) + @staticmethod def valid(line): return line and not line.startswith('#') @@ -256,8 +261,7 @@ def _from_text(cls, text): def _parse_groups(text): return ( (name, value, section) - for section, values in Sectioned.get_sections(text) - for name, value in values + for section, (name, value) in Sectioned.get_sections(text) ) @@ -573,24 +577,7 @@ def _read_egg_info_reqs(self): @classmethod def _deps_from_requires_text(cls, source): - section_pairs = cls._read_sections(source.splitlines()) - sections = { - section: list(map(operator.itemgetter('line'), results)) - for section, results in itertools.groupby( - section_pairs, operator.itemgetter('section') - ) - } - return cls._convert_egg_info_reqs_to_simple_reqs(sections) - - @staticmethod - def _read_sections(lines): - section = None - for line in filter(None, lines): - section_match = re.match(r'\[(.*)\]$', line) - if section_match: - section = section_match.group(1) - continue - yield locals() + return cls._convert_egg_info_reqs_to_simple_reqs(Sectioned.read(source)) @staticmethod def _convert_egg_info_reqs_to_simple_reqs(sections): @@ -615,9 +602,8 @@ def parse_condition(section): conditions = list(filter(None, [markers, make_condition(extra)])) return '; ' + ' and '.join(conditions) if conditions else '' - for section, deps in sections.items(): - for dep in deps: - yield dep + parse_condition(section) + for section in sections: + yield section.value + parse_condition(section.name) class DistributionFinder(MetaPathFinder): From bf6a3b13b3eec27e48abed353761a29676d6a0ff Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 29 Mar 2021 21:11:53 -0400 Subject: [PATCH 082/105] Use Pair in other places and extract it to _collections. --- importlib_metadata/__init__.py | 36 +++++++++++++----------------- importlib_metadata/_collections.py | 6 +++++ 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index b545e1c2..5abc1e9e 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -15,7 +15,7 @@ import contextlib import collections -from ._collections import FreezableDefaultDict +from ._collections import FreezableDefaultDict, Pair from ._compat import ( NullFinder, Protocol, @@ -64,26 +64,24 @@ class Sectioned: """ A simple entry point config parser for performance - >>> res = Sectioned.get_sections(Sectioned._sample) - >>> sec, pair = next(res) - >>> sec + >>> res = Sectioned.section_pairs(Sectioned._sample) + >>> item = next(res) + >>> item.name 'sec1' - >>> tuple(pair) + >>> tuple(item.value) ('a', '1') - >>> sec, pair = next(res) - >>> tuple(pair) + >>> item = next(res) + >>> tuple(item.value) ('b', '2') - >>> sec, pair = next(res) - >>> sec + >>> item = next(res) + >>> item.name 'sec2' - >>> tuple(pair) + >>> tuple(item.value) ('a', '2') >>> list(res) [] """ - Pair = collections.namedtuple('Pair', 'name value') - _sample = textwrap.dedent( """ [sec1] @@ -97,9 +95,9 @@ class Sectioned: ).lstrip() @classmethod - def get_sections(cls, text): + def section_pairs(cls, text): return ( - (section.name, cls.parse_value(section.value)) + section._replace(value=Pair.parse(section.value)) for section in cls.read(text, filter_=cls.valid) if section.name is not None ) @@ -113,16 +111,12 @@ def read(text, filter_=None): if section_match: name = value.strip('[]') continue - yield Sectioned.Pair(name, value) + yield Pair(name, value) @staticmethod def valid(line): return line and not line.startswith('#') - @staticmethod - def parse_value(line): - return map(str.strip, line.split("=", 1)) - class EntryPoint( PyPy_repr, collections.namedtuple('EntryPointBase', 'name value group') @@ -260,8 +254,8 @@ def _from_text(cls, text): @staticmethod def _parse_groups(text): return ( - (name, value, section) - for section, (name, value) in Sectioned.get_sections(text) + (item.value.name, item.value.value, item.name) + for item in Sectioned.section_pairs(text) ) diff --git a/importlib_metadata/_collections.py b/importlib_metadata/_collections.py index 6aa17c84..cf0954e1 100644 --- a/importlib_metadata/_collections.py +++ b/importlib_metadata/_collections.py @@ -22,3 +22,9 @@ def __missing__(self, key): def freeze(self): self._frozen = lambda key: self.default_factory() + + +class Pair(collections.namedtuple('Pair', 'name value')): + @classmethod + def parse(cls, text): + return cls(*map(str.strip, text.split("=", 1))) From c891d5065be9948bc593808884b85c3ad364dbfd Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 29 Mar 2021 21:26:51 -0400 Subject: [PATCH 083/105] Expand and simplify Sectioned doctest. --- importlib_metadata/__init__.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 5abc1e9e..7c5eb2c7 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -64,20 +64,27 @@ class Sectioned: """ A simple entry point config parser for performance + >>> for item in Sectioned.read(Sectioned._sample): + ... print(item) + Pair(name='sec1', value='# comments ignored') + Pair(name='sec1', value='a = 1') + Pair(name='sec1', value='b = 2') + Pair(name='sec2', value='a = 2') + >>> res = Sectioned.section_pairs(Sectioned._sample) >>> item = next(res) >>> item.name 'sec1' - >>> tuple(item.value) - ('a', '1') + >>> item.value + Pair(name='a', value='1') >>> item = next(res) - >>> tuple(item.value) - ('b', '2') + >>> item.value + Pair(name='b', value='2') >>> item = next(res) >>> item.name 'sec2' - >>> tuple(item.value) - ('a', '2') + >>> item.value + Pair(name='a', value='2') >>> list(res) [] """ From 244fc482479925b75851194989518d97061a1b28 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 29 Mar 2021 21:36:36 -0400 Subject: [PATCH 084/105] Update changelog. --- CHANGES.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 8427a258..696da526 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,8 @@ +v3.10.0 +======= + +* #295: Internal refactoring to unify section parsing logic. + v3.9.1 ====== From b18a7eebaa3ae406d96bc566f016a1f91d36ec04 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 30 Mar 2021 15:30:13 -0400 Subject: [PATCH 085/105] Extend the changelog to reference the backport. Fixes #298. --- CHANGES.rst | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 696da526..7ef6cc22 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -23,7 +23,18 @@ v3.9.0 ``warnings.filterwarnings('ignore', 'SelectableGroups dict interface')``. Preferably, switch to the ``select`` interface introduced - in 3.7.0. + in 3.7.0. See the + `entry points documentation `_ and changelog for the 3.6 + release below for more detail. + + For some use-cases, especially those that rely on + ``importlib.metadata`` in Python 3.8 and 3.9 or + those relying on older ``importlib_metadata`` (especially + on Python 3.5 and earlier), + `backports.entry_points_selectable `_ + was created to ease the transition. Please have a look + at that project if simply relying on importlib_metadata 3.6+ + is not straightforward. Background in #298. * #283: Entry point parsing no longer relies on ConfigParser and instead uses a custom, one-pass parser to load the From 69f366c8d4866cb592e0594d692ccb136a3d43f2 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Wed, 31 Mar 2021 22:53:42 -0400 Subject: [PATCH 086/105] Add keywords and Description to metadata. --- tests/fixtures.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/fixtures.py b/tests/fixtures.py index 779c1a58..508fbb5c 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -86,6 +86,10 @@ class DistInfoPkg(OnSysPath, SiteDir): Version: 1.0.0 Requires-Dist: wheel >= 1.0 Requires-Dist: pytest; extra == 'test' + Keywords: sample package + + Once upon a time + There was a distinfo pkg """, "RECORD": "mod.py,sha256=abc,20\n", "entry_points.txt": """ @@ -157,6 +161,9 @@ class EggInfoPkg(OnSysPath, SiteDir): Version: 1.0.0 Classifier: Intended Audience :: Developers Classifier: Topic :: Software Development :: Libraries + Keywords: sample package + Description: Once upon a time + There was an egginfo package """, "SOURCES.txt": """ mod.py From 51568a1a9ac1c75e843aac48ce35b29bdef462bd Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Wed, 31 Mar 2021 22:07:04 -0400 Subject: [PATCH 087/105] Add as_json function. Fixes #277. --- importlib_metadata/__init__.py | 41 ++++++++++++++++++++++++++++++++++ tests/test_api.py | 13 +++++++++++ 2 files changed, 54 insertions(+) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 7c5eb2c7..911f86c4 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -5,6 +5,7 @@ import sys import zipp import email +import string import pathlib import operator import textwrap @@ -919,3 +920,43 @@ def packages_distributions() -> Mapping[str, List[str]]: for pkg in (dist.read_text('top_level.txt') or '').split(): pkg_to_dist[pkg].append(dist.metadata['Name']) return dict(pkg_to_dist) + + +def as_json(metadata: PackageMetadata): + """ + Convert PackageMetadata to a JSON-compatible format + per PEP 0566. + """ + # TODO: Need to match case-insensitive + multiple_use = { + 'Classifier', + 'Obsoletes-Dist', + 'Platform', + 'Project-URL', + 'Provides-Dist', + 'Provides-Extra', + 'Requires-Dist', + 'Requires-External', + 'Supported-Platform', + } + + def redent(value): + "Correct for RFC822 indentation" + if not value or '\n' not in value: + return value + return textwrap.dedent(' ' * 8 + value) + + def transform(key): + value = ( + metadata.get_all(key) if key in multiple_use else redent(metadata.get(key)) + ) + if key == 'Keywords': + value = value.split(string.whitespace) + if not value and key == 'Description': + value = metadata.get_payload() + tk = key.lower().replace('-', '_') + return tk, value + + desc = ['Description'] if metadata.get_payload() else [] # type: ignore + keys = itertools.chain(metadata, desc) # type: ignore + return dict(map(transform, keys)) # type: ignore diff --git a/tests/test_api.py b/tests/test_api.py index fef99033..ad89373f 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -8,6 +8,7 @@ from importlib_metadata import ( Distribution, PackageNotFoundError, + as_json, distribution, entry_points, files, @@ -246,6 +247,18 @@ def test_more_complex_deps_requires_text(self): assert deps == expected + def test_as_json(self): + md = as_json(metadata('distinfo-pkg')) + assert 'name' in md + desc = md['description'] + assert desc.startswith('Once upon a time\nThere was') + + def test_as_json_egg_info(self): + md = as_json(metadata('egginfo-pkg')) + assert 'name' in md + desc = md['description'] + assert desc.startswith('Once upon a time\nThere was') + class LegacyDots(fixtures.DistInfoPkgWithDotLegacy, unittest.TestCase): def test_name_normalization(self): From 6eef087f7b94b0f7fdf609065ad01bbd45c65aae Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 1 Apr 2021 20:25:55 -0400 Subject: [PATCH 088/105] Implement JSON as an adapter on the PackageMetadata object, simplifying the experience for the consumer. --- importlib_metadata/__init__.py | 70 +++------------------------------ importlib_metadata/_adapters.py | 66 +++++++++++++++++++++++++++++++ importlib_metadata/_meta.py | 24 +++++++++++ tests/test_api.py | 5 +-- 4 files changed, 97 insertions(+), 68 deletions(-) create mode 100644 importlib_metadata/_adapters.py create mode 100644 importlib_metadata/_meta.py diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 911f86c4..5142ada2 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -5,7 +5,6 @@ import sys import zipp import email -import string import pathlib import operator import textwrap @@ -16,10 +15,10 @@ import contextlib import collections +from . import _adapters, _meta from ._collections import FreezableDefaultDict, Pair from ._compat import ( NullFinder, - Protocol, PyPy_repr, install, ) @@ -30,7 +29,7 @@ from importlib import import_module from importlib.abc import MetaPathFinder from itertools import starmap -from typing import Any, List, Mapping, Optional, TypeVar, Union +from typing import List, Mapping, Optional, Union __all__ = [ @@ -394,25 +393,6 @@ def __repr__(self): return ''.format(self.mode, self.value) -_T = TypeVar("_T") - - -class PackageMetadata(Protocol): - def __len__(self) -> int: - ... # pragma: no cover - - def __contains__(self, item: str) -> bool: - ... # pragma: no cover - - def __getitem__(self, key: str) -> str: - ... # pragma: no cover - - def get_all(self, name: str, failobj: _T = ...) -> Union[List[Any], _T]: - """ - Return all values associated with a possibly multi-valued key. - """ - - class Distribution: """A Python distribution package.""" @@ -497,7 +477,7 @@ def _local(cls, root='.'): return PathDistribution(zipp.Path(meta.build_as_zip(builder))) @property - def metadata(self) -> PackageMetadata: + def metadata(self) -> _meta.PackageMetadata: """Return the parsed metadata for this Distribution. The returned object will have keys that name the various bits of @@ -511,7 +491,7 @@ def metadata(self) -> PackageMetadata: # (which points to the egg-info file) attribute unchanged. or self.read_text('') ) - return email.message_from_string(text) + return _adapters.JSONMeta(email.message_from_string(text)) @property def name(self): @@ -843,7 +823,7 @@ def distributions(**kwargs): return Distribution.discover(**kwargs) -def metadata(distribution_name) -> PackageMetadata: +def metadata(distribution_name) -> _meta.PackageMetadata: """Get the metadata for the named package. :param distribution_name: The name of the distribution package to query. @@ -920,43 +900,3 @@ def packages_distributions() -> Mapping[str, List[str]]: for pkg in (dist.read_text('top_level.txt') or '').split(): pkg_to_dist[pkg].append(dist.metadata['Name']) return dict(pkg_to_dist) - - -def as_json(metadata: PackageMetadata): - """ - Convert PackageMetadata to a JSON-compatible format - per PEP 0566. - """ - # TODO: Need to match case-insensitive - multiple_use = { - 'Classifier', - 'Obsoletes-Dist', - 'Platform', - 'Project-URL', - 'Provides-Dist', - 'Provides-Extra', - 'Requires-Dist', - 'Requires-External', - 'Supported-Platform', - } - - def redent(value): - "Correct for RFC822 indentation" - if not value or '\n' not in value: - return value - return textwrap.dedent(' ' * 8 + value) - - def transform(key): - value = ( - metadata.get_all(key) if key in multiple_use else redent(metadata.get(key)) - ) - if key == 'Keywords': - value = value.split(string.whitespace) - if not value and key == 'Description': - value = metadata.get_payload() - tk = key.lower().replace('-', '_') - return tk, value - - desc = ['Description'] if metadata.get_payload() else [] # type: ignore - keys = itertools.chain(metadata, desc) # type: ignore - return dict(map(transform, keys)) # type: ignore diff --git a/importlib_metadata/_adapters.py b/importlib_metadata/_adapters.py new file mode 100644 index 00000000..0b637811 --- /dev/null +++ b/importlib_metadata/_adapters.py @@ -0,0 +1,66 @@ +import string +import textwrap +import itertools + +from . import _meta + + +class JSONMeta(_meta.PackageMetadata): + def __init__(self, orig: _meta.PackageMetadata): + self.orig = orig + + def __getitem__(self, item): + return self.orig.__getitem__(item) + + def __len__(self): + return self.orig.__len__() # pragma: nocover + + def __contains__(self, item): + return self.orig.__contains__(item) # pragma: nocover + + def __iter__(self): + return self.orig.__iter__() + + def get_all(self, name): + return self.orig.get_all(name) + + def get_payload(self): + return self.orig.get_payload() + + @property + def json(self): + """ + Convert PackageMetadata to a JSON-compatible format + per PEP 0566. + """ + # TODO: Need to match case-insensitive + multiple_use = { + 'Classifier', + 'Obsoletes-Dist', + 'Platform', + 'Project-URL', + 'Provides-Dist', + 'Provides-Extra', + 'Requires-Dist', + 'Requires-External', + 'Supported-Platform', + } + + def redent(value): + "Correct for RFC822 indentation" + if not value or '\n' not in value: + return value + return textwrap.dedent(' ' * 8 + value) + + def transform(key): + value = self.get_all(key) if key in multiple_use else redent(self[key]) + if key == 'Keywords': + value = value.split(string.whitespace) + if not value and key == 'Description': + value = self.get_payload() + tk = key.lower().replace('-', '_') + return tk, value + + desc = ['Description'] if self.get_payload() else [] + keys = itertools.chain(self, desc) + return dict(map(transform, keys)) diff --git a/importlib_metadata/_meta.py b/importlib_metadata/_meta.py new file mode 100644 index 00000000..fcb313ee --- /dev/null +++ b/importlib_metadata/_meta.py @@ -0,0 +1,24 @@ +from ._compat import Protocol +from typing import Any, List, TypeVar, Union + + +_T = TypeVar("_T") + + +class PackageMetadata(Protocol): + def __len__(self) -> int: + ... # pragma: no cover + + def __contains__(self, item: str) -> bool: + ... # pragma: no cover + + def __getitem__(self, key: str) -> str: + ... # pragma: no cover + + def get_payload(self) -> str: + ... # pragma: no cover + + def get_all(self, name: str, failobj: _T = ...) -> Union[List[Any], _T]: + """ + Return all values associated with a possibly multi-valued key. + """ diff --git a/tests/test_api.py b/tests/test_api.py index ad89373f..0f31226d 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -8,7 +8,6 @@ from importlib_metadata import ( Distribution, PackageNotFoundError, - as_json, distribution, entry_points, files, @@ -248,13 +247,13 @@ def test_more_complex_deps_requires_text(self): assert deps == expected def test_as_json(self): - md = as_json(metadata('distinfo-pkg')) + md = metadata('distinfo-pkg').json assert 'name' in md desc = md['description'] assert desc.startswith('Once upon a time\nThere was') def test_as_json_egg_info(self): - md = as_json(metadata('egginfo-pkg')) + md = metadata('egginfo-pkg').json assert 'name' in md desc = md['description'] assert desc.startswith('Once upon a time\nThere was') From b9ce1b12a4fe715b4a9754cb7194d022b28b847b Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 1 Apr 2021 20:43:16 -0400 Subject: [PATCH 089/105] Implement Message with json property as a subclass to avoid repeating the interface. --- importlib_metadata/__init__.py | 2 +- importlib_metadata/_adapters.py | 30 ++++++++---------------------- 2 files changed, 9 insertions(+), 23 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 5142ada2..dc750500 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -491,7 +491,7 @@ def metadata(self) -> _meta.PackageMetadata: # (which points to the egg-info file) attribute unchanged. or self.read_text('') ) - return _adapters.JSONMeta(email.message_from_string(text)) + return _adapters.Message(email.message_from_string(text)) @property def name(self): diff --git a/importlib_metadata/_adapters.py b/importlib_metadata/_adapters.py index 0b637811..1f53f43e 100644 --- a/importlib_metadata/_adapters.py +++ b/importlib_metadata/_adapters.py @@ -1,31 +1,17 @@ import string import textwrap import itertools +import email.message -from . import _meta +class Message(email.message.Message): + def __new__(cls, orig: email.message.Message): + res = super().__new__(cls) + vars(res).update(vars(orig)) + return res -class JSONMeta(_meta.PackageMetadata): - def __init__(self, orig: _meta.PackageMetadata): - self.orig = orig - - def __getitem__(self, item): - return self.orig.__getitem__(item) - - def __len__(self): - return self.orig.__len__() # pragma: nocover - - def __contains__(self, item): - return self.orig.__contains__(item) # pragma: nocover - - def __iter__(self): - return self.orig.__iter__() - - def get_all(self, name): - return self.orig.get_all(name) - - def get_payload(self): - return self.orig.get_payload() + def __init__(self, *args, **kwargs): + pass @property def json(self): From 5ca5fd8d9642ca88ce012cd2277eb961594f8c1b Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 1 Apr 2021 20:52:14 -0400 Subject: [PATCH 090/105] Extend PackageMetadata protocol to include the requisite interface, including some way to iterate over the keys and the new json property. --- importlib_metadata/_adapters.py | 4 ++++ importlib_metadata/_meta.py | 11 ++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/importlib_metadata/_adapters.py b/importlib_metadata/_adapters.py index 1f53f43e..5c62cf3c 100644 --- a/importlib_metadata/_adapters.py +++ b/importlib_metadata/_adapters.py @@ -13,6 +13,10 @@ def __new__(cls, orig: email.message.Message): def __init__(self, *args, **kwargs): pass + # suppress spurious error from mypy + def __iter__(self): + return super().__iter__() + @property def json(self): """ diff --git a/importlib_metadata/_meta.py b/importlib_metadata/_meta.py index fcb313ee..ec9febb5 100644 --- a/importlib_metadata/_meta.py +++ b/importlib_metadata/_meta.py @@ -1,5 +1,5 @@ from ._compat import Protocol -from typing import Any, List, TypeVar, Union +from typing import Any, Dict, Iterator, List, TypeVar, Union _T = TypeVar("_T") @@ -15,6 +15,9 @@ def __contains__(self, item: str) -> bool: def __getitem__(self, key: str) -> str: ... # pragma: no cover + def __iter__(self) -> Iterator[str]: + ... # pragma: no cover + def get_payload(self) -> str: ... # pragma: no cover @@ -22,3 +25,9 @@ def get_all(self, name: str, failobj: _T = ...) -> Union[List[Any], _T]: """ Return all values associated with a possibly multi-valued key. """ + + @property + def json(self) -> Dict[str, Union[str, List[str]]]: + """ + A JSON-compatible form of the metadata. + """ From 5ca9bc7dcf73d72260486afb28dadf5e532cf657 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 6 Apr 2021 08:57:13 -0400 Subject: [PATCH 091/105] Expand changelog around Distribution.entry_points. Closes #300. --- CHANGES.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 7ef6cc22..292c2357 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -111,11 +111,16 @@ v3.6.0 future behavior is invoked and an ``EntryPoints`` is the result. - Construction of entry points using +* #284: Construction of entry points using ``dict([EntryPoint, ...])`` is now deprecated and raises an appropriate DeprecationWarning and will be removed in a future version. +* #300: ``Distribution.entry_points`` now presents as an + ``EntryPoints`` object and access by index is no longer + allowed. If access by index is required, cast to a sequence + first. + v3.5.0 ====== From 6f754c2ae967aa6735f4eee97edbc3da96144bf2 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 5 Apr 2021 13:11:23 +0900 Subject: [PATCH 092/105] bpo-43651: PEP 597: Fix EncodingWarning in some tests (GH-25181) --- tests/fixtures.py | 2 +- tests/test_main.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/fixtures.py b/tests/fixtures.py index 508fbb5c..8ddde85a 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -257,7 +257,7 @@ def build_files(file_defs, prefix=pathlib.Path()): with full_name.open('wb') as f: f.write(contents) else: - with full_name.open('w') as f: + with full_name.open('w', encoding='utf-8') as f: f.write(DALS(contents)) diff --git a/tests/test_main.py b/tests/test_main.py index e8a66c0d..14c1660d 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -84,7 +84,7 @@ def pkg_with_dashes(site_dir): metadata_dir = site_dir / 'my_pkg.dist-info' metadata_dir.mkdir() metadata = metadata_dir / 'METADATA' - with metadata.open('w') as strm: + with metadata.open('w', encoding='utf-8') as strm: strm.write('Version: 1.0\n') return 'my-pkg' @@ -105,7 +105,7 @@ def pkg_with_mixed_case(site_dir): metadata_dir = site_dir / 'CherryPy.dist-info' metadata_dir.mkdir() metadata = metadata_dir / 'METADATA' - with metadata.open('w') as strm: + with metadata.open('w', encoding='utf-8') as strm: strm.write('Version: 1.0\n') return 'CherryPy' From 5595f2dc2a2ab4b7d1c7cf2488c427cfc134f184 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 12 Apr 2021 17:26:47 -0400 Subject: [PATCH 093/105] Re-use 'suppress' as imported. --- importlib_metadata/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 7c5eb2c7..2ba6ebfe 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -12,7 +12,6 @@ import functools import itertools import posixpath -import contextlib import collections from ._collections import FreezableDefaultDict, Pair @@ -689,7 +688,7 @@ def search(self, name): @property def mtime(self): - with contextlib.suppress(OSError): + with suppress(OSError): return os.stat(self.root).st_mtime self.lookup.cache_clear() From ed2b2c89ed1cb3506b596d4c6769691526a66fc9 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 12 Apr 2021 17:42:10 -0400 Subject: [PATCH 094/105] Update changelog. --- CHANGES.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 292c2357..a02d20e2 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,8 @@ +v3.10.1 +======= + +* Minor tweaks from CPython. + v3.10.0 ======= From e675da1402999bc69f29bf698635d5dec0c897b8 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 17 Apr 2021 20:12:17 -0400 Subject: [PATCH 095/105] Dedent values in the headers during construction. Now metadata values are normalized with leading whitespace removed and Description available even if found in the payload. --- importlib_metadata/_adapters.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/importlib_metadata/_adapters.py b/importlib_metadata/_adapters.py index 5c62cf3c..25099703 100644 --- a/importlib_metadata/_adapters.py +++ b/importlib_metadata/_adapters.py @@ -1,6 +1,5 @@ import string import textwrap -import itertools import email.message @@ -11,12 +10,24 @@ def __new__(cls, orig: email.message.Message): return res def __init__(self, *args, **kwargs): - pass + self._headers = self._repair_headers() # suppress spurious error from mypy def __iter__(self): return super().__iter__() + def _repair_headers(self): + def redent(value): + "Correct for RFC822 indentation" + if not value or '\n' not in value: + return value + return textwrap.dedent(' ' * 8 + value) + + headers = [(key, redent(value)) for key, value in vars(self)['_headers']] + if self._payload: + headers.append(('Description', self.get_payload())) + return headers + @property def json(self): """ @@ -36,21 +47,11 @@ def json(self): 'Supported-Platform', } - def redent(value): - "Correct for RFC822 indentation" - if not value or '\n' not in value: - return value - return textwrap.dedent(' ' * 8 + value) - def transform(key): - value = self.get_all(key) if key in multiple_use else redent(self[key]) + value = self.get_all(key) if key in multiple_use else self[key] if key == 'Keywords': value = value.split(string.whitespace) - if not value and key == 'Description': - value = self.get_payload() tk = key.lower().replace('-', '_') return tk, value - desc = ['Description'] if self.get_payload() else [] - keys = itertools.chain(self, desc) - return dict(map(transform, keys)) + return dict(map(transform, self)) From 84fd0137be2fb8152f0d4c1a373be17a498c1b0d Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 17 Apr 2021 20:25:14 -0400 Subject: [PATCH 096/105] Remove 'get_payload' from the PackageMetadata protocol now that Description is available. --- importlib_metadata/_meta.py | 3 --- tests/test_main.py | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/importlib_metadata/_meta.py b/importlib_metadata/_meta.py index ec9febb5..5cb690fe 100644 --- a/importlib_metadata/_meta.py +++ b/importlib_metadata/_meta.py @@ -18,9 +18,6 @@ def __getitem__(self, key: str) -> str: def __iter__(self) -> Iterator[str]: ... # pragma: no cover - def get_payload(self) -> str: - ... # pragma: no cover - def get_all(self, name: str, failobj: _T = ...) -> Union[List[Any], _T]: """ Return all values associated with a possibly multi-valued key. diff --git a/tests/test_main.py b/tests/test_main.py index e8a66c0d..d4c3cab6 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -162,7 +162,7 @@ def test_metadata_loads(self): def test_metadata_loads_egg_info(self): pkg_name = self.pkg_with_non_ascii_description_egg_info(self.site_dir) meta = metadata(pkg_name) - assert meta.get_payload() == 'pôrˈtend\n' + assert meta['Description'] == 'pôrˈtend\n' class DiscoveryTests(fixtures.EggInfoPkg, fixtures.DistInfoPkg, unittest.TestCase): From c87838e1ca29a80519a09d966cfdc9f5043699b7 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 17 Apr 2021 20:29:07 -0400 Subject: [PATCH 097/105] Update test to make the results consistent. --- tests/test_main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index d4c3cab6..e7ac4b20 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -130,7 +130,7 @@ def pkg_with_non_ascii_description(site_dir): metadata_dir.mkdir() metadata = metadata_dir / 'METADATA' with metadata.open('w', encoding='utf-8') as fp: - fp.write('Description: pôrˈtend\n') + fp.write('Description: pôrˈtend') return 'portend' @staticmethod @@ -150,7 +150,7 @@ def pkg_with_non_ascii_description_egg_info(site_dir): pôrˈtend """ - ).lstrip() + ).strip() ) return 'portend' @@ -162,7 +162,7 @@ def test_metadata_loads(self): def test_metadata_loads_egg_info(self): pkg_name = self.pkg_with_non_ascii_description_egg_info(self.site_dir) meta = metadata(pkg_name) - assert meta['Description'] == 'pôrˈtend\n' + assert meta['Description'] == 'pôrˈtend' class DiscoveryTests(fixtures.EggInfoPkg, fixtures.DistInfoPkg, unittest.TestCase): From 00bce755690745549ed00771e7d7c41e60048697 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 18 Apr 2021 09:36:50 -0400 Subject: [PATCH 098/105] Add test for keyword splitting. --- tests/test_api.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_api.py b/tests/test_api.py index 0f31226d..423a5998 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -249,12 +249,14 @@ def test_more_complex_deps_requires_text(self): def test_as_json(self): md = metadata('distinfo-pkg').json assert 'name' in md + assert md['keywords'] == ['sample', 'package'] desc = md['description'] assert desc.startswith('Once upon a time\nThere was') def test_as_json_egg_info(self): md = metadata('egginfo-pkg').json assert 'name' in md + assert md['keywords'] == ['sample', 'package'] desc = md['description'] assert desc.startswith('Once upon a time\nThere was') From 8bcd887e3f0f0df43f38b516fdac9bba7fcca638 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 18 Apr 2021 09:39:13 -0400 Subject: [PATCH 099/105] Fix splitting so test passes. --- importlib_metadata/_adapters.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/importlib_metadata/_adapters.py b/importlib_metadata/_adapters.py index 25099703..ea70f5b6 100644 --- a/importlib_metadata/_adapters.py +++ b/importlib_metadata/_adapters.py @@ -1,4 +1,4 @@ -import string +import re import textwrap import email.message @@ -50,7 +50,7 @@ def json(self): def transform(key): value = self.get_all(key) if key in multiple_use else self[key] if key == 'Keywords': - value = value.split(string.whitespace) + value = re.split(r'\s+', value) tk = key.lower().replace('-', '_') return tk, value From 008a07964fbab8b1a1dc067fc60d5a820a127da6 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 18 Apr 2021 09:43:20 -0400 Subject: [PATCH 100/105] Extend tests to capture multi-value cases. --- tests/test_api.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_api.py b/tests/test_api.py index 423a5998..bdddded8 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -252,6 +252,7 @@ def test_as_json(self): assert md['keywords'] == ['sample', 'package'] desc = md['description'] assert desc.startswith('Once upon a time\nThere was') + assert len(md['requires_dist']) == 2 def test_as_json_egg_info(self): md = metadata('egginfo-pkg').json @@ -259,6 +260,7 @@ def test_as_json_egg_info(self): assert md['keywords'] == ['sample', 'package'] desc = md['description'] assert desc.startswith('Once upon a time\nThere was') + assert len(md['classifier']) == 2 class LegacyDots(fixtures.DistInfoPkgWithDotLegacy, unittest.TestCase): From 96c0fec25bae2c997e17db423b853feef27a0b60 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 18 Apr 2021 10:03:51 -0400 Subject: [PATCH 101/105] Add test capturing expectation when metadata is not the common case. --- tests/fixtures.py | 11 +++++++++++ tests/test_api.py | 7 +++++++ 2 files changed, 18 insertions(+) diff --git a/tests/fixtures.py b/tests/fixtures.py index 508fbb5c..f9ee6a3b 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -1,5 +1,6 @@ import os import sys +import copy import shutil import pathlib import tempfile @@ -108,6 +109,16 @@ def setUp(self): super(DistInfoPkg, self).setUp() build_files(DistInfoPkg.files, self.site_dir) + def make_uppercase(self): + """ + Rewrite metadata with everything uppercase. + """ + shutil.rmtree(self.site_dir / "distinfo_pkg-1.0.0.dist-info") + files = copy.deepcopy(DistInfoPkg.files) + info = files["distinfo_pkg-1.0.0.dist-info"] + info["METADATA"] = info["METADATA"].upper() + build_files(files, self.site_dir) + class DistInfoPkgWithDot(OnSysPath, SiteDir): files: FilesDef = { diff --git a/tests/test_api.py b/tests/test_api.py index bdddded8..b3c8c2f8 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -262,6 +262,13 @@ def test_as_json_egg_info(self): assert desc.startswith('Once upon a time\nThere was') assert len(md['classifier']) == 2 + def test_as_json_odd_case(self): + self.make_uppercase() + md = metadata('distinfo-pkg').json + assert 'name' in md + assert len(md['requires_dist']) == 2 + assert md['keywords'] == ['SAMPLE', 'PACKAGE'] + class LegacyDots(fixtures.DistInfoPkgWithDotLegacy, unittest.TestCase): def test_name_normalization(self): From 401bb33804be717b1b8420efb120588c6fdd837c Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 18 Apr 2021 10:11:24 -0400 Subject: [PATCH 102/105] Honor case-insensitivity of metadata keys using FoldedCase. --- importlib_metadata/_adapters.py | 128 ++++++++++++++++++++++++++++---- 1 file changed, 115 insertions(+), 13 deletions(-) diff --git a/importlib_metadata/_adapters.py b/importlib_metadata/_adapters.py index ea70f5b6..e9233409 100644 --- a/importlib_metadata/_adapters.py +++ b/importlib_metadata/_adapters.py @@ -2,6 +2,8 @@ import textwrap import email.message +from ._functools import method_cache + class Message(email.message.Message): def __new__(cls, orig: email.message.Message): @@ -34,18 +36,22 @@ def json(self): Convert PackageMetadata to a JSON-compatible format per PEP 0566. """ - # TODO: Need to match case-insensitive - multiple_use = { - 'Classifier', - 'Obsoletes-Dist', - 'Platform', - 'Project-URL', - 'Provides-Dist', - 'Provides-Extra', - 'Requires-Dist', - 'Requires-External', - 'Supported-Platform', - } + multiple_use = set( + map( + FoldedCase, + [ + 'Classifier', + 'Obsoletes-Dist', + 'Platform', + 'Project-URL', + 'Provides-Dist', + 'Provides-Extra', + 'Requires-Dist', + 'Requires-External', + 'Supported-Platform', + ], + ) + ) def transform(key): value = self.get_all(key) if key in multiple_use else self[key] @@ -54,4 +60,100 @@ def transform(key): tk = key.lower().replace('-', '_') return tk, value - return dict(map(transform, self)) + return dict(map(transform, map(FoldedCase, self))) + + +# from jaraco.text 3.5 +class FoldedCase(str): + """ + A case insensitive string class; behaves just like str + except compares equal when the only variation is case. + + >>> s = FoldedCase('hello world') + + >>> s == 'Hello World' + True + + >>> 'Hello World' == s + True + + >>> s != 'Hello World' + False + + >>> s.index('O') + 4 + + >>> s.split('O') + ['hell', ' w', 'rld'] + + >>> sorted(map(FoldedCase, ['GAMMA', 'alpha', 'Beta'])) + ['alpha', 'Beta', 'GAMMA'] + + Sequence membership is straightforward. + + >>> "Hello World" in [s] + True + >>> s in ["Hello World"] + True + + You may test for set inclusion, but candidate and elements + must both be folded. + + >>> FoldedCase("Hello World") in {s} + True + >>> s in {FoldedCase("Hello World")} + True + + String inclusion works as long as the FoldedCase object + is on the right. + + >>> "hello" in FoldedCase("Hello World") + True + + But not if the FoldedCase object is on the left: + + >>> FoldedCase('hello') in 'Hello World' + False + + In that case, use in_: + + >>> FoldedCase('hello').in_('Hello World') + True + + >>> FoldedCase('hello') > FoldedCase('Hello') + False + """ + + def __lt__(self, other): + return self.lower() < other.lower() + + def __gt__(self, other): + return self.lower() > other.lower() + + def __eq__(self, other): + return self.lower() == other.lower() + + def __ne__(self, other): + return self.lower() != other.lower() + + def __hash__(self): + return hash(self.lower()) + + def __contains__(self, other): + return super(FoldedCase, self).lower().__contains__(other.lower()) + + def in_(self, other): + "Does self appear in other?" + return self in FoldedCase(other) + + # cache lower since it's likely to be called frequently. + @method_cache + def lower(self): + return super(FoldedCase, self).lower() + + def index(self, sub): + return self.lower().index(sub.lower()) + + def split(self, splitter=' ', maxsplit=0): + pattern = re.compile(re.escape(splitter), re.I) + return pattern.split(self, maxsplit) From cbaa4212bd76238febf10b8cca2dde642976a900 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 18 Apr 2021 10:16:29 -0400 Subject: [PATCH 103/105] Extract multiple_use_keys into a class property. --- importlib_metadata/_adapters.py | 136 ++++++-------------------------- importlib_metadata/_text.py | 99 +++++++++++++++++++++++ 2 files changed, 121 insertions(+), 114 deletions(-) create mode 100644 importlib_metadata/_text.py diff --git a/importlib_metadata/_adapters.py b/importlib_metadata/_adapters.py index e9233409..ab086180 100644 --- a/importlib_metadata/_adapters.py +++ b/importlib_metadata/_adapters.py @@ -2,10 +2,30 @@ import textwrap import email.message -from ._functools import method_cache +from ._text import FoldedCase class Message(email.message.Message): + multiple_use_keys = set( + map( + FoldedCase, + [ + 'Classifier', + 'Obsoletes-Dist', + 'Platform', + 'Project-URL', + 'Provides-Dist', + 'Provides-Extra', + 'Requires-Dist', + 'Requires-External', + 'Supported-Platform', + ], + ) + ) + """ + Keys that may be indicated multiple times per PEP 566. + """ + def __new__(cls, orig: email.message.Message): res = super().__new__(cls) vars(res).update(vars(orig)) @@ -36,124 +56,12 @@ def json(self): Convert PackageMetadata to a JSON-compatible format per PEP 0566. """ - multiple_use = set( - map( - FoldedCase, - [ - 'Classifier', - 'Obsoletes-Dist', - 'Platform', - 'Project-URL', - 'Provides-Dist', - 'Provides-Extra', - 'Requires-Dist', - 'Requires-External', - 'Supported-Platform', - ], - ) - ) def transform(key): - value = self.get_all(key) if key in multiple_use else self[key] + value = self.get_all(key) if key in self.multiple_use_keys else self[key] if key == 'Keywords': value = re.split(r'\s+', value) tk = key.lower().replace('-', '_') return tk, value return dict(map(transform, map(FoldedCase, self))) - - -# from jaraco.text 3.5 -class FoldedCase(str): - """ - A case insensitive string class; behaves just like str - except compares equal when the only variation is case. - - >>> s = FoldedCase('hello world') - - >>> s == 'Hello World' - True - - >>> 'Hello World' == s - True - - >>> s != 'Hello World' - False - - >>> s.index('O') - 4 - - >>> s.split('O') - ['hell', ' w', 'rld'] - - >>> sorted(map(FoldedCase, ['GAMMA', 'alpha', 'Beta'])) - ['alpha', 'Beta', 'GAMMA'] - - Sequence membership is straightforward. - - >>> "Hello World" in [s] - True - >>> s in ["Hello World"] - True - - You may test for set inclusion, but candidate and elements - must both be folded. - - >>> FoldedCase("Hello World") in {s} - True - >>> s in {FoldedCase("Hello World")} - True - - String inclusion works as long as the FoldedCase object - is on the right. - - >>> "hello" in FoldedCase("Hello World") - True - - But not if the FoldedCase object is on the left: - - >>> FoldedCase('hello') in 'Hello World' - False - - In that case, use in_: - - >>> FoldedCase('hello').in_('Hello World') - True - - >>> FoldedCase('hello') > FoldedCase('Hello') - False - """ - - def __lt__(self, other): - return self.lower() < other.lower() - - def __gt__(self, other): - return self.lower() > other.lower() - - def __eq__(self, other): - return self.lower() == other.lower() - - def __ne__(self, other): - return self.lower() != other.lower() - - def __hash__(self): - return hash(self.lower()) - - def __contains__(self, other): - return super(FoldedCase, self).lower().__contains__(other.lower()) - - def in_(self, other): - "Does self appear in other?" - return self in FoldedCase(other) - - # cache lower since it's likely to be called frequently. - @method_cache - def lower(self): - return super(FoldedCase, self).lower() - - def index(self, sub): - return self.lower().index(sub.lower()) - - def split(self, splitter=' ', maxsplit=0): - pattern = re.compile(re.escape(splitter), re.I) - return pattern.split(self, maxsplit) diff --git a/importlib_metadata/_text.py b/importlib_metadata/_text.py new file mode 100644 index 00000000..766979d9 --- /dev/null +++ b/importlib_metadata/_text.py @@ -0,0 +1,99 @@ +import re + +from ._functools import method_cache + + +# from jaraco.text 3.5 +class FoldedCase(str): + """ + A case insensitive string class; behaves just like str + except compares equal when the only variation is case. + + >>> s = FoldedCase('hello world') + + >>> s == 'Hello World' + True + + >>> 'Hello World' == s + True + + >>> s != 'Hello World' + False + + >>> s.index('O') + 4 + + >>> s.split('O') + ['hell', ' w', 'rld'] + + >>> sorted(map(FoldedCase, ['GAMMA', 'alpha', 'Beta'])) + ['alpha', 'Beta', 'GAMMA'] + + Sequence membership is straightforward. + + >>> "Hello World" in [s] + True + >>> s in ["Hello World"] + True + + You may test for set inclusion, but candidate and elements + must both be folded. + + >>> FoldedCase("Hello World") in {s} + True + >>> s in {FoldedCase("Hello World")} + True + + String inclusion works as long as the FoldedCase object + is on the right. + + >>> "hello" in FoldedCase("Hello World") + True + + But not if the FoldedCase object is on the left: + + >>> FoldedCase('hello') in 'Hello World' + False + + In that case, use in_: + + >>> FoldedCase('hello').in_('Hello World') + True + + >>> FoldedCase('hello') > FoldedCase('Hello') + False + """ + + def __lt__(self, other): + return self.lower() < other.lower() + + def __gt__(self, other): + return self.lower() > other.lower() + + def __eq__(self, other): + return self.lower() == other.lower() + + def __ne__(self, other): + return self.lower() != other.lower() + + def __hash__(self): + return hash(self.lower()) + + def __contains__(self, other): + return super(FoldedCase, self).lower().__contains__(other.lower()) + + def in_(self, other): + "Does self appear in other?" + return self in FoldedCase(other) + + # cache lower since it's likely to be called frequently. + @method_cache + def lower(self): + return super(FoldedCase, self).lower() + + def index(self, sub): + return self.lower().index(sub.lower()) + + def split(self, splitter=' ', maxsplit=0): + pattern = re.compile(re.escape(splitter), re.I) + return pattern.split(self, maxsplit) From c8b753ea0e196f0aedd5eaa39afc64f47f410af9 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 18 Apr 2021 10:27:41 -0400 Subject: [PATCH 104/105] Update changelog. --- CHANGES.rst | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 7ef6cc22..bd5ca54f 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,24 @@ +v4.0.0 +======= + +* #304: ``PackageMetadata`` as returned by ``metadata()`` + and ``Distribution.metadata()`` now provides normalized + metadata honoring PEP 566: + + - If a long description is provided in the payload of the + RFC 822 value, it can be retrieved as the ``Description`` + field. + - Any multi-line values in the metadata will be returned as + such. + - For any multi-line values, line continuation characters + are removed. This backward-incompatible change means + that any projects relying on the RFC 822 line continuation + characters being present must be tolerant to them having + been removed. + - Add a ``json`` property that provides the metadata + converted to a JSON-compatible form per PEP 566. + + v3.10.0 ======= From 0c86a02fbbbbdc9a888d9f2a149ec0d2cf5e7d2c Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 20 Apr 2021 21:39:56 -0400 Subject: [PATCH 105/105] Add compatibility matrix comparing importlib_metadata versions and the associated stdlib version. Fixes #306. --- CHANGES.rst | 5 +++++ README.rst | 26 +++++++++++++++++++++----- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index c83e3552..a9ef4870 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,8 @@ +v4.0.1 +======= + +* #306: Clearer guidance about compatibility in readme. + v4.0.0 ======= diff --git a/README.rst b/README.rst index 512c3cc2..986d3ca1 100644 --- a/README.rst +++ b/README.rst @@ -20,11 +20,27 @@ Library to access the metadata for a Python package. -As of Python 3.8, this functionality has been added to the -`Python standard library -`_. -This package supplies backports of that functionality including -improvements added to subsequent Python versions. +This package supplies third-party access to the functionality of +`importlib.metadata `_ +including improvements added to subsequent Python versions. + + +Compatibility +============= + +New features are introduced in this third-party library and later merged +into CPython. The following table indicates which versions of this library +were contributed to different versions in the standard library: + +.. list-table:: + :header-rows: 1 + + * - importlib_metadata + - stdlib + * - 3.10 + - 3.10 + * - 1.4 + - 3.8 Usage pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy