From 36199b01631ddab46f17fd56305ad9fae0ac3565 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius Date: Sun, 16 Jul 2023 23:42:38 +0900 Subject: [PATCH 1/9] feat: support alias rendering for python aware toolchain targets --- python/private/render_pkg_aliases.bzl | 189 ++++++++++++++++++ .../render_pkg_aliases/BUILD.bazel | 3 + .../render_pkg_aliases_test.bzl | 174 ++++++++++++++++ 3 files changed, 366 insertions(+) create mode 100644 python/private/render_pkg_aliases.bzl create mode 100644 tests/pip_hub_repository/render_pkg_aliases/BUILD.bazel create mode 100644 tests/pip_hub_repository/render_pkg_aliases/render_pkg_aliases_test.bzl diff --git a/python/private/render_pkg_aliases.bzl b/python/private/render_pkg_aliases.bzl new file mode 100644 index 0000000000..28042e0507 --- /dev/null +++ b/python/private/render_pkg_aliases.bzl @@ -0,0 +1,189 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""render_pkg_aliases is a function to generate BUILD.bazel contents used to create user-friendly aliases. + +This is used in bzlmod and non-bzlmod setups.""" + +load("//python/private:normalize_name.bzl", "normalize_name") +load(":version_label.bzl", "version_label") + +_DEFAULT = """\ +alias( + name = "{name}", + actual = "@{repo_name}_{dep}//:{target}", +)""" + +_SELECT = """\ +alias( + name = "{name}", + actual = select({{{selects}}}), +)""" + +def _render_alias( + *, + name, + repo_name, + dep, + target, + default_version, + versions, + rules_python): + """Render an alias for common targets + + If the versions is passed, then the `rules_python` must be passed as well and + an alias with a select statement based on the python version is going to be + generated. + """ + if versions == None: + return _DEFAULT.format( + name = name, + repo_name = repo_name, + dep = dep, + target = target, + ) + + # Create the alias repositories which contains different select + # statements These select statements point to the different pip + # whls that are based on a specific version of Python. + selects = {} + for full_version in versions: + condition = "@@{rules_python}//python/config_settings:is_python_{full_python_version}".format( + rules_python = rules_python, + full_python_version = full_version, + ) + actual = "@{repo_name}_{version}_{dep}//:{target}".format( + repo_name = repo_name, + version = version_label(full_version), + dep = dep, + target = target, + ) + selects[condition] = actual + + default_actual = "@{repo_name}_{version}_{dep}//:{target}".format( + repo_name = repo_name, + version = version_label(default_version), + dep = dep, + target = target, + ) + selects["//conditions:default"] = default_actual + + return _SELECT.format( + name = name, + selects = "\n{} ".format( + "".join([ + " {}: {},\n".format(repr(k), repr(v)) + for k, v in selects.items() + ]), + ), + ) + +def _render_entry_points(repo_name, dep): + return """\ +load("@{repo_name}_{dep}//:entry_points.bzl", "entry_points") + +[ + alias( + name = script, + actual = "@{repo_name}_{dep}//:" + target, + visibility = ["//visibility:public"], + ) + for script, target in entry_points.items() +] +""".format( + repo_name = repo_name, + dep = dep, + ) + +def _render_common_aliases(repo_name, name, versions = None, default_version = None, rules_python = None): + return "\n\n".join([ + """package(default_visibility = ["//visibility:public"])""", + _render_alias( + name = name, + repo_name = repo_name, + dep = name, + target = "pkg", + versions = versions, + default_version = default_version, + rules_python = rules_python, + ), + ] + [ + _render_alias( + name = target, + repo_name = repo_name, + dep = name, + target = target, + versions = versions, + default_version = default_version, + rules_python = rules_python, + ) + for target in ["pkg", "whl", "data", "dist_info"] + ]) + +def render_pkg_aliases(*, repo_name, bzl_packages = None, whl_map = None, rules_python = None, default_version = None): + """Create alias declarations for each PyPI package. + + The aliases should be appended to the pip_repository BUILD.bazel file. These aliases + allow users to use requirement() without needed a corresponding `use_repo()` for each dep + when using bzlmod. + + Args: + repo_name: the repository name of the hub repository that is visible to the users that is + also used as the prefix for the spoke repo names (e.g. "pip", "pypi"). + bzl_packages: the list of packages to setup, if not specified, whl_map.keys() will be used instead. + whl_map: the whl_map for generating Python version aware aliases. + default_version: the default version to be used for the aliases. + rules_python: the name of the rules_python workspace. + + Returns: + A dict of file paths and their contents. + """ + if not bzl_packages and whl_map: + bzl_packages = list(whl_map.keys()) + + contents = {} + for name in bzl_packages: + versions = None + if whl_map != None: + versions = whl_map[name] + name = normalize_name(name) + + filename = "{}/BUILD.bazel".format(name) + contents[filename] = _render_common_aliases( + repo_name = repo_name, + name = name, + versions = versions, + rules_python = rules_python, + default_version = default_version, + ).strip() + + if versions == None: + # NOTE: this code would be normally executed in the non-bzlmod + # scenario, where we are requesting friendly aliases to be + # generated. In that case, we will not be creating aliases for + # entry_points to leave the behaviour unchanged from previous + # rules_python versions. + continue + + # NOTE @aignas 2023-07-07: we are not creating aliases using a select + # and the version specific aliases because we would need to fetch the + # package for all versions in order to construct the said select. + for version in versions: + filename = "{}/bin_py{}/BUILD.bazel".format(name, version_label(version)) + contents[filename] = _render_entry_points( + repo_name = "{}_{}".format(repo_name, version_label(version)), + dep = name, + ).strip() + + return contents diff --git a/tests/pip_hub_repository/render_pkg_aliases/BUILD.bazel b/tests/pip_hub_repository/render_pkg_aliases/BUILD.bazel new file mode 100644 index 0000000000..f2e0126666 --- /dev/null +++ b/tests/pip_hub_repository/render_pkg_aliases/BUILD.bazel @@ -0,0 +1,3 @@ +load(":render_pkg_aliases_test.bzl", "render_pkg_aliases_test_suite") + +render_pkg_aliases_test_suite(name = "render_pkg_aliases_tests") diff --git a/tests/pip_hub_repository/render_pkg_aliases/render_pkg_aliases_test.bzl b/tests/pip_hub_repository/render_pkg_aliases/render_pkg_aliases_test.bzl new file mode 100644 index 0000000000..eed8653705 --- /dev/null +++ b/tests/pip_hub_repository/render_pkg_aliases/render_pkg_aliases_test.bzl @@ -0,0 +1,174 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"render_pkg_aliases tests" + +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python/private:render_pkg_aliases.bzl", "render_pkg_aliases") # buildifier: disable=bzl-visibility + +_tests = [] + +def _test_legacy_aliases(env): + actual = render_pkg_aliases( + bzl_packages = ["foo"], + repo_name = "pypi", + ) + + want = { + "foo/BUILD.bazel": """\ +package(default_visibility = ["//visibility:public"]) + +alias( + name = "foo", + actual = "@pypi_foo//:pkg", +) + +alias( + name = "pkg", + actual = "@pypi_foo//:pkg", +) + +alias( + name = "whl", + actual = "@pypi_foo//:whl", +) + +alias( + name = "data", + actual = "@pypi_foo//:data", +) + +alias( + name = "dist_info", + actual = "@pypi_foo//:dist_info", +)""", + } + + env.expect.that_dict(actual).contains_exactly(want) + +_tests.append(_test_legacy_aliases) + +def _test_all_legacy_aliases_are_created(env): + actual = render_pkg_aliases( + bzl_packages = ["foo", "bar"], + repo_name = "pypi", + ) + + want_files = ["bar/BUILD.bazel", "foo/BUILD.bazel"] + + env.expect.that_dict(actual).keys().contains_exactly(want_files) + +_tests.append(_test_all_legacy_aliases_are_created) + +def _test_bzlmod_aliases(env): + actual = render_pkg_aliases( + default_version = "3.2.3", + repo_name = "pypi", + rules_python = "rules_python", + whl_map = { + "bar-baz": ["3.2.3"], + }, + ) + + want = { + "bar_baz/BUILD.bazel": """\ +package(default_visibility = ["//visibility:public"]) + +alias( + name = "bar_baz", + actual = select({ + "@@rules_python//python/config_settings:is_python_3.2.3": "@pypi_32_bar_baz//:pkg", + "//conditions:default": "@pypi_32_bar_baz//:pkg", + }), +) + +alias( + name = "pkg", + actual = select({ + "@@rules_python//python/config_settings:is_python_3.2.3": "@pypi_32_bar_baz//:pkg", + "//conditions:default": "@pypi_32_bar_baz//:pkg", + }), +) + +alias( + name = "whl", + actual = select({ + "@@rules_python//python/config_settings:is_python_3.2.3": "@pypi_32_bar_baz//:whl", + "//conditions:default": "@pypi_32_bar_baz//:whl", + }), +) + +alias( + name = "data", + actual = select({ + "@@rules_python//python/config_settings:is_python_3.2.3": "@pypi_32_bar_baz//:data", + "//conditions:default": "@pypi_32_bar_baz//:data", + }), +) + +alias( + name = "dist_info", + actual = select({ + "@@rules_python//python/config_settings:is_python_3.2.3": "@pypi_32_bar_baz//:dist_info", + "//conditions:default": "@pypi_32_bar_baz//:dist_info", + }), +)""", + "bar_baz/bin_py32/BUILD.bazel": """\ +load("@pypi_32_bar_baz//:entry_points.bzl", "entry_points") + +[ + alias( + name = script, + actual = "@pypi_32_bar_baz//:" + target, + visibility = ["//visibility:public"], + ) + for script, target in entry_points.items() +]""", + } + + env.expect.that_dict(actual).contains_exactly(want) + +_tests.append(_test_bzlmod_aliases) + +def _test_bzlmod_aliases_are_created_for_all_wheels(env): + actual = render_pkg_aliases( + default_version = "3.2.3", + repo_name = "pypi", + rules_python = "rules_python", + whl_map = { + "bar": ["3.1.2", "3.2.3"], + "foo": ["3.1.2", "3.2.3"], + }, + ) + + want_files = [ + "bar/BUILD.bazel", + "bar/bin_py31/BUILD.bazel", + "bar/bin_py32/BUILD.bazel", + "foo/BUILD.bazel", + "foo/bin_py31/BUILD.bazel", + "foo/bin_py32/BUILD.bazel", + ] + + env.expect.that_dict(actual).keys().contains_exactly(want_files) + +_tests.append(_test_bzlmod_aliases_are_created_for_all_wheels) + +def render_pkg_aliases_test_suite(name): + """Create the test suite. + + Args: + name: the name of the test suite + """ + test_suite(name = name, basic_tests = _tests) From 631382259c5e86a7895bbb1252d92dde7320f0c7 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius Date: Mon, 17 Jul 2023 00:04:46 +0900 Subject: [PATCH 2/9] refactor: use render_pkg_aliases --- python/pip_install/pip_repository.bzl | 55 ++++----------------------- 1 file changed, 7 insertions(+), 48 deletions(-) diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl index 99d1fb05b1..0e80542747 100644 --- a/python/pip_install/pip_repository.bzl +++ b/python/pip_install/pip_repository.bzl @@ -21,6 +21,7 @@ load("//python/pip_install:requirements_parser.bzl", parse_requirements = "parse load("//python/pip_install/private:srcs.bzl", "PIP_INSTALL_PY_SRCS") load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") load("//python/private:normalize_name.bzl", "normalize_name") +load("//python/private:render_pkg_aliases.bzl", "render_pkg_aliases") load("//python/private:toolchains_repo.bzl", "get_host_os_arch") CPPFLAGS = "CPPFLAGS" @@ -268,56 +269,12 @@ A requirements_lock attribute must be specified, or a platform-specific lockfile """) return requirements_txt -def _pkg_aliases(rctx, repo_name, bzl_packages): - """Create alias declarations for each python dependency. - - The aliases should be appended to the pip_repository BUILD.bazel file. These aliases - allow users to use requirement() without needed a corresponding `use_repo()` for each dep - when using bzlmod. - - Args: - rctx: the repository context. - repo_name: the repository name of the parent that is visible to the users. - bzl_packages: the list of packages to setup. - """ - for name in bzl_packages: - build_content = """package(default_visibility = ["//visibility:public"]) - -alias( - name = "{name}", - actual = "@{repo_name}_{dep}//:pkg", -) - -alias( - name = "pkg", - actual = "@{repo_name}_{dep}//:pkg", -) - -alias( - name = "whl", - actual = "@{repo_name}_{dep}//:whl", -) - -alias( - name = "data", - actual = "@{repo_name}_{dep}//:data", -) - -alias( - name = "dist_info", - actual = "@{repo_name}_{dep}//:dist_info", -) -""".format( - name = name, - repo_name = repo_name, - dep = name, - ) - rctx.file("{}/BUILD.bazel".format(name), build_content) - def _create_pip_repository_bzlmod(rctx, bzl_packages, requirements): repo_name = rctx.attr.repo_name build_contents = _BUILD_FILE_CONTENTS - _pkg_aliases(rctx, repo_name, bzl_packages) + aliases = render_pkg_aliases(repo_name = repo_name, bzl_packages = bzl_packages) + for path, contents in aliases.items(): + rctx.file(path, contents) # NOTE: we are using the canonical name with the double '@' in order to # always uniquely identify a repository, as the labels are being passed as @@ -458,7 +415,9 @@ def _pip_repository_impl(rctx): config["python_interpreter_target"] = str(rctx.attr.python_interpreter_target) if rctx.attr.incompatible_generate_aliases: - _pkg_aliases(rctx, rctx.attr.name, bzl_packages) + aliases = render_pkg_aliases(repo_name = rctx.attr.name, bzl_packages = bzl_packages) + for path, contents in aliases.items(): + rctx.file(path, contents) rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS) rctx.template("requirements.bzl", rctx.attr._template, substitutions = { From 5eab849ee24f05152da7dda82514092da9e3d8b3 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius Date: Sun, 16 Jul 2023 23:42:58 +0900 Subject: [PATCH 3/9] test: add a helper to distinguish between bazel 6 --- tests/test_env.bzl | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 tests/test_env.bzl diff --git a/tests/test_env.bzl b/tests/test_env.bzl new file mode 100644 index 0000000000..ee9a32437f --- /dev/null +++ b/tests/test_env.bzl @@ -0,0 +1,27 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Functions for inspecting the test environment. + +Currently contains: +* A check to see if we are on Bazel 6.0+ +""" + +def _is_bazel_6_or_higher(): + return testing.ExecutionInfo == testing.ExecutionInfo + +test_env = struct( + is_bazel_6_or_higher = _is_bazel_6_or_higher, +) From 79fc0586d6f0c19d25688efd32c79a6d8b4fbd95 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius Date: Sun, 16 Jul 2023 23:43:27 +0900 Subject: [PATCH 4/9] feat: add a full_version helper --- python/private/full_version.bzl | 35 +++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 python/private/full_version.bzl diff --git a/python/private/full_version.bzl b/python/private/full_version.bzl new file mode 100644 index 0000000000..db4411cf79 --- /dev/null +++ b/python/private/full_version.bzl @@ -0,0 +1,35 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"A small helper to ensure that we are working with full versions." + +load("//python:versions.bzl", "MINOR_MAPPING") + +def full_version(version): + """Return a full version. + + Args: + version: the version in `X.Y` or `X.Y.Z` format. + + Returns: + a full version given the version string. If the string is already a + major version then we return it as is. + """ + parts = version.split(".") + if len(parts) == 2: + return MINOR_MAPPING[version] + elif len(parts) == 3: + return version + else: + fail("Unknown version format: {}".format(version)) From 590830bcbdf93a5731b6ee8f93d199e0df9fd50f Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius Date: Mon, 17 Jul 2023 00:10:11 +0900 Subject: [PATCH 5/9] refactor: use full_version --- python/extensions/private/pythons_hub.bzl | 11 +++-------- python/pip.bzl | 4 ++-- python/repositories.bzl | 5 ++--- 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/python/extensions/private/pythons_hub.bzl b/python/extensions/private/pythons_hub.bzl index a64f203bd6..f36ce45521 100644 --- a/python/extensions/private/pythons_hub.bzl +++ b/python/extensions/private/pythons_hub.bzl @@ -14,7 +14,8 @@ "Repo rule used by bzlmod extension to create a repo that has a map of Python interpreters and their labels" -load("//python:versions.bzl", "MINOR_MAPPING", "WINDOWS_NAME") +load("//python:versions.bzl", "WINDOWS_NAME") +load("//python/private:full_version.bzl", "full_version") load( "//python/private:toolchains_repo.bzl", "get_host_os_arch", @@ -28,12 +29,6 @@ def _have_same_length(*lists): fail("expected at least one list") return len({len(length): None for length in lists}) == 1 -def _get_version(python_version): - # we need to get the MINOR_MAPPING or use the full version - if python_version in MINOR_MAPPING: - python_version = MINOR_MAPPING[python_version] - return python_version - def _python_toolchain_build_file_content( prefixes, python_versions, @@ -55,7 +50,7 @@ def _python_toolchain_build_file_content( # build the toolchain content by calling python_toolchain_build_file_content return "\n".join([python_toolchain_build_file_content( prefix = prefixes[i], - python_version = _get_version(python_versions[i]), + python_version = full_version(python_versions[i]), set_python_version_constraint = set_python_version_constraints[i], user_repository_name = user_repository_names[i], rules_python = rules_python, diff --git a/python/pip.bzl b/python/pip.bzl index cae15919b0..352694f97f 100644 --- a/python/pip.bzl +++ b/python/pip.bzl @@ -17,7 +17,7 @@ load("//python/pip_install:pip_repository.bzl", "pip_repository", _package_annot load("//python/pip_install:repositories.bzl", "pip_install_dependencies") load("//python/pip_install:requirements.bzl", _compile_pip_requirements = "compile_pip_requirements") load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") -load(":versions.bzl", "MINOR_MAPPING") +load("//python/private:full_version.bzl", "full_version") compile_pip_requirements = _compile_pip_requirements package_annotation = _package_annotation @@ -296,7 +296,7 @@ alias( for [python_version, repo_prefix] in version_map: alias.append("""\ "@{rules_python}//python/config_settings:is_python_{full_python_version}": "{actual}",""".format( - full_python_version = MINOR_MAPPING[python_version] if python_version in MINOR_MAPPING else python_version, + full_python_version = full_version(python_version), actual = "@{repo_prefix}{wheel_name}//:{alias_name}".format( repo_prefix = repo_prefix, wheel_name = wheel_name, diff --git a/python/repositories.bzl b/python/repositories.bzl index 62d94210e0..8ac750dda6 100644 --- a/python/repositories.bzl +++ b/python/repositories.bzl @@ -21,6 +21,7 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", _http_archive = "http_archi load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") load("//python/private:coverage_deps.bzl", "coverage_dep") +load("//python/private:full_version.bzl", "full_version") load( "//python/private:toolchains_repo.bzl", "multi_toolchain_aliases", @@ -30,7 +31,6 @@ load( load( ":versions.bzl", "DEFAULT_RELEASE_BASE_URL", - "MINOR_MAPPING", "PLATFORMS", "TOOL_VERSIONS", "get_release_info", @@ -505,8 +505,7 @@ def python_register_toolchains( base_url = kwargs.pop("base_url", DEFAULT_RELEASE_BASE_URL) - if python_version in MINOR_MAPPING: - python_version = MINOR_MAPPING[python_version] + python_version = full_version(python_version) toolchain_repo_name = "{name}_toolchains".format(name = name) From d1f62cb27255dca26e35ed631917e76c77a154d8 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius Date: Mon, 17 Jul 2023 00:18:26 +0900 Subject: [PATCH 6/9] feat: generate entry_points.bzl for each whl_library --- .../tools/wheel_installer/wheel_installer.py | 42 ++++++++++++++----- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/python/pip_install/tools/wheel_installer/wheel_installer.py b/python/pip_install/tools/wheel_installer/wheel_installer.py index 9b363c3068..7d75cb31ca 100644 --- a/python/pip_install/tools/wheel_installer/wheel_installer.py +++ b/python/pip_install/tools/wheel_installer/wheel_installer.py @@ -18,7 +18,6 @@ import json import os import re -import shutil import subprocess import sys import textwrap @@ -226,7 +225,7 @@ def _generate_build_file_contents( "**/* *", "**/*.py", "**/*.pyc", - "**/*.pyc.*", # During pyc creation, temp files named *.pyc.NNNN are created + "**/*.pyc.*", # During pyc creation, temp files named *.pyc.NNNN are created # RECORD is known to contain sha256 checksums of files which might include the checksums # of generated files produced when wheels are installed. The file is ignored to avoid # Bazel caching issues. @@ -329,7 +328,7 @@ def _extract_wheel( bazel.sanitised_repo_file_label(d, repo_prefix=repo_prefix) for d in whl_deps ] - entry_points = [] + entry_points = {} for name, (module, attribute) in sorted(whl.entry_points().items()): # There is an extreme edge-case with entry_points that end with `.py` # See: https://github.com/bazelbuild/bazel/blob/09c621e4cf5b968f4c6cdf905ab142d5961f9ddc/src/test/java/com/google/devtools/build/lib/rules/python/PyBinaryConfiguredTargetTest.java#L174 @@ -341,16 +340,14 @@ def _extract_wheel( (installation_dir / entry_point_script_name).write_text( _generate_entry_point_contents(module, attribute) ) - entry_points.append( - _generate_entry_point_rule( - entry_point_target_name, - entry_point_script_name, - bazel.PY_LIBRARY_LABEL, - ) + entry_points[entry_point_without_py] = _generate_entry_point_rule( + entry_point_target_name, + entry_point_script_name, + bazel.PY_LIBRARY_LABEL, ) with open(os.path.join(installation_dir, "BUILD.bazel"), "w") as build_file: - additional_content = entry_points + additional_content = list(entry_points.values()) data = [] data_exclude = pip_data_exclude srcs_exclude = [] @@ -381,6 +378,31 @@ def _extract_wheel( ) build_file.write(contents) + with open(installation_dir / "entry_points.bzl", "w") as entry_point_file: + entry_points_str = "" + if entry_points: + entry_points_str = "\n" + "".join( + [ + f' "{script}": "{bazel.WHEEL_ENTRY_POINT_PREFIX}_{script}",\n' + for script in sorted(entry_points.keys()) + ] + ) + + contents = textwrap.dedent( + """\ + \"\"\" + This file contains the entry_point script names as a dict, where the keys + are the script names and the values are the target names. + + generated by @rules_python//python/pip_install/tools/wheel_installer/wheel_installer.py + \"\"\" + + entry_points = {{{}}} + """ + ).format(entry_points_str) + + entry_point_file.write(contents) + def main() -> None: parser = argparse.ArgumentParser( From b9a2f5f66ad58b96d815b8c2aa010f82df1ce2f9 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius Date: Mon, 17 Jul 2023 00:24:31 +0900 Subject: [PATCH 7/9] feat: consolidate bzlmod pip hub repos and support entry_points --- examples/bzlmod/MODULE.bazel | 5 +- examples/bzlmod/entry_point/BUILD.bazel | 2 +- python/extensions/pip.bzl | 45 ++------ .../extensions/private/pip_hub_repository.bzl | 95 ++++++++++++++++ .../extensions/private/requirements.bzl.tmpl | 52 +++++++++ python/pip_install/entry_point.bzl | 69 ++++++++++++ ...ub_repository_requirements_bzlmod.bzl.tmpl | 35 ------ python/pip_install/pip_repository.bzl | 102 ------------------ ...ip_repository_requirements_bzlmod.bzl.tmpl | 33 ------ .../entry_point/BUILD.bazel | 17 +++ .../entry_point/entry_point_test.bzl | 94 ++++++++++++++++ 11 files changed, 336 insertions(+), 213 deletions(-) create mode 100644 python/extensions/private/pip_hub_repository.bzl create mode 100644 python/extensions/private/requirements.bzl.tmpl create mode 100644 python/pip_install/entry_point.bzl delete mode 100644 python/pip_install/pip_hub_repository_requirements_bzlmod.bzl.tmpl delete mode 100644 python/pip_install/pip_repository_requirements_bzlmod.bzl.tmpl create mode 100644 tests/pip_hub_repository/entry_point/BUILD.bazel create mode 100644 tests/pip_hub_repository/entry_point/entry_point_test.bzl diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel index df88ae8490..7c5083c82a 100644 --- a/examples/bzlmod/MODULE.bazel +++ b/examples/bzlmod/MODULE.bazel @@ -118,10 +118,7 @@ pip.parse( "@whl_mods_hub//:wheel.json": "wheel", }, ) - -# NOTE: The pip_39 repo is only used because the plain `@pip` repo doesn't -# yet support entry points; see https://github.com/bazelbuild/rules_python/issues/1262 -use_repo(pip, "pip", "pip_39") +use_repo(pip, "pip") bazel_dep(name = "other_module", version = "", repo_name = "our_other_module") local_path_override( diff --git a/examples/bzlmod/entry_point/BUILD.bazel b/examples/bzlmod/entry_point/BUILD.bazel index f68552c3ef..dfc02b00a0 100644 --- a/examples/bzlmod/entry_point/BUILD.bazel +++ b/examples/bzlmod/entry_point/BUILD.bazel @@ -1,4 +1,4 @@ -load("@pip_39//:requirements.bzl", "entry_point") +load("@pip//:requirements.bzl", "entry_point") load("@rules_python//python:defs.bzl", "py_test") alias( diff --git a/python/extensions/pip.bzl b/python/extensions/pip.bzl index add69a4c64..699fb751f6 100644 --- a/python/extensions/pip.bzl +++ b/python/extensions/pip.bzl @@ -15,17 +15,16 @@ "pip module extension for use with bzlmod" load("@pythons_hub//:interpreters.bzl", "DEFAULT_PYTHON_VERSION", "INTERPRETER_LABELS") -load("@rules_python//python:pip.bzl", "whl_library_alias") load( "@rules_python//python/pip_install:pip_repository.bzl", "locked_requirements_label", - "pip_hub_repository_bzlmod", "pip_repository_attrs", - "pip_repository_bzlmod", "use_isolated", "whl_library", ) load("@rules_python//python/pip_install:requirements_parser.bzl", parse_requirements = "parse") +load("//python/extensions/private:pip_hub_repository.bzl", "pip_hub_repository") +load("//python/private:full_version.bzl", "full_version") load("//python/private:normalize_name.bzl", "normalize_name") load("//python/private:version_label.bzl", "version_label") @@ -111,16 +110,6 @@ def _create_versioned_pip_and_whl_repos(module_ctx, pip_attr, whl_map): requirements = parse_result.requirements extra_pip_args = pip_attr.extra_pip_args + parse_result.options - # Create the repository where users load the `requirement` macro. Under bzlmod - # this does not create the install_deps() macro. - # TODO: we may not need this repository once we have entry points - # supported. For now a user can access this repository and use - # the entrypoint functionality. - pip_repository_bzlmod( - name = pip_name, - repo_name = pip_name, - requirements_lock = pip_attr.requirements_lock, - ) if hub_name not in whl_map: whl_map[hub_name] = {} @@ -155,9 +144,9 @@ def _create_versioned_pip_and_whl_repos(module_ctx, pip_attr, whl_map): ) if whl_name not in whl_map[hub_name]: - whl_map[hub_name][whl_name] = {} + whl_map[hub_name][whl_name] = [] - whl_map[hub_name][whl_name][pip_attr.python_version] = pip_name + "_" + whl_map[hub_name][whl_name].append(full_version(pip_attr.python_version)) def _pip_impl(module_ctx): """Implementation of a class tag that creates the pip hub(s) and corresponding pip spoke, alias and whl repositories. @@ -295,32 +284,12 @@ def _pip_impl(module_ctx): _create_versioned_pip_and_whl_repos(module_ctx, pip_attr, hub_whl_map) for hub_name, whl_map in hub_whl_map.items(): - for whl_name, version_map in whl_map.items(): - if DEFAULT_PYTHON_VERSION not in version_map: - fail(( - "Default python version '{version}' missing in pip " + - "hub '{hub}': update your pip.parse() calls so that " + - 'includes `python_version = "{version}"`' - ).format( - version = DEFAULT_PYTHON_VERSION, - hub = hub_name, - )) - - # Create the alias repositories which contains different select - # statements These select statements point to the different pip - # whls that are based on a specific version of Python. - whl_library_alias( - name = hub_name + "_" + whl_name, - wheel_name = whl_name, - default_version = DEFAULT_PYTHON_VERSION, - version_map = version_map, - ) - # Create the hub repository for pip. - pip_hub_repository_bzlmod( + pip_hub_repository( name = hub_name, repo_name = hub_name, - whl_library_alias_names = whl_map.keys(), + whl_map = whl_map, + default_version = full_version(DEFAULT_PYTHON_VERSION), ) def _pip_parse_ext_attrs(): diff --git a/python/extensions/private/pip_hub_repository.bzl b/python/extensions/private/pip_hub_repository.bzl new file mode 100644 index 0000000000..b73c003500 --- /dev/null +++ b/python/extensions/private/pip_hub_repository.bzl @@ -0,0 +1,95 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +A pip_hub_repository rule used to create bzlmod hub repos for PyPI packages. + +It assumes that version aware toolchain is used and is responsible for setting up +aliases for entry points and the actual package targets. +""" + +load("//python/private:render_pkg_aliases.bzl", "render_pkg_aliases") + +_BUILD_FILE_CONTENTS = """\ +package(default_visibility = ["//visibility:public"]) + +# Ensure the `requirements.bzl` source can be accessed by stardoc, since users load() from it +exports_files(["requirements.bzl"]) +""" + +def _impl(rctx): + bzl_packages = rctx.attr.whl_map.keys() + repo_name = rctx.attr.repo_name + + aliases = render_pkg_aliases( + repo_name = repo_name, + whl_map = rctx.attr.whl_map, + default_version = rctx.attr.default_version, + rules_python = rctx.attr._template.workspace_name, + ) + for path, contents in aliases.items(): + rctx.file(path, contents) + + # NOTE: we are using the canonical name with the double '@' in order to + # always uniquely identify a repository, as the labels are being passed as + # a string and the resolution of the label happens at the call-site of the + # `requirement`, et al. macros. + macro_tmpl = "@@{name}//{{}}:{{}}".format(name = rctx.attr.name) + + rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS) + rctx.template("requirements.bzl", rctx.attr._template, substitutions = { + "%%ALL_DATA_REQUIREMENTS%%": repr([ + macro_tmpl.format(p, "data") + for p in bzl_packages + ]), + "%%ALL_REQUIREMENTS%%": repr([ + macro_tmpl.format(p, p) + for p in bzl_packages + ]), + "%%ALL_WHL_REQUIREMENTS%%": repr([ + macro_tmpl.format(p, "whl") + for p in bzl_packages + ]), + "%%DEFAULT_PY_VERSION%%": repr(rctx.attr.default_version), + "%%MACRO_TMPL%%": macro_tmpl, + "%%NAME%%": rctx.attr.name, + "%%PACKAGE_AVAILABILITY%%": repr({ + k: [v for v in versions] + for k, versions in rctx.attr.whl_map.items() + }), + "%%RULES_PYTHON%%": rctx.attr._template.workspace_name, + }) + +pip_hub_repository = repository_rule( + attrs = { + "default_version": attr.string( + mandatory = True, + doc = """\ +This is the default python version in the format of X.Y.Z. This should match +what is setup by the 'python' extension using the 'is_default = True' +setting.""", + ), + "repo_name": attr.string( + mandatory = True, + doc = "The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name.", + ), + "whl_map": attr.string_list_dict( + mandatory = True, + doc = "The wheel map where values are python versions", + ), + "_template": attr.label(default = ":requirements.bzl.tmpl"), + }, + doc = """A rule for creating bzlmod hub repo for PyPI packages. PRIVATE USE ONLY.""", + implementation = _impl, +) diff --git a/python/extensions/private/requirements.bzl.tmpl b/python/extensions/private/requirements.bzl.tmpl new file mode 100644 index 0000000000..4243990b2b --- /dev/null +++ b/python/extensions/private/requirements.bzl.tmpl @@ -0,0 +1,52 @@ +"""Starlark representation of locked requirements. + +@generated by rules_python pip.parse extension. + +This file is different from the other bzlmod template +because we do not support entry_point yet. +""" + +load("@@%%RULES_PYTHON%%//python/pip_install:entry_point.bzl", _entry_point = "entry_point") + +all_requirements = %%ALL_REQUIREMENTS%% + +all_whl_requirements = %%ALL_WHL_REQUIREMENTS%% + +all_data_requirements = %%ALL_DATA_REQUIREMENTS%% + +_default_py_version = %%DEFAULT_PY_VERSION%% +_packages = %%PACKAGE_AVAILABILITY%% + +def _clean_name(name): + return name.replace("-", "_").replace(".", "_").lower() + +def requirement(name): + return "%%MACRO_TMPL%%".format(_clean_name(name), "pkg") + +def whl_requirement(name): + return "%%MACRO_TMPL%%".format(_clean_name(name), "whl") + +def data_requirement(name): + return "%%MACRO_TMPL%%".format(_clean_name(name), "data") + +def dist_info_requirement(name): + return "%%MACRO_TMPL%%".format(_clean_name(name), "dist_info") + +def entry_point(pkg, script = None): + """Returns a select() expression to locate the version-specific entry point. + """ + # TODO: not implemented + # selects = _entry_point( + # tmpl = "@@%%NAME%%//{pkg}/bin_py{version_label}:{script}", + # pkg = _clean_name(pkg), + # script = script, + # packages = _packages, + # default_version = _default_py_version, + # ) + # if selects == None: + # fail("Package '{}' does not exist, select one from: {}".format(pkg, _packages.keys())) + + # # NOTE: We return a select() expression instead of an alias to such an expression + # # to avoid having to eagerly load all versions of the wheel. See + # # https://github.com/bazelbuild/rules_python/issues/1262 for discussion. + # return select(selects) diff --git a/python/pip_install/entry_point.bzl b/python/pip_install/entry_point.bzl new file mode 100644 index 0000000000..71bc96e68b --- /dev/null +++ b/python/pip_install/entry_point.bzl @@ -0,0 +1,69 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""entry_point macro implementation for bzlmod. PRIVATE USE ONLY. + +NOTE(2023-07-11): We cannot set the visibility of this utility function, because the hub +repo needs to be able to access this. +""" + +load("//python/private:version_label.bzl", "version_label") + +def entry_point(*, pkg, packages, default_version, tmpl, script = None): + """Return an entry_point script dictionary for a select statement. + + PRIVATE USE ONLY. + + Args: + pkg: the PyPI package name (e.g. "pylint"). + script: the script name to use (e.g. "epylint"), defaults to the `pkg` arg. + packages: the mapping of PyPI packages to python versions that are supported. + default_version: the default Python version. + tmpl: the template that will be interpolated by this function. The + following keys are going to be replaced: 'version_label', 'pkg' and + 'script'. + + Returns: + A dict that can be used in select statement or None if the pkg is not + in the supplied packages dictionary. + """ + if not script: + script = pkg + + if pkg not in packages: + # This is an error case, the caller should execute 'fail' and we are not doing it because + # we want easier testability. + return None + + selects = {} + default = "" + for full_version in packages[pkg]: + # Label() is called to evaluate this in the context of rules_python, not the pip repo + condition = str(Label("//python/config_settings:is_python_{}".format(full_version))) + + entry_point = tmpl.format( + version_label = version_label(full_version), + pkg = pkg, + script = script, + ) + + if full_version == default_version: + default = entry_point + else: + selects[condition] = entry_point + + if default: + selects["//conditions:default"] = default + + return selects diff --git a/python/pip_install/pip_hub_repository_requirements_bzlmod.bzl.tmpl b/python/pip_install/pip_hub_repository_requirements_bzlmod.bzl.tmpl deleted file mode 100644 index 4a3d512ae7..0000000000 --- a/python/pip_install/pip_hub_repository_requirements_bzlmod.bzl.tmpl +++ /dev/null @@ -1,35 +0,0 @@ -"""Starlark representation of locked requirements. - -@generated by rules_python pip_parse repository rule -from %%REQUIREMENTS_LOCK%%. - -This file is different from the other bzlmod template -because we do not support entry_point yet. -""" - -all_requirements = %%ALL_REQUIREMENTS%% - -all_whl_requirements = %%ALL_WHL_REQUIREMENTS%% - -all_data_requirements = %%ALL_DATA_REQUIREMENTS%% - -def _clean_name(name): - return name.replace("-", "_").replace(".", "_").lower() - -def requirement(name): - return "%%MACRO_TMPL%%".format(_clean_name(name), "pkg") - -def whl_requirement(name): - return "%%MACRO_TMPL%%".format(_clean_name(name), "whl") - -def data_requirement(name): - return "%%MACRO_TMPL%%".format(_clean_name(name), "data") - -def dist_info_requirement(name): - return "%%MACRO_TMPL%%".format(_clean_name(name), "dist_info") - -def entry_point(pkg, script = None): - """entry_point returns the target of the canonical label of the package entrypoints. - """ - # TODO: https://github.com/bazelbuild/rules_python/issues/1262 - print("not implemented") diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl index 0e80542747..c0a15581c8 100644 --- a/python/pip_install/pip_repository.bzl +++ b/python/pip_install/pip_repository.bzl @@ -269,108 +269,6 @@ A requirements_lock attribute must be specified, or a platform-specific lockfile """) return requirements_txt -def _create_pip_repository_bzlmod(rctx, bzl_packages, requirements): - repo_name = rctx.attr.repo_name - build_contents = _BUILD_FILE_CONTENTS - aliases = render_pkg_aliases(repo_name = repo_name, bzl_packages = bzl_packages) - for path, contents in aliases.items(): - rctx.file(path, contents) - - # NOTE: we are using the canonical name with the double '@' in order to - # always uniquely identify a repository, as the labels are being passed as - # a string and the resolution of the label happens at the call-site of the - # `requirement`, et al. macros. - macro_tmpl = "@@{name}//{{}}:{{}}".format(name = rctx.attr.name) - - rctx.file("BUILD.bazel", build_contents) - rctx.template("requirements.bzl", rctx.attr._template, substitutions = { - "%%ALL_DATA_REQUIREMENTS%%": _format_repr_list([ - macro_tmpl.format(p, "data") - for p in bzl_packages - ]), - "%%ALL_REQUIREMENTS%%": _format_repr_list([ - macro_tmpl.format(p, p) - for p in bzl_packages - ]), - "%%ALL_WHL_REQUIREMENTS%%": _format_repr_list([ - macro_tmpl.format(p, "whl") - for p in bzl_packages - ]), - "%%MACRO_TMPL%%": macro_tmpl, - "%%NAME%%": rctx.attr.name, - "%%REQUIREMENTS_LOCK%%": requirements, - }) - -def _pip_hub_repository_bzlmod_impl(rctx): - bzl_packages = rctx.attr.whl_library_alias_names - _create_pip_repository_bzlmod(rctx, bzl_packages, "") - -pip_hub_repository_bzlmod_attrs = { - "repo_name": attr.string( - mandatory = True, - doc = "The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name.", - ), - "whl_library_alias_names": attr.string_list( - mandatory = True, - doc = "The list of whl alias that we use to build aliases and the whl names", - ), - "_template": attr.label( - default = ":pip_hub_repository_requirements_bzlmod.bzl.tmpl", - ), -} - -pip_hub_repository_bzlmod = repository_rule( - attrs = pip_hub_repository_bzlmod_attrs, - doc = """A rule for bzlmod mulitple pip repository creation. PRIVATE USE ONLY.""", - implementation = _pip_hub_repository_bzlmod_impl, -) - -def _pip_repository_bzlmod_impl(rctx): - requirements_txt = locked_requirements_label(rctx, rctx.attr) - content = rctx.read(requirements_txt) - parsed_requirements_txt = parse_requirements(content) - - packages = [(normalize_name(name), requirement) for name, requirement in parsed_requirements_txt.requirements] - - bzl_packages = sorted([name for name, _ in packages]) - _create_pip_repository_bzlmod(rctx, bzl_packages, str(requirements_txt)) - -pip_repository_bzlmod_attrs = { - "repo_name": attr.string( - mandatory = True, - doc = "The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name", - ), - "requirements_darwin": attr.label( - allow_single_file = True, - doc = "Override the requirements_lock attribute when the host platform is Mac OS", - ), - "requirements_linux": attr.label( - allow_single_file = True, - doc = "Override the requirements_lock attribute when the host platform is Linux", - ), - "requirements_lock": attr.label( - allow_single_file = True, - doc = """ -A fully resolved 'requirements.txt' pip requirement file containing the transitive set of your dependencies. If this file is passed instead -of 'requirements' no resolve will take place and pip_repository will create individual repositories for each of your dependencies so that -wheels are fetched/built only for the targets specified by 'build/run/test'. -""", - ), - "requirements_windows": attr.label( - allow_single_file = True, - doc = "Override the requirements_lock attribute when the host platform is Windows", - ), - "_template": attr.label( - default = ":pip_repository_requirements_bzlmod.bzl.tmpl", - ), -} - -pip_repository_bzlmod = repository_rule( - attrs = pip_repository_bzlmod_attrs, - doc = """A rule for bzlmod pip_repository creation. Intended for private use only.""", - implementation = _pip_repository_bzlmod_impl, -) - def _pip_repository_impl(rctx): requirements_txt = locked_requirements_label(rctx, rctx.attr) content = rctx.read(requirements_txt) diff --git a/python/pip_install/pip_repository_requirements_bzlmod.bzl.tmpl b/python/pip_install/pip_repository_requirements_bzlmod.bzl.tmpl deleted file mode 100644 index 2df60b0b52..0000000000 --- a/python/pip_install/pip_repository_requirements_bzlmod.bzl.tmpl +++ /dev/null @@ -1,33 +0,0 @@ -"""Starlark representation of locked requirements. - -@generated by rules_python pip_parse repository rule -from %%REQUIREMENTS_LOCK%%. -""" - -all_requirements = %%ALL_REQUIREMENTS%% - -all_whl_requirements = %%ALL_WHL_REQUIREMENTS%% - -all_data_requirements = %%ALL_DATA_REQUIREMENTS%% - -def _clean_name(name): - return name.replace("-", "_").replace(".", "_").lower() - -def requirement(name): - return "%%MACRO_TMPL%%".format(_clean_name(name), "pkg") - -def whl_requirement(name): - return "%%MACRO_TMPL%%".format(_clean_name(name), "whl") - -def data_requirement(name): - return "%%MACRO_TMPL%%".format(_clean_name(name), "data") - -def dist_info_requirement(name): - return "%%MACRO_TMPL%%".format(_clean_name(name), "dist_info") - -def entry_point(pkg, script = None): - """entry_point returns the target of the canonical label of the package entrypoints. - """ - if not script: - script = pkg - return "@@%%NAME%%_{}//:rules_python_wheel_entry_point_{}".format(_clean_name(pkg), script) diff --git a/tests/pip_hub_repository/entry_point/BUILD.bazel b/tests/pip_hub_repository/entry_point/BUILD.bazel new file mode 100644 index 0000000000..a8a441c428 --- /dev/null +++ b/tests/pip_hub_repository/entry_point/BUILD.bazel @@ -0,0 +1,17 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load(":entry_point_test.bzl", "entry_point_test_suite") + +entry_point_test_suite(name = "entry_point_tests") diff --git a/tests/pip_hub_repository/entry_point/entry_point_test.bzl b/tests/pip_hub_repository/entry_point/entry_point_test.bzl new file mode 100644 index 0000000000..87f540dec6 --- /dev/null +++ b/tests/pip_hub_repository/entry_point/entry_point_test.bzl @@ -0,0 +1,94 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"" + +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python/pip_install:entry_point.bzl", "entry_point") +load("//tests:test_env.bzl", "test_env") + +def _label(label_str): + # Bazel 5.4 is stringifying the labels differently. + # + # This function can be removed when the minimum supported version is 6+ + if test_env.is_bazel_6_or_higher(): + return label_str + else: + return label_str.lstrip("@") + +_tests = [] + +def _test_unknown_entry_point_returns_none(env): + actual = entry_point( + pkg = "foo", + packages = {}, + tmpl = "dummy", + default_version = "dummy", + ) + + # None is returned if the package is not found, we will fail in the place + # where this is called. + want = None + + # FIXME @aignas 2023-07-11: currently the rules_testing does not accept a + # None to the dict subject. + env.expect.that_int(actual).equals(want) + +_tests.append(_test_unknown_entry_point_returns_none) + +def _test_constraint_values_are_set_correctly(env): + actual = entry_point( + pkg = "foo", + packages = {"foo": ["1.2.0", "1.2.3", "1.2.5"]}, + tmpl = "dummy", + default_version = "1.2.3", + ) + + # Python constraints are set correctly + want = { + # NOTE @aignas 2023-07-07: label will contain the rules_python + # when the macro is used outside rules_python + _label("@//python/config_settings:is_python_1.2.0"): "dummy", + _label("@//python/config_settings:is_python_1.2.5"): "dummy", + "//conditions:default": "dummy", + } + env.expect.that_dict(actual).contains_exactly(want) + +_tests.append(_test_constraint_values_are_set_correctly) + +def _test_template_is_interpolated_correctly(env): + actual = entry_point( + pkg = "foo", + script = "bar", + packages = {"foo": ["1.3.3", "1.2.5"]}, + tmpl = "pkg={pkg} script={script} version={version_label}", + default_version = "1.2.5", + ) + + # Template is interpolated correctly + want = { + _label("@//python/config_settings:is_python_1.3.3"): "pkg=foo script=bar version=13", + "//conditions:default": "pkg=foo script=bar version=12", + } + env.expect.that_dict(actual).contains_exactly(want) + +_tests.append(_test_template_is_interpolated_correctly) + +def entry_point_test_suite(name): + """Create the test suite. + + Args: + name: the name of the test suite + """ + test_suite(name = name, basic_tests = _tests) From 9d28b0f6340ed3fc8978c072961ea4ae17861047 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius Date: Mon, 17 Jul 2023 00:24:11 +0900 Subject: [PATCH 8/9] docs: regenerate docs --- docs/pip_repository.md | 46 ------------------------------------------ 1 file changed, 46 deletions(-) diff --git a/docs/pip_repository.md b/docs/pip_repository.md index 853605276f..f58d90c396 100644 --- a/docs/pip_repository.md +++ b/docs/pip_repository.md @@ -2,27 +2,6 @@ - - -## pip_hub_repository_bzlmod - -
-pip_hub_repository_bzlmod(name, repo_mapping, repo_name, whl_library_alias_names)
-
- -A rule for bzlmod mulitple pip repository creation. PRIVATE USE ONLY. - -**ATTRIBUTES** - - -| Name | Description | Type | Mandatory | Default | -| :------------- | :------------- | :------------- | :------------- | :------------- | -| name | A unique name for this repository. | Name | required | | -| repo_mapping | A dictionary from local repository name to global repository name. This allows controls over workspace dependency resolution for dependencies of this repository.<p>For example, an entry "@foo": "@bar" declares that, for any time this repository depends on @foo (such as a dependency on @foo//some:target, it should actually resolve that dependency within globally-declared @bar (@bar//some:target). | Dictionary: String -> String | required | | -| repo_name | The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name. | String | required | | -| whl_library_alias_names | The list of whl alias that we use to build aliases and the whl names | List of strings | required | | - - ## pip_repository @@ -101,31 +80,6 @@ py_binary( | timeout | Timeout (in seconds) on the rule's execution duration. | Integer | optional | 600 | - - -## pip_repository_bzlmod - -
-pip_repository_bzlmod(name, repo_mapping, repo_name, requirements_darwin, requirements_linux,
-                      requirements_lock, requirements_windows)
-
- -A rule for bzlmod pip_repository creation. Intended for private use only. - -**ATTRIBUTES** - - -| Name | Description | Type | Mandatory | Default | -| :------------- | :------------- | :------------- | :------------- | :------------- | -| name | A unique name for this repository. | Name | required | | -| repo_mapping | A dictionary from local repository name to global repository name. This allows controls over workspace dependency resolution for dependencies of this repository.<p>For example, an entry "@foo": "@bar" declares that, for any time this repository depends on @foo (such as a dependency on @foo//some:target, it should actually resolve that dependency within globally-declared @bar (@bar//some:target). | Dictionary: String -> String | required | | -| repo_name | The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name | String | required | | -| requirements_darwin | Override the requirements_lock attribute when the host platform is Mac OS | Label | optional | None | -| requirements_linux | Override the requirements_lock attribute when the host platform is Linux | Label | optional | None | -| requirements_lock | A fully resolved 'requirements.txt' pip requirement file containing the transitive set of your dependencies. If this file is passed instead of 'requirements' no resolve will take place and pip_repository will create individual repositories for each of your dependencies so that wheels are fetched/built only for the targets specified by 'build/run/test'. | Label | optional | None | -| requirements_windows | Override the requirements_lock attribute when the host platform is Windows | Label | optional | None | - - ## whl_library From 1b88ae3fc4a802002c5c3c8fce04c54f552dbb7a Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius Date: Tue, 25 Jul 2023 18:19:13 +0900 Subject: [PATCH 9/9] A different external API for entrypoints without a macro --- examples/bzlmod/MODULE.bazel | 6 ++ examples/bzlmod/entry_point/BUILD.bazel | 3 +- python/extensions/pip.bzl | 30 +++++-- .../extensions/private/pip_hub_repository.bzl | 8 ++ python/private/render_pkg_aliases.bzl | 86 ++++++++++--------- 5 files changed, 86 insertions(+), 47 deletions(-) diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel index 7c5083c82a..5d6e98f2f9 100644 --- a/examples/bzlmod/MODULE.bazel +++ b/examples/bzlmod/MODULE.bazel @@ -94,6 +94,9 @@ use_repo(pip, "whl_mods_hub") # Because we do not have a python_version defined here # pip.parse uses the python toolchain that is set as default. pip.parse( + entry_points = { + "yamllint": ["yamllint"], + }, hub_name = "pip", requirements_lock = "//:requirements_lock_3_9.txt", requirements_windows = "//:requirements_windows_3_9.txt", @@ -106,6 +109,9 @@ pip.parse( }, ) pip.parse( + entry_points = { + "yamllint": ["yamllint"], + }, hub_name = "pip", python_version = "3.10", requirements_lock = "//:requirements_lock_3_10.txt", diff --git a/examples/bzlmod/entry_point/BUILD.bazel b/examples/bzlmod/entry_point/BUILD.bazel index dfc02b00a0..8e3f3338e4 100644 --- a/examples/bzlmod/entry_point/BUILD.bazel +++ b/examples/bzlmod/entry_point/BUILD.bazel @@ -1,9 +1,8 @@ -load("@pip//:requirements.bzl", "entry_point") load("@rules_python//python:defs.bzl", "py_test") alias( name = "yamllint", - actual = entry_point("yamllint"), + actual = "@pip//yamllint/bin:yamllint", ) py_test( diff --git a/python/extensions/pip.bzl b/python/extensions/pip.bzl index 699fb751f6..2dc42420ac 100644 --- a/python/extensions/pip.bzl +++ b/python/extensions/pip.bzl @@ -111,7 +111,10 @@ def _create_versioned_pip_and_whl_repos(module_ctx, pip_attr, whl_map): extra_pip_args = pip_attr.extra_pip_args + parse_result.options if hub_name not in whl_map: - whl_map[hub_name] = {} + whl_map[hub_name] = struct( + wheels = {}, + entry_points = {}, + ) whl_modifications = {} if pip_attr.whl_modifications != None: @@ -143,10 +146,20 @@ def _create_versioned_pip_and_whl_repos(module_ctx, pip_attr, whl_map): environment = pip_attr.environment, ) - if whl_name not in whl_map[hub_name]: - whl_map[hub_name][whl_name] = [] + if whl_name not in whl_map[hub_name].wheels: + whl_map[hub_name].wheels[whl_name] = [] + + whl_map[hub_name].wheels[whl_name].append(full_version(pip_attr.python_version)) + + for whl_name, scripts in pip_attr.entry_points.items(): + if whl_name not in whl_map[hub_name].entry_points: + whl_map[hub_name].entry_points[whl_name] = {} - whl_map[hub_name][whl_name].append(full_version(pip_attr.python_version)) + for script in scripts: + if script not in whl_map[hub_name].entry_points[whl_name]: + whl_map[hub_name].entry_points[whl_name][script] = [] + + whl_map[hub_name].entry_points[whl_name][script].append(full_version(pip_attr.python_version)) def _pip_impl(module_ctx): """Implementation of a class tag that creates the pip hub(s) and corresponding pip spoke, alias and whl repositories. @@ -288,12 +301,19 @@ def _pip_impl(module_ctx): pip_hub_repository( name = hub_name, repo_name = hub_name, - whl_map = whl_map, + whl_map = whl_map.wheels, + whl_entry_points = { + whl_name: json.encode(values) + for whl_name, values in whl_map.entry_points.items() + }, default_version = full_version(DEFAULT_PYTHON_VERSION), ) def _pip_parse_ext_attrs(): attrs = dict({ + "entry_points": attr.string_list_dict( + doc = "TODO", + ), "hub_name": attr.string( mandatory = True, doc = """ diff --git a/python/extensions/private/pip_hub_repository.bzl b/python/extensions/private/pip_hub_repository.bzl index b73c003500..9e4f7551bd 100644 --- a/python/extensions/private/pip_hub_repository.bzl +++ b/python/extensions/private/pip_hub_repository.bzl @@ -35,6 +35,10 @@ def _impl(rctx): aliases = render_pkg_aliases( repo_name = repo_name, whl_map = rctx.attr.whl_map, + whl_entry_points = { + whl_name: json.decode(values) + for whl_name, values in rctx.attr.whl_entry_points.items() + }, default_version = rctx.attr.default_version, rules_python = rctx.attr._template.workspace_name, ) @@ -84,6 +88,10 @@ setting.""", mandatory = True, doc = "The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name.", ), + "whl_entry_points": attr.string_dict( + mandatory = False, + doc = "The entry points that we will create aliases for.", + ), "whl_map": attr.string_list_dict( mandatory = True, doc = "The wheel map where values are python versions", diff --git a/python/private/render_pkg_aliases.bzl b/python/private/render_pkg_aliases.bzl index 28042e0507..ca66b06719 100644 --- a/python/private/render_pkg_aliases.bzl +++ b/python/private/render_pkg_aliases.bzl @@ -37,9 +37,9 @@ def _render_alias( repo_name, dep, target, - default_version, versions, - rules_python): + rules_python, + default_version = None): """Render an alias for common targets If the versions is passed, then the `rules_python` must be passed as well and @@ -71,13 +71,14 @@ def _render_alias( ) selects[condition] = actual - default_actual = "@{repo_name}_{version}_{dep}//:{target}".format( - repo_name = repo_name, - version = version_label(default_version), - dep = dep, - target = target, - ) - selects["//conditions:default"] = default_actual + if default_version: + default_actual = "@{repo_name}_{version}_{dep}//:{target}".format( + repo_name = repo_name, + version = version_label(default_version), + dep = dep, + target = target, + ) + selects["//conditions:default"] = default_actual return _SELECT.format( name = name, @@ -89,22 +90,21 @@ def _render_alias( ), ) -def _render_entry_points(repo_name, dep): - return """\ -load("@{repo_name}_{dep}//:entry_points.bzl", "entry_points") - -[ - alias( - name = script, - actual = "@{repo_name}_{dep}//:" + target, - visibility = ["//visibility:public"], - ) - for script, target in entry_points.items() -] -""".format( - repo_name = repo_name, - dep = dep, - ) +def _render_entry_points(repo_name, dep, entry_points, default_version = None, rules_python = None, prefix = "rules_python_wheel_entry_point_"): + return "\n\n".join([ + """package(default_visibility = ["//visibility:public"])""", + ] + [ + _render_alias( + name = normalize_name(script), + repo_name = repo_name, + dep = dep, + target = prefix + normalize_name(script), + versions = versions, + default_version = default_version, + rules_python = rules_python, + ) + for script, versions in entry_points.items() + ]) def _render_common_aliases(repo_name, name, versions = None, default_version = None, rules_python = None): return "\n\n".join([ @@ -131,7 +131,14 @@ def _render_common_aliases(repo_name, name, versions = None, default_version = N for target in ["pkg", "whl", "data", "dist_info"] ]) -def render_pkg_aliases(*, repo_name, bzl_packages = None, whl_map = None, rules_python = None, default_version = None): +def render_pkg_aliases( + *, + repo_name, + bzl_packages = None, + whl_map = None, + whl_entry_points = None, + rules_python = None, + default_version = None): """Create alias declarations for each PyPI package. The aliases should be appended to the pip_repository BUILD.bazel file. These aliases @@ -155,8 +162,14 @@ def render_pkg_aliases(*, repo_name, bzl_packages = None, whl_map = None, rules_ contents = {} for name in bzl_packages: versions = None + entry_points = None + if whl_map != None: versions = whl_map[name] + + if whl_entry_points != None: + entry_points = whl_entry_points.get(name, {}) + name = normalize_name(name) filename = "{}/BUILD.bazel".format(name) @@ -168,22 +181,15 @@ def render_pkg_aliases(*, repo_name, bzl_packages = None, whl_map = None, rules_ default_version = default_version, ).strip() - if versions == None: - # NOTE: this code would be normally executed in the non-bzlmod - # scenario, where we are requesting friendly aliases to be - # generated. In that case, we will not be creating aliases for - # entry_points to leave the behaviour unchanged from previous - # rules_python versions. - continue - - # NOTE @aignas 2023-07-07: we are not creating aliases using a select - # and the version specific aliases because we would need to fetch the - # package for all versions in order to construct the said select. - for version in versions: - filename = "{}/bin_py{}/BUILD.bazel".format(name, version_label(version)) + if entry_points: + # Generate aliases where we have the select statement + filename = "{}/bin/BUILD.bazel".format(name) contents[filename] = _render_entry_points( - repo_name = "{}_{}".format(repo_name, version_label(version)), + repo_name = repo_name, dep = name, + rules_python = rules_python, + default_version = default_version, + entry_points = entry_points, ).strip() return contents pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy