Skip to content

Commit dce5120

Browse files
aignasaignas
authored
refactor: reimplement writing namespace pkgs in Starlark (bazel-contrib#2882)
With this PR I would like to facilitate the implementation of the venv layouts because we can in theory take the `srcs` and the `data` within the `py_library` and then use the `expand_template` to write the extra Python files if the namespace_pkgs flag is enabled. The old Python code has been removed and the extra generated files are written out with `bazel_skylib` `copy_file`. The implicit `namespace_pkg` init files are included to `py_library` if the `site-packages` config flag is set to false and I think this may help with continuing the implementation, but it currently is still not working as expected (see comment). Work towards bazel-contrib#2156
1 parent b40d96a commit dce5120

File tree

14 files changed

+311
-260
lines changed

14 files changed

+311
-260
lines changed

‎python/config_settings/BUILD.bazel

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -217,6 +217,15 @@ string_flag(
217217
visibility = ["//visibility:public"],
218218
)
219219

220+
config_setting(
221+
name = "is_venvs_site_packages",
222+
flag_values = {
223+
":venvs_site_packages": VenvsSitePackages.YES,
224+
},
225+
# NOTE: Only public because it is used in whl_library repos.
226+
visibility = ["//visibility:public"],
227+
)
228+
220229
define_pypi_internal_flags(
221230
name = "define_pypi_internal_flags",
222231
)

‎python/private/pypi/BUILD.bazel

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,11 @@ package(default_visibility = ["//:__subpackages__"])
1818

1919
licenses(["notice"])
2020

21+
exports_files(
22+
srcs = ["namespace_pkg_tmpl.py"],
23+
visibility = ["//visibility:public"],
24+
)
25+
2126
filegroup(
2227
name = "distribution",
2328
srcs = glob(

‎python/private/pypi/namespace_pkg_tmpl.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# __path__ manipulation added by bazel-contrib/rules_python to support namespace pkgs.
2+
__path__ = __import__("pkgutil").extend_path(__path__, __name__)

‎python/private/pypi/namespace_pkgs.bzl

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
"""Utilities to get where we should write namespace pkg paths."""
2+
3+
load("@bazel_skylib//rules:copy_file.bzl", "copy_file")
4+
5+
_ext = struct(
6+
py = ".py",
7+
pyd = ".pyd",
8+
so = ".so",
9+
pyc = ".pyc",
10+
)
11+
12+
_TEMPLATE = Label("//python/private/pypi:namespace_pkg_tmpl.py")
13+
14+
def _add_all(dirname, dirs):
15+
dir_path = "."
16+
for dir_name in dirname.split("/"):
17+
dir_path = "{}/{}".format(dir_path, dir_name)
18+
dirs[dir_path[2:]] = None
19+
20+
def get_files(*, srcs, ignored_dirnames = [], root = None):
21+
"""Get the list of filenames to write the namespace pkg files.
22+
23+
Args:
24+
srcs: {type}`src` a list of files to be passed to {bzl:obj}`py_library`
25+
as `srcs` and `data`. This is usually a result of a {obj}`glob`.
26+
ignored_dirnames: {type}`str` a list of patterns to ignore.
27+
root: {type}`str` the prefix to use as the root.
28+
29+
Returns:
30+
{type}`src` a list of paths to write the namespace pkg `__init__.py` file.
31+
"""
32+
dirs = {}
33+
ignored = {i: None for i in ignored_dirnames}
34+
35+
if root:
36+
_add_all(root, ignored)
37+
38+
for file in srcs:
39+
dirname, _, filename = file.rpartition("/")
40+
41+
if filename == "__init__.py":
42+
ignored[dirname] = None
43+
dirname, _, _ = dirname.rpartition("/")
44+
elif filename.endswith(_ext.py):
45+
pass
46+
elif filename.endswith(_ext.pyc):
47+
pass
48+
elif filename.endswith(_ext.pyd):
49+
pass
50+
elif filename.endswith(_ext.so):
51+
pass
52+
else:
53+
continue
54+
55+
if dirname in dirs or not dirname:
56+
continue
57+
58+
_add_all(dirname, dirs)
59+
60+
return sorted([d for d in dirs if d not in ignored])
61+
62+
def create_inits(**kwargs):
63+
"""Create init files and return the list to be included `py_library` srcs.
64+
65+
Args:
66+
**kwargs: passed to {obj}`get_files`.
67+
68+
Returns:
69+
{type}`list[str]` to be included as part of `py_library`.
70+
"""
71+
srcs = []
72+
for out in get_files(**kwargs):
73+
src = "{}/__init__.py".format(out)
74+
srcs.append(srcs)
75+
76+
copy_file(
77+
name = "_cp_{}_namespace".format(out),
78+
src = _TEMPLATE,
79+
out = src,
80+
**kwargs
81+
)
82+
83+
return srcs

‎python/private/pypi/whl_installer/arguments.py

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -57,11 +57,6 @@ def parser(**kwargs: Any) -> argparse.ArgumentParser:
5757
action="store",
5858
help="Additional data exclusion parameters to add to the pip packages BUILD file.",
5959
)
60-
parser.add_argument(
61-
"--enable_implicit_namespace_pkgs",
62-
action="store_true",
63-
help="Disables conversion of implicit namespace packages into pkg-util style packages.",
64-
)
6560
parser.add_argument(
6661
"--environment",
6762
action="store",

‎python/private/pypi/whl_installer/wheel_installer.py

Lines changed: 1 addition & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727

2828
from pip._vendor.packaging.utils import canonicalize_name
2929

30-
from python.private.pypi.whl_installer import arguments, namespace_pkgs, wheel
30+
from python.private.pypi.whl_installer import arguments, wheel
3131

3232

3333
def _configure_reproducible_wheels() -> None:
@@ -77,35 +77,10 @@ def _parse_requirement_for_extra(
7777
return None, None
7878

7979

80-
def _setup_namespace_pkg_compatibility(wheel_dir: str) -> None:
81-
"""Converts native namespace packages to pkgutil-style packages
82-
83-
Namespace packages can be created in one of three ways. They are detailed here:
84-
https://packaging.python.org/guides/packaging-namespace-packages/#creating-a-namespace-package
85-
86-
'pkgutil-style namespace packages' (2) and 'pkg_resources-style namespace packages' (3) works in Bazel, but
87-
'native namespace packages' (1) do not.
88-
89-
We ensure compatibility with Bazel of method 1 by converting them into method 2.
90-
91-
Args:
92-
wheel_dir: the directory of the wheel to convert
93-
"""
94-
95-
namespace_pkg_dirs = namespace_pkgs.implicit_namespace_packages(
96-
wheel_dir,
97-
ignored_dirnames=["%s/bin" % wheel_dir],
98-
)
99-
100-
for ns_pkg_dir in namespace_pkg_dirs:
101-
namespace_pkgs.add_pkgutil_style_namespace_pkg_init(ns_pkg_dir)
102-
103-
10480
def _extract_wheel(
10581
wheel_file: str,
10682
extras: Dict[str, Set[str]],
10783
enable_pipstar: bool,
108-
enable_implicit_namespace_pkgs: bool,
10984
platforms: List[wheel.Platform],
11085
installation_dir: Path = Path("."),
11186
) -> None:
@@ -116,15 +91,11 @@ def _extract_wheel(
11691
installation_dir: the destination directory for installation of the wheel.
11792
extras: a list of extras to add as dependencies for the installed wheel
11893
enable_pipstar: if true, turns off certain operations.
119-
enable_implicit_namespace_pkgs: if true, disables conversion of implicit namespace packages and will unzip as-is
12094
"""
12195

12296
whl = wheel.Wheel(wheel_file)
12397
whl.unzip(installation_dir)
12498

125-
if not enable_implicit_namespace_pkgs:
126-
_setup_namespace_pkg_compatibility(installation_dir)
127-
12899
metadata = {
129100
"entry_points": [
130101
{
@@ -168,7 +139,6 @@ def main() -> None:
168139
wheel_file=whl,
169140
extras=extras,
170141
enable_pipstar=args.enable_pipstar,
171-
enable_implicit_namespace_pkgs=args.enable_implicit_namespace_pkgs,
172142
platforms=arguments.get_platforms(args),
173143
)
174144
return

‎python/private/pypi/whl_library.bzl

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -173,9 +173,6 @@ def _parse_optional_attrs(rctx, args, extra_pip_args = None):
173173
json.encode(struct(arg = rctx.attr.pip_data_exclude)),
174174
]
175175

176-
if rctx.attr.enable_implicit_namespace_pkgs:
177-
args.append("--enable_implicit_namespace_pkgs")
178-
179176
env = {}
180177
if rctx.attr.environment != None:
181178
for key, value in rctx.attr.environment.items():
@@ -389,6 +386,8 @@ def _whl_library_impl(rctx):
389386
metadata_name = metadata.name,
390387
metadata_version = metadata.version,
391388
requires_dist = metadata.requires_dist,
389+
# TODO @aignas 2025-05-17: maybe have a build flag for this instead
390+
enable_implicit_namespace_pkgs = rctx.attr.enable_implicit_namespace_pkgs,
392391
# TODO @aignas 2025-04-14: load through the hub:
393392
annotation = None if not rctx.attr.annotation else struct(**json.decode(rctx.read(rctx.attr.annotation))),
394393
data_exclude = rctx.attr.pip_data_exclude,
@@ -457,6 +456,8 @@ def _whl_library_impl(rctx):
457456
name = whl_path.basename,
458457
dep_template = rctx.attr.dep_template or "@{}{{name}}//:{{target}}".format(rctx.attr.repo_prefix),
459458
entry_points = entry_points,
459+
# TODO @aignas 2025-05-17: maybe have a build flag for this instead
460+
enable_implicit_namespace_pkgs = rctx.attr.enable_implicit_namespace_pkgs,
460461
# TODO @aignas 2025-04-14: load through the hub:
461462
dependencies = metadata["deps"],
462463
dependencies_by_platform = metadata["deps_by_platform"],
@@ -580,7 +581,6 @@ attr makes `extra_pip_args` and `download_only` ignored.""",
580581
Label("//python/private/pypi/whl_installer:wheel.py"),
581582
Label("//python/private/pypi/whl_installer:wheel_installer.py"),
582583
Label("//python/private/pypi/whl_installer:arguments.py"),
583-
Label("//python/private/pypi/whl_installer:namespace_pkgs.py"),
584584
] + record_files.values(),
585585
),
586586
"_rule_name": attr.string(default = "whl_library"),

‎python/private/pypi/whl_library_targets.bzl

Lines changed: 35 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ load(
3030
"WHEEL_FILE_IMPL_LABEL",
3131
"WHEEL_FILE_PUBLIC_LABEL",
3232
)
33+
load(":namespace_pkgs.bzl", "create_inits")
3334
load(":pep508_deps.bzl", "deps")
3435

3536
def whl_library_targets_from_requires(
@@ -113,6 +114,7 @@ def whl_library_targets(
113114
copy_executables = {},
114115
entry_points = {},
115116
native = native,
117+
enable_implicit_namespace_pkgs = False,
116118
rules = struct(
117119
copy_file = copy_file,
118120
py_binary = py_binary,
@@ -153,6 +155,8 @@ def whl_library_targets(
153155
data: {type}`list[str]` A list of labels to include as part of the `data` attribute in `py_library`.
154156
entry_points: {type}`dict[str, str]` The mapping between the script
155157
name and the python file to use. DEPRECATED.
158+
enable_implicit_namespace_pkgs: {type}`boolean` generate __init__.py
159+
files for namespace pkgs.
156160
native: {type}`native` The native struct for overriding in tests.
157161
rules: {type}`struct` A struct with references to rules for creating targets.
158162
"""
@@ -293,6 +297,14 @@ def whl_library_targets(
293297
)
294298

295299
if hasattr(rules, "py_library"):
300+
srcs = native.glob(
301+
["site-packages/**/*.py"],
302+
exclude = srcs_exclude,
303+
# Empty sources are allowed to support wheels that don't have any
304+
# pure-Python code, e.g. pymssql, which is written in Cython.
305+
allow_empty = True,
306+
)
307+
296308
# NOTE: pyi files should probably be excluded because they're carried
297309
# by the pyi_srcs attribute. However, historical behavior included
298310
# them in data and some tools currently rely on that.
@@ -309,23 +321,31 @@ def whl_library_targets(
309321
if item not in _data_exclude:
310322
_data_exclude.append(item)
311323

324+
data = data + native.glob(
325+
["site-packages/**/*"],
326+
exclude = _data_exclude,
327+
)
328+
329+
pyi_srcs = native.glob(
330+
["site-packages/**/*.pyi"],
331+
allow_empty = True,
332+
)
333+
334+
if enable_implicit_namespace_pkgs:
335+
srcs = srcs + getattr(native, "select", select)({
336+
Label("//python/config_settings:is_venvs_site_packages"): [],
337+
"//conditions:default": create_inits(
338+
srcs = srcs + data + pyi_srcs,
339+
ignore_dirnames = [], # If you need to ignore certain folders, you can patch rules_python here to do so.
340+
root = "site-packages",
341+
),
342+
})
343+
312344
rules.py_library(
313345
name = py_library_label,
314-
srcs = native.glob(
315-
["site-packages/**/*.py"],
316-
exclude = srcs_exclude,
317-
# Empty sources are allowed to support wheels that don't have any
318-
# pure-Python code, e.g. pymssql, which is written in Cython.
319-
allow_empty = True,
320-
),
321-
pyi_srcs = native.glob(
322-
["site-packages/**/*.pyi"],
323-
allow_empty = True,
324-
),
325-
data = data + native.glob(
326-
["site-packages/**/*"],
327-
exclude = _data_exclude,
328-
),
346+
srcs = srcs,
347+
pyi_srcs = pyi_srcs,
348+
data = data,
329349
# This makes this directory a top-level in the python import
330350
# search path for anything that depends on this.
331351
imports = ["site-packages"],

‎tests/pypi/namespace_pkgs/BUILD.bazel

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
load(":namespace_pkgs_tests.bzl", "namespace_pkgs_test_suite")
2+
3+
namespace_pkgs_test_suite(
4+
name = "namespace_pkgs_tests",
5+
)

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy