fix: symlink root-level python files to the venv (bazel-contrib#2908)

aignas · web-flow · commit 3464c14c36e5 · 2025-05-28T13:53:50.000Z
As found in bazel-contrib#2882 testing, packages like `typing-extensions` which have `.py` files at the root of the `site-packages` folder don't work and it seems that the comment about `rules_python` being too eager is only half-correct. Since `namespace_pkgs` are no longer there, we can just include all of the files and if there are collisions, they will be highlighted as build errors. Now the following works: ``` bazel build //docs --@rules_python//python/config_settings:venvs_site_packages=yes ``` Work towards bazel-contrib#2156
diff --git a/.bazelrc b/.bazelrc
@@ -4,8 +4,8 @@
 # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it)
 # To update these lines, execute
 # `bazel run @rules_bazel_integration_test//tools:update_deleted_packages`
-build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma
-query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma
+build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma,tests/modules/other/nspkg_single
+query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma,tests/modules/other/nspkg_single
 
 test --test_output=errors
 
diff --git a/python/private/py_library.bzl b/python/private/py_library.bzl
@@ -253,6 +253,7 @@ def _get_site_packages_symlinks(ctx):
 
     repo_runfiles_dirname = None
     dirs_with_init = {}  # dirname -> runfile path
+    site_packages_symlinks = []
     for src in ctx.files.srcs:
         if src.extension not in PYTHON_FILE_EXTENSIONS:
             continue
@@ -261,16 +262,19 @@ def _get_site_packages_symlinks(ctx):
             continue
         path = path.removeprefix(site_packages_root)
         dir_name, _, filename = path.rpartition("/")
-        if not dir_name:
-            # This would be e.g. `site-packages/__init__.py`, which isn't valid
-            # because it's not within a directory for an importable Python package.
-            # However, the pypi integration over-eagerly adds a pkgutil-style
-            # __init__.py file during the repo phase. Just ignore them for now.
-            continue
 
-        if filename.startswith("__init__."):
+        if dir_name and filename.startswith("__init__."):
             dirs_with_init[dir_name] = None
             repo_runfiles_dirname = runfiles_root_path(ctx, src.short_path).partition("/")[0]
+        elif not dir_name:
+            repo_runfiles_dirname = runfiles_root_path(ctx, src.short_path).partition("/")[0]
+
+            # This would be files that do not have directories and we just need to add
+            # direct symlinks to them as is:
+            site_packages_symlinks.append((
+                paths.join(repo_runfiles_dirname, site_packages_root, filename),
+                filename,
+            ))
 
     # Sort so that we encounter `foo` before `foo/bar`. This ensures we
     # see the top-most explicit package first.
@@ -286,7 +290,6 @@ def _get_site_packages_symlinks(ctx):
         if not is_sub_package:
             first_level_explicit_packages.append(d)
 
-    site_packages_symlinks = []
     for dirname in first_level_explicit_packages:
         site_packages_symlinks.append((
             paths.join(repo_runfiles_dirname, site_packages_root, dirname),
diff --git a/tests/modules/other/nspkg_single/BUILD.bazel b/tests/modules/other/nspkg_single/BUILD.bazel
@@ -0,0 +1,10 @@
+load("@rules_python//python:py_library.bzl", "py_library")
+
+package(default_visibility = ["//visibility:public"])
+
+py_library(
+    name = "nspkg_single",
+    srcs = glob(["site-packages/**/*.py"]),
+    experimental_venvs_site_packages = "@rules_python//python/config_settings:venvs_site_packages",
+    imports = [package_name() + "/site-packages"],
+)
diff --git a/tests/modules/other/nspkg_single/site-packages/__init__.py b/tests/modules/other/nspkg_single/site-packages/__init__.py
@@ -0,0 +1 @@
+# empty, will not be added to the site-packages dir
diff --git a/tests/modules/other/nspkg_single/site-packages/single_file.py b/tests/modules/other/nspkg_single/site-packages/single_file.py
@@ -0,0 +1,5 @@
+__all__ = [
+    "SOMETHING",
+]
+
+SOMETHING = "nothing"
diff --git a/tests/venv_site_packages_libs/BUILD.bazel b/tests/venv_site_packages_libs/BUILD.bazel
@@ -13,5 +13,6 @@ py_reconfig_test(
         "//tests/venv_site_packages_libs/nspkg_beta",
         "@other//nspkg_delta",
         "@other//nspkg_gamma",
+        "@other//nspkg_single",
     ],
 )
diff --git a/tests/venv_site_packages_libs/bin.py b/tests/venv_site_packages_libs/bin.py
@@ -26,6 +26,7 @@ def test_imported_from_venv(self):
         self.assert_imported_from_venv("nspkg.subnspkg.beta")
         self.assert_imported_from_venv("nspkg.subnspkg.gamma")
         self.assert_imported_from_venv("nspkg.subnspkg.delta")
+        self.assert_imported_from_venv("single_file")
 
 
 if __name__ == "__main__":
diff --git a/tests/venv_site_packages_libs/nspkg_alpha/BUILD.bazel b/tests/venv_site_packages_libs/nspkg_alpha/BUILD.bazel
@@ -1,4 +1,4 @@
-load("@rules_python//python:py_library.bzl", "py_library")
+load("//python:py_library.bzl", "py_library")
 
 package(default_visibility = ["//visibility:public"])
 
diff --git a/tests/venv_site_packages_libs/venv_site_packages_pypi_test.py b/tests/venv_site_packages_libs/venv_site_packages_pypi_test.py

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+# empty, will not be added to the site-packages dir`
Original file line number	Diff line number	Diff line change
`@@ -13,5 +13,6 @@ py_reconfig_test(`
`13`	`13`	`"//tests/venv_site_packages_libs/nspkg_beta",`
`14`	`14`	`"@other//nspkg_delta",`
`15`	`15`	`"@other//nspkg_gamma",`
	`16`	`+ "@other//nspkg_single",`
`16`	`17`	`],`
`17`	`18`	`)`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-load("@rules_python//python:py_library.bzl", "py_library")`
	`1`	`+load("//python:py_library.bzl", "py_library")`
`2`	`2`
`3`	`3`	`package(default_visibility = ["//visibility:public"])`
`4`	`4`