From 71cccb3a12ebcf7c79ce8bf695da13dae02a7be6 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Thu, 24 Apr 2025 14:07:57 +0100 Subject: [PATCH 01/13] Add `android.py env` command --- Android/android.py | 97 ++++++++++++++++++++++++++++++---------------- 1 file changed, 64 insertions(+), 33 deletions(-) diff --git a/Android/android.py b/Android/android.py index 1b20820b784371..b60995a7897264 100755 --- a/Android/android.py +++ b/Android/android.py @@ -22,9 +22,13 @@ SCRIPT_NAME = Path(__file__).name ANDROID_DIR = Path(__file__).resolve().parent -CHECKOUT = ANDROID_DIR.parent +PYTHON_DIR = ANDROID_DIR.parent +in_source_tree = ( + ANDROID_DIR.name == "Android" and (PYTHON_DIR / "pyconfig.h.in").exists() +) + TESTBED_DIR = ANDROID_DIR / "testbed" -CROSS_BUILD_DIR = CHECKOUT / "cross-build" +CROSS_BUILD_DIR = PYTHON_DIR / "cross-build" HOSTS = ["aarch64-linux-android", "x86_64-linux-android"] APP_ID = "org.python.testbed" @@ -74,41 +78,62 @@ def subdir(*parts, create=False): def run(command, *, host=None, env=None, log=True, **kwargs): kwargs.setdefault("check", True) + if env is None: env = os.environ.copy() - original_env = env.copy() - if host: - env_script = ANDROID_DIR / "android-env.sh" - env_output = subprocess.run( - f"set -eu; " - f"HOST={host}; " - f"PREFIX={subdir(host)}/prefix; " - f". {env_script}; " - f"export", - check=True, shell=True, text=True, stdout=subprocess.PIPE - ).stdout - - for line in env_output.splitlines(): - # We don't require every line to match, as there may be some other - # output from installing the NDK. - if match := re.search( - "^(declare -x |export )?(\\w+)=['\"]?(.*?)['\"]?$", line - ): - key, value = match[2], match[3] - if env.get(key) != value: - print(line) - env[key] = value - - if env == original_env: - raise ValueError(f"Found no variables in {env_script.name} output:\n" - + env_output) + env.update(android_env(host)) if log: print(">", " ".join(map(str, command))) return subprocess.run(command, env=env, **kwargs) +def print_env(context): + android_env(getattr(context, "host", None)) + + +def android_env(host): + if host: + prefix = subdir(host) / "prefix" + else: + prefix = ANDROID_DIR / "prefix" + sysconfigdata_files = prefix.glob("lib/python*/_sysconfigdata__android_*.py") + host = re.fullmatch( + r"_sysconfigdata__android_(.+).py", next(sysconfigdata_files).name + )[1] + + env_script = ANDROID_DIR / "android-env.sh" + env_output = subprocess.run( + f"set -eu; " + f"export HOST={host}; " + f"PREFIX={prefix}; " + f". {env_script}; " + f"export", + check=True, shell=True, capture_output=True, text=True, + ).stdout + + env = {} + for line in env_output.splitlines(): + # We don't require every line to match, as there may be some other + # output from installing the NDK. + if match := re.search( + "^(declare -x |export )?(\\w+)=['\"]?(.*?)['\"]?$", line + ): + key, value = match[2], match[3] + if os.environ.get(key) != value: + env[key] = value + + if not env: + raise ValueError(f"Found no variables in {env_script.name} output:\n" + + env_output) + + # Format the environment so it can be pasted into a shell. + for key, value in sorted(env.items()): + print(f"export {key}={shlex.quote(value)}") + return env + + def build_python_path(): """The path to the build Python binary.""" build_dir = subdir("build") @@ -127,7 +152,7 @@ def configure_build_python(context): clean("build") os.chdir(subdir("build", create=True)) - command = [relpath(CHECKOUT / "configure")] + command = [relpath(PYTHON_DIR / "configure")] if context.args: command.extend(context.args) run(command) @@ -168,7 +193,7 @@ def configure_host_python(context): os.chdir(host_dir) command = [ # Basic cross-compiling configuration - relpath(CHECKOUT / "configure"), + relpath(PYTHON_DIR / "configure"), f"--host={context.host}", f"--build={sysconfig.get_config_var('BUILD_GNU_TYPE')}", f"--with-build-python={build_python_path()}", @@ -624,8 +649,7 @@ def parse_args(): configure_build = subcommands.add_parser("configure-build", help="Run `configure` for the " "build Python") - make_build = subcommands.add_parser("make-build", - help="Run `make` for the build Python") + subcommands.add_parser("make-build", help="Run `make` for the build Python") configure_host = subcommands.add_parser("configure-host", help="Run `configure` for Android") make_host = subcommands.add_parser("make-host", @@ -637,16 +661,22 @@ def parse_args(): test = subcommands.add_parser( "test", help="Run the test suite") package = subcommands.add_parser("package", help="Make a release package") + env = subcommands.add_parser("env", help="Print environment variables") # Common arguments for subcommand in build, configure_build, configure_host: subcommand.add_argument( "--clean", action="store_true", default=False, dest="clean", help="Delete the relevant build and prefix directories first") - for subcommand in [build, configure_host, make_host, package]: + + host_commands = [build, configure_host, make_host, package] + if in_source_tree: + host_commands.append(env) + for subcommand in host_commands: subcommand.add_argument( "host", metavar="HOST", choices=HOSTS, help="Host triplet: choices=[%(choices)s]") + for subcommand in build, configure_build, configure_host: subcommand.add_argument("args", nargs="*", help="Extra arguments to pass to `configure`") @@ -690,6 +720,7 @@ def main(): "build-testbed": build_testbed, "test": run_testbed, "package": package, + "env": print_env, } try: From e3d27acaa6619ea3a5c509657c6d9b693456eef2 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Thu, 24 Apr 2025 17:14:42 +0100 Subject: [PATCH 02/13] Miscellaneous cleanups --- Android/README.md | 8 +++++--- Android/android.py | 20 +++++++++----------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Android/README.md b/Android/README.md index 789bcbe5edff44..6cabd6ba5d6844 100644 --- a/Android/README.md +++ b/Android/README.md @@ -25,11 +25,13 @@ it: `android-sdk/cmdline-tools/latest`. * `export ANDROID_HOME=/path/to/android-sdk` -The `android.py` script also requires the following commands to be on the `PATH`: +The `android.py` script will automatically use the SDK's `sdkmanager` to install +any packages it needs. + +The script also requires the following commands to be on the `PATH`: * `curl` * `java` (or set the `JAVA_HOME` environment variable) -* `tar` ## Building @@ -97,7 +99,7 @@ similar to the `Android` directory of the CPython source tree. The Python test suite can be run on Linux, macOS, or Windows: * On Linux, the emulator needs access to the KVM virtualization interface, and - a DISPLAY environment variable pointing at an X server. + a DISPLAY environment variable pointing at an X server. Xvfb is acceptable. The test suite can usually be run on a device with 2 GB of RAM, but this is borderline, so you may need to increase it to 4 GB. As of Android diff --git a/Android/android.py b/Android/android.py index b60995a7897264..89d927c5fc5dd3 100755 --- a/Android/android.py +++ b/Android/android.py @@ -163,19 +163,19 @@ def make_build_python(context): run(["make", "-j", str(os.cpu_count())]) -def unpack_deps(host): +def unpack_deps(host, prefix_dir): deps_url = "https://github.com/beeware/cpython-android-source-deps/releases/download" for name_ver in ["bzip2-1.0.8-2", "libffi-3.4.4-3", "openssl-3.0.15-4", "sqlite-3.49.1-0", "xz-5.4.6-1"]: filename = f"{name_ver}-{host}.tar.gz" download(f"{deps_url}/{name_ver}/{filename}") - run(["tar", "-xf", filename]) + shutil.unpack_archive(filename, prefix_dir) os.remove(filename) def download(url, target_dir="."): out_path = f"{target_dir}/{basename(url)}" - run(["curl", "-Lf", "-o", out_path, url]) + run(["curl", "-Lf", "--retry", "5", "--retry-all-errors", "-o", out_path, url]) return out_path @@ -187,8 +187,7 @@ def configure_host_python(context): prefix_dir = host_dir / "prefix" if not prefix_dir.exists(): prefix_dir.mkdir() - os.chdir(prefix_dir) - unpack_deps(context.host) + unpack_deps(context.host, prefix_dir) os.chdir(host_dir) command = [ @@ -266,16 +265,15 @@ def setup_sdk(): # the Gradle wrapper is not included in the CPython repository. Instead, we # extract it from the Gradle GitHub repository. def setup_testbed(): - # The Gradle version used for the build is specified in - # testbed/gradle/wrapper/gradle-wrapper.properties. This wrapper version - # doesn't need to match, as any version of the wrapper can download any - # version of Gradle. - version = "8.9.0" paths = ["gradlew", "gradlew.bat", "gradle/wrapper/gradle-wrapper.jar"] - if all((TESTBED_DIR / path).exists() for path in paths): return + # The wrapper version isn't important, as any version of the wrapper can + # download any version of Gradle. The Gradle version actually used for the + # build is specified in testbed/gradle/wrapper/gradle-wrapper.properties. + version = "8.9.0" + for path in paths: out_path = TESTBED_DIR / path out_path.parent.mkdir(exist_ok=True) From 24b082f74ea346408effe7b9546ecf4f52737fe3 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Thu, 24 Apr 2025 17:29:13 +0100 Subject: [PATCH 03/13] Prefer 'encoding' to 'text' Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Android/android.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Android/android.py b/Android/android.py index 89d927c5fc5dd3..9845a4f22288c1 100755 --- a/Android/android.py +++ b/Android/android.py @@ -110,7 +110,7 @@ def android_env(host): f"PREFIX={prefix}; " f". {env_script}; " f"export", - check=True, shell=True, capture_output=True, text=True, + check=True, shell=True, capture_output=True, encoding='utf-8', ).stdout env = {} From b7461d3326fc96287d9fbb1e18908cc6da3483e6 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Thu, 24 Apr 2025 17:53:42 +0100 Subject: [PATCH 04/13] Clarify sysconfigdata file discovery --- Android/android.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Android/android.py b/Android/android.py index 9845a4f22288c1..b410f3e2eec22f 100755 --- a/Android/android.py +++ b/Android/android.py @@ -98,10 +98,9 @@ def android_env(host): prefix = subdir(host) / "prefix" else: prefix = ANDROID_DIR / "prefix" - sysconfigdata_files = prefix.glob("lib/python*/_sysconfigdata__android_*.py") - host = re.fullmatch( - r"_sysconfigdata__android_(.+).py", next(sysconfigdata_files).name - )[1] + sysconfig_files = prefix.glob("lib/python*/_sysconfigdata__android_*.py") + sysconfig_filename = next(sysconfig_files).name + host = re.fullmatch(r"_sysconfigdata__android_(.+).py", sysconfig_filename)[1] env_script = ANDROID_DIR / "android-env.sh" env_output = subprocess.run( From c7cdb984dfc413313eeea10f57905d008d0507f6 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Fri, 25 Apr 2025 16:54:09 +0100 Subject: [PATCH 05/13] Rename api_level environment variable to ANDROID_API_LEVEL --- Android/android-env.sh | 4 ++-- Android/testbed/app/build.gradle.kts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Android/android-env.sh b/Android/android-env.sh index bab4130c9e92d0..ae1385034a37f2 100644 --- a/Android/android-env.sh +++ b/Android/android-env.sh @@ -3,7 +3,7 @@ : "${HOST:?}" # GNU target triplet # You may also override the following: -: "${api_level:=24}" # Minimum Android API level the build will run on +: "${ANDROID_API_LEVEL:=24}" # Minimum Android API level the build will run on : "${PREFIX:-}" # Path in which to find required libraries @@ -43,7 +43,7 @@ fi toolchain=$(echo "$ndk"/toolchains/llvm/prebuilt/*) export AR="$toolchain/bin/llvm-ar" export AS="$toolchain/bin/llvm-as" -export CC="$toolchain/bin/${clang_triplet}${api_level}-clang" +export CC="$toolchain/bin/${clang_triplet}${ANDROID_API_LEVEL}-clang" export CXX="${CC}++" export LD="$toolchain/bin/ld" export NM="$toolchain/bin/llvm-nm" diff --git a/Android/testbed/app/build.gradle.kts b/Android/testbed/app/build.gradle.kts index c627cb1b0e0b22..2a284f619db9ec 100644 --- a/Android/testbed/app/build.gradle.kts +++ b/Android/testbed/app/build.gradle.kts @@ -85,7 +85,7 @@ android { minSdk = androidEnvFile.useLines { for (line in it) { - """api_level:=(\d+)""".toRegex().find(line)?.let { + """ANDROID_API_LEVEL:=(\d+)""".toRegex().find(line)?.let { return@useLines it.groupValues[1].toInt() } } From f29e1779c31c1deb8a09d83959724cdea5acaeb9 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Wed, 30 Apr 2025 10:45:21 +0100 Subject: [PATCH 06/13] Environment variable cleanups --- Android/android.py | 38 ++++++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/Android/android.py b/Android/android.py index b410f3e2eec22f..0722da2df2ba8a 100755 --- a/Android/android.py +++ b/Android/android.py @@ -82,15 +82,30 @@ def run(command, *, host=None, env=None, log=True, **kwargs): if env is None: env = os.environ.copy() if host: - env.update(android_env(host)) + # The -I and -L arguments used when building Python should not be reused + # when building third-party extension modules, so pass them via the + # NODIST environment variables. + host_env = android_env(host) + for name in ["CFLAGS", "CXXFLAGS", "LDFLAGS"]: + flags = [] + nodist = [] + for word in host_env[name].split(): + (nodist if word.startswith(("-I", "-L")) else flags).append(word) + host_env[name] = " ".join(flags) + host_env[f"{name}_NODIST"] = " ".join(nodist) + + print_env(host_env) + env.update(host_env) if log: print(">", " ".join(map(str, command))) return subprocess.run(command, env=env, **kwargs) -def print_env(context): - android_env(getattr(context, "host", None)) +# Format the environment so it can be pasted into a shell. +def print_env(env): + for key, value in sorted(env.items()): + print(f"export {key}={shlex.quote(value)}") def android_env(host): @@ -126,10 +141,6 @@ def android_env(host): if not env: raise ValueError(f"Found no variables in {env_script.name} output:\n" + env_output) - - # Format the environment so it can be pasted into a shell. - for key, value in sorted(env.items()): - print(f"export {key}={shlex.quote(value)}") return env @@ -220,9 +231,12 @@ def make_host_python(context): for pattern in ("include/python*", "lib/libpython*", "lib/python*"): delete_glob(f"{prefix_dir}/{pattern}") + # The Android environment variables were already captured in the Makefile by + # `configure`, and passing them again when running `make` may cause some + # flags to be duplicated. So we don't use the `host` argument here. os.chdir(host_dir) - run(["make", "-j", str(os.cpu_count())], host=context.host) - run(["make", "install", f"prefix={prefix_dir}"], host=context.host) + run(["make", "-j", str(os.cpu_count())]) + run(["make", "install", f"prefix={prefix_dir}"]) def build_all(context): @@ -628,6 +642,10 @@ def package(context): print(f"Wrote {package_path}") +def env(context): + print_env(android_env(getattr(context, "host", None))) + + # Handle SIGTERM the same way as SIGINT. This ensures that if we're terminated # by the buildbot worker, we'll make an attempt to clean up our subprocesses. def install_signal_handler(): @@ -717,7 +735,7 @@ def main(): "build-testbed": build_testbed, "test": run_testbed, "package": package, - "env": print_env, + "env": env, } try: From fc8c1e1fd7e8bf947ea16b0cc47efd336f8c45d9 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Fri, 2 May 2025 15:57:49 +0100 Subject: [PATCH 07/13] Add `android.py test` -c and -m options --- Android/android.py | 68 ++++++++++++++----- .../java/org/python/testbed/PythonSuite.kt | 8 +-- .../java/org/python/testbed/MainActivity.kt | 27 ++++++-- .../{main.py => android_testbed_main.py} | 10 ++- 4 files changed, 82 insertions(+), 31 deletions(-) rename Android/testbed/app/src/main/python/{main.py => android_testbed_main.py} (84%) diff --git a/Android/android.py b/Android/android.py index 0722da2df2ba8a..eafbc56f46fecf 100755 --- a/Android/android.py +++ b/Android/android.py @@ -98,10 +98,19 @@ def run(command, *, host=None, env=None, log=True, **kwargs): env.update(host_env) if log: - print(">", " ".join(map(str, command))) + print(">", join_command(command)) return subprocess.run(command, env=env, **kwargs) +# Format a command so it can be copied into a shell. Like shlex.join, but also +# accepts arguments which are Paths, or a single string/Path outside of a list. +def join_command(args): + if isinstance(args, (str, Path)): + return str(args) + else: + return shlex.join(map(str, args)) + + # Format the environment so it can be pasted into a shell. def print_env(env): for key, value in sorted(env.items()): @@ -512,24 +521,42 @@ async def gradle_task(context): task_prefix = "connected" env["ANDROID_SERIAL"] = context.connected + hidden_output = [] + + def log(line): + # Gradle may take several minutes to install SDK packages, so it's worth + # showing those messages even in non-verbose mode. + if context.verbose or line.startswith('Preparing "Install'): + sys.stdout.write(line) + else: + hidden_output.append(line) + + if context.command: + mode = "-c" + module = context.command + else: + mode = "-m" + module = context.module or "test" + args = [ gradlew, "--console", "plain", f"{task_prefix}DebugAndroidTest", - "-Pandroid.testInstrumentationRunnerArguments.pythonArgs=" - + shlex.join(context.args), + ] + [ + f"-Pandroid.testInstrumentationRunnerArguments.python{name}={value}" + for name, value in [ + ("Mode", mode), + ("Module", module), + ("Args", join_command(context.args)), + ] ] - hidden_output = [] + log("> " + join_command(args)) + try: async with async_process( *args, cwd=TESTBED_DIR, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) as process: while line := (await process.stdout.readline()).decode(*DECODE_ARGS): - # Gradle may take several minutes to install SDK packages, so - # it's worth showing those messages even in non-verbose mode. - if context.verbose or line.startswith('Preparing "Install'): - sys.stdout.write(line) - else: - hidden_output.append(line) + log(line) status = await wait_for(process.wait(), timeout=1) if status == 0: @@ -701,6 +728,7 @@ def parse_args(): "-v", "--verbose", action="count", default=0, help="Show Gradle output, and non-Python logcat messages. " "Use twice to include high-volume messages which are rarely useful.") + device_group = test.add_mutually_exclusive_group(required=True) device_group.add_argument( "--connected", metavar="SERIAL", help="Run on a connected device. " @@ -708,8 +736,17 @@ def parse_args(): device_group.add_argument( "--managed", metavar="NAME", help="Run on a Gradle-managed device. " "These are defined in `managedDevices` in testbed/app/build.gradle.kts.") + + mode_group = test.add_mutually_exclusive_group() + mode_group.add_argument( + "-c", dest="command", help="Execute the given Python code.") + mode_group.add_argument( + "-m", dest="module", help="Execute the module with the given name.") + test.epilog = ( + "If neither -c nor -m are passed, the default is '-m test', which will " + "run Python's own test suite.") test.add_argument( - "args", nargs="*", help=f"Arguments for `python -m test`. " + "args", nargs="*", help=f"Arguments to add to sys.argv. " f"Separate them from {SCRIPT_NAME}'s own arguments with `--`.") return parser.parse_args() @@ -756,14 +793,9 @@ def print_called_process_error(e): if not content.endswith("\n"): stream.write("\n") - # Format the command so it can be copied into a shell. shlex uses single - # quotes, so we surround the whole command with double quotes. - args_joined = ( - e.cmd if isinstance(e.cmd, str) - else " ".join(shlex.quote(str(arg)) for arg in e.cmd) - ) + # shlex uses single quotes, so we surround the command with double quotes. print( - f'Command "{args_joined}" returned exit status {e.returncode}' + f'Command "{join_command(e.cmd)}" returned exit status {e.returncode}' ) diff --git a/Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt b/Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt index 0e888ab71d87da..94be52dd2dc870 100644 --- a/Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt +++ b/Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt @@ -17,11 +17,11 @@ class PythonSuite { fun testPython() { val start = System.currentTimeMillis() try { - val context = + val status = PythonTestRunner( InstrumentationRegistry.getInstrumentation().targetContext - val args = - InstrumentationRegistry.getArguments().getString("pythonArgs", "") - val status = PythonTestRunner(context).run(args) + ).run( + InstrumentationRegistry.getArguments() + ) assertEquals(0, status) } finally { // Make sure the process lives long enough for the test script to diff --git a/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt b/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt index c4bf6cbe83d8cd..db716d2b49e49c 100644 --- a/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt +++ b/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt @@ -15,17 +15,29 @@ class MainActivity : AppCompatActivity() { override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) setContentView(R.layout.activity_main) - val status = PythonTestRunner(this).run("-W -uall") + val status = PythonTestRunner(this).run("-m", "test", "-W -uall") findViewById(R.id.tvHello).text = "Exit status $status" } } class PythonTestRunner(val context: Context) { - /** @param args Extra arguments for `python -m test`. - * @return The Python exit status: zero if the tests passed, nonzero if - * they failed. */ - fun run(args: String = "") : Int { + fun run(instrumentationArgs: Bundle) = run( + instrumentationArgs.getString("pythonMode")!!, + instrumentationArgs.getString("pythonModule")!!, + instrumentationArgs.getString("pythonArgs")!!, + ) + + /** Run Python. + * + * @param mode Either "-c" or "-m". + * @param module Python statements for "-c" mode, or a module name for + * "-m" mode. + * @param args Arguments to add to sys.argv. Will be parsed by `shlex.split`. + * @return The Python exit status: zero on success, nonzero on failure. */ + fun run(mode: String, module: String, args: String = "") : Int { + Os.setenv("PYTHON_MODE", mode, true) + Os.setenv("PYTHON_MODULE", module, true) Os.setenv("PYTHON_ARGS", args, true) // Python needs this variable to help it find the temporary directory, @@ -36,8 +48,9 @@ class PythonTestRunner(val context: Context) { System.loadLibrary("main_activity") redirectStdioToLogcat() - // The main module is in src/main/python/main.py. - return runPython(pythonHome.toString(), "main") + // The main module is in src/main/python. We don't simply call it + // "main", as that could clash with third-party test code. + return runPython(pythonHome.toString(), "android_testbed_main") } private fun extractAssets() : File { diff --git a/Android/testbed/app/src/main/python/main.py b/Android/testbed/app/src/main/python/android_testbed_main.py similarity index 84% rename from Android/testbed/app/src/main/python/main.py rename to Android/testbed/app/src/main/python/android_testbed_main.py index d6941b14412fcc..71ac47e5734306 100644 --- a/Android/testbed/app/src/main/python/main.py +++ b/Android/testbed/app/src/main/python/android_testbed_main.py @@ -26,7 +26,13 @@ # test_signals in test_threadsignals.py. signal.pthread_sigmask(signal.SIG_UNBLOCK, [signal.SIGUSR1]) +mode = os.environ["PYTHON_MODE"] +module = os.environ["PYTHON_MODULE"] sys.argv[1:] = shlex.split(os.environ["PYTHON_ARGS"]) -# The test module will call sys.exit to indicate whether the tests passed. -runpy.run_module("test") +if mode == "-c": + exec(module, {}) +elif mode == "-m": + runpy.run_module(module, run_name="__main__", alter_sys=True) +else: + raise ValueError(f"unknown mode: {mode}") From b273bc743dacb24128ed4bb11b6c4c0b4d498222 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Sat, 3 May 2025 14:29:55 +0100 Subject: [PATCH 08/13] Add `android.py test` --site-packages and --cwd options --- Android/README.md | 4 ++ Android/android.py | 54 +++++++++++-------- Android/testbed/app/build.gradle.kts | 18 +++++++ .../java/org/python/testbed/MainActivity.kt | 4 +- .../src/main/python/android_testbed_main.py | 10 ++++ Doc/using/android.rst | 9 ++++ 6 files changed, 76 insertions(+), 23 deletions(-) diff --git a/Android/README.md b/Android/README.md index 6cabd6ba5d6844..c42eb627006e6a 100644 --- a/Android/README.md +++ b/Android/README.md @@ -156,6 +156,10 @@ repository's `Lib` directory will be picked up immediately. Changes in C files, and architecture-specific files such as sysconfigdata, will not take effect until you re-run `android.py make-host` or `build`. +The testbed app can also be used to test third-party packages. For more details, +run `android.py test --help`, paying attention to the options `--site-packages`, +`--cwd`, `-c` and `-m`. + ## Using in your own app diff --git a/Android/android.py b/Android/android.py index eafbc56f46fecf..40c962ed4715a4 100755 --- a/Android/android.py +++ b/Android/android.py @@ -14,7 +14,7 @@ from contextlib import asynccontextmanager from datetime import datetime, timezone from glob import glob -from os.path import basename, relpath +from os.path import abspath, basename, relpath from pathlib import Path from subprocess import CalledProcessError from tempfile import TemporaryDirectory @@ -541,12 +541,17 @@ def log(line): args = [ gradlew, "--console", "plain", f"{task_prefix}DebugAndroidTest", ] + [ + # Build-time properties + f"-Ppython.{name}={value}" + for name, value in [ + ("sitePackages", context.site_packages), ("cwd", context.cwd) + ] if value + ] + [ + # Runtime properties f"-Pandroid.testInstrumentationRunnerArguments.python{name}={value}" for name, value in [ - ("Mode", mode), - ("Module", module), - ("Args", join_command(context.args)), - ] + ("Mode", mode), ("Module", module), ("Args", join_command(context.args)) + ] if value ] log("> " + join_command(args)) @@ -684,24 +689,24 @@ def signal_handler(*args): def parse_args(): parser = argparse.ArgumentParser() - subcommands = parser.add_subparsers(dest="subcommand") + subcommands = parser.add_subparsers(dest="subcommand", required=True) # Subcommands - build = subcommands.add_parser("build", help="Build everything") - configure_build = subcommands.add_parser("configure-build", - help="Run `configure` for the " - "build Python") - subcommands.add_parser("make-build", help="Run `make` for the build Python") - configure_host = subcommands.add_parser("configure-host", - help="Run `configure` for Android") - make_host = subcommands.add_parser("make-host", - help="Run `make` for Android") - subcommands.add_parser( - "clean", help="Delete all build and prefix directories") + build = subcommands.add_parser( + "build", help="Run configure-build, make-build, configure-host and " + "make-host") + configure_build = subcommands.add_parser( + "configure-build", help="Run `configure` for the build Python") subcommands.add_parser( - "build-testbed", help="Build the testbed app") - test = subcommands.add_parser( - "test", help="Run the test suite") + "make-build", help="Run `make` for the build Python") + configure_host = subcommands.add_parser( + "configure-host", help="Run `configure` for Android") + make_host = subcommands.add_parser( + "make-host", help="Run `make` for Android") + + subcommands.add_parser("clean", help="Delete all build directories") + subcommands.add_parser("build-testbed", help="Build the testbed app") + test = subcommands.add_parser("test", help="Run the testbed app") package = subcommands.add_parser("package", help="Make a release package") env = subcommands.add_parser("env", help="Print environment variables") @@ -709,7 +714,7 @@ def parse_args(): for subcommand in build, configure_build, configure_host: subcommand.add_argument( "--clean", action="store_true", default=False, dest="clean", - help="Delete the relevant build and prefix directories first") + help="Delete the relevant build directories first") host_commands = [build, configure_host, make_host, package] if in_source_tree: @@ -737,6 +742,13 @@ def parse_args(): "--managed", metavar="NAME", help="Run on a Gradle-managed device. " "These are defined in `managedDevices` in testbed/app/build.gradle.kts.") + test.add_argument( + "--site-packages", metavar="DIR", type=abspath, + help="Directory to copy as the app's site-packages.") + test.add_argument( + "--cwd", metavar="DIR", type=abspath, + help="Directory to copy as the app's working directory.") + mode_group = test.add_mutually_exclusive_group() mode_group.add_argument( "-c", dest="command", help="Execute the given Python code.") diff --git a/Android/testbed/app/build.gradle.kts b/Android/testbed/app/build.gradle.kts index 2a284f619db9ec..92cffd61f86876 100644 --- a/Android/testbed/app/build.gradle.kts +++ b/Android/testbed/app/build.gradle.kts @@ -205,11 +205,29 @@ androidComponents.onVariants { variant -> into("site-packages") { from("$projectDir/src/main/python") + + val sitePackages = findProperty("python.sitePackages") as String? + if (!sitePackages.isNullOrEmpty()) { + if (!file(sitePackages).exists()) { + throw GradleException("$sitePackages does not exist") + } + from(sitePackages) + } } duplicatesStrategy = DuplicatesStrategy.EXCLUDE exclude("**/__pycache__") } + + into("cwd") { + val cwd = findProperty("python.cwd") as String? + if (!cwd.isNullOrEmpty()) { + if (!file(cwd).exists()) { + throw GradleException("$cwd does not exist") + } + from(cwd) + } + } } } diff --git a/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt b/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt index db716d2b49e49c..ef28948486fb52 100644 --- a/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt +++ b/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt @@ -25,7 +25,7 @@ class PythonTestRunner(val context: Context) { fun run(instrumentationArgs: Bundle) = run( instrumentationArgs.getString("pythonMode")!!, instrumentationArgs.getString("pythonModule")!!, - instrumentationArgs.getString("pythonArgs")!!, + instrumentationArgs.getString("pythonArgs") ?: "", ) /** Run Python. @@ -35,7 +35,7 @@ class PythonTestRunner(val context: Context) { * "-m" mode. * @param args Arguments to add to sys.argv. Will be parsed by `shlex.split`. * @return The Python exit status: zero on success, nonzero on failure. */ - fun run(mode: String, module: String, args: String = "") : Int { + fun run(mode: String, module: String, args: String) : Int { Os.setenv("PYTHON_MODE", mode, true) Os.setenv("PYTHON_MODULE", module, true) Os.setenv("PYTHON_ARGS", args, true) diff --git a/Android/testbed/app/src/main/python/android_testbed_main.py b/Android/testbed/app/src/main/python/android_testbed_main.py index 71ac47e5734306..31b8e5343a8449 100644 --- a/Android/testbed/app/src/main/python/android_testbed_main.py +++ b/Android/testbed/app/src/main/python/android_testbed_main.py @@ -30,9 +30,19 @@ module = os.environ["PYTHON_MODULE"] sys.argv[1:] = shlex.split(os.environ["PYTHON_ARGS"]) +cwd = f"{sys.prefix}/cwd" +if not os.path.exists(cwd): + # Empty directories are lost in the asset packing/unpacking process. + os.mkdir(cwd) +os.chdir(cwd) + if mode == "-c": + # In -c mode, sys.path starts with an empty string, which means whatever the current + # working directory is at the moment of each import. + sys.path.insert(0, "") exec(module, {}) elif mode == "-m": + sys.path.insert(0, os.getcwd()) runpy.run_module(module, run_name="__main__", alter_sys=True) else: raise ValueError(f"unknown mode: {mode}") diff --git a/Doc/using/android.rst b/Doc/using/android.rst index 65bf23dc994856..cb762310328f1c 100644 --- a/Doc/using/android.rst +++ b/Doc/using/android.rst @@ -63,3 +63,12 @@ link to the relevant file. * Add code to your app to :source:`start Python in embedded mode `. This will need to be C code called via JNI. + +Building a Python package for Android +------------------------------------- + +Python packages can be built for Android as wheels and released on PyPI. The +recommended tool for doing this is `cibuildwheel +`__, which automates +all the details of setting up a cross-compilation environment, building the +wheel, and testing it on an emulator. From f36273a2cf393339338a319f54553ab1f5871835 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Thu, 29 May 2025 14:37:21 +0100 Subject: [PATCH 09/13] Revert use of NODIST environment variables --- Android/android.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/Android/android.py b/Android/android.py index 40c962ed4715a4..5e986e59110efd 100755 --- a/Android/android.py +++ b/Android/android.py @@ -78,22 +78,11 @@ def subdir(*parts, create=False): def run(command, *, host=None, env=None, log=True, **kwargs): kwargs.setdefault("check", True) - if env is None: env = os.environ.copy() + if host: - # The -I and -L arguments used when building Python should not be reused - # when building third-party extension modules, so pass them via the - # NODIST environment variables. host_env = android_env(host) - for name in ["CFLAGS", "CXXFLAGS", "LDFLAGS"]: - flags = [] - nodist = [] - for word in host_env[name].split(): - (nodist if word.startswith(("-I", "-L")) else flags).append(word) - host_env[name] = " ".join(flags) - host_env[f"{name}_NODIST"] = " ".join(nodist) - print_env(host_env) env.update(host_env) From 51e1460c5646d597e9bd76bd57bef4c0fb31d1ad Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Mon, 2 Jun 2025 22:24:25 +0100 Subject: [PATCH 10/13] Update to current Android Gradle plugin version, which gives better error messages when failing to start the emulator --- Android/testbed/build.gradle.kts | 2 +- Android/testbed/gradle/wrapper/gradle-wrapper.properties | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Android/testbed/build.gradle.kts b/Android/testbed/build.gradle.kts index 4d1d6f87594da3..451517b3f1aeab 100644 --- a/Android/testbed/build.gradle.kts +++ b/Android/testbed/build.gradle.kts @@ -1,5 +1,5 @@ // Top-level build file where you can add configuration options common to all sub-projects/modules. plugins { - id("com.android.application") version "8.6.1" apply false + id("com.android.application") version "8.10.0" apply false id("org.jetbrains.kotlin.android") version "1.9.22" apply false } diff --git a/Android/testbed/gradle/wrapper/gradle-wrapper.properties b/Android/testbed/gradle/wrapper/gradle-wrapper.properties index 36529c896426b0..5d42fbae084da1 100644 --- a/Android/testbed/gradle/wrapper/gradle-wrapper.properties +++ b/Android/testbed/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ #Mon Feb 19 20:29:06 GMT 2024 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists From 497e84c92cec3823aa6add8283862f060fd2d7b0 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Tue, 3 Jun 2025 19:26:36 +0100 Subject: [PATCH 11/13] Update to an NDK version which is pre-installed on GitHub Actions --- Android/android-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Android/android-env.sh b/Android/android-env.sh index ae1385034a37f2..7b381a013cf0ba 100644 --- a/Android/android-env.sh +++ b/Android/android-env.sh @@ -24,7 +24,7 @@ fail() { # * https://android.googlesource.com/platform/ndk/+/ndk-rXX-release/docs/BuildSystemMaintainers.md # where XX is the NDK version. Do a diff against the version you're upgrading from, e.g.: # https://android.googlesource.com/platform/ndk/+/ndk-r25-release..ndk-r26-release/docs/BuildSystemMaintainers.md -ndk_version=27.1.12297006 +ndk_version=27.2.12479018 ndk=$ANDROID_HOME/ndk/$ndk_version if ! [ -e "$ndk" ]; then From ddfb875de4d4974f67e4dd8c7421b9b8f68cc126 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Tue, 3 Jun 2025 19:28:08 +0100 Subject: [PATCH 12/13] Logging improvements --- Android/android.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Android/android.py b/Android/android.py index 5e986e59110efd..c8f235724f7d9e 100755 --- a/Android/android.py +++ b/Android/android.py @@ -264,7 +264,12 @@ def setup_sdk(): if not all((android_home / "licenses" / path).exists() for path in [ "android-sdk-arm-dbt-license", "android-sdk-license" ]): - run([sdkmanager, "--licenses"], text=True, input="y\n" * 100) + run( + [sdkmanager, "--licenses"], + text=True, + capture_output=True, + input="y\n" * 100, + ) # Gradle may install this automatically, but we can't rely on that because # we need to run adb within the logcat task. @@ -542,6 +547,8 @@ def log(line): ("Mode", mode), ("Module", module), ("Args", join_command(context.args)) ] if value ] + if context.verbose >= 2: + args.append("--info") log("> " + join_command(args)) try: From 6f185281d13784b0472bcf34bf8d9fd03a722435 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Wed, 4 Jun 2025 15:22:39 +0100 Subject: [PATCH 13/13] Update to bzip2-1.0.8-3 --- Android/android.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Android/android.py b/Android/android.py index c8f235724f7d9e..551168fc4b2f5a 100755 --- a/Android/android.py +++ b/Android/android.py @@ -172,12 +172,13 @@ def make_build_python(context): def unpack_deps(host, prefix_dir): + os.chdir(prefix_dir) deps_url = "https://github.com/beeware/cpython-android-source-deps/releases/download" - for name_ver in ["bzip2-1.0.8-2", "libffi-3.4.4-3", "openssl-3.0.15-4", + for name_ver in ["bzip2-1.0.8-3", "libffi-3.4.4-3", "openssl-3.0.15-4", "sqlite-3.49.1-0", "xz-5.4.6-1"]: filename = f"{name_ver}-{host}.tar.gz" download(f"{deps_url}/{name_ver}/{filename}") - shutil.unpack_archive(filename, prefix_dir) + shutil.unpack_archive(filename) os.remove(filename) pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy