Add testbed implementation

encukou · May 1, 2024 · Apr 14, 2024 · Apr 14, 2024 · Apr 14, 2024 · Apr 14, 2024
commit 4a6affebf98d19e3384adc9607ea9b9471b4f71c
diff --git a/Android/testbed/app/build.gradle.kts b/Android/testbed/app/build.gradle.kts
@@ -1,8 +1,28 @@
+import com.android.build.api.variant.*
+
 plugins {
     id("com.android.application")
     id("org.jetbrains.kotlin.android")
 }
 
+val PYTHON_DIR = File(projectDir, "../../..").canonicalPath
+val PYTHON_CROSS_DIR = "$PYTHON_DIR/cross-build"
+val ABIS = mapOf(
+    "arm64-v8a" to "aarch64-linux-android",
+    "x86_64" to "x86_64-linux-android",
+)
+
+val PYTHON_VERSION = File("$PYTHON_DIR/Include/patchlevel.h").useLines {
+    for (line in it) {
+        val match = """#define PY_VERSION\s+"(\d+\.\d+)""".toRegex().find(line)
+        if (match != null) {
+            return@useLines match.groupValues[1]
+        }
+    }
+    throw GradleException("Failed to find Python version")
+}
+
+
 android {
     namespace = "org.python.testbed"
     compileSdk = 34
@@ -15,8 +35,21 @@ android {
         versionName = "1.0"
 
         testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner"
+
+        ndk.abiFilters.addAll(ABIS.keys)
+        externalNativeBuild.cmake.arguments(
+            "-DPYTHON_CROSS_DIR=$PYTHON_CROSS_DIR",
+            "-DPYTHON_VERSION=$PYTHON_VERSION")
+    }
+
+    externalNativeBuild.cmake {
+        path("src/main/c/CMakeLists.txt")
     }
 
+    // Set this property to something non-empty, otherwise it'll use the default
+    // list, which ignores asset directories beginning with an underscore.
+    aaptOptions.ignoreAssetsPattern = ".git"
+
     buildTypes {
         release {
             isMinifyEnabled = false
@@ -44,4 +77,69 @@ dependencies {
     testImplementation("junit:junit:4.13.2")
     androidTestImplementation("androidx.test.ext:junit:1.1.5")
     androidTestImplementation("androidx.test.espresso:espresso-core:3.5.1")
-}
+}
+
+
+// Create some custom tasks to copy Python and its standard library from
+// elsewhere in the repository.
+androidComponents.onVariants { variant ->
+    generateTask(variant, variant.sources.assets!!) {
+        into("python") {
+            for (triplet in ABIS.values) {
+                for (subDir in listOf("include", "lib")) {
+                    into(subDir) {
+                        from("$PYTHON_CROSS_DIR/$triplet/prefix/$subDir")
+                        include("python$PYTHON_VERSION/**")
+                        duplicatesStrategy = DuplicatesStrategy.EXCLUDE
+                    }
+                }
+            }
+            into("lib/python$PYTHON_VERSION") {
+                // Uncomment this to pick up edits from the source directory
+                // without having to rerun `make install`.
+                // from("$PYTHON_DIR/Lib")
+                // duplicatesStrategy = DuplicatesStrategy.INCLUDE
+
+                into("site-packages") {
+                    from("$projectDir/src/main/python")
+                }
+            }
+        }
+        exclude("**/__pycache__")
+    }
+
+    generateTask(variant, variant.sources.jniLibs!!) {
+        for ((abi, triplet) in ABIS.entries) {
+            into(abi) {
+                from("$PYTHON_CROSS_DIR/$triplet/prefix/lib")
+                include("libpython*.*.so")
+                include("lib*_python.so")
+            }
+        }
+    }
+}
+
+
+fun generateTask(
+    variant: ApplicationVariant, directories: SourceDirectories,
+    configure: GenerateTask.() -> Unit
+) {
+    val taskName = "generate" +
+        listOf(variant.name, "Python", directories.name)
+            .map { it.replaceFirstChar(Char::uppercase) }
+            .joinToString("")
+
+    directories.addGeneratedSourceDirectory(
+        tasks.register<GenerateTask>(taskName) {
+            into(outputDir)
+            configure()
+        },
+        GenerateTask::outputDir)
+}
+
+
+// addGeneratedSourceDirectory requires the task to have a DirectoryProperty.
+abstract class GenerateTask: Sync() {
+    @get:OutputDirectory
+    abstract val outputDir: DirectoryProperty
+}
diff --git a/Android/testbed/app/src/main/AndroidManifest.xml b/Android/testbed/app/src/main/AndroidManifest.xml
@@ -2,6 +2,8 @@
 <manifest xmlns:android="http://schemas.android.com/apk/res/android"
     xmlns:tools="http://schemas.android.com/tools">
 
+    <uses-permission android:name="android.permission.INTERNET"/>
+
     <application
         android:allowBackup="true"
         android:dataExtractionRules="@xml/data_extraction_rules"

diff --git a/Android/testbed/app/src/main/c/CMakeLists.txt b/Android/testbed/app/src/main/c/CMakeLists.txt
@@ -0,0 +1,9 @@
+cmake_minimum_required(VERSION 3.4.1)
+project(testbed)
+
+set(PREFIX_DIR ${PYTHON_CROSS_DIR}/${CMAKE_LIBRARY_ARCHITECTURE}/prefix)
+include_directories(${PREFIX_DIR}/include/python${PYTHON_VERSION})
+link_directories(${PREFIX_DIR}/lib)
+link_libraries(log python${PYTHON_VERSION})
+
+add_library(main_activity SHARED main_activity.c)
diff --git a/Android/testbed/app/src/main/c/main_activity.c b/Android/testbed/app/src/main/c/main_activity.c
@@ -0,0 +1,147 @@
+#include <android/log.h>
+#include <errno.h>
+#include <jni.h>
+#include <pthread.h>
+#include <Python.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+
+
+static void throw_runtime_exception(JNIEnv *env, const char *message) {
+    (*env)->ThrowNew(
+        env,
+        (*env)->FindClass(env, "java/lang/RuntimeException"),
+        message);
+}
+
+
+// --- Stdio redirection ------------------------------------------------------
+
+// Most apps won't need this, because the Python-level sys.stdout and sys.stderr
+// are redirected to the Android logcat by Python itself. However, in the
+// testbed it's useful to redirect the native streams as well, to debug problems
+// in the Python startup or redirection process.
+//
+// Based on
+// https://github.com/beeware/briefcase-android-gradle-template/blob/v0.3.11/%7B%7B%20cookiecutter.safe_formal_name%20%7D%7D/app/src/main/cpp/native-lib.cpp
+
+typedef struct {
+    FILE *file;
+    int fd;
+    android_LogPriority priority;
+    char *tag;
+    int pipe[2];
+} StreamInfo;
+
+static StreamInfo STREAMS[] = {
+    {stdout, STDOUT_FILENO, ANDROID_LOG_INFO, "native.stdout", {-1, -1}},
+    {stderr, STDERR_FILENO, ANDROID_LOG_WARN, "native.stderr", {-1, -1}},
+    {NULL, -1, ANDROID_LOG_UNKNOWN, NULL, {-1, -1}},
+};
+
+// The maximum length of a log message in bytes, including the level marker and
+// tag, is defined as LOGGER_ENTRY_MAX_PAYLOAD in
+// platform/system/logging/liblog/include/log/log.h. As of API level 30, messages
+// longer than this will be be truncated by logcat. This limit has already been
+// reduced at least once in the history of Android (from 4076 to 4068 between API
+// level 23 and 26), so leave some headroom.
+static const int MAX_BYTES_PER_WRITE = 4000;
+
+static void *redirection_thread(void *arg) {
+    StreamInfo *si = (StreamInfo*)arg;
+    ssize_t read_size;
+    char buf[MAX_BYTES_PER_WRITE];
+    while ((read_size = read(si->pipe[0], buf, sizeof buf - 1)) > 0) {
+        buf[read_size] = '\0'; /* add null-terminator */
+        __android_log_write(si->priority, si->tag, buf);
+    }
+    return 0;
+}
+
+static char *redirect_stream(StreamInfo *si) {
+    /* make the FILE unbuffered, to ensure messages are never lost */
+    if (setvbuf(si->file, 0, _IONBF, 0)) {
+        return "setvbuf";
+    }
+
+    /* create the pipe and redirect the file descriptor */
+    if (pipe(si->pipe)) {
+        return "pipe";
+    }
+    if (dup2(si->pipe[1], si->fd) == -1) {
+        return "dup2";
+    }
+
+    /* start the logging thread */
+    pthread_t thr;
+    if ((errno = pthread_create(&thr, 0, redirection_thread, si))) {
+        return "pthread_create";
+    }
+    if ((errno = pthread_detach(thr))) {
+        return "pthread_detach";
+    }
+    return 0;
+}
+
+JNIEXPORT void JNICALL Java_org_python_testbed_MainActivity_redirectStdioToLogcat(
+    JNIEnv *env, jobject obj
+) {
+    for (StreamInfo *si = STREAMS; si->file; si++) {
+        char *error_prefix;
+        if ((error_prefix = redirect_stream(si))) {
+            char error_message[1024];
+            snprintf(error_message, sizeof(error_message),
+                     "%s: %s", error_prefix, strerror(errno));
+            throw_runtime_exception(env, error_message);
+            return;
+        }
+    }
+}
+
+
+// --- Python intialization ----------------------------------------------------
+
+static PyStatus set_config_string(
+    JNIEnv *env, PyConfig *config, wchar_t **config_str, jstring value
+) {
+    const char *value_utf8 = (*env)->GetStringUTFChars(env, value, NULL);
+    PyStatus status = PyConfig_SetBytesString(config, config_str, value_utf8);
+    (*env)->ReleaseStringUTFChars(env, value, value_utf8);
+    return status;
+}
+
+static void throw_status(JNIEnv *env, PyStatus status) {
+    throw_runtime_exception(env, status.err_msg ? status.err_msg : "");
+}
+
+JNIEXPORT void JNICALL Java_org_python_testbed_MainActivity_runPython(
+    JNIEnv *env, jobject obj, jstring home, jstring runModule
+) {
+    PyConfig config;
+    PyStatus status;
+    PyConfig_InitIsolatedConfig(&config);
+
+    status = set_config_string(env, &config, &config.home, home);
+    if (PyStatus_Exception(status)) {
+        throw_status(env, status);
+        return;
+    }
+
+    status = set_config_string(env, &config, &config.run_module, runModule);
+    if (PyStatus_Exception(status)) {
+        throw_status(env, status);
+        return;
+    }
+
+    // Some tests generate SIGPIPE and SIGXFSZ, which should be ignored.
+    config.install_signal_handlers = 1;
+
+    status = Py_InitializeFromConfig(&config);
+    if (PyStatus_Exception(status)) {
+        throw_status(env, status);
+        return;
+    }
+
+    Py_RunMain();
+}
diff --git a/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt b/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt
@@ -1,11 +1,61 @@
 package org.python.testbed
 
-import androidx.appcompat.app.AppCompatActivity
-import android.os.Bundle
+import android.os.*
+import android.system.Os
+import android.widget.TextView
+import androidx.appcompat.app.*
+import java.io.*
 
 class MainActivity : AppCompatActivity() {
     override fun onCreate(savedInstanceState: Bundle?) {
         super.onCreate(savedInstanceState)
         setContentView(R.layout.activity_main)
+
+        // Python needs this variable to help it find the temporary directory,
+        // but Android only sets it on API level 33 and later.
+        Os.setenv("TMPDIR", cacheDir.toString(), false)
+
+        val pythonHome = extractAssets()
+        System.loadLibrary("main_activity")
+        redirectStdioToLogcat()
+        runPython(pythonHome.toString(), "main")
+        findViewById<TextView>(R.id.tvHello).text = "Python complete"
+    }
+
+    private fun extractAssets() : File {
+        val pythonHome = File(filesDir, "python")
+        if (pythonHome.exists() && !pythonHome.deleteRecursively()) {
+            throw RuntimeException("Failed to delete $pythonHome")
+        }
+        extractAssetDir("python", filesDir)
+        return pythonHome
     }
+
+    private fun extractAssetDir(path: String, targetDir: File) {
+        val names = assets.list(path)
+            ?: throw RuntimeException("Failed to list $path")
+        val targetSubdir = File(targetDir, path)
+        if (!targetSubdir.mkdirs()) {
+            throw RuntimeException("Failed to create $targetSubdir")
+        }
+
+        for (name in names) {
+            val subPath = "$path/$name"
+            val input: InputStream
+            try {
+                input = assets.open(subPath)
+            } catch (e: FileNotFoundException) {
+                extractAssetDir(subPath, targetDir)
+                continue
+            }
+            input.use {
+                File(targetSubdir, name).outputStream().use { output ->
+                    input.copyTo(output)
+                }
+            }
+        }
+    }
+
+    private external fun redirectStdioToLogcat()
+    private external fun runPython(home: String, runModule: String)
 }
diff --git a/Android/testbed/app/src/main/python/main.py b/Android/testbed/app/src/main/python/main.py
@@ -0,0 +1,17 @@
+import runpy
+import signal
+import sys
+
+# Some tests use SIGUSR1, but that's blocked by default in an Android app in
+# order to make it available to `sigwait` in the "Signal Catcher" thread. That
+# thread's functionality is only relevant to the JVM ("forcing GC (no HPROF) and
+# profile save"), so disabling it should not weaken the tests.
+signal.pthread_sigmask(signal.SIG_UNBLOCK, [signal.SIGUSR1])
+
+# To run specific tests, or pass any other arguments to the test suite, edit
+# this command line.
+sys.argv[1:] = [
+    "--use", "all,-cpu",
+    "--verbose3",
+]
+runpy.run_module("test")
diff --git a/Android/testbed/app/src/main/res/layout/activity_main.xml b/Android/testbed/app/src/main/res/layout/activity_main.xml
@@ -7,6 +7,7 @@
     tools:context=".MainActivity">
 
     <TextView
+        android:id="@+id/tvHello"
         android:layout_width="wrap_content"
         android:layout_height="wrap_content"
         android:text="Hello World!"