[Android] Add testing support

This adds support for running tests for the stdlib built
for Android, both on the host machine and on an Android device.

The Android variant of Swift may be built and tested using
the following `build-script` invocation:

```
$ utils/build-script \
  -R \                                           # Build in ReleaseAssert mode.
  -T \                                           # Run all tests.
  --android \                                    # Build for Android.
  --android-deploy-device-path /data/local/tmp \ # Temporary directory on the device where Android tests are run.
  --android-ndk ~/android-ndk-r10e \             # Path to an Android NDK.
  --android-ndk-version 21 \
  --android-icu-uc ~/libicu-android/armeabi-v7a/libicuuc.so \
  --android-icu-uc-include ~/libicu-android/armeabi-v7a/icu/source/common \
  --android-icu-i18n ~/libicu-android/armeabi-v7a/libicui18n.so \
  --android-icu-i18n-include ~/libicu-android/armeabi-v7a/icu/source/i18n/
```

See docs/Testing.rst for more details.

Author: modocache

CMakeLists.txt

@@ -160,6 +160,8 @@ set(SWIFT_ANDROID_ICU_I18N "" CACHE STRING
   "Path to a directory containing libicui18n.so")
 set(SWIFT_ANDROID_ICU_I18N_INCLUDE "" CACHE STRING
   "Path to a directory containing headers libicui18n")
+set(SWIFT_ANDROID_DEPLOY_DEVICE_PATH "" CACHE STRING
+  "Path on an Android device where build products will be pushed. These are used when running the test suite against the device")
 
 #
 # User-configurable Darwin-specific options.

docs/Android.md

@@ -2,7 +2,10 @@
 
 The Swift stdlib can be compiled for Android armv7 targets, which makes it
 possible to execute Swift code on a mobile device running Android. This guide
-explains how to run a simple "Hello, world" program on your Android device.
+explains:
+
+1. How to run a simple "Hello, world" program on your Android device.
+2. How to run the Swift test suite, targeting Android, and on an Android device.
 
 If you encounter any problems following the instructions below, please file a
 bug using https://bugs.swift.org/.
@@ -39,7 +42,7 @@ To follow along with this guide, you'll need:
    turn on remote debugging by following the official instructions:
    https://developer.chrome.com/devtools/docs/remote-debugging.
 
-## "Hello, world" on Android
+## Part One: "Hello, world" on Android
 
 ### 1. Downloading (or building) the Swift Android stdlib dependencies
 
@@ -171,3 +174,28 @@ Hello, Android
 
 Congratulations! You've just run your first Swift program on Android.
 
+## Part Two: Running the Swift test suite hosted on an Android device
+
+When running the test suite, build products are automatically pushed to your
+device. As in part one, you'll need to connect your Android device via USB:
+
+1. Connect your Android device to your computer via USB. Ensure that remote
+   debugging is enabled for that device by following the official instructions:
+   https://developer.chrome.com/devtools/docs/remote-debugging.
+2. Confirm the device is connected by running `adb devices`. You should see
+   your device listed.
+3. Run the tests using the build script:
+
+```
+$ utils/build-script \
+  -R \                                           # Build in ReleaseAssert mode.
+  -T \                                           # Run all tests.
+  --android \                                    # Build for Android.
+  --android-deploy-device-path /data/local/tmp \ # Temporary directory on the device where Android tests are run.
+  --android-ndk ~/android-ndk-r10e \             # Path to an Android NDK.
+  --android-ndk-version 21 \
+  --android-icu-uc ~/libicu-android/armeabi-v7a/libicuuc.so \
+  --android-icu-uc-include ~/libicu-android/armeabi-v7a/icu/source/common \
+  --android-icu-i18n ~/libicu-android/armeabi-v7a/libicui18n.so \
+  --android-icu-i18n-include ~/libicu-android/armeabi-v7a/icu/source/i18n/
+```

test/1_stdlib/InputStream.swift.gyb

@@ -16,6 +16,10 @@
 // RUN: %S/../../utils/line-directive %t/InputStream.swift -- %target-run %t/a.out
 // REQUIRES: executable_test
 
+// FIXME: The Android test runner is incapable of running this test, which
+//        relies on stdin input.
+// UNSUPPORTED: OS=linux-androideabi
+
 import StdlibUnittest
 
 

test/CMakeLists.txt

@@ -212,7 +212,29 @@ if(PYTHONINTERP_FOUND)
 
         set(command_upload_stdlib)
         if("${SDK}" STREQUAL "IOS" OR "${SDK}" STREQUAL "TVOS" OR "${SDK}" STREQUAL "WATCHOS")
-          # These are supported testing SDKs.
+          # These are supported testing SDKs, but their implementation of
+          # `command_upload_stdlib` is hidden.
+        elseif("${SDK}" STREQUAL "ANDROID")
+          # Warning: This step will fail if you do not have an Android device
+          #          connected via USB. See docs/Android.md for details on
+          #          how to run the test suite for Android.
+          set(command_upload_stdlib
+              COMMAND
+                  # Reboot the device and remove everything in its tmp
+                  # directory. Build products and test executables are pushed
+                  # to that directory when running the test suite.
+                  ${PYTHON_EXECUTABLE} "${SWIFT_SOURCE_DIR}/utils/android/adb_clean.py"
+              COMMAND
+                  ${PYTHON_EXECUTABLE} "${SWIFT_SOURCE_DIR}/utils/android/adb_push_built_products.py"
+                  --ndk "${SWIFT_ANDROID_NDK_PATH}"
+                  --destination "${SWIFT_ANDROID_DEPLOY_DEVICE_PATH}"
+                  # Build products like libswiftCore.so.
+                  "${SWIFTLIB_DIR}/android"
+                  # These two directories may contain the same libraries,
+                  # but upload both to device just in case. Duplicates will be
+                  # overwritten, and uploading doesn't take very long anyway.
+                  "${SWIFT_ANDROID_ICU_UC}"
+                  "${SWIFT_ANDROID_ICU_I18N}")
         endif()
 
         foreach(test_subset ${TEST_SUBSETS})

test/lit.cfg

@@ -741,6 +741,69 @@ elif run_os == 'linux-gnu' or run_os == 'linux-gnueabihf' or run_os == 'freebsd'
     config.target_ld = (
         "ld -L%s" %
         (os.path.join(test_resource_dir, config.target_sdk_name)))
+elif run_os == 'linux-androideabi':
+    lit_config.note("Testing Android " + config.variant_triple)
+    config.target_object_format = "elf"
+    config.target_dylib_extension = "so"
+    config.target_runtime = "native"
+    config.target_swift_autolink_extract = inferSwiftBinary("swift-autolink-extract")
+    config.target_sdk_name = "android"
+    android_linker_opt = "-L {libcxx} -L {libgcc}".format(
+        libcxx=os.path.join(config.android_ndk_path,
+                            "sources", "cxx-stl", "llvm-libc++", "libs",
+                            "armeabi-v7a"),
+        libgcc=os.path.join(config.android_ndk_path,
+                            "toolchains",
+                            "arm-linux-androideabi-{}".format(
+                                config.android_ndk_gcc_version),
+                            "prebuilt", "linux-x86_64", "lib", "gcc",
+                            "arm-linux-androideabi",
+                            config.android_ndk_gcc_version))
+    config.target_build_swift = (
+        '%s -target %s -sdk %s %s -Xlinker -pie %s %s %s %s'
+        % (config.swiftc, config.variant_triple, config.variant_sdk,
+           android_linker_opt, resource_dir_opt, mcp_opt,
+           config.swift_test_options, swift_execution_tests_extra_flags))
+    config.target_swift_frontend = (
+        '%s -frontend -target %s -sdk %s %s %s'
+        % (config.swift, config.variant_triple, config.variant_sdk,
+           android_linker_opt, resource_dir_opt))
+    subst_target_swift_frontend_mock_sdk = config.target_swift_frontend
+    subst_target_swift_frontend_mock_sdk_after = ""
+    config.target_run = os.path.join(
+        config.swift_src_root, 'utils', 'android', 'adb_test_runner.py')
+    # FIXME: Include -sdk in this invocation.
+    config.target_sil_opt = (
+        '%s -target %s %s %s' %
+        (config.sil_opt, config.variant_triple, resource_dir_opt, mcp_opt))
+    config.target_swift_ide_test = (
+        '%s -target %s %s %s %s' %
+        (config.swift_ide_test, config.variant_triple, resource_dir_opt,
+         mcp_opt, ccp_opt))
+    subst_target_swift_ide_test_mock_sdk = config.target_swift_ide_test
+    subst_target_swift_ide_test_mock_sdk_after = ""
+    config.target_swiftc_driver = (
+        "%s -target %s -sdk %s %s %s %s" %
+        (config.swiftc, config.variant_triple, config.variant_sdk,
+         android_linker_opt, resource_dir_opt, mcp_opt))
+    config.target_swift_modulewrap = (
+        '%s -modulewrap -target %s' %
+        (config.swiftc, config.variant_triple))
+    config.target_clang = (
+        "clang++ -target %s %s" %
+        (config.variant_triple, clang_mcp_opt))
+    config.target_ld = "{} -L{}".format(
+        os.path.join(
+            config.android_ndk_path,
+            'toolchains',
+            'arm-linux-androideabi-{}'.format(config.android_ndk_gcc_version),
+            'prebuilt',
+            'linux-x86_64',
+            'arm-linux-androideabi',
+            'bin'),
+        os.path.join(test_resource_dir, config.target_sdk_name))
+    # The Swift interpreter is not available when targeting Android.
+    config.available_features.remove('swift_interpreter')
 
 else:
     lit_config.fatal("Don't know how to define target_run and "

test/lit.site.cfg.in

@@ -18,6 +18,8 @@ config.variant_sdk = "@VARIANT_SDK@"
 config.variant_suffix = "@VARIANT_SUFFIX@"
 config.swiftlib_dir = "@LIT_SWIFTLIB_DIR@"
 config.darwin_xcrun_toolchain = "@SWIFT_DARWIN_XCRUN_TOOLCHAIN@"
+config.android_ndk_path = "@SWIFT_ANDROID_NDK_PATH@"
+config.android_ndk_gcc_version = "@SWIFT_ANDROID_NDK_GCC_VERSION@"
 
 config.coverage_mode = "@SWIFT_ANALYZE_CODE_COVERAGE@"
 

utils/android/adb/__init__.py

@@ -0,0 +1,152 @@
+# adb/commands.py - Run executables on an Android device -*- python -*-
+#
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See http://swift.org/LICENSE.txt for license information
+# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+#
+# ----------------------------------------------------------------------------
+#
+# Push executables to an Android device and run them, capturing their output
+# and exit code.
+#
+# ----------------------------------------------------------------------------
+
+from __future__ import print_function
+
+import subprocess
+import tempfile
+import uuid
+
+
+# A temporary directory on the Android device.
+DEVICE_TEMP_DIR = '/data/local/tmp'
+
+
+def shell(args):
+    """
+    Execute 'adb shell' with the given arguments.
+
+    Raise an exception if 'adb shell' returns a non-zero exit code.
+    Note that this only occurs if communication with the connected device
+    fails, not if the command run on the device fails.
+    """
+    return subprocess.check_output(['adb', 'shell'] + args)
+
+
+def rmdir(path):
+    """Remove all files in the device directory at `path`."""
+    shell(['rm', '-rf', '{}/*'.format(path)])
+
+
+def push(local_path, device_path):
+    """Move the file at the given local path to the path on the device."""
+    return subprocess.check_output(['adb', 'push', local_path, device_path],
+                                   stderr=subprocess.STDOUT).strip()
+
+
+def reboot():
+    """Reboot the connected Android device, waiting for it to return online."""
+    subprocess.check_call(['adb', 'reboot'])
+    subprocess.check_call(['adb', 'wait-for-device'])
+
+
+def _create_executable_on_device(device_path, contents):
+    _, tmp = tempfile.mkstemp()
+    with open(tmp, 'w') as f:
+        f.write(contents)
+    push(tmp, device_path)
+    shell(['chmod', '755', device_path])
+
+
+def execute_on_device(executable_path, executable_arguments):
+    """
+    Run an executable on an Android device.
+
+    Push an executable at the given 'executable_path' to an Android device,
+    then execute that executable on the device, passing any additional
+    'executable_arguments'. Return 0 if the executable succeeded when run on
+    device, and 1 otherwise.
+
+    This function is not as simple as calling 'adb shell', for two reasons:
+
+    1. 'adb shell' can only take input up to a certain length, so it fails for
+       long executable names or when a large amount of arguments are passed to
+       the executable. This function attempts to limit the size of any string
+       passed to 'adb shell'.
+    2. 'adb shell' ignores the exit code of any command it runs. This function
+       therefore uses its own mechanisms to determine whether the executable
+       had a successful exit code when run on device.
+    """
+    # We'll be running the executable in a temporary directory in
+    # /data/local/tmp. `adb shell` has trouble with commands that
+    # exceed a certain length, so to err on the safe side we only
+    # use the first 10 characters of the UUID.
+    uuid_dir = '{}/{}'.format(DEVICE_TEMP_DIR, str(uuid.uuid4())[:10])
+    shell(['mkdir', '-p', uuid_dir])
+
+    # `adb` can only handle commands under a certain length. No matter what the
+    # original executable's name, on device we call it `__executable`.
+    executable = '{}/__executable'.format(uuid_dir)
+    push(executable_path, executable)
+
+    # When running the executable on the device, we need to pass it the same
+    # arguments, as well as specify the correct LD_LIBRARY_PATH. Save these
+    # to a file we can easily call multiple times.
+    executable_with_args = '{}/__executable_with_args'.format(uuid_dir)
+    _create_executable_on_device(
+        executable_with_args,
+        'LD_LIBRARY_PATH={uuid_dir}:{tmp_dir} '
+        '{executable} {executable_arguments}'.format(
+            uuid_dir=uuid_dir,
+            tmp_dir=DEVICE_TEMP_DIR,
+            executable=executable,
+            executable_arguments=' '.join(executable_arguments)))
+
+    # Write the output from the test executable to a file named '__stdout', and
+    # if the test executable succeeds, write 'SUCCEEDED' to a file
+    # named '__succeeded'. We do this because `adb shell` does not report
+    # the exit code of the command it executes on the device, so instead we
+    # check the '__succeeded' file for our string.
+    executable_stdout = '{}/__stdout'.format(uuid_dir)
+    succeeded_token = 'SUCCEEDED'
+    executable_succeeded = '{}/__succeeded'.format(uuid_dir)
+    executable_piped = '{}/__executable_piped'.format(uuid_dir)
+    _create_executable_on_device(
+        executable_piped,
+        '{executable_with_args} > {executable_stdout} && '
+        'echo "{succeeded_token}" > {executable_succeeded}'.format(
+            executable_with_args=executable_with_args,
+            executable_stdout=executable_stdout,
+            succeeded_token=succeeded_token,
+            executable_succeeded=executable_succeeded))
+
+    # We've pushed everything we need to the device.
+    # Now execute the wrapper script.
+    shell([executable_piped])
+
+    # Grab the results of running the executable on device.
+    stdout = shell(['cat', executable_stdout])
+    exitcode = shell(['cat', executable_succeeded])
+    if not exitcode.startswith(succeeded_token):
+        debug_command = '$ adb shell {}'.format(executable_with_args)
+        print('Executable exited with a non-zero code on the Android device.\n'
+              'Device stdout:\n'
+              '{stdout}\n'
+              'To debug, run:\n'
+              '{debug_command}\n'.format(
+                  stdout=stdout,
+                  debug_command=debug_command))
+
+        # Exit early so that the output isn't passed to FileCheck, nor are any
+        # temporary directories removed; this allows the user to re-run
+        # the executable on the device.
+        return 1
+
+    print(stdout)
+
+    shell(['rm', '-rf', uuid_dir])
+    return 0

utils/android/adb/commands.py

@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+# adb_reboot.py - Reboots and cleans an Android device. -*- python -*-
+#
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See http://swift.org/LICENSE.txt for license information
+# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+
+from adb.commands import DEVICE_TEMP_DIR, reboot, rmdir
+
+
+if __name__ == '__main__':
+    reboot()
+    rmdir(DEVICE_TEMP_DIR)

utils/android/adb_clean.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+# adb_push_build_products.py - Push libraries to Android device -*- python -*-
+#
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See http://swift.org/LICENSE.txt for license information
+# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+
+import sys
+
+from adb_push_built_products.main import main
+
+
+if __name__ == '__main__':
+    sys.exit(main())