run-test: Create runner scripts as part of the build.

The generated zip file includes json file which includes
the commands needed to execute the run-tests as bash script.
The CI just needs to push the data to the device and run the script.
This is just the first step, the scripts are currently unused.

This is implemented by running testrunner.py from the soong build step,
with ART_TEST_RUN_FROM_SOONG set so that the script can adjust behaviour
accordingly (adb is not available; nor is the full Android source tree).

For the time being, this generates arm-specific script,
although it should be easy to make the scrip arch-agnostic.

Test: The commands are exactly identical as during test.py execution.

Change-Id: I8427dd68ffab0815f23034920860f2616755d8f8

6 files changed, 153 insertions, 73 deletions

diff --git a/test/Android.run-test.bp b/test/Android.run-test.bp
index 25594ffbd8..d27153ae11 100644
--- a/test/Android.run-test.bp
+++ b/test/Android.run-test.bp
@@ -2659,6 +2659,11 @@ genrule_defaults {
         "988-method-trace/trace_fib.cc",
         "1953-pop-frame/src/art/Test1953.java",
         "1953-pop-frame/src/art/SuspendEvents.java",
+        // Files needed to generate runner scripts.
+        "testrunner/*.py",
+        "knownfailures.json",
+        "default_run.py",
+        "run-test",
     ],
     tools: [
         "android-smali",
@@ -5775,6 +5780,11 @@ genrule_defaults {
         "988-method-trace/trace_fib.cc",
         "1953-pop-frame/src/art/Test1953.java",
         "1953-pop-frame/src/art/SuspendEvents.java",
+        // Files needed to generate runner scripts.
+        "testrunner/*.py",
+        "knownfailures.json",
+        "default_run.py",
+        "run-test",
     ],
     tools: [
         "android-smali",
@@ -8891,6 +8901,11 @@ genrule_defaults {
         "988-method-trace/trace_fib.cc",
         "1953-pop-frame/src/art/Test1953.java",
         "1953-pop-frame/src/art/SuspendEvents.java",
+        // Files needed to generate runner scripts.
+        "testrunner/*.py",
+        "knownfailures.json",
+        "default_run.py",
+        "run-test",
     ],
     tools: [
         "android-smali",
diff --git a/test/Android.run-test.bp.py b/test/Android.run-test.bp.py
index ef422b1a0b..e1a3707a2b 100755
--- a/test/Android.run-test.bp.py
+++ b/test/Android.run-test.bp.py
@@ -131,6 +131,11 @@ def main():
                 "988-method-trace/trace_fib.cc",
                 "1953-pop-frame/src/art/Test1953.java",
                 "1953-pop-frame/src/art/SuspendEvents.java",
+                // Files needed to generate runner scripts.
+                "testrunner/*.py",
+                "knownfailures.json",
+                "default_run.py",
+                "run-test",
             ],
             tools: [
                 "android-smali",
diff --git a/test/run-test b/test/run-test
index 3cf5bd820d..c9609ca4e7 100755
--- a/test/run-test
+++ b/test/run-test
@@ -24,10 +24,11 @@ from default_run import get_target_arch
 from importlib.machinery import SourceFileLoader
 from inspect import currentframe, getframeinfo, FrameInfo
 from pathlib import Path
-from shutil import copyfile
+from shutil import copyfile, copytree
 from testrunner import env
 from typing import Optional, Dict, List
 from zipfile import ZipFile
+from hashlib import sha1
 
 COLOR = (os.environ.get("LUCI_CONTEXT") == None)  # Disable colors on LUCI.
 COLOR_BLUE = '\033[94m' if COLOR else ''
@@ -107,10 +108,9 @@ if True:
   os.chdir(progdir)
   test_dir = "test-{}".format(os.getpid())
   TMPDIR = os.environ.get("TMPDIR")
-  USER = os.environ.get("USER")
   PYTHON3 = os.environ.get("PYTHON3")
   if not TMPDIR:
-    tmp_dir = f"/tmp/{USER}/{test_dir}"
+    tmp_dir = f"/tmp/art/{test_dir}"
   else:
     tmp_dir = f"{TMPDIR}/{test_dir}"
   checker = f"{progdir}/../tools/checker/checker.py"
@@ -336,7 +336,7 @@ if True:
   argp.add_argument("--random-profile", action='store_true')
   argp.add_argument("--dex2oat-jobs", type=int,
                     help="Number of dex2oat jobs.")
-  argp.add_argument("--create-runner", action='store_true',
+  argp.add_argument("--create-runner", type=Path, metavar='output_dir',
                     help="Creates a runner script for use with other tools.")
   argp.add_argument("--dev", action='store_true',
                     help="Development mode (dumps to stdout).")
@@ -691,10 +691,6 @@ if True:
       sys.exit(1)
     run_args += ["--no-image"]
 
-  if create_runner and target_mode:
-    error("--create-runner does not function for non --host tests")
-    usage = True
-
   if dev_mode and update_mode:
     error("--dev and --update are mutually exclusive")
     usage = True
@@ -761,26 +757,38 @@ if True:
   resource.setrlimit(resource.RLIMIT_FSIZE, (file_ulimit * 1024, resource.RLIM_INFINITY))
 
   # Extract run-test data from the zip file.
-  shutil.rmtree(tmp_dir)
-  os.makedirs(f"{tmp_dir}/.unzipped")
-  os.chdir(tmp_dir)
-  m = re.match("[0-9]*([0-9][0-9])-.*", TEST_NAME)
-  assert m, "Can not find test number in " + TEST_NAME
-  SHARD = "HiddenApi" if "hiddenapi" in TEST_NAME else m.group(1)
-  if target_mode:
-    zip_file = f"{ANDROID_HOST_OUT}/etc/art/art-run-test-target-data-shard{SHARD}.zip"
-    zip_entry = f"target/{TEST_NAME}/"
-  elif runtime == "jvm":
-    zip_file = f"{ANDROID_HOST_OUT}/etc/art/art-run-test-jvm-data-shard{SHARD}.zip"
-    zip_entry = f"jvm/{TEST_NAME}/"
-  else:
-    zip_file = f"{ANDROID_HOST_OUT}/etc/art/art-run-test-host-data-shard{SHARD}.zip"
-    zip_entry = f"host/{TEST_NAME}/"
-  zip = ZipFile(zip_file, "r")
-  zip_entries = [e for e in zip.namelist() if e.startswith(zip_entry)]
-  zip.extractall(Path(tmp_dir) / ".unzipped", members=zip_entries)
-  for entry in (Path(tmp_dir) / ".unzipped" / zip_entry).iterdir():
-    entry.rename(Path(tmp_dir) / entry.name)
+  def unzip():
+    shutil.rmtree(tmp_dir)
+    if env.ART_TEST_RUN_FROM_SOONG:
+      # We already have the unzipped copy of the data.
+      assert target_mode
+      src = Path(ANDROID_BUILD_TOP) / "out" / "zip" / "target" / TEST_NAME
+      assert src.exists(), src
+      copytree(src, tmp_dir)
+      os.chdir(tmp_dir)
+      return
+    os.makedirs(f"{tmp_dir}/.unzipped")
+    os.chdir(tmp_dir)
+    m = re.match("[0-9]*([0-9][0-9])-.*", TEST_NAME)
+    assert m, "Can not find test number in " + TEST_NAME
+    SHARD = "HiddenApi" if "hiddenapi" in TEST_NAME else m.group(1)
+    zip_dir = f"{ANDROID_HOST_OUT}/etc/art"
+    if target_mode:
+      zip_file = f"{zip_dir}/art-run-test-target-data-shard{SHARD}.zip"
+      zip_entry = f"target/{TEST_NAME}/"
+    elif runtime == "jvm":
+      zip_file = f"{zip_dir}/art-run-test-jvm-data-shard{SHARD}.zip"
+      zip_entry = f"jvm/{TEST_NAME}/"
+    else:
+      zip_file = f"{zip_dir}/art-run-test-host-data-shard{SHARD}.zip"
+      zip_entry = f"host/{TEST_NAME}/"
+    zip = ZipFile(zip_file, "r")
+    zip_entries = [e for e in zip.namelist() if e.startswith(zip_entry)]
+    zip.extractall(Path(tmp_dir) / ".unzipped", members=zip_entries)
+    for entry in (Path(tmp_dir) / ".unzipped" / zip_entry).iterdir():
+      entry.rename(Path(tmp_dir) / entry.name)
+
+  unzip()
 
   def clean_up(passed: bool):
     if always_clean or (passed and not never_clean):
@@ -797,8 +805,6 @@ if True:
       print(f"{TEST_NAME} files left in {tmp_dir} on host" +
             (f" and in {chroot_dex_location} on target" if target_mode else ""))
     atexit.unregister(clean_up)
-  # TODO: Run this in global try-finally once the script is more refactored.
-  atexit.register(clean_up, passed=False)
 
   ctx = RunTestContext(Path(tmp_dir), target_mode, chroot, DEX_LOCATION, TEST_NAME)
   td_info = f"{test_dir}/{info}"
@@ -850,19 +856,33 @@ if True:
   print(f"{test_dir}: Create runner script...")
   runner = create_runner_script()
 
+  if args.create_runner:
+    # TODO: Generate better unique names.
+    name = [a for a in sys.argv[1:] if not a.startswith("--create-runner")]
+    hash = sha1((" ".join(name)).encode()).digest().hex()[:8]
+    dst = args.create_runner / f"{TEST_NAME}-{hash}"
+    assert not dst.exists(), dst
+    copyfile(runner, dst)
+
   # Script debugging feature - just export the runner script into a directory,
   # so that it can be compared before/after runner script refactoring.
   save_runner_dir = os.environ.get("RUN_TEST_DEBUG__SAVE_RUNNER_DIR")
   if save_runner_dir:
-    name = urllib.parse.quote(" ".join(sys.argv[1:]), safe=' ')
+    name = [a for a in sys.argv[1:] if not a.startswith("--create-runner")]
+    name = urllib.parse.quote(" ".join(name), safe=' ')
     dst = Path(save_runner_dir) / TEST_NAME / name
     os.makedirs(dst.parent, exist_ok=True)
     txt = runner.read_text()
     txt = txt.replace(Path(tmp_dir).name, "${TMP_DIR}")  # Make it deterministic.
     txt = re.sub('\[run-test:\d+\]', '[run-test:(line-number)]', txt)
     dst.write_text(txt)
+
+  if args.create_runner or save_runner_dir:
     sys.exit(0)
 
+  # TODO: Run this in global try-finally once the script is more refactored.
+  atexit.register(clean_up, passed=False)
+
   print(f"{test_dir}: Run...")
   if target_mode:
     # Prepare the on-device test directory
diff --git a/test/run_test_build.py b/test/run_test_build.py
index ca4634d5ce..c795b7df93 100755
--- a/test/run_test_build.py
+++ b/test/run_test_build.py
@@ -19,14 +19,11 @@ This scripts compiles Java files which are needed to execute run-tests.
 It is intended to be used only from soong genrule.
 """
 
-import argparse
 import functools
-import glob
+import json
 import os
 import pathlib
 import re
-import shlex
-import shutil
 import subprocess
 import sys
 import zipfile
@@ -35,15 +32,15 @@ from argparse import ArgumentParser
 from concurrent.futures import ThreadPoolExecutor
 from fcntl import lockf, LOCK_EX, LOCK_NB
 from importlib.machinery import SourceFileLoader
-from os import environ, getcwd, chdir, cpu_count, chmod
+from os import environ, getcwd, cpu_count
 from os.path import relpath
 from pathlib import Path
 from pprint import pprint
-from re import match
 from shutil import copytree, rmtree
-from subprocess import run
+from subprocess import PIPE, run
 from tempfile import TemporaryDirectory, NamedTemporaryFile
 from typing import Dict, List, Union, Set, Optional
+from multiprocessing import cpu_count
 
 USE_RBE = 100  # Percentage of tests that can use RBE (between 0 and 100)
 
@@ -513,6 +510,30 @@ class BuildTestContext:
       else:
         zip(Path(self.test_name + ".jar"), Path("classes.dex"))
 
+# Create bash scripts that can fully execute the run tests.
+# This can be used in CI to execute the tests without running `testrunner.py`.
+# This takes into account any custom behaviour defined in per-test `run.py`.
+# We generate distinct scripts for all of the pre-defined variants.
+def create_ci_runner_scripts(mode, test_names):
+  with TemporaryDirectory() as tmpdir:
+    python = sys.executable
+    script = 'art/test/testrunner/testrunner.py'
+    envs = {
+      "ANDROID_BUILD_TOP": str(Path(getcwd()).absolute()),
+      "ART_TEST_RUN_FROM_SOONG": "true",
+      # TODO: Make the runner scripts target agnostic.
+      #       The only dependency is setting of "-Djava.library.path".
+      "TARGET_ARCH": "arm64",
+      "TARGET_2ND_ARCH": "arm",
+    }
+    args = [
+      f"--run-test-option=--create-runner={tmpdir}",
+      f"-j={cpu_count()}",
+      f"--{mode}",
+    ]
+    run([python, script] + args + test_names, env=envs, check=True)
+    runners = {r.name: r.read_text().split("\n") for r in Path(tmpdir).glob("*")}
+    return [{"name": name, "runner": bash} for name, bash in runners.items()]
 
 # If we build just individual shard, we want to split the work among all the cores,
 # but if the build system builds all shards, we don't want to overload the machine.
@@ -550,11 +571,7 @@ def main() -> None:
   android_build_top = Path(getcwd()).absolute()
   ziproot = args.out.absolute().parent / "zip"
   test_dir_regex = re.compile(args.test_dir_regex) if args.test_dir_regex else re.compile(".*")
-  srcdirs = set(
-    s.parents[-4].absolute()
-    for s in args.srcs
-    if test_dir_regex.search(str(s))
-  )
+  srcdirs = set(s.parents[-4].absolute() for s in args.srcs if test_dir_regex.search(str(s)))
 
   # Special hidden-api shard: If the --hiddenapi flag is provided, build only
   # hiddenapi tests. Otherwise exclude all hiddenapi tests from normal shards.
@@ -575,19 +592,26 @@ def main() -> None:
     os.chdir(invalid_tmpdir)
     os.chmod(invalid_tmpdir, 0)
     with ThreadPoolExecutor(cpu_count() if use_multiprocessing(args.mode) else 1) as pool:
-      jobs = {}
-      for ctx in tests:
-        jobs[ctx.test_name] = pool.submit(ctx.build)
+      jobs = {ctx.test_name: pool.submit(ctx.build) for ctx in tests}
       for test_name, job in jobs.items():
         try:
           job.result()
         except Exception as e:
           raise Exception("Failed to build " + test_name) from e
 
-  # Create the final zip file which contains the content of the temporary directory.
-  proc = run([android_build_top / args.soong_zip, "-o", android_build_top / args.out,
-              "-C", ziproot, "-D", ziproot], check=True)
+  if args.mode == "target":
+    os.chdir(android_build_top)
+    test_names = [ctx.test_name for ctx in tests]
+    data = create_ci_runner_scripts(args.mode, test_names)
+    dst = ziproot / "runner" / args.out.with_suffix(".json").name
+    dst.parent.mkdir(parents=True)
+    text = json.dumps(data, ensure_ascii=False, indent=2)
+    Path(dst).write_text(text)
 
+  # Create the final zip file which contains the content of the temporary directory.
+  soong_zip = android_build_top / args.soong_zip
+  zip_file = android_build_top / args.out
+  run([soong_zip, "-o", zip_file, "-C", ziproot, "-D", ziproot], check=True)
 
 if __name__ == "__main__":
   main()
diff --git a/test/testrunner/env.py b/test/testrunner/env.py
index de24b4cde6..e3fd9b11ae 100644
--- a/test/testrunner/env.py
+++ b/test/testrunner/env.py
@@ -17,17 +17,21 @@ import re
 import tempfile
 import subprocess
 
-# begin import $ANDROID_BUILD_TOP/art/tools/build/var_cache.py
-_THIS_DIR = os.path.dirname(os.path.realpath(__file__))
-_TOP = os.path.join(_THIS_DIR, "../../..")
-_VAR_CACHE_DIR = os.path.join(_TOP, "art/tools/build/")
+_env = dict(os.environ)
 
-import sys
-sys.path.append(_VAR_CACHE_DIR)
-import var_cache  # type: ignore
-# end import var_cache.py
+# Check if we are running from the build system.
+ART_TEST_RUN_FROM_SOONG = os.environ.get("ART_TEST_RUN_FROM_SOONG")
 
-_env = dict(os.environ)
+if not ART_TEST_RUN_FROM_SOONG:
+  # begin import $ANDROID_BUILD_TOP/art/tools/build/var_cache.py
+  _THIS_DIR = os.path.dirname(os.path.realpath(__file__))
+  _TOP = os.path.join(_THIS_DIR, "../../..")
+  _VAR_CACHE_DIR = os.path.join(_TOP, "art/tools/build/")
+
+  import sys
+  sys.path.append(_VAR_CACHE_DIR)
+  import var_cache  # type: ignore
+  # end import var_cache.py
 
 def _getEnvBoolean(var, default):
   val = _env.get(var)
@@ -39,6 +43,8 @@ def _getEnvBoolean(var, default):
   return default
 
 def _get_build_var(var_name):
+  if ART_TEST_RUN_FROM_SOONG:
+    return _env.get(var_name)
   return var_cache.get_build_var(var_name)
 
 def _get_build_var_boolean(var, default):
@@ -62,7 +68,8 @@ def _get_android_build_top():
     path_to_top = os.path.realpath(path_to_top)
 
   if not os.path.exists(os.path.join(path_to_top, 'build/envsetup.sh')):
-    raise AssertionError("env.py must be located inside an android source tree")
+    if not ART_TEST_RUN_FROM_SOONG:
+      raise AssertionError("env.py must be located inside an android source tree")
 
   return path_to_top
 
@@ -129,18 +136,18 @@ else:
   ART_PHONY_TEST_HOST_SUFFIX = "64"
   ART_2ND_PHONY_TEST_HOST_SUFFIX = "32"
 
-HOST_OUT_EXECUTABLES = os.path.join(ANDROID_BUILD_TOP,
-                                    _get_build_var("HOST_OUT_EXECUTABLES"))
+if HOST_OUT_EXECUTABLES := _get_build_var("HOST_OUT_EXECUTABLES"):
+  HOST_OUT_EXECUTABLES = os.path.join(ANDROID_BUILD_TOP, HOST_OUT_EXECUTABLES)
 
-# Set up default values for $D8, $SMALI, etc to the $HOST_OUT_EXECUTABLES/$name path.
-for tool in ['smali', 'jasmin', 'd8']:
-  os.environ.setdefault(tool.upper(), HOST_OUT_EXECUTABLES + '/' + tool)
+  # Set up default values for $D8, $SMALI, etc to the $HOST_OUT_EXECUTABLES/$name path.
+  for tool in ['smali', 'jasmin', 'd8']:
+    os.environ.setdefault(tool.upper(), HOST_OUT_EXECUTABLES + '/' + tool)
 
-ANDROID_JAVA_TOOLCHAIN = os.path.join(ANDROID_BUILD_TOP,
-                                     _get_build_var('ANDROID_JAVA_TOOLCHAIN'))
+if ANDROID_JAVA_TOOLCHAIN := _get_build_var('ANDROID_JAVA_TOOLCHAIN'):
+  ANDROID_JAVA_TOOLCHAIN = os.path.join(ANDROID_BUILD_TOP, ANDROID_JAVA_TOOLCHAIN)
 
-# include platform prebuilt java, javac, etc in $PATH.
-os.environ['PATH'] = ANDROID_JAVA_TOOLCHAIN + ':' + os.environ['PATH']
+  # include platform prebuilt java, javac, etc in $PATH.
+  os.environ['PATH'] = ANDROID_JAVA_TOOLCHAIN + ':' + os.environ['PATH']
 
 DIST_DIR = _get_build_var('DIST_DIR')
 SOONG_OUT_DIR = _get_build_var('SOONG_OUT_DIR')
diff --git a/test/testrunner/testrunner.py b/test/testrunner/testrunner.py
index 7971b2ed8b..5892345067 100755
--- a/test/testrunner/testrunner.py
+++ b/test/testrunner/testrunner.py
@@ -82,6 +82,7 @@ from target_config import target_config
 from device_config import device_config
 from typing import Dict, Set, List
 from functools import lru_cache
+from pathlib import Path
 
 # TODO: make it adjustable per tests and for buildbots
 #
@@ -351,6 +352,8 @@ def get_device_name():
   """
   Gets the value of ro.product.name from remote device (unless running on a VM).
   """
+  if env.ART_TEST_RUN_FROM_SOONG:
+    return "target"  # We can't use adb during build.
   if env.ART_TEST_ON_VM:
     return subprocess.Popen(f"{env.ART_SSH_CMD} uname -a".split(),
                             stdout = subprocess.PIPE,
@@ -570,7 +573,9 @@ def run_tests(tests):
 
       # Run the run-test script using the prebuilt python.
       python3_bin = env.ANDROID_BUILD_TOP + "/prebuilts/build-tools/path/linux-x86/python3"
-      run_test_sh = env.ANDROID_BUILD_TOP + '/art/test/run-test'
+      run_test_sh = str(Path(__file__).parent.parent / 'run-test')
+      if not os.path.exists(python3_bin):
+        python3_bin = sys.executable  # Fallback to current python if we are in a sandbox.
       args_test = [python3_bin, run_test_sh] + args_test + extra_arguments[target] + [test]
       return executor.submit(run_test, args_test, test, variant_set, test_name)
 
@@ -815,7 +820,7 @@ def get_disabled_test_info(device_name):
     The method returns a dict of tests mapped to the variants list
     for which the test should not be run.
   """
-  known_failures_file = env.ANDROID_BUILD_TOP + '/art/test/knownfailures.json'
+  known_failures_file = Path(__file__).parent.parent / 'knownfailures.json'
   with open(known_failures_file) as known_failures_json:
     known_failures_info = json.loads(known_failures_json.read())
 
@@ -849,6 +854,8 @@ def get_disabled_test_info(device_name):
     if check_env_vars(env_vars):
       for test in tests:
         if test not in RUN_TEST_SET:
+          if env.ART_TEST_RUN_FROM_SOONG:
+            continue  # Soong can see only sub-set of the tests within the shard.
           raise ValueError('%s is not a valid run-test' % (
               test))
         if test in disabled_test_info:
@@ -931,7 +938,9 @@ def parse_variants(variants):
     variant_list.add(frozenset(variant))
   return variant_list
 
-def print_text(output):
+def print_text(output, error=False):
+  if env.ART_TEST_RUN_FROM_SOONG and not error:
+    return  # Be quiet during build.
   sys.stdout.write(output)
   sys.stdout.flush()
 
@@ -962,9 +971,9 @@ def print_analysis():
 
   # Prints the list of failed tests, if any.
   if failed_tests:
-    print_text(COLOR_ERROR + 'FAILED: ' + COLOR_NORMAL + '\n')
+    print_text(COLOR_ERROR + 'FAILED: ' + COLOR_NORMAL + '\n', error=True)
     for test_info in failed_tests:
-      print_text(('%s\n%s\n' % (test_info[0], test_info[1])))
+      print_text(('%s\n%s\n' % (test_info[0], test_info[1])), error=True)
     print_text(COLOR_ERROR + '----------' + COLOR_NORMAL + '\n')
     for failed_test in sorted([test_info[0] for test_info in failed_tests]):
       print_text(('%s\n' % (failed_test)))