testrunner.py: Add --build-only and --skip-build options.

The build stage is more expensive than the test stage
and we do it redundantly repeatedly for all variants.

Make it possible to separate the build and test stages,
which will allow us to do the build just once on buildbots.

The proper solution is to move all the build work to soong,
and we will be able to remove this code again at that point.

Bug: 188631922
Test: testrunner.py --host --build-only && \
      testrunner.py --host --skip-build
Change-Id: I542feac03acc25a853dbf7f1a2e5587a2c5d6d7a

3 files changed, 102 insertions, 42 deletions

diff --git a/test/run-test b/test/run-test
index 9352314686..c968bee460 100755
--- a/test/run-test
+++ b/test/run-test
@@ -39,6 +39,7 @@ if [ -z "$TMPDIR" ]; then
 else
   tmp_dir="${TMPDIR}/${test_dir}"
 fi
+build_path="${tmp_dir}-build"
 checker="${progdir}/../tools/checker/checker.py"
 export JAVA="java"
 export JAVAC="javac -g -Xlint:-options -source 1.8 -target 1.8"
@@ -148,6 +149,7 @@ relocate="no"
 runtime="art"
 usage="no"
 build_only="no"
+skip_build="no"
 suffix64=""
 trace="false"
 trace_stream="false"
@@ -381,11 +383,23 @@ while true; do
     elif [ "x$1" = "x--build-only" ]; then
         build_only="yes"
         shift
-    elif [ "x$1" = "x--output-path" ]; then
+    elif [ "x$1" = "x--skip-build" ]; then
+        skip_build="yes"
+        shift
+    elif [ "x$1" = "x--build-path" ]; then
+        shift
+        build_path=$1
+        if [ "x$build_path" = "x" ]; then
+            echo "$0 missing argument to --build-path" 1>&2
+            usage="yes"
+            break
+        fi
+        shift
+    elif [ "x$1" = "x--temp-path" ]; then
         shift
         tmp_dir=$1
         if [ "x$tmp_dir" = "x" ]; then
-            echo "$0 missing argument to --output-path" 1>&2
+            echo "$0 missing argument to --temp-path" 1>&2
             usage="yes"
             break
         fi
@@ -796,6 +810,9 @@ if [ "$usage" = "yes" ]; then
         echo "                          Use the given binary as gdbserver."
         echo "    --gdb-arg             Pass an option to gdb or gdbserver."
         echo "    --build-only          Build test files only (off by default)."
+        echo "    --skip-build          Assume that test files are already built (off by default)."
+        echo "    --build-path [path]   Location where to store or expect the build files."
+        echo "    --temp-path [path]    Location where to execute the tests."
         echo "    --interpreter         Enable interpreter only mode (off by default)."
         echo "    --jit                 Enable jit (off by default)."
         echo "    --optimizing          Enable optimizing compiler (default)."
@@ -825,8 +842,6 @@ if [ "$usage" = "yes" ]; then
         echo "    --use-java-home       Use the JAVA_HOME environment variable"
         echo "                          to find the java compiler and runtime"
         echo "                          (if applicable) to run the test with."
-        echo "    --output-path [path]  Location where to store the build" \
-             "files."
         echo "    --64                  Run the test in 64-bit mode"
         echo "    --bionic              Use the (host, 64-bit only) linux_bionic libc runtime"
         echo "    --runtime-zipapex [file]"
@@ -885,35 +900,36 @@ for td_file in "$td_info" "$td_expected_stdout" "$td_expected_stderr"; do
     fi
 done
 
-# copy the test to a temp dir and run it
+if [ "$skip_build" = "no" ]; then
+  echo "${test_dir}: building..." 1>&2
 
-echo "${test_dir}: building..." 1>&2
+  rm -rf "$build_path"
+  cp -LRp "$test_dir" "$build_path"
+  cd "$build_path"
 
-rm -rf "$tmp_dir"
-cp -LRp "$test_dir" "$tmp_dir"
-cd "$tmp_dir"
+  if [ '!' -r "$build" ]; then
+      cp "${progdir}/etc/default-build" build
+  else
+      cp "${progdir}/etc/default-build" .
+  fi
 
-if [ '!' -r "$build" ]; then
-    cp "${progdir}/etc/default-build" build
-else
-    cp "${progdir}/etc/default-build" .
-fi
+  if [ '!' -r "$run" ]; then
+      cp "${progdir}/etc/default-run" run
+  else
+      cp "${progdir}/etc/default-run" .
+  fi
 
-if [ '!' -r "$run" ]; then
-    cp "${progdir}/etc/default-run" run
-else
-    cp "${progdir}/etc/default-run" .
-fi
+  if [ '!' -r "$check_cmd" ]; then
+      cp "${progdir}/etc/default-check" check
+  else
+      cp "${progdir}/etc/default-check" .
+  fi
 
-if [ '!' -r "$check_cmd" ]; then
-    cp "${progdir}/etc/default-check" check
-else
-    cp "${progdir}/etc/default-check" .
+  chmod 755 "$build"
+  chmod 755 "$run"
+  chmod 755 "$check_cmd"
 fi
-
-chmod 755 "$build"
-chmod 755 "$run"
-chmod 755 "$check_cmd"
+cd "$build_path"
 
 export TEST_NAME=`basename ${test_dir}`
 
@@ -965,14 +981,30 @@ if [[ "$dev_mode" == "yes" ]]; then
   build_args="$build_args --dev"
 fi
 
+# Build needed files, and copy them to the directory that will run the tests.
+if [ "$skip_build" = "yes" ]; then
+  # Assume the directory already contains all the needed files.
+  # Load the exit code, since builds can intentionally fail.
+  build_exit=`cat ./build_exit_code`
+elif [ "$dev_mode" = "yes" ]; then
+  "./${build}" $build_args
+  build_exit="$?"
+  echo "build exit status: $build_exit" 1>&2
+  echo "$build_exit" > ./build_exit_code
+else
+  "./${build}" $build_args >"$build_stdout" 2>"$build_stderr"
+  build_exit="$?"
+  echo "$build_exit" > ./build_exit_code
+fi
+rm -rf "$tmp_dir"
+cp -LRp "$build_path" "$tmp_dir"
+cd "$tmp_dir"
+
 good="no"
 good_build="yes"
 good_run="yes"
 export TEST_RUNTIME="${runtime}"
 if [ "$dev_mode" = "yes" ]; then
-    "./${build}" $build_args
-    build_exit="$?"
-    echo "build exit status: $build_exit" 1>&2
     if [ "$build_exit" = '0' ]; then
         echo "${test_dir}: running..." 1>&2
         "./${run}" "${run_args[@]}" "$@"
@@ -996,8 +1028,6 @@ if [ "$dev_mode" = "yes" ]; then
         echo "run exit status: $run_exit" 1>&2
     fi
 elif [ "$update_mode" = "yes" ]; then
-    "./${build}" $build_args >"$build_stdout" 2>"$build_stderr"
-    build_exit="$?"
     if [ "$build_exit" = '0' ]; then
         echo "${test_dir}: running..." 1>&2
         "./${run}" "${run_args[@]}" "$@" >"$test_stdout" 2>"$test_stderr"
@@ -1021,8 +1051,6 @@ elif [ "$update_mode" = "yes" ]; then
     fi
 elif [ "$build_only" = "yes" ]; then
     good="yes"
-    "./${build}" $build_args >"$build_stdout" 2>"$build_stderr"
-    build_exit="$?"
     if [ "$build_exit" '!=' '0' ]; then
         cp "$build_stdout" "$test_stdout"
         diff --strip-trailing-cr -q "$expected_stdout" "$test_stdout" >/dev/null
@@ -1046,8 +1074,6 @@ elif [ "$build_only" = "yes" ]; then
       | xargs rm -rf
     exit 0
 else
-    "./${build}" $build_args >"$build_stdout" 2>"$build_stderr"
-    build_exit="$?"
     if [ "$build_exit" = '0' ]; then
         echo "${test_dir}: running..." 1>&2
         "./${run}" "${run_args[@]}" "$@" >"$test_stdout" 2>"$test_stderr"
@@ -1165,6 +1191,10 @@ fi
 # Clean up test files.
 if [ "$always_clean" = "yes" -o "$good" = "yes" ] && [ "$never_clean" = "no" ]; then
     cd "$oldwd"
+    # Clean up build files only if we created them.
+    if [ "$skip_build" = "no" ]; then
+      rm -rf "$build_path"
+    fi
     rm -rf "$tmp_dir"
     if [ "$target_mode" = "yes" -a "$build_exit" = "0" ]; then
         adb shell rm -rf $chroot_dex_location
diff --git a/test/testrunner/env.py b/test/testrunner/env.py
index 40f750b984..319e1a76b0 100644
--- a/test/testrunner/env.py
+++ b/test/testrunner/env.py
@@ -71,6 +71,9 @@ ANDROID_BUILD_TOP = _get_android_build_top()
 # Directory used for temporary test files on the host.
 ART_HOST_TEST_DIR = tempfile.mkdtemp(prefix = 'test-art-')
 
+# Directory used to store files build by the run-test script.
+ART_TEST_RUN_TEST_BUILD_PATH = _env.get('ART_TEST_RUN_TEST_BUILD_PATH')
+
 # Keep going after encountering a test failure?
 ART_TEST_KEEP_GOING = _getEnvBoolean('ART_TEST_KEEP_GOING', True)
 
diff --git a/test/testrunner/testrunner.py b/test/testrunner/testrunner.py
index c531d2e94a..987cf712bf 100755
--- a/test/testrunner/testrunner.py
+++ b/test/testrunner/testrunner.py
@@ -194,6 +194,8 @@ class ChildProcessTracker(object):
 
 child_process_tracker = ChildProcessTracker()
 
+# Keep track of the already executed build scripts
+finished_build_script = {}
 
 def setup_csv_result():
   """Set up the CSV output if required."""
@@ -568,10 +570,27 @@ def run_tests(tests):
       if address_size == '64':
         options_test += ' --64'
 
-      # TODO(http://36039166): This is a temporary solution to
-      # fix build breakages.
-      options_test = (' --output-path %s') % (
-          tempfile.mkdtemp(dir=env.ART_HOST_TEST_DIR)) + options_test
+      # Make it possible to split the test to two passes: build only and test only.
+      # This is useful to avoid building identical files many times for the test combinations.
+      # We can remove this once we move the build script fully to soong.
+      global build_only
+      global skip_build
+      if (build_only or skip_build) and not is_test_disabled(test, variant_set):
+        assert(env.ART_TEST_RUN_TEST_BUILD_PATH)  # Persistent storage between the passes.
+        build_path = os.path.join(env.ART_TEST_RUN_TEST_BUILD_PATH, test)
+        if build_only and finished_build_script.setdefault(test, test_name) != test_name:
+          return None  # Different combination already build the needed files for this test.
+        os.makedirs(build_path, exist_ok=True)
+        if build_only:
+          options_test += ' --build-only'
+        if skip_build:
+          options_test += ' --skip-build'
+      else:
+        build_path = tempfile.mkdtemp(dir=env.ART_HOST_TEST_DIR)
+
+      # b/36039166: Note that the path lengths must kept reasonably short.
+      temp_path = tempfile.mkdtemp(dir=env.ART_HOST_TEST_DIR)
+      options_test = '--build-path {} --temp-path {} '.format(build_path, temp_path) + options_test
 
       run_test_sh = env.ANDROID_BUILD_TOP + '/art/test/run-test'
       command = ' '.join((run_test_sh, options_test, ' '.join(extra_arguments[target]), test))
@@ -594,7 +613,7 @@ def run_tests(tests):
 
       try:
         tests_done = 0
-        for test_future in concurrent.futures.as_completed(test_futures):
+        for test_future in concurrent.futures.as_completed(f for f in test_futures if f):
           (test, status, failure_info, test_time) = test_future.result()
           tests_done += 1
           print_test_info(tests_done, test, status, failure_info, test_time)
@@ -1091,6 +1110,8 @@ def parse_option():
   global with_agent
   global zipapex_loc
   global csv_result
+  global build_only
+  global skip_build
 
   parser = argparse.ArgumentParser(description="Runs all or a subset of the ART test suite.")
   parser.add_argument('-t', '--test', action='append', dest='tests', help='name(s) of the test(s)')
@@ -1127,7 +1148,11 @@ def parse_option():
                             help="""Pass an option, unaltered, to the run-test script.
                             This should be enclosed in single-quotes to allow for spaces. The option
                             will be split using shlex.split() prior to invoking run-test.
-                            Example \"--run-test-option='--with-agent libtifast.so=MethodExit'\"""")
+                            Example \"--run-test-option='--with-agent libtifast.so=MethodExit'\".""")
+  global_group.add_argument('--build-only', action='store_true', dest='build_only',
+                            help="""Only execute the build commands in the run-test script""")
+  global_group.add_argument('--skip-build', action='store_true', dest='skip_build',
+                            help="""Skip the builds command in the run-test script""")
   global_group.add_argument('--with-agent', action='append', dest='with_agent',
                             help="""Pass an agent to be attached to the runtime""")
   global_group.add_argument('--runtime-option', action='append', dest='runtime_option',
@@ -1195,6 +1220,8 @@ def parse_option():
   with_agent = options['with_agent'];
   run_test_option = sum(map(shlex.split, options['run_test_option']), [])
   zipapex_loc = options['runtime_zipapex']
+  build_only = options['build_only']
+  skip_build = options['skip_build']
 
   timeout = options['timeout']
   if options['dex2oat_jobs']: