Run libcore tests in parallel

Bug: 142039427
Test: run-libcore-tests.py --mode=host
Test: run-libcore-tests.py --mode=device
Change-Id: Idc36d8402d80ea3901784bb0196b1b11123f1efd
diff --git a/tools/run-libcore-tests.py b/tools/run-libcore-tests.py
index e6fbc59..661a98a 100755
--- a/tools/run-libcore-tests.py
+++ b/tools/run-libcore-tests.py
@@ -18,7 +18,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sys, os, argparse, subprocess, shlex
+import sys, os, argparse, subprocess, shlex, re, concurrent.futures, multiprocessing
 
 def parse_args():
   parser = argparse.ArgumentParser(description="Run libcore tests using the vogar testing tool.")
@@ -26,6 +26,8 @@
                       help='Specify where tests should be run.')
   parser.add_argument('--variant', choices=['X32', 'X64'],
                       help='Which dalvikvm variant to execute with.')
+  parser.add_argument('-j', '--jobs', type=int,
+                      help='Number of tests to run simultaneously.')
   parser.add_argument('--timeout', type=int,
                       help='How long to run the test before aborting (seconds).')
   parser.add_argument('--debug', action='store_true',
@@ -47,14 +49,29 @@
 ANDROID_PRODUCT_OUT = os.environ.get("ANDROID_PRODUCT_OUT")
 
 LIBCORE_TEST_NAMES = [
+  # Naive critical path optimization: Run the longest tests first.
+  "org.apache.harmony.tests.java.util",  # 90min under gcstress
+  "libcore.java.lang",                   # 90min under gcstress
+  "jsr166",                              # 60min under gcstress
+  "libcore.java.util",                   # 60min under gcstress
+  "libcore.java.math",                   # 50min under gcstress
+  "org.apache.harmony.crypto",           # 30min under gcstress
+  "org.apache.harmony.tests.java.io",    # 30min under gcstress
+  "org.apache.harmony.tests.java.text",  # 30min under gcstress
+  # Split highmemorytest to individual classes since it is too big.
+  "libcore.highmemorytest.java.text.DateFormatTest",
+  "libcore.highmemorytest.java.text.DecimalFormatTest",
+  "libcore.highmemorytest.java.text.SimpleDateFormatTest",
+  "libcore.highmemorytest.java.time.format.DateTimeFormatterTest",
+  "libcore.highmemorytest.java.util.CalendarTest",
+  "libcore.highmemorytest.java.util.CurrencyTest",
+  "libcore.highmemorytest.libcore.icu.LocaleDataTest",
+  # All other tests in alphabetical order.
   "libcore.android.system",
   "libcore.build",
   "libcore.dalvik.system",
   "libcore.java.awt",
-  "libcore.java.lang",
-  "libcore.java.math",
   "libcore.java.text",
-  "libcore.java.util",
   "libcore.javax.crypto",
   "libcore.javax.net",
   "libcore.javax.security",
@@ -67,26 +84,20 @@
   "libcore.libcore.reflect",
   "libcore.libcore.util",
   "libcore.sun.invoke",
-  "libcore.sun.net",
   "libcore.sun.misc",
+  "libcore.sun.net",
   "libcore.sun.security",
   "libcore.sun.util",
   "libcore.xml",
   "org.apache.harmony.annotation",
-  "org.apache.harmony.crypto",
   "org.apache.harmony.luni",
   "org.apache.harmony.nio",
   "org.apache.harmony.regex",
   "org.apache.harmony.testframework",
-  "org.apache.harmony.tests.java.io",
   "org.apache.harmony.tests.java.lang",
   "org.apache.harmony.tests.java.math",
-  "org.apache.harmony.tests.java.util",
-  "org.apache.harmony.tests.java.text",
   "org.apache.harmony.tests.javax.security",
   "tests.java.lang.String",
-  "jsr166",
-  "libcore.highmemorytest",
 ]
 # "org.apache.harmony.security",  # We don't have rights to revert changes in case of failures.
 
@@ -137,10 +148,10 @@
   test_names = list(LIBCORE_TEST_NAMES)
   # See b/78228743 and b/178351808.
   if args.gcstress or args.debug or args.mode == "jvm":
-    test_names.remove("libcore.highmemorytest")
+    test_names = list(t for t in test_names if not t.startswith("libcore.highmemorytest"))
   return test_names
 
-def get_vogar_command(test_names):
+def get_vogar_command(test_name):
   cmd = ["vogar"]
   if args.mode == "device":
     cmd.append("--mode=device --vm-arg -Ximage:/apex/com.android.art/javalib/boot.art")
@@ -162,10 +173,12 @@
 
   if args.mode == "device":
     if ART_TEST_CHROOT:
-      cmd.append(f"--chroot {ART_TEST_CHROOT} --device-dir=/tmp")
+      cmd.append(f"--chroot {ART_TEST_CHROOT} --device-dir=/tmp/vogar/test-{test_name}")
     else:
-      cmd.append("--device-dir=/data/local/tmp")
+      cmd.append("--device-dir=/data/local/tmp/vogar/test-{test_name}")
     cmd.append(f"--vm-command={ART_TEST_ANDROID_ROOT}/bin/art")
+  else:
+    cmd.append(f"--device-dir=/tmp/vogar/test-{test_name}")
 
   if args.mode != "jvm":
     cmd.append("--timeout {}".format(get_timeout_secs()))
@@ -188,9 +201,20 @@
 
   cmd.extend("--expectations " + f for f in get_expected_failures())
   cmd.extend("--classpath " + get_jar_filename(cp) for cp in CLASSPATH)
-  cmd.extend(test_names)
+  cmd.append(test_name)
   return cmd
 
+def get_target_cpu_count():
+  adb_command = 'adb shell cat /sys/devices/system/cpu/present'
+  with subprocess.Popen(adb_command.split(),
+                        stderr=subprocess.STDOUT,
+                        stdout=subprocess.PIPE,
+                        universal_newlines=True) as proc:
+    assert(proc.wait() == 0)  # Check the exit code.
+    match = re.match(r'\d*-(\d*)', proc.stdout.read())
+    assert(match)
+    return int(match.group(1)) + 1  # Add one to convert from "last-index" to "count"
+
 def main():
   global args
   args = parse_args()
@@ -201,12 +225,36 @@
     if not os.path.exists(jar):
       raise AssertionError(f"Missing {jar}. Run buildbot-build.sh first.")
 
-  cmd = " ".join(get_vogar_command(get_test_names()))
-  print(cmd)
-  if not args.dry_run:
-    with subprocess.Popen(shlex.split(cmd)) as proc:
-      exit_code = proc.wait()
-      sys.exit(exit_code)
+  if not args.jobs:
+    args.jobs = get_target_cpu_count() if args.mode == "device" else multiprocessing.cpu_count()
+
+  def run_test(test_name):
+    cmd = " ".join(get_vogar_command(test_name))
+    if args.dry_run:
+      return test_name, cmd, "Dry-run: skipping execution", 0
+    with subprocess.Popen(shlex.split(cmd),
+                          stderr=subprocess.STDOUT,
+                          stdout=subprocess.PIPE,
+                          universal_newlines=True) as proc:
+      return test_name, cmd, proc.communicate()[0], proc.wait()
+
+  failed_regex = re.compile(r"^.* FAIL \(EXEC_FAILED\)$", re.MULTILINE)
+  failed_tests, max_exit_code = [], 0
+  with concurrent.futures.ThreadPoolExecutor(max_workers=args.jobs) as pool:
+    futures = [pool.submit(run_test, test_name) for test_name in get_test_names()]
+    print(f"Running {len(futures)} tasks on {args.jobs} core(s)...")
+    for i, future in enumerate(concurrent.futures.as_completed(futures)):
+      test_name, cmd, stdout, exit_code = future.result()
+      print(f"\n[{i+1}/{len(futures)}] {test_name} " + ("FAIL" if exit_code != 0 else "PASS"))
+      if exit_code != 0 or args.dry_run:
+        print(cmd)
+        print(stdout)
+      else:
+        print(stdout.strip().split("\n")[-1])  # Vogar final summary line.
+      failed_tests.extend(failed_regex.findall(stdout))
+      max_exit_code = max(max_exit_code, exit_code)
+  print("\n" + "\n".join(failed_tests))
+  sys.exit(max_exit_code)
 
 if __name__ == '__main__':
   main()