ART Test Runner

The testrunner runs the ART run-tests by simply invoking the script.
It fetches the list of eligible tests from art/test directory, and list of
disabled tests from art/test/knownfailures.json. It runs the tests by
invoking art/test/run-test script and checks the exit code to decide if
the test passed or failed.

Before invoking the script, first build all the tests dependencies by
building 'test-art-host-run-test-dependencies' for host tests,
'test-art-target-run-test-dependencies' for target tests, and
'test-art-run-test-dependencies' for building dependencies for both.
There are various options to invoke the script which are:
-t: Either the test name as in art/test or the test name including the variant
    information. Eg, "-t 001-HelloWorld",
    "-t test-art-host-run-test-debug-prebuild-optimizing-relocate-ntrace-cms-checkjni-picimage-npictest-ndebuggable-001-HelloWorld32"
-j: Number of thread workers to be used. Eg - "-j64"
--dry-run: Instead of running the test name, just print its name.
--verbose
-b To build the dependencies before running the test.

To specify any specific variants for the test, use --<<variant-name>>.
For eg, for compiler type as optimizing, use --optimizing.

In the end, the script will print the failed and skipped tests if any.

New targets - test-art-host-run-test-dependencies and
test-art-target-run-test-dependencies have been added. The targets build
dependencies for host and target respectively.

Test: No tests
Change-Id: Ia4f13ee3444c2c733342c046ef1582517459fc9b

1 files changed, 174 insertions, 0 deletions

diff --git a/test/testrunner/env.py b/test/testrunner/env.py
new file mode 100644
index 0000000000..e015d7427c
--- /dev/null
+++ b/test/testrunner/env.py
@@ -0,0 +1,174 @@
+#!/usr/bin/env python
+#
+# Copyright 2017, The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import tempfile
+import subprocess
+
+env = dict(os.environ)
+
+def getEnvBoolean(var, default):
+  val = env.get(var)
+  if val:
+    if val == "True" or val == "true":
+      return True
+    if val == "False" or val == "false":
+      return False
+  return default
+
+def get_build_var(var_name):
+  # The command is taken from build/envsetup.sh to fetch build variables.
+  command = ("CALLED_FROM_SETUP=true BUILD_SYSTEM=build/core "
+             "make --no-print-directory -C \"%s\" -f build/core/config.mk "
+             "dumpvar-%s") % (ANDROID_BUILD_TOP, var_name)
+  config = subprocess.Popen(command, stdout=subprocess.PIPE,
+                            shell=True).communicate()[0]
+  return config.strip()
+
+def get_env(key):
+  return env.get(key)
+
+ANDROID_BUILD_TOP = env.get('ANDROID_BUILD_TOP', os.getcwd())
+
+# Directory used for temporary test files on the host.
+ART_HOST_TEST_DIR = tempfile.mkdtemp(prefix = 'test-art-')
+
+# Keep going after encountering a test failure?
+ART_TEST_KEEP_GOING = getEnvBoolean('ART_TEST_KEEP_GOING', True)
+
+# Do you want all tests, even those that are time consuming?
+ART_TEST_FULL = getEnvBoolean('ART_TEST_FULL', False)
+
+# Do you want interpreter tests run?
+ART_TEST_INTERPRETER = getEnvBoolean('ART_TEST_INTERPRETER', ART_TEST_FULL)
+ART_TEST_INTERPRETER_ACCESS_CHECKS = getEnvBoolean('ART_TEST_INTERPRETER_ACCESS_CHECKS',
+                                                   ART_TEST_FULL)
+
+# Do you want JIT tests run?
+ART_TEST_JIT = getEnvBoolean('ART_TEST_JIT', ART_TEST_FULL)
+
+# Do you want optimizing compiler tests run?
+ART_TEST_OPTIMIZING = getEnvBoolean('ART_TEST_OPTIMIZING', True)
+
+# Do you want to test the optimizing compiler with graph coloring register allocation?
+ART_TEST_OPTIMIZING_GRAPH_COLOR = getEnvBoolean('ART_TEST_OPTIMIZING_GRAPH_COLOR', ART_TEST_FULL)
+
+# Do we want to test a non-PIC-compiled core image?
+ART_TEST_NPIC_IMAGE = getEnvBoolean('ART_TEST_NPIC_IMAGE', ART_TEST_FULL)
+
+# Do we want to test PIC-compiled tests ("apps")?
+ART_TEST_PIC_TEST = getEnvBoolean('ART_TEST_PIC_TEST', ART_TEST_FULL)
+# Do you want tracing tests run?
+ART_TEST_TRACE = getEnvBoolean('ART_TEST_TRACE', ART_TEST_FULL)
+
+# Do you want tracing tests (streaming mode) run?
+ART_TEST_TRACE_STREAM = getEnvBoolean('ART_TEST_TRACE_STREAM', ART_TEST_FULL)
+
+# Do you want tests with GC verification enabled run?
+ART_TEST_GC_VERIFY = getEnvBoolean('ART_TEST_GC_VERIFY', ART_TEST_FULL)
+
+# Do you want tests with the GC stress mode enabled run?
+ART_TEST_GC_STRESS = getEnvBoolean('ART_TEST_GC_STRESS', ART_TEST_FULL)
+
+# Do you want tests with the JNI forcecopy mode enabled run?
+ART_TEST_JNI_FORCECOPY = getEnvBoolean('ART_TEST_JNI_FORCECOPY', ART_TEST_FULL)
+
+# Do you want run-tests with relocation disabled run?
+ART_TEST_RUN_TEST_RELOCATE = getEnvBoolean('ART_TEST_RUN_TEST_RELOCATE', ART_TEST_FULL)
+
+# Do you want run-tests with prebuilding?
+ART_TEST_RUN_TEST_PREBUILD = getEnvBoolean('ART_TEST_RUN_TEST_PREBUILD', True)
+
+# Do you want run-tests with no prebuilding enabled run?
+ART_TEST_RUN_TEST_NO_PREBUILD = getEnvBoolean('ART_TEST_RUN_TEST_NO_PREBUILD', ART_TEST_FULL)
+
+# Do you want run-tests with a pregenerated core.art?
+ART_TEST_RUN_TEST_IMAGE = getEnvBoolean('ART_TEST_RUN_TEST_IMAGE', True)
+
+# Do you want run-tests without a pregenerated core.art?
+ART_TEST_RUN_TEST_NO_IMAGE = getEnvBoolean('ART_TEST_RUN_TEST_NO_IMAGE', ART_TEST_FULL)
+
+# Do you want run-tests with relocation enabled but patchoat failing?
+ART_TEST_RUN_TEST_RELOCATE_NO_PATCHOAT = getEnvBoolean('ART_TEST_RUN_TEST_RELOCATE_NO_PATCHOAT',
+                                                       ART_TEST_FULL)
+
+# Do you want run-tests without a dex2oat?
+ART_TEST_RUN_TEST_NO_DEX2OAT = getEnvBoolean('ART_TEST_RUN_TEST_NO_DEX2OAT', ART_TEST_FULL)
+
+# Do you want run-tests with libartd.so?
+ART_TEST_RUN_TEST_DEBUG = getEnvBoolean('ART_TEST_RUN_TEST_DEBUG', True)
+
+# Do you want run-tests with libart.so?
+ART_TEST_RUN_TEST_NDEBUG = getEnvBoolean('ART_TEST_RUN_TEST_NDEBUG', ART_TEST_FULL)
+
+# Do you want failed tests to have their artifacts cleaned up?
+ART_TEST_RUN_TEST_ALWAYS_CLEAN = getEnvBoolean('ART_TEST_RUN_TEST_ALWAYS_CLEAN', True)
+
+# Do you want run-tests with the --debuggable flag
+ART_TEST_RUN_TEST_DEBUGGABLE = getEnvBoolean('ART_TEST_RUN_TEST_DEBUGGABLE', ART_TEST_FULL)
+
+# Do you want to test multi-part boot-image functionality?
+ART_TEST_RUN_TEST_MULTI_IMAGE = getEnvBoolean('ART_TEST_RUN_TEST_MULTI_IMAGE', ART_TEST_FULL)
+
+ART_TEST_DEBUG_GC = getEnvBoolean('ART_TEST_DEBUG_GC', False)
+
+ART_TEST_BISECTION = getEnvBoolean('ART_TEST_BISECTION', False)
+
+DEX2OAT_HOST_INSTRUCTION_SET_FEATURES = env.get('DEX2OAT_HOST_INSTRUCTION_SET_FEATURES')
+
+# Do you want run-tests with the host/target's second arch?
+ART_TEST_RUN_TEST_2ND_ARCH = getEnvBoolean('ART_TEST_RUN_TEST_2ND_ARCH', True)
+
+HOST_2ND_ARCH_PREFIX = get_build_var('HOST_2ND_ARCH_PREFIX')
+HOST_2ND_ARCH_PREFIX_DEX2OAT_HOST_INSTRUCTION_SET_FEATURES = env.get(
+  HOST_2ND_ARCH_PREFIX + 'DEX2OAT_HOST_INSTRUCTION_SET_FEATURES')
+
+ART_TEST_ANDROID_ROOT = env.get('ART_TEST_ANDROID_ROOT')
+
+ART_TEST_WITH_STRACE = getEnvBoolean('ART_TEST_DEBUG_GC', False)
+
+TARGET_2ND_ARCH = get_build_var('TARGET_2ND_ARCH')
+TARGET_ARCH = get_build_var('TARGET_ARCH')
+if TARGET_2ND_ARCH:
+  if "64" in TARGET_ARCH:
+    ART_PHONY_TEST_TARGET_SUFFIX = "64"
+    _2ND_ART_PHONY_TEST_TARGET_SUFFIX = "32"
+  else:
+    ART_PHONY_TEST_TARGET_SUFFIX = "32"
+    _2ND_ART_PHONY_TEST_TARGET_SUFFIX = ""
+else:
+  if "64" in TARGET_ARCH:
+    ART_PHONY_TEST_TARGET_SUFFIX = "64"
+    _2ND_ART_PHONY_TEST_TARGET_SUFFIX = ""
+  else:
+    ART_PHONY_TEST_TARGET_SUFFIX = "32"
+    _2ND_ART_PHONY_TEST_TARGET_SUFFIX = ""
+
+HOST_PREFER_32_BIT = get_build_var('HOST_PREFER_32_BIT')
+if HOST_PREFER_32_BIT == "true":
+  ART_PHONY_TEST_HOST_SUFFIX = "32"
+  _2ND_ART_PHONY_TEST_HOST_SUFFIX = ""
+else:
+  ART_PHONY_TEST_HOST_SUFFIX = "64"
+  _2ND_ART_PHONY_TEST_HOST_SUFFIX = "32"
+
+HOST_OUT_EXECUTABLES = ('%s/%s') % (ANDROID_BUILD_TOP,
+                                    get_build_var("HOST_OUT_EXECUTABLES"))
+os.environ['JACK'] = HOST_OUT_EXECUTABLES + '/jack'
+os.environ['DX'] = HOST_OUT_EXECUTABLES + '/dx'
+os.environ['SMALI'] = HOST_OUT_EXECUTABLES + '/smali'
+os.environ['JASMIN'] = HOST_OUT_EXECUTABLES + '/jasmin'
+os.environ['DXMERGER'] = HOST_OUT_EXECUTABLES + '/dexmerger'