ART Test Runner

The testrunner runs the ART run-tests by simply invoking the script.
It fetches the list of eligible tests from art/test directory, and list of
disabled tests from art/test/knownfailures.json. It runs the tests by
invoking art/test/run-test script and checks the exit code to decide if
the test passed or failed.

Before invoking the script, first build all the tests dependencies by
building 'test-art-host-run-test-dependencies' for host tests,
'test-art-target-run-test-dependencies' for target tests, and
'test-art-run-test-dependencies' for building dependencies for both.
There are various options to invoke the script which are:
-t: Either the test name as in art/test or the test name including the variant
    information. Eg, "-t 001-HelloWorld",
    "-t test-art-host-run-test-debug-prebuild-optimizing-relocate-ntrace-cms-checkjni-picimage-npictest-ndebuggable-001-HelloWorld32"
-j: Number of thread workers to be used. Eg - "-j64"
--dry-run: Instead of running the test name, just print its name.
--verbose
-b To build the dependencies before running the test.

To specify any specific variants for the test, use --<<variant-name>>.
For eg, for compiler type as optimizing, use --optimizing.

In the end, the script will print the failed and skipped tests if any.

New targets - test-art-host-run-test-dependencies and
test-art-target-run-test-dependencies have been added. The targets build
dependencies for host and target respectively.

Test: No tests
Change-Id: Ia4f13ee3444c2c733342c046ef1582517459fc9b

1 files changed, 794 insertions, 0 deletions

diff --git a/test/testrunner/testrunner.py b/test/testrunner/testrunner.py
new file mode 100755
index 0000000000..fd3ebd1d4d
--- /dev/null
+++ b/test/testrunner/testrunner.py
@@ -0,0 +1,794 @@
+#!/usr/bin/env python
+#
+# Copyright 2017, The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""ART Run-Test TestRunner
+
+The testrunner runs the ART run-tests by simply invoking the script.
+It fetches the list of eligible tests from art/test directory, and list of
+disabled tests from art/test/knownfailures.json. It runs the tests by
+invoking art/test/run-test script and checks the exit value to decide if the
+test passed or failed.
+
+Before invoking the script, first build all the tests dependencies.
+There are two major build targets for building target and host tests
+dependencies:
+1) test-art-host-run-test
+2) test-art-target-run-test
+
+There are various options to invoke the script which are:
+-t: Either the test name as in art/test or the test name including the variant
+    information. Eg, "-t 001-HelloWorld",
+    "-t test-art-host-run-test-debug-prebuild-optimizing-relocate-ntrace-cms-checkjni-picimage-npictest-ndebuggable-001-HelloWorld32"
+-j: Number of thread workers to be used. Eg - "-j64"
+--dry-run: Instead of running the test name, just print its name.
+--verbose
+-b / --build-dependencies: to build the dependencies before running the test
+
+To specify any specific variants for the test, use --<<variant-name>>.
+For eg, for compiler type as optimizing, use --optimizing.
+
+
+In the end, the script will print the failed and skipped tests if any.
+
+"""
+import fnmatch
+import itertools
+import json
+from optparse import OptionParser
+import os
+import re
+import subprocess
+import sys
+import threading
+import time
+
+import env
+
+TARGET_TYPES = set()
+RUN_TYPES = set()
+PREBUILD_TYPES = set()
+COMPILER_TYPES = set()
+RELOCATE_TYPES = set()
+TRACE_TYPES = set()
+GC_TYPES = set()
+JNI_TYPES = set()
+IMAGE_TYPES = set()
+PICTEST_TYPES = set()
+DEBUGGABLE_TYPES = set()
+ADDRESS_SIZES = set()
+OPTIMIZING_COMPILER_TYPES = set()
+ADDRESS_SIZES_TARGET = {'host': set(), 'target': set()}
+
+# DISABLED_TEST_CONTAINER holds information about the disabled tests. It is a map
+# that has key as the test name (like 001-HelloWorld), and value as set of
+# variants that the test is disabled for.
+DISABLED_TEST_CONTAINER = {}
+
+# The Dict contains the list of all possible variants for a given type. For example,
+# for key TARGET, the value would be target and host. The list is used to parse
+# the test name given as the argument to run.
+VARIANT_TYPE_DICT = {}
+
+# The set contains all the variants of each time.
+TOTAL_VARIANTS_SET = set()
+
+# The colors are used in the output. When a test passes, COLOR_PASS is used,
+# and so on.
+COLOR_ERROR = '\033[91m'
+COLOR_PASS = '\033[92m'
+COLOR_SKIP = '\033[93m'
+COLOR_NORMAL = '\033[0m'
+
+# The mutex object is used by the threads for exclusive access of test_count
+# to make any changes in its value.
+test_count_mutex = threading.Lock()
+# The set contains the list of all the possible run tests that are in art/test
+# directory.
+RUN_TEST_SET = set()
+# The semaphore object is used by the testrunner to limit the number of
+# threads to the user requested concurrency value.
+semaphore = threading.Semaphore(1)
+# The mutex object is used to provide exclusive access to a thread to print
+# its output.
+print_mutex = threading.Lock()
+failed_tests = []
+skipped_tests = []
+
+# Flags
+n_thread = 1
+test_count = 0
+total_test_count = 0
+verbose = False
+last_print_length = 0
+dry_run = False
+build = False
+gdb = False
+gdb_arg = ''
+stop_testrunner = False
+
+def gather_test_info():
+  """The method gathers test information about the test to be run which includes
+  generating the list of total tests from the art/test directory and the list
+  of disabled test. It also maps various variants to types.
+  """
+  global TOTAL_VARIANTS_SET
+  global DISABLED_TEST_CONTAINER
+  # TODO: Avoid duplication of the variant names in different lists.
+  VARIANT_TYPE_DICT['pictest'] = {'pictest', 'npictest'}
+  VARIANT_TYPE_DICT['run'] = {'ndebug', 'debug'}
+  VARIANT_TYPE_DICT['target'] = {'target', 'host'}
+  VARIANT_TYPE_DICT['trace'] = {'trace', 'ntrace', 'stream'}
+  VARIANT_TYPE_DICT['image'] = {'picimage', 'no-image', 'npicimage',
+                                'multinpicimage', 'multipicimage'}
+  VARIANT_TYPE_DICT['debuggable'] = {'ndebuggable', 'debuggable'}
+  VARIANT_TYPE_DICT['gc'] = {'gcstress', 'gcverify', 'cms'}
+  VARIANT_TYPE_DICT['prebuild'] = {'no-prebuild', 'no-dex2oat', 'prebuild'}
+  VARIANT_TYPE_DICT['relocate'] = {'relocate-npatchoat', 'relocate', 'no-relocate'}
+  VARIANT_TYPE_DICT['jni'] = {'jni', 'forcecopy', 'checkjni'}
+  VARIANT_TYPE_DICT['address_sizes'] = {'64', '32'}
+  VARIANT_TYPE_DICT['compiler'] = {'interp-ac', 'interpreter', 'jit', 'optimizing',
+                              'regalloc_gc'}
+
+  for v_type in VARIANT_TYPE_DICT:
+    TOTAL_VARIANTS_SET = TOTAL_VARIANTS_SET.union(VARIANT_TYPE_DICT.get(v_type))
+
+  test_dir = env.ANDROID_BUILD_TOP + '/art/test'
+  for f in os.listdir(test_dir):
+    if fnmatch.fnmatch(f, '[0-9]*'):
+      RUN_TEST_SET.add(f)
+  DISABLED_TEST_CONTAINER = get_disabled_test_info()
+
+
+def setup_test_env():
+  """The method sets default value for the various variants of the tests if they
+  are already not set.
+  """
+  if env.ART_TEST_BISECTION:
+    env.ART_TEST_RUN_TEST_NO_PREBUILD = True
+    env.ART_TEST_RUN_TEST_PREBUILD = False
+    # Bisection search writes to standard output.
+    env.ART_TEST_QUIET = False
+
+  if not TARGET_TYPES:
+    TARGET_TYPES.add('host')
+    TARGET_TYPES.add('target')
+
+  if env.ART_TEST_RUN_TEST_PREBUILD:
+    PREBUILD_TYPES.add('prebuild')
+  if env.ART_TEST_RUN_TEST_NO_PREBUILD:
+    PREBUILD_TYPES.add('no-prebuild')
+  if env.ART_TEST_RUN_TEST_NO_DEX2OAT:
+    PREBUILD_TYPES.add('no-dex2oat')
+
+  if env.ART_TEST_INTERPRETER_ACCESS_CHECKS:
+    COMPILER_TYPES.add('interp-ac')
+  if env.ART_TEST_INTERPRETER:
+    COMPILER_TYPES.add('interpreter')
+  if env.ART_TEST_JIT:
+    COMPILER_TYPES.add('jit')
+
+  if env.ART_TEST_OPTIMIZING:
+    COMPILER_TYPES.add('optimizing')
+    OPTIMIZING_COMPILER_TYPES.add('optimizing')
+  if env.ART_TEST_OPTIMIZING_GRAPH_COLOR:
+    COMPILER_TYPES.add('regalloc_gc')
+    OPTIMIZING_COMPILER_TYPES.add('regalloc_gc')
+
+  if not RELOCATE_TYPES:
+    RELOCATE_TYPES.add('no-relocate')
+  if env.ART_TEST_RUN_TEST_RELOCATE:
+    RELOCATE_TYPES.add('relocate')
+  if env.ART_TEST_RUN_TEST_RELOCATE_NO_PATCHOAT:
+    RELOCATE_TYPES.add('relocate-npatchoat')
+
+  if not TRACE_TYPES:
+    TRACE_TYPES.add('ntrace')
+  if env.ART_TEST_TRACE:
+    TRACE_TYPES.add('trace')
+  if env.ART_TEST_TRACE_STREAM:
+    TRACE_TYPES.add('stream')
+
+  if not GC_TYPES:
+    GC_TYPES.add('cms')
+  if env.ART_TEST_GC_STRESS:
+    GC_TYPES.add('gcstress')
+  if env.ART_TEST_GC_VERIFY:
+    GC_TYPES.add('gcverify')
+
+  if not JNI_TYPES:
+    JNI_TYPES.add('checkjni')
+  if env.ART_TEST_JNI_FORCECOPY:
+    JNI_TYPES.add('forcecopy')
+
+  if env.ART_TEST_RUN_TEST_IMAGE:
+    IMAGE_TYPES.add('picimage')
+  if env.ART_TEST_RUN_TEST_NO_IMAGE:
+    IMAGE_TYPES.add('no-image')
+  if env.ART_TEST_RUN_TEST_MULTI_IMAGE:
+    IMAGE_TYPES.add('multipicimage')
+  if env.ART_TEST_NPIC_IMAGE:
+    IMAGE_TYPES.add('npicimage')
+  if env.ART_TEST_RUN_TEST_MULTI_IMAGE:
+    IMAGE_TYPES.add('multinpicimage')
+
+  if not PICTEST_TYPES:
+    PICTEST_TYPES.add('npictest')
+  if env.ART_TEST_PIC_TEST:
+    PICTEST_TYPES.add('pictest')
+
+  if env.ART_TEST_RUN_TEST_DEBUG:
+    RUN_TYPES.add('debug')
+  if env.ART_TEST_RUN_TEST_NDEBUG:
+    RUN_TYPES.add('ndebug')
+
+  if not DEBUGGABLE_TYPES:
+    DEBUGGABLE_TYPES.add('ndebuggable')
+
+  if env.ART_TEST_RUN_TEST_DEBUGGABLE:
+    DEBUGGABLE_TYPES.add('debuggable')
+
+  if not ADDRESS_SIZES:
+    ADDRESS_SIZES_TARGET['target'].add(env.ART_PHONY_TEST_TARGET_SUFFIX)
+    ADDRESS_SIZES_TARGET['host'].add(env.ART_PHONY_TEST_HOST_SUFFIX)
+    if env.ART_TEST_RUN_TEST_2ND_ARCH:
+      ADDRESS_SIZES_TARGET['host'].add(env._2ND_ART_PHONY_TEST_HOST_SUFFIX)
+      ADDRESS_SIZES_TARGET['target'].add(env._2ND_ART_PHONY_TEST_TARGET_SUFFIX)
+  else:
+    ADDRESS_SIZES_TARGET['host'] = ADDRESS_SIZES_TARGET['host'].union(ADDRESS_SIZES)
+    ADDRESS_SIZES_TARGET['target'] = ADDRESS_SIZES_TARGET['target'].union(ADDRESS_SIZES)
+
+  global semaphore
+  semaphore = threading.Semaphore(n_thread)
+
+
+def run_tests(tests):
+  """Creates thread workers to run the tests.
+
+  The method generates command and thread worker to run the tests. Depending on
+  the user input for the number of threads to be used, the method uses a
+  semaphore object to keep a count in control for the thread workers. When a new
+  worker is created, it acquires the semaphore object, and when the number of
+  workers reaches the maximum allowed concurrency, the method wait for an
+  existing thread worker to release the semaphore object. Worker releases the
+  semaphore object when they finish printing the output.
+
+  Args:
+    tests: The set of tests to be run.
+  """
+  options_all = ''
+  global total_test_count
+  total_test_count = len(tests)
+  total_test_count *= len(RUN_TYPES)
+  total_test_count *= len(PREBUILD_TYPES)
+  total_test_count *= len(RELOCATE_TYPES)
+  total_test_count *= len(TRACE_TYPES)
+  total_test_count *= len(GC_TYPES)
+  total_test_count *= len(JNI_TYPES)
+  total_test_count *= len(IMAGE_TYPES)
+  total_test_count *= len(PICTEST_TYPES)
+  total_test_count *= len(DEBUGGABLE_TYPES)
+  total_test_count *= len(COMPILER_TYPES)
+  target_address_combinations = 0
+  for target in TARGET_TYPES:
+    for address_size in ADDRESS_SIZES_TARGET[target]:
+      target_address_combinations += 1
+  total_test_count *= target_address_combinations
+
+  if env.ART_TEST_WITH_STRACE:
+    options_all += ' --strace'
+
+  if env.ART_TEST_RUN_TEST_ALWAYS_CLEAN:
+    options_all += ' --always-clean'
+
+  if env.ART_TEST_BISECTION:
+    options_all += ' --bisection-search'
+
+  if env.ART_TEST_ANDROID_ROOT:
+    options_all += ' --android-root ' + env.ART_TEST_ANDROID_ROOT
+
+  if gdb:
+    options_all += ' --gdb'
+    if gdb_arg:
+      options_all += ' --gdb-arg ' + gdb_arg
+
+  config = itertools.product(tests, TARGET_TYPES, RUN_TYPES, PREBUILD_TYPES,
+                             COMPILER_TYPES, RELOCATE_TYPES, TRACE_TYPES,
+                             GC_TYPES, JNI_TYPES, IMAGE_TYPES, PICTEST_TYPES,
+                             DEBUGGABLE_TYPES)
+
+  for test, target, run, prebuild, compiler, relocate, trace, gc, \
+      jni, image, pictest, debuggable in config:
+    for address_size in ADDRESS_SIZES_TARGET[target]:
+      if stop_testrunner:
+        # When ART_TEST_KEEP_GOING is set to false, then as soon as a test
+        # fails, stop_testrunner is set to True. When this happens, the method
+        # stops creating any any thread and wait for all the exising threads
+        # to end.
+        while threading.active_count() > 2:
+          time.sleep(0.1)
+          return
+      test_name = 'test-art-'
+      test_name += target + '-run-test-'
+      test_name += run + '-'
+      test_name += prebuild + '-'
+      test_name += compiler + '-'
+      test_name += relocate + '-'
+      test_name += trace + '-'
+      test_name += gc + '-'
+      test_name += jni + '-'
+      test_name += image + '-'
+      test_name += pictest + '-'
+      test_name += debuggable + '-'
+      test_name += test
+      test_name += address_size
+
+      variant_set = {target, run, prebuild, compiler, relocate, trace, gc, jni,
+                     image, pictest, debuggable, address_size}
+
+      options_test = options_all
+
+      if target == 'host':
+        options_test += ' --host'
+
+      if run == 'ndebug':
+        options_test += ' -O'
+
+      if prebuild == 'prebuild':
+        options_test += ' --prebuild'
+      elif prebuild == 'no-prebuild':
+        options_test += ' --no-prebuild'
+      elif prebuild == 'no-dex2oat':
+        options_test += ' --no-prebuild --no-dex2oat'
+
+      if compiler == 'optimizing':
+        options_test += ' --optimizing'
+      elif compiler == 'regalloc_gc':
+        options_test += ' --optimizing -Xcompiler-option --register-allocation-strategy=graph-color'
+      elif compiler == 'interpreter':
+        options_test += ' --interpreter'
+      elif compiler == 'interp-ac':
+        options_test += ' --interpreter --verify-soft-fail'
+      elif compiler == 'jit':
+        options_test += ' --jit'
+
+      if relocate == 'relocate':
+        options_test += ' --relocate'
+      elif relocate == 'no-relocate':
+        options_test += ' --no-relocate'
+      elif relocate == 'relocate-npatchoat':
+        options_test += ' --relocate --no-patchoat'
+
+      if trace == 'trace':
+        options_test += ' --trace'
+      elif trace == 'stream':
+        options_test += ' --trace --stream'
+
+      if gc == 'gcverify':
+        options_test += ' --gcverify'
+      elif gc == 'gcstress':
+        options_test += ' --gcstress'
+
+      if jni == 'forcecopy':
+        options_test += ' --runtime-option -Xjniopts:forcecopy'
+      elif jni == 'checkjni':
+        options_test += ' --runtime-option -Xcheck:jni'
+
+      if image == 'no-image':
+        options_test += ' --no-image'
+      elif image == 'npicimage':
+        options_test += ' --npic-image'
+      elif image == 'multinpicimage':
+        options_test += ' --npic-image --multi-image'
+      elif image == 'multipicimage':
+        options_test += ' --multi-image'
+
+      if pictest == 'pictest':
+        options_test += ' --pic-test'
+
+      if debuggable == 'debuggable':
+        options_test += ' --debuggable'
+
+      if address_size == '64':
+        options_test += ' --64'
+
+        if env.DEX2OAT_HOST_INSTRUCTION_SET_FEATURES:
+          options_test += ' --instruction-set-features' + env.DEX2OAT_HOST_INSTRUCTION_SET_FEATURES
+
+      elif address_size == '32':
+        if env.HOST_2ND_ARCH_PREFIX_DEX2OAT_HOST_INSTRUCTION_SET_FEATURES:
+          options_test += ' --instruction-set-features ' + \
+                          env.HOST_2ND_ARCH_PREFIX_DEX2OAT_HOST_INSTRUCTION_SET_FEATURES
+
+      options_test = (' --output-path %s/run-test-output/%s') % (
+        env.ART_HOST_TEST_DIR, test_name) + options_test
+
+      run_test_sh = env.ANDROID_BUILD_TOP + '/art/test/run-test'
+      command = run_test_sh + ' ' + options_test + ' ' + test
+
+      semaphore.acquire()
+      worker = threading.Thread(target=run_test, args=(command, test, variant_set, test_name))
+      worker.daemon = True
+      worker.start()
+
+  while threading.active_count() > 2:
+    time.sleep(0.1)
+
+
+def run_test(command, test, test_variant, test_name):
+  """Runs the test.
+
+  It invokes art/test/run-test script to run the test. The output of the script
+  is checked, and if it ends with "Succeeded!", it assumes that the tests
+  passed, otherwise, put it in the list of failed test. Before actually running
+  the test, it also checks if the test is placed in the list of disabled tests,
+  and if yes, it skips running it, and adds the test in the list of skipped
+  tests. The method uses print_text method to actually print the output. After
+  successfully running and capturing the output for the test, it releases the
+  semaphore object.
+
+  Args:
+    command: The command to be used to invoke the script
+    test: The name of the test without the variant information.
+    test_variant: The set of variant for the test.
+    test_name: The name of the test along with the variants.
+  """
+  global last_print_length
+  global test_count
+  global stop_testrunner
+  if is_test_disabled(test, test_variant):
+    test_skipped = True
+  else:
+    test_skipped = False
+    proc = subprocess.Popen(command.split(), stderr=subprocess.PIPE)
+    script_output = proc.stderr.read().strip()
+    test_passed = not proc.wait()
+
+  # If verbose is set to True, every test information is printed on a new line.
+  # If not, the information is printed on the same line overriding the
+  # previous test output.
+  if not verbose:
+    suffix = '\r'
+    prefix = ' ' * last_print_length + '\r'
+  else:
+    suffix = '\n'
+    prefix = ''
+  test_count_mutex.acquire()
+  test_count += 1
+  percent = (test_count * 100) / total_test_count
+  out = '[ ' + str(percent) + '% ' + str(test_count) + '/' + str(total_test_count) + ' ] '
+  test_count_mutex.release()
+  out += test_name + ' '
+  if not test_skipped:
+    if test_passed:
+      out += COLOR_PASS + 'PASS' + COLOR_NORMAL
+    else:
+      out += COLOR_ERROR + 'FAIL' + COLOR_NORMAL
+      failed_tests.append(test_name)
+      if verbose:
+        out += '\n' + command + '\n' + script_output
+      if not env.ART_TEST_KEEP_GOING:
+        stop_testrunner = True
+  elif not dry_run:
+    out += COLOR_SKIP + 'SKIP' + COLOR_NORMAL
+    skipped_tests.append(test_name)
+  last_print_length = len(out)
+  print_mutex.acquire()
+  print_text(prefix + out + suffix)
+  print_mutex.release()
+  semaphore.release()
+
+
+def get_disabled_test_info():
+  """Generate set of known failures.
+
+  It parses the art/test/knownfailures.json file to generate the list of
+  disabled tests.
+
+  Returns:
+    The method returns a dict of tests mapped to the variants list
+    for which the test should not be run.
+  """
+  known_failures_file = env.ANDROID_BUILD_TOP + '/art/test/knownfailures.json'
+  with open(known_failures_file) as known_failures_json:
+    known_failures_info = json.loads(known_failures_json.read())
+
+  disabled_test_info = {}
+  for failure in known_failures_info:
+    tests = failure.get('test')
+    if tests:
+      tests = [tests]
+    else:
+      tests = failure.get('tests', [])
+    variants = parse_variants(failure.get('variant'))
+    env_vars = failure.get('env_vars')
+    if check_env_vars(env_vars):
+      for test in tests:
+        if test in disabled_test_info:
+          disabled_test_info[test] = disabled_test_info[test].union(variants)
+        else:
+          disabled_test_info[test] = variants
+  return disabled_test_info
+
+
+def check_env_vars(env_vars):
+  """Checks if the env variables are set as required to run the test.
+
+  Returns:
+    True if all the env variables are set as required, otherwise False.
+  """
+
+  if not env_vars:
+    return True
+  for key in env_vars:
+    if env.get_env(key) != env_vars.get(key):
+      return False
+  return True
+
+
+def is_test_disabled(test, variant_set):
+  """Checks if the test along with the variant_set is disabled.
+
+  Args:
+    test: The name of the test as in art/test directory.
+    variant_set: Variants to be used for the test.
+  Returns:
+    True, if the test is disabled.
+  """
+  if dry_run:
+    return True
+  variants_list = DISABLED_TEST_CONTAINER.get(test, {})
+  for variants in variants_list:
+    variants_present = True
+    for variant in variants:
+      if variant not in variant_set:
+        variants_present = False
+        break
+    if variants_present:
+      return True
+  return False
+
+
+def parse_variants(variants):
+  """Parse variants fetched from art/test/knownfailures.json.
+  """
+  if not variants:
+    variants = ''
+    for variant in TOTAL_VARIANTS_SET:
+      variants += variant
+      variants += '|'
+    variants = variants[:-1]
+  variant_list = set()
+  or_variants = variants.split('|')
+  for or_variant in or_variants:
+    and_variants = or_variant.split('&')
+    variant = set()
+    for and_variant in and_variants:
+      and_variant = and_variant.strip()
+      variant.add(and_variant)
+    variant_list.add(frozenset(variant))
+  return variant_list
+
+def print_text(output):
+  sys.stdout.write(output)
+  sys.stdout.flush()
+
+def print_analysis():
+  if not verbose:
+    print_text(' ' * last_print_length + '\r')
+  if skipped_tests:
+    print_text(COLOR_SKIP + 'SKIPPED TESTS' + COLOR_NORMAL + '\n')
+    for test in skipped_tests:
+      print_text(test + '\n')
+    print_text('\n')
+
+  if failed_tests:
+    print_text(COLOR_ERROR + 'FAILED TESTS' + COLOR_NORMAL + '\n')
+    for test in failed_tests:
+      print_text(test + '\n')
+
+
+def parse_test_name(test_name):
+  """Parses the testname provided by the user.
+  It supports two types of test_name:
+  1) Like 001-HelloWorld. In this case, it will just verify if the test actually
+  exists and if it does, it returns the testname.
+  2) Like test-art-host-run-test-debug-prebuild-interpreter-no-relocate-ntrace-cms-checkjni-picimage-npictest-ndebuggable-001-HelloWorld32
+  In this case, it will parse all the variants and check if they are placed
+  correctly. If yes, it will set the various VARIANT_TYPES to use the
+  variants required to run the test. Again, it returns the test_name
+  without the variant information like 001-HelloWorld.
+  """
+  if test_name in RUN_TEST_SET:
+    return {test_name}
+
+  regex = '^test-art-'
+  regex += '(' + '|'.join(VARIANT_TYPE_DICT['target']) + ')-'
+  regex += 'run-test-'
+  regex += '(' + '|'.join(VARIANT_TYPE_DICT['run']) + ')-'
+  regex += '(' + '|'.join(VARIANT_TYPE_DICT['prebuild']) + ')-'
+  regex += '(' + '|'.join(VARIANT_TYPE_DICT['compiler']) + ')-'
+  regex += '(' + '|'.join(VARIANT_TYPE_DICT['relocate']) + ')-'
+  regex += '(' + '|'.join(VARIANT_TYPE_DICT['trace']) + ')-'
+  regex += '(' + '|'.join(VARIANT_TYPE_DICT['gc']) + ')-'
+  regex += '(' + '|'.join(VARIANT_TYPE_DICT['jni']) + ')-'
+  regex += '(' + '|'.join(VARIANT_TYPE_DICT['image']) + ')-'
+  regex += '(' + '|'.join(VARIANT_TYPE_DICT['pictest']) + ')-'
+  regex += '(' + '|'.join(VARIANT_TYPE_DICT['debuggable']) + ')-'
+  regex += '(' + '|'.join(RUN_TEST_SET) + ')'
+  regex += '(' + '|'.join(VARIANT_TYPE_DICT['address_sizes']) + ')$'
+  match = re.match(regex, test_name)
+  if match:
+    TARGET_TYPES.add(match.group(1))
+    RUN_TYPES.add(match.group(2))
+    PREBUILD_TYPES.add(match.group(3))
+    COMPILER_TYPES.add(match.group(4))
+    RELOCATE_TYPES.add(match.group(5))
+    TRACE_TYPES.add(match.group(6))
+    GC_TYPES.add(match.group(7))
+    JNI_TYPES.add(match.group(8))
+    IMAGE_TYPES.add(match.group(9))
+    PICTEST_TYPES.add(match.group(10))
+    DEBUGGABLE_TYPES.add(match.group(11))
+    ADDRESS_SIZES.add(match.group(13))
+    return {match.group(12)}
+
+
+def parse_option():
+  global verbose
+  global dry_run
+  global n_thread
+  global build
+  global gdb
+  global gdb_arg
+
+  parser = OptionParser()
+  parser.add_option('-t', '--test', dest='test', help='name of the test')
+  parser.add_option('-j', type='int', dest='n_thread')
+  for variant in TOTAL_VARIANTS_SET:
+    flag = '--' + variant
+    flag_dest = variant.replace('-', '_')
+    if variant == '32' or variant == '64':
+      flag_dest = 'n' + flag_dest
+    parser.add_option(flag, action='store_true', dest=flag_dest)
+  parser.add_option('--verbose', '-v', action='store_true', dest='verbose')
+  parser.add_option('--dry-run', action='store_true', dest='dry_run')
+  parser.add_option('-b', '--build-dependencies', action='store_true', dest='build')
+  parser.add_option('--gdb', action='store_true', dest='gdb')
+  parser.add_option('--gdb-arg', dest='gdb_arg')
+
+  options = parser.parse_args()[0]
+  test = ''
+  if options.test:
+    test = parse_test_name(options.test)
+  if options.pictest:
+    PICTEST_TYPES.add('pictest')
+  if options.ndebug:
+    RUN_TYPES.add('ndebug')
+  if options.interp_ac:
+    COMPILER_TYPES.add('interp-ac')
+  if options.picimage:
+    IMAGE_TYPES.add('picimage')
+  if options.n64:
+    ADDRESS_SIZES.add('64')
+  if options.interpreter:
+    COMPILER_TYPES.add('interpreter')
+  if options.jni:
+    JNI_TYPES.add('jni')
+  if options.relocate_npatchoat:
+    RELOCATE_TYPES.add('relocate-npatchoat')
+  if options.no_prebuild:
+    PREBUILD_TYPES.add('no-prebuild')
+  if options.npictest:
+    PICTEST_TYPES.add('npictest')
+  if options.no_dex2oat:
+    PREBUILD_TYPES.add('no-dex2oat')
+  if options.jit:
+    COMPILER_TYPES.add('jit')
+  if options.relocate:
+    RELOCATE_TYPES.add('relocate')
+  if options.ndebuggable:
+    DEBUGGABLE_TYPES.add('ndebuggable')
+  if options.no_image:
+    IMAGE_TYPES.add('no-image')
+  if options.optimizing:
+    COMPILER_TYPES.add('optimizing')
+  if options.trace:
+    TRACE_TYPES.add('trace')
+  if options.gcstress:
+    GC_TYPES.add('gcstress')
+  if options.no_relocate:
+    RELOCATE_TYPES.add('no-relocate')
+  if options.target:
+    TARGET_TYPES.add('target')
+  if options.forcecopy:
+    JNI_TYPES.add('forcecopy')
+  if options.n32:
+    ADDRESS_SIZES.add('32')
+  if options.host:
+    TARGET_TYPES.add('host')
+  if options.gcverify:
+    GC_TYPES.add('gcverify')
+  if options.debuggable:
+    DEBUGGABLE_TYPES.add('debuggable')
+  if options.prebuild:
+    PREBUILD_TYPES.add('prebuild')
+  if options.debug:
+    RUN_TYPES.add('debug')
+  if options.checkjni:
+    JNI_TYPES.add('checkjni')
+  if options.ntrace:
+    TRACE_TYPES.add('ntrace')
+  if options.cms:
+    GC_TYPES.add('cms')
+  if options.npicimage:
+    IMAGE_TYPES.add('npicimage')
+  if options.multinpicimage:
+    IMAGE_TYPES.add('multinpicimage')
+  if options.multipicimage:
+    IMAGE_TYPES.add('multipicimage')
+  if options.verbose:
+    verbose = True
+  if options.n_thread:
+    n_thread = max(1, options.n_thread)
+  if options.dry_run:
+    dry_run = True
+    verbose = True
+  if options.build:
+    build = True
+  if options.gdb:
+    n_thread = 1
+    gdb = True
+    if options.gdb_arg:
+      gdb_arg = options.gdb_arg
+
+  return test
+
+def main():
+  gather_test_info()
+  user_requested_test = parse_option()
+  setup_test_env()
+  if build:
+    build_targets = ''
+    if 'host' in TARGET_TYPES:
+      build_targets += 'test-art-host-run-test-dependencies'
+    if 'target' in TARGET_TYPES:
+      build_targets += 'test-art-target-run-test-dependencies'
+    build_command = 'make -j' + str(n_thread) + ' ' + build_targets
+    if subprocess.call(build_command.split()):
+      sys.exit(1)
+  if user_requested_test:
+    test_runner_thread = threading.Thread(target=run_tests, args=(user_requested_test,))
+  else:
+    test_runner_thread = threading.Thread(target=run_tests, args=(RUN_TEST_SET,))
+  test_runner_thread.daemon = True
+  try:
+    test_runner_thread.start()
+    while threading.active_count() > 1:
+      time.sleep(0.1)
+    print_analysis()
+    if failed_tests:
+      sys.exit(1)
+    sys.exit(0)
+  except SystemExit:
+    pass
+  except:
+    print "hello"
+    print_analysis()
+    sys.exit(1)
+
+
+if __name__ == '__main__':
+  main()