diff options
| -rw-r--r-- | test/Android.run-test.mk | 63 | ||||
| -rw-r--r-- | test/knownfailures.json | 365 | ||||
| -rw-r--r-- | test/testrunner/env.py | 174 | ||||
| -rwxr-xr-x | test/testrunner/testrunner.py | 794 |
4 files changed, 1396 insertions, 0 deletions
diff --git a/test/Android.run-test.mk b/test/Android.run-test.mk index 742353da46..dbf516a301 100644 --- a/test/Android.run-test.mk +++ b/test/Android.run-test.mk @@ -863,6 +863,69 @@ ART_TEST_HOST_RUN_TEST_DEPENDENCIES += \ endif +# Host executables. +host_prereq_rules := $(ART_TEST_HOST_RUN_TEST_DEPENDENCIES) + +# Classpath for Jack compilation for host. +host_prereq_rules += $(HOST_JACK_CLASSPATH_DEPENDENCIES) + +# Required for dx, jasmin, smali, dexmerger, jack. +host_prereq_rules += $(TEST_ART_RUN_TEST_DEPENDENCIES) + +host_prereq_rules += $(HOST_OUT_EXECUTABLES)/hprof-conv + +# Classpath for Jack compilation for target. +target_prereq_rules := $(TARGET_JACK_CLASSPATH_DEPENDENCIES) + +# Sync test files to the target, depends upon all things that must be pushed +#to the target. +target_prereq_rules += test-art-target-sync + +define core-image-dependencies + image_suffix := $(3) + ifeq ($(3),regalloc_gc) + image_suffix:=optimizing + else + ifeq ($(3),jit) + image_suffix:=interpreter + endif + endif + ifeq ($(2),no-image) + $(1)_prereq_rules += $$($(call name-to-var,$(1))_CORE_IMAGE_$$(image_suffix)_pic_$(4)) + else + ifeq ($(2),npicimage) + $(1)_prereq_rules += $$($(call name-to-var,$(1))_CORE_IMAGE_$$(image_suffix)_no-pic_$(4)) + else + ifeq ($(2),picimage) + $(1)_prereq_rules += $$($(call name-to-var,$(1))_CORE_IMAGE_$$(image_suffix)_pic_$(4)) + else + ifeq ($(2),multinpicimage) + $(1)_prereq_rules += $$($(call name-to-var,$(1))_CORE_IMAGE_$$(image_suffix)_no-pic_multi_$(4)) + else + ifeq ($(2),multipicimage) + $(1)_prereq_rules += $$($(call name-to-var,$(1))_CORE_IMAGE_$$(image_suffix)_pic_multi_$(4)) + endif + endif + endif + endif + endif +endef + +# Add core image dependencies required for given target - HOST or TARGET, +# IMAGE_TYPE, COMPILER_TYPE and ADDRESS_SIZE to the prereq_rules. +$(foreach target, $(TARGET_TYPES), \ + $(foreach image, $(IMAGE_TYPES), \ + $(foreach compiler, $(COMPILER_TYPES), \ + $(foreach address_size, $(ALL_ADDRESS_SIZES), $(eval \ + $(call core-image-dependencies,$(target),$(image),$(compiler),$(address_size))))))) + +test-art-host-run-test-dependencies : $(host_prereq_rules) +test-art-target-run-test-dependencies : $(target_prereq_rules) +test-art-run-test-dependencies : test-art-host-run-test-dependencies test-art-target-run-test-dependencies + +host_prereq_rules := +target_prereq_rules := + # Create a rule to build and run a tests following the form: # test-art-{1: host or target}-run-test-{2: debug ndebug}-{3: prebuild no-prebuild no-dex2oat}- # {4: interpreter optimizing jit interp-ac}- diff --git a/test/knownfailures.json b/test/knownfailures.json new file mode 100644 index 0000000000..98d56d4d9a --- /dev/null +++ b/test/knownfailures.json @@ -0,0 +1,365 @@ +[ + { + "test": "153-reference-stress", + "description": ["Disable 153-reference-stress temporarily until a fix", + "arrives."], + "bug": "http://b/33389022" + }, + { + "test": "080-oom-fragmentation", + "description": "Disable 080-oom-fragmentation due to flakes.", + "bug": "http://b/33795328" + }, + { + "tests": ["497-inlining-and-class-loader", + "542-unresolved-access-check"], + "description": ["Disable 497-inlining-and-class-loader and ", + "542-unresolved-access-check until they are rewritten.", + "These tests use a broken class loader that tries to", + "register a dex file that's already registered with a", + "different loader."], + "bug": "http://b/34193123" + }, + { + "test": "149-suspend-all-stress", + "description": "Disable 149-suspend-all-stress, its output is flaky", + "bug": "http://b/28988206" + }, + { + "test": "577-profile-foreign-dex", + "description": "Disable 577-profile-foreign-dex", + "bug": "http://b/27454772" + }, + { + "tests": ["002-sleep", + "053-wait-some", + "055-enum-performance", + "133-static-invoke-super"], + "description": ["Tests that are timing sensitive and flaky on heavily", + "loaded systems."] + }, + { + "test": "147-stripped-dex-fallback", + "variant": "target", + "description": ["147-stripped-dex-fallback isn't supported on device", + "because --strip-dex requires the zip command."] + }, + { + "test": "569-checker-pattern-replacement", + "variant": "target", + "description": ["569-checker-pattern-replacement tests behaviour", + "present only on host."] + }, + { + "tests": ["116-nodex2oat", + "118-noimage-dex2oat", + "134-nodex2oat-nofallback"], + "variant": "prebuild", + "description": ["Note 116-nodex2oat is not broken per-se it just", + "doesn't (and isn't meant to) work with --prebuild."] + }, + { + "test": "554-jit-profile-file", + "variant": "no-prebuild | interpreter", + "description": ["554-jit-profile-file is disabled because it needs a", + "primary oat file to know what it should save."] + }, + { + "tests": ["529-checker-unresolved", "555-checker-regression-x86const"], + "variant": "no-prebuild", + "bug": "http://b/27784033" + }, + { + "tests": ["117-nopatchoat", + "147-stripped-dex-fallback", + "608-checker-unresolved-lse"], + "variant": "no-prebuild" + }, + { + "tests": ["117-nopatchoat", + "118-noimage-dex2oat", + "119-noimage-patchoat", + "554-jit-profile-file"], + "variant": "no-relocate", + "description": ["117-nopatchoat is not broken per-se it just doesn't", + "work (and isn't meant to) without --prebuild", + "--relocate"] + }, + { + "test": "137-cfi", + "variant": "interp-ac", + "description": ["Temporarily disable some broken tests when forcing", + "access checks in interpreter"], + "bug": "http://b/22414682" + }, + { + "test" : "629-vdex-speed", + "variant": "interp-ac | no-dex2oat | interpreter | jit | relocate-npatchoat", + "description": "629 requires compilation." + }, + { + "test": "137-cfi", + "variant": "gcstress", + "description": ["137-cfi needs to unwind a second forked process. We're", + "using a primitive sleep to wait till we hope the", + "second process got into the expected state. The", + "slowness of gcstress makes this bad."] + }, + { + "tests": ["908-gc-start-finish", + "913-heaps"], + "variant": "gcstress", + "description": ["908-gc-start-finish expects GCs only to be run at", + "clear points. The reduced heap size makes this", + "non-deterministic. Same for 913."] + }, + { + "test": "961-default-iface-resolution-gen", + "variant": "gcstress", + "description": ["961-default-iface-resolution-gen and", + "964-default-iface-init-genare very long tests that", + "often will take more than the timeout to run when", + "gcstress is enabled. This is because gcstress slows", + "down allocations significantly which these tests do a", + "lot."] + }, + { + "tests": ["964-default-iface-init-gen", + "154-gc-loop"], + "variant": "gcstress" + }, + { + "test": "115-native-bridge", + "variant": "target", + "description": ["115-native-bridge setup is complicated. Need to", + "implement it correctly for the target."] + }, + { + "test": "130-hprof", + "variant": "target", + "desription": ["130-hprof dumps the heap and runs hprof-conv to check", + "whether the file is somewhat readable. Thi is only", + "possible on the host. TODO: Turn off all the other", + "combinations, this is more about testing actual ART", + "code. A gtest is very hard to write here, as (for a", + "complete test) JDWP must be set up."] + }, + { + "test": "131-structural-change", + "variant": "debug", + "description": ["131 is an old test. The functionality has been", + "implemented at an earlier stage and is checked", + "in tests 138. Blacklisted for debug builds since", + "these builds have duplicate classes checks which", + "punt to interpreter"] + }, + { + "test": "138-duplicate-classes-check", + "variant": "ndebug", + "description": ["Turned on for debug builds since debug builds have", + "duplicate classes checks enabled"], + "bug": "http://b/2133391" + }, + { + "test": "147-stripped-dex-fallback", + "variant": "no-dex2oat | no-image | relocate-npatchoat", + "description": ["147-stripped-dex-fallback is disabled because it", + "requires --prebuild."] + }, + { + "test": "554-jit-profile-file", + "variant": "no-dex2oat | no-image | relocate-npatchoat", + "description": ["554-jit-profile-file is disabled because it needs a", + "primary oat file to know what it should save."] + }, + { + "tests": ["116-nodex2oat", + "117-nopatchoat", + "118-noimage-dex2oat", + "119-noimage-patchoat", + "137-cfi", + "138-duplicate-classes-check2"], + "variant": "no-dex2oat | no-image | relocate-npatchoat", + "description": ["All these tests check that we have sane behavior if we", + "don't have a patchoat or dex2oat. Therefore we", + "shouldn't run them in situations where we actually", + "don't have these since they explicitly test for them.", + "These all also assume we have an image."] + }, + { + "tests": ["137-cfi", + "138-duplicate-classes-check", + "018-stack-overflow", + "961-default-iface-resolution-gen", + "964-default-iface-init"], + "variant": "no-image", + "description": ["This test fails without an image. 018, 961, 964 often", + "time out."], + "bug": "http://b/34369284" + }, + { + "test": "137-cfi", + "description": ["This test unrolls and expects managed frames, but", + "tracing means we run the interpreter."], + "variant": "trace | stream" + }, + { + "tests": ["802-deoptimization", + "570-checker-osr"], + "description": ["This test dynamically enables tracing to force a", + "deoptimization. This makes the test meaningless", + "when already tracing, and writes an error message", + "that we do not want to check for."], + "variant": "trace | stream" + }, + { + "test": "130-hprof", + "description": "130 occasional timeout", + "bug": "http://b/32383962", + "variant": "trace | stream" + }, + { + "tests": ["087-gc-after-link", + "141-class-unload"], + "variant": "trace | stream" + }, + { + "tests": ["604-hot-static-interface", + "612-jit-dex-cache", + "613-inlining-dex-cache", + "616-cha", + "626-set-resolved-string"], + "variant": "trace | stream", + "description": ["These tests expect JIT compilation, which is", + "suppressed when tracing."] + }, + { + "test": "137-cfi", + "description": ["CFI unwinding expects managed frames, and the test", + "does not iterate enough to even compile. JIT also", + "uses Generic JNI instead of the JNI compiler."], + "variant": "interpreter | jit" + }, + { + "test": "902-hello-transformation", + "description": "Test 902 hits races with the JIT compiler.", + "bug": "http://b/32821077", + "variant": "jit" + }, + { + "test": "906-iterate-heap", + "description": ["Test 906 iterates the heap filtering with different", + "options. No instances should be created between those", + "runs to be able to have precise checks."], + "variant": "jit" + }, + { + "tests": ["914-hello-obsolescence", + "915-obsolete-2", + "917-fields-transformation", + "919-obsolete-fields"], + "description": ["Test 914, 915, 917, & 919 are very sensitive to the", + "exact state of the stack, including the jit-inserted", + "runtime frames. This causes them to be somewhat flaky", + "as JIT tests. This should be fixed once b/33630159 or", + "b/33616143 are resolved but until then just disable", + "them. Test 916 already checks this feature for JIT", + "use cases in a way that is resilient to the jit", + "frames."], + "variant": "jit" + }, + { + "tests": ["926-multi-obsolescence", + "904-object-allocation"], + "variant": "jit" + }, + { + "test": "912-classes", + "variant": "jit", + "bug": "http://b/34655682" + }, + { + "tests": ["570-checker-select", + "484-checker-register-hints"], + "description": ["These tests were based on the linear scan allocator,", + "which makes different decisions than the graph", + "coloring allocator. (These attempt to test for code", + "quality, not correctness.)"], + "variant": "regalloc_gc" + }, + { + "tests": ["454-get-vreg", + "457-regs", + "602-deoptimizeable"], + "description": ["Tests that should fail when the optimizing compiler ", + "compiles them non-debuggable."], + "variant": "optimizing & ndebuggable | regalloc_gc & ndebuggable" + }, + { + "test": "596-app-images", + "variant": "npictest" + }, + { + "test": "055-enum-performance", + "variant": "optimizing | regalloc_gc", + "description": ["055: Exceeds run time limits due to heap poisoning ", + "instrumentation (on ARM and ARM64 devices)."] + }, + { + "test": "909-attach-agent", + "variant": "debuggable", + "description": "Tests that check semantics for a non-debuggable app." + }, + { + "test": "137-cfi", + "variant": "debuggable", + "description": ["The test relies on AOT code and debuggable makes us", + "JIT always."] + }, + { + "tests": ["000-nop", + "134-nodex2oat-nofallback", + "147-stripped-dex-fallback", + "595-profile-saving"], + "description": "The doesn't compile anything", + "env_vars": {"ART_TEST_BISECTION": "true"}, + "variant": "optimizing | regalloc_gc" + }, + { + "tests": "089-many-methods", + "description": "The test tests a build failure", + "env_vars": {"ART_TEST_BISECTION": "true"}, + "variant": "optimizing | regalloc_gc" + }, + { + "tests": ["018-stack-overflow", + "116-nodex2oat", + "117-nopatchoat", + "118-noimage-dex2oat", + "119-noimage-patchoat", + "126-miranda-multidex", + "137-cfi"], + "description": "The test run dalvikvm more than once.", + "env_vars": {"ART_TEST_BISECTION": "true"}, + "variant": "optimizing | regalloc_gc" + }, + { + "tests": ["115-native-bridge", + "088-monitor-verification"], + "description": "The test assume they are always compiled.", + "env_vars": {"ART_TEST_BISECTION": "true"}, + "variant": "optimizing | regalloc_gc" + }, + { + "test": "055-enum-performance", + "description": ["The test tests performance which degrades during", + "bisecting."], + "env_vars": {"ART_TEST_BISECTION": "true"}, + "variant": "optimizing | regalloc_gc" + }, + { + "test": "537-checker-arraycopy", + "env_vars": {"ART_USE_READ_BARRIER": "true"}, + "variant": "interpreter | optimizing | regalloc_gc | jit" + } +] diff --git a/test/testrunner/env.py b/test/testrunner/env.py new file mode 100644 index 0000000000..e015d7427c --- /dev/null +++ b/test/testrunner/env.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python +# +# Copyright 2017, The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import tempfile +import subprocess + +env = dict(os.environ) + +def getEnvBoolean(var, default): + val = env.get(var) + if val: + if val == "True" or val == "true": + return True + if val == "False" or val == "false": + return False + return default + +def get_build_var(var_name): + # The command is taken from build/envsetup.sh to fetch build variables. + command = ("CALLED_FROM_SETUP=true BUILD_SYSTEM=build/core " + "make --no-print-directory -C \"%s\" -f build/core/config.mk " + "dumpvar-%s") % (ANDROID_BUILD_TOP, var_name) + config = subprocess.Popen(command, stdout=subprocess.PIPE, + shell=True).communicate()[0] + return config.strip() + +def get_env(key): + return env.get(key) + +ANDROID_BUILD_TOP = env.get('ANDROID_BUILD_TOP', os.getcwd()) + +# Directory used for temporary test files on the host. +ART_HOST_TEST_DIR = tempfile.mkdtemp(prefix = 'test-art-') + +# Keep going after encountering a test failure? +ART_TEST_KEEP_GOING = getEnvBoolean('ART_TEST_KEEP_GOING', True) + +# Do you want all tests, even those that are time consuming? +ART_TEST_FULL = getEnvBoolean('ART_TEST_FULL', False) + +# Do you want interpreter tests run? +ART_TEST_INTERPRETER = getEnvBoolean('ART_TEST_INTERPRETER', ART_TEST_FULL) +ART_TEST_INTERPRETER_ACCESS_CHECKS = getEnvBoolean('ART_TEST_INTERPRETER_ACCESS_CHECKS', + ART_TEST_FULL) + +# Do you want JIT tests run? +ART_TEST_JIT = getEnvBoolean('ART_TEST_JIT', ART_TEST_FULL) + +# Do you want optimizing compiler tests run? +ART_TEST_OPTIMIZING = getEnvBoolean('ART_TEST_OPTIMIZING', True) + +# Do you want to test the optimizing compiler with graph coloring register allocation? +ART_TEST_OPTIMIZING_GRAPH_COLOR = getEnvBoolean('ART_TEST_OPTIMIZING_GRAPH_COLOR', ART_TEST_FULL) + +# Do we want to test a non-PIC-compiled core image? +ART_TEST_NPIC_IMAGE = getEnvBoolean('ART_TEST_NPIC_IMAGE', ART_TEST_FULL) + +# Do we want to test PIC-compiled tests ("apps")? +ART_TEST_PIC_TEST = getEnvBoolean('ART_TEST_PIC_TEST', ART_TEST_FULL) +# Do you want tracing tests run? +ART_TEST_TRACE = getEnvBoolean('ART_TEST_TRACE', ART_TEST_FULL) + +# Do you want tracing tests (streaming mode) run? +ART_TEST_TRACE_STREAM = getEnvBoolean('ART_TEST_TRACE_STREAM', ART_TEST_FULL) + +# Do you want tests with GC verification enabled run? +ART_TEST_GC_VERIFY = getEnvBoolean('ART_TEST_GC_VERIFY', ART_TEST_FULL) + +# Do you want tests with the GC stress mode enabled run? +ART_TEST_GC_STRESS = getEnvBoolean('ART_TEST_GC_STRESS', ART_TEST_FULL) + +# Do you want tests with the JNI forcecopy mode enabled run? +ART_TEST_JNI_FORCECOPY = getEnvBoolean('ART_TEST_JNI_FORCECOPY', ART_TEST_FULL) + +# Do you want run-tests with relocation disabled run? +ART_TEST_RUN_TEST_RELOCATE = getEnvBoolean('ART_TEST_RUN_TEST_RELOCATE', ART_TEST_FULL) + +# Do you want run-tests with prebuilding? +ART_TEST_RUN_TEST_PREBUILD = getEnvBoolean('ART_TEST_RUN_TEST_PREBUILD', True) + +# Do you want run-tests with no prebuilding enabled run? +ART_TEST_RUN_TEST_NO_PREBUILD = getEnvBoolean('ART_TEST_RUN_TEST_NO_PREBUILD', ART_TEST_FULL) + +# Do you want run-tests with a pregenerated core.art? +ART_TEST_RUN_TEST_IMAGE = getEnvBoolean('ART_TEST_RUN_TEST_IMAGE', True) + +# Do you want run-tests without a pregenerated core.art? +ART_TEST_RUN_TEST_NO_IMAGE = getEnvBoolean('ART_TEST_RUN_TEST_NO_IMAGE', ART_TEST_FULL) + +# Do you want run-tests with relocation enabled but patchoat failing? +ART_TEST_RUN_TEST_RELOCATE_NO_PATCHOAT = getEnvBoolean('ART_TEST_RUN_TEST_RELOCATE_NO_PATCHOAT', + ART_TEST_FULL) + +# Do you want run-tests without a dex2oat? +ART_TEST_RUN_TEST_NO_DEX2OAT = getEnvBoolean('ART_TEST_RUN_TEST_NO_DEX2OAT', ART_TEST_FULL) + +# Do you want run-tests with libartd.so? +ART_TEST_RUN_TEST_DEBUG = getEnvBoolean('ART_TEST_RUN_TEST_DEBUG', True) + +# Do you want run-tests with libart.so? +ART_TEST_RUN_TEST_NDEBUG = getEnvBoolean('ART_TEST_RUN_TEST_NDEBUG', ART_TEST_FULL) + +# Do you want failed tests to have their artifacts cleaned up? +ART_TEST_RUN_TEST_ALWAYS_CLEAN = getEnvBoolean('ART_TEST_RUN_TEST_ALWAYS_CLEAN', True) + +# Do you want run-tests with the --debuggable flag +ART_TEST_RUN_TEST_DEBUGGABLE = getEnvBoolean('ART_TEST_RUN_TEST_DEBUGGABLE', ART_TEST_FULL) + +# Do you want to test multi-part boot-image functionality? +ART_TEST_RUN_TEST_MULTI_IMAGE = getEnvBoolean('ART_TEST_RUN_TEST_MULTI_IMAGE', ART_TEST_FULL) + +ART_TEST_DEBUG_GC = getEnvBoolean('ART_TEST_DEBUG_GC', False) + +ART_TEST_BISECTION = getEnvBoolean('ART_TEST_BISECTION', False) + +DEX2OAT_HOST_INSTRUCTION_SET_FEATURES = env.get('DEX2OAT_HOST_INSTRUCTION_SET_FEATURES') + +# Do you want run-tests with the host/target's second arch? +ART_TEST_RUN_TEST_2ND_ARCH = getEnvBoolean('ART_TEST_RUN_TEST_2ND_ARCH', True) + +HOST_2ND_ARCH_PREFIX = get_build_var('HOST_2ND_ARCH_PREFIX') +HOST_2ND_ARCH_PREFIX_DEX2OAT_HOST_INSTRUCTION_SET_FEATURES = env.get( + HOST_2ND_ARCH_PREFIX + 'DEX2OAT_HOST_INSTRUCTION_SET_FEATURES') + +ART_TEST_ANDROID_ROOT = env.get('ART_TEST_ANDROID_ROOT') + +ART_TEST_WITH_STRACE = getEnvBoolean('ART_TEST_DEBUG_GC', False) + +TARGET_2ND_ARCH = get_build_var('TARGET_2ND_ARCH') +TARGET_ARCH = get_build_var('TARGET_ARCH') +if TARGET_2ND_ARCH: + if "64" in TARGET_ARCH: + ART_PHONY_TEST_TARGET_SUFFIX = "64" + _2ND_ART_PHONY_TEST_TARGET_SUFFIX = "32" + else: + ART_PHONY_TEST_TARGET_SUFFIX = "32" + _2ND_ART_PHONY_TEST_TARGET_SUFFIX = "" +else: + if "64" in TARGET_ARCH: + ART_PHONY_TEST_TARGET_SUFFIX = "64" + _2ND_ART_PHONY_TEST_TARGET_SUFFIX = "" + else: + ART_PHONY_TEST_TARGET_SUFFIX = "32" + _2ND_ART_PHONY_TEST_TARGET_SUFFIX = "" + +HOST_PREFER_32_BIT = get_build_var('HOST_PREFER_32_BIT') +if HOST_PREFER_32_BIT == "true": + ART_PHONY_TEST_HOST_SUFFIX = "32" + _2ND_ART_PHONY_TEST_HOST_SUFFIX = "" +else: + ART_PHONY_TEST_HOST_SUFFIX = "64" + _2ND_ART_PHONY_TEST_HOST_SUFFIX = "32" + +HOST_OUT_EXECUTABLES = ('%s/%s') % (ANDROID_BUILD_TOP, + get_build_var("HOST_OUT_EXECUTABLES")) +os.environ['JACK'] = HOST_OUT_EXECUTABLES + '/jack' +os.environ['DX'] = HOST_OUT_EXECUTABLES + '/dx' +os.environ['SMALI'] = HOST_OUT_EXECUTABLES + '/smali' +os.environ['JASMIN'] = HOST_OUT_EXECUTABLES + '/jasmin' +os.environ['DXMERGER'] = HOST_OUT_EXECUTABLES + '/dexmerger' diff --git a/test/testrunner/testrunner.py b/test/testrunner/testrunner.py new file mode 100755 index 0000000000..fd3ebd1d4d --- /dev/null +++ b/test/testrunner/testrunner.py @@ -0,0 +1,794 @@ +#!/usr/bin/env python +# +# Copyright 2017, The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""ART Run-Test TestRunner + +The testrunner runs the ART run-tests by simply invoking the script. +It fetches the list of eligible tests from art/test directory, and list of +disabled tests from art/test/knownfailures.json. It runs the tests by +invoking art/test/run-test script and checks the exit value to decide if the +test passed or failed. + +Before invoking the script, first build all the tests dependencies. +There are two major build targets for building target and host tests +dependencies: +1) test-art-host-run-test +2) test-art-target-run-test + +There are various options to invoke the script which are: +-t: Either the test name as in art/test or the test name including the variant + information. Eg, "-t 001-HelloWorld", + "-t test-art-host-run-test-debug-prebuild-optimizing-relocate-ntrace-cms-checkjni-picimage-npictest-ndebuggable-001-HelloWorld32" +-j: Number of thread workers to be used. Eg - "-j64" +--dry-run: Instead of running the test name, just print its name. +--verbose +-b / --build-dependencies: to build the dependencies before running the test + +To specify any specific variants for the test, use --<<variant-name>>. +For eg, for compiler type as optimizing, use --optimizing. + + +In the end, the script will print the failed and skipped tests if any. + +""" +import fnmatch +import itertools +import json +from optparse import OptionParser +import os +import re +import subprocess +import sys +import threading +import time + +import env + +TARGET_TYPES = set() +RUN_TYPES = set() +PREBUILD_TYPES = set() +COMPILER_TYPES = set() +RELOCATE_TYPES = set() +TRACE_TYPES = set() +GC_TYPES = set() +JNI_TYPES = set() +IMAGE_TYPES = set() +PICTEST_TYPES = set() +DEBUGGABLE_TYPES = set() +ADDRESS_SIZES = set() +OPTIMIZING_COMPILER_TYPES = set() +ADDRESS_SIZES_TARGET = {'host': set(), 'target': set()} + +# DISABLED_TEST_CONTAINER holds information about the disabled tests. It is a map +# that has key as the test name (like 001-HelloWorld), and value as set of +# variants that the test is disabled for. +DISABLED_TEST_CONTAINER = {} + +# The Dict contains the list of all possible variants for a given type. For example, +# for key TARGET, the value would be target and host. The list is used to parse +# the test name given as the argument to run. +VARIANT_TYPE_DICT = {} + +# The set contains all the variants of each time. +TOTAL_VARIANTS_SET = set() + +# The colors are used in the output. When a test passes, COLOR_PASS is used, +# and so on. +COLOR_ERROR = '\033[91m' +COLOR_PASS = '\033[92m' +COLOR_SKIP = '\033[93m' +COLOR_NORMAL = '\033[0m' + +# The mutex object is used by the threads for exclusive access of test_count +# to make any changes in its value. +test_count_mutex = threading.Lock() +# The set contains the list of all the possible run tests that are in art/test +# directory. +RUN_TEST_SET = set() +# The semaphore object is used by the testrunner to limit the number of +# threads to the user requested concurrency value. +semaphore = threading.Semaphore(1) +# The mutex object is used to provide exclusive access to a thread to print +# its output. +print_mutex = threading.Lock() +failed_tests = [] +skipped_tests = [] + +# Flags +n_thread = 1 +test_count = 0 +total_test_count = 0 +verbose = False +last_print_length = 0 +dry_run = False +build = False +gdb = False +gdb_arg = '' +stop_testrunner = False + +def gather_test_info(): + """The method gathers test information about the test to be run which includes + generating the list of total tests from the art/test directory and the list + of disabled test. It also maps various variants to types. + """ + global TOTAL_VARIANTS_SET + global DISABLED_TEST_CONTAINER + # TODO: Avoid duplication of the variant names in different lists. + VARIANT_TYPE_DICT['pictest'] = {'pictest', 'npictest'} + VARIANT_TYPE_DICT['run'] = {'ndebug', 'debug'} + VARIANT_TYPE_DICT['target'] = {'target', 'host'} + VARIANT_TYPE_DICT['trace'] = {'trace', 'ntrace', 'stream'} + VARIANT_TYPE_DICT['image'] = {'picimage', 'no-image', 'npicimage', + 'multinpicimage', 'multipicimage'} + VARIANT_TYPE_DICT['debuggable'] = {'ndebuggable', 'debuggable'} + VARIANT_TYPE_DICT['gc'] = {'gcstress', 'gcverify', 'cms'} + VARIANT_TYPE_DICT['prebuild'] = {'no-prebuild', 'no-dex2oat', 'prebuild'} + VARIANT_TYPE_DICT['relocate'] = {'relocate-npatchoat', 'relocate', 'no-relocate'} + VARIANT_TYPE_DICT['jni'] = {'jni', 'forcecopy', 'checkjni'} + VARIANT_TYPE_DICT['address_sizes'] = {'64', '32'} + VARIANT_TYPE_DICT['compiler'] = {'interp-ac', 'interpreter', 'jit', 'optimizing', + 'regalloc_gc'} + + for v_type in VARIANT_TYPE_DICT: + TOTAL_VARIANTS_SET = TOTAL_VARIANTS_SET.union(VARIANT_TYPE_DICT.get(v_type)) + + test_dir = env.ANDROID_BUILD_TOP + '/art/test' + for f in os.listdir(test_dir): + if fnmatch.fnmatch(f, '[0-9]*'): + RUN_TEST_SET.add(f) + DISABLED_TEST_CONTAINER = get_disabled_test_info() + + +def setup_test_env(): + """The method sets default value for the various variants of the tests if they + are already not set. + """ + if env.ART_TEST_BISECTION: + env.ART_TEST_RUN_TEST_NO_PREBUILD = True + env.ART_TEST_RUN_TEST_PREBUILD = False + # Bisection search writes to standard output. + env.ART_TEST_QUIET = False + + if not TARGET_TYPES: + TARGET_TYPES.add('host') + TARGET_TYPES.add('target') + + if env.ART_TEST_RUN_TEST_PREBUILD: + PREBUILD_TYPES.add('prebuild') + if env.ART_TEST_RUN_TEST_NO_PREBUILD: + PREBUILD_TYPES.add('no-prebuild') + if env.ART_TEST_RUN_TEST_NO_DEX2OAT: + PREBUILD_TYPES.add('no-dex2oat') + + if env.ART_TEST_INTERPRETER_ACCESS_CHECKS: + COMPILER_TYPES.add('interp-ac') + if env.ART_TEST_INTERPRETER: + COMPILER_TYPES.add('interpreter') + if env.ART_TEST_JIT: + COMPILER_TYPES.add('jit') + + if env.ART_TEST_OPTIMIZING: + COMPILER_TYPES.add('optimizing') + OPTIMIZING_COMPILER_TYPES.add('optimizing') + if env.ART_TEST_OPTIMIZING_GRAPH_COLOR: + COMPILER_TYPES.add('regalloc_gc') + OPTIMIZING_COMPILER_TYPES.add('regalloc_gc') + + if not RELOCATE_TYPES: + RELOCATE_TYPES.add('no-relocate') + if env.ART_TEST_RUN_TEST_RELOCATE: + RELOCATE_TYPES.add('relocate') + if env.ART_TEST_RUN_TEST_RELOCATE_NO_PATCHOAT: + RELOCATE_TYPES.add('relocate-npatchoat') + + if not TRACE_TYPES: + TRACE_TYPES.add('ntrace') + if env.ART_TEST_TRACE: + TRACE_TYPES.add('trace') + if env.ART_TEST_TRACE_STREAM: + TRACE_TYPES.add('stream') + + if not GC_TYPES: + GC_TYPES.add('cms') + if env.ART_TEST_GC_STRESS: + GC_TYPES.add('gcstress') + if env.ART_TEST_GC_VERIFY: + GC_TYPES.add('gcverify') + + if not JNI_TYPES: + JNI_TYPES.add('checkjni') + if env.ART_TEST_JNI_FORCECOPY: + JNI_TYPES.add('forcecopy') + + if env.ART_TEST_RUN_TEST_IMAGE: + IMAGE_TYPES.add('picimage') + if env.ART_TEST_RUN_TEST_NO_IMAGE: + IMAGE_TYPES.add('no-image') + if env.ART_TEST_RUN_TEST_MULTI_IMAGE: + IMAGE_TYPES.add('multipicimage') + if env.ART_TEST_NPIC_IMAGE: + IMAGE_TYPES.add('npicimage') + if env.ART_TEST_RUN_TEST_MULTI_IMAGE: + IMAGE_TYPES.add('multinpicimage') + + if not PICTEST_TYPES: + PICTEST_TYPES.add('npictest') + if env.ART_TEST_PIC_TEST: + PICTEST_TYPES.add('pictest') + + if env.ART_TEST_RUN_TEST_DEBUG: + RUN_TYPES.add('debug') + if env.ART_TEST_RUN_TEST_NDEBUG: + RUN_TYPES.add('ndebug') + + if not DEBUGGABLE_TYPES: + DEBUGGABLE_TYPES.add('ndebuggable') + + if env.ART_TEST_RUN_TEST_DEBUGGABLE: + DEBUGGABLE_TYPES.add('debuggable') + + if not ADDRESS_SIZES: + ADDRESS_SIZES_TARGET['target'].add(env.ART_PHONY_TEST_TARGET_SUFFIX) + ADDRESS_SIZES_TARGET['host'].add(env.ART_PHONY_TEST_HOST_SUFFIX) + if env.ART_TEST_RUN_TEST_2ND_ARCH: + ADDRESS_SIZES_TARGET['host'].add(env._2ND_ART_PHONY_TEST_HOST_SUFFIX) + ADDRESS_SIZES_TARGET['target'].add(env._2ND_ART_PHONY_TEST_TARGET_SUFFIX) + else: + ADDRESS_SIZES_TARGET['host'] = ADDRESS_SIZES_TARGET['host'].union(ADDRESS_SIZES) + ADDRESS_SIZES_TARGET['target'] = ADDRESS_SIZES_TARGET['target'].union(ADDRESS_SIZES) + + global semaphore + semaphore = threading.Semaphore(n_thread) + + +def run_tests(tests): + """Creates thread workers to run the tests. + + The method generates command and thread worker to run the tests. Depending on + the user input for the number of threads to be used, the method uses a + semaphore object to keep a count in control for the thread workers. When a new + worker is created, it acquires the semaphore object, and when the number of + workers reaches the maximum allowed concurrency, the method wait for an + existing thread worker to release the semaphore object. Worker releases the + semaphore object when they finish printing the output. + + Args: + tests: The set of tests to be run. + """ + options_all = '' + global total_test_count + total_test_count = len(tests) + total_test_count *= len(RUN_TYPES) + total_test_count *= len(PREBUILD_TYPES) + total_test_count *= len(RELOCATE_TYPES) + total_test_count *= len(TRACE_TYPES) + total_test_count *= len(GC_TYPES) + total_test_count *= len(JNI_TYPES) + total_test_count *= len(IMAGE_TYPES) + total_test_count *= len(PICTEST_TYPES) + total_test_count *= len(DEBUGGABLE_TYPES) + total_test_count *= len(COMPILER_TYPES) + target_address_combinations = 0 + for target in TARGET_TYPES: + for address_size in ADDRESS_SIZES_TARGET[target]: + target_address_combinations += 1 + total_test_count *= target_address_combinations + + if env.ART_TEST_WITH_STRACE: + options_all += ' --strace' + + if env.ART_TEST_RUN_TEST_ALWAYS_CLEAN: + options_all += ' --always-clean' + + if env.ART_TEST_BISECTION: + options_all += ' --bisection-search' + + if env.ART_TEST_ANDROID_ROOT: + options_all += ' --android-root ' + env.ART_TEST_ANDROID_ROOT + + if gdb: + options_all += ' --gdb' + if gdb_arg: + options_all += ' --gdb-arg ' + gdb_arg + + config = itertools.product(tests, TARGET_TYPES, RUN_TYPES, PREBUILD_TYPES, + COMPILER_TYPES, RELOCATE_TYPES, TRACE_TYPES, + GC_TYPES, JNI_TYPES, IMAGE_TYPES, PICTEST_TYPES, + DEBUGGABLE_TYPES) + + for test, target, run, prebuild, compiler, relocate, trace, gc, \ + jni, image, pictest, debuggable in config: + for address_size in ADDRESS_SIZES_TARGET[target]: + if stop_testrunner: + # When ART_TEST_KEEP_GOING is set to false, then as soon as a test + # fails, stop_testrunner is set to True. When this happens, the method + # stops creating any any thread and wait for all the exising threads + # to end. + while threading.active_count() > 2: + time.sleep(0.1) + return + test_name = 'test-art-' + test_name += target + '-run-test-' + test_name += run + '-' + test_name += prebuild + '-' + test_name += compiler + '-' + test_name += relocate + '-' + test_name += trace + '-' + test_name += gc + '-' + test_name += jni + '-' + test_name += image + '-' + test_name += pictest + '-' + test_name += debuggable + '-' + test_name += test + test_name += address_size + + variant_set = {target, run, prebuild, compiler, relocate, trace, gc, jni, + image, pictest, debuggable, address_size} + + options_test = options_all + + if target == 'host': + options_test += ' --host' + + if run == 'ndebug': + options_test += ' -O' + + if prebuild == 'prebuild': + options_test += ' --prebuild' + elif prebuild == 'no-prebuild': + options_test += ' --no-prebuild' + elif prebuild == 'no-dex2oat': + options_test += ' --no-prebuild --no-dex2oat' + + if compiler == 'optimizing': + options_test += ' --optimizing' + elif compiler == 'regalloc_gc': + options_test += ' --optimizing -Xcompiler-option --register-allocation-strategy=graph-color' + elif compiler == 'interpreter': + options_test += ' --interpreter' + elif compiler == 'interp-ac': + options_test += ' --interpreter --verify-soft-fail' + elif compiler == 'jit': + options_test += ' --jit' + + if relocate == 'relocate': + options_test += ' --relocate' + elif relocate == 'no-relocate': + options_test += ' --no-relocate' + elif relocate == 'relocate-npatchoat': + options_test += ' --relocate --no-patchoat' + + if trace == 'trace': + options_test += ' --trace' + elif trace == 'stream': + options_test += ' --trace --stream' + + if gc == 'gcverify': + options_test += ' --gcverify' + elif gc == 'gcstress': + options_test += ' --gcstress' + + if jni == 'forcecopy': + options_test += ' --runtime-option -Xjniopts:forcecopy' + elif jni == 'checkjni': + options_test += ' --runtime-option -Xcheck:jni' + + if image == 'no-image': + options_test += ' --no-image' + elif image == 'npicimage': + options_test += ' --npic-image' + elif image == 'multinpicimage': + options_test += ' --npic-image --multi-image' + elif image == 'multipicimage': + options_test += ' --multi-image' + + if pictest == 'pictest': + options_test += ' --pic-test' + + if debuggable == 'debuggable': + options_test += ' --debuggable' + + if address_size == '64': + options_test += ' --64' + + if env.DEX2OAT_HOST_INSTRUCTION_SET_FEATURES: + options_test += ' --instruction-set-features' + env.DEX2OAT_HOST_INSTRUCTION_SET_FEATURES + + elif address_size == '32': + if env.HOST_2ND_ARCH_PREFIX_DEX2OAT_HOST_INSTRUCTION_SET_FEATURES: + options_test += ' --instruction-set-features ' + \ + env.HOST_2ND_ARCH_PREFIX_DEX2OAT_HOST_INSTRUCTION_SET_FEATURES + + options_test = (' --output-path %s/run-test-output/%s') % ( + env.ART_HOST_TEST_DIR, test_name) + options_test + + run_test_sh = env.ANDROID_BUILD_TOP + '/art/test/run-test' + command = run_test_sh + ' ' + options_test + ' ' + test + + semaphore.acquire() + worker = threading.Thread(target=run_test, args=(command, test, variant_set, test_name)) + worker.daemon = True + worker.start() + + while threading.active_count() > 2: + time.sleep(0.1) + + +def run_test(command, test, test_variant, test_name): + """Runs the test. + + It invokes art/test/run-test script to run the test. The output of the script + is checked, and if it ends with "Succeeded!", it assumes that the tests + passed, otherwise, put it in the list of failed test. Before actually running + the test, it also checks if the test is placed in the list of disabled tests, + and if yes, it skips running it, and adds the test in the list of skipped + tests. The method uses print_text method to actually print the output. After + successfully running and capturing the output for the test, it releases the + semaphore object. + + Args: + command: The command to be used to invoke the script + test: The name of the test without the variant information. + test_variant: The set of variant for the test. + test_name: The name of the test along with the variants. + """ + global last_print_length + global test_count + global stop_testrunner + if is_test_disabled(test, test_variant): + test_skipped = True + else: + test_skipped = False + proc = subprocess.Popen(command.split(), stderr=subprocess.PIPE) + script_output = proc.stderr.read().strip() + test_passed = not proc.wait() + + # If verbose is set to True, every test information is printed on a new line. + # If not, the information is printed on the same line overriding the + # previous test output. + if not verbose: + suffix = '\r' + prefix = ' ' * last_print_length + '\r' + else: + suffix = '\n' + prefix = '' + test_count_mutex.acquire() + test_count += 1 + percent = (test_count * 100) / total_test_count + out = '[ ' + str(percent) + '% ' + str(test_count) + '/' + str(total_test_count) + ' ] ' + test_count_mutex.release() + out += test_name + ' ' + if not test_skipped: + if test_passed: + out += COLOR_PASS + 'PASS' + COLOR_NORMAL + else: + out += COLOR_ERROR + 'FAIL' + COLOR_NORMAL + failed_tests.append(test_name) + if verbose: + out += '\n' + command + '\n' + script_output + if not env.ART_TEST_KEEP_GOING: + stop_testrunner = True + elif not dry_run: + out += COLOR_SKIP + 'SKIP' + COLOR_NORMAL + skipped_tests.append(test_name) + last_print_length = len(out) + print_mutex.acquire() + print_text(prefix + out + suffix) + print_mutex.release() + semaphore.release() + + +def get_disabled_test_info(): + """Generate set of known failures. + + It parses the art/test/knownfailures.json file to generate the list of + disabled tests. + + Returns: + The method returns a dict of tests mapped to the variants list + for which the test should not be run. + """ + known_failures_file = env.ANDROID_BUILD_TOP + '/art/test/knownfailures.json' + with open(known_failures_file) as known_failures_json: + known_failures_info = json.loads(known_failures_json.read()) + + disabled_test_info = {} + for failure in known_failures_info: + tests = failure.get('test') + if tests: + tests = [tests] + else: + tests = failure.get('tests', []) + variants = parse_variants(failure.get('variant')) + env_vars = failure.get('env_vars') + if check_env_vars(env_vars): + for test in tests: + if test in disabled_test_info: + disabled_test_info[test] = disabled_test_info[test].union(variants) + else: + disabled_test_info[test] = variants + return disabled_test_info + + +def check_env_vars(env_vars): + """Checks if the env variables are set as required to run the test. + + Returns: + True if all the env variables are set as required, otherwise False. + """ + + if not env_vars: + return True + for key in env_vars: + if env.get_env(key) != env_vars.get(key): + return False + return True + + +def is_test_disabled(test, variant_set): + """Checks if the test along with the variant_set is disabled. + + Args: + test: The name of the test as in art/test directory. + variant_set: Variants to be used for the test. + Returns: + True, if the test is disabled. + """ + if dry_run: + return True + variants_list = DISABLED_TEST_CONTAINER.get(test, {}) + for variants in variants_list: + variants_present = True + for variant in variants: + if variant not in variant_set: + variants_present = False + break + if variants_present: + return True + return False + + +def parse_variants(variants): + """Parse variants fetched from art/test/knownfailures.json. + """ + if not variants: + variants = '' + for variant in TOTAL_VARIANTS_SET: + variants += variant + variants += '|' + variants = variants[:-1] + variant_list = set() + or_variants = variants.split('|') + for or_variant in or_variants: + and_variants = or_variant.split('&') + variant = set() + for and_variant in and_variants: + and_variant = and_variant.strip() + variant.add(and_variant) + variant_list.add(frozenset(variant)) + return variant_list + +def print_text(output): + sys.stdout.write(output) + sys.stdout.flush() + +def print_analysis(): + if not verbose: + print_text(' ' * last_print_length + '\r') + if skipped_tests: + print_text(COLOR_SKIP + 'SKIPPED TESTS' + COLOR_NORMAL + '\n') + for test in skipped_tests: + print_text(test + '\n') + print_text('\n') + + if failed_tests: + print_text(COLOR_ERROR + 'FAILED TESTS' + COLOR_NORMAL + '\n') + for test in failed_tests: + print_text(test + '\n') + + +def parse_test_name(test_name): + """Parses the testname provided by the user. + It supports two types of test_name: + 1) Like 001-HelloWorld. In this case, it will just verify if the test actually + exists and if it does, it returns the testname. + 2) Like test-art-host-run-test-debug-prebuild-interpreter-no-relocate-ntrace-cms-checkjni-picimage-npictest-ndebuggable-001-HelloWorld32 + In this case, it will parse all the variants and check if they are placed + correctly. If yes, it will set the various VARIANT_TYPES to use the + variants required to run the test. Again, it returns the test_name + without the variant information like 001-HelloWorld. + """ + if test_name in RUN_TEST_SET: + return {test_name} + + regex = '^test-art-' + regex += '(' + '|'.join(VARIANT_TYPE_DICT['target']) + ')-' + regex += 'run-test-' + regex += '(' + '|'.join(VARIANT_TYPE_DICT['run']) + ')-' + regex += '(' + '|'.join(VARIANT_TYPE_DICT['prebuild']) + ')-' + regex += '(' + '|'.join(VARIANT_TYPE_DICT['compiler']) + ')-' + regex += '(' + '|'.join(VARIANT_TYPE_DICT['relocate']) + ')-' + regex += '(' + '|'.join(VARIANT_TYPE_DICT['trace']) + ')-' + regex += '(' + '|'.join(VARIANT_TYPE_DICT['gc']) + ')-' + regex += '(' + '|'.join(VARIANT_TYPE_DICT['jni']) + ')-' + regex += '(' + '|'.join(VARIANT_TYPE_DICT['image']) + ')-' + regex += '(' + '|'.join(VARIANT_TYPE_DICT['pictest']) + ')-' + regex += '(' + '|'.join(VARIANT_TYPE_DICT['debuggable']) + ')-' + regex += '(' + '|'.join(RUN_TEST_SET) + ')' + regex += '(' + '|'.join(VARIANT_TYPE_DICT['address_sizes']) + ')$' + match = re.match(regex, test_name) + if match: + TARGET_TYPES.add(match.group(1)) + RUN_TYPES.add(match.group(2)) + PREBUILD_TYPES.add(match.group(3)) + COMPILER_TYPES.add(match.group(4)) + RELOCATE_TYPES.add(match.group(5)) + TRACE_TYPES.add(match.group(6)) + GC_TYPES.add(match.group(7)) + JNI_TYPES.add(match.group(8)) + IMAGE_TYPES.add(match.group(9)) + PICTEST_TYPES.add(match.group(10)) + DEBUGGABLE_TYPES.add(match.group(11)) + ADDRESS_SIZES.add(match.group(13)) + return {match.group(12)} + + +def parse_option(): + global verbose + global dry_run + global n_thread + global build + global gdb + global gdb_arg + + parser = OptionParser() + parser.add_option('-t', '--test', dest='test', help='name of the test') + parser.add_option('-j', type='int', dest='n_thread') + for variant in TOTAL_VARIANTS_SET: + flag = '--' + variant + flag_dest = variant.replace('-', '_') + if variant == '32' or variant == '64': + flag_dest = 'n' + flag_dest + parser.add_option(flag, action='store_true', dest=flag_dest) + parser.add_option('--verbose', '-v', action='store_true', dest='verbose') + parser.add_option('--dry-run', action='store_true', dest='dry_run') + parser.add_option('-b', '--build-dependencies', action='store_true', dest='build') + parser.add_option('--gdb', action='store_true', dest='gdb') + parser.add_option('--gdb-arg', dest='gdb_arg') + + options = parser.parse_args()[0] + test = '' + if options.test: + test = parse_test_name(options.test) + if options.pictest: + PICTEST_TYPES.add('pictest') + if options.ndebug: + RUN_TYPES.add('ndebug') + if options.interp_ac: + COMPILER_TYPES.add('interp-ac') + if options.picimage: + IMAGE_TYPES.add('picimage') + if options.n64: + ADDRESS_SIZES.add('64') + if options.interpreter: + COMPILER_TYPES.add('interpreter') + if options.jni: + JNI_TYPES.add('jni') + if options.relocate_npatchoat: + RELOCATE_TYPES.add('relocate-npatchoat') + if options.no_prebuild: + PREBUILD_TYPES.add('no-prebuild') + if options.npictest: + PICTEST_TYPES.add('npictest') + if options.no_dex2oat: + PREBUILD_TYPES.add('no-dex2oat') + if options.jit: + COMPILER_TYPES.add('jit') + if options.relocate: + RELOCATE_TYPES.add('relocate') + if options.ndebuggable: + DEBUGGABLE_TYPES.add('ndebuggable') + if options.no_image: + IMAGE_TYPES.add('no-image') + if options.optimizing: + COMPILER_TYPES.add('optimizing') + if options.trace: + TRACE_TYPES.add('trace') + if options.gcstress: + GC_TYPES.add('gcstress') + if options.no_relocate: + RELOCATE_TYPES.add('no-relocate') + if options.target: + TARGET_TYPES.add('target') + if options.forcecopy: + JNI_TYPES.add('forcecopy') + if options.n32: + ADDRESS_SIZES.add('32') + if options.host: + TARGET_TYPES.add('host') + if options.gcverify: + GC_TYPES.add('gcverify') + if options.debuggable: + DEBUGGABLE_TYPES.add('debuggable') + if options.prebuild: + PREBUILD_TYPES.add('prebuild') + if options.debug: + RUN_TYPES.add('debug') + if options.checkjni: + JNI_TYPES.add('checkjni') + if options.ntrace: + TRACE_TYPES.add('ntrace') + if options.cms: + GC_TYPES.add('cms') + if options.npicimage: + IMAGE_TYPES.add('npicimage') + if options.multinpicimage: + IMAGE_TYPES.add('multinpicimage') + if options.multipicimage: + IMAGE_TYPES.add('multipicimage') + if options.verbose: + verbose = True + if options.n_thread: + n_thread = max(1, options.n_thread) + if options.dry_run: + dry_run = True + verbose = True + if options.build: + build = True + if options.gdb: + n_thread = 1 + gdb = True + if options.gdb_arg: + gdb_arg = options.gdb_arg + + return test + +def main(): + gather_test_info() + user_requested_test = parse_option() + setup_test_env() + if build: + build_targets = '' + if 'host' in TARGET_TYPES: + build_targets += 'test-art-host-run-test-dependencies' + if 'target' in TARGET_TYPES: + build_targets += 'test-art-target-run-test-dependencies' + build_command = 'make -j' + str(n_thread) + ' ' + build_targets + if subprocess.call(build_command.split()): + sys.exit(1) + if user_requested_test: + test_runner_thread = threading.Thread(target=run_tests, args=(user_requested_test,)) + else: + test_runner_thread = threading.Thread(target=run_tests, args=(RUN_TEST_SET,)) + test_runner_thread.daemon = True + try: + test_runner_thread.start() + while threading.active_count() > 1: + time.sleep(0.1) + print_analysis() + if failed_tests: + sys.exit(1) + sys.exit(0) + except SystemExit: + pass + except: + print "hello" + print_analysis() + sys.exit(1) + + +if __name__ == '__main__': + main() |