diff options
| -rw-r--r-- | test/Android.run-test.mk | 1181 | ||||
| -rw-r--r-- | test/knownfailures.json | 359 | ||||
| -rw-r--r-- | test/testrunner/env.py | 174 | ||||
| -rwxr-xr-x | test/testrunner/testrunner.py | 794 |
4 files changed, 1039 insertions, 1469 deletions
diff --git a/test/Android.run-test.mk b/test/Android.run-test.mk index 770fc21102..742353da46 100644 --- a/test/Android.run-test.mk +++ b/test/Android.run-test.mk @@ -14,6 +14,7 @@ # LOCAL_PATH := $(call my-dir) + include art/build/Android.common_test.mk # List of all tests of the form 003-omnibus-opcodes. @@ -106,53 +107,690 @@ TEST_ART_RUN_TEST_BUILD_RULES := # General rules to build and run a run-test. TARGET_TYPES := host target - -PREBUILD_TYPES := prebuild -PREBUILD_TYPES += no-prebuild -PREBUILD_TYPES += no-dex2oat - -COMPILER_TYPES := interp-ac -COMPILER_TYPES += interpreter -COMPILER_TYPES += jit -COMPILER_TYPES += optimizing -COMPILER_TYPES += regalloc_gc - -RELOCATE_TYPES := relocate -RELOCATE_TYPES += no-relocate -RELOCATE_TYPES += relocate-npatchoat - +PREBUILD_TYPES := +ifeq ($(ART_TEST_RUN_TEST_PREBUILD),true) + PREBUILD_TYPES += prebuild +endif +ifeq ($(ART_TEST_RUN_TEST_NO_PREBUILD),true) + PREBUILD_TYPES += no-prebuild +endif +ifeq ($(ART_TEST_RUN_TEST_NO_DEX2OAT),true) + PREBUILD_TYPES += no-dex2oat +endif +COMPILER_TYPES := +ifeq ($(ART_TEST_INTERPRETER_ACCESS_CHECKS),true) + COMPILER_TYPES += interp-ac +endif +ifeq ($(ART_TEST_INTERPRETER),true) + COMPILER_TYPES += interpreter +endif +ifeq ($(ART_TEST_JIT),true) + COMPILER_TYPES += jit +endif +OPTIMIZING_COMPILER_TYPES := +ifeq ($(ART_TEST_OPTIMIZING),true) + COMPILER_TYPES += optimizing + OPTIMIZING_COMPILER_TYPES += optimizing +endif +ifeq ($(ART_TEST_OPTIMIZING_GRAPH_COLOR),true) + COMPILER_TYPES += regalloc_gc + OPTIMIZING_COMPILER_TYPES += regalloc_gc +endif +RELOCATE_TYPES := no-relocate +ifeq ($(ART_TEST_RUN_TEST_RELOCATE),true) + RELOCATE_TYPES += relocate +endif +ifeq ($(ART_TEST_RUN_TEST_RELOCATE_NO_PATCHOAT),true) + RELOCATE_TYPES += relocate-npatchoat +endif TRACE_TYPES := ntrace -TRACE_TYPES += trace -TRACE_TYPES += stream - +ifeq ($(ART_TEST_TRACE),true) + TRACE_TYPES += trace +endif +ifeq ($(ART_TEST_TRACE_STREAM),true) + TRACE_TYPES += stream +endif GC_TYPES := cms -GC_TYPES += gcstress -GC_TYPES += gcverify - +ifeq ($(ART_TEST_GC_STRESS),true) + GC_TYPES += gcstress +endif +ifeq ($(ART_TEST_GC_VERIFY),true) + GC_TYPES += gcverify +endif JNI_TYPES := checkjni -JNI_TYPES += forcecopy - +ifeq ($(ART_TEST_JNI_FORCECOPY),true) + JNI_TYPES += forcecopy +endif +ifeq ($(ART_TEST_RUN_TEST_IMAGE),true) IMAGE_TYPES := picimage -IMAGE_TYPES += no-image -IMAGE_TYPES += multipicimage -IMAGE_TYPES += npicimage -IMAGE_TYPES += multinpicimage - +endif +ifeq ($(ART_TEST_RUN_TEST_NO_IMAGE),true) + IMAGE_TYPES += no-image +endif +ifeq ($(ART_TEST_RUN_TEST_MULTI_IMAGE),true) + IMAGE_TYPES := multipicimage +endif +ifeq ($(ART_TEST_NPIC_IMAGE),true) + IMAGE_TYPES += npicimage + ifeq ($(ART_TEST_RUN_TEST_MULTI_IMAGE),true) + IMAGE_TYPES := multinpicimage + endif +endif PICTEST_TYPES := npictest -PICTEST_TYPES += pictest - -RUN_TYPES := debug -RUN_TYPES += ndebug - +ifeq ($(ART_TEST_PIC_TEST),true) + PICTEST_TYPES += pictest +endif +RUN_TYPES := +ifeq ($(ART_TEST_RUN_TEST_DEBUG),true) + RUN_TYPES += debug +endif +ifeq ($(ART_TEST_RUN_TEST_NDEBUG),true) + RUN_TYPES += ndebug +endif DEBUGGABLE_TYPES := ndebuggable +ifeq ($(ART_TEST_RUN_TEST_DEBUGGABLE),true) DEBUGGABLE_TYPES += debuggable - +endif +ADDRESS_SIZES_TARGET := $(ART_PHONY_TEST_TARGET_SUFFIX) +ADDRESS_SIZES_HOST := $(ART_PHONY_TEST_HOST_SUFFIX) +ifeq ($(ART_TEST_RUN_TEST_2ND_ARCH),true) + ADDRESS_SIZES_TARGET += $(2ND_ART_PHONY_TEST_TARGET_SUFFIX) + ADDRESS_SIZES_HOST += $(2ND_ART_PHONY_TEST_HOST_SUFFIX) +endif ALL_ADDRESS_SIZES := 64 32 +# List all run test names with number arguments agreeing with the comment above. +define all-run-test-names + $(foreach target, $(1), \ + $(foreach run-type, $(2), \ + $(foreach prebuild, $(3), \ + $(foreach compiler, $(4), \ + $(foreach relocate, $(5), \ + $(foreach trace, $(6), \ + $(foreach gc, $(7), \ + $(foreach jni, $(8), \ + $(foreach image, $(9), \ + $(foreach pictest, $(10), \ + $(foreach debuggable, $(11), \ + $(foreach test, $(12), \ + $(foreach address_size, $(13), \ + test-art-$(target)-run-test-$(run-type)-$(prebuild)-$(compiler)-$(relocate)-$(trace)-$(gc)-$(jni)-$(image)-$(pictest)-$(debuggable)-$(test)$(address_size) \ + ))))))))))))) +endef # all-run-test-names + +# To generate a full list or tests: +# $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES),$(COMPILER_TYPES), \ +# $(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES),$(IMAGE_TYPES), \ +# $(PICTEST_TYPES),$(DEBUGGABLE_TYPES),$(TEST_ART_RUN_TESTS),$(ALL_ADDRESS_SIZES)) + +# Convert's a rule name to the form used in variables, e.g. no-relocate to NO_RELOCATE define name-to-var $(shell echo $(1) | tr '[:lower:]' '[:upper:]' | tr '-' '_') endef # name-to-var +# Disable 153-reference-stress temporarily until a fix arrives. b/33389022. +# Disable 080-oom-fragmentation due to flakes. b/33795328 +# Disable 497-inlining-and-class-loader and 542-unresolved-access-check until +# they are rewritten. These tests use a broken class loader that tries to +# register a dex file that's already registered with a different loader. +# b/34193123 +ART_TEST_RUN_TEST_SKIP += \ + 153-reference-stress \ + 080-oom-fragmentation \ + 497-inlining-and-class-loader \ + 542-unresolved-access-check + +ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \ + $(COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \ + $(IMAGE_TYPES), $(PICTEST_TYPES), $(DEBUGGABLE_TYPES), $(ART_TEST_RUN_TEST_SKIP), $(ALL_ADDRESS_SIZES)) + + +# Disable 149-suspend-all-stress, its output is flaky (b/28988206). +# Disable 577-profile-foreign-dex (b/27454772). +TEST_ART_BROKEN_ALL_TARGET_TESTS := \ + 149-suspend-all-stress \ + 577-profile-foreign-dex \ + +ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \ + $(COMPILER_TYPES), $(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \ + $(IMAGE_TYPES),$(PICTEST_TYPES),$(DEBUGGABLE_TYPES), $(TEST_ART_BROKEN_ALL_TARGET_TESTS), \ + $(ALL_ADDRESS_SIZES)) + +TEST_ART_BROKEN_ALL_TARGET_TESTS := + +# Tests that are timing sensitive and flaky on heavily loaded systems. +TEST_ART_TIMING_SENSITIVE_RUN_TESTS := \ + 002-sleep \ + 053-wait-some \ + 055-enum-performance \ + 133-static-invoke-super + +# disable timing sensitive tests on "dist" builds. +ifdef dist_goal + ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \ + $(COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \ + $(IMAGE_TYPES), $(PICTEST_TYPES), $(DEBUGGABLE_TYPES), $(TEST_ART_TIMING_SENSITIVE_RUN_TESTS), $(ALL_ADDRESS_SIZES)) +endif + +# 147-stripped-dex-fallback isn't supported on device because --strip-dex +# requires the zip command. +# 569-checker-pattern-replacement tests behaviour present only on host. +TEST_ART_BROKEN_TARGET_TESTS := \ + 147-stripped-dex-fallback \ + 569-checker-pattern-replacement + +ifneq (,$(filter target,$(TARGET_TYPES))) + ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,target,$(RUN_TYPES),$(PREBUILD_TYPES), \ + $(COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \ + $(IMAGE_TYPES), $(PICTEST_TYPES), $(DEBUGGABLE_TYPES), $(TEST_ART_BROKEN_TARGET_TESTS), $(ALL_ADDRESS_SIZES)) +endif + +TEST_ART_BROKEN_TARGET_TESTS := + +# Tests that require python3. +TEST_ART_PYTHON3_DEPENDENCY_RUN_TESTS := \ + 960-default-smali \ + 961-default-iface-resolution-gen \ + 964-default-iface-init-gen \ + 968-default-partial-compile-gen \ + 969-iface-super \ + 970-iface-super-resolution-gen \ + 971-iface-super + +# Check if we have python3 to run our tests. +ifeq ($(wildcard /usr/bin/python3),) + $(warning "No python3 found. Disabling tests: $(TEST_ART_PYTHON3_DEPENDENCY_RUN_TESTS)") + + # Currently disable tests requiring python3 when it is not installed. + ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \ + $(COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \ + $(IMAGE_TYPES), $(PICTEST_TYPES), $(DEBUGGABLE_TYPES), $(TEST_ART_PYTHON3_DEPENDENCY_RUN_TESTS), $(ALL_ADDRESS_SIZES)) +endif + +TEST_ART_TIMING_SENSITIVE_RUN_TESTS := + +# Note 116-nodex2oat is not broken per-se it just doesn't (and isn't meant to) work with --prebuild. +TEST_ART_BROKEN_PREBUILD_RUN_TESTS := \ + 116-nodex2oat \ + 118-noimage-dex2oat \ + 134-nodex2oat-nofallback + +ifneq (,$(filter prebuild,$(PREBUILD_TYPES))) + ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),prebuild, \ + $(COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \ + $(IMAGE_TYPES), $(PICTEST_TYPES), $(DEBUGGABLE_TYPES), $(TEST_ART_BROKEN_PREBUILD_RUN_TESTS), $(ALL_ADDRESS_SIZES)) +endif + +TEST_ART_BROKEN_PREBUILD_RUN_TESTS := + +# 554-jit-profile-file is disabled because it needs a primary oat file to know what it should save. +# 529 and 555: b/27784033 +TEST_ART_BROKEN_NO_PREBUILD_TESTS := \ + 117-nopatchoat \ + 147-stripped-dex-fallback \ + 554-jit-profile-file \ + 529-checker-unresolved \ + 555-checker-regression-x86const \ + 608-checker-unresolved-lse + +ifneq (,$(filter no-prebuild,$(PREBUILD_TYPES))) + ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),no-prebuild, \ + $(COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \ + $(IMAGE_TYPES), $(PICTEST_TYPES), $(DEBUGGABLE_TYPES), $(TEST_ART_BROKEN_NO_PREBUILD_TESTS), $(ALL_ADDRESS_SIZES)) +endif + +TEST_ART_BROKEN_NO_PREBUILD_TESTS := + +# Note 117-nopatchoat is not broken per-se it just doesn't work (and isn't meant to) without +# --prebuild --relocate +TEST_ART_BROKEN_NO_RELOCATE_TESTS := \ + 117-nopatchoat \ + 118-noimage-dex2oat \ + 119-noimage-patchoat \ + 554-jit-profile-file + +ifneq (,$(filter no-relocate,$(RELOCATE_TYPES))) + ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \ + $(COMPILER_TYPES), no-relocate,$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \ + $(IMAGE_TYPES), $(PICTEST_TYPES), $(DEBUGGABLE_TYPES), $(TEST_ART_BROKEN_NO_RELOCATE_TESTS), $(ALL_ADDRESS_SIZES)) +endif + +TEST_ART_BROKEN_NO_RELOCATE_TESTS := + +# Temporarily disable some broken tests when forcing access checks in interpreter b/22414682 +# 629 requires compilation. +TEST_ART_BROKEN_INTERPRETER_ACCESS_CHECK_TESTS := \ + 137-cfi \ + 629-vdex-speed + +ifneq (,$(filter interp-ac,$(COMPILER_TYPES))) + ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \ + interp-ac,$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \ + $(IMAGE_TYPES), $(PICTEST_TYPES), $(DEBUGGABLE_TYPES), $(TEST_ART_BROKEN_INTERPRETER_ACCESS_CHECK_TESTS), $(ALL_ADDRESS_SIZES)) +endif + +TEST_ART_BROKEN_INTERPRETER_ACCESS_CHECK_TESTS := + +# Tests that are broken with GC stress. +# * 137-cfi needs to unwind a second forked process. We're using a primitive sleep to wait till we +# hope the second process got into the expected state. The slowness of gcstress makes this bad. +# * 908-gc-start-finish expects GCs only to be run at clear points. The reduced heap size makes +# this non-deterministic. Same for 913. +# * 961-default-iface-resolution-gen and 964-default-iface-init-genare very long tests that often +# will take more than the timeout to run when gcstress is enabled. This is because gcstress +# slows down allocations significantly which these tests do a lot. +TEST_ART_BROKEN_GCSTRESS_RUN_TESTS := \ + 137-cfi \ + 154-gc-loop \ + 908-gc-start-finish \ + 913-heaps \ + 961-default-iface-resolution-gen \ + 964-default-iface-init-gen + +ifneq (,$(filter gcstress,$(GC_TYPES))) + ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \ + $(COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),gcstress,$(JNI_TYPES), \ + $(IMAGE_TYPES), $(PICTEST_TYPES), $(DEBUGGABLE_TYPES), $(TEST_ART_BROKEN_GCSTRESS_RUN_TESTS), $(ALL_ADDRESS_SIZES)) +endif + +TEST_ART_BROKEN_GCSTRESS_RUN_TESTS := + +# 115-native-bridge setup is complicated. Need to implement it correctly for the target. +ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,target,$(RUN_TYPES),$(PREBUILD_TYPES),$(COMPILER_TYPES), \ + $(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES),$(IMAGE_TYPES),$(PICTEST_TYPES),$(DEBUGGABLE_TYPES), 115-native-bridge, \ + $(ALL_ADDRESS_SIZES)) + +# 130-hprof dumps the heap and runs hprof-conv to check whether the file is somewhat readable. This +# is only possible on the host. +# TODO: Turn off all the other combinations, this is more about testing actual ART code. A gtest is +# very hard to write here, as (for a complete test) JDWP must be set up. +ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,target,$(RUN_TYPES),$(PREBUILD_TYPES), \ + $(COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES),$(IMAGE_TYPES), \ + $(PICTEST_TYPES),$(DEBUGGABLE_TYPES),130-hprof,$(ALL_ADDRESS_SIZES)) + +# 131 is an old test. The functionality has been implemented at an earlier stage and is checked +# in tests 138. Blacklisted for debug builds since these builds have duplicate classes checks which +# punt to interpreter. +ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),debug,$(PREBUILD_TYPES), \ + $(COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES),$(IMAGE_TYPES), \ + $(PICTEST_TYPES),$(DEBUGGABLE_TYPES),131-structural-change,$(ALL_ADDRESS_SIZES)) + +# 138-duplicate-classes-check. Turned on for debug builds since debug builds have duplicate classes +# checks enabled, b/2133391. +ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),ndebug,$(PREBUILD_TYPES), \ + $(COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES),$(IMAGE_TYPES), \ + $(PICTEST_TYPES),$(DEBUGGABLE_TYPES),138-duplicate-classes-check,$(ALL_ADDRESS_SIZES)) + +# All these tests check that we have sane behavior if we don't have a patchoat or dex2oat. +# Therefore we shouldn't run them in situations where we actually don't have these since they +# explicitly test for them. These all also assume we have an image. +# 147-stripped-dex-fallback is disabled because it requires --prebuild. +# 554-jit-profile-file is disabled because it needs a primary oat file to know what it should save. +# 629-vdex-speed requires compiled code. +TEST_ART_BROKEN_FALLBACK_RUN_TESTS := \ + 116-nodex2oat \ + 117-nopatchoat \ + 118-noimage-dex2oat \ + 119-noimage-patchoat \ + 137-cfi \ + 138-duplicate-classes-check2 \ + 147-stripped-dex-fallback \ + 554-jit-profile-file \ + 629-vdex-speed + +# This test fails without an image. +# 018, 961, 964, 968 often time out. b/34369284 +TEST_ART_BROKEN_NO_IMAGE_RUN_TESTS := \ + 137-cfi \ + 138-duplicate-classes-check \ + 018-stack-overflow \ + 961-default-iface-resolution-gen \ + 964-default-iface-init \ + 968-default-partial-compile-gen \ + +ifneq (,$(filter no-dex2oat,$(PREBUILD_TYPES))) + ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),no-dex2oat, \ + $(COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES),$(IMAGE_TYPES), \ + $(PICTEST_TYPES),$(DEBUGGABLE_TYPES), $(TEST_ART_BROKEN_FALLBACK_RUN_TESTS),$(ALL_ADDRESS_SIZES)) +endif + + +ifneq (,$(filter no-image,$(IMAGE_TYPES))) + ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \ + $(COMPILER_TYPES), $(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES),no-image, \ + $(PICTEST_TYPES), $(DEBUGGABLE_TYPES), $(TEST_ART_BROKEN_FALLBACK_RUN_TESTS),$(ALL_ADDRESS_SIZES)) + ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \ + $(COMPILER_TYPES), $(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES),no-image, \ + $(PICTEST_TYPES), $(DEBUGGABLE_TYPES), $(TEST_ART_BROKEN_NO_IMAGE_RUN_TESTS),$(ALL_ADDRESS_SIZES)) +endif + +ifneq (,$(filter relocate-npatchoat,$(RELOCATE_TYPES))) + ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \ + $(COMPILER_TYPES), relocate-npatchoat,$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \ + $(IMAGE_TYPES),$(PICTEST_TYPES),$(DEBUGGABLE_TYPES), $(TEST_ART_BROKEN_FALLBACK_RUN_TESTS),$(ALL_ADDRESS_SIZES)) +endif + +TEST_ART_BROKEN_FALLBACK_RUN_TESTS := + +# 137: +# This test unrolls and expects managed frames, but tracing means we run the interpreter. +# 802 and 570-checker-osr: +# This test dynamically enables tracing to force a deoptimization. This makes the test meaningless +# when already tracing, and writes an error message that we do not want to check for. +# 130 occasional timeout b/32383962. +# 629 requires compilation. +TEST_ART_BROKEN_TRACING_RUN_TESTS := \ + 087-gc-after-link \ + 130-hprof \ + 137-cfi \ + 141-class-unload \ + 570-checker-osr \ + 629-vdex-speed \ + 802-deoptimization + +ifneq (,$(filter trace stream,$(TRACE_TYPES))) + ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \ + $(COMPILER_TYPES),$(RELOCATE_TYPES),trace stream,$(GC_TYPES),$(JNI_TYPES),$(IMAGE_TYPES), \ + $(PICTEST_TYPES),$(DEBUGGABLE_TYPES), $(TEST_ART_BROKEN_TRACING_RUN_TESTS),$(ALL_ADDRESS_SIZES)) +endif + +TEST_ART_BROKEN_TRACING_RUN_TESTS := + +# These tests expect JIT compilation, which is suppressed when tracing. +TEST_ART_BROKEN_JIT_TRACING_RUN_TESTS := \ + 604-hot-static-interface \ + 612-jit-dex-cache \ + 613-inlining-dex-cache \ + 616-cha \ + 626-set-resolved-string \ + +ifneq (,$(filter trace stream,$(TRACE_TYPES))) + ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \ + jit,$(RELOCATE_TYPES),trace stream,$(GC_TYPES),$(JNI_TYPES),$(IMAGE_TYPES), \ + $(PICTEST_TYPES),$(DEBUGGABLE_TYPES), $(TEST_ART_BROKEN_JIT_TRACING_RUN_TESTS),$(ALL_ADDRESS_SIZES)) +endif + +TEST_ART_BROKEN_JIT_TRACING_RUN_TESTS := + +# Known broken tests for the interpreter. +# CFI unwinding expects managed frames. +# 629 requires compilation. +TEST_ART_BROKEN_INTERPRETER_RUN_TESTS := \ + 137-cfi \ + 554-jit-profile-file \ + 629-vdex-speed + +ifneq (,$(filter interpreter,$(COMPILER_TYPES))) + ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \ + interpreter,$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \ + $(IMAGE_TYPES),$(PICTEST_TYPES),$(DEBUGGABLE_TYPES),$(TEST_ART_BROKEN_INTERPRETER_RUN_TESTS),$(ALL_ADDRESS_SIZES)) +endif + +TEST_ART_BROKEN_INTERPRETER_RUN_TESTS := + +# Known broken tests for the JIT. +# CFI unwinding expects managed frames, and the test does not iterate enough to even compile. JIT +# also uses Generic JNI instead of the JNI compiler. +# Test 906 iterates the heap filtering with different options. No instances should be created +# between those runs to be able to have precise checks. +# Test 902 hits races with the JIT compiler. b/32821077 +# Test 629 requires compilation. +# Test 914, 915, 917, & 919 are very sensitive to the exact state of the stack, +# including the jit-inserted runtime frames. This causes them to be somewhat +# flaky as JIT tests. This should be fixed once b/33630159 or b/33616143 are +# resolved but until then just disable them. Test 916 already checks this +# feature for JIT use cases in a way that is resilient to the jit frames. +# 912: b/34655682 +TEST_ART_BROKEN_JIT_RUN_TESTS := \ + 137-cfi \ + 629-vdex-speed \ + 902-hello-transformation \ + 904-object-allocation \ + 906-iterate-heap \ + 912-classes \ + 914-hello-obsolescence \ + 915-obsolete-2 \ + 917-fields-transformation \ + 919-obsolete-fields \ + 926-multi-obsolescence \ + +ifneq (,$(filter jit,$(COMPILER_TYPES))) + ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \ + jit,$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \ + $(IMAGE_TYPES),$(PICTEST_TYPES),$(DEBUGGABLE_TYPES),$(TEST_ART_BROKEN_JIT_RUN_TESTS),$(ALL_ADDRESS_SIZES)) +endif + +TEST_ART_BROKEN_JIT_RUN_TESTS := + +# Known broken tests for the graph coloring register allocator. +# These tests were based on the linear scan allocator, which makes different decisions than +# the graph coloring allocator. (These attempt to test for code quality, not correctness.) +TEST_ART_BROKEN_OPTIMIZING_GRAPH_COLOR := \ + 570-checker-select \ + 484-checker-register-hints + +ifneq (,$(filter regalloc_gc,$(COMPILER_TYPES))) + ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \ + regalloc_gc,$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \ + $(IMAGE_TYPES),$(PICTEST_TYPES),$(DEBUGGABLE_TYPES), \ + $(TEST_ART_BROKEN_OPTIMIZING_GRAPH_COLOR),$(ALL_ADDRESS_SIZES)) +endif + +# Known broken tests for the mips32 optimizing compiler backend. +TEST_ART_BROKEN_OPTIMIZING_MIPS_RUN_TESTS := \ + +ifeq (mips,$(TARGET_ARCH)) + ifneq (,$(filter $(OPTIMIZING_COMPILER_TYPES),$(COMPILER_TYPES))) + ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,target,$(RUN_TYPES),$(PREBUILD_TYPES), \ + $(OPTIMIZING_COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \ + $(IMAGE_TYPES),$(PICTEST_TYPES),$(DEBUGGABLE_TYPES), \ + $(TEST_ART_BROKEN_OPTIMIZING_MIPS_RUN_TESTS),$(ALL_ADDRESS_SIZES)) + endif +endif + +TEST_ART_BROKEN_OPTIMIZING_MIPS_RUN_TESTS := + +# Known broken tests for the mips64 optimizing compiler backend. +TEST_ART_BROKEN_OPTIMIZING_MIPS64_RUN_TESTS := \ + +ifeq (mips64,$(TARGET_ARCH)) + ifneq (,$(filter $(OPTIMIZING_COMPILER_TYPES),$(COMPILER_TYPES))) + ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,target,$(RUN_TYPES),$(PREBUILD_TYPES), \ + $(OPTIMIZING_COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \ + $(IMAGE_TYPES),$(PICTEST_TYPES),$(DEBUGGABLE_TYPES), \ + $(TEST_ART_BROKEN_OPTIMIZING_MIPS64_RUN_TESTS),$(ALL_ADDRESS_SIZES)) + endif +endif + +TEST_ART_BROKEN_OPTIMIZING_MIPS64_RUN_TESTS := + +# Tests that should fail when the optimizing compiler compiles them non-debuggable. +TEST_ART_BROKEN_OPTIMIZING_NONDEBUGGABLE_RUN_TESTS := \ + 454-get-vreg \ + 457-regs \ + 602-deoptimizeable + +ifneq (,$(filter $(OPTIMIZING_COMPILER_TYPES),$(COMPILER_TYPES))) + ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \ + $(OPTIMIZING_COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \ + $(IMAGE_TYPES),$(PICTEST_TYPES),ndebuggable,$(TEST_ART_BROKEN_OPTIMIZING_NONDEBUGGABLE_RUN_TESTS),$(ALL_ADDRESS_SIZES)) +endif + +TEST_ART_BROKEN_OPTIMIZING_NONDEBUGGABLE_RUN_TESTS := + +# Tests that should fail when the optimizing compiler compiles them debuggable. +TEST_ART_BROKEN_OPTIMIZING_DEBUGGABLE_RUN_TESTS := \ + +ifneq (,$(filter $(OPTIMIZING_COMPILER_TYPES),$(COMPILER_TYPES))) + ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \ + $(OPTIMIZING_COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \ + $(IMAGE_TYPES),$(PICTEST_TYPES),debuggable,$(TEST_ART_BROKEN_OPTIMIZING_DEBUGGABLE_RUN_TESTS),$(ALL_ADDRESS_SIZES)) +endif + +TEST_ART_BROKEN_OPTIMIZING_DEBUGGABLE_RUN_TESTS := + +# Tests that should fail in the read barrier configuration with the interpreter. +TEST_ART_BROKEN_INTERPRETER_READ_BARRIER_RUN_TESTS := + +# Tests that should fail in the read barrier configuration with the Optimizing compiler (AOT). +TEST_ART_BROKEN_OPTIMIZING_READ_BARRIER_RUN_TESTS := + +# Tests that should fail in the read barrier configuration with JIT (Optimizing compiler). +TEST_ART_BROKEN_JIT_READ_BARRIER_RUN_TESTS := + +# Tests failing in non-Baker read barrier configurations with the Optimizing compiler (AOT). +# 537: Expects an array copy to be intrinsified, but calling-on-slowpath intrinsics are not yet +# handled in non-Baker read barrier configurations. +TEST_ART_BROKEN_OPTIMIZING_NON_BAKER_READ_BARRIER_RUN_TESTS := \ + 537-checker-arraycopy + +# Tests failing in non-Baker read barrier configurations with JIT (Optimizing compiler). +# 537: Expects an array copy to be intrinsified, but calling-on-slowpath intrinsics are not yet +# handled in non-Baker read barrier configurations. +TEST_ART_BROKEN_JIT_NON_BAKER_READ_BARRIER_RUN_TESTS := \ + 537-checker-arraycopy + +ifeq ($(ART_USE_READ_BARRIER),true) + ifneq (,$(filter interpreter,$(COMPILER_TYPES))) + ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES), \ + $(PREBUILD_TYPES),interpreter,$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES), \ + $(JNI_TYPES),$(IMAGE_TYPES),$(PICTEST_TYPES),$(DEBUGGABLE_TYPES), \ + $(TEST_ART_BROKEN_INTERPRETER_READ_BARRIER_RUN_TESTS),$(ALL_ADDRESS_SIZES)) + endif + + ifneq (,$(filter $(OPTIMIZING_COMPILER_TYPES),$(COMPILER_TYPES))) + ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES), \ + $(PREBUILD_TYPES),$(OPTIMIZING_COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES), \ + $(GC_TYPES),$(JNI_TYPES),$(IMAGE_TYPES),$(PICTEST_TYPES),$(DEBUGGABLE_TYPES), \ + $(TEST_ART_BROKEN_OPTIMIZING_READ_BARRIER_RUN_TESTS),$(ALL_ADDRESS_SIZES)) + ifneq ($(ART_READ_BARRIER_TYPE),BAKER) + ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES), \ + $(PREBUILD_TYPES),$(OPTIMIZING_COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES), \ + $(GC_TYPES),$(JNI_TYPES),$(IMAGE_TYPES),$(PICTEST_TYPES),$(DEBUGGABLE_TYPES), \ + $(TEST_ART_BROKEN_OPTIMIZING_NON_BAKER_READ_BARRIER_RUN_TESTS),$(ALL_ADDRESS_SIZES)) + endif + endif + + ifneq (,$(filter jit,$(COMPILER_TYPES))) + ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES), \ + $(PREBUILD_TYPES),jit,$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES), \ + $(JNI_TYPES),$(IMAGE_TYPES),$(PICTEST_TYPES),$(DEBUGGABLE_TYPES), \ + $(TEST_ART_BROKEN_JIT_READ_BARRIER_RUN_TESTS),$(ALL_ADDRESS_SIZES)) + ifneq ($(ART_READ_BARRIER_TYPE),BAKER) + ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES), \ + $(PREBUILD_TYPES),jit,$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES), \ + $(JNI_TYPES),$(IMAGE_TYPES),$(PICTEST_TYPES),$(DEBUGGABLE_TYPES), \ + $(TEST_ART_BROKEN_JIT_NON_BAKER_READ_BARRIER_RUN_TESTS),$(ALL_ADDRESS_SIZES)) + endif + endif +endif + +TEST_ART_BROKEN_OPTIMIZING_READ_BARRIER_RUN_TESTS := +TEST_ART_BROKEN_JIT_READ_BARRIER_RUN_TESTS := + +TEST_ART_BROKEN_NPIC_RUN_TESTS := 596-app-images +ifneq (,$(filter npictest,$(PICTEST_TYPES))) + ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \ + ${COMPILER_TYPES},$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \ + $(IMAGE_TYPES),npictest,$(DEBUGGABLE_TYPES),$(TEST_ART_BROKEN_NPIC_RUN_TESTS),$(ALL_ADDRESS_SIZES)) +endif + +# Tests that should fail in the heap poisoning configuration with the Optimizing compiler. +# 055: Exceeds run time limits due to heap poisoning instrumentation (on ARM and ARM64 devices). +TEST_ART_BROKEN_OPTIMIZING_HEAP_POISONING_RUN_TESTS := \ + 055-enum-performance + +ifeq ($(ART_HEAP_POISONING),true) + ifneq (,$(filter $(OPTIMIZING_COMPILER_TYPES),$(COMPILER_TYPES))) + ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES), \ + $(PREBUILD_TYPES),$(OPTIMIZING_COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \ + $(IMAGE_TYPES),$(PICTEST_TYPES),$(DEBUGGABLE_TYPES), \ + $(TEST_ART_BROKEN_OPTIMIZING_HEAP_POISONING_RUN_TESTS),$(ALL_ADDRESS_SIZES)) + endif +endif + +TEST_ART_BROKEN_OPTIMIZING_HEAP_POISONING_RUN_TESTS := + +# 909: Tests that check semantics for a non-debuggable app. +# 137: relies on AOT code and debuggable makes us JIT always. +TEST_ART_BROKEN_DEBUGGABLE_RUN_TESTS := \ + 137-cfi \ + 909-attach-agent \ + +ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \ + $(COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \ + $(IMAGE_TYPES),$(PICTEST_TYPES),debuggable,$(TEST_ART_BROKEN_DEBUGGABLE_RUN_TESTS),$(ALL_ADDRESS_SIZES)) + +TEST_ART_BROKEN_DEBUGGABLE_RUN_TESTS := + +# Tests incompatible with bisection bug search. Sorted by incompatibility reason. +# 000 through 595 do not compile anything. 089 tests a build failure. 018 through 137 +# run dalvikvm more than once. 115 and 088 assume they are always compiled. +# 055 tests performance which is degraded during bisecting. +TEST_ART_INCOMPATIBLE_BISECTION_SEARCH_RUN_TESTS := \ + 000-nop \ + 134-nodex2oat-nofallback \ + 147-stripped-dex-fallback \ + 595-profile-saving \ + \ + 089-many-methods \ + \ + 018-stack-overflow \ + 116-nodex2oat \ + 117-nopatchoat \ + 118-noimage-dex2oat \ + 119-noimage-patchoat \ + 126-miranda-multidex \ + 137-cfi \ + \ + 115-native-bridge \ + 088-monitor-verification \ + \ + 055-enum-performance + +ifeq ($(ART_TEST_BISECTION),true) + ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES), \ + $(PREBUILD_TYPES),$(OPTIMIZING_COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \ + $(IMAGE_TYPES),$(PICTEST_TYPES),$(DEBUGGABLE_TYPES), \ + $(TEST_ART_INCOMPATIBLE_BISECTION_SEARCH_RUN_TESTS),$(ALL_ADDRESS_SIZES)) +endif + +# Clear variables ahead of appending to them when defining tests. +$(foreach target, $(TARGET_TYPES), $(eval ART_RUN_TEST_$(call name-to-var,$(target))_RULES :=)) +$(foreach target, $(TARGET_TYPES), \ + $(foreach prebuild, $(PREBUILD_TYPES), \ + $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(prebuild))_RULES :=))) +$(foreach target, $(TARGET_TYPES), \ + $(foreach compiler, $(COMPILER_TYPES), \ + $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(compiler))_RULES :=))) +$(foreach target, $(TARGET_TYPES), \ + $(foreach relocate, $(RELOCATE_TYPES), \ + $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(relocate))_RULES :=))) +$(foreach target, $(TARGET_TYPES), \ + $(foreach trace, $(TRACE_TYPES), \ + $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(trace))_RULES :=))) +$(foreach target, $(TARGET_TYPES), \ + $(foreach gc, $(GC_TYPES), \ + $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(gc))_RULES :=))) +$(foreach target, $(TARGET_TYPES), \ + $(foreach jni, $(JNI_TYPES), \ + $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(jni))_RULES :=))) +$(foreach target, $(TARGET_TYPES), \ + $(foreach image, $(IMAGE_TYPES), \ + $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(image))_RULES :=))) +$(foreach target, $(TARGET_TYPES), \ + $(foreach test, $(TEST_ART_RUN_TESTS), \ + $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(test))_RULES :=))) +$(foreach target, $(TARGET_TYPES), \ + $(foreach address_size, $(ALL_ADDRESS_SIZES), \ + $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(address_size))_RULES :=))) +$(foreach target, $(TARGET_TYPES), \ + $(foreach run_type, $(RUN_TYPES), \ + $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(run_type))_RULES :=))) +$(foreach target, $(TARGET_TYPES), \ + $(foreach debuggable_type, $(DEBUGGABLE_TYPES), \ + $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(debuggable_type))_RULES :=))) + # We need dex2oat and dalvikvm on the target as well as the core images (all images as we sync # only once). TEST_ART_TARGET_SYNC_DEPS += $(ART_TARGET_EXECUTABLES) $(TARGET_CORE_IMG_OUTS) @@ -225,148 +863,405 @@ ART_TEST_HOST_RUN_TEST_DEPENDENCIES += \ endif -host_prereq_rules := $(ART_TEST_HOST_RUN_TEST_DEPENDENCIES) -host_prereq_rules += $(HOST_JACK_CLASSPATH_DEPENDENCIES) -host_prereq_rules += $(HOST_CORE_IMAGE_OUT) -host_prereq_rules += $(TEST_ART_RUN_TEST_DEPENDENCIES) -host_prereq_rules += $(HOST_OUT_EXECUTABLES)/hprof-conv - -target_prereq_rules := $(TARGET_JACK_CLASSPATH_DEPENDENCIES) -target_prereq_rules += test-art-target-sync +# Create a rule to build and run a tests following the form: +# test-art-{1: host or target}-run-test-{2: debug ndebug}-{3: prebuild no-prebuild no-dex2oat}- +# {4: interpreter optimizing jit interp-ac}- +# {5: relocate nrelocate relocate-npatchoat}- +# {6: trace or ntrace}-{7: gcstress gcverify cms}-{8: forcecopy checkjni jni}- +# {9: no-image image picimage}-{10: pictest npictest}- +# {11: ndebuggable debuggable}-{12: test name}{13: 32 or 64} +define define-test-art-run-test + run_test_options := + prereq_rule := + test_groups := + uc_host_or_target := + jack_classpath := + ifeq ($(ART_TEST_WITH_STRACE),true) + run_test_options += --strace + endif + ifeq ($(ART_TEST_RUN_TEST_ALWAYS_CLEAN),true) + run_test_options += --always-clean + endif + ifeq ($(ART_TEST_BISECTION),true) + run_test_options += --bisection-search + endif + ifeq ($(1),host) + uc_host_or_target := HOST + test_groups := ART_RUN_TEST_HOST_RULES + run_test_options += --host + prereq_rule := $(ART_TEST_HOST_RUN_TEST_DEPENDENCIES) $(HOST_JACK_CLASSPATH_DEPENDENCIES) + jack_classpath := $(HOST_JACK_CLASSPATH) + else + ifeq ($(1),target) + uc_host_or_target := TARGET + test_groups := ART_RUN_TEST_TARGET_RULES + prereq_rule := test-art-target-sync $(TARGET_JACK_CLASSPATH_DEPENDENCIES) + jack_classpath := $(TARGET_JACK_CLASSPATH) + else + $$(error found $(1) expected $(TARGET_TYPES)) + endif + endif + ifeq ($(2),debug) + test_groups += ART_RUN_TEST_$$(uc_host_or_target)_DEBUG_RULES + else + ifeq ($(2),ndebug) + test_groups += ART_RUN_TEST_$$(uc_host_or_target)_RELEASE_RULES + run_test_options += -O + else + $$(error found $(2) expected $(RUN_TYPES)) + endif + endif + ifeq ($(3),prebuild) + test_groups += ART_RUN_TEST_$$(uc_host_or_target)_PREBUILD_RULES + run_test_options += --prebuild + else + ifeq ($(3),no-prebuild) + test_groups += ART_RUN_TEST_$$(uc_host_or_target)_NO_PREBUILD_RULES + run_test_options += --no-prebuild + else + ifeq ($(3),no-dex2oat) + test_groups += ART_RUN_TEST_$$(uc_host_or_target)_NO_DEX2OAT_RULES + run_test_options += --no-prebuild --no-dex2oat + else + $$(error found $(3) expected $(PREBUILD_TYPES)) + endif + endif + endif + ifeq ($(4),optimizing) + test_groups += ART_RUN_TEST_$$(uc_host_or_target)_OPTIMIZING_RULES + run_test_options += --optimizing + else ifeq ($(4),regalloc_gc) + test_groups += ART_RUN_TEST_$$(uc_host_or_target)_OPTIMIZING_GRAPH_COLOR_RULES + run_test_options += --optimizing -Xcompiler-option --register-allocation-strategy=graph-color + else + ifeq ($(4),interpreter) + test_groups += ART_RUN_TEST_$$(uc_host_or_target)_INTERPRETER_RULES + run_test_options += --interpreter + else ifeq ($(4),interp-ac) + test_groups += ART_RUN_TEST_$$(uc_host_or_target)_INTERPRETER_ACCESS_CHECKS_RULES + run_test_options += --interpreter --verify-soft-fail + else + ifeq ($(4),jit) + test_groups += ART_RUN_TEST_$$(uc_host_or_target)_JIT_RULES + run_test_options += --jit + else + $$(error found $(4) expected $(COMPILER_TYPES)) + endif + endif + endif -define core-image-dependencies - ifeq ($(2),no-image) - $(1)_prereq_rules += $($(call name-to-var,$(1))_CORE_IMAGE_$(3)_pic_$(4)) + ifeq ($(5),relocate) + test_groups += ART_RUN_TEST_$$(uc_host_or_target)_RELOCATE_RULES + run_test_options += --relocate + else + ifeq ($(5),no-relocate) + test_groups += ART_RUN_TEST_$$(uc_host_or_target)_NO_RELOCATE_RULES + run_test_options += --no-relocate + else + ifeq ($(5),relocate-npatchoat) + test_groups += ART_RUN_TEST_$$(uc_host_or_target)_RELOCATE_NO_PATCHOAT_RULES + run_test_options += --relocate --no-patchoat + else + $$(error found $(5) expected $(RELOCATE_TYPES)) + endif + endif + endif + ifeq ($(6),trace) + test_groups += ART_RUN_TEST_$$(uc_host_or_target)_TRACE_RULES + run_test_options += --trace else - ifeq ($(2),npicimage) - $(1)_prereq_rules += $($(call name-to-var,$(1))_CORE_IMAGE_$(3)_no-pic_$(4)) + ifeq ($(6),ntrace) + test_groups += ART_RUN_TEST_$$(uc_host_or_target)_NO_TRACE_RULES else - ifeq ($(2),picimage) - $(1)_prereq_rules += $($(call name-to-var,$(1))_CORE_IMAGE_$(3)_pic_$(4)) + ifeq ($(6),stream) + # Group streaming under normal tracing rules. + test_groups += ART_RUN_TEST_$$(uc_host_or_target)_TRACE_RULES + run_test_options += --trace --stream else - ifeq ($(2),multinpicimage) - $(1)_prereq_rules += $($(call name-to-var,$(1))_CORE_IMAGE_$(3)_no-pic_multi_$(4)) + $$(error found $(6) expected $(TRACE_TYPES)) + endif + endif + endif + ifeq ($(7),gcverify) + test_groups += ART_RUN_TEST_$$(uc_host_or_target)_GCVERIFY_RULES + run_test_options += --gcverify + else + ifeq ($(7),gcstress) + test_groups += ART_RUN_TEST_$$(uc_host_or_target)_GCSTRESS_RULES + run_test_options += --gcstress + else + ifeq ($(7),cms) + test_groups += ART_RUN_TEST_$$(uc_host_or_target)_CMS_RULES + else + $$(error found $(7) expected $(GC_TYPES)) + endif + endif + endif + ifeq ($(8),forcecopy) + test_groups += ART_RUN_TEST_$$(uc_host_or_target)_FORCECOPY_RULES + run_test_options += --runtime-option -Xjniopts:forcecopy + ifneq ($$(ART_TEST_JNI_FORCECOPY),true) + skip_test := true + endif + else + ifeq ($(8),checkjni) + test_groups += ART_RUN_TEST_$$(uc_host_or_target)_CHECKJNI_RULES + run_test_options += --runtime-option -Xcheck:jni + else + ifeq ($(8),jni) + test_groups += ART_RUN_TEST_$$(uc_host_or_target)_JNI_RULES + else + $$(error found $(8) expected $(JNI_TYPES)) + endif + endif + endif + image_suffix := $(4) + ifeq ($(4),regalloc_gc) + # Graph coloring tests share the image_suffix with optimizing tests. + image_suffix := optimizing + else + ifeq ($(4),jit) + # JIT tests share the image_suffix with interpreter tests. + image_suffix := interpreter + endif + endif + ifeq ($(9),no-image) + test_groups += ART_RUN_TEST_$$(uc_host_or_target)_NO_IMAGE_RULES + run_test_options += --no-image + # Add the core dependency. This is required for pre-building. + # Use the PIC image, as it is the default in run-test, to match dependencies. + ifeq ($(1),host) + prereq_rule += $$(HOST_CORE_IMAGE_$$(image_suffix)_pic_$(13)) + else + prereq_rule += $$(TARGET_CORE_IMAGE_$$(image_suffix)_pic_$(13)) + endif + else + ifeq ($(9),npicimage) + test_groups += ART_RUN_TEST_$$(uc_host_or_target)_IMAGE_RULES + run_test_options += --npic-image + # Add the core dependency. + ifeq ($(1),host) + prereq_rule += $$(HOST_CORE_IMAGE_$$(image_suffix)_no-pic_$(13)) + else + prereq_rule += $$(TARGET_CORE_IMAGE_$$(image_suffix)_no-pic_$(13)) + endif + else + ifeq ($(9),picimage) + test_groups += ART_RUN_TEST_$$(uc_host_or_target)_PICIMAGE_RULES + ifeq ($(1),host) + prereq_rule += $$(HOST_CORE_IMAGE_$$(image_suffix)_pic_$(13)) else - ifeq ($(2),multipicimage) - $(1)_prereq_rules += $($(call name-to-var,$(1))_CORE_IMAGE_$(3)_pic_multi_$(4)) + prereq_rule += $$(TARGET_CORE_IMAGE_$$(image_suffix)_pic_$(13)) + endif + else + ifeq ($(9),multinpicimage) + test_groups += ART_RUN_TEST_$$(uc_host_or_target)_IMAGE_RULES + run_test_options += --npic-image --multi-image + ifeq ($(1),host) + prereq_rule += $$(HOST_CORE_IMAGE_$$(image_suffix)_no-pic_multi_$(13)) + else + prereq_rule += $$(TARGET_CORE_IMAGE_$$(image_suffix)_no-pic_multi_$(13)) + endif + else + ifeq ($(9),multipicimage) + test_groups += ART_RUN_TEST_$$(uc_host_or_target)_PICIMAGE_RULES + run_test_options += --multi-image + ifeq ($(1),host) + prereq_rule += $$(HOST_CORE_IMAGE_$$(image_suffix)_pic_multi_$(13)) + else + prereq_rule += $$(TARGET_CORE_IMAGE_$$(image_suffix)_pic_multi_$(13)) + endif + else + $$(error found $(9) expected $(IMAGE_TYPES)) endif endif endif endif endif -endef + ifeq ($(10),pictest) + run_test_options += --pic-test + else + ifeq ($(10),npictest) + # Nothing to be done. + else + $$(error found $(10) expected $(PICTEST_TYPES)) + endif + endif + ifeq ($(11),debuggable) + test_groups += ART_RUN_TEST_$$(uc_host_or_target)_DEBUGGABLE_RULES + run_test_options += --debuggable + else + ifeq ($(11),ndebuggable) + test_groups += ART_RUN_TEST_$$(uc_host_or_target)_NONDEBUGGABLE_RULES + # Nothing to be done. + else + $$(error found $(11) expected $(DEBUGGABLE_TYPES)) + endif + endif + # $(12) is the test name. + test_groups += ART_RUN_TEST_$$(uc_host_or_target)_$(call name-to-var,$(12))_RULES + ifeq ($(13),64) + test_groups += ART_RUN_TEST_$$(uc_host_or_target)_64_RULES + run_test_options += --64 + else + ifeq ($(13),32) + test_groups += ART_RUN_TEST_$$(uc_host_or_target)_32_RULES + else + $$(error found $(13) expected $(ALL_ADDRESS_SIZES)) + endif + endif + # Override of host instruction-set-features. Required to test advanced x86 intrinsics. The + # conditionals aren't really correct, they will fail to do the right thing on a 32-bit only + # host. However, this isn't common enough to worry here and make the conditions complicated. + ifneq ($(DEX2OAT_HOST_INSTRUCTION_SET_FEATURES),) + ifeq ($(13),64) + run_test_options += --instruction-set-features $(DEX2OAT_HOST_INSTRUCTION_SET_FEATURES) + endif + endif + ifneq ($($(HOST_2ND_ARCH_VAR_PREFIX)DEX2OAT_HOST_INSTRUCTION_SET_FEATURES),) + ifeq ($(13),32) + run_test_options += --instruction-set-features $($(HOST_2ND_ARCH_VAR_PREFIX)DEX2OAT_HOST_INSTRUCTION_SET_FEATURES) + endif + endif + run_test_rule_name := test-art-$(1)-run-test-$(2)-$(3)-$(4)-$(5)-$(6)-$(7)-$(8)-$(9)-$(10)-$(11)-$(12)$(13) + run_test_options := --output-path $(ART_HOST_TEST_DIR)/run-test-output/$$(run_test_rule_name) \ + $$(run_test_options) + ifneq ($(ART_TEST_ANDROID_ROOT),) + run_test_options := --android-root $(ART_TEST_ANDROID_ROOT) $$(run_test_options) + endif + ifeq ($(ART_TEST_QUIET),true) + run_test_options += --quiet + endif +$$(run_test_rule_name): PRIVATE_RUN_TEST_OPTIONS := $$(run_test_options) +$$(run_test_rule_name): PRIVATE_JACK_CLASSPATH := $$(jack_classpath) +.PHONY: $$(run_test_rule_name) +$$(run_test_rule_name): $(TEST_ART_RUN_TEST_DEPENDENCIES) $(HOST_OUT_EXECUTABLES)/hprof-conv $$(prereq_rule) | $(TEST_ART_RUN_TEST_ORDERONLY_DEPENDENCIES) + $(hide) $$(call ART_TEST_SKIP,$$@) && \ + DX=$(abspath $(DX)) \ + JASMIN=$(abspath $(HOST_OUT_EXECUTABLES)/jasmin) \ + SMALI=$(abspath $(HOST_OUT_EXECUTABLES)/smali) \ + DXMERGER=$(abspath $(HOST_OUT_EXECUTABLES)/dexmerger) \ + JACK_VERSION=$(JACK_DEFAULT_VERSION) \ + JACK=$(abspath $(JACK)) \ + JACK_VERSION=$(JACK_DEFAULT_VERSION) \ + JACK_CLASSPATH=$$(PRIVATE_JACK_CLASSPATH) \ + art/test/run-test $$(PRIVATE_RUN_TEST_OPTIONS) $(12) \ + && $$(call ART_TEST_PASSED,$$@) || $$(call ART_TEST_FAILED,$$@) + $$(hide) (echo $(MAKECMDGOALS) | grep -q $$@ && \ + echo "run-test run as top-level target, removing test directory $(ART_HOST_TEST_DIR)" && \ + rm -r $(ART_HOST_TEST_DIR)) || true + + $$(foreach test_group,$$(test_groups), $$(eval $$(value test_group) += $$(run_test_rule_name))) + + # Clear locally defined variables. + uc_host_or_target := + test_groups := + run_test_options := + run_test_rule_name := + prereq_rule := + jack_classpath := +endef # define-test-art-run-test -# Generate list of dependencies required for given target - HOST or TARGET, IMAGE_TYPE, -# COMPILER_TYPE and ADDRESS_SIZE. $(foreach target, $(TARGET_TYPES), \ - $(foreach image, $(IMAGE_TYPES), \ - $(foreach compiler, $(COMPILER_TYPES), \ - $(foreach address_size, $(ALL_ADDRESS_SIZES), $(eval \ - $(call core-image-dependencies,$(target),$(image),$(compiler),$(address_size))))))) - -test-art-host-run-test-dependencies : $(host_prereq_rules) -test-art-target-run-test-dependencies : $(target_prereq_rules) -test-art-run-test-dependencies : test-art-host-run-test-dependencies test-art-target-run-test-dependencies - -# Create a rule to build and run a test group of the following form: -# test-art-{1: host target}-run-test-{2: prebuild no-prebuild no-dex2oat \ -# interp-ac interpreter jit optimizing regalloc_gc relocate no-relocate \ -# relocate-npatchoat ntrace trace stream cms gcstress gcverify checkjni \ -# forcecopy picimage no-image multinpicimage npictest pictest debug ndebug \ -# ndebuggable debuggable} -# -# test-art-{1: host target}-run-test{2: 32 64} + $(foreach test, $(TEST_ART_RUN_TESTS), \ + $(foreach run_type, $(RUN_TYPES), \ + $(foreach address_size, $(ADDRESS_SIZES_$(call name-to-var,$(target))), \ + $(foreach prebuild, $(PREBUILD_TYPES), \ + $(foreach compiler, $(COMPILER_TYPES), \ + $(foreach relocate, $(RELOCATE_TYPES), \ + $(foreach trace, $(TRACE_TYPES), \ + $(foreach gc, $(GC_TYPES), \ + $(foreach jni, $(JNI_TYPES), \ + $(foreach image, $(IMAGE_TYPES), \ + $(foreach pictest, $(PICTEST_TYPES), \ + $(foreach debuggable, $(DEBUGGABLE_TYPES), \ + $(eval $(call define-test-art-run-test,$(target),$(run_type),$(prebuild),$(compiler),$(relocate),$(trace),$(gc),$(jni),$(image),$(pictest),$(debuggable),$(test),$(address_size))) \ + ))))))))))))) +define-test-art-run-test := + +# Define a phony rule whose purpose is to test its prerequisites. +# $(1): host or target +# $(2): list of prerequisites define define-test-art-run-test-group - ifneq (,$(filter $(2),$(ALL_ADDRESS_SIZES))) - build_target := test-art-$(1)-run-test$(2) - else - build_target := test-art-$(1)-run-test-$(2) - endif - # Python testrunner support multi-threaded test run, however, to invoke the script, one - # has to provide the concurrency level for the run. To provide that value, the concurrency - # level of the make invocation is used. The make invocation information is extracted through - # `ps` and subsequently the number of threads is determined. - kati_pid = $(shell echo $$PPID) - make_pid = $$(shell ps -o ppid= -p $$(kati_pid)) - grep_build_targets_from_ps_keyword := -e - grep_build_targets_from_ps_keyword += $$(shell echo $(MAKECMDGOALS) | sed -e "s/\s\+/ -e /g") - ps_data := $$(shell ps aux | grep $$(make_pid) | grep $$(grep_build_targets_from_ps_keyword) ) - concurrency := $$(shell [[ "$$(ps_data)" =~ -[a-zA-Z]*j[[:space:]]*[0-9]+ ]] && [[ $$$$BASH_REMATCH =~ [0-9]+ ]] && echo $$$$BASH_REMATCH || echo 1) - - .PHONY: $$(build_target) - $$(build_target) : args := --$(1) --$(2) --verbose -j$$(concurrency) - $$(build_target) : test-art-$(1)-run-test-dependencies - ./art/test/testrunner/testrunner.py $$(args) - build_target := - concurrency := - ps_data := - kati_pid := - make_pid := - grep_build_targets_from_ps_keyword := +.PHONY: $(1) +$(1): $(2) + $(hide) $$(call ART_TEST_PREREQ_FINISHED,$$@) + endef # define-test-art-run-test-group + +$(foreach target, $(TARGET_TYPES), $(eval \ + $(call define-test-art-run-test-group,test-art-$(target)-run-test,$(ART_RUN_TEST_$(call name-to-var,$(target))_RULES)))) $(foreach target, $(TARGET_TYPES), \ $(foreach prebuild, $(PREBUILD_TYPES), $(eval \ - $(call define-test-art-run-test-group,$(target),$(prebuild))))) + $(call define-test-art-run-test-group,test-art-$(target)-run-test-$(prebuild),$(ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(prebuild))_RULES))))) $(foreach target, $(TARGET_TYPES), \ $(foreach run-type, $(RUN_TYPES), $(eval \ - $(call define-test-art-run-test-group,$(target),$(run-type))))) + $(call define-test-art-run-test-group,test-art-$(target)-run-test-$(run-type),$(ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(run-type))_RULES))))) $(foreach target, $(TARGET_TYPES), \ $(foreach compiler, $(COMPILER_TYPES), $(eval \ - $(call define-test-art-run-test-group,$(target),$(compiler))))) + $(call define-test-art-run-test-group,test-art-$(target)-run-test-$(compiler),$(ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(compiler))_RULES))))) $(foreach target, $(TARGET_TYPES), \ $(foreach relocate, $(RELOCATE_TYPES), $(eval \ - $(call define-test-art-run-test-group,$(target),$(relocate))))) + $(call define-test-art-run-test-group,test-art-$(target)-run-test-$(relocate),$(ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(relocate))_RULES))))) $(foreach target, $(TARGET_TYPES), \ $(foreach trace, $(TRACE_TYPES), $(eval \ - $(call define-test-art-run-test-group,$(target),$(trace))))) + $(call define-test-art-run-test-group,test-art-$(target)-run-test-$(trace),$(ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(trace))_RULES))))) $(foreach target, $(TARGET_TYPES), \ $(foreach gc, $(GC_TYPES), $(eval \ - $(call define-test-art-run-test-group,$(target),$(gc))))) + $(call define-test-art-run-test-group,test-art-$(target)-run-test-$(gc),$(ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(gc))_RULES))))) $(foreach target, $(TARGET_TYPES), \ $(foreach jni, $(JNI_TYPES), $(eval \ - $(call define-test-art-run-test-group,$(target),$(jni))))) + $(call define-test-art-run-test-group,test-art-$(target)-run-test-$(jni),$(ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(jni))_RULES))))) $(foreach target, $(TARGET_TYPES), \ $(foreach debuggable, $(DEBUGGABLE_TYPES), $(eval \ - $(call define-test-art-run-test-group,$(target),$(debuggable))))) + $(call define-test-art-run-test-group,test-art-$(target)-run-test-$(debuggable),$(ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(debuggable))_RULES))))) $(foreach target, $(TARGET_TYPES), \ $(foreach image, $(IMAGE_TYPES), $(eval \ - $(call define-test-art-run-test-group,$(target),$(image))))) + $(call define-test-art-run-test-group,test-art-$(target)-run-test-$(image),$(ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(image))_RULES))))) +$(foreach target, $(TARGET_TYPES), \ + $(foreach test, $(TEST_ART_RUN_TESTS), $(eval \ + $(call define-test-art-run-test-group,test-art-$(target)-run-test-$(test),$(ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(test))_RULES))))) $(foreach target, $(TARGET_TYPES), \ $(foreach address_size, $(ADDRESS_SIZES_$(call name-to-var,$(target))), $(eval \ - $(call define-test-art-run-test-group,$(target),$(address_size))))) - -# Create a rule to build and run a test group of the following form: -# test-art-{1: host target}-run-test -define define-test-art-host-or-target-run-test-group - build_target := test-art-$(1)-run-test - .PHONY: $$(build_target) - # Python testrunner support multi-threaded test run, however, to invoke the script, one - # has to provide the concurrency level for the run. To provide that value, the concurrency - # level of the make invocation is used. The make invocation information is extracted through - # `ps` and subsequently the number of threads is determined. - kati_pid = $(shell echo $$PPID) - make_pid = $$(shell ps -o ppid= -p $$(kati_pid)) - grep_build_targets_from_ps_keyword := -e - grep_build_targets_from_ps_keyword += $$(shell echo $(MAKECMDGOALS) | sed -e "s/\s\+/ -e /g") - ps_data := $$(shell ps aux | grep $$(make_pid) | grep $$(grep_build_targets_from_ps_keyword)) - concurrency := $$(shell [[ "$$(ps_data)" =~ -[a-zA-Z]*j[[:space:]]*[0-9]+ ]] && [[ $$$$BASH_REMATCH =~ [0-9]+ ]] && echo $$$$BASH_REMATCH || echo 1) - - $$(build_target) : args := --$(1) --verbose -j$$(concurrency) - $$(build_target) : test-art-$(1)-run-test-dependencies - ./art/test/testrunner/testrunner.py $$(args) - build_target := - args := - grep_build_targets_from_ps_keyword := - ps_data := - concurrency := - kati_pid := - make_pid := -endef # define-test-art-host-or-target-run-test-group - -$(foreach target, $(TARGET_TYPES), $(eval \ - $(call define-test-art-host-or-target-run-test-group,$(target)))) + $(call define-test-art-run-test-group,test-art-$(target)-run-test$(address_size),$(ART_RUN_TEST_$(call name-to-var,$(target))_$(address_size)_RULES))))) +# Clear variables now we're finished with them. +$(foreach target, $(TARGET_TYPES), $(eval ART_RUN_TEST_$(call name-to-var,$(target))_RULES :=)) +$(foreach target, $(TARGET_TYPES), \ + $(foreach prebuild, $(PREBUILD_TYPES), \ + $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(prebuild))_RULES :=))) +$(foreach target, $(TARGET_TYPES), \ + $(foreach compiler, $(COMPILER_TYPES), \ + $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(compiler))_RULES :=))) +$(foreach target, $(TARGET_TYPES), \ + $(foreach relocate, $(RELOCATE_TYPES), \ + $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(relocate))_RULES :=))) +$(foreach target, $(TARGET_TYPES), \ + $(foreach trace, $(TRACE_TYPES), \ + $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(trace))_RULES :=))) +$(foreach target, $(TARGET_TYPES), \ + $(foreach gc, $(GC_TYPES), \ + $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(gc))_RULES :=))) +$(foreach target, $(TARGET_TYPES), \ + $(foreach jni, $(JNI_TYPES), \ + $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(jni))_RULES :=))) +$(foreach target, $(TARGET_TYPES), \ + $(foreach debuggable, $(DEBUGGABLE_TYPES), \ + $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(debuggable))_RULES :=))) +$(foreach target, $(TARGET_TYPES), \ + $(foreach image, $(IMAGE_TYPES), \ + $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(image))_RULES :=))) +$(foreach target, $(TARGET_TYPES), \ + $(foreach test, $(TEST_ART_RUN_TESTS), \ + $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(test))_RULES :=))) +$(foreach target, $(TARGET_TYPES), \ + $(foreach address_size, $(ALL_ADDRESS_SIZES), \ + $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(address_size))_RULES :=))) +$(foreach target, $(TARGET_TYPES), \ + $(foreach run_type, $(RUN_TYPES), \ + $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(run_type))_RULES :=))) +define-test-art-run-test-group := TARGET_TYPES := PREBUILD_TYPES := COMPILER_TYPES := @@ -375,8 +1270,10 @@ TRACE_TYPES := GC_TYPES := JNI_TYPES := IMAGE_TYPES := -PICTEST_TYPES := +ADDRESS_SIZES_TARGET := +ADDRESS_SIZES_HOST := +ALL_ADDRESS_SIZES := RUN_TYPES := DEBUGGABLE_TYPES := -ALL_ADDRESS_SIZES := + LOCAL_PATH := diff --git a/test/knownfailures.json b/test/knownfailures.json deleted file mode 100644 index 3cfff43451..0000000000 --- a/test/knownfailures.json +++ /dev/null @@ -1,359 +0,0 @@ -[ - { - "test": "153-reference-stress", - "description": ["Disable 153-reference-stress temporarily until a fix", - "arrives."], - "bug": "http://b/33389022" - }, - { - "test": "080-oom-fragmentation", - "description": "Disable 080-oom-fragmentation due to flakes.", - "bug": "http://b/33795328" - }, - { - "tests": ["497-inlining-and-class-loader", - "542-unresolved-access-check"], - "description": ["Disable 497-inlining-and-class-loader and ", - "542-unresolved-access-check until they are rewritten.", - "These tests use a broken class loader that tries to", - "register a dex file that's already registered with a", - "different loader."], - "bug": "http://b/34193123" - }, - { - "test": "149-suspend-all-stress", - "description": "Disable 149-suspend-all-stress, its output is flaky", - "bug": "http://b/28988206" - }, - { - "test": "577-profile-foreign-dex", - "description": "Disable 577-profile-foreign-dex", - "bug": "http://b/27454772" - }, - { - "tests": ["002-sleep", - "053-wait-some", - "055-enum-performance", - "133-static-invoke-super"], - "description": ["Tests that are timing sensitive and flaky on heavily", - "loaded systems."] - }, - { - "test": "147-stripped-dex-fallback", - "variant": "target", - "description": ["147-stripped-dex-fallback isn't supported on device", - "because --strip-dex requires the zip command."] - }, - { - "test": "569-checker-pattern-replacement", - "variant": "target", - "description": ["569-checker-pattern-replacement tests behaviour", - "present only on host."] - }, - { - "tests": ["116-nodex2oat", - "118-noimage-dex2oat", - "134-nodex2oat-nofallback"], - "variant": "prebuild", - "description": ["Note 116-nodex2oat is not broken per-se it just", - "doesn't (and isn't meant to) work with --prebuild."] - }, - { - "test": "554-jit-profile-file", - "variant": "no-prebuild | interpreter", - "description": ["554-jit-profile-file is disabled because it needs a", - "primary oat file to know what it should save."] - }, - { - "tests": ["529-checker-unresolved", "555-checker-regression-x86const"], - "variant": "no-prebuild", - "bug": "http://b/27784033" - }, - { - "tests": ["117-nopatchoat", - "147-stripped-dex-fallback", - "608-checker-unresolved-lse"], - "variant": "no-prebuild" - }, - { - "tests": ["117-nopatchoat", - "118-noimage-dex2oat", - "119-noimage-patchoat", - "554-jit-profile-file"], - "variant": "no-relocate", - "description": ["117-nopatchoat is not broken per-se it just doesn't", - "work (and isn't meant to) without --prebuild", - "--relocate"] - }, - { - "test": "137-cfi", - "variant": "interp-ac", - "description": ["Temporarily disable some broken tests when forcing", - "access checks in interpreter"], - "bug": "http://b/22414682" - }, - { - "test" : "629-vdex-speed", - "variant": "interp-ac | no-dex2oat | interpreter | jit", - "description": "629 requires compilation." - }, - { - "test": "137-cfi", - "variant": "gcstress", - "description": ["137-cfi needs to unwind a second forked process. We're", - "using a primitive sleep to wait till we hope the", - "second process got into the expected state. The", - "slowness of gcstress makes this bad."] - }, - { - "tests": ["908-gc-start-finish", - "913-heaps"], - "variant": "gcstress", - "description": ["908-gc-start-finish expects GCs only to be run at", - "clear points. The reduced heap size makes this", - "non-deterministic. Same for 913."] - }, - { - "test": "961-default-iface-resolution-gen", - "variant": "gcstress", - "description": ["961-default-iface-resolution-gen and", - "964-default-iface-init-genare very long tests that", - "often will take more than the timeout to run when", - "gcstress is enabled. This is because gcstress slows", - "down allocations significantly which these tests do a", - "lot."] - }, - { - "tests": ["964-default-iface-init-gen", - "154-gc-loop"], - "variant": "gcstress" - }, - { - "test": "115-native-bridge", - "variant": "target", - "description": ["115-native-bridge setup is complicated. Need to", - "implement it correctly for the target."] - }, - { - "test": "130-hprof", - "variant": "target", - "desription": ["130-hprof dumps the heap and runs hprof-conv to check", - "whether the file is somewhat readable. Thi is only", - "possible on the host. TODO: Turn off all the other", - "combinations, this is more about testing actual ART", - "code. A gtest is very hard to write here, as (for a", - "complete test) JDWP must be set up."] - }, - { - "test": "131-structural-change", - "variant": "debug", - "description": ["131 is an old test. The functionality has been", - "implemented at an earlier stage and is checked", - "in tests 138. Blacklisted for debug builds since", - "these builds have duplicate classes checks which", - "punt to interpreter"] - }, - { - "test": "138-duplicate-classes-check", - "variant": "ndebug", - "description": ["Turned on for debug builds since debug builds have", - "duplicate classes checks enabled"], - "bug": "http://b/2133391" - }, - { - "test": "147-stripped-dex-fallback", - "variant": "no-dex2oat | no-image", - "description": ["147-stripped-dex-fallback is disabled because it", - "requires --prebuild."] - }, - { - "test": "554-jit-profile-file", - "variant": "no-dex2oat | no-image | relocate-npatchoat", - "description": ["554-jit-profile-file is disabled because it needs a", - "primary oat file to know what it should save."] - }, - { - "tests": ["116-nodex2oat", - "117-nopatchoat", - "118-noimage-dex2oat", - "119-noimage-patchoat", - "137-cfi", - "138-duplicate-classes-check2"], - "variant": "no-dex2oat | no-image", - "description": ["All these tests check that we have sane behavior if we", - "don't have a patchoat or dex2oat. Therefore we", - "shouldn't run them in situations where we actually", - "don't have these since they explicitly test for them.", - "These all also assume we have an image."] - }, - { - "tests": ["137-cfi", - "138-duplicate-classes-check", - "018-stack-overflow", - "961-default-iface-resolution-gen", - "964-default-iface-init"], - "variant": "no-image", - "description": ["This test fails without an image. 018, 961, 964 often", - "time out."], - "bug": "http://b/34369284" - }, - { - "test": "137-cfi", - "description": ["This test unrolls and expects managed frames, but", - "tracing means we run the interpreter."], - "variant": "trace | stream" - }, - { - "tests": ["802-deoptimization", - "570-checker-osr"], - "description": ["This test dynamically enables tracing to force a", - "deoptimization. This makes the test meaningless", - "when already tracing, and writes an error message", - "that we do not want to check for."], - "variant": "trace | stream" - }, - { - "test": "130-hprof", - "description": "130 occasional timeout", - "bug": "http://b/32383962", - "variant": "trace | stream" - }, - { - "tests": ["087-gc-after-link", - "141-class-unload"], - "variant": "trace | stream" - }, - { - "tests": ["604-hot-static-interface", - "612-jit-dex-cache", - "613-inlining-dex-cache", - "616-cha", - "626-set-resolved-string"], - "variant": "trace | stream", - "description": ["These tests expect JIT compilation, which is", - "suppressed when tracing."] - }, - { - "test": "137-cfi", - "description": ["CFI unwinding expects managed frames, and the test", - "does not iterate enough to even compile. JIT also", - "uses Generic JNI instead of the JNI compiler."], - "variant": "interpreter | jit" - }, - { - "test": "902-hello-transformation", - "description": "Test 902 hits races with the JIT compiler.", - "bug": "http://b/32821077", - "variant": "jit" - }, - { - "test": "906-iterate-heap", - "description": ["Test 906 iterates the heap filtering with different", - "options. No instances should be created between those", - "runs to be able to have precise checks."], - "variant": "jit" - }, - { - "tests": ["914-hello-obsolescence", - "915-obsolete-2", - "917-fields-transformation", - "919-obsolete-fields"], - "description": ["Test 914, 915, 917, & 919 are very sensitive to the", - "exact state of the stack, including the jit-inserted", - "runtime frames. This causes them to be somewhat flaky", - "as JIT tests. This should be fixed once b/33630159 or", - "b/33616143 are resolved but until then just disable", - "them. Test 916 already checks this feature for JIT", - "use cases in a way that is resilient to the jit", - "frames."], - "variant": "jit" - }, - { - "tests": ["926-multi-obsolescence", - "904-object-allocation"], - "variant": "jit" - }, - { - "test": "912-classes", - "variant": "jit", - "bug": "http://b/34655682" - }, - { - "tests": ["570-checker-select", - "484-checker-register-hints"], - "description": ["These tests were based on the linear scan allocator,", - "which makes different decisions than the graph", - "coloring allocator. (These attempt to test for code", - "quality, not correctness.)"], - "variant": "regalloc_gc" - }, - { - "tests": ["454-get-vreg", - "457-regs", - "602-deoptimizeable"], - "description": ["Tests that should fail when the optimizing compiler ", - "compiles them non-debuggable."], - "variant": "optimizing & ndebuggable | regalloc_gc & ndebuggable" - }, - { - "test": "596-app-images", - "variant": "npictest" - }, - { - "test": "055-enum-performance", - "variant": "optimizing | regalloc_gc", - "description": ["055: Exceeds run time limits due to heap poisoning ", - "instrumentation (on ARM and ARM64 devices)."] - }, - { - "test": "909-attach-agent", - "variant": "debuggable", - "description": "Tests that check semantics for a non-debuggable app." - }, - { - "tests": ["000-nop", - "134-nodex2oat-nofallback", - "147-stripped-dex-fallback", - "595-profile-saving"], - "description": "The doesn't compile anything", - "env_vars": {"ART_TEST_BISECTION": "true"}, - "variant": "optimizing | regalloc_gc" - }, - { - "tests": "089-many-methods", - "description": "The test tests a build failure", - "env_vars": {"ART_TEST_BISECTION": "true"}, - "variant": "optimizing | regalloc_gc" - }, - { - "tests": ["018-stack-overflow", - "116-nodex2oat", - "117-nopatchoat", - "118-noimage-dex2oat", - "119-noimage-patchoat", - "126-miranda-multidex", - "137-cfi"], - "description": "The test run dalvikvm more than once.", - "env_vars": {"ART_TEST_BISECTION": "true"}, - "variant": "optimizing | regalloc_gc" - }, - { - "tests": ["115-native-bridge", - "088-monitor-verification"], - "description": "The test assume they are always compiled.", - "env_vars": {"ART_TEST_BISECTION": "true"}, - "variant": "optimizing | regalloc_gc" - }, - { - "test": "055-enum-performance", - "description": ["The test tests performance which degrades during", - "bisecting."], - "env_vars": {"ART_TEST_BISECTION": "true"}, - "variant": "optimizing | regalloc_gc" - }, - { - "test": "537-checker-arraycopy", - "env_vars": {"ART_USE_READ_BARRIER": "true"}, - "variant": "interpreter | optimizing | regalloc_gc | jit" - } -] diff --git a/test/testrunner/env.py b/test/testrunner/env.py deleted file mode 100644 index e015d7427c..0000000000 --- a/test/testrunner/env.py +++ /dev/null @@ -1,174 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2017, The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import tempfile -import subprocess - -env = dict(os.environ) - -def getEnvBoolean(var, default): - val = env.get(var) - if val: - if val == "True" or val == "true": - return True - if val == "False" or val == "false": - return False - return default - -def get_build_var(var_name): - # The command is taken from build/envsetup.sh to fetch build variables. - command = ("CALLED_FROM_SETUP=true BUILD_SYSTEM=build/core " - "make --no-print-directory -C \"%s\" -f build/core/config.mk " - "dumpvar-%s") % (ANDROID_BUILD_TOP, var_name) - config = subprocess.Popen(command, stdout=subprocess.PIPE, - shell=True).communicate()[0] - return config.strip() - -def get_env(key): - return env.get(key) - -ANDROID_BUILD_TOP = env.get('ANDROID_BUILD_TOP', os.getcwd()) - -# Directory used for temporary test files on the host. -ART_HOST_TEST_DIR = tempfile.mkdtemp(prefix = 'test-art-') - -# Keep going after encountering a test failure? -ART_TEST_KEEP_GOING = getEnvBoolean('ART_TEST_KEEP_GOING', True) - -# Do you want all tests, even those that are time consuming? -ART_TEST_FULL = getEnvBoolean('ART_TEST_FULL', False) - -# Do you want interpreter tests run? -ART_TEST_INTERPRETER = getEnvBoolean('ART_TEST_INTERPRETER', ART_TEST_FULL) -ART_TEST_INTERPRETER_ACCESS_CHECKS = getEnvBoolean('ART_TEST_INTERPRETER_ACCESS_CHECKS', - ART_TEST_FULL) - -# Do you want JIT tests run? -ART_TEST_JIT = getEnvBoolean('ART_TEST_JIT', ART_TEST_FULL) - -# Do you want optimizing compiler tests run? -ART_TEST_OPTIMIZING = getEnvBoolean('ART_TEST_OPTIMIZING', True) - -# Do you want to test the optimizing compiler with graph coloring register allocation? -ART_TEST_OPTIMIZING_GRAPH_COLOR = getEnvBoolean('ART_TEST_OPTIMIZING_GRAPH_COLOR', ART_TEST_FULL) - -# Do we want to test a non-PIC-compiled core image? -ART_TEST_NPIC_IMAGE = getEnvBoolean('ART_TEST_NPIC_IMAGE', ART_TEST_FULL) - -# Do we want to test PIC-compiled tests ("apps")? -ART_TEST_PIC_TEST = getEnvBoolean('ART_TEST_PIC_TEST', ART_TEST_FULL) -# Do you want tracing tests run? -ART_TEST_TRACE = getEnvBoolean('ART_TEST_TRACE', ART_TEST_FULL) - -# Do you want tracing tests (streaming mode) run? -ART_TEST_TRACE_STREAM = getEnvBoolean('ART_TEST_TRACE_STREAM', ART_TEST_FULL) - -# Do you want tests with GC verification enabled run? -ART_TEST_GC_VERIFY = getEnvBoolean('ART_TEST_GC_VERIFY', ART_TEST_FULL) - -# Do you want tests with the GC stress mode enabled run? -ART_TEST_GC_STRESS = getEnvBoolean('ART_TEST_GC_STRESS', ART_TEST_FULL) - -# Do you want tests with the JNI forcecopy mode enabled run? -ART_TEST_JNI_FORCECOPY = getEnvBoolean('ART_TEST_JNI_FORCECOPY', ART_TEST_FULL) - -# Do you want run-tests with relocation disabled run? -ART_TEST_RUN_TEST_RELOCATE = getEnvBoolean('ART_TEST_RUN_TEST_RELOCATE', ART_TEST_FULL) - -# Do you want run-tests with prebuilding? -ART_TEST_RUN_TEST_PREBUILD = getEnvBoolean('ART_TEST_RUN_TEST_PREBUILD', True) - -# Do you want run-tests with no prebuilding enabled run? -ART_TEST_RUN_TEST_NO_PREBUILD = getEnvBoolean('ART_TEST_RUN_TEST_NO_PREBUILD', ART_TEST_FULL) - -# Do you want run-tests with a pregenerated core.art? -ART_TEST_RUN_TEST_IMAGE = getEnvBoolean('ART_TEST_RUN_TEST_IMAGE', True) - -# Do you want run-tests without a pregenerated core.art? -ART_TEST_RUN_TEST_NO_IMAGE = getEnvBoolean('ART_TEST_RUN_TEST_NO_IMAGE', ART_TEST_FULL) - -# Do you want run-tests with relocation enabled but patchoat failing? -ART_TEST_RUN_TEST_RELOCATE_NO_PATCHOAT = getEnvBoolean('ART_TEST_RUN_TEST_RELOCATE_NO_PATCHOAT', - ART_TEST_FULL) - -# Do you want run-tests without a dex2oat? -ART_TEST_RUN_TEST_NO_DEX2OAT = getEnvBoolean('ART_TEST_RUN_TEST_NO_DEX2OAT', ART_TEST_FULL) - -# Do you want run-tests with libartd.so? -ART_TEST_RUN_TEST_DEBUG = getEnvBoolean('ART_TEST_RUN_TEST_DEBUG', True) - -# Do you want run-tests with libart.so? -ART_TEST_RUN_TEST_NDEBUG = getEnvBoolean('ART_TEST_RUN_TEST_NDEBUG', ART_TEST_FULL) - -# Do you want failed tests to have their artifacts cleaned up? -ART_TEST_RUN_TEST_ALWAYS_CLEAN = getEnvBoolean('ART_TEST_RUN_TEST_ALWAYS_CLEAN', True) - -# Do you want run-tests with the --debuggable flag -ART_TEST_RUN_TEST_DEBUGGABLE = getEnvBoolean('ART_TEST_RUN_TEST_DEBUGGABLE', ART_TEST_FULL) - -# Do you want to test multi-part boot-image functionality? -ART_TEST_RUN_TEST_MULTI_IMAGE = getEnvBoolean('ART_TEST_RUN_TEST_MULTI_IMAGE', ART_TEST_FULL) - -ART_TEST_DEBUG_GC = getEnvBoolean('ART_TEST_DEBUG_GC', False) - -ART_TEST_BISECTION = getEnvBoolean('ART_TEST_BISECTION', False) - -DEX2OAT_HOST_INSTRUCTION_SET_FEATURES = env.get('DEX2OAT_HOST_INSTRUCTION_SET_FEATURES') - -# Do you want run-tests with the host/target's second arch? -ART_TEST_RUN_TEST_2ND_ARCH = getEnvBoolean('ART_TEST_RUN_TEST_2ND_ARCH', True) - -HOST_2ND_ARCH_PREFIX = get_build_var('HOST_2ND_ARCH_PREFIX') -HOST_2ND_ARCH_PREFIX_DEX2OAT_HOST_INSTRUCTION_SET_FEATURES = env.get( - HOST_2ND_ARCH_PREFIX + 'DEX2OAT_HOST_INSTRUCTION_SET_FEATURES') - -ART_TEST_ANDROID_ROOT = env.get('ART_TEST_ANDROID_ROOT') - -ART_TEST_WITH_STRACE = getEnvBoolean('ART_TEST_DEBUG_GC', False) - -TARGET_2ND_ARCH = get_build_var('TARGET_2ND_ARCH') -TARGET_ARCH = get_build_var('TARGET_ARCH') -if TARGET_2ND_ARCH: - if "64" in TARGET_ARCH: - ART_PHONY_TEST_TARGET_SUFFIX = "64" - _2ND_ART_PHONY_TEST_TARGET_SUFFIX = "32" - else: - ART_PHONY_TEST_TARGET_SUFFIX = "32" - _2ND_ART_PHONY_TEST_TARGET_SUFFIX = "" -else: - if "64" in TARGET_ARCH: - ART_PHONY_TEST_TARGET_SUFFIX = "64" - _2ND_ART_PHONY_TEST_TARGET_SUFFIX = "" - else: - ART_PHONY_TEST_TARGET_SUFFIX = "32" - _2ND_ART_PHONY_TEST_TARGET_SUFFIX = "" - -HOST_PREFER_32_BIT = get_build_var('HOST_PREFER_32_BIT') -if HOST_PREFER_32_BIT == "true": - ART_PHONY_TEST_HOST_SUFFIX = "32" - _2ND_ART_PHONY_TEST_HOST_SUFFIX = "" -else: - ART_PHONY_TEST_HOST_SUFFIX = "64" - _2ND_ART_PHONY_TEST_HOST_SUFFIX = "32" - -HOST_OUT_EXECUTABLES = ('%s/%s') % (ANDROID_BUILD_TOP, - get_build_var("HOST_OUT_EXECUTABLES")) -os.environ['JACK'] = HOST_OUT_EXECUTABLES + '/jack' -os.environ['DX'] = HOST_OUT_EXECUTABLES + '/dx' -os.environ['SMALI'] = HOST_OUT_EXECUTABLES + '/smali' -os.environ['JASMIN'] = HOST_OUT_EXECUTABLES + '/jasmin' -os.environ['DXMERGER'] = HOST_OUT_EXECUTABLES + '/dexmerger' diff --git a/test/testrunner/testrunner.py b/test/testrunner/testrunner.py deleted file mode 100755 index f237268f6d..0000000000 --- a/test/testrunner/testrunner.py +++ /dev/null @@ -1,794 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2017, The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""ART Run-Test TestRunner - -The testrunner runs the ART run-tests by simply invoking the script. -It fetches the list of eligible tests from art/test directory, and list of -disabled tests from art/test/knownfailures.json. It runs the tests by -invoking art/test/run-test script and checks the exit value to decide if the -test passed or failed. - -Before invoking the script, first build all the tests dependencies. -There are two major build targets for building target and host tests -dependencies: -1) test-art-host-run-test -2) test-art-target-run-test - -There are various options to invoke the script which are: --t: Either the test name as in art/test or the test name including the variant - information. Eg, "-t 001-HelloWorld", - "-t test-art-host-run-test-debug-prebuild-optimizing-relocate-ntrace-cms-checkjni-picimage-npictest-ndebuggable-001-HelloWorld32" --j: Number of thread workers to be used. Eg - "-j64" ---dry-run: Instead of running the test name, just print its name. ---verbose --b / --build-dependencies: to build the dependencies before running the test - -To specify any specific variants for the test, use --<<variant-name>>. -For eg, for compiler type as optimizing, use --optimizing. - - -In the end, the script will print the failed and skipped tests if any. - -""" -import fnmatch -import itertools -import json -from optparse import OptionParser -import os -import re -import subprocess -import sys -import threading -import time - -import env - -TARGET_TYPES = set() -RUN_TYPES = set() -PREBUILD_TYPES = set() -COMPILER_TYPES = set() -RELOCATE_TYPES = set() -TRACE_TYPES = set() -GC_TYPES = set() -JNI_TYPES = set() -IMAGE_TYPES = set() -PICTEST_TYPES = set() -DEBUGGABLE_TYPES = set() -ADDRESS_SIZES = set() -OPTIMIZING_COMPILER_TYPES = set() -ADDRESS_SIZES_TARGET = {'host': set(), 'target': set()} - -# DISABLED_TEST_CONTAINER holds information about the disabled tests. It is a map -# that has key as the test name (like 001-HelloWorld), and value as set of -# variants that the test is disabled for. -DISABLED_TEST_CONTAINER = {} - -# The Dict contains the list of all possible variants for a given type. For example, -# for key TARGET, the value would be target and host. The list is used to parse -# the test name given as the argument to run. -VARIANT_TYPE_DICT = {} - -# The set contains all the variants of each time. -TOTAL_VARIANTS_SET = set() - -# The colors are used in the output. When a test passes, COLOR_PASS is used, -# and so on. -COLOR_ERROR = '\033[91m' -COLOR_PASS = '\033[92m' -COLOR_SKIP = '\033[93m' -COLOR_NORMAL = '\033[0m' - -# The mutex object is used by the threads for exclusive access of test_count -# to make any changes in its value. -test_count_mutex = threading.Lock() -# The set contains the list of all the possible run tests that are in art/test -# directory. -RUN_TEST_SET = set() -# The semaphore object is used by the testrunner to limit the number of -# threads to the user requested concurrency value. -semaphore = threading.Semaphore(1) -# The mutex object is used to provide exclusive access to a thread to print -# its output. -print_mutex = threading.Lock() -failed_tests = [] -skipped_tests = [] - -# Flags -n_thread = 1 -test_count = 0 -total_test_count = 0 -verbose = False -last_print_length = 0 -dry_run = False -build = False -gdb = False -gdb_arg = '' -stop_testrunner = False - -def gather_test_info(): - """The method gathers test information about the test to be run which includes - generating the list of total tests from the art/test directory and the list - of disabled test. It also maps various variants to types. - """ - global TOTAL_VARIANTS_SET - global DISABLED_TEST_CONTAINER - # TODO: Avoid duplication of the variant names in different lists. - VARIANT_TYPE_DICT['pictest'] = {'pictest', 'npictest'} - VARIANT_TYPE_DICT['run'] = {'ndebug', 'debug'} - VARIANT_TYPE_DICT['target'] = {'target', 'host'} - VARIANT_TYPE_DICT['trace'] = {'trace', 'ntrace', 'stream'} - VARIANT_TYPE_DICT['image'] = {'picimage', 'no-image', 'npicimage', - 'multinpicimage', 'multipicimage'} - VARIANT_TYPE_DICT['debuggable'] = {'ndebuggable', 'debuggable'} - VARIANT_TYPE_DICT['gc'] = {'gcstress', 'gcverify', 'cms'} - VARIANT_TYPE_DICT['prebuild'] = {'no-prebuild', 'no-dex2oat', 'prebuild'} - VARIANT_TYPE_DICT['relocate'] = {'relocate-npatchoat', 'relocate', 'no-relocate'} - VARIANT_TYPE_DICT['jni'] = {'jni', 'forcecopy', 'checkjni'} - VARIANT_TYPE_DICT['address_sizes'] = {'64', '32'} - VARIANT_TYPE_DICT['compiler'] = {'interp-ac', 'interpreter', 'jit', 'optimizing', - 'regalloc_gc'} - - for v_type in VARIANT_TYPE_DICT: - TOTAL_VARIANTS_SET = TOTAL_VARIANTS_SET.union(VARIANT_TYPE_DICT.get(v_type)) - - test_dir = env.ANDROID_BUILD_TOP + '/art/test' - for f in os.listdir(test_dir): - if fnmatch.fnmatch(f, '[0-9]*'): - RUN_TEST_SET.add(f) - DISABLED_TEST_CONTAINER = get_disabled_test_info() - - -def setup_test_env(): - """The method sets default value for the various variants of the tests if they - are already not set. - """ - if env.ART_TEST_BISECTION: - env.ART_TEST_RUN_TEST_NO_PREBUILD = True - env.ART_TEST_RUN_TEST_PREBUILD = False - # Bisection search writes to standard output. - env.ART_TEST_QUIET = False - - if not TARGET_TYPES: - TARGET_TYPES.add('host') - TARGET_TYPES.add('target') - - if not PREBUILD_TYPES: - if env.ART_TEST_RUN_TEST_PREBUILD: - PREBUILD_TYPES.add('prebuild') - if env.ART_TEST_RUN_TEST_NO_PREBUILD: - PREBUILD_TYPES.add('no-prebuild') - if env.ART_TEST_RUN_TEST_NO_DEX2OAT: - PREBUILD_TYPES.add('no-dex2oat') - - if not COMPILER_TYPES: - if env.ART_TEST_INTERPRETER_ACCESS_CHECKS: - COMPILER_TYPES.add('interp-ac') - if env.ART_TEST_INTERPRETER: - COMPILER_TYPES.add('interpreter') - if env.ART_TEST_JIT: - COMPILER_TYPES.add('jit') - - if env.ART_TEST_OPTIMIZING: - COMPILER_TYPES.add('optimizing') - OPTIMIZING_COMPILER_TYPES.add('optimizing') - if env.ART_TEST_OPTIMIZING_GRAPH_COLOR: - COMPILER_TYPES.add('regalloc_gc') - OPTIMIZING_COMPILER_TYPES.add('regalloc_gc') - - if not RELOCATE_TYPES: - RELOCATE_TYPES.add('no-relocate') - if env.ART_TEST_RUN_TEST_RELOCATE: - RELOCATE_TYPES.add('relocate') - if env.ART_TEST_RUN_TEST_RELOCATE_NO_PATCHOAT: - RELOCATE_TYPES.add('relocate-npatchoat') - - if not TRACE_TYPES: - TRACE_TYPES.add('ntrace') - if env.ART_TEST_TRACE: - TRACE_TYPES.add('trace') - if env.ART_TEST_TRACE_STREAM: - TRACE_TYPES.add('stream') - - if not GC_TYPES: - GC_TYPES.add('cms') - if env.ART_TEST_GC_STRESS: - GC_TYPES.add('gcstress') - if env.ART_TEST_GC_VERIFY: - GC_TYPES.add('gcverify') - - if not JNI_TYPES: - JNI_TYPES.add('checkjni') - if env.ART_TEST_JNI_FORCECOPY: - JNI_TYPES.add('forcecopy') - - if not IMAGE_TYPES: - IMAGE_TYPES.add('picimage') - if env.ART_TEST_RUN_TEST_NO_IMAGE: - IMAGE_TYPES.add('no-image') - if env.ART_TEST_RUN_TEST_MULTI_IMAGE: - IMAGE_TYPES.add('multipicimage') - if env.ART_TEST_NPIC_IMAGE: - IMAGE_TYPES.add('npicimage') - if env.ART_TEST_RUN_TEST_MULTI_IMAGE: - IMAGE_TYPES.add('multinpicimage') - - if not PICTEST_TYPES: - PICTEST_TYPES.add('npictest') - if env.ART_TEST_PIC_TEST: - PICTEST_TYPES.add('pictest') - - if not RUN_TYPES: - if env.ART_TEST_RUN_TEST_DEBUG: - RUN_TYPES.add('debug') - if env.ART_TEST_RUN_TEST_NDEBUG: - RUN_TYPES.add('ndebug') - - if not DEBUGGABLE_TYPES: - DEBUGGABLE_TYPES.add('ndebuggable') - - if env.ART_TEST_RUN_TEST_DEBUGGABLE: - DEBUGGABLE_TYPES.add('debuggable') - - if not ADDRESS_SIZES: - ADDRESS_SIZES_TARGET['target'].add(env.ART_PHONY_TEST_TARGET_SUFFIX) - ADDRESS_SIZES_TARGET['host'].add(env.ART_PHONY_TEST_HOST_SUFFIX) - if env.ART_TEST_RUN_TEST_2ND_ARCH: - ADDRESS_SIZES_TARGET['host'].add(env._2ND_ART_PHONY_TEST_HOST_SUFFIX) - ADDRESS_SIZES_TARGET['target'].add(env._2ND_ART_PHONY_TEST_TARGET_SUFFIX) - else: - ADDRESS_SIZES_TARGET['host'] = ADDRESS_SIZES_TARGET['host'].union(ADDRESS_SIZES) - ADDRESS_SIZES_TARGET['target'] = ADDRESS_SIZES_TARGET['target'].union(ADDRESS_SIZES) - - global semaphore - semaphore = threading.Semaphore(n_thread) - - -def run_tests(tests): - """Creates thread workers to run the tests. - - The method generates command and thread worker to run the tests. Depending on - the user input for the number of threads to be used, the method uses a - semaphore object to keep a count in control for the thread workers. When a new - worker is created, it acquires the semaphore object, and when the number of - workers reaches the maximum allowed concurrency, the method wait for an - existing thread worker to release the semaphore object. Worker releases the - semaphore object when they finish printing the output. - - Args: - tests: The set of tests to be run. - """ - options_all = '' - global total_test_count - total_test_count = len(tests) - total_test_count *= len(RUN_TYPES) - total_test_count *= len(PREBUILD_TYPES) - total_test_count *= len(RELOCATE_TYPES) - total_test_count *= len(TRACE_TYPES) - total_test_count *= len(GC_TYPES) - total_test_count *= len(JNI_TYPES) - total_test_count *= len(IMAGE_TYPES) - total_test_count *= len(PICTEST_TYPES) - total_test_count *= len(DEBUGGABLE_TYPES) - total_test_count *= len(COMPILER_TYPES) - target_address_combinations = 0 - for target in TARGET_TYPES: - for address_size in ADDRESS_SIZES_TARGET[target]: - target_address_combinations += 1 - total_test_count *= target_address_combinations - - if env.ART_TEST_WITH_STRACE: - options_all += ' --strace' - - if env.ART_TEST_RUN_TEST_ALWAYS_CLEAN: - options_all += ' --always-clean' - - if env.ART_TEST_BISECTION: - options_all += ' --bisection-search' - - if env.ART_TEST_ANDROID_ROOT: - options_all += ' --android-root ' + env.ART_TEST_ARDROID_ROOT - - if gdb: - options_all += ' --gdb' - if gdb_arg: - options_all += ' --gdb-arg ' + gdb_arg - - config = itertools.product(tests, TARGET_TYPES, RUN_TYPES, PREBUILD_TYPES, - COMPILER_TYPES, RELOCATE_TYPES, TRACE_TYPES, - GC_TYPES, JNI_TYPES, IMAGE_TYPES, PICTEST_TYPES, - DEBUGGABLE_TYPES) - - for test, target, run, prebuild, compiler, relocate, trace, gc, \ - jni, image, pictest, debuggable in config: - for address_size in ADDRESS_SIZES_TARGET[target]: - if stop_testrunner: - # When ART_TEST_KEEP_GOING is set to false, then as soon as a test - # fails, stop_testrunner is set to True. When this happens, the method - # stops creating any any thread and wait for all the exising threads - # to end. - while threading.active_count() > 2: - time.sleep(0.1) - return - test_name = 'test-art-' - test_name += target + '-run-test-' - test_name += run + '-' - test_name += prebuild + '-' - test_name += compiler + '-' - test_name += relocate + '-' - test_name += trace + '-' - test_name += gc + '-' - test_name += jni + '-' - test_name += image + '-' - test_name += pictest + '-' - test_name += debuggable + '-' - test_name += test - test_name += address_size - - variant_set = {target, run, prebuild, compiler, relocate, trace, gc, jni, - image, pictest, debuggable, address_size} - - options_test = options_all - - if target == 'host': - options_test += ' --host' - - if run == 'ndebug': - options_test += ' -O' - - if prebuild == 'prebuild': - options_test += ' --prebuild' - elif prebuild == 'no-prebuild': - options_test += ' --no-prebuild' - elif prebuild == 'no-dex2oat': - options_test += ' --no-prebuild --no-dex2oat' - - if compiler == 'optimizing': - options_test += ' --optimizing' - elif compiler == 'regalloc_gc': - options_test += ' --optimizing -Xcompiler-option --register-allocation-strategy=graph-color' - elif compiler == 'interpreter': - options_test += ' --interpreter' - elif compiler == 'interp-ac': - options_test += ' --interpreter --verify-soft-fail' - elif compiler == 'jit': - options_test += ' --jit' - - if relocate == 'relocate': - options_test += ' --relocate' - elif relocate == 'no-relocate': - options_test += ' --no-relocate' - elif relocate == 'relocate-npatchoat': - options_test += ' --relocate --no-patchoat' - - if trace == 'trace': - options_test += ' --trace' - elif trace == 'stream': - options_test += ' --trace --stream' - - if gc == 'gcverify': - options_test += ' --gcverify' - elif gc == 'gcstress': - options_test += ' --gcstress' - - if jni == 'forcecopy': - options_test += ' --runtime-option -Xjniopts:forcecopy' - elif jni == 'checkjni': - options_test += ' --runtime-option -Xcheck:jni' - - if image == 'no-image': - options_test += ' --no-image' - elif image == 'npicimage': - options_test += ' --npic-image' - elif image == 'multinpicimage': - options_test += ' --npic-image --multi-image' - elif image == 'multipicimage': - options_test += ' --multi-image' - - if pictest == 'pictest': - options_test += ' --pic-test' - - if debuggable == 'debuggable': - options_test += ' --debuggable' - - if address_size == '64': - options_test += ' --64' - - if env.DEX2OAT_HOST_INSTRUCTION_SET_FEATURES: - options_test += ' --instruction-set-features' + env.DEX2OAT_HOST_INSTRUCTION_SET_FEATURES - - elif address_size == '32': - if env.HOST_2ND_ARCH_PREFIX_DEX2OAT_HOST_INSTRUCTION_SET_FEATURES: - options_test += ' --instruction-set-features ' + \ - env.HOST_2ND_ARCH_PREFIX_DEX2OAT_HOST_INSTRUCTION_SET_FEATURES - - options_test = (' --output-path %s/run-test-output/%s') % ( - env.ART_HOST_TEST_DIR, test_name) + options_test - - run_test_sh = env.ANDROID_BUILD_TOP + '/art/test/run-test' - command = run_test_sh + ' ' + options_test + ' ' + test - - semaphore.acquire() - worker = threading.Thread(target=run_test, args=(command, test, variant_set, test_name)) - worker.daemon = True - worker.start() - - while threading.active_count() > 2: - time.sleep(0.1) - - -def run_test(command, test, test_variant, test_name): - """Runs the test. - - It invokes art/test/run-test script to run the test. The output of the script - is checked, and if it ends with "Succeeded!", it assumes that the tests - passed, otherwise, put it in the list of failed test. Before actually running - the test, it also checks if the test is placed in the list of disabled tests, - and if yes, it skips running it, and adds the test in the list of skipped - tests. The method uses print_text method to actually print the output. After - successfully running and capturing the output for the test, it releases the - semaphore object. - - Args: - command: The command to be used to invoke the script - test: The name of the test without the variant information. - test_variant: The set of variant for the test. - test_name: The name of the test along with the variants. - """ - global last_print_length - global test_count - global stop_testrunner - if is_test_disabled(test, test_variant): - test_skipped = True - else: - test_skipped = False - proc = subprocess.Popen(command.split(), stderr=subprocess.PIPE) - script_output = proc.stderr.read().strip() - test_passed = not proc.wait() - - # If verbose is set to True, every test information is printed on a new line. - # If not, the information is printed on the same line overriding the - # previous test output. - if not verbose: - suffix = '\r' - prefix = ' ' * last_print_length + '\r' - else: - suffix = '\n' - prefix = '' - test_count_mutex.acquire() - test_count += 1 - percent = (test_count * 100) / total_test_count - out = '[ ' + str(percent) + '% ' + str(test_count) + '/' + str(total_test_count) + ' ] ' - test_count_mutex.release() - out += test_name + ' ' - if not test_skipped: - if test_passed: - out += COLOR_PASS + 'PASS' + COLOR_NORMAL - else: - out += COLOR_ERROR + 'FAIL' + COLOR_NORMAL - failed_tests.append(test_name) - if verbose: - out += '\n' + command + '\n' + script_output - if not env.ART_TEST_KEEP_GOING: - stop_testrunner = True - elif not dry_run: - out += COLOR_SKIP + 'SKIP' + COLOR_NORMAL - skipped_tests.append(test_name) - last_print_length = len(out) - print_mutex.acquire() - print_text(prefix + out + suffix) - print_mutex.release() - semaphore.release() - - -def get_disabled_test_info(): - """Generate set of known failures. - - It parses the art/test/knownfailures.json file to generate the list of - disabled tests. - - Returns: - The method returns a dict of tests mapped to the variants list - for which the test should not be run. - """ - known_failures_file = env.ANDROID_BUILD_TOP + '/art/test/knownfailures.json' - with open(known_failures_file) as known_failures_json: - known_failures_info = json.loads(known_failures_json.read()) - - disabled_test_info = {} - for failure in known_failures_info: - tests = failure.get('test') - if tests: - tests = [tests] - else: - tests = failure.get('tests', []) - variants = parse_variants(failure.get('variant')) - env_vars = failure.get('env_vars') - if check_env_vars(env_vars): - for test in tests: - if test in disabled_test_info: - disabled_test_info[test] = disabled_test_info[test].union(variants) - else: - disabled_test_info[test] = variants - return disabled_test_info - - -def check_env_vars(env_vars): - """Checks if the env variables are set as required to run the test. - - Returns: - True if all the env variables are set as required, otherwise False. - """ - - if not env_vars: - return True - for key in env_vars: - if env.get_env(key) != env_vars.get(key): - return False - return True - - -def is_test_disabled(test, variant_set): - """Checks if the test along with the variant_set is disabled. - - Args: - test: The name of the test as in art/test directory. - variant_set: Variants to be used for the test. - Returns: - True, if the test is disabled. - """ - if dry_run: - return True - variants_list = DISABLED_TEST_CONTAINER.get(test, {}) - for variants in variants_list: - variants_present = True - for variant in variants: - if variant not in variant_set: - variants_present = False - break - if variants_present: - return True - return False - - -def parse_variants(variants): - """Parse variants fetched from art/test/knownfailures.json. - """ - if not variants: - variants = '' - for variant in TOTAL_VARIANTS_SET: - variants += variant - variants += '|' - variants = variants[:-1] - variant_list = set() - or_variants = variants.split('|') - for or_variant in or_variants: - and_variants = or_variant.split('&') - variant = set() - for and_variant in and_variants: - and_variant = and_variant.strip() - variant.add(and_variant) - variant_list.add(frozenset(variant)) - return variant_list - - -def print_text(output): - sys.stdout.write(output) - sys.stdout.flush() - - -def print_analysis(): - if not verbose: - print_text(' ' * last_print_length + '\r') - if skipped_tests: - print_text(COLOR_SKIP + 'SKIPPED TESTS' + COLOR_NORMAL + '\n') - for test in skipped_tests: - print_text(test + '\n') - print_text('\n') - - if failed_tests: - print_text(COLOR_ERROR + 'FAILED TESTS' + COLOR_NORMAL + '\n') - for test in failed_tests: - print_text(test + '\n') - - -def parse_test_name(test_name): - """Parses the testname provided by the user. - It supports two types of test_name: - 1) Like 001-HelloWorld. In this case, it will just verify if the test actually - exists and if it does, it returns the testname. - 2) Like test-art-host-run-test-debug-prebuild-interpreter-no-relocate-ntrace-cms-checkjni-picimage-npictest-ndebuggable-001-HelloWorld32 - In this case, it will parse all the variants and check if they are placed - correctly. If yes, it will set the various VARIANT_TYPES to use the - variants required to run the test. Again, it returns the test_name - without the variant information like 001-HelloWorld. - """ - if test_name in RUN_TEST_SET: - return {test_name} - - regex = '^test-art-' - regex += '(' + '|'.join(VARIANT_TYPE_DICT['target']) + ')-' - regex += 'run-test-' - regex += '(' + '|'.join(VARIANT_TYPE_DICT['run']) + ')-' - regex += '(' + '|'.join(VARIANT_TYPE_DICT['prebuild']) + ')-' - regex += '(' + '|'.join(VARIANT_TYPE_DICT['compiler']) + ')-' - regex += '(' + '|'.join(VARIANT_TYPE_DICT['relocate']) + ')-' - regex += '(' + '|'.join(VARIANT_TYPE_DICT['trace']) + ')-' - regex += '(' + '|'.join(VARIANT_TYPE_DICT['gc']) + ')-' - regex += '(' + '|'.join(VARIANT_TYPE_DICT['jni']) + ')-' - regex += '(' + '|'.join(VARIANT_TYPE_DICT['image']) + ')-' - regex += '(' + '|'.join(VARIANT_TYPE_DICT['pictest']) + ')-' - regex += '(' + '|'.join(VARIANT_TYPE_DICT['debuggable']) + ')-' - regex += '(' + '|'.join(RUN_TEST_SET) + ')' - regex += '(' + '|'.join(VARIANT_TYPE_DICT['address_sizes']) + ')$' - match = re.match(regex, test_name) - if match: - TARGET_TYPES.add(match.group(1)) - RUN_TYPES.add(match.group(2)) - PREBUILD_TYPES.add(match.group(3)) - COMPILER_TYPES.add(match.group(4)) - RELOCATE_TYPES.add(match.group(5)) - TRACE_TYPES.add(match.group(6)) - GC_TYPES.add(match.group(7)) - JNI_TYPES.add(match.group(8)) - IMAGE_TYPES.add(match.group(9)) - PICTEST_TYPES.add(match.group(10)) - DEBUGGABLE_TYPES.add(match.group(11)) - ADDRESS_SIZES.add(match.group(13)) - return {match.group(12)} - - -def parse_option(): - global verbose - global dry_run - global n_thread - global build - global gdb - global gdb_arg - - parser = OptionParser() - parser.add_option('-t', '--test', dest='test', help='name of the test') - parser.add_option('-j', type='int', dest='n_thread') - for variant in TOTAL_VARIANTS_SET: - flag = '--' + variant - flag_dest = variant.replace('-', '_') - if variant == '32' or variant == '64': - flag_dest = 'n' + flag_dest - parser.add_option(flag, action='store_true', dest=flag_dest) - parser.add_option('--verbose', '-v', action='store_true', dest='verbose') - parser.add_option('--dry-run', action='store_true', dest='dry_run') - parser.add_option('-b', '--build-dependencies', action='store_true', dest='build') - parser.add_option('--gdb', action='store_true', dest='gdb') - parser.add_option('--gdb-arg', dest='gdb_arg') - - options = parser.parse_args()[0] - test = '' - if options.test: - test = parse_test_name(options.test) - if options.pictest: - PICTEST_TYPES.add('pictest') - if options.ndebug: - RUN_TYPES.add('ndebug') - if options.interp_ac: - COMPILER_TYPES.add('interp-ac') - if options.picimage: - IMAGE_TYPES.add('picimage') - if options.n64: - ADDRESS_SIZES.add('64') - if options.interpreter: - COMPILER_TYPES.add('interpreter') - if options.jni: - JNI_TYPES.add('jni') - if options.relocate_npatchoat: - RELOCATE_TYPES.add('relocate-npatchoat') - if options.no_prebuild: - PREBUILD_TYPES.add('no-prebuild') - if options.npictest: - PICTEST_TYPES.add('npictest') - if options.no_dex2oat: - PREBUILD_TYPES.add('no-dex2oat') - if options.jit: - COMPILER_TYPES.add('jit') - if options.relocate: - RELOCATE_TYPES.add('relocate') - if options.ndebuggable: - DEBUGGABLE_TYPES.add('ndebuggable') - if options.no_image: - IMAGE_TYPES.add('no-image') - if options.optimizing: - COMPILER_TYPES.add('optimizing') - if options.trace: - TRACE_TYPES.add('trace') - if options.gcstress: - GC_TYPES.add('gcstress') - if options.no_relocate: - RELOCATE_TYPES.add('no-relocate') - if options.target: - TARGET_TYPES.add('target') - if options.forcecopy: - JNI_TYPES.add('forcecopy') - if options.n32: - ADDRESS_SIZES.add('32') - if options.host: - TARGET_TYPES.add('host') - if options.gcverify: - GC_TYPES.add('gcverify') - if options.debuggable: - DEBUGGABLE_TYPES.add('debuggable') - if options.prebuild: - PREBUILD_TYPES.add('prebuild') - if options.debug: - RUN_TYPES.add('debug') - if options.checkjni: - JNI_TYPES.add('checkjni') - if options.ntrace: - TRACE_TYPES.add('ntrace') - if options.cms: - GC_TYPES.add('cms') - if options.npicimage: - IMAGE_TYPES.add('npicimage') - if options.multinpicimage: - IMAGE_TYPES.add('multinpicimage') - if options.multipicimage: - IMAGE_TYPES.add('multipicimage') - if options.verbose: - verbose = True - if options.n_thread: - n_thread = max(1, options.n_thread) - if options.dry_run: - dry_run = True - verbose = True - if options.build: - build = True - if options.gdb: - n_thread = 1 - gdb = True - if options.gdb_arg: - gdb_arg = options.gdb_arg - - return test - -def main(): - gather_test_info() - user_requested_test = parse_option() - setup_test_env() - if build: - build_targets = '' - if 'host' in TARGET_TYPES: - build_targets += 'test-art-host-run-test-dependencies' - if 'target' in TARGET_TYPES: - build_targets += 'test-art-target-run-test-dependencies' - build_command = 'make -j' + str(n_thread) + ' ' + build_targets - if subprocess.call(build_command.split()): - sys.exit(1) - if user_requested_test: - test_runner_thread = threading.Thread(target=run_tests, args=(user_requested_test,)) - else: - test_runner_thread = threading.Thread(target=run_tests, args=(RUN_TEST_SET,)) - test_runner_thread.daemon = True - try: - test_runner_thread.start() - while threading.active_count() > 1: - time.sleep(0.1) - print_analysis() - if failed_tests: - sys.exit(1) - sys.exit(0) - except KeyboardInterrupt: - print_analysis() - -if __name__ == '__main__': - main() |