diff options
Diffstat (limited to 'test/testrunner/testrunner.py')
| -rwxr-xr-x | test/testrunner/testrunner.py | 338 |
1 files changed, 115 insertions, 223 deletions
diff --git a/test/testrunner/testrunner.py b/test/testrunner/testrunner.py index f425097b57..2a772ffa1f 100755 --- a/test/testrunner/testrunner.py +++ b/test/testrunner/testrunner.py @@ -45,6 +45,7 @@ In the end, the script will print the failed and skipped tests if any. """ import argparse +import collections import fnmatch import itertools import json @@ -60,21 +61,6 @@ import time import env from target_config import target_config -TARGET_TYPES = set() -RUN_TYPES = set() -PREBUILD_TYPES = set() -COMPILER_TYPES = set() -RELOCATE_TYPES = set() -TRACE_TYPES = set() -GC_TYPES = set() -JNI_TYPES = set() -IMAGE_TYPES = set() -PICTEST_TYPES = set() -DEBUGGABLE_TYPES = set() -ADDRESS_SIZES = set() -OPTIMIZING_COMPILER_TYPES = set() -JVMTI_TYPES = set() -ADDRESS_SIZES_TARGET = {'host': set(), 'target': set()} # timeout for individual tests. # TODO: make it adjustable per tests and for buildbots timeout = 3000 # 50 minutes @@ -128,6 +114,12 @@ gdb = False gdb_arg = '' stop_testrunner = False dex2oat_jobs = -1 # -1 corresponds to default threads for dex2oat +run_all_configs = False + +# Dict to store user requested test variants. +# key: variant_type. +# value: set of variants user wants to run of type <key>. +_user_input_variants = collections.defaultdict(set) def gather_test_info(): """The method gathers test information about the test to be run which includes @@ -151,7 +143,7 @@ def gather_test_info(): VARIANT_TYPE_DICT['jvmti'] = {'no-jvmti', 'jvmti-stress', 'redefine-stress', 'trace-stress', 'field-stress', 'step-stress'} VARIANT_TYPE_DICT['compiler'] = {'interp-ac', 'interpreter', 'jit', 'optimizing', - 'regalloc_gc', 'speed-profile'} + 'regalloc_gc', 'speed-profile'} for v_type in VARIANT_TYPE_DICT: TOTAL_VARIANTS_SET = TOTAL_VARIANTS_SET.union(VARIANT_TYPE_DICT.get(v_type)) @@ -173,106 +165,75 @@ def setup_test_env(): # Bisection search writes to standard output. env.ART_TEST_QUIET = False - if not TARGET_TYPES: - TARGET_TYPES.add('host') - TARGET_TYPES.add('target') - - if env.ART_TEST_RUN_TEST_NO_PREBUILD: - PREBUILD_TYPES.add('no-prebuild') - if env.ART_TEST_RUN_TEST_NO_DEX2OAT: - PREBUILD_TYPES.add('no-dex2oat') - if env.ART_TEST_RUN_TEST_PREBUILD or not PREBUILD_TYPES: # Default - PREBUILD_TYPES.add('prebuild') - - if env.ART_TEST_INTERPRETER_ACCESS_CHECKS: - COMPILER_TYPES.add('interp-ac') - if env.ART_TEST_INTERPRETER: - COMPILER_TYPES.add('interpreter') - if env.ART_TEST_JIT: - COMPILER_TYPES.add('jit') - if env.ART_TEST_OPTIMIZING_GRAPH_COLOR: - COMPILER_TYPES.add('regalloc_gc') - OPTIMIZING_COMPILER_TYPES.add('regalloc_gc') - if env.ART_TEST_OPTIMIZING: - COMPILER_TYPES.add('optimizing') - OPTIMIZING_COMPILER_TYPES.add('optimizing') - if env.ART_TEST_SPEED_PROFILE: - COMPILER_TYPES.add('speed-profile') + global _user_input_variants + global run_all_configs + if run_all_configs: + target_types = _user_input_variants['target'] + _user_input_variants = VARIANT_TYPE_DICT + _user_input_variants['target'] = target_types + + if not _user_input_variants['target']: + _user_input_variants['target'].add('host') + _user_input_variants['target'].add('target') + + if not _user_input_variants['prebuild']: # Default + _user_input_variants['prebuild'].add('prebuild') # By default only run without jvmti - if not JVMTI_TYPES: - JVMTI_TYPES.add('no-jvmti') + if not _user_input_variants['jvmti']: + _user_input_variants['jvmti'].add('no-jvmti') # By default we run all 'compiler' variants. - if not COMPILER_TYPES: - COMPILER_TYPES.add('optimizing') - COMPILER_TYPES.add('jit') - COMPILER_TYPES.add('interpreter') - COMPILER_TYPES.add('interp-ac') - COMPILER_TYPES.add('speed-profile') - OPTIMIZING_COMPILER_TYPES.add('optimizing') - - if env.ART_TEST_RUN_TEST_RELOCATE: - RELOCATE_TYPES.add('relocate') - if env.ART_TEST_RUN_TEST_RELOCATE_NO_PATCHOAT: - RELOCATE_TYPES.add('relocate-npatchoat') - if not RELOCATE_TYPES: # Default - RELOCATE_TYPES.add('no-relocate') - - if env.ART_TEST_TRACE: - TRACE_TYPES.add('trace') - if env.ART_TEST_TRACE_STREAM: - TRACE_TYPES.add('stream') - if not TRACE_TYPES: # Default - TRACE_TYPES.add('ntrace') - - if env.ART_TEST_GC_STRESS: - GC_TYPES.add('gcstress') - if env.ART_TEST_GC_VERIFY: - GC_TYPES.add('gcverify') - if not GC_TYPES: # Default - GC_TYPES.add('cms') - - if env.ART_TEST_JNI_FORCECOPY: - JNI_TYPES.add('forcecopy') - if not JNI_TYPES: # Default - JNI_TYPES.add('checkjni') - - if env.ART_TEST_RUN_TEST_NO_IMAGE: - IMAGE_TYPES.add('no-image') - if env.ART_TEST_RUN_TEST_MULTI_IMAGE: - IMAGE_TYPES.add('multipicimage') - if env.ART_TEST_RUN_TEST_IMAGE or not IMAGE_TYPES: # Default - IMAGE_TYPES.add('picimage') - - if env.ART_TEST_PIC_TEST: - PICTEST_TYPES.add('pictest') - if not PICTEST_TYPES: # Default - PICTEST_TYPES.add('npictest') - - if env.ART_TEST_RUN_TEST_NDEBUG: - RUN_TYPES.add('ndebug') - if env.ART_TEST_RUN_TEST_DEBUG or not RUN_TYPES: # Default - RUN_TYPES.add('debug') - - if env.ART_TEST_RUN_TEST_DEBUGGABLE: - DEBUGGABLE_TYPES.add('debuggable') - if not DEBUGGABLE_TYPES: # Default - DEBUGGABLE_TYPES.add('ndebuggable') - - if not ADDRESS_SIZES: - ADDRESS_SIZES_TARGET['target'].add(env.ART_PHONY_TEST_TARGET_SUFFIX) - ADDRESS_SIZES_TARGET['host'].add(env.ART_PHONY_TEST_HOST_SUFFIX) + if not _user_input_variants['compiler']: + _user_input_variants['compiler'].add('optimizing') + _user_input_variants['compiler'].add('jit') + _user_input_variants['compiler'].add('interpreter') + _user_input_variants['compiler'].add('interp-ac') + _user_input_variants['compiler'].add('speed-profile') + + if not _user_input_variants['relocate']: # Default + _user_input_variants['relocate'].add('no-relocate') + + if not _user_input_variants['trace']: # Default + _user_input_variants['trace'].add('ntrace') + + if not _user_input_variants['gc']: # Default + _user_input_variants['gc'].add('cms') + + if not _user_input_variants['jni']: # Default + _user_input_variants['jni'].add('checkjni') + + if not _user_input_variants['image']: # Default + _user_input_variants['image'].add('picimage') + + + if not _user_input_variants['pictest']: # Default + _user_input_variants['pictest'].add('npictest') + + if not _user_input_variants['debuggable']: # Default + _user_input_variants['debuggable'].add('ndebuggable') + + if not _user_input_variants['run']: # Default + _user_input_variants['run'].add('debug') + + _user_input_variants['address_sizes_target'] = collections.defaultdict(set) + if not _user_input_variants['address_sizes']: + _user_input_variants['address_sizes_target']['target'].add( + env.ART_PHONY_TEST_TARGET_SUFFIX) + _user_input_variants['address_sizes_target']['host'].add( + env.ART_PHONY_TEST_HOST_SUFFIX) if env.ART_TEST_RUN_TEST_2ND_ARCH: - ADDRESS_SIZES_TARGET['host'].add(env.ART_2ND_PHONY_TEST_HOST_SUFFIX) - ADDRESS_SIZES_TARGET['target'].add(env.ART_2ND_PHONY_TEST_TARGET_SUFFIX) + _user_input_variants['address_sizes_target']['host'].add( + env.ART_2ND_PHONY_TEST_HOST_SUFFIX) + _user_input_variants['address_sizes_target']['target'].add( + env.ART_2ND_PHONY_TEST_TARGET_SUFFIX) else: - ADDRESS_SIZES_TARGET['host'] = ADDRESS_SIZES_TARGET['host'].union(ADDRESS_SIZES) - ADDRESS_SIZES_TARGET['target'] = ADDRESS_SIZES_TARGET['target'].union(ADDRESS_SIZES) + _user_input_variants['address_sizes_target']['host'] = _user_input_variants['address_sizes'] + _user_input_variants['address_sizes_target']['target'] = _user_input_variants['address_sizes'] global n_thread if n_thread is -1: - if 'target' in TARGET_TYPES: + if 'target' in _user_input_variants['target']: n_thread = get_default_threads('target') else: n_thread = get_default_threads('host') @@ -308,20 +269,12 @@ def run_tests(tests): options_all = '' global total_test_count total_test_count = len(tests) - total_test_count *= len(RUN_TYPES) - total_test_count *= len(PREBUILD_TYPES) - total_test_count *= len(RELOCATE_TYPES) - total_test_count *= len(TRACE_TYPES) - total_test_count *= len(GC_TYPES) - total_test_count *= len(JNI_TYPES) - total_test_count *= len(IMAGE_TYPES) - total_test_count *= len(PICTEST_TYPES) - total_test_count *= len(DEBUGGABLE_TYPES) - total_test_count *= len(COMPILER_TYPES) - total_test_count *= len(JVMTI_TYPES) + for variant_type in VARIANT_TYPE_DICT: + if not (variant_type == 'target' or 'address_sizes' in variant_type): + total_test_count *= len(_user_input_variants[variant_type]) target_address_combinations = 0 - for target in TARGET_TYPES: - for address_size in ADDRESS_SIZES_TARGET[target]: + for target in _user_input_variants['target']: + for address_size in _user_input_variants['address_sizes_target'][target]: target_address_combinations += 1 total_test_count *= target_address_combinations @@ -345,14 +298,16 @@ def run_tests(tests): if dex2oat_jobs != -1: options_all += ' --dex2oat-jobs ' + str(dex2oat_jobs) - config = itertools.product(tests, TARGET_TYPES, RUN_TYPES, PREBUILD_TYPES, - COMPILER_TYPES, RELOCATE_TYPES, TRACE_TYPES, - GC_TYPES, JNI_TYPES, IMAGE_TYPES, PICTEST_TYPES, - DEBUGGABLE_TYPES, JVMTI_TYPES) + config = itertools.product(tests, _user_input_variants['target'], _user_input_variants['run'], + _user_input_variants['prebuild'], _user_input_variants['compiler'], + _user_input_variants['relocate'], _user_input_variants['trace'], + _user_input_variants['gc'], _user_input_variants['jni'], + _user_input_variants['image'], _user_input_variants['pictest'], + _user_input_variants['debuggable'], _user_input_variants['jvmti']) for test, target, run, prebuild, compiler, relocate, trace, gc, \ jni, image, pictest, debuggable, jvmti in config: - for address_size in ADDRESS_SIZES_TARGET[target]: + for address_size in _user_input_variants['address_sizes_target'][target]: if stop_testrunner: # When ART_TEST_KEEP_GOING is set to false, then as soon as a test # fails, stop_testrunner is set to True. When this happens, the method @@ -577,11 +532,10 @@ def print_test_info(test_name, result, failed_test_info=""): total_test_count) if result == 'FAIL' or result == 'TIMEOUT': - info += ('%s %s %s\n%s\n') % ( + info += ('%s %s %s\n') % ( progress_info, test_name, - COLOR_ERROR + result + COLOR_NORMAL, - failed_test_info) + COLOR_ERROR + result + COLOR_NORMAL) else: result_text = '' if result == 'PASS': @@ -617,6 +571,7 @@ def print_test_info(test_name, result, failed_test_info=""): def verify_knownfailure_entry(entry): supported_field = { 'tests' : (list, str), + 'test_patterns' : (list,), 'description' : (list, str), 'bug' : (str,), 'variant' : (str,), @@ -650,6 +605,11 @@ def get_disabled_test_info(): tests = failure.get('tests', []) if isinstance(tests, str): tests = [tests] + patterns = failure.get("test_patterns", []) + if (not isinstance(patterns, list)): + raise ValueError("test_patters is not a list in %s" % failure) + + tests += [f for f in RUN_TEST_SET if any(re.match(pat, f) is not None for pat in patterns)] variants = parse_variants(failure.get('variant')) env_vars = failure.get('env_vars') @@ -804,19 +764,19 @@ def parse_test_name(test_name): regex += '(' + '|'.join(VARIANT_TYPE_DICT['address_sizes']) + ')$' match = re.match(regex, test_name) if match: - TARGET_TYPES.add(match.group(1)) - RUN_TYPES.add(match.group(2)) - PREBUILD_TYPES.add(match.group(3)) - COMPILER_TYPES.add(match.group(4)) - RELOCATE_TYPES.add(match.group(5)) - TRACE_TYPES.add(match.group(6)) - GC_TYPES.add(match.group(7)) - JNI_TYPES.add(match.group(8)) - IMAGE_TYPES.add(match.group(9)) - PICTEST_TYPES.add(match.group(10)) - DEBUGGABLE_TYPES.add(match.group(11)) - JVMTI_TYPES.add(match.group(12)) - ADDRESS_SIZES.add(match.group(14)) + _user_input_variants['target'].add(match.group(1)) + _user_input_variants['run'].add(match.group(2)) + _user_input_variants['prebuild'].add(match.group(3)) + _user_input_variants['compiler'].add(match.group(4)) + _user_input_variants['relocate'].add(match.group(5)) + _user_input_variants['trace'].add(match.group(6)) + _user_input_variants['gc'].add(match.group(7)) + _user_input_variants['jni'].add(match.group(8)) + _user_input_variants['image'].add(match.group(9)) + _user_input_variants['pictest'].add(match.group(10)) + _user_input_variants['debuggable'].add(match.group(11)) + _user_input_variants['jvmti'].add(match.group(12)) + _user_input_variants['address_sizes'].add(match.group(14)) return {match.group(13)} raise ValueError(test_name + " is not a valid test") @@ -865,6 +825,7 @@ def parse_option(): global gdb_arg global timeout global dex2oat_jobs + global run_all_configs parser = argparse.ArgumentParser(description="Runs all or a subset of the ART test suite.") parser.add_argument('-t', '--test', dest='test', help='name of the test') @@ -872,10 +833,7 @@ def parse_option(): parser.add_argument('--timeout', default=timeout, type=int, dest='timeout') for variant in TOTAL_VARIANTS_SET: flag = '--' + variant - flag_dest = variant.replace('-', '_') - if variant == '32' or variant == '64': - flag_dest = 'n' + flag_dest - parser.add_argument(flag, action='store_true', dest=flag_dest) + parser.add_argument(flag, action='store_true', dest=variant) parser.add_argument('--verbose', '-v', action='store_true', dest='verbose') parser.add_argument('--dry-run', action='store_true', dest='dry_run') parser.add_argument("--skip", action="append", dest="skips", default=[], @@ -894,6 +852,8 @@ def parse_option(): parser.add_argument('--gdb-arg', dest='gdb_arg') parser.add_argument('--dex2oat-jobs', type=int, dest='dex2oat_jobs', help='Number of dex2oat jobs') + parser.add_argument('-a', '--all', action='store_true', dest='run_all', + help="Run all the possible configurations for the input test set") options = vars(parser.parse_args()) if options['build_target']: @@ -904,82 +864,12 @@ def parse_option(): env.EXTRA_DISABLED_TESTS.update(set(options['skips'])) if options['test']: test = parse_test_name(options['test']) - if options['pictest']: - PICTEST_TYPES.add('pictest') - if options['ndebug']: - RUN_TYPES.add('ndebug') - if options['interp_ac']: - COMPILER_TYPES.add('interp-ac') - if options['picimage']: - IMAGE_TYPES.add('picimage') - if options['n64']: - ADDRESS_SIZES.add('64') - if options['interpreter']: - COMPILER_TYPES.add('interpreter') - if options['jni']: - JNI_TYPES.add('jni') - if options['relocate_npatchoat']: - RELOCATE_TYPES.add('relocate-npatchoat') - if options['no_prebuild']: - PREBUILD_TYPES.add('no-prebuild') - if options['npictest']: - PICTEST_TYPES.add('npictest') - if options['no_dex2oat']: - PREBUILD_TYPES.add('no-dex2oat') - if options['jit']: - COMPILER_TYPES.add('jit') - if options['relocate']: - RELOCATE_TYPES.add('relocate') - if options['ndebuggable']: - DEBUGGABLE_TYPES.add('ndebuggable') - if options['no_image']: - IMAGE_TYPES.add('no-image') - if options['optimizing']: - COMPILER_TYPES.add('optimizing') - if options['speed_profile']: - COMPILER_TYPES.add('speed-profile') - if options['trace']: - TRACE_TYPES.add('trace') - if options['gcstress']: - GC_TYPES.add('gcstress') - if options['no_relocate']: - RELOCATE_TYPES.add('no-relocate') - if options['target']: - TARGET_TYPES.add('target') - if options['forcecopy']: - JNI_TYPES.add('forcecopy') - if options['n32']: - ADDRESS_SIZES.add('32') - if options['host']: - TARGET_TYPES.add('host') - if options['gcverify']: - GC_TYPES.add('gcverify') - if options['debuggable']: - DEBUGGABLE_TYPES.add('debuggable') - if options['prebuild']: - PREBUILD_TYPES.add('prebuild') - if options['debug']: - RUN_TYPES.add('debug') - if options['checkjni']: - JNI_TYPES.add('checkjni') - if options['ntrace']: - TRACE_TYPES.add('ntrace') - if options['cms']: - GC_TYPES.add('cms') - if options['multipicimage']: - IMAGE_TYPES.add('multipicimage') - if options['jvmti_stress']: - JVMTI_TYPES.add('jvmti-stress') - if options['redefine_stress']: - JVMTI_TYPES.add('redefine-stress') - if options['field_stress']: - JVMTI_TYPES.add('field-stress') - if options['step_stress']: - JVMTI_TYPES.add('step-stress') - if options['trace_stress']: - JVMTI_TYPES.add('trace-stress') - if options['no_jvmti']: - JVMTI_TYPES.add('no-jvmti') + + for variant_type in VARIANT_TYPE_DICT: + for variant in VARIANT_TYPE_DICT[variant_type]: + if options.get(variant): + _user_input_variants[variant_type].add(variant) + if options['verbose']: verbose = True if options['n_thread']: @@ -996,6 +886,8 @@ def parse_option(): timeout = options['timeout'] if options['dex2oat_jobs']: dex2oat_jobs = options['dex2oat_jobs'] + if options['run_all']: + run_all_configs = True return test @@ -1005,9 +897,9 @@ def main(): setup_test_env() if build: build_targets = '' - if 'host' in TARGET_TYPES: + if 'host' in _user_input_variants['target']: build_targets += 'test-art-host-run-test-dependencies' - if 'target' in TARGET_TYPES: + if 'target' in _user_input_variants['target']: build_targets += 'test-art-target-run-test-dependencies' build_command = 'make' build_command += ' -j' |