diff options
Diffstat (limited to 'test/testrunner/testrunner.py')
| -rwxr-xr-x | test/testrunner/testrunner.py | 269 |
1 files changed, 196 insertions, 73 deletions
diff --git a/test/testrunner/testrunner.py b/test/testrunner/testrunner.py index c22b0be9f4..6a8b0aeb06 100755 --- a/test/testrunner/testrunner.py +++ b/test/testrunner/testrunner.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # Copyright 2017, The Android Open Source Project # @@ -48,14 +48,17 @@ import argparse import fnmatch import itertools import json +import multiprocessing import os import re import subprocess import sys +import tempfile import threading import time import env +from target_config import target_config TARGET_TYPES = set() RUN_TYPES = set() @@ -71,6 +74,9 @@ DEBUGGABLE_TYPES = set() ADDRESS_SIZES = set() OPTIMIZING_COMPILER_TYPES = set() ADDRESS_SIZES_TARGET = {'host': set(), 'target': set()} +# timeout for individual tests. +# TODO: make it adjustable per tests and for buildbots +timeout = 3000 # 50 minutes # DISABLED_TEST_CONTAINER holds information about the disabled tests. It is a map # that has key as the test name (like 001-HelloWorld), and value as set of @@ -111,7 +117,7 @@ failed_tests = [] skipped_tests = [] # Flags -n_thread = 1 +n_thread = -1 test_count = 0 total_test_count = 0 verbose = False @@ -141,7 +147,7 @@ def gather_test_info(): VARIANT_TYPE_DICT['jni'] = {'jni', 'forcecopy', 'checkjni'} VARIANT_TYPE_DICT['address_sizes'] = {'64', '32'} VARIANT_TYPE_DICT['compiler'] = {'interp-ac', 'interpreter', 'jit', 'optimizing', - 'regalloc_gc'} + 'regalloc_gc', 'speed-profile'} for v_type in VARIANT_TYPE_DICT: TOTAL_VARIANTS_SET = TOTAL_VARIANTS_SET.union(VARIANT_TYPE_DICT.get(v_type)) @@ -183,9 +189,20 @@ def setup_test_env(): if env.ART_TEST_OPTIMIZING_GRAPH_COLOR: COMPILER_TYPES.add('regalloc_gc') OPTIMIZING_COMPILER_TYPES.add('regalloc_gc') - if env.ART_TEST_OPTIMIZING or not COMPILER_TYPES: # Default + if env.ART_TEST_OPTIMIZING: COMPILER_TYPES.add('optimizing') OPTIMIZING_COMPILER_TYPES.add('optimizing') + if env.ART_TEST_SPEED_PROFILE: + COMPILER_TYPES.add('speed-profile') + + # By default we run all 'compiler' variants. + if not COMPILER_TYPES: + COMPILER_TYPES.add('optimizing') + COMPILER_TYPES.add('jit') + COMPILER_TYPES.add('interpreter') + COMPILER_TYPES.add('interp-ac') + COMPILER_TYPES.add('speed-profile') + OPTIMIZING_COMPILER_TYPES.add('optimizing') if env.ART_TEST_RUN_TEST_RELOCATE: RELOCATE_TYPES.add('relocate') @@ -245,9 +262,26 @@ def setup_test_env(): ADDRESS_SIZES_TARGET['host'] = ADDRESS_SIZES_TARGET['host'].union(ADDRESS_SIZES) ADDRESS_SIZES_TARGET['target'] = ADDRESS_SIZES_TARGET['target'].union(ADDRESS_SIZES) + global n_thread + if n_thread is -1: + if 'target' in TARGET_TYPES: + n_thread = get_default_threads('target') + else: + n_thread = get_default_threads('host') + global semaphore semaphore = threading.Semaphore(n_thread) + if not sys.stdout.isatty(): + global COLOR_ERROR + global COLOR_PASS + global COLOR_SKIP + global COLOR_NORMAL + COLOR_ERROR = '' + COLOR_PASS = '' + COLOR_SKIP = '' + COLOR_NORMAL = '' + def run_tests(tests): """Creates thread workers to run the tests. @@ -358,6 +392,8 @@ def run_tests(tests): options_test += ' --interpreter --verify-soft-fail' elif compiler == 'jit': options_test += ' --jit' + elif compiler == 'speed-profile': + options_test += ' --random-profile' if relocate == 'relocate': options_test += ' --relocate' @@ -403,8 +439,10 @@ def run_tests(tests): options_test += ' --instruction-set-features ' + \ env.HOST_2ND_ARCH_PREFIX_DEX2OAT_HOST_INSTRUCTION_SET_FEATURES - options_test = (' --output-path %s/run-test-output/%s') % ( - env.ART_HOST_TEST_DIR, test_name) + options_test + # TODO(http://36039166): This is a temporary solution to + # fix build breakages. + options_test = (' --output-path %s') % ( + tempfile.mkdtemp(dir=env.ART_HOST_TEST_DIR)) + options_test run_test_sh = env.ANDROID_BUILD_TOP + '/art/test/run-test' command = run_test_sh + ' ' + options_test + ' ' + test @@ -442,15 +480,15 @@ def run_test(command, test, test_variant, test_name): test_skipped = True else: test_skipped = False - proc = subprocess.Popen(command.split(), stderr=subprocess.STDOUT, stdout=subprocess.PIPE) - script_output = proc.stdout.read().strip() + proc = subprocess.Popen(command.split(), stderr=subprocess.STDOUT, stdout=subprocess.PIPE, universal_newlines=True) + script_output = proc.communicate(timeout=timeout)[0] test_passed = not proc.wait() if not test_skipped: if test_passed: print_test_info(test_name, 'PASS') else: - failed_tests.append(test_name) + failed_tests.append((test_name, script_output)) if not env.ART_TEST_KEEP_GOING: stop_testrunner = True print_test_info(test_name, 'FAIL', ('%s\n%s') % ( @@ -460,9 +498,14 @@ def run_test(command, test, test_variant, test_name): skipped_tests.append(test_name) else: print_test_info(test_name, '') - except Exception, e: - failed_tests.append(test_name) - print_text(('%s\n%s\n') % (command, str(e))) + except subprocess.TimeoutExpired as e: + failed_tests.append((test_name, 'Timed out in %d seconds' % timeout)) + print_test_info(test_name, 'TIMEOUT', 'Timed out in %d seconds\n%s' % ( + timeout, command)) + except Exception as e: + failed_tests.append((test_name, str(e))) + print_test_info(test_name, 'FAIL', + ('%s\n%s\n\n') % (command, str(e))) finally: semaphore.release() @@ -499,11 +542,11 @@ def print_test_info(test_name, result, failed_test_info=""): test_count, total_test_count) - if result == "FAIL": + if result == 'FAIL' or result == 'TIMEOUT': info += ('%s %s %s\n%s\n') % ( progress_info, test_name, - COLOR_ERROR + 'FAIL' + COLOR_NORMAL, + COLOR_ERROR + result + COLOR_NORMAL, failed_test_info) else: result_text = '' @@ -524,20 +567,35 @@ def print_test_info(test_name, result, failed_test_info=""): allowed_test_length = console_width - total_output_length test_name_len = len(test_name) if allowed_test_length < test_name_len: - test_name = ('%s...%s') % ( - test_name[:(allowed_test_length - 3)/2], - test_name[-(allowed_test_length - 3)/2:]) + test_name = ('...%s') % ( + test_name[-(allowed_test_length - 3):]) info += ('%s %s %s') % ( progress_info, test_name, result_text) print_text(info) - except Exception, e: + except Exception as e: print_text(('%s\n%s\n') % (test_name, str(e))) failed_tests.append(test_name) finally: print_mutex.release() +def verify_knownfailure_entry(entry): + supported_field = { + 'tests' : (list, str), + 'description' : (list, str), + 'bug' : (str,), + 'variant' : (str,), + 'env_vars' : (dict,), + } + for field in entry: + field_type = type(entry[field]) + if field_type not in supported_field[field]: + raise ValueError('%s is not supported type for %s\n%s' % ( + str(field_type), + field, + str(entry))) + def get_disabled_test_info(): """Generate set of known failures. @@ -554,15 +612,18 @@ def get_disabled_test_info(): disabled_test_info = {} for failure in known_failures_info: - tests = failure.get('test') - if tests: + verify_knownfailure_entry(failure) + tests = failure.get('tests', []) + if isinstance(tests, str): tests = [tests] - else: - tests = failure.get('tests', []) variants = parse_variants(failure.get('variant')) env_vars = failure.get('env_vars') + if check_env_vars(env_vars): for test in tests: + if test not in RUN_TEST_SET: + raise ValueError('%s is not a valid run-test' % ( + test)) if test in disabled_test_info: disabled_test_info[test] = disabled_test_info[test].union(variants) else: @@ -626,6 +687,9 @@ def parse_variants(variants): variant = set() for and_variant in and_variants: and_variant = and_variant.strip() + if and_variant not in TOTAL_VARIANTS_SET: + raise ValueError('%s is not a valid variant' % ( + and_variant)) variant.add(and_variant) variant_list.add(frozenset(variant)) return variant_list @@ -642,16 +706,29 @@ def print_analysis(): console_width = int(os.popen('stty size', 'r').read().split()[1]) eraser_text = '\r' + ' ' * console_width + '\r' print_text(eraser_text) + + # Prints information about the total tests run. + # E.g., "2/38 (5%) tests passed". + passed_test_count = total_test_count - len(skipped_tests) - len(failed_tests) + passed_test_information = ('%d/%d (%d%%) %s passed.\n') % ( + passed_test_count, + total_test_count, + (passed_test_count*100)/total_test_count, + 'tests' if passed_test_count > 1 else 'test') + print_text(passed_test_information) + + # Prints the list of skipped tests, if any. if skipped_tests: - print_text(COLOR_SKIP + 'SKIPPED TESTS' + COLOR_NORMAL + '\n') + print_text(COLOR_SKIP + 'SKIPPED TESTS: ' + COLOR_NORMAL + '\n') for test in skipped_tests: print_text(test + '\n') print_text('\n') + # Prints the list of failed tests, if any. if failed_tests: - print_text(COLOR_ERROR + 'FAILED TESTS' + COLOR_NORMAL + '\n') - for test in failed_tests: - print_text(test + '\n') + print_text(COLOR_ERROR + 'FAILED: ' + COLOR_NORMAL + '\n') + for test_info in failed_tests: + print_text(('%s\n%s\n' % (test_info[0], test_info[1]))) def parse_test_name(test_name): @@ -705,6 +782,33 @@ def parse_test_name(test_name): raise ValueError(test_name + " is not a valid test") +def setup_env_for_build_target(build_target, parser, options): + """Setup environment for the build target + + The method setup environment for the master-art-host targets. + """ + os.environ.update(build_target['env']) + os.environ['SOONG_ALLOW_MISSING_DEPENDENCIES'] = 'true' + print_text('%s\n' % (str(os.environ))) + + target_options = vars(parser.parse_args(build_target['flags'])) + target_options['host'] = True + target_options['verbose'] = True + target_options['build'] = True + target_options['n_thread'] = options['n_thread'] + target_options['dry_run'] = options['dry_run'] + + return target_options + +def get_default_threads(target): + if target is 'target': + adb_command = 'adb shell cat /sys/devices/system/cpu/present' + cpu_info_proc = subprocess.Popen(adb_command.split(), stdout=subprocess.PIPE) + cpu_info = cpu_info_proc.stdout.read() + return int(cpu_info.split('-')[1]) + else: + return multiprocessing.cpu_count() + def parse_option(): global verbose global dry_run @@ -712,10 +816,12 @@ def parse_option(): global build global gdb global gdb_arg + global timeout parser = argparse.ArgumentParser(description="Runs all or a subset of the ART test suite.") parser.add_argument('-t', '--test', dest='test', help='name of the test') parser.add_argument('-j', type=int, dest='n_thread') + parser.add_argument('--timeout', default=timeout, type=int, dest='timeout') for variant in TOTAL_VARIANTS_SET: flag = '--' + variant flag_dest = variant.replace('-', '_') @@ -726,91 +832,106 @@ def parse_option(): parser.add_argument('--dry-run', action='store_true', dest='dry_run') parser.add_argument("--skip", action="append", dest="skips", default=[], help="Skip the given test in all circumstances.") - parser.add_argument('-b', '--build-dependencies', action='store_true', dest='build') + parser.add_argument('--no-build-dependencies', + action='store_false', dest='build', + help="Don't build dependencies under any circumstances. This is the " + + "behavior if ART_TEST_RUN_TEST_ALWAYS_BUILD is not set to 'true'.") + parser.add_argument('-b', '--build-dependencies', + action='store_true', dest='build', + help="Build dependencies under all circumstances. By default we will " + + "not build dependencies unless ART_TEST_RUN_TEST_BUILD=true.") + parser.add_argument('--build-target', dest='build_target', help='master-art-host targets') + parser.set_defaults(build = env.ART_TEST_RUN_TEST_BUILD) parser.add_argument('--gdb', action='store_true', dest='gdb') parser.add_argument('--gdb-arg', dest='gdb_arg') - options = parser.parse_args() + options = vars(parser.parse_args()) + if options['build_target']: + options = setup_env_for_build_target(target_config[options['build_target']], + parser, options) + test = '' - env.EXTRA_DISABLED_TESTS.update(set(options.skips)) - if options.test: - test = parse_test_name(options.test) - if options.pictest: + env.EXTRA_DISABLED_TESTS.update(set(options['skips'])) + if options['test']: + test = parse_test_name(options['test']) + if options['pictest']: PICTEST_TYPES.add('pictest') - if options.ndebug: + if options['ndebug']: RUN_TYPES.add('ndebug') - if options.interp_ac: + if options['interp_ac']: COMPILER_TYPES.add('interp-ac') - if options.picimage: + if options['picimage']: IMAGE_TYPES.add('picimage') - if options.n64: + if options['n64']: ADDRESS_SIZES.add('64') - if options.interpreter: + if options['interpreter']: COMPILER_TYPES.add('interpreter') - if options.jni: + if options['jni']: JNI_TYPES.add('jni') - if options.relocate_npatchoat: + if options['relocate_npatchoat']: RELOCATE_TYPES.add('relocate-npatchoat') - if options.no_prebuild: + if options['no_prebuild']: PREBUILD_TYPES.add('no-prebuild') - if options.npictest: + if options['npictest']: PICTEST_TYPES.add('npictest') - if options.no_dex2oat: + if options['no_dex2oat']: PREBUILD_TYPES.add('no-dex2oat') - if options.jit: + if options['jit']: COMPILER_TYPES.add('jit') - if options.relocate: + if options['relocate']: RELOCATE_TYPES.add('relocate') - if options.ndebuggable: + if options['ndebuggable']: DEBUGGABLE_TYPES.add('ndebuggable') - if options.no_image: + if options['no_image']: IMAGE_TYPES.add('no-image') - if options.optimizing: + if options['optimizing']: COMPILER_TYPES.add('optimizing') - if options.trace: + if options['speed_profile']: + COMPILER_TYPES.add('speed-profile') + if options['trace']: TRACE_TYPES.add('trace') - if options.gcstress: + if options['gcstress']: GC_TYPES.add('gcstress') - if options.no_relocate: + if options['no_relocate']: RELOCATE_TYPES.add('no-relocate') - if options.target: + if options['target']: TARGET_TYPES.add('target') - if options.forcecopy: + if options['forcecopy']: JNI_TYPES.add('forcecopy') - if options.n32: + if options['n32']: ADDRESS_SIZES.add('32') - if options.host: + if options['host']: TARGET_TYPES.add('host') - if options.gcverify: + if options['gcverify']: GC_TYPES.add('gcverify') - if options.debuggable: + if options['debuggable']: DEBUGGABLE_TYPES.add('debuggable') - if options.prebuild: + if options['prebuild']: PREBUILD_TYPES.add('prebuild') - if options.debug: + if options['debug']: RUN_TYPES.add('debug') - if options.checkjni: + if options['checkjni']: JNI_TYPES.add('checkjni') - if options.ntrace: + if options['ntrace']: TRACE_TYPES.add('ntrace') - if options.cms: + if options['cms']: GC_TYPES.add('cms') - if options.multipicimage: + if options['multipicimage']: IMAGE_TYPES.add('multipicimage') - if options.verbose: + if options['verbose']: verbose = True - if options.n_thread: - n_thread = max(1, options.n_thread) - if options.dry_run: + if options['n_thread']: + n_thread = max(1, options['n_thread']) + if options['dry_run']: dry_run = True verbose = True - if options.build: - build = True - if options.gdb: + build = options['build'] + if options['gdb']: n_thread = 1 gdb = True - if options.gdb_arg: - gdb_arg = options.gdb_arg + if options['gdb_arg']: + gdb_arg = options['gdb_arg'] + timeout = options['timeout'] return test @@ -825,9 +946,11 @@ def main(): if 'target' in TARGET_TYPES: build_targets += 'test-art-target-run-test-dependencies' build_command = 'make' - build_command += ' -j' + str(n_thread) + build_command += ' -j' build_command += ' -C ' + env.ANDROID_BUILD_TOP build_command += ' ' + build_targets + # Add 'dist' to avoid Jack issues b/36169180. + build_command += ' dist' if subprocess.call(build_command.split()): sys.exit(1) if user_requested_test: @@ -840,7 +963,7 @@ def main(): while threading.active_count() > 1: time.sleep(0.1) print_analysis() - except Exception, e: + except Exception as e: print_analysis() print_text(str(e)) sys.exit(1) |