diff options
author | 2016-09-15 17:15:50 +0000 | |
---|---|---|
committer | 2016-09-15 17:15:50 +0000 | |
commit | eb1aa7da786086bef8c22c76e1d68e8eae7300bf (patch) | |
tree | 942c30d473c8fa1d36f87adf20117695419f7545 | |
parent | fad6f4dbf914418e7b16fc78a77578edfa618b72 (diff) | |
parent | 0d0fd4a6bcf3b1223f1f5ed31d61aadfcfe79bc9 (diff) |
Merge "Integrate bisection search with javafuzz"
-rw-r--r-- | tools/bisection_search/README.md | 59 | ||||
-rwxr-xr-x | tools/bisection_search/bisection_search.py | 64 | ||||
-rwxr-xr-x | tools/bisection_search/common.py | 143 | ||||
-rwxr-xr-x | tools/javafuzz/run_java_fuzz_test.py | 481 |
4 files changed, 507 insertions, 240 deletions
diff --git a/tools/bisection_search/README.md b/tools/bisection_search/README.md index a7485c2bb5..64ccb206c4 100644 --- a/tools/bisection_search/README.md +++ b/tools/bisection_search/README.md @@ -15,29 +15,54 @@ incorrect output. Prints Mi and Pj. How to run Bisection Bug Search =============================== +There are two supported invocation modes: + +1. Regular invocation, dalvikvm command is constructed internally: + + ./bisection_search.py -cp classes.dex --expected-output out_int --class Test + +2. Raw-cmd invocation, dalvikvm command is accepted as an argument. The command + has to start with an executable. + + Extra dalvikvm arguments will be placed on second position in the command + by default. {ARGS} tag can be used to specify a custom position. + + ./bisection_search.py --raw-cmd='run.sh -cp classes.dex Test' --expected-retcode SUCCESS + ./bisection_search.py --raw-cmd='/bin/sh art {ARGS} -cp classes.dex Test' --expected-retcode SUCCESS + +Help: + bisection_search.py [-h] [-cp CLASSPATH] [--class CLASSNAME] [--lib LIB] - [--dalvikvm-option [OPT [OPT ...]]] [--arg [ARG [ARG ...]]] - [--image IMAGE] [--raw-cmd RAW_CMD] - [--64] [--device] [--expected-output EXPECTED_OUTPUT] - [--check-script CHECK_SCRIPT] [--verbose] + [--dalvikvm-option [OPT [OPT ...]]] [--arg [ARG [ARG ...]]] + [--image IMAGE] [--raw-cmd RAW_CMD] + [--64] [--device] [--device-serial DEVICE_SERIAL] + [--expected-output EXPECTED_OUTPUT] + [--expected-retcode {SUCCESS,TIMEOUT,ERROR}] + [--check-script CHECK_SCRIPT] [--logfile LOGFILE] [--cleanup] + [--timeout TIMEOUT] [--verbose] Tool for finding compiler bugs. Either --raw-cmd or both -cp and --class are required. optional arguments: - -h, --help show this help message and exit + -h, --help show this help message and exit dalvikvm command options: - -cp CLASSPATH, --classpath CLASSPATH classpath - --class CLASSNAME name of main class - --lib LIB lib to use, default: libart.so - --dalvikvm-option [OPT [OPT ...]] additional dalvikvm option - --arg [ARG [ARG ...]] argument passed to test - --image IMAGE path to image - --raw-cmd RAW_CMD bisect with this command, ignore other command options + -cp CLASSPATH, --classpath CLASSPATH classpath + --class CLASSNAME name of main class + --lib LIB lib to use, default: libart.so + --dalvikvm-option [OPT [OPT ...]] additional dalvikvm option + --arg [ARG [ARG ...]] argument passed to test + --image IMAGE path to image + --raw-cmd RAW_CMD bisect with this command, ignore other command options bisection options: - --64 x64 mode - --device run on device - --expected-output EXPECTED_OUTPUT file containing expected output - --check-script CHECK_SCRIPT script comparing output and expected output - --verbose enable verbose output + --64 x64 mode + --device run on device + --device-serial DEVICE_SERIAL device serial number, implies --device + --expected-output EXPECTED_OUTPUT file containing expected output + --expected-retcode {SUCCESS,TIMEOUT,ERROR} expected normalized return code + --check-script CHECK_SCRIPT script comparing output and expected output + --logfile LOGFILE custom logfile location + --cleanup clean up after bisecting + --timeout TIMEOUT if timeout seconds pass assume test failed + --verbose enable verbose output diff --git a/tools/bisection_search/bisection_search.py b/tools/bisection_search/bisection_search.py index 110ef82433..0d36aa4622 100755 --- a/tools/bisection_search/bisection_search.py +++ b/tools/bisection_search/bisection_search.py @@ -34,6 +34,7 @@ from common import DeviceTestEnv from common import FatalError from common import GetEnvVariableOrError from common import HostTestEnv +from common import RetCode # Passes that are never disabled during search process because disabling them @@ -51,6 +52,10 @@ MANDATORY_PASSES = ['dex_cache_array_fixups_arm', NON_PASSES = ['builder', 'prepare_for_register_allocation', 'liveness', 'register'] +# If present in raw cmd, this tag will be replaced with runtime arguments +# controlling the bisection search. Otherwise arguments will be placed on second +# position in the command. +RAW_CMD_RUNTIME_ARGS_TAG = '{ARGS}' class Dex2OatWrapperTestable(object): """Class representing a testable compilation. @@ -58,21 +63,29 @@ class Dex2OatWrapperTestable(object): Accepts filters on compiled methods and optimization passes. """ - def __init__(self, base_cmd, test_env, output_checker=None, verbose=False): + def __init__(self, base_cmd, test_env, expected_retcode=None, + output_checker=None, verbose=False): """Constructor. Args: base_cmd: list of strings, base command to run. test_env: ITestEnv. + expected_retcode: RetCode, expected normalized return code. output_checker: IOutputCheck, output checker. verbose: bool, enable verbose output. """ self._base_cmd = base_cmd self._test_env = test_env + self._expected_retcode = expected_retcode self._output_checker = output_checker self._compiled_methods_path = self._test_env.CreateFile('compiled_methods') self._passes_to_run_path = self._test_env.CreateFile('run_passes') self._verbose = verbose + if RAW_CMD_RUNTIME_ARGS_TAG in self._base_cmd: + self._arguments_position = self._base_cmd.index(RAW_CMD_RUNTIME_ARGS_TAG) + self._base_cmd.pop(self._arguments_position) + else: + self._arguments_position = 1 def Test(self, compiled_methods, passes_to_run=None): """Tests compilation with compiled_methods and run_passes switches active. @@ -95,8 +108,11 @@ class Dex2OatWrapperTestable(object): verbose_compiler=False) (output, ret_code) = self._test_env.RunCommand( cmd, {'ANDROID_LOG_TAGS': '*:e'}) - res = ((self._output_checker is None and ret_code == 0) - or self._output_checker.Check(output)) + res = True + if self._expected_retcode: + res = self._expected_retcode == ret_code + if self._output_checker: + res = res and self._output_checker.Check(output) if self._verbose: print('Test passed: {0}.'.format(res)) return res @@ -142,8 +158,8 @@ class Dex2OatWrapperTestable(object): def _PrepareCmd(self, compiled_methods=None, passes_to_run=None, verbose_compiler=False): """Prepare command to run.""" - cmd = [self._base_cmd[0]] - # insert additional arguments + cmd = self._base_cmd[0:self._arguments_position] + # insert additional arguments before the first argument if compiled_methods is not None: self._test_env.WriteLines(self._compiled_methods_path, compiled_methods) cmd += ['-Xcompiler-option', '--compiled-methods={0}'.format( @@ -155,7 +171,7 @@ class Dex2OatWrapperTestable(object): if verbose_compiler: cmd += ['-Xcompiler-option', '--runtime-arg', '-Xcompiler-option', '-verbose:compiler', '-Xcompiler-option', '-j1'] - cmd += self._base_cmd[1:] + cmd += self._base_cmd[self._arguments_position:] return cmd @@ -299,7 +315,7 @@ def PrepareParser(): command_opts.add_argument('-cp', '--classpath', type=str, help='classpath') command_opts.add_argument('--class', dest='classname', type=str, help='name of main class') - command_opts.add_argument('--lib', dest='lib', type=str, default='libart.so', + command_opts.add_argument('--lib', type=str, default='libart.so', help='lib to use, default: libart.so') command_opts.add_argument('--dalvikvm-option', dest='dalvikvm_opts', metavar='OPT', nargs='*', default=[], @@ -307,7 +323,7 @@ def PrepareParser(): command_opts.add_argument('--arg', dest='test_args', nargs='*', default=[], metavar='ARG', help='argument passed to test') command_opts.add_argument('--image', type=str, help='path to image') - command_opts.add_argument('--raw-cmd', dest='raw_cmd', type=str, + command_opts.add_argument('--raw-cmd', type=str, help='bisect with this command, ignore other ' 'command options') bisection_opts = parser.add_argument_group('bisection options') @@ -315,11 +331,22 @@ def PrepareParser(): default=False, help='x64 mode') bisection_opts.add_argument( '--device', action='store_true', default=False, help='run on device') + bisection_opts.add_argument( + '--device-serial', help='device serial number, implies --device') bisection_opts.add_argument('--expected-output', type=str, help='file containing expected output') bisection_opts.add_argument( - '--check-script', dest='check_script', type=str, + '--expected-retcode', type=str, help='expected normalized return code', + choices=[RetCode.SUCCESS.name, RetCode.TIMEOUT.name, RetCode.ERROR.name]) + bisection_opts.add_argument( + '--check-script', type=str, help='script comparing output and expected output') + bisection_opts.add_argument( + '--logfile', type=str, help='custom logfile location') + bisection_opts.add_argument('--cleanup', action='store_true', + default=False, help='clean up after bisecting') + bisection_opts.add_argument('--timeout', type=int, default=60, + help='if timeout seconds pass assume test failed') bisection_opts.add_argument('--verbose', action='store_true', default=False, help='enable verbose output') return parser @@ -351,15 +378,24 @@ def main(): args = parser.parse_args() if not args.raw_cmd and (not args.classpath or not args.classname): parser.error('Either --raw-cmd or both -cp and --class are required') + if args.device_serial: + args.device = True + if args.expected_retcode: + args.expected_retcode = RetCode[args.expected_retcode] + if not args.expected_retcode and not args.check_script: + args.expected_retcode = RetCode.SUCCESS # Prepare environment classpath = args.classpath if args.device: - test_env = DeviceTestEnv() + test_env = DeviceTestEnv( + 'bisection_search_', args.cleanup, args.logfile, args.timeout, + args.device_serial) if classpath: classpath = test_env.PushClasspath(classpath) else: - test_env = HostTestEnv(args.x64) + test_env = HostTestEnv( + 'bisection_search_', args.cleanup, args.logfile, args.timeout, args.x64) base_cmd = PrepareBaseCommand(args, classpath) output_checker = None if args.expected_output: @@ -372,11 +408,11 @@ def main(): # Perform the search try: - testable = Dex2OatWrapperTestable(base_cmd, test_env, output_checker, - args.verbose) + testable = Dex2OatWrapperTestable(base_cmd, test_env, args.expected_retcode, + output_checker, args.verbose) (method, opt_pass) = BugSearch(testable) except Exception as e: - print('Error. Refer to logfile: {0}'.format(test_env.logfile.name)) + print('Error occurred.\nLogfile: {0}'.format(test_env.logfile.name)) test_env.logfile.write('Exception: {0}\n'.format(e)) raise diff --git a/tools/bisection_search/common.py b/tools/bisection_search/common.py index d5029bb970..b69b60668b 100755 --- a/tools/bisection_search/common.py +++ b/tools/bisection_search/common.py @@ -18,7 +18,9 @@ import abc import os +import signal import shlex +import shutil from subprocess import check_call from subprocess import PIPE @@ -29,6 +31,9 @@ from subprocess import TimeoutExpired from tempfile import mkdtemp from tempfile import NamedTemporaryFile +from enum import Enum +from enum import unique + # Temporary directory path on device. DEVICE_TMP_PATH = '/data/local/tmp' @@ -36,6 +41,16 @@ DEVICE_TMP_PATH = '/data/local/tmp' DALVIK_CACHE_ARCHS = ['arm', 'arm64', 'x86', 'x86_64'] +@unique +class RetCode(Enum): + """Enum representing normalized return codes.""" + SUCCESS = 0 + TIMEOUT = 1 + ERROR = 2 + NOTCOMPILED = 3 + NOTRUN = 4 + + def GetEnvVariableOrError(variable_name): """Gets value of an environmental variable. @@ -70,6 +85,37 @@ def _DexArchCachePaths(android_data_path): for arch in DALVIK_CACHE_ARCHS) +def RunCommandForOutput(cmd, env, stdout, stderr, timeout=60): + """Runs command piping output to files, stderr or stdout. + + Args: + cmd: list of strings, command to run. + env: shell environment to run the command with. + stdout: file handle or one of Subprocess.PIPE, Subprocess.STDOUT, + Subprocess.DEVNULL, see Popen. + stderr: file handle or one of Subprocess.PIPE, Subprocess.STDOUT, + Subprocess.DEVNULL, see Popen. + timeout: int, timeout in seconds. + + Returns: + tuple (string, string, RetCode) stdout output, stderr output, normalized + return code. + """ + proc = Popen(cmd, stdout=stdout, stderr=stderr, env=env, + universal_newlines=True, start_new_session=True) + try: + (output, stderr_output) = proc.communicate(timeout=timeout) + if proc.returncode == 0: + retcode = RetCode.SUCCESS + else: + retcode = RetCode.ERROR + except TimeoutExpired: + os.killpg(os.getpgid(proc.pid), signal.SIGTERM) + (output, stderr_output) = proc.communicate() + retcode = RetCode.TIMEOUT + return (output, stderr_output, retcode) + + def _RunCommandForOutputAndLog(cmd, env, logfile, timeout=60): """Runs command and logs its output. Returns the output. @@ -77,28 +123,19 @@ def _RunCommandForOutputAndLog(cmd, env, logfile, timeout=60): cmd: list of strings, command to run. env: shell environment to run the command with. logfile: file handle to logfile. - timeout: int, timeout in seconds + timeout: int, timeout in seconds. Returns: - tuple (string, string, int) stdout output, stderr output, return code. + tuple (string, string, RetCode) stdout output, stderr output, normalized + return code. """ - proc = Popen(cmd, stderr=STDOUT, stdout=PIPE, env=env, - universal_newlines=True) - timeouted = False - try: - (output, _) = proc.communicate(timeout=timeout) - except TimeoutExpired: - timeouted = True - proc.kill() - (output, _) = proc.communicate() + (output, _, retcode) = RunCommandForOutput(cmd, env, PIPE, STDOUT, timeout) logfile.write('Command:\n{0}\n{1}\nReturn code: {2}\n'.format( - _CommandListToCommandString(cmd), output, - 'TIMEOUT' if timeouted else proc.returncode)) - ret_code = 1 if timeouted else proc.returncode - return (output, ret_code) + CommandListToCommandString(cmd), output, retcode)) + return (output, retcode) -def _CommandListToCommandString(cmd): +def CommandListToCommandString(cmd): """Converts shell command represented as list of strings to a single string. Each element of the list is wrapped in double quotes. @@ -109,7 +146,7 @@ def _CommandListToCommandString(cmd): Returns: string, shell command. """ - return ' '.join(['"{0}"'.format(segment) for segment in cmd]) + return ' '.join([shlex.quote(segment) for segment in cmd]) class FatalError(Exception): @@ -175,14 +212,24 @@ class HostTestEnv(ITestEnv): For methods documentation see base class. """ - def __init__(self, x64): + def __init__(self, directory_prefix, cleanup=True, logfile_path=None, + timeout=60, x64=False): """Constructor. Args: + directory_prefix: string, prefix for environment directory name. + cleanup: boolean, if True remove test directory in destructor. + logfile_path: string, can be used to specify custom logfile location. + timeout: int, seconds, time to wait for single test run to finish. x64: boolean, whether to setup in x64 mode. """ - self._env_path = mkdtemp(dir='/tmp/', prefix='bisection_search_') - self._logfile = open('{0}/log'.format(self._env_path), 'w+') + self._cleanup = cleanup + self._timeout = timeout + self._env_path = mkdtemp(dir='/tmp/', prefix=directory_prefix) + if logfile_path is None: + self._logfile = open('{0}/log'.format(self._env_path), 'w+') + else: + self._logfile = open(logfile_path, 'w+') os.mkdir('{0}/dalvik-cache'.format(self._env_path)) for arch_cache_path in _DexArchCachePaths(self._env_path): os.mkdir(arch_cache_path) @@ -199,6 +246,10 @@ class HostTestEnv(ITestEnv): # Using dlopen requires load bias on the host. self._shell_env['LD_USE_LOAD_BIAS'] = '1' + def __del__(self): + if self._cleanup: + shutil.rmtree(self._env_path) + def CreateFile(self, name=None): if name is None: f = NamedTemporaryFile(dir=self._env_path, delete=False) @@ -217,7 +268,7 @@ class HostTestEnv(ITestEnv): self._EmptyDexCache() env = self._shell_env.copy() env.update(env_updates) - return _RunCommandForOutputAndLog(cmd, env, self._logfile) + return _RunCommandForOutputAndLog(cmd, env, self._logfile, self._timeout) @property def logfile(self): @@ -239,16 +290,28 @@ class HostTestEnv(ITestEnv): class DeviceTestEnv(ITestEnv): """Device test environment. Concrete implementation of ITestEnv. - Makes use of HostTestEnv to maintain a test directory on host. Creates an - on device test directory which is kept in sync with the host one. - For methods documentation see base class. """ - def __init__(self): - """Constructor.""" - self._host_env_path = mkdtemp(dir='/tmp/', prefix='bisection_search_') - self._logfile = open('{0}/log'.format(self._host_env_path), 'w+') + def __init__(self, directory_prefix, cleanup=True, logfile_path=None, + timeout=60, specific_device=None): + """Constructor. + + Args: + directory_prefix: string, prefix for environment directory name. + cleanup: boolean, if True remove test directory in destructor. + logfile_path: string, can be used to specify custom logfile location. + timeout: int, seconds, time to wait for single test run to finish. + specific_device: string, serial number of device to use. + """ + self._cleanup = cleanup + self._timeout = timeout + self._specific_device = specific_device + self._host_env_path = mkdtemp(dir='/tmp/', prefix=directory_prefix) + if logfile_path is None: + self._logfile = open('{0}/log'.format(self._host_env_path), 'w+') + else: + self._logfile = open(logfile_path, 'w+') self._device_env_path = '{0}/{1}'.format( DEVICE_TMP_PATH, os.path.basename(self._host_env_path)) self._shell_env = os.environ.copy() @@ -257,6 +320,13 @@ class DeviceTestEnv(ITestEnv): for arch_cache_path in _DexArchCachePaths(self._device_env_path): self._AdbMkdir(arch_cache_path) + def __del__(self): + if self._cleanup: + shutil.rmtree(self._host_env_path) + check_call(shlex.split( + 'adb shell if [ -d "{0}" ]; then rm -rf "{0}"; fi' + .format(self._device_env_path))) + def CreateFile(self, name=None): with NamedTemporaryFile(mode='w') as temp_file: self._AdbPush(temp_file.name, self._device_env_path) @@ -279,11 +349,18 @@ class DeviceTestEnv(ITestEnv): env_updates['ANDROID_DATA'] = self._device_env_path env_updates_cmd = ' '.join(['{0}={1}'.format(var, val) for var, val in env_updates.items()]) - cmd = _CommandListToCommandString(cmd) - cmd = ('adb shell "logcat -c && {0} {1} ; logcat -d -s dex2oat:* dex2oatd:*' - '| grep -v "^---------" 1>&2"').format(env_updates_cmd, cmd) - return _RunCommandForOutputAndLog( - shlex.split(cmd), self._shell_env, self._logfile) + cmd = CommandListToCommandString(cmd) + adb = 'adb' + if self._specific_device: + adb += ' -s ' + self._specific_device + cmd = '{0} shell "logcat -c && {1} {2}"'.format( + adb, env_updates_cmd, cmd) + (output, retcode) = _RunCommandForOutputAndLog( + shlex.split(cmd), self._shell_env, self._logfile, self._timeout) + logcat_cmd = 'adb shell "logcat -d -s -b main dex2oat:* dex2oatd:*"' + (err_output, _) = _RunCommandForOutputAndLog( + shlex.split(logcat_cmd), self._shell_env, self._logfile) + return (output + err_output, retcode) @property def logfile(self): diff --git a/tools/javafuzz/run_java_fuzz_test.py b/tools/javafuzz/run_java_fuzz_test.py index 5f527b804b..085471fc39 100755 --- a/tools/javafuzz/run_java_fuzz_test.py +++ b/tools/javafuzz/run_java_fuzz_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3.4 # # Copyright (C) 2016 The Android Open Source Project # @@ -16,68 +16,71 @@ import abc import argparse +import filecmp + +from glob import glob + +import os +import shlex +import shutil import subprocess import sys -import os from tempfile import mkdtemp -from threading import Timer -# Normalized return codes. -EXIT_SUCCESS = 0 -EXIT_TIMEOUT = 1 -EXIT_NOTCOMPILED = 2 -EXIT_NOTRUN = 3 + +sys.path.append(os.path.dirname(os.path.dirname(__file__))) + +from bisection_search.common import RetCode +from bisection_search.common import CommandListToCommandString +from bisection_search.common import FatalError +from bisection_search.common import GetEnvVariableOrError +from bisection_search.common import RunCommandForOutput +from bisection_search.common import DeviceTestEnv + +# Return codes supported by bisection bug search. +BISECTABLE_RET_CODES = (RetCode.SUCCESS, RetCode.ERROR, RetCode.TIMEOUT) # # Utility methods. # -def RunCommand(cmd, args, out, err, timeout = 5): + +def RunCommand(cmd, out, err, timeout=5): """Executes a command, and returns its return code. Args: - cmd: string, a command to execute - args: string, arguments to pass to command (or None) + cmd: list of strings, a command to execute out: string, file name to open for stdout (or None) err: string, file name to open for stderr (or None) timeout: int, time out in seconds Returns: - return code of running command (forced EXIT_TIMEOUT on timeout) + RetCode, return code of running command (forced RetCode.TIMEOUT + on timeout) """ - cmd = 'exec ' + cmd # preserve pid - if args != None: - cmd = cmd + ' ' + args - outf = None - if out != None: + devnull = subprocess.DEVNULL + outf = devnull + if out is not None: outf = open(out, mode='w') - errf = None - if err != None: + errf = devnull + if err is not None: errf = open(err, mode='w') - proc = subprocess.Popen(cmd, stdout=outf, stderr=errf, shell=True) - timer = Timer(timeout, proc.kill) # enforces timeout - timer.start() - proc.communicate() - if timer.is_alive(): - timer.cancel() - returncode = proc.returncode - else: - returncode = EXIT_TIMEOUT - if outf != None: + (_, _, retcode) = RunCommandForOutput(cmd, None, outf, errf, timeout) + if outf != devnull: outf.close() - if errf != None: + if errf != devnull: errf.close() - return returncode + return retcode + def GetJackClassPath(): """Returns Jack's classpath.""" - top = os.environ.get('ANDROID_BUILD_TOP') - if top == None: - raise FatalError('Cannot find AOSP build top') + top = GetEnvVariableOrError('ANDROID_BUILD_TOP') libdir = top + '/out/host/common/obj/JAVA_LIBRARIES' return libdir + '/core-libart-hostdex_intermediates/classes.jack:' \ + libdir + '/core-oj-hostdex_intermediates/classes.jack' + def GetExecutionModeRunner(device, mode): """Returns a runner for the given execution mode. @@ -92,49 +95,44 @@ def GetExecutionModeRunner(device, mode): if mode == 'ri': return TestRunnerRIOnHost() if mode == 'hint': - return TestRunnerArtOnHost(True) + return TestRunnerArtIntOnHost() if mode == 'hopt': - return TestRunnerArtOnHost(False) + return TestRunnerArtOptOnHost() if mode == 'tint': - return TestRunnerArtOnTarget(device, True) + return TestRunnerArtIntOnTarget(device) if mode == 'topt': - return TestRunnerArtOnTarget(device, False) + return TestRunnerArtOptOnTarget(device) raise FatalError('Unknown execution mode') -def GetReturnCode(retc): - """Returns a string representation of the given normalized return code. - Args: - retc: int, normalized return code - Returns: - string representation of normalized return code - Raises: - FatalError: error for unknown normalized return code - """ - if retc == EXIT_SUCCESS: - return 'SUCCESS' - if retc == EXIT_TIMEOUT: - return 'TIMED-OUT' - if retc == EXIT_NOTCOMPILED: - return 'NOT-COMPILED' - if retc == EXIT_NOTRUN: - return 'NOT-RUN' - raise FatalError('Unknown normalized return code') - # # Execution mode classes. # + class TestRunner(object): """Abstraction for running a test in a particular execution mode.""" __meta_class__ = abc.ABCMeta - def GetDescription(self): + @abc.abstractproperty + def description(self): """Returns a description string of the execution mode.""" - return self._description - def GetId(self): + @abc.abstractproperty + def id(self): """Returns a short string that uniquely identifies the execution mode.""" - return self._id + + @property + def output_file(self): + return self.id + '_out.txt' + + @abc.abstractmethod + def GetBisectionSearchArgs(self): + """Get arguments to pass to bisection search tool. + + Returns: + list of strings - arguments for bisection search tool, or None if + runner is not bisectable + """ @abc.abstractmethod def CompileAndRunTest(self): @@ -142,8 +140,7 @@ class TestRunner(object): Ensures that the current Test.java in the temporary directory is compiled and executed under the current execution mode. On success, transfers the - generated output to the file GetId()_out.txt in the temporary directory. - Cleans up after itself. + generated output to the file self.output_file in the temporary directory. Most nonzero return codes are assumed non-divergent, since systems may exit in different ways. This is enforced by normalizing return codes. @@ -151,112 +148,196 @@ class TestRunner(object): Returns: normalized return code """ - pass + class TestRunnerRIOnHost(TestRunner): """Concrete test runner of the reference implementation on host.""" - def __init__(self): - """Constructor for the RI tester.""" - self._description = 'RI on host' - self._id = 'RI' + @property + def description(self): + return 'RI on host' + + @property + def id(self): + return 'RI' def CompileAndRunTest(self): - if RunCommand('javac', 'Test.java', - out=None, err=None, timeout=30) == EXIT_SUCCESS: - retc = RunCommand('java', 'Test', 'RI_run_out.txt', err=None) - if retc != EXIT_SUCCESS and retc != EXIT_TIMEOUT: - retc = EXIT_NOTRUN + if RunCommand(['javac', 'Test.java'], + out=None, err=None, timeout=30) == RetCode.SUCCESS: + retc = RunCommand(['java', 'Test'], self.output_file, err=None) else: - retc = EXIT_NOTCOMPILED - # Cleanup and return. - RunCommand('rm', '-f Test.class', out=None, err=None) + retc = RetCode.NOTCOMPILED return retc + def GetBisectionSearchArgs(self): + return None + + class TestRunnerArtOnHost(TestRunner): - """Concrete test runner of Art on host (interpreter or optimizing).""" + """Abstract test runner of Art on host.""" - def __init__(self, interpreter): + def __init__(self, extra_args=None): """Constructor for the Art on host tester. Args: - interpreter: boolean, selects between interpreter or optimizing + extra_args: list of strings, extra arguments for dalvikvm """ - self._art_args = '-cp classes.dex Test' - if interpreter: - self._description = 'Art interpreter on host' - self._id = 'HInt' - self._art_args = '-Xint ' + self._art_args - else: - self._description = 'Art optimizing on host' - self._id = 'HOpt' - self._jack_args = '-cp ' + GetJackClassPath() + ' --output-dex . Test.java' + self._art_cmd = ['/bin/bash', 'art', '-cp', 'classes.dex'] + if extra_args is not None: + self._art_cmd += extra_args + self._art_cmd.append('Test') + self._jack_args = ['-cp', GetJackClassPath(), '--output-dex', '.', + 'Test.java'] def CompileAndRunTest(self): - if RunCommand('jack', self._jack_args, - out=None, err='jackerr.txt', timeout=30) == EXIT_SUCCESS: - out = self.GetId() + '_run_out.txt' - retc = RunCommand('art', self._art_args, out, 'arterr.txt') - if retc != EXIT_SUCCESS and retc != EXIT_TIMEOUT: - retc = EXIT_NOTRUN + if RunCommand(['jack'] + self._jack_args, out=None, err='jackerr.txt', + timeout=30) == RetCode.SUCCESS: + retc = RunCommand(self._art_cmd, self.output_file, 'arterr.txt') else: - retc = EXIT_NOTCOMPILED - # Cleanup and return. - RunCommand('rm', '-rf classes.dex jackerr.txt arterr.txt android-data*', - out=None, err=None) + retc = RetCode.NOTCOMPILED return retc -# TODO: very rough first version without proper cache, -# reuse staszkiewicz' module for properly setting up dalvikvm on target. + +class TestRunnerArtIntOnHost(TestRunnerArtOnHost): + """Concrete test runner of interpreter mode Art on host.""" + + def __init__(self): + """Constructor.""" + super().__init__(['-Xint']) + + @property + def description(self): + return 'Art interpreter on host' + + @property + def id(self): + return 'HInt' + + def GetBisectionSearchArgs(self): + return None + + +class TestRunnerArtOptOnHost(TestRunnerArtOnHost): + """Concrete test runner of optimizing compiler mode Art on host.""" + + def __init__(self): + """Constructor.""" + super().__init__(None) + + @property + def description(self): + return 'Art optimizing on host' + + @property + def id(self): + return 'HOpt' + + def GetBisectionSearchArgs(self): + cmd_str = CommandListToCommandString( + self._art_cmd[0:2] + ['{ARGS}'] + self._art_cmd[2:]) + return ['--raw-cmd={0}'.format(cmd_str), '--timeout', str(30)] + + class TestRunnerArtOnTarget(TestRunner): - """Concrete test runner of Art on target (interpreter or optimizing).""" + """Abstract test runner of Art on target.""" - def __init__(self, device, interpreter): + def __init__(self, device, extra_args=None): """Constructor for the Art on target tester. Args: device: string, target device serial number (or None) - interpreter: boolean, selects between interpreter or optimizing + extra_args: list of strings, extra arguments for dalvikvm """ - self._dalvik_args = 'shell dalvikvm -cp /data/local/tmp/classes.dex Test' - if interpreter: - self._description = 'Art interpreter on target' - self._id = 'TInt' - self._dalvik_args = '-Xint ' + self._dalvik_args - else: - self._description = 'Art optimizing on target' - self._id = 'TOpt' - self._adb = 'adb' - if device != None: - self._adb = self._adb + ' -s ' + device - self._jack_args = '-cp ' + GetJackClassPath() + ' --output-dex . Test.java' + self._test_env = DeviceTestEnv('javafuzz_', specific_device=device) + self._dalvik_cmd = ['dalvikvm'] + if extra_args is not None: + self._dalvik_cmd += extra_args + self._device = device + self._jack_args = ['-cp', GetJackClassPath(), '--output-dex', '.', + 'Test.java'] + self._device_classpath = None def CompileAndRunTest(self): - if RunCommand('jack', self._jack_args, - out=None, err='jackerr.txt', timeout=30) == EXIT_SUCCESS: - if RunCommand(self._adb, 'push classes.dex /data/local/tmp/', - 'adb.txt', err=None) != EXIT_SUCCESS: - raise FatalError('Cannot push to target device') - out = self.GetId() + '_run_out.txt' - retc = RunCommand(self._adb, self._dalvik_args, out, err=None) - if retc != EXIT_SUCCESS and retc != EXIT_TIMEOUT: - retc = EXIT_NOTRUN + if RunCommand(['jack'] + self._jack_args, out=None, err='jackerr.txt', + timeout=30) == RetCode.SUCCESS: + self._device_classpath = self._test_env.PushClasspath('classes.dex') + cmd = self._dalvik_cmd + ['-cp', self._device_classpath, 'Test'] + (output, retc) = self._test_env.RunCommand( + cmd, {'ANDROID_LOG_TAGS': '*:s'}) + with open(self.output_file, 'w') as run_out: + run_out.write(output) else: - retc = EXIT_NOTCOMPILED - # Cleanup and return. - RunCommand('rm', '-f classes.dex jackerr.txt adb.txt', - out=None, err=None) - RunCommand(self._adb, 'shell rm -f /data/local/tmp/classes.dex', - out=None, err=None) + retc = RetCode.NOTCOMPILED return retc + def GetBisectionSearchArgs(self): + cmd_str = CommandListToCommandString( + self._dalvik_cmd + ['-cp',self._device_classpath, 'Test']) + cmd = ['--raw-cmd={0}'.format(cmd_str), '--timeout', str(30)] + if self._device: + cmd += ['--device-serial', self._device] + else: + cmd.append('--device') + return cmd + + +class TestRunnerArtIntOnTarget(TestRunnerArtOnTarget): + """Concrete test runner of interpreter mode Art on target.""" + + def __init__(self, device): + """Constructor. + + Args: + device: string, target device serial number (or None) + """ + super().__init__(device, ['-Xint']) + + @property + def description(self): + return 'Art interpreter on target' + + @property + def id(self): + return 'TInt' + + def GetBisectionSearchArgs(self): + return None + + +class TestRunnerArtOptOnTarget(TestRunnerArtOnTarget): + """Concrete test runner of optimizing compiler mode Art on target.""" + + def __init__(self, device): + """Constructor. + + Args: + device: string, target device serial number (or None) + """ + super().__init__(device, None) + + @property + def description(self): + return 'Art optimizing on target' + + @property + def id(self): + return 'TOpt' + + def GetBisectionSearchArgs(self): + cmd_str = CommandListToCommandString( + self._dalvik_cmd + ['-cp', self._device_classpath, 'Test']) + cmd = ['--raw-cmd={0}'.format(cmd_str), '--timeout', str(30)] + if self._device: + cmd += ['--device-serial', self._device] + else: + cmd.append('--device') + return cmd + + # # Tester classes. # -class FatalError(Exception): - """Fatal error in the tester.""" - pass class JavaFuzzTester(object): """Tester that runs JavaFuzz many times and report divergences.""" @@ -265,10 +346,10 @@ class JavaFuzzTester(object): """Constructor for the tester. Args: - num_tests: int, number of tests to run - device: string, target device serial number (or None) - mode1: string, execution mode for first runner - mode2: string, execution mode for second runner + num_tests: int, number of tests to run + device: string, target device serial number (or None) + mode1: string, execution mode for first runner + mode2: string, execution mode for second runner """ self._num_tests = num_tests self._device = device @@ -291,8 +372,9 @@ class JavaFuzzTester(object): FatalError: error when temp directory cannot be constructed """ self._save_dir = os.getcwd() - self._tmp_dir = mkdtemp(dir="/tmp/") - if self._tmp_dir == None: + self._results_dir = mkdtemp(dir='/tmp/') + self._tmp_dir = mkdtemp(dir=self._results_dir) + if self._tmp_dir is None or self._results_dir is None: raise FatalError('Cannot obtain temp directory') os.chdir(self._tmp_dir) return self @@ -300,37 +382,38 @@ class JavaFuzzTester(object): def __exit__(self, etype, evalue, etraceback): """On exit, re-enters previously saved current directory and cleans up.""" os.chdir(self._save_dir) + shutil.rmtree(self._tmp_dir) if self._num_divergences == 0: - RunCommand('rm', '-rf ' + self._tmp_dir, out=None, err=None) + shutil.rmtree(self._results_dir) def Run(self): """Runs JavaFuzz many times and report divergences.""" - print - print '**\n**** JavaFuzz Testing\n**' - print - print '#Tests :', self._num_tests - print 'Device :', self._device - print 'Directory :', self._tmp_dir - print 'Exec-mode1:', self._runner1.GetDescription() - print 'Exec-mode2:', self._runner2.GetDescription() + print() + print('**\n**** JavaFuzz Testing\n**') + print() + print('#Tests :', self._num_tests) + print('Device :', self._device) + print('Directory :', self._results_dir) + print('Exec-mode1:', self._runner1.description) + print('Exec-mode2:', self._runner2.description) print self.ShowStats() for self._test in range(1, self._num_tests + 1): self.RunJavaFuzzTest() self.ShowStats() if self._num_divergences == 0: - print '\n\nsuccess (no divergences)\n' + print('\n\nsuccess (no divergences)\n') else: - print '\n\nfailure (divergences)\n' + print('\n\nfailure (divergences)\n') def ShowStats(self): """Shows current statistics (on same line) while tester is running.""" - print '\rTests:', self._test, \ - 'Success:', self._num_success, \ - 'Not-compiled:', self._num_not_compiled, \ - 'Not-run:', self._num_not_run, \ - 'Timed-out:', self._num_timed_out, \ - 'Divergences:', self._num_divergences, + print('\rTests:', self._test, \ + 'Success:', self._num_success, \ + 'Not-compiled:', self._num_not_compiled, \ + 'Not-run:', self._num_not_run, \ + 'Timed-out:', self._num_timed_out, \ + 'Divergences:', self._num_divergences, end='') sys.stdout.flush() def RunJavaFuzzTest(self): @@ -347,8 +430,7 @@ class JavaFuzzTester(object): Raises: FatalError: error when javafuzz fails """ - if RunCommand('javafuzz', args=None, - out='Test.java', err=None) != EXIT_SUCCESS: + if RunCommand(['javafuzz'], out='Test.java', err=None) != RetCode.SUCCESS: raise FatalError('Unexpected error while running JavaFuzz') def CheckForDivergence(self, retc1, retc2): @@ -360,38 +442,85 @@ class JavaFuzzTester(object): """ if retc1 == retc2: # Non-divergent in return code. - if retc1 == EXIT_SUCCESS: + if retc1 == RetCode.SUCCESS: # Both compilations and runs were successful, inspect generated output. - args = self._runner1.GetId() + '_run_out.txt ' \ - + self._runner2.GetId() + '_run_out.txt' - if RunCommand('diff', args, out=None, err=None) != EXIT_SUCCESS: - self.ReportDivergence('divergence in output') + runner1_out = self._runner1.output_file + runner2_out = self._runner2.output_file + if not filecmp.cmp(runner1_out, runner2_out, shallow=False): + self.ReportDivergence(retc1, retc2, is_output_divergence=True) else: self._num_success += 1 - elif retc1 == EXIT_TIMEOUT: + elif retc1 == RetCode.TIMEOUT: self._num_timed_out += 1 - elif retc1 == EXIT_NOTCOMPILED: + elif retc1 == RetCode.NOTCOMPILED: self._num_not_compiled += 1 else: self._num_not_run += 1 else: # Divergent in return code. - self.ReportDivergence('divergence in return code: ' + - GetReturnCode(retc1) + ' vs. ' + - GetReturnCode(retc2)) + self.ReportDivergence(retc1, retc2, is_output_divergence=False) + + def GetCurrentDivergenceDir(self): + return self._results_dir + '/divergence' + str(self._num_divergences) - def ReportDivergence(self, reason): + def ReportDivergence(self, retc1, retc2, is_output_divergence): """Reports and saves a divergence.""" self._num_divergences += 1 - print '\n', self._test, reason + print('\n' + str(self._num_divergences), end='') + if is_output_divergence: + print(' divergence in output') + else: + print(' divergence in return code: ' + retc1.name + ' vs. ' + + retc2.name) # Save. - ddir = 'divergence' + str(self._test) - RunCommand('mkdir', ddir, out=None, err=None) - RunCommand('mv', 'Test.java *.txt ' + ddir, out=None, err=None) + ddir = self.GetCurrentDivergenceDir() + os.mkdir(ddir) + for f in glob('*.txt') + ['Test.java']: + shutil.copy(f, ddir) + # Maybe run bisection bug search. + if retc1 in BISECTABLE_RET_CODES and retc2 in BISECTABLE_RET_CODES: + self.MaybeBisectDivergence(retc1, retc2, is_output_divergence) + + def RunBisectionSearch(self, args, expected_retcode, expected_output, + runner_id): + ddir = self.GetCurrentDivergenceDir() + outfile_path = ddir + '/' + runner_id + '_bisection_out.txt' + logfile_path = ddir + '/' + runner_id + '_bisection_log.txt' + errfile_path = ddir + '/' + runner_id + '_bisection_err.txt' + args = list(args) + ['--logfile', logfile_path, '--cleanup'] + args += ['--expected-retcode', expected_retcode.name] + if expected_output: + args += ['--expected-output', expected_output] + bisection_search_path = os.path.join( + GetEnvVariableOrError('ANDROID_BUILD_TOP'), + 'art/tools/bisection_search/bisection_search.py') + if RunCommand([bisection_search_path] + args, out=outfile_path, + err=errfile_path, timeout=300) == RetCode.TIMEOUT: + print('Bisection search TIMEOUT') + + def MaybeBisectDivergence(self, retc1, retc2, is_output_divergence): + bisection_args1 = self._runner1.GetBisectionSearchArgs() + bisection_args2 = self._runner2.GetBisectionSearchArgs() + if is_output_divergence: + maybe_output1 = self._runner1.output_file + maybe_output2 = self._runner2.output_file + else: + maybe_output1 = maybe_output2 = None + if bisection_args1 is not None: + self.RunBisectionSearch(bisection_args1, retc2, maybe_output2, + self._runner1.id) + if bisection_args2 is not None: + self.RunBisectionSearch(bisection_args2, retc1, maybe_output1, + self._runner2.id) def CleanupTest(self): """Cleans up after a single test run.""" - RunCommand('rm', '-f Test.java *.txt', out=None, err=None) + for file_name in os.listdir(self._tmp_dir): + file_path = os.path.join(self._tmp_dir, file_name) + if os.path.isfile(file_path): + os.unlink(file_path) + elif os.path.isdir(file_path): + shutil.rmtree(file_path) def main(): @@ -406,11 +535,11 @@ def main(): help='execution mode 2 (default: hopt)') args = parser.parse_args() if args.mode1 == args.mode2: - raise FatalError("Identical execution modes given") + raise FatalError('Identical execution modes given') # Run the JavaFuzz tester. with JavaFuzzTester(args.num_tests, args.device, args.mode1, args.mode2) as fuzzer: fuzzer.Run() -if __name__ == "__main__": +if __name__ == '__main__': main() |