| #!/usr/bin/env python3 |
| # |
| # Copyright (C) 2016 The Android Open Source Project |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| import abc |
| import argparse |
| import filecmp |
| import os |
| import shlex |
| import shutil |
| import subprocess |
| import sys |
| |
| from glob import glob |
| from subprocess import DEVNULL |
| from tempfile import mkdtemp |
| |
| sys.path.append(os.path.dirname(os.path.dirname( |
| os.path.realpath(__file__)))) |
| |
| from common.common import RetCode |
| from common.common import CommandListToCommandString |
| from common.common import FatalError |
| from common.common import GetEnvVariableOrError |
| from common.common import RunCommand |
| from common.common import RunCommandForOutput |
| from common.common import DeviceTestEnv |
| |
| # Return codes supported by bisection bug search. |
| BISECTABLE_RET_CODES = (RetCode.SUCCESS, RetCode.ERROR, RetCode.TIMEOUT) |
| |
| |
| def GetExecutionModeRunner(dexer, debug_info, device, mode): |
| """Returns a runner for the given execution mode. |
| |
| Args: |
| dexer: string, defines dexer |
| debug_info: boolean, if True include debugging info |
| device: string, target device serial number (or None) |
| mode: string, execution mode |
| Returns: |
| TestRunner with given execution mode |
| Raises: |
| FatalError: error for unknown execution mode |
| """ |
| if mode == 'ri': |
| return TestRunnerRIOnHost(debug_info) |
| if mode == 'hint': |
| return TestRunnerArtIntOnHost(dexer, debug_info) |
| if mode == 'hopt': |
| return TestRunnerArtOptOnHost(dexer, debug_info) |
| if mode == 'tint': |
| return TestRunnerArtIntOnTarget(dexer, debug_info, device) |
| if mode == 'topt': |
| return TestRunnerArtOptOnTarget(dexer, debug_info, device) |
| raise FatalError('Unknown execution mode') |
| |
| |
| # |
| # Execution mode classes. |
| # |
| |
| |
| class TestRunner(object): |
| """Abstraction for running a test in a particular execution mode.""" |
| __meta_class__ = abc.ABCMeta |
| |
| @abc.abstractproperty |
| def description(self): |
| """Returns a description string of the execution mode.""" |
| |
| @abc.abstractproperty |
| def id(self): |
| """Returns a short string that uniquely identifies the execution mode.""" |
| |
| @property |
| def output_file(self): |
| return self.id + '_out.txt' |
| |
| @abc.abstractmethod |
| def GetBisectionSearchArgs(self): |
| """Get arguments to pass to bisection search tool. |
| |
| Returns: |
| list of strings - arguments for bisection search tool, or None if |
| runner is not bisectable |
| """ |
| |
| @abc.abstractmethod |
| def CompileAndRunTest(self): |
| """Compile and run the generated test. |
| |
| Ensures that the current Test.java in the temporary directory is compiled |
| and executed under the current execution mode. On success, transfers the |
| generated output to the file self.output_file in the temporary directory. |
| |
| Most nonzero return codes are assumed non-divergent, since systems may |
| exit in different ways. This is enforced by normalizing return codes. |
| |
| Returns: |
| normalized return code |
| """ |
| |
| |
| class TestRunnerWithHostCompilation(TestRunner): |
| """Abstract test runner that supports compilation on host.""" |
| |
| def __init__(self, dexer, debug_info): |
| """Constructor for the runner with host compilation. |
| |
| Args: |
| dexer: string, defines dexer |
| debug_info: boolean, if True include debugging info |
| """ |
| self._dexer = dexer |
| self._debug_info = debug_info |
| |
| def CompileOnHost(self): |
| if self._dexer == 'dx' or self._dexer == 'd8': |
| dbg = '-g' if self._debug_info else '-g:none' |
| if RunCommand(['javac', '--release=8', dbg, 'Test.java'], |
| out=None, err=None, timeout=30) == RetCode.SUCCESS: |
| dx = 'dx' if self._dexer == 'dx' else 'd8-compat-dx' |
| retc = RunCommand([dx, '--dex', '--output=classes.dex'] + glob('*.class'), |
| out=None, err='dxerr.txt', timeout=30) |
| else: |
| retc = RetCode.NOTCOMPILED |
| else: |
| raise FatalError('Unknown dexer: ' + self._dexer) |
| return retc |
| |
| |
| class TestRunnerRIOnHost(TestRunner): |
| """Concrete test runner of the reference implementation on host.""" |
| |
| def __init__(self, debug_info): |
| """Constructor for the runner with host compilation. |
| |
| Args: |
| debug_info: boolean, if True include debugging info |
| """ |
| self._debug_info = debug_info |
| |
| @property |
| def description(self): |
| return 'RI on host' |
| |
| @property |
| def id(self): |
| return 'RI' |
| |
| def CompileAndRunTest(self): |
| dbg = '-g' if self._debug_info else '-g:none' |
| if RunCommand(['javac', '--release=8', dbg, 'Test.java'], |
| out=None, err=None, timeout=30) == RetCode.SUCCESS: |
| retc = RunCommand(['java', 'Test'], self.output_file, err=None) |
| else: |
| retc = RetCode.NOTCOMPILED |
| return retc |
| |
| def GetBisectionSearchArgs(self): |
| return None |
| |
| |
| class TestRunnerArtOnHost(TestRunnerWithHostCompilation): |
| """Abstract test runner of Art on host.""" |
| |
| def __init__(self, dexer, debug_info, extra_args=None): |
| """Constructor for the Art on host tester. |
| |
| Args: |
| dexer: string, defines dexer |
| debug_info: boolean, if True include debugging info |
| extra_args: list of strings, extra arguments for dalvikvm |
| """ |
| super().__init__(dexer, debug_info) |
| self._art_cmd = ['/bin/bash', 'art', '-cp', 'classes.dex'] |
| if extra_args is not None: |
| self._art_cmd += extra_args |
| self._art_cmd.append('Test') |
| |
| def CompileAndRunTest(self): |
| if self.CompileOnHost() == RetCode.SUCCESS: |
| retc = RunCommand(self._art_cmd, self.output_file, 'arterr.txt') |
| else: |
| retc = RetCode.NOTCOMPILED |
| return retc |
| |
| |
| class TestRunnerArtIntOnHost(TestRunnerArtOnHost): |
| """Concrete test runner of interpreter mode Art on host.""" |
| |
| def __init__(self, dexer, debug_info): |
| """Constructor for the Art on host tester (interpreter). |
| |
| Args: |
| dexer: string, defines dexer |
| debug_info: boolean, if True include debugging info |
| """ |
| super().__init__(dexer, debug_info, ['-Xint']) |
| |
| @property |
| def description(self): |
| return 'Art interpreter on host' |
| |
| @property |
| def id(self): |
| return 'HInt' |
| |
| def GetBisectionSearchArgs(self): |
| return None |
| |
| |
| class TestRunnerArtOptOnHost(TestRunnerArtOnHost): |
| """Concrete test runner of optimizing compiler mode Art on host.""" |
| |
| def __init__(self, dexer, debug_info): |
| """Constructor for the Art on host tester (optimizing). |
| |
| Args: |
| dexer: string, defines dexer |
| debug_info: boolean, if True include debugging info |
| """ |
| super().__init__(dexer, debug_info, None) |
| |
| @property |
| def description(self): |
| return 'Art optimizing on host' |
| |
| @property |
| def id(self): |
| return 'HOpt' |
| |
| def GetBisectionSearchArgs(self): |
| cmd_str = CommandListToCommandString( |
| self._art_cmd[0:2] + ['{ARGS}'] + self._art_cmd[2:]) |
| return ['--raw-cmd={0}'.format(cmd_str), '--timeout', str(30)] |
| |
| |
| class TestRunnerArtOnTarget(TestRunnerWithHostCompilation): |
| """Abstract test runner of Art on target.""" |
| |
| def __init__(self, dexer, debug_info, device, extra_args=None): |
| """Constructor for the Art on target tester. |
| |
| Args: |
| dexer: string, defines dexer |
| debug_info: boolean, if True include debugging info |
| device: string, target device serial number (or None) |
| extra_args: list of strings, extra arguments for dalvikvm |
| """ |
| super().__init__(dexer, debug_info) |
| self._test_env = DeviceTestEnv('jfuzz_', specific_device=device) |
| self._dalvik_cmd = ['dalvikvm'] |
| if extra_args is not None: |
| self._dalvik_cmd += extra_args |
| self._device = device |
| self._device_classpath = None |
| |
| def CompileAndRunTest(self): |
| if self.CompileOnHost() == RetCode.SUCCESS: |
| self._device_classpath = self._test_env.PushClasspath('classes.dex') |
| cmd = self._dalvik_cmd + ['-cp', self._device_classpath, 'Test'] |
| (output, retc) = self._test_env.RunCommand( |
| cmd, {'ANDROID_LOG_TAGS': '*:s'}) |
| with open(self.output_file, 'w') as run_out: |
| run_out.write(output) |
| else: |
| retc = RetCode.NOTCOMPILED |
| return retc |
| |
| def GetBisectionSearchArgs(self): |
| cmd_str = CommandListToCommandString( |
| self._dalvik_cmd + ['-cp',self._device_classpath, 'Test']) |
| cmd = ['--raw-cmd={0}'.format(cmd_str), '--timeout', str(30)] |
| if self._device: |
| cmd += ['--device-serial', self._device] |
| else: |
| cmd.append('--device') |
| return cmd |
| |
| |
| class TestRunnerArtIntOnTarget(TestRunnerArtOnTarget): |
| """Concrete test runner of interpreter mode Art on target.""" |
| |
| def __init__(self, dexer, debug_info, device): |
| """Constructor for the Art on target tester (interpreter). |
| |
| Args: |
| dexer: string, defines dexer |
| debug_info: boolean, if True include debugging info |
| device: string, target device serial number (or None) |
| """ |
| super().__init__(dexer, debug_info, device, ['-Xint']) |
| |
| @property |
| def description(self): |
| return 'Art interpreter on target' |
| |
| @property |
| def id(self): |
| return 'TInt' |
| |
| def GetBisectionSearchArgs(self): |
| return None |
| |
| |
| class TestRunnerArtOptOnTarget(TestRunnerArtOnTarget): |
| """Concrete test runner of optimizing compiler mode Art on target.""" |
| |
| def __init__(self, dexer, debug_info, device): |
| """Constructor for the Art on target tester (optimizing). |
| |
| Args: |
| dexer: string, defines dexer |
| debug_info: boolean, if True include debugging info |
| device: string, target device serial number (or None) |
| """ |
| super().__init__(dexer, debug_info, device, None) |
| |
| @property |
| def description(self): |
| return 'Art optimizing on target' |
| |
| @property |
| def id(self): |
| return 'TOpt' |
| |
| def GetBisectionSearchArgs(self): |
| cmd_str = CommandListToCommandString( |
| self._dalvik_cmd + ['-cp', self._device_classpath, 'Test']) |
| cmd = ['--raw-cmd={0}'.format(cmd_str), '--timeout', str(30)] |
| if self._device: |
| cmd += ['--device-serial', self._device] |
| else: |
| cmd.append('--device') |
| return cmd |
| |
| |
| # |
| # Tester class. |
| # |
| |
| |
| class JFuzzTester(object): |
| """Tester that runs JFuzz many times and report divergences.""" |
| |
| def __init__(self, num_tests, device, mode1, mode2, jfuzz_args, |
| report_script, true_divergence_only, dexer, debug_info): |
| """Constructor for the tester. |
| |
| Args: |
| num_tests: int, number of tests to run |
| device: string, target device serial number (or None) |
| mode1: string, execution mode for first runner |
| mode2: string, execution mode for second runner |
| jfuzz_args: list of strings, additional arguments for jfuzz |
| report_script: string, path to script called for each divergence |
| true_divergence_only: boolean, if True don't bisect timeout divergences |
| dexer: string, defines dexer |
| debug_info: boolean, if True include debugging info |
| """ |
| self._num_tests = num_tests |
| self._device = device |
| self._runner1 = GetExecutionModeRunner(dexer, debug_info, device, mode1) |
| self._runner2 = GetExecutionModeRunner(dexer, debug_info, device, mode2) |
| self._jfuzz_args = jfuzz_args |
| self._report_script = report_script |
| self._true_divergence_only = true_divergence_only |
| self._dexer = dexer |
| self._debug_info = debug_info |
| self._save_dir = None |
| self._results_dir = None |
| self._jfuzz_dir = None |
| # Statistics. |
| self._test = 0 |
| self._num_success = 0 |
| self._num_not_compiled = 0 |
| self._num_not_run = 0 |
| self._num_timed_out = 0 |
| self._num_divergences = 0 |
| |
| def __enter__(self): |
| """On entry, enters new temp directory after saving current directory. |
| |
| Raises: |
| FatalError: error when temp directory cannot be constructed |
| """ |
| self._save_dir = os.getcwd() |
| self._results_dir = mkdtemp(dir='/tmp/') |
| self._jfuzz_dir = mkdtemp(dir=self._results_dir) |
| if self._results_dir is None or self._jfuzz_dir is None: |
| raise FatalError('Cannot obtain temp directory') |
| os.chdir(self._jfuzz_dir) |
| return self |
| |
| def __exit__(self, etype, evalue, etraceback): |
| """On exit, re-enters previously saved current directory and cleans up.""" |
| os.chdir(self._save_dir) |
| shutil.rmtree(self._jfuzz_dir) |
| if self._num_divergences == 0: |
| shutil.rmtree(self._results_dir) |
| |
| def Run(self): |
| """Runs JFuzz many times and report divergences.""" |
| print() |
| print('**\n**** JFuzz Testing\n**') |
| print() |
| print('#Tests :', self._num_tests) |
| print('Device :', self._device) |
| print('Directory :', self._results_dir) |
| print('Exec-mode1:', self._runner1.description) |
| print('Exec-mode2:', self._runner2.description) |
| print('Dexer :', self._dexer) |
| print('Debug-info:', self._debug_info) |
| print() |
| self.ShowStats() |
| for self._test in range(1, self._num_tests + 1): |
| self.RunJFuzzTest() |
| self.ShowStats() |
| if self._num_divergences == 0: |
| print('\n\nsuccess (no divergences)\n') |
| else: |
| print('\n\nfailure (divergences)\n') |
| |
| def ShowStats(self): |
| """Shows current statistics (on same line) while tester is running.""" |
| print('\rTests:', self._test, |
| 'Success:', self._num_success, |
| 'Not-compiled:', self._num_not_compiled, |
| 'Not-run:', self._num_not_run, |
| 'Timed-out:', self._num_timed_out, |
| 'Divergences:', self._num_divergences, |
| end='') |
| sys.stdout.flush() |
| |
| def RunJFuzzTest(self): |
| """Runs a single JFuzz test, comparing two execution modes.""" |
| self.ConstructTest() |
| retc1 = self._runner1.CompileAndRunTest() |
| retc2 = self._runner2.CompileAndRunTest() |
| self.CheckForDivergence(retc1, retc2) |
| self.CleanupTest() |
| |
| def ConstructTest(self): |
| """Use JFuzz to generate next Test.java test. |
| |
| Raises: |
| FatalError: error when jfuzz fails |
| """ |
| if (RunCommand(['jfuzz'] + self._jfuzz_args, out='Test.java', err=None) |
| != RetCode.SUCCESS): |
| raise FatalError('Unexpected error while running JFuzz') |
| |
| def CheckForDivergence(self, retc1, retc2): |
| """Checks for divergences and updates statistics. |
| |
| Args: |
| retc1: int, normalized return code of first runner |
| retc2: int, normalized return code of second runner |
| """ |
| if retc1 == retc2: |
| # No divergence in return code. |
| if retc1 == RetCode.SUCCESS: |
| # Both compilations and runs were successful, inspect generated output. |
| runner1_out = self._runner1.output_file |
| runner2_out = self._runner2.output_file |
| if not filecmp.cmp(runner1_out, runner2_out, shallow=False): |
| # Divergence in output. |
| self.ReportDivergence(retc1, retc2, is_output_divergence=True) |
| else: |
| # No divergence in output. |
| self._num_success += 1 |
| elif retc1 == RetCode.TIMEOUT: |
| self._num_timed_out += 1 |
| elif retc1 == RetCode.NOTCOMPILED: |
| self._num_not_compiled += 1 |
| else: |
| self._num_not_run += 1 |
| else: |
| # Divergence in return code. |
| if self._true_divergence_only: |
| # When only true divergences are requested, any divergence in return |
| # code where one is a time out is treated as a regular time out. |
| if RetCode.TIMEOUT in (retc1, retc2): |
| self._num_timed_out += 1 |
| return |
| # When only true divergences are requested, a runtime crash in just |
| # the RI is treated as if not run at all. |
| if retc1 == RetCode.ERROR and retc2 == RetCode.SUCCESS: |
| if self._runner1.GetBisectionSearchArgs() is None: |
| self._num_not_run += 1 |
| return |
| self.ReportDivergence(retc1, retc2, is_output_divergence=False) |
| |
| def GetCurrentDivergenceDir(self): |
| return self._results_dir + '/divergence' + str(self._num_divergences) |
| |
| def ReportDivergence(self, retc1, retc2, is_output_divergence): |
| """Reports and saves a divergence.""" |
| self._num_divergences += 1 |
| print('\n' + str(self._num_divergences), end='') |
| if is_output_divergence: |
| print(' divergence in output') |
| else: |
| print(' divergence in return code: ' + retc1.name + ' vs. ' + |
| retc2.name) |
| # Save. |
| ddir = self.GetCurrentDivergenceDir() |
| os.mkdir(ddir) |
| for f in glob('*.txt') + ['Test.java']: |
| shutil.copy(f, ddir) |
| # Maybe run bisection bug search. |
| if retc1 in BISECTABLE_RET_CODES and retc2 in BISECTABLE_RET_CODES: |
| self.MaybeBisectDivergence(retc1, retc2, is_output_divergence) |
| # Call reporting script. |
| if self._report_script: |
| self.RunReportScript(retc1, retc2, is_output_divergence) |
| |
| def RunReportScript(self, retc1, retc2, is_output_divergence): |
| """Runs report script.""" |
| try: |
| title = "Divergence between {0} and {1} (found with fuzz testing)".format( |
| self._runner1.description, self._runner2.description) |
| # Prepare divergence comment. |
| jfuzz_cmd_and_version = subprocess.check_output( |
| ['grep', '-o', 'jfuzz.*', 'Test.java'], universal_newlines=True) |
| (jfuzz_cmd_str, jfuzz_ver) = jfuzz_cmd_and_version.split('(') |
| # Strip right parenthesis and new line. |
| jfuzz_ver = jfuzz_ver[:-2] |
| jfuzz_args = ['\'-{0}\''.format(arg) |
| for arg in jfuzz_cmd_str.strip().split(' -')][1:] |
| wrapped_args = ['--jfuzz_arg={0}'.format(opt) for opt in jfuzz_args] |
| repro_cmd_str = (os.path.basename(__file__) + |
| ' --num_tests=1 --dexer=' + self._dexer + |
| (' --debug_info ' if self._debug_info else ' ') + |
| ' '.join(wrapped_args)) |
| comment = 'jfuzz {0}\nReproduce test:\n{1}\nReproduce divergence:\n{2}\n'.format( |
| jfuzz_ver, jfuzz_cmd_str, repro_cmd_str) |
| if is_output_divergence: |
| (output, _, _) = RunCommandForOutput( |
| ['diff', self._runner1.output_file, self._runner2.output_file], |
| None, subprocess.PIPE, subprocess.STDOUT) |
| comment += 'Diff:\n' + output |
| else: |
| comment += '{0} vs {1}\n'.format(retc1, retc2) |
| # Prepare report script command. |
| script_cmd = [self._report_script, title, comment] |
| ddir = self.GetCurrentDivergenceDir() |
| bisection_out_files = glob(ddir + '/*_bisection_out.txt') |
| if bisection_out_files: |
| script_cmd += ['--bisection_out', bisection_out_files[0]] |
| subprocess.check_call(script_cmd, stdout=DEVNULL, stderr=DEVNULL) |
| except subprocess.CalledProcessError as err: |
| print('Failed to run report script.\n', err) |
| |
| def RunBisectionSearch(self, args, expected_retcode, expected_output, |
| runner_id): |
| ddir = self.GetCurrentDivergenceDir() |
| outfile_path = ddir + '/' + runner_id + '_bisection_out.txt' |
| logfile_path = ddir + '/' + runner_id + '_bisection_log.txt' |
| errfile_path = ddir + '/' + runner_id + '_bisection_err.txt' |
| args = list(args) + ['--logfile', logfile_path, '--cleanup'] |
| args += ['--expected-retcode', expected_retcode.name] |
| if expected_output: |
| args += ['--expected-output', expected_output] |
| bisection_search_path = os.path.join( |
| GetEnvVariableOrError('ANDROID_BUILD_TOP'), |
| 'art/tools/bisection_search/bisection_search.py') |
| if RunCommand([bisection_search_path] + args, out=outfile_path, |
| err=errfile_path, timeout=300) == RetCode.TIMEOUT: |
| print('Bisection search TIMEOUT') |
| |
| def MaybeBisectDivergence(self, retc1, retc2, is_output_divergence): |
| bisection_args1 = self._runner1.GetBisectionSearchArgs() |
| bisection_args2 = self._runner2.GetBisectionSearchArgs() |
| if is_output_divergence: |
| maybe_output1 = self._runner1.output_file |
| maybe_output2 = self._runner2.output_file |
| else: |
| maybe_output1 = maybe_output2 = None |
| if bisection_args1 is not None: |
| self.RunBisectionSearch(bisection_args1, retc2, maybe_output2, |
| self._runner1.id) |
| if bisection_args2 is not None: |
| self.RunBisectionSearch(bisection_args2, retc1, maybe_output1, |
| self._runner2.id) |
| |
| def CleanupTest(self): |
| """Cleans up after a single test run.""" |
| for file_name in os.listdir(self._jfuzz_dir): |
| file_path = os.path.join(self._jfuzz_dir, file_name) |
| if os.path.isfile(file_path): |
| os.unlink(file_path) |
| elif os.path.isdir(file_path): |
| shutil.rmtree(file_path) |
| |
| |
| def main(): |
| # Handle arguments. |
| parser = argparse.ArgumentParser() |
| parser.add_argument('--num_tests', default=10000, type=int, |
| help='number of tests to run') |
| parser.add_argument('--device', help='target device serial number') |
| parser.add_argument('--mode1', default='ri', |
| help='execution mode 1 (default: ri)') |
| parser.add_argument('--mode2', default='hopt', |
| help='execution mode 2 (default: hopt)') |
| parser.add_argument('--report_script', |
| help='script called for each divergence') |
| parser.add_argument('--jfuzz_arg', default=[], dest='jfuzz_args', |
| action='append', |
| help='argument for jfuzz') |
| parser.add_argument('--true_divergence', default=False, action='store_true', |
| help='do not bisect timeout divergences') |
| parser.add_argument('--dexer', default='dx', type=str, |
| help='defines dexer as dx or d8 (default: dx)') |
| parser.add_argument('--debug_info', default=False, action='store_true', |
| help='include debugging info') |
| args = parser.parse_args() |
| if args.mode1 == args.mode2: |
| raise FatalError('Identical execution modes given') |
| # Run the JFuzz tester. |
| with JFuzzTester(args.num_tests, |
| args.device, |
| args.mode1, args.mode2, |
| args.jfuzz_args, |
| args.report_script, |
| args.true_divergence, |
| args.dexer, |
| args.debug_info) as fuzzer: |
| fuzzer.Run() |
| |
| if __name__ == '__main__': |
| main() |