Renaming tool: jfuzz
Test: run_jfuzz_test.py
Change-Id: I0efa31c3f1c30344c796a9077488e7e467d2456e
diff --git a/tools/jfuzz/run_jfuzz_test.py b/tools/jfuzz/run_jfuzz_test.py
new file mode 100755
index 0000000..cf2364b
--- /dev/null
+++ b/tools/jfuzz/run_jfuzz_test.py
@@ -0,0 +1,507 @@
+#!/usr/bin/env python3.4
+#
+# Copyright (C) 2016 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import abc
+import argparse
+import filecmp
+import os
+import shlex
+import shutil
+import sys
+
+from glob import glob
+from tempfile import mkdtemp
+
+sys.path.append(os.path.dirname(os.path.dirname(
+ os.path.realpath(__file__))))
+
+from common.common import RetCode
+from common.common import CommandListToCommandString
+from common.common import FatalError
+from common.common import GetJackClassPath
+from common.common import GetEnvVariableOrError
+from common.common import RunCommand
+from common.common import DeviceTestEnv
+
+# Return codes supported by bisection bug search.
+BISECTABLE_RET_CODES = (RetCode.SUCCESS, RetCode.ERROR, RetCode.TIMEOUT)
+
+
+def GetExecutionModeRunner(device, mode):
+ """Returns a runner for the given execution mode.
+
+ Args:
+ device: string, target device serial number (or None)
+ mode: string, execution mode
+ Returns:
+ TestRunner with given execution mode
+ Raises:
+ FatalError: error for unknown execution mode
+ """
+ if mode == 'ri':
+ return TestRunnerRIOnHost()
+ if mode == 'hint':
+ return TestRunnerArtIntOnHost()
+ if mode == 'hopt':
+ return TestRunnerArtOptOnHost()
+ if mode == 'tint':
+ return TestRunnerArtIntOnTarget(device)
+ if mode == 'topt':
+ return TestRunnerArtOptOnTarget(device)
+ raise FatalError('Unknown execution mode')
+
+
+#
+# Execution mode classes.
+#
+
+
+class TestRunner(object):
+ """Abstraction for running a test in a particular execution mode."""
+ __meta_class__ = abc.ABCMeta
+
+ @abc.abstractproperty
+ def description(self):
+ """Returns a description string of the execution mode."""
+
+ @abc.abstractproperty
+ def id(self):
+ """Returns a short string that uniquely identifies the execution mode."""
+
+ @property
+ def output_file(self):
+ return self.id + '_out.txt'
+
+ @abc.abstractmethod
+ def GetBisectionSearchArgs(self):
+ """Get arguments to pass to bisection search tool.
+
+ Returns:
+ list of strings - arguments for bisection search tool, or None if
+ runner is not bisectable
+ """
+
+ @abc.abstractmethod
+ def CompileAndRunTest(self):
+ """Compile and run the generated test.
+
+ Ensures that the current Test.java in the temporary directory is compiled
+ and executed under the current execution mode. On success, transfers the
+ generated output to the file self.output_file in the temporary directory.
+
+ Most nonzero return codes are assumed non-divergent, since systems may
+ exit in different ways. This is enforced by normalizing return codes.
+
+ Returns:
+ normalized return code
+ """
+
+
+class TestRunnerRIOnHost(TestRunner):
+ """Concrete test runner of the reference implementation on host."""
+
+ @property
+ def description(self):
+ return 'RI on host'
+
+ @property
+ def id(self):
+ return 'RI'
+
+ def CompileAndRunTest(self):
+ if RunCommand(['javac', 'Test.java'],
+ out=None, err=None, timeout=30) == RetCode.SUCCESS:
+ retc = RunCommand(['java', 'Test'], self.output_file, err=None)
+ else:
+ retc = RetCode.NOTCOMPILED
+ return retc
+
+ def GetBisectionSearchArgs(self):
+ return None
+
+
+class TestRunnerArtOnHost(TestRunner):
+ """Abstract test runner of Art on host."""
+
+ def __init__(self, extra_args=None):
+ """Constructor for the Art on host tester.
+
+ Args:
+ extra_args: list of strings, extra arguments for dalvikvm
+ """
+ self._art_cmd = ['/bin/bash', 'art', '-cp', 'classes.dex']
+ if extra_args is not None:
+ self._art_cmd += extra_args
+ self._art_cmd.append('Test')
+ self._jack_args = ['-cp', GetJackClassPath(), '--output-dex', '.',
+ 'Test.java']
+
+ def CompileAndRunTest(self):
+ if RunCommand(['jack'] + self._jack_args, out=None, err='jackerr.txt',
+ timeout=30) == RetCode.SUCCESS:
+ retc = RunCommand(self._art_cmd, self.output_file, 'arterr.txt')
+ else:
+ retc = RetCode.NOTCOMPILED
+ return retc
+
+
+class TestRunnerArtIntOnHost(TestRunnerArtOnHost):
+ """Concrete test runner of interpreter mode Art on host."""
+
+ def __init__(self):
+ """Constructor."""
+ super().__init__(['-Xint'])
+
+ @property
+ def description(self):
+ return 'Art interpreter on host'
+
+ @property
+ def id(self):
+ return 'HInt'
+
+ def GetBisectionSearchArgs(self):
+ return None
+
+
+class TestRunnerArtOptOnHost(TestRunnerArtOnHost):
+ """Concrete test runner of optimizing compiler mode Art on host."""
+
+ def __init__(self):
+ """Constructor."""
+ super().__init__(None)
+
+ @property
+ def description(self):
+ return 'Art optimizing on host'
+
+ @property
+ def id(self):
+ return 'HOpt'
+
+ def GetBisectionSearchArgs(self):
+ cmd_str = CommandListToCommandString(
+ self._art_cmd[0:2] + ['{ARGS}'] + self._art_cmd[2:])
+ return ['--raw-cmd={0}'.format(cmd_str), '--timeout', str(30)]
+
+
+class TestRunnerArtOnTarget(TestRunner):
+ """Abstract test runner of Art on target."""
+
+ def __init__(self, device, extra_args=None):
+ """Constructor for the Art on target tester.
+
+ Args:
+ device: string, target device serial number (or None)
+ extra_args: list of strings, extra arguments for dalvikvm
+ """
+ self._test_env = DeviceTestEnv('jfuzz_', specific_device=device)
+ self._dalvik_cmd = ['dalvikvm']
+ if extra_args is not None:
+ self._dalvik_cmd += extra_args
+ self._device = device
+ self._jack_args = ['-cp', GetJackClassPath(), '--output-dex', '.',
+ 'Test.java']
+ self._device_classpath = None
+
+ def CompileAndRunTest(self):
+ if RunCommand(['jack'] + self._jack_args, out=None, err='jackerr.txt',
+ timeout=30) == RetCode.SUCCESS:
+ self._device_classpath = self._test_env.PushClasspath('classes.dex')
+ cmd = self._dalvik_cmd + ['-cp', self._device_classpath, 'Test']
+ (output, retc) = self._test_env.RunCommand(
+ cmd, {'ANDROID_LOG_TAGS': '*:s'})
+ with open(self.output_file, 'w') as run_out:
+ run_out.write(output)
+ else:
+ retc = RetCode.NOTCOMPILED
+ return retc
+
+ def GetBisectionSearchArgs(self):
+ cmd_str = CommandListToCommandString(
+ self._dalvik_cmd + ['-cp',self._device_classpath, 'Test'])
+ cmd = ['--raw-cmd={0}'.format(cmd_str), '--timeout', str(30)]
+ if self._device:
+ cmd += ['--device-serial', self._device]
+ else:
+ cmd.append('--device')
+ return cmd
+
+
+class TestRunnerArtIntOnTarget(TestRunnerArtOnTarget):
+ """Concrete test runner of interpreter mode Art on target."""
+
+ def __init__(self, device):
+ """Constructor.
+
+ Args:
+ device: string, target device serial number (or None)
+ """
+ super().__init__(device, ['-Xint'])
+
+ @property
+ def description(self):
+ return 'Art interpreter on target'
+
+ @property
+ def id(self):
+ return 'TInt'
+
+ def GetBisectionSearchArgs(self):
+ return None
+
+
+class TestRunnerArtOptOnTarget(TestRunnerArtOnTarget):
+ """Concrete test runner of optimizing compiler mode Art on target."""
+
+ def __init__(self, device):
+ """Constructor.
+
+ Args:
+ device: string, target device serial number (or None)
+ """
+ super().__init__(device, None)
+
+ @property
+ def description(self):
+ return 'Art optimizing on target'
+
+ @property
+ def id(self):
+ return 'TOpt'
+
+ def GetBisectionSearchArgs(self):
+ cmd_str = CommandListToCommandString(
+ self._dalvik_cmd + ['-cp', self._device_classpath, 'Test'])
+ cmd = ['--raw-cmd={0}'.format(cmd_str), '--timeout', str(30)]
+ if self._device:
+ cmd += ['--device-serial', self._device]
+ else:
+ cmd.append('--device')
+ return cmd
+
+
+#
+# Tester class.
+#
+
+
+class JFuzzTester(object):
+ """Tester that runs JFuzz many times and report divergences."""
+
+ def __init__(self, num_tests, device, mode1, mode2):
+ """Constructor for the tester.
+
+ Args:
+ num_tests: int, number of tests to run
+ device: string, target device serial number (or None)
+ mode1: string, execution mode for first runner
+ mode2: string, execution mode for second runner
+ """
+ self._num_tests = num_tests
+ self._device = device
+ self._runner1 = GetExecutionModeRunner(device, mode1)
+ self._runner2 = GetExecutionModeRunner(device, mode2)
+ self._save_dir = None
+ self._results_dir = None
+ self._jfuzz_dir = None
+ # Statistics.
+ self._test = 0
+ self._num_success = 0
+ self._num_not_compiled = 0
+ self._num_not_run = 0
+ self._num_timed_out = 0
+ self._num_divergences = 0
+
+ def __enter__(self):
+ """On entry, enters new temp directory after saving current directory.
+
+ Raises:
+ FatalError: error when temp directory cannot be constructed
+ """
+ self._save_dir = os.getcwd()
+ self._results_dir = mkdtemp(dir='/tmp/')
+ self._jfuzz_dir = mkdtemp(dir=self._results_dir)
+ if self._results_dir is None or self._jfuzz_dir is None:
+ raise FatalError('Cannot obtain temp directory')
+ os.chdir(self._jfuzz_dir)
+ return self
+
+ def __exit__(self, etype, evalue, etraceback):
+ """On exit, re-enters previously saved current directory and cleans up."""
+ os.chdir(self._save_dir)
+ shutil.rmtree(self._jfuzz_dir)
+ if self._num_divergences == 0:
+ shutil.rmtree(self._results_dir)
+
+ def Run(self):
+ """Runs JFuzz many times and report divergences."""
+ print()
+ print('**\n**** JFuzz Testing\n**')
+ print()
+ print('#Tests :', self._num_tests)
+ print('Device :', self._device)
+ print('Directory :', self._results_dir)
+ print('Exec-mode1:', self._runner1.description)
+ print('Exec-mode2:', self._runner2.description)
+ print()
+ self.ShowStats()
+ for self._test in range(1, self._num_tests + 1):
+ self.RunJFuzzTest()
+ self.ShowStats()
+ if self._num_divergences == 0:
+ print('\n\nsuccess (no divergences)\n')
+ else:
+ print('\n\nfailure (divergences)\n')
+
+ def ShowStats(self):
+ """Shows current statistics (on same line) while tester is running."""
+ print('\rTests:', self._test,
+ 'Success:', self._num_success,
+ 'Not-compiled:', self._num_not_compiled,
+ 'Not-run:', self._num_not_run,
+ 'Timed-out:', self._num_timed_out,
+ 'Divergences:', self._num_divergences,
+ end='')
+ sys.stdout.flush()
+
+ def RunJFuzzTest(self):
+ """Runs a single JFuzz test, comparing two execution modes."""
+ self.ConstructTest()
+ retc1 = self._runner1.CompileAndRunTest()
+ retc2 = self._runner2.CompileAndRunTest()
+ self.CheckForDivergence(retc1, retc2)
+ self.CleanupTest()
+
+ def ConstructTest(self):
+ """Use JFuzz to generate next Test.java test.
+
+ Raises:
+ FatalError: error when jfuzz fails
+ """
+ if RunCommand(['jfuzz'], out='Test.java', err=None) != RetCode.SUCCESS:
+ raise FatalError('Unexpected error while running JFuzz')
+
+ def CheckForDivergence(self, retc1, retc2):
+ """Checks for divergences and updates statistics.
+
+ Args:
+ retc1: int, normalized return code of first runner
+ retc2: int, normalized return code of second runner
+ """
+ if retc1 == retc2:
+ # Non-divergent in return code.
+ if retc1 == RetCode.SUCCESS:
+ # Both compilations and runs were successful, inspect generated output.
+ runner1_out = self._runner1.output_file
+ runner2_out = self._runner2.output_file
+ if not filecmp.cmp(runner1_out, runner2_out, shallow=False):
+ self.ReportDivergence(retc1, retc2, is_output_divergence=True)
+ else:
+ self._num_success += 1
+ elif retc1 == RetCode.TIMEOUT:
+ self._num_timed_out += 1
+ elif retc1 == RetCode.NOTCOMPILED:
+ self._num_not_compiled += 1
+ else:
+ self._num_not_run += 1
+ else:
+ # Divergent in return code.
+ self.ReportDivergence(retc1, retc2, is_output_divergence=False)
+
+ def GetCurrentDivergenceDir(self):
+ return self._results_dir + '/divergence' + str(self._num_divergences)
+
+ def ReportDivergence(self, retc1, retc2, is_output_divergence):
+ """Reports and saves a divergence."""
+ self._num_divergences += 1
+ print('\n' + str(self._num_divergences), end='')
+ if is_output_divergence:
+ print(' divergence in output')
+ else:
+ print(' divergence in return code: ' + retc1.name + ' vs. ' +
+ retc2.name)
+ # Save.
+ ddir = self.GetCurrentDivergenceDir()
+ os.mkdir(ddir)
+ for f in glob('*.txt') + ['Test.java']:
+ shutil.copy(f, ddir)
+ # Maybe run bisection bug search.
+ if retc1 in BISECTABLE_RET_CODES and retc2 in BISECTABLE_RET_CODES:
+ self.MaybeBisectDivergence(retc1, retc2, is_output_divergence)
+
+ def RunBisectionSearch(self, args, expected_retcode, expected_output,
+ runner_id):
+ ddir = self.GetCurrentDivergenceDir()
+ outfile_path = ddir + '/' + runner_id + '_bisection_out.txt'
+ logfile_path = ddir + '/' + runner_id + '_bisection_log.txt'
+ errfile_path = ddir + '/' + runner_id + '_bisection_err.txt'
+ args = list(args) + ['--logfile', logfile_path, '--cleanup']
+ args += ['--expected-retcode', expected_retcode.name]
+ if expected_output:
+ args += ['--expected-output', expected_output]
+ bisection_search_path = os.path.join(
+ GetEnvVariableOrError('ANDROID_BUILD_TOP'),
+ 'art/tools/bisection_search/bisection_search.py')
+ if RunCommand([bisection_search_path] + args, out=outfile_path,
+ err=errfile_path, timeout=300) == RetCode.TIMEOUT:
+ print('Bisection search TIMEOUT')
+
+ def MaybeBisectDivergence(self, retc1, retc2, is_output_divergence):
+ bisection_args1 = self._runner1.GetBisectionSearchArgs()
+ bisection_args2 = self._runner2.GetBisectionSearchArgs()
+ if is_output_divergence:
+ maybe_output1 = self._runner1.output_file
+ maybe_output2 = self._runner2.output_file
+ else:
+ maybe_output1 = maybe_output2 = None
+ if bisection_args1 is not None:
+ self.RunBisectionSearch(bisection_args1, retc2, maybe_output2,
+ self._runner1.id)
+ if bisection_args2 is not None:
+ self.RunBisectionSearch(bisection_args2, retc1, maybe_output1,
+ self._runner2.id)
+
+ def CleanupTest(self):
+ """Cleans up after a single test run."""
+ for file_name in os.listdir(self._jfuzz_dir):
+ file_path = os.path.join(self._jfuzz_dir, file_name)
+ if os.path.isfile(file_path):
+ os.unlink(file_path)
+ elif os.path.isdir(file_path):
+ shutil.rmtree(file_path)
+
+
+def main():
+ # Handle arguments.
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--num_tests', default=10000,
+ type=int, help='number of tests to run')
+ parser.add_argument('--device', help='target device serial number')
+ parser.add_argument('--mode1', default='ri',
+ help='execution mode 1 (default: ri)')
+ parser.add_argument('--mode2', default='hopt',
+ help='execution mode 2 (default: hopt)')
+ args = parser.parse_args()
+ if args.mode1 == args.mode2:
+ raise FatalError('Identical execution modes given')
+ # Run the JFuzz tester.
+ with JFuzzTester(args.num_tests, args.device,
+ args.mode1, args.mode2) as fuzzer:
+ fuzzer.Run()
+
+if __name__ == '__main__':
+ main()