Integrate bisection search with javafuzz
This CL makes javafuzz call bisection search on failing tests.
Three switches were added to bisection_search, --logfile which can be
used to provide custom logfile destination, --never-clean which
disables automatic cleanup of bisection directory and --timeout
which allows user to specify maximum time in seconds to wait for
a single test run.
ITestEnv subclasses were updated to integrate with javafuzz.
run_java_fuzz_test.py is now reusing code from bisection_search
module. It also better matches python style guidelines.
Change-Id: Ie41653b045469f2ceb352fd35fb4099842bb5bc3
diff --git a/tools/javafuzz/run_java_fuzz_test.py b/tools/javafuzz/run_java_fuzz_test.py
index 5f527b8..085471f 100755
--- a/tools/javafuzz/run_java_fuzz_test.py
+++ b/tools/javafuzz/run_java_fuzz_test.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3.4
#
# Copyright (C) 2016 The Android Open Source Project
#
@@ -16,68 +16,71 @@
import abc
import argparse
+import filecmp
+
+from glob import glob
+
+import os
+import shlex
+import shutil
import subprocess
import sys
-import os
from tempfile import mkdtemp
-from threading import Timer
-# Normalized return codes.
-EXIT_SUCCESS = 0
-EXIT_TIMEOUT = 1
-EXIT_NOTCOMPILED = 2
-EXIT_NOTRUN = 3
+
+sys.path.append(os.path.dirname(os.path.dirname(__file__)))
+
+from bisection_search.common import RetCode
+from bisection_search.common import CommandListToCommandString
+from bisection_search.common import FatalError
+from bisection_search.common import GetEnvVariableOrError
+from bisection_search.common import RunCommandForOutput
+from bisection_search.common import DeviceTestEnv
+
+# Return codes supported by bisection bug search.
+BISECTABLE_RET_CODES = (RetCode.SUCCESS, RetCode.ERROR, RetCode.TIMEOUT)
#
# Utility methods.
#
-def RunCommand(cmd, args, out, err, timeout = 5):
+
+def RunCommand(cmd, out, err, timeout=5):
"""Executes a command, and returns its return code.
Args:
- cmd: string, a command to execute
- args: string, arguments to pass to command (or None)
+ cmd: list of strings, a command to execute
out: string, file name to open for stdout (or None)
err: string, file name to open for stderr (or None)
timeout: int, time out in seconds
Returns:
- return code of running command (forced EXIT_TIMEOUT on timeout)
+ RetCode, return code of running command (forced RetCode.TIMEOUT
+ on timeout)
"""
- cmd = 'exec ' + cmd # preserve pid
- if args != None:
- cmd = cmd + ' ' + args
- outf = None
- if out != None:
+ devnull = subprocess.DEVNULL
+ outf = devnull
+ if out is not None:
outf = open(out, mode='w')
- errf = None
- if err != None:
+ errf = devnull
+ if err is not None:
errf = open(err, mode='w')
- proc = subprocess.Popen(cmd, stdout=outf, stderr=errf, shell=True)
- timer = Timer(timeout, proc.kill) # enforces timeout
- timer.start()
- proc.communicate()
- if timer.is_alive():
- timer.cancel()
- returncode = proc.returncode
- else:
- returncode = EXIT_TIMEOUT
- if outf != None:
+ (_, _, retcode) = RunCommandForOutput(cmd, None, outf, errf, timeout)
+ if outf != devnull:
outf.close()
- if errf != None:
+ if errf != devnull:
errf.close()
- return returncode
+ return retcode
+
def GetJackClassPath():
"""Returns Jack's classpath."""
- top = os.environ.get('ANDROID_BUILD_TOP')
- if top == None:
- raise FatalError('Cannot find AOSP build top')
+ top = GetEnvVariableOrError('ANDROID_BUILD_TOP')
libdir = top + '/out/host/common/obj/JAVA_LIBRARIES'
return libdir + '/core-libart-hostdex_intermediates/classes.jack:' \
+ libdir + '/core-oj-hostdex_intermediates/classes.jack'
+
def GetExecutionModeRunner(device, mode):
"""Returns a runner for the given execution mode.
@@ -92,49 +95,44 @@
if mode == 'ri':
return TestRunnerRIOnHost()
if mode == 'hint':
- return TestRunnerArtOnHost(True)
+ return TestRunnerArtIntOnHost()
if mode == 'hopt':
- return TestRunnerArtOnHost(False)
+ return TestRunnerArtOptOnHost()
if mode == 'tint':
- return TestRunnerArtOnTarget(device, True)
+ return TestRunnerArtIntOnTarget(device)
if mode == 'topt':
- return TestRunnerArtOnTarget(device, False)
+ return TestRunnerArtOptOnTarget(device)
raise FatalError('Unknown execution mode')
-def GetReturnCode(retc):
- """Returns a string representation of the given normalized return code.
- Args:
- retc: int, normalized return code
- Returns:
- string representation of normalized return code
- Raises:
- FatalError: error for unknown normalized return code
- """
- if retc == EXIT_SUCCESS:
- return 'SUCCESS'
- if retc == EXIT_TIMEOUT:
- return 'TIMED-OUT'
- if retc == EXIT_NOTCOMPILED:
- return 'NOT-COMPILED'
- if retc == EXIT_NOTRUN:
- return 'NOT-RUN'
- raise FatalError('Unknown normalized return code')
-
#
# Execution mode classes.
#
+
class TestRunner(object):
"""Abstraction for running a test in a particular execution mode."""
__meta_class__ = abc.ABCMeta
- def GetDescription(self):
+ @abc.abstractproperty
+ def description(self):
"""Returns a description string of the execution mode."""
- return self._description
- def GetId(self):
+ @abc.abstractproperty
+ def id(self):
"""Returns a short string that uniquely identifies the execution mode."""
- return self._id
+
+ @property
+ def output_file(self):
+ return self.id + '_out.txt'
+
+ @abc.abstractmethod
+ def GetBisectionSearchArgs(self):
+ """Get arguments to pass to bisection search tool.
+
+ Returns:
+ list of strings - arguments for bisection search tool, or None if
+ runner is not bisectable
+ """
@abc.abstractmethod
def CompileAndRunTest(self):
@@ -142,8 +140,7 @@
Ensures that the current Test.java in the temporary directory is compiled
and executed under the current execution mode. On success, transfers the
- generated output to the file GetId()_out.txt in the temporary directory.
- Cleans up after itself.
+ generated output to the file self.output_file in the temporary directory.
Most nonzero return codes are assumed non-divergent, since systems may
exit in different ways. This is enforced by normalizing return codes.
@@ -151,112 +148,196 @@
Returns:
normalized return code
"""
- pass
+
class TestRunnerRIOnHost(TestRunner):
"""Concrete test runner of the reference implementation on host."""
- def __init__(self):
- """Constructor for the RI tester."""
- self._description = 'RI on host'
- self._id = 'RI'
+ @property
+ def description(self):
+ return 'RI on host'
+
+ @property
+ def id(self):
+ return 'RI'
def CompileAndRunTest(self):
- if RunCommand('javac', 'Test.java',
- out=None, err=None, timeout=30) == EXIT_SUCCESS:
- retc = RunCommand('java', 'Test', 'RI_run_out.txt', err=None)
- if retc != EXIT_SUCCESS and retc != EXIT_TIMEOUT:
- retc = EXIT_NOTRUN
+ if RunCommand(['javac', 'Test.java'],
+ out=None, err=None, timeout=30) == RetCode.SUCCESS:
+ retc = RunCommand(['java', 'Test'], self.output_file, err=None)
else:
- retc = EXIT_NOTCOMPILED
- # Cleanup and return.
- RunCommand('rm', '-f Test.class', out=None, err=None)
+ retc = RetCode.NOTCOMPILED
return retc
-class TestRunnerArtOnHost(TestRunner):
- """Concrete test runner of Art on host (interpreter or optimizing)."""
+ def GetBisectionSearchArgs(self):
+ return None
- def __init__(self, interpreter):
+
+class TestRunnerArtOnHost(TestRunner):
+ """Abstract test runner of Art on host."""
+
+ def __init__(self, extra_args=None):
"""Constructor for the Art on host tester.
Args:
- interpreter: boolean, selects between interpreter or optimizing
+ extra_args: list of strings, extra arguments for dalvikvm
"""
- self._art_args = '-cp classes.dex Test'
- if interpreter:
- self._description = 'Art interpreter on host'
- self._id = 'HInt'
- self._art_args = '-Xint ' + self._art_args
- else:
- self._description = 'Art optimizing on host'
- self._id = 'HOpt'
- self._jack_args = '-cp ' + GetJackClassPath() + ' --output-dex . Test.java'
+ self._art_cmd = ['/bin/bash', 'art', '-cp', 'classes.dex']
+ if extra_args is not None:
+ self._art_cmd += extra_args
+ self._art_cmd.append('Test')
+ self._jack_args = ['-cp', GetJackClassPath(), '--output-dex', '.',
+ 'Test.java']
def CompileAndRunTest(self):
- if RunCommand('jack', self._jack_args,
- out=None, err='jackerr.txt', timeout=30) == EXIT_SUCCESS:
- out = self.GetId() + '_run_out.txt'
- retc = RunCommand('art', self._art_args, out, 'arterr.txt')
- if retc != EXIT_SUCCESS and retc != EXIT_TIMEOUT:
- retc = EXIT_NOTRUN
+ if RunCommand(['jack'] + self._jack_args, out=None, err='jackerr.txt',
+ timeout=30) == RetCode.SUCCESS:
+ retc = RunCommand(self._art_cmd, self.output_file, 'arterr.txt')
else:
- retc = EXIT_NOTCOMPILED
- # Cleanup and return.
- RunCommand('rm', '-rf classes.dex jackerr.txt arterr.txt android-data*',
- out=None, err=None)
+ retc = RetCode.NOTCOMPILED
return retc
-# TODO: very rough first version without proper cache,
-# reuse staszkiewicz' module for properly setting up dalvikvm on target.
-class TestRunnerArtOnTarget(TestRunner):
- """Concrete test runner of Art on target (interpreter or optimizing)."""
- def __init__(self, device, interpreter):
+class TestRunnerArtIntOnHost(TestRunnerArtOnHost):
+ """Concrete test runner of interpreter mode Art on host."""
+
+ def __init__(self):
+ """Constructor."""
+ super().__init__(['-Xint'])
+
+ @property
+ def description(self):
+ return 'Art interpreter on host'
+
+ @property
+ def id(self):
+ return 'HInt'
+
+ def GetBisectionSearchArgs(self):
+ return None
+
+
+class TestRunnerArtOptOnHost(TestRunnerArtOnHost):
+ """Concrete test runner of optimizing compiler mode Art on host."""
+
+ def __init__(self):
+ """Constructor."""
+ super().__init__(None)
+
+ @property
+ def description(self):
+ return 'Art optimizing on host'
+
+ @property
+ def id(self):
+ return 'HOpt'
+
+ def GetBisectionSearchArgs(self):
+ cmd_str = CommandListToCommandString(
+ self._art_cmd[0:2] + ['{ARGS}'] + self._art_cmd[2:])
+ return ['--raw-cmd={0}'.format(cmd_str), '--timeout', str(30)]
+
+
+class TestRunnerArtOnTarget(TestRunner):
+ """Abstract test runner of Art on target."""
+
+ def __init__(self, device, extra_args=None):
"""Constructor for the Art on target tester.
Args:
device: string, target device serial number (or None)
- interpreter: boolean, selects between interpreter or optimizing
+ extra_args: list of strings, extra arguments for dalvikvm
"""
- self._dalvik_args = 'shell dalvikvm -cp /data/local/tmp/classes.dex Test'
- if interpreter:
- self._description = 'Art interpreter on target'
- self._id = 'TInt'
- self._dalvik_args = '-Xint ' + self._dalvik_args
- else:
- self._description = 'Art optimizing on target'
- self._id = 'TOpt'
- self._adb = 'adb'
- if device != None:
- self._adb = self._adb + ' -s ' + device
- self._jack_args = '-cp ' + GetJackClassPath() + ' --output-dex . Test.java'
+ self._test_env = DeviceTestEnv('javafuzz_', specific_device=device)
+ self._dalvik_cmd = ['dalvikvm']
+ if extra_args is not None:
+ self._dalvik_cmd += extra_args
+ self._device = device
+ self._jack_args = ['-cp', GetJackClassPath(), '--output-dex', '.',
+ 'Test.java']
+ self._device_classpath = None
def CompileAndRunTest(self):
- if RunCommand('jack', self._jack_args,
- out=None, err='jackerr.txt', timeout=30) == EXIT_SUCCESS:
- if RunCommand(self._adb, 'push classes.dex /data/local/tmp/',
- 'adb.txt', err=None) != EXIT_SUCCESS:
- raise FatalError('Cannot push to target device')
- out = self.GetId() + '_run_out.txt'
- retc = RunCommand(self._adb, self._dalvik_args, out, err=None)
- if retc != EXIT_SUCCESS and retc != EXIT_TIMEOUT:
- retc = EXIT_NOTRUN
+ if RunCommand(['jack'] + self._jack_args, out=None, err='jackerr.txt',
+ timeout=30) == RetCode.SUCCESS:
+ self._device_classpath = self._test_env.PushClasspath('classes.dex')
+ cmd = self._dalvik_cmd + ['-cp', self._device_classpath, 'Test']
+ (output, retc) = self._test_env.RunCommand(
+ cmd, {'ANDROID_LOG_TAGS': '*:s'})
+ with open(self.output_file, 'w') as run_out:
+ run_out.write(output)
else:
- retc = EXIT_NOTCOMPILED
- # Cleanup and return.
- RunCommand('rm', '-f classes.dex jackerr.txt adb.txt',
- out=None, err=None)
- RunCommand(self._adb, 'shell rm -f /data/local/tmp/classes.dex',
- out=None, err=None)
+ retc = RetCode.NOTCOMPILED
return retc
+ def GetBisectionSearchArgs(self):
+ cmd_str = CommandListToCommandString(
+ self._dalvik_cmd + ['-cp',self._device_classpath, 'Test'])
+ cmd = ['--raw-cmd={0}'.format(cmd_str), '--timeout', str(30)]
+ if self._device:
+ cmd += ['--device-serial', self._device]
+ else:
+ cmd.append('--device')
+ return cmd
+
+
+class TestRunnerArtIntOnTarget(TestRunnerArtOnTarget):
+ """Concrete test runner of interpreter mode Art on target."""
+
+ def __init__(self, device):
+ """Constructor.
+
+ Args:
+ device: string, target device serial number (or None)
+ """
+ super().__init__(device, ['-Xint'])
+
+ @property
+ def description(self):
+ return 'Art interpreter on target'
+
+ @property
+ def id(self):
+ return 'TInt'
+
+ def GetBisectionSearchArgs(self):
+ return None
+
+
+class TestRunnerArtOptOnTarget(TestRunnerArtOnTarget):
+ """Concrete test runner of optimizing compiler mode Art on target."""
+
+ def __init__(self, device):
+ """Constructor.
+
+ Args:
+ device: string, target device serial number (or None)
+ """
+ super().__init__(device, None)
+
+ @property
+ def description(self):
+ return 'Art optimizing on target'
+
+ @property
+ def id(self):
+ return 'TOpt'
+
+ def GetBisectionSearchArgs(self):
+ cmd_str = CommandListToCommandString(
+ self._dalvik_cmd + ['-cp', self._device_classpath, 'Test'])
+ cmd = ['--raw-cmd={0}'.format(cmd_str), '--timeout', str(30)]
+ if self._device:
+ cmd += ['--device-serial', self._device]
+ else:
+ cmd.append('--device')
+ return cmd
+
+
#
# Tester classes.
#
-class FatalError(Exception):
- """Fatal error in the tester."""
- pass
class JavaFuzzTester(object):
"""Tester that runs JavaFuzz many times and report divergences."""
@@ -265,10 +346,10 @@
"""Constructor for the tester.
Args:
- num_tests: int, number of tests to run
- device: string, target device serial number (or None)
- mode1: string, execution mode for first runner
- mode2: string, execution mode for second runner
+ num_tests: int, number of tests to run
+ device: string, target device serial number (or None)
+ mode1: string, execution mode for first runner
+ mode2: string, execution mode for second runner
"""
self._num_tests = num_tests
self._device = device
@@ -291,8 +372,9 @@
FatalError: error when temp directory cannot be constructed
"""
self._save_dir = os.getcwd()
- self._tmp_dir = mkdtemp(dir="/tmp/")
- if self._tmp_dir == None:
+ self._results_dir = mkdtemp(dir='/tmp/')
+ self._tmp_dir = mkdtemp(dir=self._results_dir)
+ if self._tmp_dir is None or self._results_dir is None:
raise FatalError('Cannot obtain temp directory')
os.chdir(self._tmp_dir)
return self
@@ -300,37 +382,38 @@
def __exit__(self, etype, evalue, etraceback):
"""On exit, re-enters previously saved current directory and cleans up."""
os.chdir(self._save_dir)
+ shutil.rmtree(self._tmp_dir)
if self._num_divergences == 0:
- RunCommand('rm', '-rf ' + self._tmp_dir, out=None, err=None)
+ shutil.rmtree(self._results_dir)
def Run(self):
"""Runs JavaFuzz many times and report divergences."""
- print
- print '**\n**** JavaFuzz Testing\n**'
- print
- print '#Tests :', self._num_tests
- print 'Device :', self._device
- print 'Directory :', self._tmp_dir
- print 'Exec-mode1:', self._runner1.GetDescription()
- print 'Exec-mode2:', self._runner2.GetDescription()
+ print()
+ print('**\n**** JavaFuzz Testing\n**')
+ print()
+ print('#Tests :', self._num_tests)
+ print('Device :', self._device)
+ print('Directory :', self._results_dir)
+ print('Exec-mode1:', self._runner1.description)
+ print('Exec-mode2:', self._runner2.description)
print
self.ShowStats()
for self._test in range(1, self._num_tests + 1):
self.RunJavaFuzzTest()
self.ShowStats()
if self._num_divergences == 0:
- print '\n\nsuccess (no divergences)\n'
+ print('\n\nsuccess (no divergences)\n')
else:
- print '\n\nfailure (divergences)\n'
+ print('\n\nfailure (divergences)\n')
def ShowStats(self):
"""Shows current statistics (on same line) while tester is running."""
- print '\rTests:', self._test, \
- 'Success:', self._num_success, \
- 'Not-compiled:', self._num_not_compiled, \
- 'Not-run:', self._num_not_run, \
- 'Timed-out:', self._num_timed_out, \
- 'Divergences:', self._num_divergences,
+ print('\rTests:', self._test, \
+ 'Success:', self._num_success, \
+ 'Not-compiled:', self._num_not_compiled, \
+ 'Not-run:', self._num_not_run, \
+ 'Timed-out:', self._num_timed_out, \
+ 'Divergences:', self._num_divergences, end='')
sys.stdout.flush()
def RunJavaFuzzTest(self):
@@ -347,8 +430,7 @@
Raises:
FatalError: error when javafuzz fails
"""
- if RunCommand('javafuzz', args=None,
- out='Test.java', err=None) != EXIT_SUCCESS:
+ if RunCommand(['javafuzz'], out='Test.java', err=None) != RetCode.SUCCESS:
raise FatalError('Unexpected error while running JavaFuzz')
def CheckForDivergence(self, retc1, retc2):
@@ -360,38 +442,85 @@
"""
if retc1 == retc2:
# Non-divergent in return code.
- if retc1 == EXIT_SUCCESS:
+ if retc1 == RetCode.SUCCESS:
# Both compilations and runs were successful, inspect generated output.
- args = self._runner1.GetId() + '_run_out.txt ' \
- + self._runner2.GetId() + '_run_out.txt'
- if RunCommand('diff', args, out=None, err=None) != EXIT_SUCCESS:
- self.ReportDivergence('divergence in output')
+ runner1_out = self._runner1.output_file
+ runner2_out = self._runner2.output_file
+ if not filecmp.cmp(runner1_out, runner2_out, shallow=False):
+ self.ReportDivergence(retc1, retc2, is_output_divergence=True)
else:
self._num_success += 1
- elif retc1 == EXIT_TIMEOUT:
+ elif retc1 == RetCode.TIMEOUT:
self._num_timed_out += 1
- elif retc1 == EXIT_NOTCOMPILED:
+ elif retc1 == RetCode.NOTCOMPILED:
self._num_not_compiled += 1
else:
self._num_not_run += 1
else:
# Divergent in return code.
- self.ReportDivergence('divergence in return code: ' +
- GetReturnCode(retc1) + ' vs. ' +
- GetReturnCode(retc2))
+ self.ReportDivergence(retc1, retc2, is_output_divergence=False)
- def ReportDivergence(self, reason):
+ def GetCurrentDivergenceDir(self):
+ return self._results_dir + '/divergence' + str(self._num_divergences)
+
+ def ReportDivergence(self, retc1, retc2, is_output_divergence):
"""Reports and saves a divergence."""
self._num_divergences += 1
- print '\n', self._test, reason
+ print('\n' + str(self._num_divergences), end='')
+ if is_output_divergence:
+ print(' divergence in output')
+ else:
+ print(' divergence in return code: ' + retc1.name + ' vs. ' +
+ retc2.name)
# Save.
- ddir = 'divergence' + str(self._test)
- RunCommand('mkdir', ddir, out=None, err=None)
- RunCommand('mv', 'Test.java *.txt ' + ddir, out=None, err=None)
+ ddir = self.GetCurrentDivergenceDir()
+ os.mkdir(ddir)
+ for f in glob('*.txt') + ['Test.java']:
+ shutil.copy(f, ddir)
+ # Maybe run bisection bug search.
+ if retc1 in BISECTABLE_RET_CODES and retc2 in BISECTABLE_RET_CODES:
+ self.MaybeBisectDivergence(retc1, retc2, is_output_divergence)
+
+ def RunBisectionSearch(self, args, expected_retcode, expected_output,
+ runner_id):
+ ddir = self.GetCurrentDivergenceDir()
+ outfile_path = ddir + '/' + runner_id + '_bisection_out.txt'
+ logfile_path = ddir + '/' + runner_id + '_bisection_log.txt'
+ errfile_path = ddir + '/' + runner_id + '_bisection_err.txt'
+ args = list(args) + ['--logfile', logfile_path, '--cleanup']
+ args += ['--expected-retcode', expected_retcode.name]
+ if expected_output:
+ args += ['--expected-output', expected_output]
+ bisection_search_path = os.path.join(
+ GetEnvVariableOrError('ANDROID_BUILD_TOP'),
+ 'art/tools/bisection_search/bisection_search.py')
+ if RunCommand([bisection_search_path] + args, out=outfile_path,
+ err=errfile_path, timeout=300) == RetCode.TIMEOUT:
+ print('Bisection search TIMEOUT')
+
+ def MaybeBisectDivergence(self, retc1, retc2, is_output_divergence):
+ bisection_args1 = self._runner1.GetBisectionSearchArgs()
+ bisection_args2 = self._runner2.GetBisectionSearchArgs()
+ if is_output_divergence:
+ maybe_output1 = self._runner1.output_file
+ maybe_output2 = self._runner2.output_file
+ else:
+ maybe_output1 = maybe_output2 = None
+ if bisection_args1 is not None:
+ self.RunBisectionSearch(bisection_args1, retc2, maybe_output2,
+ self._runner1.id)
+ if bisection_args2 is not None:
+ self.RunBisectionSearch(bisection_args2, retc1, maybe_output1,
+ self._runner2.id)
def CleanupTest(self):
"""Cleans up after a single test run."""
- RunCommand('rm', '-f Test.java *.txt', out=None, err=None)
+ for file_name in os.listdir(self._tmp_dir):
+ file_path = os.path.join(self._tmp_dir, file_name)
+ if os.path.isfile(file_path):
+ os.unlink(file_path)
+ elif os.path.isdir(file_path):
+ shutil.rmtree(file_path)
def main():
@@ -406,11 +535,11 @@
help='execution mode 2 (default: hopt)')
args = parser.parse_args()
if args.mode1 == args.mode2:
- raise FatalError("Identical execution modes given")
+ raise FatalError('Identical execution modes given')
# Run the JavaFuzz tester.
with JavaFuzzTester(args.num_tests, args.device,
args.mode1, args.mode2) as fuzzer:
fuzzer.Run()
-if __name__ == "__main__":
+if __name__ == '__main__':
main()