Integrate bisection search with javafuzz
This CL makes javafuzz call bisection search on failing tests.
Three switches were added to bisection_search, --logfile which can be
used to provide custom logfile destination, --never-clean which
disables automatic cleanup of bisection directory and --timeout
which allows user to specify maximum time in seconds to wait for
a single test run.
ITestEnv subclasses were updated to integrate with javafuzz.
run_java_fuzz_test.py is now reusing code from bisection_search
module. It also better matches python style guidelines.
Change-Id: Ie41653b045469f2ceb352fd35fb4099842bb5bc3
diff --git a/tools/bisection_search/README.md b/tools/bisection_search/README.md
index a7485c2..64ccb20 100644
--- a/tools/bisection_search/README.md
+++ b/tools/bisection_search/README.md
@@ -15,29 +15,54 @@
How to run Bisection Bug Search
===============================
+There are two supported invocation modes:
+
+1. Regular invocation, dalvikvm command is constructed internally:
+
+ ./bisection_search.py -cp classes.dex --expected-output out_int --class Test
+
+2. Raw-cmd invocation, dalvikvm command is accepted as an argument. The command
+ has to start with an executable.
+
+ Extra dalvikvm arguments will be placed on second position in the command
+ by default. {ARGS} tag can be used to specify a custom position.
+
+ ./bisection_search.py --raw-cmd='run.sh -cp classes.dex Test' --expected-retcode SUCCESS
+ ./bisection_search.py --raw-cmd='/bin/sh art {ARGS} -cp classes.dex Test' --expected-retcode SUCCESS
+
+Help:
+
bisection_search.py [-h] [-cp CLASSPATH] [--class CLASSNAME] [--lib LIB]
- [--dalvikvm-option [OPT [OPT ...]]] [--arg [ARG [ARG ...]]]
- [--image IMAGE] [--raw-cmd RAW_CMD]
- [--64] [--device] [--expected-output EXPECTED_OUTPUT]
- [--check-script CHECK_SCRIPT] [--verbose]
+ [--dalvikvm-option [OPT [OPT ...]]] [--arg [ARG [ARG ...]]]
+ [--image IMAGE] [--raw-cmd RAW_CMD]
+ [--64] [--device] [--device-serial DEVICE_SERIAL]
+ [--expected-output EXPECTED_OUTPUT]
+ [--expected-retcode {SUCCESS,TIMEOUT,ERROR}]
+ [--check-script CHECK_SCRIPT] [--logfile LOGFILE] [--cleanup]
+ [--timeout TIMEOUT] [--verbose]
Tool for finding compiler bugs. Either --raw-cmd or both -cp and --class are required.
optional arguments:
- -h, --help show this help message and exit
+ -h, --help show this help message and exit
dalvikvm command options:
- -cp CLASSPATH, --classpath CLASSPATH classpath
- --class CLASSNAME name of main class
- --lib LIB lib to use, default: libart.so
- --dalvikvm-option [OPT [OPT ...]] additional dalvikvm option
- --arg [ARG [ARG ...]] argument passed to test
- --image IMAGE path to image
- --raw-cmd RAW_CMD bisect with this command, ignore other command options
+ -cp CLASSPATH, --classpath CLASSPATH classpath
+ --class CLASSNAME name of main class
+ --lib LIB lib to use, default: libart.so
+ --dalvikvm-option [OPT [OPT ...]] additional dalvikvm option
+ --arg [ARG [ARG ...]] argument passed to test
+ --image IMAGE path to image
+ --raw-cmd RAW_CMD bisect with this command, ignore other command options
bisection options:
- --64 x64 mode
- --device run on device
- --expected-output EXPECTED_OUTPUT file containing expected output
- --check-script CHECK_SCRIPT script comparing output and expected output
- --verbose enable verbose output
+ --64 x64 mode
+ --device run on device
+ --device-serial DEVICE_SERIAL device serial number, implies --device
+ --expected-output EXPECTED_OUTPUT file containing expected output
+ --expected-retcode {SUCCESS,TIMEOUT,ERROR} expected normalized return code
+ --check-script CHECK_SCRIPT script comparing output and expected output
+ --logfile LOGFILE custom logfile location
+ --cleanup clean up after bisecting
+ --timeout TIMEOUT if timeout seconds pass assume test failed
+ --verbose enable verbose output
diff --git a/tools/bisection_search/bisection_search.py b/tools/bisection_search/bisection_search.py
index 110ef82..0d36aa4 100755
--- a/tools/bisection_search/bisection_search.py
+++ b/tools/bisection_search/bisection_search.py
@@ -34,6 +34,7 @@
from common import FatalError
from common import GetEnvVariableOrError
from common import HostTestEnv
+from common import RetCode
# Passes that are never disabled during search process because disabling them
@@ -51,6 +52,10 @@
NON_PASSES = ['builder', 'prepare_for_register_allocation',
'liveness', 'register']
+# If present in raw cmd, this tag will be replaced with runtime arguments
+# controlling the bisection search. Otherwise arguments will be placed on second
+# position in the command.
+RAW_CMD_RUNTIME_ARGS_TAG = '{ARGS}'
class Dex2OatWrapperTestable(object):
"""Class representing a testable compilation.
@@ -58,21 +63,29 @@
Accepts filters on compiled methods and optimization passes.
"""
- def __init__(self, base_cmd, test_env, output_checker=None, verbose=False):
+ def __init__(self, base_cmd, test_env, expected_retcode=None,
+ output_checker=None, verbose=False):
"""Constructor.
Args:
base_cmd: list of strings, base command to run.
test_env: ITestEnv.
+ expected_retcode: RetCode, expected normalized return code.
output_checker: IOutputCheck, output checker.
verbose: bool, enable verbose output.
"""
self._base_cmd = base_cmd
self._test_env = test_env
+ self._expected_retcode = expected_retcode
self._output_checker = output_checker
self._compiled_methods_path = self._test_env.CreateFile('compiled_methods')
self._passes_to_run_path = self._test_env.CreateFile('run_passes')
self._verbose = verbose
+ if RAW_CMD_RUNTIME_ARGS_TAG in self._base_cmd:
+ self._arguments_position = self._base_cmd.index(RAW_CMD_RUNTIME_ARGS_TAG)
+ self._base_cmd.pop(self._arguments_position)
+ else:
+ self._arguments_position = 1
def Test(self, compiled_methods, passes_to_run=None):
"""Tests compilation with compiled_methods and run_passes switches active.
@@ -95,8 +108,11 @@
verbose_compiler=False)
(output, ret_code) = self._test_env.RunCommand(
cmd, {'ANDROID_LOG_TAGS': '*:e'})
- res = ((self._output_checker is None and ret_code == 0)
- or self._output_checker.Check(output))
+ res = True
+ if self._expected_retcode:
+ res = self._expected_retcode == ret_code
+ if self._output_checker:
+ res = res and self._output_checker.Check(output)
if self._verbose:
print('Test passed: {0}.'.format(res))
return res
@@ -142,8 +158,8 @@
def _PrepareCmd(self, compiled_methods=None, passes_to_run=None,
verbose_compiler=False):
"""Prepare command to run."""
- cmd = [self._base_cmd[0]]
- # insert additional arguments
+ cmd = self._base_cmd[0:self._arguments_position]
+ # insert additional arguments before the first argument
if compiled_methods is not None:
self._test_env.WriteLines(self._compiled_methods_path, compiled_methods)
cmd += ['-Xcompiler-option', '--compiled-methods={0}'.format(
@@ -155,7 +171,7 @@
if verbose_compiler:
cmd += ['-Xcompiler-option', '--runtime-arg', '-Xcompiler-option',
'-verbose:compiler', '-Xcompiler-option', '-j1']
- cmd += self._base_cmd[1:]
+ cmd += self._base_cmd[self._arguments_position:]
return cmd
@@ -299,7 +315,7 @@
command_opts.add_argument('-cp', '--classpath', type=str, help='classpath')
command_opts.add_argument('--class', dest='classname', type=str,
help='name of main class')
- command_opts.add_argument('--lib', dest='lib', type=str, default='libart.so',
+ command_opts.add_argument('--lib', type=str, default='libart.so',
help='lib to use, default: libart.so')
command_opts.add_argument('--dalvikvm-option', dest='dalvikvm_opts',
metavar='OPT', nargs='*', default=[],
@@ -307,7 +323,7 @@
command_opts.add_argument('--arg', dest='test_args', nargs='*', default=[],
metavar='ARG', help='argument passed to test')
command_opts.add_argument('--image', type=str, help='path to image')
- command_opts.add_argument('--raw-cmd', dest='raw_cmd', type=str,
+ command_opts.add_argument('--raw-cmd', type=str,
help='bisect with this command, ignore other '
'command options')
bisection_opts = parser.add_argument_group('bisection options')
@@ -315,11 +331,22 @@
default=False, help='x64 mode')
bisection_opts.add_argument(
'--device', action='store_true', default=False, help='run on device')
+ bisection_opts.add_argument(
+ '--device-serial', help='device serial number, implies --device')
bisection_opts.add_argument('--expected-output', type=str,
help='file containing expected output')
bisection_opts.add_argument(
- '--check-script', dest='check_script', type=str,
+ '--expected-retcode', type=str, help='expected normalized return code',
+ choices=[RetCode.SUCCESS.name, RetCode.TIMEOUT.name, RetCode.ERROR.name])
+ bisection_opts.add_argument(
+ '--check-script', type=str,
help='script comparing output and expected output')
+ bisection_opts.add_argument(
+ '--logfile', type=str, help='custom logfile location')
+ bisection_opts.add_argument('--cleanup', action='store_true',
+ default=False, help='clean up after bisecting')
+ bisection_opts.add_argument('--timeout', type=int, default=60,
+ help='if timeout seconds pass assume test failed')
bisection_opts.add_argument('--verbose', action='store_true',
default=False, help='enable verbose output')
return parser
@@ -351,15 +378,24 @@
args = parser.parse_args()
if not args.raw_cmd and (not args.classpath or not args.classname):
parser.error('Either --raw-cmd or both -cp and --class are required')
+ if args.device_serial:
+ args.device = True
+ if args.expected_retcode:
+ args.expected_retcode = RetCode[args.expected_retcode]
+ if not args.expected_retcode and not args.check_script:
+ args.expected_retcode = RetCode.SUCCESS
# Prepare environment
classpath = args.classpath
if args.device:
- test_env = DeviceTestEnv()
+ test_env = DeviceTestEnv(
+ 'bisection_search_', args.cleanup, args.logfile, args.timeout,
+ args.device_serial)
if classpath:
classpath = test_env.PushClasspath(classpath)
else:
- test_env = HostTestEnv(args.x64)
+ test_env = HostTestEnv(
+ 'bisection_search_', args.cleanup, args.logfile, args.timeout, args.x64)
base_cmd = PrepareBaseCommand(args, classpath)
output_checker = None
if args.expected_output:
@@ -372,11 +408,11 @@
# Perform the search
try:
- testable = Dex2OatWrapperTestable(base_cmd, test_env, output_checker,
- args.verbose)
+ testable = Dex2OatWrapperTestable(base_cmd, test_env, args.expected_retcode,
+ output_checker, args.verbose)
(method, opt_pass) = BugSearch(testable)
except Exception as e:
- print('Error. Refer to logfile: {0}'.format(test_env.logfile.name))
+ print('Error occurred.\nLogfile: {0}'.format(test_env.logfile.name))
test_env.logfile.write('Exception: {0}\n'.format(e))
raise
diff --git a/tools/bisection_search/common.py b/tools/bisection_search/common.py
index d5029bb..b69b606 100755
--- a/tools/bisection_search/common.py
+++ b/tools/bisection_search/common.py
@@ -18,7 +18,9 @@
import abc
import os
+import signal
import shlex
+import shutil
from subprocess import check_call
from subprocess import PIPE
@@ -29,6 +31,9 @@
from tempfile import mkdtemp
from tempfile import NamedTemporaryFile
+from enum import Enum
+from enum import unique
+
# Temporary directory path on device.
DEVICE_TMP_PATH = '/data/local/tmp'
@@ -36,6 +41,16 @@
DALVIK_CACHE_ARCHS = ['arm', 'arm64', 'x86', 'x86_64']
+@unique
+class RetCode(Enum):
+ """Enum representing normalized return codes."""
+ SUCCESS = 0
+ TIMEOUT = 1
+ ERROR = 2
+ NOTCOMPILED = 3
+ NOTRUN = 4
+
+
def GetEnvVariableOrError(variable_name):
"""Gets value of an environmental variable.
@@ -70,6 +85,37 @@
for arch in DALVIK_CACHE_ARCHS)
+def RunCommandForOutput(cmd, env, stdout, stderr, timeout=60):
+ """Runs command piping output to files, stderr or stdout.
+
+ Args:
+ cmd: list of strings, command to run.
+ env: shell environment to run the command with.
+ stdout: file handle or one of Subprocess.PIPE, Subprocess.STDOUT,
+ Subprocess.DEVNULL, see Popen.
+ stderr: file handle or one of Subprocess.PIPE, Subprocess.STDOUT,
+ Subprocess.DEVNULL, see Popen.
+ timeout: int, timeout in seconds.
+
+ Returns:
+ tuple (string, string, RetCode) stdout output, stderr output, normalized
+ return code.
+ """
+ proc = Popen(cmd, stdout=stdout, stderr=stderr, env=env,
+ universal_newlines=True, start_new_session=True)
+ try:
+ (output, stderr_output) = proc.communicate(timeout=timeout)
+ if proc.returncode == 0:
+ retcode = RetCode.SUCCESS
+ else:
+ retcode = RetCode.ERROR
+ except TimeoutExpired:
+ os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
+ (output, stderr_output) = proc.communicate()
+ retcode = RetCode.TIMEOUT
+ return (output, stderr_output, retcode)
+
+
def _RunCommandForOutputAndLog(cmd, env, logfile, timeout=60):
"""Runs command and logs its output. Returns the output.
@@ -77,28 +123,19 @@
cmd: list of strings, command to run.
env: shell environment to run the command with.
logfile: file handle to logfile.
- timeout: int, timeout in seconds
+ timeout: int, timeout in seconds.
Returns:
- tuple (string, string, int) stdout output, stderr output, return code.
+ tuple (string, string, RetCode) stdout output, stderr output, normalized
+ return code.
"""
- proc = Popen(cmd, stderr=STDOUT, stdout=PIPE, env=env,
- universal_newlines=True)
- timeouted = False
- try:
- (output, _) = proc.communicate(timeout=timeout)
- except TimeoutExpired:
- timeouted = True
- proc.kill()
- (output, _) = proc.communicate()
+ (output, _, retcode) = RunCommandForOutput(cmd, env, PIPE, STDOUT, timeout)
logfile.write('Command:\n{0}\n{1}\nReturn code: {2}\n'.format(
- _CommandListToCommandString(cmd), output,
- 'TIMEOUT' if timeouted else proc.returncode))
- ret_code = 1 if timeouted else proc.returncode
- return (output, ret_code)
+ CommandListToCommandString(cmd), output, retcode))
+ return (output, retcode)
-def _CommandListToCommandString(cmd):
+def CommandListToCommandString(cmd):
"""Converts shell command represented as list of strings to a single string.
Each element of the list is wrapped in double quotes.
@@ -109,7 +146,7 @@
Returns:
string, shell command.
"""
- return ' '.join(['"{0}"'.format(segment) for segment in cmd])
+ return ' '.join([shlex.quote(segment) for segment in cmd])
class FatalError(Exception):
@@ -175,14 +212,24 @@
For methods documentation see base class.
"""
- def __init__(self, x64):
+ def __init__(self, directory_prefix, cleanup=True, logfile_path=None,
+ timeout=60, x64=False):
"""Constructor.
Args:
+ directory_prefix: string, prefix for environment directory name.
+ cleanup: boolean, if True remove test directory in destructor.
+ logfile_path: string, can be used to specify custom logfile location.
+ timeout: int, seconds, time to wait for single test run to finish.
x64: boolean, whether to setup in x64 mode.
"""
- self._env_path = mkdtemp(dir='/tmp/', prefix='bisection_search_')
- self._logfile = open('{0}/log'.format(self._env_path), 'w+')
+ self._cleanup = cleanup
+ self._timeout = timeout
+ self._env_path = mkdtemp(dir='/tmp/', prefix=directory_prefix)
+ if logfile_path is None:
+ self._logfile = open('{0}/log'.format(self._env_path), 'w+')
+ else:
+ self._logfile = open(logfile_path, 'w+')
os.mkdir('{0}/dalvik-cache'.format(self._env_path))
for arch_cache_path in _DexArchCachePaths(self._env_path):
os.mkdir(arch_cache_path)
@@ -199,6 +246,10 @@
# Using dlopen requires load bias on the host.
self._shell_env['LD_USE_LOAD_BIAS'] = '1'
+ def __del__(self):
+ if self._cleanup:
+ shutil.rmtree(self._env_path)
+
def CreateFile(self, name=None):
if name is None:
f = NamedTemporaryFile(dir=self._env_path, delete=False)
@@ -217,7 +268,7 @@
self._EmptyDexCache()
env = self._shell_env.copy()
env.update(env_updates)
- return _RunCommandForOutputAndLog(cmd, env, self._logfile)
+ return _RunCommandForOutputAndLog(cmd, env, self._logfile, self._timeout)
@property
def logfile(self):
@@ -239,16 +290,28 @@
class DeviceTestEnv(ITestEnv):
"""Device test environment. Concrete implementation of ITestEnv.
- Makes use of HostTestEnv to maintain a test directory on host. Creates an
- on device test directory which is kept in sync with the host one.
-
For methods documentation see base class.
"""
- def __init__(self):
- """Constructor."""
- self._host_env_path = mkdtemp(dir='/tmp/', prefix='bisection_search_')
- self._logfile = open('{0}/log'.format(self._host_env_path), 'w+')
+ def __init__(self, directory_prefix, cleanup=True, logfile_path=None,
+ timeout=60, specific_device=None):
+ """Constructor.
+
+ Args:
+ directory_prefix: string, prefix for environment directory name.
+ cleanup: boolean, if True remove test directory in destructor.
+ logfile_path: string, can be used to specify custom logfile location.
+ timeout: int, seconds, time to wait for single test run to finish.
+ specific_device: string, serial number of device to use.
+ """
+ self._cleanup = cleanup
+ self._timeout = timeout
+ self._specific_device = specific_device
+ self._host_env_path = mkdtemp(dir='/tmp/', prefix=directory_prefix)
+ if logfile_path is None:
+ self._logfile = open('{0}/log'.format(self._host_env_path), 'w+')
+ else:
+ self._logfile = open(logfile_path, 'w+')
self._device_env_path = '{0}/{1}'.format(
DEVICE_TMP_PATH, os.path.basename(self._host_env_path))
self._shell_env = os.environ.copy()
@@ -257,6 +320,13 @@
for arch_cache_path in _DexArchCachePaths(self._device_env_path):
self._AdbMkdir(arch_cache_path)
+ def __del__(self):
+ if self._cleanup:
+ shutil.rmtree(self._host_env_path)
+ check_call(shlex.split(
+ 'adb shell if [ -d "{0}" ]; then rm -rf "{0}"; fi'
+ .format(self._device_env_path)))
+
def CreateFile(self, name=None):
with NamedTemporaryFile(mode='w') as temp_file:
self._AdbPush(temp_file.name, self._device_env_path)
@@ -279,11 +349,18 @@
env_updates['ANDROID_DATA'] = self._device_env_path
env_updates_cmd = ' '.join(['{0}={1}'.format(var, val) for var, val
in env_updates.items()])
- cmd = _CommandListToCommandString(cmd)
- cmd = ('adb shell "logcat -c && {0} {1} ; logcat -d -s dex2oat:* dex2oatd:*'
- '| grep -v "^---------" 1>&2"').format(env_updates_cmd, cmd)
- return _RunCommandForOutputAndLog(
- shlex.split(cmd), self._shell_env, self._logfile)
+ cmd = CommandListToCommandString(cmd)
+ adb = 'adb'
+ if self._specific_device:
+ adb += ' -s ' + self._specific_device
+ cmd = '{0} shell "logcat -c && {1} {2}"'.format(
+ adb, env_updates_cmd, cmd)
+ (output, retcode) = _RunCommandForOutputAndLog(
+ shlex.split(cmd), self._shell_env, self._logfile, self._timeout)
+ logcat_cmd = 'adb shell "logcat -d -s -b main dex2oat:* dex2oatd:*"'
+ (err_output, _) = _RunCommandForOutputAndLog(
+ shlex.split(logcat_cmd), self._shell_env, self._logfile)
+ return (output + err_output, retcode)
@property
def logfile(self):