Integrate bisection search with javafuzz
This CL makes javafuzz call bisection search on failing tests.
Three switches were added to bisection_search, --logfile which can be
used to provide custom logfile destination, --never-clean which
disables automatic cleanup of bisection directory and --timeout
which allows user to specify maximum time in seconds to wait for
a single test run.
ITestEnv subclasses were updated to integrate with javafuzz.
run_java_fuzz_test.py is now reusing code from bisection_search
module. It also better matches python style guidelines.
Change-Id: Ie41653b045469f2ceb352fd35fb4099842bb5bc3
diff --git a/tools/bisection_search/bisection_search.py b/tools/bisection_search/bisection_search.py
index 110ef82..0d36aa4 100755
--- a/tools/bisection_search/bisection_search.py
+++ b/tools/bisection_search/bisection_search.py
@@ -34,6 +34,7 @@
from common import FatalError
from common import GetEnvVariableOrError
from common import HostTestEnv
+from common import RetCode
# Passes that are never disabled during search process because disabling them
@@ -51,6 +52,10 @@
NON_PASSES = ['builder', 'prepare_for_register_allocation',
'liveness', 'register']
+# If present in raw cmd, this tag will be replaced with runtime arguments
+# controlling the bisection search. Otherwise arguments will be placed on second
+# position in the command.
+RAW_CMD_RUNTIME_ARGS_TAG = '{ARGS}'
class Dex2OatWrapperTestable(object):
"""Class representing a testable compilation.
@@ -58,21 +63,29 @@
Accepts filters on compiled methods and optimization passes.
"""
- def __init__(self, base_cmd, test_env, output_checker=None, verbose=False):
+ def __init__(self, base_cmd, test_env, expected_retcode=None,
+ output_checker=None, verbose=False):
"""Constructor.
Args:
base_cmd: list of strings, base command to run.
test_env: ITestEnv.
+ expected_retcode: RetCode, expected normalized return code.
output_checker: IOutputCheck, output checker.
verbose: bool, enable verbose output.
"""
self._base_cmd = base_cmd
self._test_env = test_env
+ self._expected_retcode = expected_retcode
self._output_checker = output_checker
self._compiled_methods_path = self._test_env.CreateFile('compiled_methods')
self._passes_to_run_path = self._test_env.CreateFile('run_passes')
self._verbose = verbose
+ if RAW_CMD_RUNTIME_ARGS_TAG in self._base_cmd:
+ self._arguments_position = self._base_cmd.index(RAW_CMD_RUNTIME_ARGS_TAG)
+ self._base_cmd.pop(self._arguments_position)
+ else:
+ self._arguments_position = 1
def Test(self, compiled_methods, passes_to_run=None):
"""Tests compilation with compiled_methods and run_passes switches active.
@@ -95,8 +108,11 @@
verbose_compiler=False)
(output, ret_code) = self._test_env.RunCommand(
cmd, {'ANDROID_LOG_TAGS': '*:e'})
- res = ((self._output_checker is None and ret_code == 0)
- or self._output_checker.Check(output))
+ res = True
+ if self._expected_retcode:
+ res = self._expected_retcode == ret_code
+ if self._output_checker:
+ res = res and self._output_checker.Check(output)
if self._verbose:
print('Test passed: {0}.'.format(res))
return res
@@ -142,8 +158,8 @@
def _PrepareCmd(self, compiled_methods=None, passes_to_run=None,
verbose_compiler=False):
"""Prepare command to run."""
- cmd = [self._base_cmd[0]]
- # insert additional arguments
+ cmd = self._base_cmd[0:self._arguments_position]
+ # insert additional arguments before the first argument
if compiled_methods is not None:
self._test_env.WriteLines(self._compiled_methods_path, compiled_methods)
cmd += ['-Xcompiler-option', '--compiled-methods={0}'.format(
@@ -155,7 +171,7 @@
if verbose_compiler:
cmd += ['-Xcompiler-option', '--runtime-arg', '-Xcompiler-option',
'-verbose:compiler', '-Xcompiler-option', '-j1']
- cmd += self._base_cmd[1:]
+ cmd += self._base_cmd[self._arguments_position:]
return cmd
@@ -299,7 +315,7 @@
command_opts.add_argument('-cp', '--classpath', type=str, help='classpath')
command_opts.add_argument('--class', dest='classname', type=str,
help='name of main class')
- command_opts.add_argument('--lib', dest='lib', type=str, default='libart.so',
+ command_opts.add_argument('--lib', type=str, default='libart.so',
help='lib to use, default: libart.so')
command_opts.add_argument('--dalvikvm-option', dest='dalvikvm_opts',
metavar='OPT', nargs='*', default=[],
@@ -307,7 +323,7 @@
command_opts.add_argument('--arg', dest='test_args', nargs='*', default=[],
metavar='ARG', help='argument passed to test')
command_opts.add_argument('--image', type=str, help='path to image')
- command_opts.add_argument('--raw-cmd', dest='raw_cmd', type=str,
+ command_opts.add_argument('--raw-cmd', type=str,
help='bisect with this command, ignore other '
'command options')
bisection_opts = parser.add_argument_group('bisection options')
@@ -315,11 +331,22 @@
default=False, help='x64 mode')
bisection_opts.add_argument(
'--device', action='store_true', default=False, help='run on device')
+ bisection_opts.add_argument(
+ '--device-serial', help='device serial number, implies --device')
bisection_opts.add_argument('--expected-output', type=str,
help='file containing expected output')
bisection_opts.add_argument(
- '--check-script', dest='check_script', type=str,
+ '--expected-retcode', type=str, help='expected normalized return code',
+ choices=[RetCode.SUCCESS.name, RetCode.TIMEOUT.name, RetCode.ERROR.name])
+ bisection_opts.add_argument(
+ '--check-script', type=str,
help='script comparing output and expected output')
+ bisection_opts.add_argument(
+ '--logfile', type=str, help='custom logfile location')
+ bisection_opts.add_argument('--cleanup', action='store_true',
+ default=False, help='clean up after bisecting')
+ bisection_opts.add_argument('--timeout', type=int, default=60,
+ help='if timeout seconds pass assume test failed')
bisection_opts.add_argument('--verbose', action='store_true',
default=False, help='enable verbose output')
return parser
@@ -351,15 +378,24 @@
args = parser.parse_args()
if not args.raw_cmd and (not args.classpath or not args.classname):
parser.error('Either --raw-cmd or both -cp and --class are required')
+ if args.device_serial:
+ args.device = True
+ if args.expected_retcode:
+ args.expected_retcode = RetCode[args.expected_retcode]
+ if not args.expected_retcode and not args.check_script:
+ args.expected_retcode = RetCode.SUCCESS
# Prepare environment
classpath = args.classpath
if args.device:
- test_env = DeviceTestEnv()
+ test_env = DeviceTestEnv(
+ 'bisection_search_', args.cleanup, args.logfile, args.timeout,
+ args.device_serial)
if classpath:
classpath = test_env.PushClasspath(classpath)
else:
- test_env = HostTestEnv(args.x64)
+ test_env = HostTestEnv(
+ 'bisection_search_', args.cleanup, args.logfile, args.timeout, args.x64)
base_cmd = PrepareBaseCommand(args, classpath)
output_checker = None
if args.expected_output:
@@ -372,11 +408,11 @@
# Perform the search
try:
- testable = Dex2OatWrapperTestable(base_cmd, test_env, output_checker,
- args.verbose)
+ testable = Dex2OatWrapperTestable(base_cmd, test_env, args.expected_retcode,
+ output_checker, args.verbose)
(method, opt_pass) = BugSearch(testable)
except Exception as e:
- print('Error. Refer to logfile: {0}'.format(test_env.logfile.name))
+ print('Error occurred.\nLogfile: {0}'.format(test_env.logfile.name))
test_env.logfile.write('Exception: {0}\n'.format(e))
raise