Prepare bisection search for runtest integration
Prepares bisection_search.py for run-test integration.
Introduces a --raw-cmd switch which accepts a full dalvikvm
command.
Fixes few bugs.
Test: ./art/tools/bisection-search/bisection_test.py
manual testing
Change-Id: I0272d21fe1e73278837bce76ad3eef632993fbe2
diff --git a/tools/bisection-search/README.md b/tools/bisection-search/README.md
index 857c930..a7485c2 100644
--- a/tools/bisection-search/README.md
+++ b/tools/bisection-search/README.md
@@ -15,29 +15,29 @@
How to run Bisection Bug Search
===============================
- bisection_search.py [-h] -cp CLASSPATH
- [--expected-output EXPECTED_OUTPUT] [--device]
- [--lib LIB] [--64]
- [--dalvikvm-option [OPTION [OPTION ...]]]
- [--arg [TEST_ARGS [TEST_ARGS ...]]] [--image IMAGE]
- [--verbose]
- classname
+ bisection_search.py [-h] [-cp CLASSPATH] [--class CLASSNAME] [--lib LIB]
+ [--dalvikvm-option [OPT [OPT ...]]] [--arg [ARG [ARG ...]]]
+ [--image IMAGE] [--raw-cmd RAW_CMD]
+ [--64] [--device] [--expected-output EXPECTED_OUTPUT]
+ [--check-script CHECK_SCRIPT] [--verbose]
- positional arguments:
- classname name of class to run
+ Tool for finding compiler bugs. Either --raw-cmd or both -cp and --class are required.
optional arguments:
- -h, --help show this help message and exit
- -cp CLASSPATH, --classpath CLASSPATH
- classpath
- --expected-output EXPECTED_OUTPUT
- file containing expected output
- --device run on device
- --lib LIB lib to use, default: libart.so
- --64 x64 mode
- --dalvikvm-option [OPTION [OPTION ...]]
- additional dalvikvm option
- --arg [TEST_ARGS [TEST_ARGS ...]]
- argument to pass to program
- --image IMAGE path to image
- --verbose enable verbose output
+ -h, --help show this help message and exit
+
+ dalvikvm command options:
+ -cp CLASSPATH, --classpath CLASSPATH classpath
+ --class CLASSNAME name of main class
+ --lib LIB lib to use, default: libart.so
+ --dalvikvm-option [OPT [OPT ...]] additional dalvikvm option
+ --arg [ARG [ARG ...]] argument passed to test
+ --image IMAGE path to image
+ --raw-cmd RAW_CMD bisect with this command, ignore other command options
+
+ bisection options:
+ --64 x64 mode
+ --device run on device
+ --expected-output EXPECTED_OUTPUT file containing expected output
+ --check-script CHECK_SCRIPT script comparing output and expected output
+ --verbose enable verbose output
diff --git a/tools/bisection-search/bisection_search.py b/tools/bisection-search/bisection_search.py
index d6c1749..110ef82 100755
--- a/tools/bisection-search/bisection_search.py
+++ b/tools/bisection-search/bisection_search.py
@@ -22,15 +22,20 @@
./bisection-search.py -cp classes.dex --expected-output output Test
"""
+import abc
import argparse
import re
+import shlex
+from subprocess import call
import sys
+from tempfile import NamedTemporaryFile
from common import DeviceTestEnv
from common import FatalError
from common import GetEnvVariableOrError
from common import HostTestEnv
+
# Passes that are never disabled during search process because disabling them
# would compromise correctness.
MANDATORY_PASSES = ['dex_cache_array_fixups_arm',
@@ -53,23 +58,18 @@
Accepts filters on compiled methods and optimization passes.
"""
- def __init__(self, base_cmd, test_env, class_name, args,
- expected_output=None, verbose=False):
+ def __init__(self, base_cmd, test_env, output_checker=None, verbose=False):
"""Constructor.
Args:
base_cmd: list of strings, base command to run.
test_env: ITestEnv.
- class_name: string, name of class to run.
- args: list of strings, program arguments to pass.
- expected_output: string, expected output to compare against or None.
+ output_checker: IOutputCheck, output checker.
verbose: bool, enable verbose output.
"""
self._base_cmd = base_cmd
self._test_env = test_env
- self._class_name = class_name
- self._args = args
- self._expected_output = expected_output
+ self._output_checker = output_checker
self._compiled_methods_path = self._test_env.CreateFile('compiled_methods')
self._passes_to_run_path = self._test_env.CreateFile('run_passes')
self._verbose = verbose
@@ -88,14 +88,15 @@
True if test passes with given settings. False otherwise.
"""
if self._verbose:
- print('Testing methods: {0} passes:{1}.'.format(
+ print('Testing methods: {0} passes: {1}.'.format(
compiled_methods, passes_to_run))
cmd = self._PrepareCmd(compiled_methods=compiled_methods,
passes_to_run=passes_to_run,
- verbose_compiler=True)
- (output, _, ret_code) = self._test_env.RunCommand(cmd)
- res = ret_code == 0 and (self._expected_output is None
- or output == self._expected_output)
+ verbose_compiler=False)
+ (output, ret_code) = self._test_env.RunCommand(
+ cmd, {'ANDROID_LOG_TAGS': '*:e'})
+ res = ((self._output_checker is None and ret_code == 0)
+ or self._output_checker.Check(output))
if self._verbose:
print('Test passed: {0}.'.format(res))
return res
@@ -110,8 +111,8 @@
FatalError: An error occurred when retrieving methods list.
"""
cmd = self._PrepareCmd(verbose_compiler=True)
- (_, err_output, _) = self._test_env.RunCommand(cmd)
- match_methods = re.findall(r'Building ([^\n]+)\n', err_output)
+ (output, _) = self._test_env.RunCommand(cmd, {'ANDROID_LOG_TAGS': '*:i'})
+ match_methods = re.findall(r'Building ([^\n]+)\n', output)
if not match_methods:
raise FatalError('Failed to retrieve methods list. '
'Not recognized output format.')
@@ -131,8 +132,8 @@
"""
cmd = self._PrepareCmd(compiled_methods=[compiled_method],
verbose_compiler=True)
- (_, err_output, _) = self._test_env.RunCommand(cmd)
- match_passes = re.findall(r'Starting pass: ([^\n]+)\n', err_output)
+ (output, _) = self._test_env.RunCommand(cmd, {'ANDROID_LOG_TAGS': '*:i'})
+ match_passes = re.findall(r'Starting pass: ([^\n]+)\n', output)
if not match_passes:
raise FatalError('Failed to retrieve passes list. '
'Not recognized output format.')
@@ -141,7 +142,8 @@
def _PrepareCmd(self, compiled_methods=None, passes_to_run=None,
verbose_compiler=False):
"""Prepare command to run."""
- cmd = list(self._base_cmd)
+ cmd = [self._base_cmd[0]]
+ # insert additional arguments
if compiled_methods is not None:
self._test_env.WriteLines(self._compiled_methods_path, compiled_methods)
cmd += ['-Xcompiler-option', '--compiled-methods={0}'.format(
@@ -152,12 +154,78 @@
self._passes_to_run_path)]
if verbose_compiler:
cmd += ['-Xcompiler-option', '--runtime-arg', '-Xcompiler-option',
- '-verbose:compiler']
- cmd += ['-classpath', self._test_env.classpath, self._class_name]
- cmd += self._args
+ '-verbose:compiler', '-Xcompiler-option', '-j1']
+ cmd += self._base_cmd[1:]
return cmd
+class IOutputCheck(object):
+ """Abstract output checking class.
+
+ Checks if output is correct.
+ """
+ __meta_class__ = abc.ABCMeta
+
+ @abc.abstractmethod
+ def Check(self, output):
+ """Check if output is correct.
+
+ Args:
+ output: string, output to check.
+
+ Returns:
+ boolean, True if output is correct, False otherwise.
+ """
+
+
+class EqualsOutputCheck(IOutputCheck):
+ """Concrete output checking class checking for equality to expected output."""
+
+ def __init__(self, expected_output):
+ """Constructor.
+
+ Args:
+ expected_output: string, expected output.
+ """
+ self._expected_output = expected_output
+
+ def Check(self, output):
+ """See base class."""
+ return self._expected_output == output
+
+
+class ExternalScriptOutputCheck(IOutputCheck):
+ """Concrete output checking class calling an external script.
+
+ The script should accept two arguments, path to expected output and path to
+ program output. It should exit with 0 return code if outputs are equivalent
+ and with different return code otherwise.
+ """
+
+ def __init__(self, script_path, expected_output_path, logfile):
+ """Constructor.
+
+ Args:
+ script_path: string, path to checking script.
+ expected_output_path: string, path to file with expected output.
+ logfile: file handle, logfile.
+ """
+ self._script_path = script_path
+ self._expected_output_path = expected_output_path
+ self._logfile = logfile
+
+ def Check(self, output):
+ """See base class."""
+ ret_code = None
+ with NamedTemporaryFile(mode='w', delete=False) as temp_file:
+ temp_file.write(output)
+ temp_file.flush()
+ ret_code = call(
+ [self._script_path, self._expected_output_path, temp_file.name],
+ stdout=self._logfile, stderr=self._logfile, universal_newlines=True)
+ return ret_code == 0
+
+
def BinarySearch(start, end, test):
"""Binary search integers using test function to guide the process."""
while start < end:
@@ -200,9 +268,9 @@
all_methods = testable.GetAllMethods()
faulty_method_idx = BinarySearch(
0,
- len(all_methods),
+ len(all_methods) + 1,
lambda mid: testable.Test(all_methods[0:mid]))
- if faulty_method_idx == len(all_methods):
+ if faulty_method_idx == len(all_methods) + 1:
return (None, None)
if faulty_method_idx == 0:
raise FatalError('Testable fails with no methods compiled. '
@@ -211,72 +279,100 @@
all_passes = testable.GetAllPassesForMethod(faulty_method)
faulty_pass_idx = BinarySearch(
0,
- len(all_passes),
+ len(all_passes) + 1,
lambda mid: testable.Test([faulty_method],
FilterPasses(all_passes, mid)))
if faulty_pass_idx == 0:
return (faulty_method, None)
- assert faulty_pass_idx != len(all_passes), 'Method must fail for some passes.'
+ assert faulty_pass_idx != len(all_passes) + 1, ('Method must fail for some '
+ 'passes.')
faulty_pass = all_passes[faulty_pass_idx - 1]
return (faulty_method, faulty_pass)
def PrepareParser():
"""Prepares argument parser."""
- parser = argparse.ArgumentParser()
- parser.add_argument(
- '-cp', '--classpath', required=True, type=str, help='classpath')
- parser.add_argument('--expected-output', type=str,
- help='file containing expected output')
- parser.add_argument(
+ parser = argparse.ArgumentParser(
+ description='Tool for finding compiler bugs. Either --raw-cmd or both '
+ '-cp and --class are required.')
+ command_opts = parser.add_argument_group('dalvikvm command options')
+ command_opts.add_argument('-cp', '--classpath', type=str, help='classpath')
+ command_opts.add_argument('--class', dest='classname', type=str,
+ help='name of main class')
+ command_opts.add_argument('--lib', dest='lib', type=str, default='libart.so',
+ help='lib to use, default: libart.so')
+ command_opts.add_argument('--dalvikvm-option', dest='dalvikvm_opts',
+ metavar='OPT', nargs='*', default=[],
+ help='additional dalvikvm option')
+ command_opts.add_argument('--arg', dest='test_args', nargs='*', default=[],
+ metavar='ARG', help='argument passed to test')
+ command_opts.add_argument('--image', type=str, help='path to image')
+ command_opts.add_argument('--raw-cmd', dest='raw_cmd', type=str,
+ help='bisect with this command, ignore other '
+ 'command options')
+ bisection_opts = parser.add_argument_group('bisection options')
+ bisection_opts.add_argument('--64', dest='x64', action='store_true',
+ default=False, help='x64 mode')
+ bisection_opts.add_argument(
'--device', action='store_true', default=False, help='run on device')
- parser.add_argument('classname', type=str, help='name of class to run')
- parser.add_argument('--lib', dest='lib', type=str, default='libart.so',
- help='lib to use, default: libart.so')
- parser.add_argument('--64', dest='x64', action='store_true',
- default=False, help='x64 mode')
- parser.add_argument('--dalvikvm-option', dest='dalvikvm_opts',
- metavar='OPTION', nargs='*', default=[],
- help='additional dalvikvm option')
- parser.add_argument('--arg', dest='test_args', nargs='*', default=[],
- help='argument to pass to program')
- parser.add_argument('--image', type=str, help='path to image')
- parser.add_argument('--verbose', action='store_true',
- default=False, help='enable verbose output')
+ bisection_opts.add_argument('--expected-output', type=str,
+ help='file containing expected output')
+ bisection_opts.add_argument(
+ '--check-script', dest='check_script', type=str,
+ help='script comparing output and expected output')
+ bisection_opts.add_argument('--verbose', action='store_true',
+ default=False, help='enable verbose output')
return parser
+def PrepareBaseCommand(args, classpath):
+ """Prepares base command used to run test."""
+ if args.raw_cmd:
+ return shlex.split(args.raw_cmd)
+ else:
+ base_cmd = ['dalvikvm64'] if args.x64 else ['dalvikvm32']
+ if not args.device:
+ base_cmd += ['-XXlib:{0}'.format(args.lib)]
+ if not args.image:
+ image_path = '{0}/framework/core-optimizing-pic.art'.format(
+ GetEnvVariableOrError('ANDROID_HOST_OUT'))
+ else:
+ image_path = args.image
+ base_cmd += ['-Ximage:{0}'.format(image_path)]
+ if args.dalvikvm_opts:
+ base_cmd += args.dalvikvm_opts
+ base_cmd += ['-cp', classpath, args.classname] + args.test_args
+ return base_cmd
+
+
def main():
# Parse arguments
parser = PrepareParser()
args = parser.parse_args()
+ if not args.raw_cmd and (not args.classpath or not args.classname):
+ parser.error('Either --raw-cmd or both -cp and --class are required')
# Prepare environment
- if args.expected_output is not None:
- with open(args.expected_output, 'r') as f:
- expected_output = f.read()
- else:
- expected_output = None
+ classpath = args.classpath
if args.device:
- run_cmd = ['dalvikvm64'] if args.x64 else ['dalvikvm32']
- test_env = DeviceTestEnv(args.classpath)
+ test_env = DeviceTestEnv()
+ if classpath:
+ classpath = test_env.PushClasspath(classpath)
else:
- run_cmd = ['dalvikvm64'] if args.x64 else ['dalvikvm32']
- run_cmd += ['-XXlib:{0}'.format(args.lib)]
- if not args.image:
- image_path = '{0}/framework/core-optimizing-pic.art'.format(
- GetEnvVariableOrError('ANDROID_HOST_OUT'))
+ test_env = HostTestEnv(args.x64)
+ base_cmd = PrepareBaseCommand(args, classpath)
+ output_checker = None
+ if args.expected_output:
+ if args.check_script:
+ output_checker = ExternalScriptOutputCheck(
+ args.check_script, args.expected_output, test_env.logfile)
else:
- image_path = args.image
- run_cmd += ['-Ximage:{0}'.format(image_path)]
- if args.dalvikvm_opts:
- run_cmd += args.dalvikvm_opts
- test_env = HostTestEnv(args.classpath, args.x64)
+ with open(args.expected_output, 'r') as expected_output_file:
+ output_checker = EqualsOutputCheck(expected_output_file.read())
# Perform the search
try:
- testable = Dex2OatWrapperTestable(run_cmd, test_env, args.classname,
- args.test_args, expected_output,
+ testable = Dex2OatWrapperTestable(base_cmd, test_env, output_checker,
args.verbose)
(method, opt_pass) = BugSearch(testable)
except Exception as e:
diff --git a/tools/bisection-search/common.py b/tools/bisection-search/common.py
index 8361fc9..d5029bb 100755
--- a/tools/bisection-search/common.py
+++ b/tools/bisection-search/common.py
@@ -23,6 +23,7 @@
from subprocess import check_call
from subprocess import PIPE
from subprocess import Popen
+from subprocess import STDOUT
from subprocess import TimeoutExpired
from tempfile import mkdtemp
@@ -81,19 +82,20 @@
Returns:
tuple (string, string, int) stdout output, stderr output, return code.
"""
- proc = Popen(cmd, stderr=PIPE, stdout=PIPE, env=env, universal_newlines=True)
+ proc = Popen(cmd, stderr=STDOUT, stdout=PIPE, env=env,
+ universal_newlines=True)
timeouted = False
try:
- (output, err_output) = proc.communicate(timeout=timeout)
+ (output, _) = proc.communicate(timeout=timeout)
except TimeoutExpired:
timeouted = True
proc.kill()
- (output, err_output) = proc.communicate()
- logfile.write('Command:\n{0}\n{1}{2}\nReturn code: {3}\n'.format(
- _CommandListToCommandString(cmd), err_output, output,
+ (output, _) = proc.communicate()
+ logfile.write('Command:\n{0}\n{1}\nReturn code: {2}\n'.format(
+ _CommandListToCommandString(cmd), output,
'TIMEOUT' if timeouted else proc.returncode))
ret_code = 1 if timeouted else proc.returncode
- return (output, err_output, ret_code)
+ return (output, ret_code)
def _CommandListToCommandString(cmd):
@@ -148,21 +150,18 @@
"""
@abc.abstractmethod
- def RunCommand(self, cmd):
- """Runs command in environment.
+ def RunCommand(self, cmd, env_updates=None):
+ """Runs command in environment with updated environmental variables.
Args:
- cmd: string, command to run.
-
+ cmd: list of strings, command to run.
+ env_updates: dict, string to string, maps names of variables to their
+ updated values.
Returns:
tuple (string, string, int) stdout output, stderr output, return code.
"""
@abc.abstractproperty
- def classpath(self):
- """Gets environment specific classpath with test class."""
-
- @abc.abstractproperty
def logfile(self):
"""Gets file handle to logfile residing on host."""
@@ -176,14 +175,12 @@
For methods documentation see base class.
"""
- def __init__(self, classpath, x64):
+ def __init__(self, x64):
"""Constructor.
Args:
- classpath: string, classpath with test class.
x64: boolean, whether to setup in x64 mode.
"""
- self._classpath = classpath
self._env_path = mkdtemp(dir='/tmp/', prefix='bisection_search_')
self._logfile = open('{0}/log'.format(self._env_path), 'w+')
os.mkdir('{0}/dalvik-cache'.format(self._env_path))
@@ -197,6 +194,7 @@
self._shell_env['ANDROID_DATA'] = self._env_path
self._shell_env['ANDROID_ROOT'] = android_root
self._shell_env['LD_LIBRARY_PATH'] = library_path
+ self._shell_env['DYLD_LIBRARY_PATH'] = library_path
self._shell_env['PATH'] = (path + ':' + self._shell_env['PATH'])
# Using dlopen requires load bias on the host.
self._shell_env['LD_USE_LOAD_BIAS'] = '1'
@@ -213,13 +211,13 @@
f.writelines('{0}\n'.format(line) for line in lines)
return
- def RunCommand(self, cmd):
+ def RunCommand(self, cmd, env_updates=None):
+ if not env_updates:
+ env_updates = {}
self._EmptyDexCache()
- return _RunCommandForOutputAndLog(cmd, self._shell_env, self._logfile)
-
- @property
- def classpath(self):
- return self._classpath
+ env = self._shell_env.copy()
+ env.update(env_updates)
+ return _RunCommandForOutputAndLog(cmd, env, self._logfile)
@property
def logfile(self):
@@ -247,32 +245,18 @@
For methods documentation see base class.
"""
- def __init__(self, classpath):
- """Constructor.
-
- Args:
- classpath: string, classpath with test class.
- """
+ def __init__(self):
+ """Constructor."""
self._host_env_path = mkdtemp(dir='/tmp/', prefix='bisection_search_')
self._logfile = open('{0}/log'.format(self._host_env_path), 'w+')
self._device_env_path = '{0}/{1}'.format(
DEVICE_TMP_PATH, os.path.basename(self._host_env_path))
- self._classpath = os.path.join(
- self._device_env_path, os.path.basename(classpath))
- self._shell_env = os.environ
+ self._shell_env = os.environ.copy()
self._AdbMkdir('{0}/dalvik-cache'.format(self._device_env_path))
for arch_cache_path in _DexArchCachePaths(self._device_env_path):
self._AdbMkdir(arch_cache_path)
- paths = classpath.split(':')
- device_paths = []
- for path in paths:
- device_paths.append('{0}/{1}'.format(
- self._device_env_path, os.path.basename(path)))
- self._AdbPush(path, self._device_env_path)
- self._classpath = ':'.join(device_paths)
-
def CreateFile(self, name=None):
with NamedTemporaryFile(mode='w') as temp_file:
self._AdbPush(temp_file.name, self._device_env_path)
@@ -283,25 +267,47 @@
def WriteLines(self, file_path, lines):
with NamedTemporaryFile(mode='w') as temp_file:
temp_file.writelines('{0}\n'.format(line) for line in lines)
+ temp_file.flush()
self._AdbPush(temp_file.name, file_path)
return
- def RunCommand(self, cmd):
+ def RunCommand(self, cmd, env_updates=None):
+ if not env_updates:
+ env_updates = {}
self._EmptyDexCache()
+ if 'ANDROID_DATA' not in env_updates:
+ env_updates['ANDROID_DATA'] = self._device_env_path
+ env_updates_cmd = ' '.join(['{0}={1}'.format(var, val) for var, val
+ in env_updates.items()])
cmd = _CommandListToCommandString(cmd)
- cmd = ('adb shell "logcat -c && ANDROID_DATA={0} {1} && '
- 'logcat -d dex2oat:* *:S 1>&2"').format(self._device_env_path, cmd)
- return _RunCommandForOutputAndLog(shlex.split(cmd), self._shell_env,
- self._logfile)
-
- @property
- def classpath(self):
- return self._classpath
+ cmd = ('adb shell "logcat -c && {0} {1} ; logcat -d -s dex2oat:* dex2oatd:*'
+ '| grep -v "^---------" 1>&2"').format(env_updates_cmd, cmd)
+ return _RunCommandForOutputAndLog(
+ shlex.split(cmd), self._shell_env, self._logfile)
@property
def logfile(self):
return self._logfile
+ def PushClasspath(self, classpath):
+ """Push classpath to on-device test directory.
+
+ Classpath can contain multiple colon separated file paths, each file is
+ pushed. Returns analogous classpath with paths valid on device.
+
+ Args:
+ classpath: string, classpath in format 'a/b/c:d/e/f'.
+ Returns:
+ string, classpath valid on device.
+ """
+ paths = classpath.split(':')
+ device_paths = []
+ for path in paths:
+ device_paths.append('{0}/{1}'.format(
+ self._device_env_path, os.path.basename(path)))
+ self._AdbPush(path, self._device_env_path)
+ return ':'.join(device_paths)
+
def _AdbPush(self, what, where):
check_call(shlex.split('adb push "{0}" "{1}"'.format(what, where)),
stdout=self._logfile, stderr=self._logfile)