Merge "Prepare bisection search for runtest integration"

3 files changed, 237 insertions, 135 deletions

diff --git a/tools/bisection-search/README.md b/tools/bisection-search/README.md
index 857c93075e..a7485c2bb5 100644
--- a/tools/bisection-search/README.md
+++ b/tools/bisection-search/README.md
@@ -15,29 +15,29 @@ incorrect output. Prints Mi and Pj.
 How to run Bisection Bug Search
 ===============================
 
-    bisection_search.py [-h] -cp CLASSPATH
-                        [--expected-output EXPECTED_OUTPUT] [--device]
-                        [--lib LIB] [--64]
-                        [--dalvikvm-option [OPTION [OPTION ...]]]
-                        [--arg [TEST_ARGS [TEST_ARGS ...]]] [--image IMAGE]
-                        [--verbose]
-                        classname
+    bisection_search.py [-h] [-cp CLASSPATH] [--class CLASSNAME] [--lib LIB]
+                               [--dalvikvm-option [OPT [OPT ...]]] [--arg [ARG [ARG ...]]]
+                               [--image IMAGE] [--raw-cmd RAW_CMD]
+                               [--64] [--device] [--expected-output EXPECTED_OUTPUT]
+                               [--check-script CHECK_SCRIPT] [--verbose]
 
-    positional arguments:
-      classname             name of class to run
+    Tool for finding compiler bugs. Either --raw-cmd or both -cp and --class are required.
 
     optional arguments:
-      -h, --help            show this help message and exit
-      -cp CLASSPATH, --classpath CLASSPATH
-                            classpath
-      --expected-output EXPECTED_OUTPUT
-                            file containing expected output
-      --device              run on device
-      --lib LIB             lib to use, default: libart.so
-      --64                  x64 mode
-      --dalvikvm-option [OPTION [OPTION ...]]
-                            additional dalvikvm option
-      --arg [TEST_ARGS [TEST_ARGS ...]]
-                            argument to pass to program
-      --image IMAGE         path to image
-      --verbose             enable verbose output
+      -h, --help                            show this help message and exit
+
+    dalvikvm command options:
+      -cp CLASSPATH, --classpath CLASSPATH  classpath
+      --class CLASSNAME                     name of main class
+      --lib LIB                             lib to use, default: libart.so
+      --dalvikvm-option [OPT [OPT ...]]     additional dalvikvm option
+      --arg [ARG [ARG ...]]                 argument passed to test
+      --image IMAGE                         path to image
+      --raw-cmd RAW_CMD                     bisect with this command, ignore other command options
+
+    bisection options:
+      --64                                  x64 mode
+      --device                              run on device
+      --expected-output EXPECTED_OUTPUT     file containing expected output
+      --check-script CHECK_SCRIPT           script comparing output and expected output
+      --verbose                             enable verbose output
diff --git a/tools/bisection-search/bisection_search.py b/tools/bisection-search/bisection_search.py
index d6c1749b60..110ef82433 100755
--- a/tools/bisection-search/bisection_search.py
+++ b/tools/bisection-search/bisection_search.py
@@ -22,15 +22,20 @@ Example usage:
 ./bisection-search.py -cp classes.dex --expected-output output Test
 """
 
+import abc
 import argparse
 import re
+import shlex
+from subprocess import call
 import sys
+from tempfile import NamedTemporaryFile
 
 from common import DeviceTestEnv
 from common import FatalError
 from common import GetEnvVariableOrError
 from common import HostTestEnv
 
+
 # Passes that are never disabled during search process because disabling them
 # would compromise correctness.
 MANDATORY_PASSES = ['dex_cache_array_fixups_arm',
@@ -53,23 +58,18 @@ class Dex2OatWrapperTestable(object):
   Accepts filters on compiled methods and optimization passes.
   """
 
-  def __init__(self, base_cmd, test_env, class_name, args,
-               expected_output=None, verbose=False):
+  def __init__(self, base_cmd, test_env, output_checker=None, verbose=False):
     """Constructor.
 
     Args:
       base_cmd: list of strings, base command to run.
       test_env: ITestEnv.
-      class_name: string, name of class to run.
-      args: list of strings, program arguments to pass.
-      expected_output: string, expected output to compare against or None.
+      output_checker: IOutputCheck, output checker.
       verbose: bool, enable verbose output.
     """
     self._base_cmd = base_cmd
     self._test_env = test_env
-    self._class_name = class_name
-    self._args = args
-    self._expected_output = expected_output
+    self._output_checker = output_checker
     self._compiled_methods_path = self._test_env.CreateFile('compiled_methods')
     self._passes_to_run_path = self._test_env.CreateFile('run_passes')
     self._verbose = verbose
@@ -88,14 +88,15 @@ class Dex2OatWrapperTestable(object):
       True if test passes with given settings. False otherwise.
     """
     if self._verbose:
-      print('Testing methods: {0} passes:{1}.'.format(
+      print('Testing methods: {0} passes: {1}.'.format(
           compiled_methods, passes_to_run))
     cmd = self._PrepareCmd(compiled_methods=compiled_methods,
                            passes_to_run=passes_to_run,
-                           verbose_compiler=True)
-    (output, _, ret_code) = self._test_env.RunCommand(cmd)
-    res = ret_code == 0 and (self._expected_output is None
-                             or output == self._expected_output)
+                           verbose_compiler=False)
+    (output, ret_code) = self._test_env.RunCommand(
+        cmd, {'ANDROID_LOG_TAGS': '*:e'})
+    res = ((self._output_checker is None and ret_code == 0)
+           or self._output_checker.Check(output))
     if self._verbose:
       print('Test passed: {0}.'.format(res))
     return res
@@ -110,8 +111,8 @@ class Dex2OatWrapperTestable(object):
       FatalError: An error occurred when retrieving methods list.
     """
     cmd = self._PrepareCmd(verbose_compiler=True)
-    (_, err_output, _) = self._test_env.RunCommand(cmd)
-    match_methods = re.findall(r'Building ([^\n]+)\n', err_output)
+    (output, _) = self._test_env.RunCommand(cmd, {'ANDROID_LOG_TAGS': '*:i'})
+    match_methods = re.findall(r'Building ([^\n]+)\n', output)
     if not match_methods:
       raise FatalError('Failed to retrieve methods list. '
                        'Not recognized output format.')
@@ -131,8 +132,8 @@ class Dex2OatWrapperTestable(object):
     """
     cmd = self._PrepareCmd(compiled_methods=[compiled_method],
                            verbose_compiler=True)
-    (_, err_output, _) = self._test_env.RunCommand(cmd)
-    match_passes = re.findall(r'Starting pass: ([^\n]+)\n', err_output)
+    (output, _) = self._test_env.RunCommand(cmd, {'ANDROID_LOG_TAGS': '*:i'})
+    match_passes = re.findall(r'Starting pass: ([^\n]+)\n', output)
     if not match_passes:
       raise FatalError('Failed to retrieve passes list. '
                        'Not recognized output format.')
@@ -141,7 +142,8 @@ class Dex2OatWrapperTestable(object):
   def _PrepareCmd(self, compiled_methods=None, passes_to_run=None,
                   verbose_compiler=False):
     """Prepare command to run."""
-    cmd = list(self._base_cmd)
+    cmd = [self._base_cmd[0]]
+    # insert additional arguments
     if compiled_methods is not None:
       self._test_env.WriteLines(self._compiled_methods_path, compiled_methods)
       cmd += ['-Xcompiler-option', '--compiled-methods={0}'.format(
@@ -152,12 +154,78 @@ class Dex2OatWrapperTestable(object):
           self._passes_to_run_path)]
     if verbose_compiler:
       cmd += ['-Xcompiler-option', '--runtime-arg', '-Xcompiler-option',
-              '-verbose:compiler']
-    cmd += ['-classpath', self._test_env.classpath, self._class_name]
-    cmd += self._args
+              '-verbose:compiler', '-Xcompiler-option', '-j1']
+    cmd += self._base_cmd[1:]
     return cmd
 
 
+class IOutputCheck(object):
+  """Abstract output checking class.
+
+  Checks if output is correct.
+  """
+  __meta_class__ = abc.ABCMeta
+
+  @abc.abstractmethod
+  def Check(self, output):
+    """Check if output is correct.
+
+    Args:
+      output: string, output to check.
+
+    Returns:
+      boolean, True if output is correct, False otherwise.
+    """
+
+
+class EqualsOutputCheck(IOutputCheck):
+  """Concrete output checking class checking for equality to expected output."""
+
+  def __init__(self, expected_output):
+    """Constructor.
+
+    Args:
+      expected_output: string, expected output.
+    """
+    self._expected_output = expected_output
+
+  def Check(self, output):
+    """See base class."""
+    return self._expected_output == output
+
+
+class ExternalScriptOutputCheck(IOutputCheck):
+  """Concrete output checking class calling an external script.
+
+  The script should accept two arguments, path to expected output and path to
+  program output. It should exit with 0 return code if outputs are equivalent
+  and with different return code otherwise.
+  """
+
+  def __init__(self, script_path, expected_output_path, logfile):
+    """Constructor.
+
+    Args:
+      script_path: string, path to checking script.
+      expected_output_path: string, path to file with expected output.
+      logfile: file handle, logfile.
+    """
+    self._script_path = script_path
+    self._expected_output_path = expected_output_path
+    self._logfile = logfile
+
+  def Check(self, output):
+    """See base class."""
+    ret_code = None
+    with NamedTemporaryFile(mode='w', delete=False) as temp_file:
+      temp_file.write(output)
+      temp_file.flush()
+      ret_code = call(
+          [self._script_path, self._expected_output_path, temp_file.name],
+          stdout=self._logfile, stderr=self._logfile, universal_newlines=True)
+    return ret_code == 0
+
+
 def BinarySearch(start, end, test):
   """Binary search integers using test function to guide the process."""
   while start < end:
@@ -200,9 +268,9 @@ def BugSearch(testable):
   all_methods = testable.GetAllMethods()
   faulty_method_idx = BinarySearch(
       0,
-      len(all_methods),
+      len(all_methods) + 1,
       lambda mid: testable.Test(all_methods[0:mid]))
-  if faulty_method_idx == len(all_methods):
+  if faulty_method_idx == len(all_methods) + 1:
     return (None, None)
   if faulty_method_idx == 0:
     raise FatalError('Testable fails with no methods compiled. '
@@ -211,72 +279,100 @@ def BugSearch(testable):
   all_passes = testable.GetAllPassesForMethod(faulty_method)
   faulty_pass_idx = BinarySearch(
       0,
-      len(all_passes),
+      len(all_passes) + 1,
       lambda mid: testable.Test([faulty_method],
                                 FilterPasses(all_passes, mid)))
   if faulty_pass_idx == 0:
     return (faulty_method, None)
-  assert faulty_pass_idx != len(all_passes), 'Method must fail for some passes.'
+  assert faulty_pass_idx != len(all_passes) + 1, ('Method must fail for some '
+                                                  'passes.')
   faulty_pass = all_passes[faulty_pass_idx - 1]
   return (faulty_method, faulty_pass)
 
 
 def PrepareParser():
   """Prepares argument parser."""
-  parser = argparse.ArgumentParser()
-  parser.add_argument(
-      '-cp', '--classpath', required=True, type=str, help='classpath')
-  parser.add_argument('--expected-output', type=str,
-                      help='file containing expected output')
-  parser.add_argument(
+  parser = argparse.ArgumentParser(
+      description='Tool for finding compiler bugs. Either --raw-cmd or both '
+                  '-cp and --class are required.')
+  command_opts = parser.add_argument_group('dalvikvm command options')
+  command_opts.add_argument('-cp', '--classpath', type=str, help='classpath')
+  command_opts.add_argument('--class', dest='classname', type=str,
+                            help='name of main class')
+  command_opts.add_argument('--lib', dest='lib', type=str, default='libart.so',
+                            help='lib to use, default: libart.so')
+  command_opts.add_argument('--dalvikvm-option', dest='dalvikvm_opts',
+                            metavar='OPT', nargs='*', default=[],
+                            help='additional dalvikvm option')
+  command_opts.add_argument('--arg', dest='test_args', nargs='*', default=[],
+                            metavar='ARG', help='argument passed to test')
+  command_opts.add_argument('--image', type=str, help='path to image')
+  command_opts.add_argument('--raw-cmd', dest='raw_cmd', type=str,
+                            help='bisect with this command, ignore other '
+                                 'command options')
+  bisection_opts = parser.add_argument_group('bisection options')
+  bisection_opts.add_argument('--64', dest='x64', action='store_true',
+                              default=False, help='x64 mode')
+  bisection_opts.add_argument(
       '--device', action='store_true', default=False, help='run on device')
-  parser.add_argument('classname', type=str, help='name of class to run')
-  parser.add_argument('--lib', dest='lib', type=str, default='libart.so',
-                      help='lib to use, default: libart.so')
-  parser.add_argument('--64', dest='x64', action='store_true',
-                      default=False, help='x64 mode')
-  parser.add_argument('--dalvikvm-option', dest='dalvikvm_opts',
-                      metavar='OPTION', nargs='*', default=[],
-                      help='additional dalvikvm option')
-  parser.add_argument('--arg', dest='test_args', nargs='*', default=[],
-                      help='argument to pass to program')
-  parser.add_argument('--image', type=str, help='path to image')
-  parser.add_argument('--verbose', action='store_true',
-                      default=False, help='enable verbose output')
+  bisection_opts.add_argument('--expected-output', type=str,
+                              help='file containing expected output')
+  bisection_opts.add_argument(
+      '--check-script', dest='check_script', type=str,
+      help='script comparing output and expected output')
+  bisection_opts.add_argument('--verbose', action='store_true',
+                              default=False, help='enable verbose output')
   return parser
 
 
+def PrepareBaseCommand(args, classpath):
+  """Prepares base command used to run test."""
+  if args.raw_cmd:
+    return shlex.split(args.raw_cmd)
+  else:
+    base_cmd = ['dalvikvm64'] if args.x64 else ['dalvikvm32']
+    if not args.device:
+      base_cmd += ['-XXlib:{0}'.format(args.lib)]
+      if not args.image:
+        image_path = '{0}/framework/core-optimizing-pic.art'.format(
+            GetEnvVariableOrError('ANDROID_HOST_OUT'))
+      else:
+        image_path = args.image
+      base_cmd += ['-Ximage:{0}'.format(image_path)]
+    if args.dalvikvm_opts:
+      base_cmd += args.dalvikvm_opts
+    base_cmd += ['-cp', classpath, args.classname] + args.test_args
+  return base_cmd
+
+
 def main():
   # Parse arguments
   parser = PrepareParser()
   args = parser.parse_args()
+  if not args.raw_cmd and (not args.classpath or not args.classname):
+    parser.error('Either --raw-cmd or both -cp and --class are required')
 
   # Prepare environment
-  if args.expected_output is not None:
-    with open(args.expected_output, 'r') as f:
-      expected_output = f.read()
-  else:
-    expected_output = None
+  classpath = args.classpath
   if args.device:
-    run_cmd = ['dalvikvm64'] if args.x64 else ['dalvikvm32']
-    test_env = DeviceTestEnv(args.classpath)
+    test_env = DeviceTestEnv()
+    if classpath:
+      classpath = test_env.PushClasspath(classpath)
   else:
-    run_cmd = ['dalvikvm64'] if args.x64 else ['dalvikvm32']
-    run_cmd += ['-XXlib:{0}'.format(args.lib)]
-    if not args.image:
-      image_path = '{0}/framework/core-optimizing-pic.art'.format(
-          GetEnvVariableOrError('ANDROID_HOST_OUT'))
+    test_env = HostTestEnv(args.x64)
+  base_cmd = PrepareBaseCommand(args, classpath)
+  output_checker = None
+  if args.expected_output:
+    if args.check_script:
+      output_checker = ExternalScriptOutputCheck(
+          args.check_script, args.expected_output, test_env.logfile)
     else:
-      image_path = args.image
-    run_cmd += ['-Ximage:{0}'.format(image_path)]
-    if args.dalvikvm_opts:
-      run_cmd += args.dalvikvm_opts
-    test_env = HostTestEnv(args.classpath, args.x64)
+      with open(args.expected_output, 'r') as expected_output_file:
+        output_checker = EqualsOutputCheck(expected_output_file.read())
 
   # Perform the search
   try:
-    testable = Dex2OatWrapperTestable(run_cmd, test_env, args.classname,
-                                      args.test_args, expected_output,
+    testable = Dex2OatWrapperTestable(base_cmd, test_env, output_checker,
                                       args.verbose)
     (method, opt_pass) = BugSearch(testable)
   except Exception as e:
diff --git a/tools/bisection-search/common.py b/tools/bisection-search/common.py
index 8361fc9e94..d5029bb970 100755
--- a/tools/bisection-search/common.py
+++ b/tools/bisection-search/common.py
@@ -23,6 +23,7 @@ import shlex
 from subprocess import check_call
 from subprocess import PIPE
 from subprocess import Popen
+from subprocess import STDOUT
 from subprocess import TimeoutExpired
 
 from tempfile import mkdtemp
@@ -81,19 +82,20 @@ def _RunCommandForOutputAndLog(cmd, env, logfile, timeout=60):
   Returns:
    tuple (string, string, int) stdout output, stderr output, return code.
   """
-  proc = Popen(cmd, stderr=PIPE, stdout=PIPE, env=env, universal_newlines=True)
+  proc = Popen(cmd, stderr=STDOUT, stdout=PIPE, env=env,
+               universal_newlines=True)
   timeouted = False
   try:
-    (output, err_output) = proc.communicate(timeout=timeout)
+    (output, _) = proc.communicate(timeout=timeout)
   except TimeoutExpired:
     timeouted = True
     proc.kill()
-    (output, err_output) = proc.communicate()
-  logfile.write('Command:\n{0}\n{1}{2}\nReturn code: {3}\n'.format(
-      _CommandListToCommandString(cmd), err_output, output,
+    (output, _) = proc.communicate()
+  logfile.write('Command:\n{0}\n{1}\nReturn code: {2}\n'.format(
+      _CommandListToCommandString(cmd), output,
       'TIMEOUT' if timeouted else proc.returncode))
   ret_code = 1 if timeouted else proc.returncode
-  return (output, err_output, ret_code)
+  return (output, ret_code)
 
 
 def _CommandListToCommandString(cmd):
@@ -148,21 +150,18 @@ class ITestEnv(object):
     """
 
   @abc.abstractmethod
-  def RunCommand(self, cmd):
-    """Runs command in environment.
+  def RunCommand(self, cmd, env_updates=None):
+    """Runs command in environment with updated environmental variables.
 
     Args:
-      cmd: string, command to run.
-
+      cmd: list of strings, command to run.
+      env_updates: dict, string to string, maps names of variables to their
+        updated values.
     Returns:
       tuple (string, string, int) stdout output, stderr output, return code.
     """
 
   @abc.abstractproperty
-  def classpath(self):
-    """Gets environment specific classpath with test class."""
-
-  @abc.abstractproperty
   def logfile(self):
     """Gets file handle to logfile residing on host."""
 
@@ -176,14 +175,12 @@ class HostTestEnv(ITestEnv):
   For methods documentation see base class.
   """
 
-  def __init__(self, classpath, x64):
+  def __init__(self, x64):
     """Constructor.
 
     Args:
-      classpath: string, classpath with test class.
       x64: boolean, whether to setup in x64 mode.
     """
-    self._classpath = classpath
     self._env_path = mkdtemp(dir='/tmp/', prefix='bisection_search_')
     self._logfile = open('{0}/log'.format(self._env_path), 'w+')
     os.mkdir('{0}/dalvik-cache'.format(self._env_path))
@@ -197,6 +194,7 @@ class HostTestEnv(ITestEnv):
     self._shell_env['ANDROID_DATA'] = self._env_path
     self._shell_env['ANDROID_ROOT'] = android_root
     self._shell_env['LD_LIBRARY_PATH'] = library_path
+    self._shell_env['DYLD_LIBRARY_PATH'] = library_path
     self._shell_env['PATH'] = (path + ':' + self._shell_env['PATH'])
     # Using dlopen requires load bias on the host.
     self._shell_env['LD_USE_LOAD_BIAS'] = '1'
@@ -213,13 +211,13 @@ class HostTestEnv(ITestEnv):
       f.writelines('{0}\n'.format(line) for line in lines)
     return
 
-  def RunCommand(self, cmd):
+  def RunCommand(self, cmd, env_updates=None):
+    if not env_updates:
+      env_updates = {}
     self._EmptyDexCache()
-    return _RunCommandForOutputAndLog(cmd, self._shell_env, self._logfile)
-
-  @property
-  def classpath(self):
-    return self._classpath
+    env = self._shell_env.copy()
+    env.update(env_updates)
+    return _RunCommandForOutputAndLog(cmd, env, self._logfile)
 
   @property
   def logfile(self):
@@ -247,32 +245,18 @@ class DeviceTestEnv(ITestEnv):
   For methods documentation see base class.
   """
 
-  def __init__(self, classpath):
-    """Constructor.
-
-    Args:
-      classpath: string, classpath with test class.
-    """
+  def __init__(self):
+    """Constructor."""
     self._host_env_path = mkdtemp(dir='/tmp/', prefix='bisection_search_')
     self._logfile = open('{0}/log'.format(self._host_env_path), 'w+')
     self._device_env_path = '{0}/{1}'.format(
         DEVICE_TMP_PATH, os.path.basename(self._host_env_path))
-    self._classpath = os.path.join(
-        self._device_env_path, os.path.basename(classpath))
-    self._shell_env = os.environ
+    self._shell_env = os.environ.copy()
 
     self._AdbMkdir('{0}/dalvik-cache'.format(self._device_env_path))
     for arch_cache_path in _DexArchCachePaths(self._device_env_path):
       self._AdbMkdir(arch_cache_path)
 
-    paths = classpath.split(':')
-    device_paths = []
-    for path in paths:
-      device_paths.append('{0}/{1}'.format(
-          self._device_env_path, os.path.basename(path)))
-      self._AdbPush(path, self._device_env_path)
-    self._classpath = ':'.join(device_paths)
-
   def CreateFile(self, name=None):
     with NamedTemporaryFile(mode='w') as temp_file:
       self._AdbPush(temp_file.name, self._device_env_path)
@@ -283,25 +267,47 @@ class DeviceTestEnv(ITestEnv):
   def WriteLines(self, file_path, lines):
     with NamedTemporaryFile(mode='w') as temp_file:
       temp_file.writelines('{0}\n'.format(line) for line in lines)
+      temp_file.flush()
       self._AdbPush(temp_file.name, file_path)
     return
 
-  def RunCommand(self, cmd):
+  def RunCommand(self, cmd, env_updates=None):
+    if not env_updates:
+      env_updates = {}
     self._EmptyDexCache()
+    if 'ANDROID_DATA' not in env_updates:
+      env_updates['ANDROID_DATA'] = self._device_env_path
+    env_updates_cmd = ' '.join(['{0}={1}'.format(var, val) for var, val
+                                in env_updates.items()])
     cmd = _CommandListToCommandString(cmd)
-    cmd = ('adb shell "logcat -c && ANDROID_DATA={0} {1} && '
-           'logcat -d dex2oat:* *:S 1>&2"').format(self._device_env_path, cmd)
-    return _RunCommandForOutputAndLog(shlex.split(cmd), self._shell_env,
-                                      self._logfile)
-
-  @property
-  def classpath(self):
-    return self._classpath
+    cmd = ('adb shell "logcat -c && {0} {1} ; logcat -d -s dex2oat:* dex2oatd:*'
+           '| grep -v "^---------" 1>&2"').format(env_updates_cmd, cmd)
+    return _RunCommandForOutputAndLog(
+        shlex.split(cmd), self._shell_env, self._logfile)
 
   @property
   def logfile(self):
     return self._logfile
 
+  def PushClasspath(self, classpath):
+    """Push classpath to on-device test directory.
+
+    Classpath can contain multiple colon separated file paths, each file is
+    pushed. Returns analogous classpath with paths valid on device.
+
+    Args:
+      classpath: string, classpath in format 'a/b/c:d/e/f'.
+    Returns:
+      string, classpath valid on device.
+    """
+    paths = classpath.split(':')
+    device_paths = []
+    for path in paths:
+      device_paths.append('{0}/{1}'.format(
+          self._device_env_path, os.path.basename(path)))
+      self._AdbPush(path, self._device_env_path)
+    return ':'.join(device_paths)
+
   def _AdbPush(self, what, where):
     check_call(shlex.split('adb push "{0}" "{1}"'.format(what, where)),
                stdout=self._logfile, stderr=self._logfile)