diff options
| -rwxr-xr-x | tools/bisection_search/bisection_search.py | 22 | ||||
| -rw-r--r-- | tools/common/__init__.py | 17 | ||||
| -rwxr-xr-x | tools/common/common.py (renamed from tools/bisection_search/common.py) | 44 | ||||
| -rw-r--r-- | tools/javafuzz/README.md | 22 | ||||
| -rwxr-xr-x | tools/javafuzz/run_dex_fuzz_test.py | 139 | ||||
| -rwxr-xr-x | tools/javafuzz/run_java_fuzz_test.py | 96 |
6 files changed, 256 insertions, 84 deletions
diff --git a/tools/bisection_search/bisection_search.py b/tools/bisection_search/bisection_search.py index b7f190714e..c5971e6a41 100755 --- a/tools/bisection_search/bisection_search.py +++ b/tools/bisection_search/bisection_search.py @@ -24,18 +24,23 @@ Example usage: import abc import argparse +import os import re import shlex -from subprocess import call import sys + +from subprocess import call from tempfile import NamedTemporaryFile -from common import DeviceTestEnv -from common import FatalError -from common import GetEnvVariableOrError -from common import HostTestEnv -from common import LogSeverity -from common import RetCode +sys.path.append(os.path.dirname(os.path.dirname( + os.path.realpath(__file__)))) + +from common.common import DeviceTestEnv +from common.common import FatalError +from common.common import GetEnvVariableOrError +from common.common import HostTestEnv +from common.common import LogSeverity +from common.common import RetCode # Passes that are never disabled during search process because disabling them @@ -157,8 +162,7 @@ class Dex2OatWrapperTestable(object): 'Not recognized output format.') return [p for p in match_passes if p not in NON_PASSES] - def _PrepareCmd(self, compiled_methods=None, passes_to_run=None, - verbose_compiler=False): + def _PrepareCmd(self, compiled_methods=None, passes_to_run=None): """Prepare command to run.""" cmd = self._base_cmd[0:self._arguments_position] # insert additional arguments before the first argument diff --git a/tools/common/__init__.py b/tools/common/__init__.py new file mode 100644 index 0000000000..3955c712ab --- /dev/null +++ b/tools/common/__init__.py @@ -0,0 +1,17 @@ +# +# Copyright (C) 2016 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This file is intentionally left empty. It indicates that the directory is a Python package.
\ No newline at end of file diff --git a/tools/bisection_search/common.py b/tools/common/common.py index 3d92ee5a60..b822dcadb7 100755 --- a/tools/bisection_search/common.py +++ b/tools/common/common.py @@ -23,6 +23,10 @@ import shlex import shutil import time +from enum import Enum +from enum import unique + +from subprocess import DEVNULL from subprocess import check_call from subprocess import PIPE from subprocess import Popen @@ -32,9 +36,6 @@ from subprocess import TimeoutExpired from tempfile import mkdtemp from tempfile import NamedTemporaryFile -from enum import Enum -from enum import unique - # Temporary directory path on device. DEVICE_TMP_PATH = '/data/local/tmp' @@ -91,7 +92,7 @@ class LogSeverity(Enum): def __lt__(self, other): if self.__class__ is other.__class__: return self.value < other.value - return NotImplemented + return NotImplemented def GetEnvVariableOrError(variable_name): @@ -115,6 +116,14 @@ def GetEnvVariableOrError(variable_name): return top +def GetJackClassPath(): + """Returns Jack's classpath.""" + top = GetEnvVariableOrError('ANDROID_BUILD_TOP') + libdir = top + '/out/host/common/obj/JAVA_LIBRARIES' + return libdir + '/core-libart-hostdex_intermediates/classes.jack:' \ + + libdir + '/core-oj-hostdex_intermediates/classes.jack' + + def _DexArchCachePaths(android_data_path): """Returns paths to architecture specific caches. @@ -172,6 +181,33 @@ def _LogCmdOutput(logfile, cmd, output, retcode): CommandListToCommandString(cmd), output, retcode)) +def RunCommand(cmd, out, err, timeout=5): + """Executes a command, and returns its return code. + + Args: + cmd: list of strings, a command to execute + out: string, file name to open for stdout (or None) + err: string, file name to open for stderr (or None) + timeout: int, time out in seconds + Returns: + RetCode, return code of running command (forced RetCode.TIMEOUT + on timeout) + """ + devnull = DEVNULL + outf = devnull + if out is not None: + outf = open(out, mode='w') + errf = devnull + if err is not None: + errf = open(err, mode='w') + (_, _, retcode) = RunCommandForOutput(cmd, None, outf, errf, timeout) + if outf != devnull: + outf.close() + if errf != devnull: + errf.close() + return retcode + + def CommandListToCommandString(cmd): """Converts shell command represented as list of strings to a single string. diff --git a/tools/javafuzz/README.md b/tools/javafuzz/README.md index b08075a9d8..a70e4c117f 100644 --- a/tools/javafuzz/README.md +++ b/tools/javafuzz/README.md @@ -8,7 +8,7 @@ using the optimizing compiler, using an external reference implementation, or using various target architectures. Any difference between the outputs (**divergence**) may indicate a bug in one of the execution modes. -JavaFuzz can be combined with dexfuzz to get multi-layered fuzz testing. +JavaFuzz can be combined with DexFuzz to get multi-layered fuzz testing. How to run JavaFuzz =================== @@ -36,11 +36,11 @@ a fixed testing class named Test. So a typical test run looks as follows. jack -cp ${JACK_CLASSPATH} --output-dex . Test.java art -classpath classes.dex Test -How to start the JavaFuzz tests -=============================== +How to start JavaFuzz testing +============================= run_java_fuzz_test.py - [--num_tests=#TESTS] + [--num_tests=NUM_TESTS] [--device=DEVICE] [--mode1=MODE] [--mode2=MODE] @@ -56,6 +56,20 @@ where tint = Art interpreter on target topt = Art optimizing on target +How to start Java/DexFuzz testing (multi-layered) +================================================= + + run_dex_fuzz_test.py + [--num_tests=NUM_TESTS] + [--num_inputs=NUM_INPUTS] + [--device=DEVICE] + +where + + --num_tests : number of tests to run (10000 by default) + --num_inputs: number of JavaFuzz programs to generate + --device : target device serial number (passed to adb -s) + Background ========== diff --git a/tools/javafuzz/run_dex_fuzz_test.py b/tools/javafuzz/run_dex_fuzz_test.py new file mode 100755 index 0000000000..ff87aa41f0 --- /dev/null +++ b/tools/javafuzz/run_dex_fuzz_test.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3.4 +# +# Copyright (C) 2016 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import shutil +import sys + +from subprocess import check_call +from tempfile import mkdtemp + +sys.path.append(os.path.dirname(os.path.dirname( + os.path.realpath(__file__)))) + +from common.common import FatalError +from common.common import GetJackClassPath +from common.common import RetCode +from common.common import RunCommand + + +# +# Tester class. +# + + +class DexFuzzTester(object): + """Tester that feeds JavaFuzz programs into DexFuzz testing.""" + + def __init__(self, num_tests, num_inputs, device): + """Constructor for the tester. + + Args: + num_tests: int, number of tests to run + num_inputs: int, number of JavaFuzz programs to generate + device: string, target device serial number (or None) + """ + self._num_tests = num_tests + self._num_inputs = num_inputs + self._device = device + self._save_dir = None + self._results_dir = None + self._dexfuzz_dir = None + self._inputs_dir = None + + def __enter__(self): + """On entry, enters new temp directory after saving current directory. + + Raises: + FatalError: error when temp directory cannot be constructed + """ + self._save_dir = os.getcwd() + self._results_dir = mkdtemp(dir='/tmp/') + self._dexfuzz_dir = mkdtemp(dir=self._results_dir) + self._inputs_dir = mkdtemp(dir=self._dexfuzz_dir) + if self._results_dir is None or self._dexfuzz_dir is None or \ + self._inputs_dir is None: + raise FatalError('Cannot obtain temp directory') + os.chdir(self._dexfuzz_dir) + return self + + def __exit__(self, etype, evalue, etraceback): + """On exit, re-enters previously saved current directory and cleans up.""" + os.chdir(self._save_dir) + # TODO: detect divergences or shutil.rmtree(self._results_dir) + + def Run(self): + """Feeds JavaFuzz programs into DexFuzz testing.""" + print() + print('**\n**** JavaFuzz Testing\n**') + print() + print('#Tests :', self._num_tests) + print('Device :', self._device) + print('Directory :', self._results_dir) + print() + self.GenerateJavaFuzzPrograms() + self.RunDexFuzz() + + + def GenerateJavaFuzzPrograms(self): + """Generates JavaFuzzPrograms. + + Raises: + FatalError: error when generation fails + """ + os.chdir(self._inputs_dir) + for i in range(1, self._num_inputs + 1): + jack_args = ['-cp', GetJackClassPath(), '--output-dex', '.', 'Test.java'] + if RunCommand(['javafuzz'], out='Test.java', err=None) != RetCode.SUCCESS: + raise FatalError('Unexpected error while running JavaFuzz') + if RunCommand(['jack'] + jack_args, out=None, err='jackerr.txt', + timeout=30) != RetCode.SUCCESS: + raise FatalError('Unexpected error while running Jack') + shutil.move('Test.java', '../Test' + str(i) + '.java') + shutil.move('classes.dex', 'classes' + str(i) + '.dex') + os.unlink('jackerr.txt') + + def RunDexFuzz(self): + """Starts the DexFuzz testing.""" + os.chdir(self._dexfuzz_dir) + os.environ['ANDROID_DATA'] = self._dexfuzz_dir + dexfuzz_args = ['--inputs=' + self._inputs_dir, '--execute', + '--execute-class=Test', '--repeat=' + str(self._num_tests), + '--dump-output', '--interpreter', '--optimizing'] + if self._device is not None: + dexfuzz_args += ['--device=' + self._device, '--allarm'] + else: + dexfuzz_args += ['--host'] # Assume host otherwise. + check_call(['dexfuzz'] + dexfuzz_args) + # TODO: summarize findings. + + +def main(): + # Handle arguments. + parser = argparse.ArgumentParser() + parser.add_argument('--num_tests', default=10000, + type=int, help='number of tests to run') + parser.add_argument('--num_inputs', default=50, + type=int, help='number of JavaFuzz program to generate') + parser.add_argument('--device', help='target device serial number') + args = parser.parse_args() + # Run the DexFuzz tester. + with DexFuzzTester(args.num_tests, args.num_inputs, args.device) as fuzzer: + fuzzer.Run() + +if __name__ == '__main__': + main() diff --git a/tools/javafuzz/run_java_fuzz_test.py b/tools/javafuzz/run_java_fuzz_test.py index 51d00be373..6cf3e85684 100755 --- a/tools/javafuzz/run_java_fuzz_test.py +++ b/tools/javafuzz/run_java_fuzz_test.py @@ -17,69 +17,28 @@ import abc import argparse import filecmp - -from glob import glob - import os import shlex import shutil -import subprocess import sys +from glob import glob from tempfile import mkdtemp sys.path.append(os.path.dirname(os.path.dirname( os.path.realpath(__file__)))) -from bisection_search.common import RetCode -from bisection_search.common import CommandListToCommandString -from bisection_search.common import FatalError -from bisection_search.common import GetEnvVariableOrError -from bisection_search.common import RunCommandForOutput -from bisection_search.common import DeviceTestEnv +from common.common import RetCode +from common.common import CommandListToCommandString +from common.common import FatalError +from common.common import GetJackClassPath +from common.common import GetEnvVariableOrError +from common.common import RunCommand +from common.common import DeviceTestEnv # Return codes supported by bisection bug search. BISECTABLE_RET_CODES = (RetCode.SUCCESS, RetCode.ERROR, RetCode.TIMEOUT) -# -# Utility methods. -# - - -def RunCommand(cmd, out, err, timeout=5): - """Executes a command, and returns its return code. - - Args: - cmd: list of strings, a command to execute - out: string, file name to open for stdout (or None) - err: string, file name to open for stderr (or None) - timeout: int, time out in seconds - Returns: - RetCode, return code of running command (forced RetCode.TIMEOUT - on timeout) - """ - devnull = subprocess.DEVNULL - outf = devnull - if out is not None: - outf = open(out, mode='w') - errf = devnull - if err is not None: - errf = open(err, mode='w') - (_, _, retcode) = RunCommandForOutput(cmd, None, outf, errf, timeout) - if outf != devnull: - outf.close() - if errf != devnull: - errf.close() - return retcode - - -def GetJackClassPath(): - """Returns Jack's classpath.""" - top = GetEnvVariableOrError('ANDROID_BUILD_TOP') - libdir = top + '/out/host/common/obj/JAVA_LIBRARIES' - return libdir + '/core-libart-hostdex_intermediates/classes.jack:' \ - + libdir + '/core-oj-hostdex_intermediates/classes.jack' - def GetExecutionModeRunner(device, mode): """Returns a runner for the given execution mode. @@ -104,6 +63,7 @@ def GetExecutionModeRunner(device, mode): return TestRunnerArtOptOnTarget(device) raise FatalError('Unknown execution mode') + # # Execution mode classes. # @@ -335,7 +295,7 @@ class TestRunnerArtOptOnTarget(TestRunnerArtOnTarget): # -# Tester classes. +# Tester class. # @@ -356,7 +316,8 @@ class JavaFuzzTester(object): self._runner1 = GetExecutionModeRunner(device, mode1) self._runner2 = GetExecutionModeRunner(device, mode2) self._save_dir = None - self._tmp_dir = None + self._results_dir = None + self._javafuzz_dir = None # Statistics. self._test = 0 self._num_success = 0 @@ -373,16 +334,16 @@ class JavaFuzzTester(object): """ self._save_dir = os.getcwd() self._results_dir = mkdtemp(dir='/tmp/') - self._tmp_dir = mkdtemp(dir=self._results_dir) - if self._tmp_dir is None or self._results_dir is None: + self._javafuzz_dir = mkdtemp(dir=self._results_dir) + if self._results_dir is None or self._javafuzz_dir is None: raise FatalError('Cannot obtain temp directory') - os.chdir(self._tmp_dir) + os.chdir(self._javafuzz_dir) return self def __exit__(self, etype, evalue, etraceback): """On exit, re-enters previously saved current directory and cleans up.""" os.chdir(self._save_dir) - shutil.rmtree(self._tmp_dir) + shutil.rmtree(self._javafuzz_dir) if self._num_divergences == 0: shutil.rmtree(self._results_dir) @@ -408,12 +369,13 @@ class JavaFuzzTester(object): def ShowStats(self): """Shows current statistics (on same line) while tester is running.""" - print('\rTests:', self._test, \ - 'Success:', self._num_success, \ - 'Not-compiled:', self._num_not_compiled, \ - 'Not-run:', self._num_not_run, \ - 'Timed-out:', self._num_timed_out, \ - 'Divergences:', self._num_divergences, end='') + print('\rTests:', self._test, + 'Success:', self._num_success, + 'Not-compiled:', self._num_not_compiled, + 'Not-run:', self._num_not_run, + 'Timed-out:', self._num_timed_out, + 'Divergences:', self._num_divergences, + end='') sys.stdout.flush() def RunJavaFuzzTest(self): @@ -515,12 +477,12 @@ class JavaFuzzTester(object): def CleanupTest(self): """Cleans up after a single test run.""" - for file_name in os.listdir(self._tmp_dir): - file_path = os.path.join(self._tmp_dir, file_name) - if os.path.isfile(file_path): - os.unlink(file_path) - elif os.path.isdir(file_path): - shutil.rmtree(file_path) + for file_name in os.listdir(self._javafuzz_dir): + file_path = os.path.join(self._javafuzz_dir, file_name) + if os.path.isfile(file_path): + os.unlink(file_path) + elif os.path.isdir(file_path): + shutil.rmtree(file_path) def main(): |