tests/DumpRenderTree/assets/run_layout_tests.py - LeafOS-Project/android_frameworks_base - Gitiles

 #!/usr/bin/python

 """Run layout tests using Android emulator and instrumentation.

   First, you need to get an SD card or sdcard image that has layout tests on it.
   Layout tests are in following directory:
     /sdcard/android/layout_tests
   For example, /sdcard/android/layout_tests/fast

   Usage:
     Run all tests under fast/ directory:
       run_layout_tests.py, or
       run_layout_tests.py fast

     Run all tests under a sub directory:
       run_layout_tests.py fast/dom

     Run a single test:
       run_layout_tests.py fast/dom/

   After a merge, if there are changes of layout tests in SD card, you need to
   use --refresh-test-list option *once* to re-generate test list on the card.

   Some other options are:
     --rebaseline generates expected layout tests results under /sdcard/android/expected_result/
     --time-out-ms (default is 8000 millis) for each test
     --adb-options="-e" passes option string to adb
     --results-directory=..., (default is ./layout-test-results) directory name under which results are stored.
     --js-engine the JavaScript engine currently in use, determines which set of Android-specific expected results we should use, should be 'jsc' or 'v8'
 """

 import logging
 import optparse
 import os
 import subprocess
 import sys
 import time

 def CountLineNumber(filename):
   """Compute the number of lines in a given file.

   Args:
     filename: a file name related to the current directory.
   """

   fp = open(os.path.abspath(filename), "r");
   lines = 0
   for line in fp.readlines():
     lines = lines + 1
   fp.close()
   return lines

 def DumpRenderTreeFinished(adb_cmd):
   """ Check if DumpRenderTree finished running tests

   Args:
     output: adb_cmd string
   """

   # pull /sdcard/android/running_test.txt, if the content is "#DONE", it's done
   shell_cmd_str = adb_cmd + " shell cat /sdcard/android/running_test.txt"
   adb_output = subprocess.Popen(shell_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
   return adb_output.strip() == "#DONE"

 def DiffResults(marker, new_results, old_results, diff_results, strip_reason,
                 new_count_first=True):
    """ Given two result files, generate diff and
        write to diff_results file. All arguments are absolute paths
        to files.
    """
    old_file = open(old_results, "r")
    new_file = open(new_results, "r")
    diff_file = open(diff_results, "a")

    # Read lines from each file
    ndict = new_file.readlines()
    cdict = old_file.readlines()

    # Write marker to diff file
    diff_file.writelines(marker + "\n")
    diff_file.writelines("###############\n")

    # Strip reason from result lines
    if strip_reason is True:
      for i in range(0, len(ndict)):
        ndict[i] = ndict[i].split(' ')[0] + "\n"
      for i in range(0, len(cdict)):
        cdict[i] = cdict[i].split(' ')[0] + "\n"

    params = {
        "new": [0, ndict, cdict, "+"],
        "miss": [0, cdict, ndict, "-"]
        }
    if new_count_first:
      order = ["new", "miss"]
    else:
      order = ["miss", "new"]

    for key in order:
      for line in params[key][1]:
        if line not in params[key][2]:
          if line[-1] != "\n":
            line += "\n";
          diff_file.writelines(params[key][3] + line)
          params[key][0] += 1

    logging.info(marker + "  >>> " + str(params["new"][0]) + " new, " +
                 str(params["miss"][0]) + " misses")

    diff_file.writelines("\n\n")

    old_file.close()
    new_file.close()
    diff_file.close()
    return

 def CompareResults(ref_dir, results_dir):
   """Compare results in two directories

   Args:
     ref_dir: the reference directory having layout results as references
     results_dir: the results directory
   """
   logging.info("Comparing results to " + ref_dir)

   diff_result = os.path.join(results_dir, "layout_tests_diff.txt")
   if os.path.exists(diff_result):
     os.remove(diff_result)

   files=["crashed", "failed", "passed", "nontext"]
   for f in files:
     result_file_name = "layout_tests_" + f + ".txt"
     DiffResults(f, os.path.join(results_dir, result_file_name),
                 os.path.join(ref_dir, result_file_name), diff_result,
                 False, f != "passed")
   logging.info("Detailed diffs are in " + diff_result)

 def main(options, args):
   """Run the tests. Will call sys.exit when complete.

   Args:
     options: a dictionary of command line options
     args: a list of sub directories or files to test
   """

   # Set up logging format.
   log_level = logging.INFO
   if options.verbose:
     log_level = logging.DEBUG
   logging.basicConfig(level=log_level,
                       format='%(message)s')

   # Include all tests if none are specified.
   if not args:
     path = '/';
   else:
     path = ' '.join(args);

   adb_cmd = "adb ";
   if options.adb_options:
     adb_cmd += options.adb_options

   # Re-generate the test list if --refresh-test-list is on
   if options.refresh_test_list:
     logging.info("Generating test list.");
     generate_test_list_cmd_str = adb_cmd + " shell am instrument -e class com.android.dumprendertree.LayoutTestsAutoTest#generateTestList -e path \"" + path + "\" -w com.android.dumprendertree/.LayoutTestsAutoRunner"
     adb_output = subprocess.Popen(generate_test_list_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]

     if adb_output.find('Process crashed') != -1:
        logging.info("Aborting because cannot generate test list.\n" + adb_output)
        sys.exit(1)


   logging.info("Running tests")

   # Count crashed tests.
   crashed_tests = []

   timeout_ms = '30000'
   if options.time_out_ms:
     timeout_ms = options.time_out_ms

   # Run test until it's done

   run_layout_test_cmd_prefix = adb_cmd + " shell am instrument"

   run_layout_test_cmd_postfix = " -e path \"" + path + "\" -e timeout " + timeout_ms
   if options.rebaseline:
     run_layout_test_cmd_postfix += " -e rebaseline true"

   # If the JS engine is not specified on the command line, try reading the
   # JS_ENGINE environment  variable, which is used by the build system in
   # external/webkit/Android.mk.
   js_engine = options.js_engine
   if not js_engine and os.environ.has_key('JS_ENGINE'):
     js_engine = os.environ['JS_ENGINE']
   if js_engine:
     run_layout_test_cmd_postfix += " -e jsengine " + js_engine

   run_layout_test_cmd_postfix += " -w com.android.dumprendertree/.LayoutTestsAutoRunner"

   # Call LayoutTestsAutoTest::startLayoutTests.
   run_layout_test_cmd = run_layout_test_cmd_prefix + " -e class com.android.dumprendertree.LayoutTestsAutoTest#startLayoutTests" + run_layout_test_cmd_postfix

   adb_output = subprocess.Popen(run_layout_test_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
   while not DumpRenderTreeFinished(adb_cmd):
     # Get the running_test.txt
     logging.error("DumpRenderTree crashed, output:\n" + adb_output)

     shell_cmd_str = adb_cmd + " shell cat /sdcard/android/running_test.txt"
     crashed_test = ""
     while not crashed_test:
       (crashed_test, err) = subprocess.Popen(
           shell_cmd_str, shell=True, stdout=subprocess.PIPE,
           stderr=subprocess.PIPE).communicate()
       crashed_test = crashed_test.strip()
       if not crashed_test:
         logging.error('Cannot get crashed test name, device offline?')
         logging.error('stderr: ' + err)
         logging.error('retrying in 10s...')
         time.sleep(10)

     logging.info(crashed_test + " CRASHED");
     crashed_tests.append(crashed_test);

     logging.info("Resuming layout test runner...");
     # Call LayoutTestsAutoTest::resumeLayoutTests
     run_layout_test_cmd = run_layout_test_cmd_prefix + " -e class com.android.dumprendertree.LayoutTestsAutoTest#resumeLayoutTests" + run_layout_test_cmd_postfix

     adb_output = subprocess.Popen(run_layout_test_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]

   if adb_output.find('INSTRUMENTATION_FAILED') != -1:
     logging.error("Error happened : " + adb_output)
     sys.exit(1)

   logging.debug(adb_output);
   logging.info("Done\n");

   # Pull results from /sdcard
   results_dir = options.results_directory
   if not os.path.exists(results_dir):
     os.makedirs(results_dir)
   if not os.path.isdir(results_dir):
     logging.error("Cannot create results dir: " + results_dir);
     sys.exit(1);

   result_files = ["/sdcard/layout_tests_passed.txt",
                   "/sdcard/layout_tests_failed.txt",
                   "/sdcard/layout_tests_ignored.txt",
                   "/sdcard/layout_tests_nontext.txt"]
   for file in result_files:
     shell_cmd_str = adb_cmd + " pull " + file + " " + results_dir
     adb_output = subprocess.Popen(shell_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
     logging.debug(adb_output)

   # Create the crash list.
   fp = open(results_dir + "/layout_tests_crashed.txt", "w");
   for crashed_test in crashed_tests:
     fp.writelines(crashed_test + '\n')
   fp.close()

   # Count the number of tests in each category.
   passed_tests = CountLineNumber(results_dir + "/layout_tests_passed.txt")
   logging.info(str(passed_tests) + " passed")
   failed_tests = CountLineNumber(results_dir + "/layout_tests_failed.txt")
   logging.info(str(failed_tests) + " failed")
   ignored_tests = CountLineNumber(results_dir + "/layout_tests_ignored.txt")
   logging.info(str(ignored_tests) + " ignored results")
   crashed_tests = CountLineNumber(results_dir + "/layout_tests_crashed.txt")
   logging.info(str(crashed_tests) + " crashed")
   nontext_tests = CountLineNumber(results_dir + "/layout_tests_nontext.txt")
   logging.info(str(nontext_tests) + " no dumpAsText")
   logging.info(str(passed_tests + failed_tests + ignored_tests + crashed_tests + nontext_tests) + " TOTAL")

   logging.info("Results are stored under: " + results_dir + "\n")

   # Comparing results to references to find new fixes and regressions.
   results_dir = os.path.abspath(options.results_directory)
   ref_dir = options.ref_directory

   # if ref_dir is null, cannonify ref_dir to the script dir.
   if not ref_dir:
     script_self = sys.argv[0]
     script_dir = os.path.dirname(script_self)
     ref_dir = os.path.join(script_dir, "results")

   ref_dir = os.path.abspath(ref_dir)

   CompareResults(ref_dir, results_dir)

 if '__main__' == __name__:
   option_parser = optparse.OptionParser()
   option_parser.add_option("", "--rebaseline", action="store_true",
                            default=False,
                            help="generate expected results for those tests not having one")
   option_parser.add_option("", "--time-out-ms",
                            default=None,
                            help="set the timeout for each test")
   option_parser.add_option("", "--verbose", action="store_true",
                            default=False,
                            help="include debug-level logging")
   option_parser.add_option("", "--refresh-test-list", action="store_true",
                            default=False,
                            help="re-generate test list, it may take some time.")
   option_parser.add_option("", "--adb-options",
                            default=None,
                            help="pass options to adb, such as -d -e, etc");
   option_parser.add_option("", "--results-directory",
                            default="layout-test-results",
                            help="directory which results are stored.")
   option_parser.add_option("", "--ref-directory",
                            default=None,
                            dest="ref_directory",
                            help="directory where reference results are stored.")
   option_parser.add_option("", "--js-engine",
                            default=None,
                            help="The JavaScript engine currently in use, which determines which set of Android-specific expected results we should use. Should be 'jsc' or 'v8'.");

   options, args = option_parser.parse_args();
   main(options, args)
	#!/usr/bin/python

	"""Run layout tests using Android emulator and instrumentation.

	First, you need to get an SD card or sdcard image that has layout tests on it.
	Layout tests are in following directory:
	/sdcard/android/layout_tests
	For example, /sdcard/android/layout_tests/fast

	Usage:
	Run all tests under fast/ directory:
	run_layout_tests.py, or
	run_layout_tests.py fast

	Run all tests under a sub directory:
	run_layout_tests.py fast/dom

	Run a single test:
	run_layout_tests.py fast/dom/

	After a merge, if there are changes of layout tests in SD card, you need to
	use --refresh-test-list option once to re-generate test list on the card.

	Some other options are:
	--rebaseline generates expected layout tests results under /sdcard/android/expected_result/
	--time-out-ms (default is 8000 millis) for each test
	--adb-options="-e" passes option string to adb
	--results-directory=..., (default is ./layout-test-results) directory name under which results are stored.
	--js-engine the JavaScript engine currently in use, determines which set of Android-specific expected results we should use, should be 'jsc' or 'v8'
	"""

	import logging
	import optparse
	import os
	import subprocess
	import sys
	import time

	def CountLineNumber(filename):
	"""Compute the number of lines in a given file.

	Args:
	filename: a file name related to the current directory.
	"""

	fp = open(os.path.abspath(filename), "r");
	lines = 0
	for line in fp.readlines():
	lines = lines + 1
	fp.close()
	return lines

	def DumpRenderTreeFinished(adb_cmd):
	""" Check if DumpRenderTree finished running tests

	Args:
	output: adb_cmd string
	"""

	# pull /sdcard/android/running_test.txt, if the content is "#DONE", it's done
	shell_cmd_str = adb_cmd + " shell cat /sdcard/android/running_test.txt"
	adb_output = subprocess.Popen(shell_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
	return adb_output.strip() == "#DONE"

	def DiffResults(marker, new_results, old_results, diff_results, strip_reason,
	new_count_first=True):
	""" Given two result files, generate diff and
	write to diff_results file. All arguments are absolute paths
	to files.
	"""
	old_file = open(old_results, "r")
	new_file = open(new_results, "r")
	diff_file = open(diff_results, "a")

	# Read lines from each file
	ndict = new_file.readlines()
	cdict = old_file.readlines()

	# Write marker to diff file
	diff_file.writelines(marker + "\n")
	diff_file.writelines("###############\n")

	# Strip reason from result lines
	if strip_reason is True:
	for i in range(0, len(ndict)):
	ndict[i] = ndict[i].split(' ')[0] + "\n"
	for i in range(0, len(cdict)):
	cdict[i] = cdict[i].split(' ')[0] + "\n"

	params = {
	"new": [0, ndict, cdict, "+"],
	"miss": [0, cdict, ndict, "-"]
	}
	if new_count_first:
	order = ["new", "miss"]
	else:
	order = ["miss", "new"]

	for key in order:
	for line in params[key][1]:
	if line not in params[key][2]:
	if line[-1] != "\n":
	line += "\n";
	diff_file.writelines(params[key][3] + line)
	params[key][0] += 1

	logging.info(marker + " >>> " + str(params["new"][0]) + " new, " +
	str(params["miss"][0]) + " misses")

	diff_file.writelines("\n\n")

	old_file.close()
	new_file.close()
	diff_file.close()
	return

	def CompareResults(ref_dir, results_dir):
	"""Compare results in two directories

	Args:
	ref_dir: the reference directory having layout results as references
	results_dir: the results directory
	"""
	logging.info("Comparing results to " + ref_dir)

	diff_result = os.path.join(results_dir, "layout_tests_diff.txt")
	if os.path.exists(diff_result):
	os.remove(diff_result)

	files=["crashed", "failed", "passed", "nontext"]
	for f in files:
	result_file_name = "layout_tests_" + f + ".txt"
	DiffResults(f, os.path.join(results_dir, result_file_name),
	os.path.join(ref_dir, result_file_name), diff_result,
	False, f != "passed")
	logging.info("Detailed diffs are in " + diff_result)

	def main(options, args):
	"""Run the tests. Will call sys.exit when complete.

	Args:
	options: a dictionary of command line options
	args: a list of sub directories or files to test
	"""

	# Set up logging format.
	log_level = logging.INFO
	if options.verbose:
	log_level = logging.DEBUG
	logging.basicConfig(level=log_level,
	format='%(message)s')

	# Include all tests if none are specified.
	if not args:
	path = '/';
	else:
	path = ' '.join(args);

	adb_cmd = "adb ";
	if options.adb_options:
	adb_cmd += options.adb_options

	# Re-generate the test list if --refresh-test-list is on
	if options.refresh_test_list:
	logging.info("Generating test list.");
	generate_test_list_cmd_str = adb_cmd + " shell am instrument -e class com.android.dumprendertree.LayoutTestsAutoTest#generateTestList -e path \"" + path + "\" -w com.android.dumprendertree/.LayoutTestsAutoRunner"
	adb_output = subprocess.Popen(generate_test_list_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]

	if adb_output.find('Process crashed') != -1:
	logging.info("Aborting because cannot generate test list.\n" + adb_output)
	sys.exit(1)


	logging.info("Running tests")

	# Count crashed tests.
	crashed_tests = []

	timeout_ms = '30000'
	if options.time_out_ms:
	timeout_ms = options.time_out_ms

	# Run test until it's done

	run_layout_test_cmd_prefix = adb_cmd + " shell am instrument"

	run_layout_test_cmd_postfix = " -e path \"" + path + "\" -e timeout " + timeout_ms
	if options.rebaseline:
	run_layout_test_cmd_postfix += " -e rebaseline true"

	# If the JS engine is not specified on the command line, try reading the
	# JS_ENGINE environment variable, which is used by the build system in
	# external/webkit/Android.mk.
	js_engine = options.js_engine
	if not js_engine and os.environ.has_key('JS_ENGINE'):
	js_engine = os.environ['JS_ENGINE']
	if js_engine:
	run_layout_test_cmd_postfix += " -e jsengine " + js_engine

	run_layout_test_cmd_postfix += " -w com.android.dumprendertree/.LayoutTestsAutoRunner"

	# Call LayoutTestsAutoTest::startLayoutTests.
	run_layout_test_cmd = run_layout_test_cmd_prefix + " -e class com.android.dumprendertree.LayoutTestsAutoTest#startLayoutTests" + run_layout_test_cmd_postfix

	adb_output = subprocess.Popen(run_layout_test_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
	while not DumpRenderTreeFinished(adb_cmd):
	# Get the running_test.txt
	logging.error("DumpRenderTree crashed, output:\n" + adb_output)

	shell_cmd_str = adb_cmd + " shell cat /sdcard/android/running_test.txt"
	crashed_test = ""
	while not crashed_test:
	(crashed_test, err) = subprocess.Popen(
	shell_cmd_str, shell=True, stdout=subprocess.PIPE,
	stderr=subprocess.PIPE).communicate()
	crashed_test = crashed_test.strip()
	if not crashed_test:
	logging.error('Cannot get crashed test name, device offline?')
	logging.error('stderr: ' + err)
	logging.error('retrying in 10s...')
	time.sleep(10)

	logging.info(crashed_test + " CRASHED");
	crashed_tests.append(crashed_test);

	logging.info("Resuming layout test runner...");
	# Call LayoutTestsAutoTest::resumeLayoutTests
	run_layout_test_cmd = run_layout_test_cmd_prefix + " -e class com.android.dumprendertree.LayoutTestsAutoTest#resumeLayoutTests" + run_layout_test_cmd_postfix

	adb_output = subprocess.Popen(run_layout_test_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]

	if adb_output.find('INSTRUMENTATION_FAILED') != -1:
	logging.error("Error happened : " + adb_output)
	sys.exit(1)

	logging.debug(adb_output);
	logging.info("Done\n");

	# Pull results from /sdcard
	results_dir = options.results_directory
	if not os.path.exists(results_dir):
	os.makedirs(results_dir)
	if not os.path.isdir(results_dir):
	logging.error("Cannot create results dir: " + results_dir);
	sys.exit(1);

	result_files = ["/sdcard/layout_tests_passed.txt",
	"/sdcard/layout_tests_failed.txt",
	"/sdcard/layout_tests_ignored.txt",
	"/sdcard/layout_tests_nontext.txt"]
	for file in result_files:
	shell_cmd_str = adb_cmd + " pull " + file + " " + results_dir
	adb_output = subprocess.Popen(shell_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
	logging.debug(adb_output)

	# Create the crash list.
	fp = open(results_dir + "/layout_tests_crashed.txt", "w");
	for crashed_test in crashed_tests:
	fp.writelines(crashed_test + '\n')
	fp.close()

	# Count the number of tests in each category.
	passed_tests = CountLineNumber(results_dir + "/layout_tests_passed.txt")
	logging.info(str(passed_tests) + " passed")
	failed_tests = CountLineNumber(results_dir + "/layout_tests_failed.txt")
	logging.info(str(failed_tests) + " failed")
	ignored_tests = CountLineNumber(results_dir + "/layout_tests_ignored.txt")
	logging.info(str(ignored_tests) + " ignored results")
	crashed_tests = CountLineNumber(results_dir + "/layout_tests_crashed.txt")
	logging.info(str(crashed_tests) + " crashed")
	nontext_tests = CountLineNumber(results_dir + "/layout_tests_nontext.txt")
	logging.info(str(nontext_tests) + " no dumpAsText")
	logging.info(str(passed_tests + failed_tests + ignored_tests + crashed_tests + nontext_tests) + " TOTAL")

	logging.info("Results are stored under: " + results_dir + "\n")

	# Comparing results to references to find new fixes and regressions.
	results_dir = os.path.abspath(options.results_directory)
	ref_dir = options.ref_directory

	# if ref_dir is null, cannonify ref_dir to the script dir.
	if not ref_dir:
	script_self = sys.argv[0]
	script_dir = os.path.dirname(script_self)
	ref_dir = os.path.join(script_dir, "results")

	ref_dir = os.path.abspath(ref_dir)

	CompareResults(ref_dir, results_dir)

	if '__main__' == __name__:
	option_parser = optparse.OptionParser()
	option_parser.add_option("", "--rebaseline", action="store_true",
	default=False,
	help="generate expected results for those tests not having one")
	option_parser.add_option("", "--time-out-ms",
	default=None,
	help="set the timeout for each test")
	option_parser.add_option("", "--verbose", action="store_true",
	default=False,
	help="include debug-level logging")
	option_parser.add_option("", "--refresh-test-list", action="store_true",
	default=False,
	help="re-generate test list, it may take some time.")
	option_parser.add_option("", "--adb-options",
	default=None,
	help="pass options to adb, such as -d -e, etc");
	option_parser.add_option("", "--results-directory",
	default="layout-test-results",
	help="directory which results are stored.")
	option_parser.add_option("", "--ref-directory",
	default=None,
	dest="ref_directory",
	help="directory where reference results are stored.")
	option_parser.add_option("", "--js-engine",
	default=None,
	help="The JavaScript engine currently in use, which determines which set of Android-specific expected results we should use. Should be 'jsc' or 'v8'.");

	options, args = option_parser.parse_args();
	main(options, args)