Run test: Execute scripts via runner bash script.
Simplify all run tests to simple execution pattern:
adb push <inputs> && adb shell <runner> && adb pull <outputs>
This dramatically reduces the number of adb commands, and
it will make it trivial to reimplement the above steps in tradefed.
Follow the above logic for host and jvm as well for consistency.
Test: ./art/test.py -r --all-target --optimizing --64
Change-Id: I4b54a4d32195c440ca43133bbbe988ae24e626af
diff --git a/test/default_run.py b/test/default_run.py
index 3f311a8..87664aa 100755
--- a/test/default_run.py
+++ b/test/default_run.py
@@ -17,7 +17,7 @@
from argparse import ArgumentParser, BooleanOptionalAction, Namespace
from os import path
from os.path import isfile, isdir, basename
-from subprocess import DEVNULL, PIPE, STDOUT
+from subprocess import check_output, DEVNULL, PIPE, STDOUT
from tempfile import NamedTemporaryFile
from testrunner import env
from typing import List
@@ -213,75 +213,6 @@
else:
setattr(args, name, new_value)
- # Script debugging: Record executed commands into the given directory.
- # This is useful to ensure that changes to the script don't change behaviour.
- # (the commands are appended so the directory needs to be cleared before run)
- ART_TEST_CMD_DIR = os.environ.get("ART_TEST_CMD_DIR")
-
- # Script debugging: Record executed commands, but don't actually run the main test.
- # This makes it possible the extract the test commands without waiting for days.
- # This will make tests fail since there is no stdout. Use with large -j value.
- ART_TEST_DRY_RUN = os.environ.get("ART_TEST_DRY_RUN")
-
- def run(cmdline: str,
- env={},
- check=True,
- parse_exit_code_from_stdout=False,
- expected_exit_code=0,
- save_cmd=True) -> subprocess.CompletedProcess:
- env.setdefault("PATH", PATH) # Ensure that PATH is always set.
- env = {k: v for k, v in env.items() if v != None} # Filter "None" entries.
- if ART_TEST_CMD_DIR and save_cmd and cmdline != "true":
- tmp = os.environ["DEX_LOCATION"]
- with open(
- os.path.join(ART_TEST_CMD_DIR, os.environ["FULL_TEST_NAME"]),
- "a") as f:
- # Replace DEX_LOCATION (which is randomly generated temporary directory),
- # with a deterministic placeholder so that we can do a diff from run to run.
- f.write("\n".join(
- k + ":" + v.replace(tmp, "<tmp>") for k, v in env.items()) + "\n\n")
- f.write(re.sub(" +", "\n", cmdline).replace(tmp, "<tmp>") + "\n\n")
- if ART_TEST_DRY_RUN and ("dalvikvm" in cmdline or
- "adb shell chroot" in cmdline):
- cmdline = "true" # We still need to run some command, so run the no-op "true" binary instead.
-
- if cmdline != "true":
- print(f"{COLOR_BLUE}$ {cmdline}{COLOR_NORMAL}")
- proc = subprocess.run([cmdline],
- shell=True,
- executable="/bin/bash",
- env=env,
- encoding="utf8",
- capture_output=True)
- if proc.stdout:
- print(proc.stdout.strip(), flush=True)
- if proc.stderr:
- print(proc.stderr.strip(), flush=True)
-
- # ADB forwards exit code from the executed command, but if ADB process itself crashes,
- # it will also return non-zero exit code, and we can not distinguish those two cases.
- # As a work-around, we wrap the executed command so that it always returns 0 exit code
- # and we also make it print the actual exit code as the last line of its stdout.
- if parse_exit_code_from_stdout:
- assert proc.returncode == 0, f"ADB failed (exit code {proc.returncode})"
- found = re.search("exit_code=([0-9]+)$", proc.stdout)
- assert found, "Expected exit code as the last line of stdout"
- proc.stdout = proc.stdout[:found.start(0)] # Remove the exit code from stdout.
- proc.returncode = int(found.group(1)) # Use it as if it was the process exit code.
-
- # Check the exit code.
- if (check and proc.returncode != expected_exit_code):
- suffix = ""
- if proc.returncode == 124:
- suffix = " (TIME OUT)"
- elif expected_exit_code != 0:
- suffix = " (expected {})".format(expected_exit_code)
- print(f"{COLOR_RED}{TEST_NAME} FAILED: Command returned exit code "
- f"{proc.returncode}{suffix}{COLOR_NORMAL}", file=sys.stderr)
- sys.exit(1)
-
- return proc
-
# Store copy of stdout&stderr of command in files so that we can diff them later.
# This may run under 'adb shell' so we are limited only to 'sh' shell feature set.
def tee(cmd: str):
@@ -290,48 +221,8 @@
cmd = f"({cmd} | tee -a {DEX_LOCATION}/{basename(args.stderr_file)}) 3>&1 1>&2 2>&3"
return f"set -o pipefail; {cmd}" # Use exit code of first failure in piped command.
- class Adb():
-
- def __init__(self):
- self.env = {
- "ADB_VENDOR_KEYS": os.environ.get("ADB_VENDOR_KEYS"),
- "ANDROID_SERIAL": os.environ.get("ANDROID_SERIAL"),
- "PATH": os.environ.get("PATH"),
- }
-
- def root(self) -> None:
- run("adb root", self.env)
-
- def wait_for_device(self) -> None:
- run("adb wait-for-device", self.env)
-
- def shell(self, cmdline: str, **kwargs) -> subprocess.CompletedProcess:
- return run(f"adb shell '{cmdline}; echo exit_code=$?'", self.env,
- parse_exit_code_from_stdout=True, **kwargs)
-
- def push(self, src: str, dst: str, **kwargs) -> None:
- run(f"adb push {src} {dst}", self.env, **kwargs)
-
- def pull(self, src: str, dst: str, **kwargs) -> None:
- run(f"adb pull {src} {dst}", self.env, **kwargs)
-
- adb = Adb()
-
local_path = os.path.dirname(__file__)
- # Check that stdout is connected to a terminal and that we have at least 1 color.
- # This ensures that if the stdout is not connected to a terminal and instead
- # the stdout will be used for a log, it will not append the color characters.
- bold_red = ""
- if sys.stdout.isatty():
- if int(subprocess.run(["tput", "colors"], capture_output=True).stdout) >= 1:
- bold_red = subprocess.run(["tput", "bold"],
- text=True,
- capture_output=True).stdout.strip()
- bold_red += subprocess.run(["tput", "setaf", "1"],
- text=True,
- capture_output=True).stdout.strip()
-
ANDROID_BUILD_TOP = os.environ.get("ANDROID_BUILD_TOP")
ANDROID_DATA = os.environ.get("ANDROID_DATA")
ANDROID_HOST_OUT = os.environ["ANDROID_HOST_OUT"]
@@ -588,12 +479,7 @@
# If running on device, determine the ISA of the device.
if not HOST and not USE_JVM:
- ISA = run(
- f"{ANDROID_BUILD_TOP}/art/test/utils/get-device-isa {GET_DEVICE_ISA_BITNESS_FLAG}",
- adb.env,
- save_cmd=False).stdout.strip()
- target_arch = get_target_arch(args.is64) # Should return the same ISA.
- assert ISA == target_arch, f"{ISA} vs {target_arch}"
+ ISA = get_target_arch(args.is64)
if not USE_JVM:
FLAGS += f" {ANDROID_FLAGS}"
@@ -613,24 +499,17 @@
DEX_OPTIMIZE = "-Xdexopt:verified"
else:
DEX_OPTIMIZE = "-Xdexopt:all"
- print("Performing optimizations")
else:
DEX_OPTIMIZE = "-Xdexopt:none"
- print("Skipping optimizations")
if VERIFY == "y":
JVM_VERIFY_ARG = "-Xverify:all"
- print("Performing verification")
elif VERIFY == "s":
JVM_VERIFY_ARG = "Xverify:all"
DEX_VERIFY = "-Xverify:softfail"
- print("Forcing verification to be soft fail")
else: # VERIFY == "n"
DEX_VERIFY = "-Xverify:none"
JVM_VERIFY_ARG = "-Xverify:none"
- print("Skipping verification")
-
- print("------------------------------")
if DEBUGGER == "y":
# Use this instead for ddms and connect by running 'ddms':
@@ -721,34 +600,21 @@
FLAGS += f" -agentpath:{agent}={agent_args}"
if USE_JVM:
- env = {
- "ANDROID_I18N_ROOT": ANDROID_I18N_ROOT,
- "DEX_LOCATION": DEX_LOCATION,
- "JAVA_HOME": os.environ["JAVA_HOME"],
- "LANG":
- "en_US.UTF-8", # Needed to enable unicode and make the output is deterministic.
- "LD_LIBRARY_PATH": f"{ANDROID_HOST_OUT}/lib64",
- }
+ ctx.export(
+ ANDROID_I18N_ROOT = ANDROID_I18N_ROOT,
+ DEX_LOCATION = DEX_LOCATION,
+ JAVA_HOME = os.environ["JAVA_HOME"],
+ LANG = "en_US.UTF-8", # Needed to enable unicode and make the output is deterministic.
+ LD_LIBRARY_PATH = f"{ANDROID_HOST_OUT}/lib64",
+ )
# Some jvmti tests are flaky without -Xint on the RI.
if IS_JVMTI_TEST:
FLAGS += " -Xint"
# Xmx is necessary since we don't pass down the ART flags to JVM.
# We pass the classes2 path whether it's used (src-multidex) or not.
cmdline = f"{JAVA} {DEBUGGER_OPTS} {JVM_VERIFY_ARG} -Xmx256m -classpath classes:classes2 {FLAGS} {MAIN} {ARGS}"
- if CREATE_RUNNER:
- with open("runit.sh", "w") as f:
- f.write("#!/bin/bash")
- print(f"export LD_LIBRARY_PATH=\"{LD_LIBRARY_PATH}\"")
- f.write(cmdline)
- os.chmod("runit.sh", 0o777)
- pwd = os.getcwd()
- print(f"Runnable test script written to {pwd}/runit.sh")
- return
- else:
- run(tee(cmdline),
- env,
- expected_exit_code=args.expected_exit_code)
- return
+ ctx.run(tee(cmdline), expected_exit_code=args.expected_exit_code)
+ return
b_path = get_apex_bootclasspath(HOST)
b_path_locations = get_apex_bootclasspath_locations(HOST)
@@ -779,29 +645,14 @@
if USE_GDB_DEX2OAT:
assert HOST, "The --gdb-dex2oat option is not yet implemented for target."
+ assert not USE_GDBSERVER, "Not supported"
if USE_GDB:
- assert not USE_GDBSERVER, "Cannot pass both --gdb and --gdbserver at the same time!"
if not HOST:
# We might not have any hostname resolution if we are using a chroot.
GDB = f"{GDBSERVER_DEVICE} --no-startup-with-shell 127.0.0.1{GDBSERVER_PORT}"
else:
- if run("uname").stdout.strip() == "Darwin":
- GDB = "lldb"
- GDB_ARGS += f" -- {DALVIKVM}"
- DALVIKVM = ""
- else:
- GDB = "gdb"
- GDB_ARGS += f" --args {DALVIKVM}"
- # Enable for Emacs "M-x gdb" support. TODO: allow extra gdb arguments on command line.
- # gdbargs=f"--annotate=3 {gdbargs}"
- elif USE_GDBSERVER:
- if not HOST:
- # We might not have any hostname resolution if we are using a chroot.
- GDB = f"{GDBSERVER_DEVICE} --no-startup-with-shell 127.0.0.1{GDBSERVER_PORT}"
- else:
- GDB = f"{GDBSERVER_HOST} {GDBSERVER_PORT}"
-
- assert shutil.which(GDBSERVER_HOST), f"{GDBSERVER_HOST} is not available"
+ GDB = "gdb"
+ GDB_ARGS += f" --args {DALVIKVM}"
if INTERPRETER:
INT_OPTS += " -Xint"
@@ -845,8 +696,7 @@
# full path to dex, stripping leading '/', appending '@classes.vdex' and changing every
# remaining '/' into '@'.
if HOST:
- max_filename_size = int(
- run(f"getconf NAME_MAX {DEX_LOCATION}", save_cmd=False).stdout)
+ max_filename_size = int(check_output(f"getconf NAME_MAX {DEX_LOCATION}", shell=True))
else:
# There is no getconf on device, fallback to standard value.
# See NAME_MAX in kernel <linux/limits.h>
@@ -912,7 +762,6 @@
ANDROID_ART_BIN_DIR = f"{DEX_LOCATION}/zipapex/bin"
# Force since some tests manually run this file twice.
# If the {RUN} is executed multiple times we don't need to recreate the link
- installapex_test_cmdline = f"test -L {DEX_LOCATION}/zipapex"
installapex_cmdline = f"ln -s -f --verbose {EXTRACTED_ZIPAPEX_LOC} {DEX_LOCATION}/zipapex"
# PROFILE takes precedence over RANDOM_PROFILE, since PROFILE tests require a
@@ -939,10 +788,10 @@
def get_prebuilt_lldb_path():
CLANG_BASE = "prebuilts/clang/host"
- CLANG_VERSION = run(
+ CLANG_VERSION = check_output(
f"{ANDROID_BUILD_TOP}/build/soong/scripts/get_clang_version.py"
- ).stdout.strip()
- uname = run("uname -s").stdout.strip()
+ ).strip()
+ uname = check_output("uname -s", shell=True).strip()
if uname == "Darwin":
PREBUILT_NAME = "darwin-x86"
elif uname == "Linux":
@@ -969,7 +818,7 @@
# Set the current terminfo directory to TERMINFO so that LLDB can read the
# termcap database.
- terminfo = re.search("/.*/terminfo/", run("infocmp", save_cmd=False).stdout)
+ terminfo = re.search("/.*/terminfo/", check_output("infocmp"))
if terminfo:
os.environ["TERMINFO"] = terminfo[0]
@@ -1213,50 +1062,32 @@
# Note: Using "--foreground" to not propagate the signal to children, i.e., the runtime.
timeout_prefix = f"timeout --foreground -k 120s {TIME_OUT_VALUE}s {timeout_dumper_cmd} {cmdline}"
- env = {
- "ASAN_OPTIONS": RUN_TEST_ASAN_OPTIONS,
- "ANDROID_DATA": DEX_LOCATION,
- "DEX_LOCATION": DEX_LOCATION,
- "ANDROID_ROOT": ANDROID_ROOT,
- "ANDROID_I18N_ROOT": ANDROID_I18N_ROOT,
- "ANDROID_ART_ROOT": ANDROID_ART_ROOT,
- "ANDROID_TZDATA_ROOT": ANDROID_TZDATA_ROOT,
- "ANDROID_LOG_TAGS": ANDROID_LOG_TAGS,
- "LD_LIBRARY_PATH": LD_LIBRARY_PATH,
- "NATIVELOADER_DEFAULT_NAMESPACE_LIBS": NATIVELOADER_DEFAULT_NAMESPACE_LIBS,
- "PATH": f"{PREPEND_TARGET_PATH}:$PATH",
- } # pyformat: disable
-
- def run_cmd(cmdline: str, env={}, **kwargs) -> subprocess.CompletedProcess:
- if cmdline == "true" or DRY_RUN:
- return run('true') # Noop command which just executes the linux 'true' binary.
- cmdline = (f"cd {DEX_LOCATION} && " +
- "".join(f"export {k}={v} && " for k, v in env.items()) +
- cmdline)
- # Create a script with the command. The command can get longer than the longest
- # allowed adb command and there is no way to get the exit status from a adb shell command.
- with NamedTemporaryFile(mode="w") as cmdfile:
- cmdfile.write("echo '$$ {}'\n".format(cmdline.replace("'", r"'\''")))
- cmdfile.write(cmdline)
- cmdfile.flush()
- adb.push(
- cmdfile.name, f"{CHROOT_DEX_LOCATION}/cmdline.sh", save_cmd=False)
- chroot_prefix = f"chroot {CHROOT}" if CHROOT else ""
- return adb.shell(f"{chroot_prefix} sh {DEX_LOCATION}/cmdline.sh", **kwargs)
+ ctx.export(
+ ASAN_OPTIONS = RUN_TEST_ASAN_OPTIONS,
+ ANDROID_DATA = DEX_LOCATION,
+ DEX_LOCATION = DEX_LOCATION,
+ ANDROID_ROOT = ANDROID_ROOT,
+ ANDROID_I18N_ROOT = ANDROID_I18N_ROOT,
+ ANDROID_ART_ROOT = ANDROID_ART_ROOT,
+ ANDROID_TZDATA_ROOT = ANDROID_TZDATA_ROOT,
+ ANDROID_LOG_TAGS = ANDROID_LOG_TAGS,
+ LD_LIBRARY_PATH = LD_LIBRARY_PATH,
+ NATIVELOADER_DEFAULT_NAMESPACE_LIBS = NATIVELOADER_DEFAULT_NAMESPACE_LIBS,
+ PATH = f"{PREPEND_TARGET_PATH}:$PATH",
+ )
if USE_GDB or USE_GDBSERVER:
print(f"Forward {GDBSERVER_PORT} to local port and connect GDB")
- run_cmd(f"rm -rf {DEX_LOCATION}/{{oat,dalvik-cache}}/ && mkdir -p {mkdir_locations}")
- run_cmd(f"{profman_cmdline}", env)
- run_cmd(f"{dex2oat_cmdline}", env)
- run_cmd(f"{dm_cmdline}", env)
- run_cmd(f"{vdex_cmdline}", env)
- run_cmd(f"{strip_cmdline}")
- run_cmd(f"{sync_cmdline}")
- run_cmd(tee(f"{timeout_prefix} {dalvikvm_cmdline}"),
- env,
- expected_exit_code=args.expected_exit_code)
+ ctx.run(f"rm -rf {DEX_LOCATION}/{{oat,dalvik-cache}}/ && mkdir -p {mkdir_locations}")
+ ctx.run(f"{profman_cmdline}")
+ ctx.run(f"{dex2oat_cmdline}", desc="Dex2oat")
+ ctx.run(f"{dm_cmdline}")
+ ctx.run(f"{vdex_cmdline}")
+ ctx.run(f"{strip_cmdline}")
+ ctx.run(f"{sync_cmdline}")
+ ctx.run(tee(f"{timeout_prefix} {dalvikvm_cmdline}"),
+ expected_exit_code=args.expected_exit_code, desc="DalvikVM")
else:
# Host run.
if USE_ZIPAPEX or USE_EXRACTED_ZIPAPEX:
@@ -1265,22 +1096,23 @@
else:
LD_LIBRARY_PATH = f"{ANDROID_ROOT}/{LIBRARY_DIRECTORY}:{ANDROID_ROOT}/{TEST_DIRECTORY}"
- env = {
- "ANDROID_PRINTF_LOG": "brief",
- "ASAN_OPTIONS": RUN_TEST_ASAN_OPTIONS,
- "ANDROID_DATA": DEX_LOCATION,
- "DEX_LOCATION": DEX_LOCATION,
- "ANDROID_ROOT": ANDROID_ROOT,
- "ANDROID_I18N_ROOT": ANDROID_I18N_ROOT,
- "ANDROID_ART_ROOT": ANDROID_ART_ROOT,
- "ANDROID_TZDATA_ROOT": ANDROID_TZDATA_ROOT,
- "ANDROID_LOG_TAGS": ANDROID_LOG_TAGS,
- "LD_LIBRARY_PATH": LD_LIBRARY_PATH,
- "PATH": f"{PATH}:{ANDROID_ART_BIN_DIR}",
- # Temporarily disable address space layout randomization (ASLR).
- # This is needed on the host so that the linker loads core.oat at the necessary address.
- "LD_USE_LOAD_BIAS": "1",
- }
+ ctx.export(
+ ANDROID_PRINTF_LOG = "brief",
+ ASAN_OPTIONS = RUN_TEST_ASAN_OPTIONS,
+ ANDROID_DATA = DEX_LOCATION,
+ DEX_LOCATION = DEX_LOCATION,
+ ANDROID_ROOT = ANDROID_ROOT,
+ ANDROID_I18N_ROOT = ANDROID_I18N_ROOT,
+ ANDROID_ART_ROOT = ANDROID_ART_ROOT,
+ ANDROID_TZDATA_ROOT = ANDROID_TZDATA_ROOT,
+ ANDROID_LOG_TAGS = ANDROID_LOG_TAGS,
+ LD_LIBRARY_PATH = LD_LIBRARY_PATH,
+ PATH = f"{PATH}:{ANDROID_ART_BIN_DIR}",
+ # Temporarily disable address space layout randomization (ASLR).
+ # This is needed on the host so that the linker loads core.oat at the necessary address.
+ LD_USE_LOAD_BIAS = "1",
+ TERM = os.environ.get("TERM", ""), # Needed for GDB
+ )
cmdline = dalvikvm_cmdline
@@ -1313,75 +1145,39 @@
# Make sure we delete any existing compiler artifacts.
# This enables tests to call the RUN script multiple times in a row
# without worrying about interference.
- shutil.rmtree(f"{DEX_LOCATION}/oat", ignore_errors=True)
- shutil.rmtree(f"{DEX_LOCATION}/dalvik-cache/", ignore_errors=True)
+ ctx.run(f"rm -rf {DEX_LOCATION}/{{oat,dalvik-cache}}/")
- run(f"mkdir -p {mkdir_locations}", save_cmd=False)
- run(setupapex_cmdline)
- if run(installapex_test_cmdline, check=False).returncode != 0:
- run(installapex_cmdline)
- run(linkroot_cmdline)
- run(linkroot_overlay_cmdline)
- run(profman_cmdline, env)
- run(dex2oat_cmdline, env)
- run(dm_cmdline, env)
- run(vdex_cmdline, env)
- run(strip_cmdline)
- run(sync_cmdline)
+ ctx.run(f"mkdir -p {mkdir_locations}")
+ ctx.run(setupapex_cmdline)
+ if USE_EXTRACTED_ZIPAPEX:
+ ctx.run(installapex_cmdline)
+ ctx.run(linkroot_cmdline)
+ ctx.run(linkroot_overlay_cmdline)
+ ctx.run(profman_cmdline)
+ ctx.run(dex2oat_cmdline, desc="Dex2oat")
+ ctx.run(dm_cmdline)
+ ctx.run(vdex_cmdline)
+ ctx.run(strip_cmdline)
+ ctx.run(sync_cmdline)
- if CREATE_RUNNER:
- with open(f"{DEX_LOCATION}/runit.sh", "w") as f:
- f.write("#!/bin/bash")
- for var in ("ANDROID_PRINTF_LOG ANDROID_DATA ANDROID_ROOT "
- "ANDROID_I18N_ROOT ANDROID_TZDATA_ROOT ANDROID_ART_ROOT "
- "LD_LIBRARY_PATH DYLD_LIBRARY_PATH PATH LD_USE_LOAD_BIAS"
- ).split(" "):
- value = os.environ.get(var, "")
- f.write(f'export {var}="{value}"')
- f.write(cmdline)
- os.chmod("{DEX_LOCATION}/runit.sh", 0o777)
- print(f"Runnable test script written to {DEX_LOCATION}/runit.sh")
if DRY_RUN:
return
if USE_GDB:
# When running under gdb, we cannot do piping and grepping...
- env["TERM"] = os.environ.get("TERM", "")
- subprocess.run(cmdline, env=env, shell=True)
- elif USE_GDBSERVER:
- print(f"Connect to {GDBSERVER_PORT}")
- # When running under gdb, we cannot do piping and grepping...
- subprocess.run(cmdline, env=env, shell=True)
+ ctx.run(cmdline)
else:
- if TIME_OUT != "gdb":
- run(tee(cmdline),
- env,
- expected_exit_code=args.expected_exit_code)
+ ctx.run(tee(cmdline), expected_exit_code=args.expected_exit_code, desc="DalvikVM")
- # Remove unwanted log messages from stderr before diffing with the expected output.
- # NB: The unwanted log line can be interleaved in the middle of wanted stderr printf.
- # In particular, unhandled exception is printed using several unterminated printfs.
- ALL_LOG_TAGS = ["V", "D", "I", "W", "E", "F", "S"]
- skip_tag_set = "|".join(ALL_LOG_TAGS[:ALL_LOG_TAGS.index(args.diff_min_log_tag.upper())])
- skip_reg_exp = fr'[[:alnum:]]+ ({skip_tag_set}) #-# #:#:# [^\n]*\n'.replace('#', '[0-9]+')
- run(fr"sed -i -z -E 's/{skip_reg_exp}//g' '{args.stderr_file}'")
- if not HAVE_IMAGE:
- message = "(Unable to open file|Could not create image space)"
- run(fr"sed -i -E '/^dalvikvm(|32|64) E .* {message}/d' '{args.stderr_file}'")
- if ANDROID_LOG_TAGS != "*:i" and "D" in skip_tag_set:
- run(fr"sed -i -E '/^(Time zone|I18n) APEX ICU file found/d' '{args.stderr_file}'")
- else:
- # With a thread dump that uses gdb if a timeout.
- proc = run(cmdline, check=False)
- # TODO: Spawn a watcher process.
- raise Exception("Not implemented")
- # ( sleep {TIME_OUT_VALUE} && \
- # echo "##### Thread dump using gdb on test timeout" && \
- # ( gdb -q -p {pid} --eval-command="info thread" --eval-command="thread apply all bt" \
- # --eval-command="call exit(124)" --eval-command=quit || \
- # kill {pid} )) 2> /dev/null & watcher=$!
- test_exit_status = proc.returncode
- # pkill -P {watcher} 2> /dev/null # kill the sleep which will in turn end the watcher as well
- if proc.returncode == 124 and TIME_OUT == "timeout":
- print("\e[91mTEST TIMED OUT!\e[0m", file=sys.stderr)
- assert proc.returncode == args.expected_exit_code, f"exit code: {proc.returncode}"
+ # Remove unwanted log messages from stderr before diffing with the expected output.
+ # NB: The unwanted log line can be interleaved in the middle of wanted stderr printf.
+ # In particular, unhandled exception is printed using several unterminated printfs.
+ ALL_LOG_TAGS = ["V", "D", "I", "W", "E", "F", "S"]
+ skip_tag_set = "|".join(ALL_LOG_TAGS[:ALL_LOG_TAGS.index(args.diff_min_log_tag.upper())])
+ skip_reg_exp = fr'[[:alnum:]]+ ({skip_tag_set}) #-# #:#:# [^\n]*\n'.replace('#', '[0-9]+')
+ ctx.run(fr"sed -i -z -E 's/{skip_reg_exp}//g' '{args.stderr_file}'")
+ if not HAVE_IMAGE:
+ message = "(Unable to open file|Could not create image space)"
+ ctx.run(fr"sed -i -E '/^dalvikvm(|32|64) E .* {message}/d' '{args.stderr_file}'")
+ if ANDROID_LOG_TAGS != "*:i" and "D" in skip_tag_set:
+ ctx.run(fr"sed -i -E '/^(Time zone|I18n) APEX ICU file found/d' '{args.stderr_file}'")
diff --git a/test/run-test b/test/run-test
index 4612622..e283079 100755
--- a/test/run-test
+++ b/test/run-test
@@ -24,7 +24,7 @@
from pathlib import Path
from shutil import copyfile
from testrunner import env
-from typing import Optional
+from typing import Optional, Dict, List
from zipfile import ZipFile
COLOR = (os.environ.get("LUCI_CONTEXT") == None) # Disable colors on LUCI.
@@ -47,27 +47,51 @@
# Context passed to individual tests to let them customize the behaviour.
class RunTestContext:
- def __init__(self, tmp_dir: Path, target: bool, chroot, dex_location) -> None:
+ def __init__(self, tmp_dir: Path, target: bool, chroot, dex_location, test_name) -> None:
self.env = Environment()
self.target = target
self.chroot = chroot
self.dex_location = dex_location
+ self.test_name = test_name
# Note: The expected path can be modified by the tests.
self.expected_stdout = tmp_dir / "expected-stdout.txt"
self.expected_stderr = tmp_dir / "expected-stderr.txt"
+ self.runner: List[str] = ["#!/bin/bash"]
+
def echo(self, text):
self.run(f"echo {text} > {test_stdout}")
- # Let the test execute arbitrary bash command.
- # The command is executed in the test directory (on device for target tests).
- def run(self, cmd, check=True):
- if self.target:
- cmd = "sh -c 'cd {}; {}'".format(self.dex_location, cmd.replace("'", r"'\''"))
- cmd = "adb shell 'chroot {} {}'".format(self.chroot, cmd.replace("'", r"'\''"))
- print(f"{COLOR_BLUE}$ {cmd}{COLOR_NORMAL}", flush=True)
- subprocess.run(cmd, shell=True, cwd=tmp_dir, stderr=subprocess.STDOUT, check=check)
+ def export(self, **env: str) -> None:
+ self.runner.append("")
+ for name, value in env.items():
+ self.runner.append(f"export {name}={value}")
+
+ # Add "runner" script command. It is not executed now.
+ # All "runner" commands are executed later via single bash call.
+ def run(self, cmd: str, check: bool=True, expected_exit_code: int=0, desc:str = None) -> None:
+ if cmd == "true":
+ return
+ cmd_esc = cmd.replace("'", r"'\''")
+ self.runner.append("")
+ self.runner.append(f"echo '{COLOR_BLUE}$$ {cmd_esc}{COLOR_NORMAL}'")
+ self.runner.append(cmd)
+
+ # Check the exit code.
+ if check:
+ caller = getframeinfo(currentframe().f_back) # type: ignore
+ source = "{}:{}".format(Path(caller.filename).name, caller.lineno)
+ msg = f"{self.test_name} FAILED: [{source}] "
+ msg += "{} returned exit code ${{exit_code}}.".format(desc or "Command")
+ if expected_exit_code:
+ msg += f" Expected {expected_exit_code}."
+ self.runner.append(
+ f"exit_code=$?; if [ $exit_code -ne {expected_exit_code} ]; then "
+ f"echo {COLOR_RED}{msg}{COLOR_NORMAL}; exit 100; "
+ f"fi; ")
+ else:
+ self.runner.append("true; # Ignore previous exit code")
# Execute the default runner (possibly with modified arguments).
def default_run(self, args, **kwargs):
@@ -91,23 +115,18 @@
def fail(message: str, caller:Optional[FrameInfo]=None):
caller = caller or getframeinfo(currentframe().f_back) # type: ignore
- source = "{}:{}".format(Path(caller.filename).relative_to(oldwd), caller.lineno)
- print(f"\n{COLOR_RED}{TEST_NAME} FAILED: [{source}] {message}{COLOR_NORMAL}",
+ assert caller
+ source = "{}:{}".format(Path(caller.filename).name, caller.lineno)
+ print(f"{COLOR_RED}{TEST_NAME} FAILED: [{source}] {message}{COLOR_NORMAL}",
file=sys.stderr)
sys.exit(1)
- def run(cmdline: str,
- check=True,
- fail_message=None,
- capture_output=True) -> subprocess.CompletedProcess:
+ def run(cmdline: str, check=True, fail_message=None) -> subprocess.CompletedProcess:
+ print(f"{COLOR_BLUE}$ {cmdline}{COLOR_NORMAL}", flush=True)
proc = subprocess.run([cmdline],
shell=True,
- encoding="utf8",
- capture_output=capture_output)
- if (check and proc.returncode != 0) or (quiet == "no"):
- print(f"{COLOR_BLUE}$ {cmdline}{COLOR_NORMAL}")
- print(proc.stdout or "", file=sys.stdout, end="", flush=True)
- print(COLOR_RED + (proc.stderr or "") + COLOR_NORMAL, file=sys.stderr, end="", flush=True)
+ executable="/bin/bash",
+ stderr=subprocess.STDOUT)
if (check and proc.returncode != 0):
if fail_message:
# If we have custom fail message, exit without printing the full backtrace.
@@ -638,40 +657,7 @@
# Try to map the suffix64 flag and what we find in {ANDROID_PRODUCT_OUT}/data/art-test to an architecture name.
def guess_target_arch_name():
- # Check whether this is a device with native bridge. Currently this is hardcoded
- # to x86 + arm.
- guess_path = f"{chroot}/system/framework/art_boot_images"
- # Use check=False because if grep does not match anything, it returns exit code 1.
- x86_arm = run(
- f"adb shell ls {guess_path} | sort | grep -E '^(arm|x86)$'",
- check=False).stdout
- # Collapse line-breaks into spaces
- x86_arm = x86_arm.replace("\n", " ")
- if x86_arm == "arm x86":
- error("Native-bridge configuration detected.")
- # We only support the main arch for tests.
- if suffix64 == "64":
- target_arch_name = ""
- else:
- target_arch_name = "x86"
- else:
- # Use check=False because if grep does not match anything, it returns exit code 1.
- grep32bit = run(
- f"adb shell ls {guess_path} | grep -E '^(arm|x86)$'",
- check=False).stdout
- grep64bit = run(
- f"adb shell ls {guess_path} | grep -E '^(arm64|x86_64)$'",
- check=False).stdout
- if suffix64 == "64":
- target_arch_name = grep64bit
- else:
- target_arch_name = grep32bit
-
- # Check that the heuristics matches build configuration.
- arch = get_target_arch(suffix64 == "64")
- assert target_arch_name.strip() == arch, f"{target_arch_name.strip()} != {arch}"
-
- return target_arch_name.strip()
+ return get_target_arch(suffix64 == "64")
def guess_host_arch_name():
if suffix64 == "64":
@@ -963,7 +949,7 @@
# TODO: Run this in global try-finally once the script is more refactored.
atexit.register(clean_up, passed=False)
- ctx = RunTestContext(Path(tmp_dir), target_mode == "yes", chroot, DEX_LOCATION)
+ ctx = RunTestContext(Path(tmp_dir), target_mode == "yes", chroot, DEX_LOCATION, TEST_NAME)
td_info = f"{test_dir}/{info}"
for td_file in [td_info, ctx.expected_stdout, ctx.expected_stderr]:
assert os.access(td_file, os.R_OK)
@@ -971,46 +957,62 @@
joined_run_args = " ".join(run_args)
joined_args = " ".join(args)
- # Execute the "run" method in the per-test specific script file.
- def run_test_script():
+ # Create runner (bash script that executes the whole test)
+ def create_runner_script() -> Path:
parsed_args = default_run_module.parse_args(shlex.split(" ".join(run_args + args)))
parsed_args.stdout_file = os.path.join(DEX_LOCATION, test_stdout)
parsed_args.stderr_file = os.path.join(DEX_LOCATION, test_stderr)
- script = os.path.join(tmp_dir, "run.py")
- if os.path.exists(script):
- module = SourceFileLoader("run_" + TEST_NAME, script).load_module()
+ ctx.run(f"cd {DEX_LOCATION}")
+ if target_mode != "yes":
+ # Make "out" directory accessible from test directory.
+ ctx.run(f"ln -s -f -t {DEX_LOCATION} {ANDROID_BUILD_TOP}/out")
+ # Clear the stdout/stderr files (create empty files).
+ ctx.run(f"echo -n > {test_stdout} && echo -n > {test_stderr}")
+
+ script = Path(tmp_dir) / "run.py"
+ if script.exists():
+ module = SourceFileLoader("run_" + TEST_NAME, str(script)).load_module()
module.run(ctx, parsed_args)
else:
default_run_module.default_run(ctx, parsed_args)
+ runner = Path(tmp_dir) / "run.sh"
+ runner.write_text("\n".join(ctx.runner))
+ runner.chmod(0o777)
+ return runner
+
# Test might not execute anything but we still expect the output files to exist.
Path(test_stdout).touch()
Path(test_stderr).touch()
export("TEST_RUNTIME", runtime)
- print(f"{test_dir}: running...")
+ print(f"{test_dir}: Create runner script...")
+ runner = create_runner_script()
+
+ print(f"{test_dir}: Run...")
if target_mode == "yes":
# Prepare the on-device test directory
run("adb root")
run("adb wait-for-device")
run(f"adb shell 'rm -rf {chroot_dex_location} && mkdir -p {chroot_dex_location}'")
- push_files = list(Path(".").glob(f"{TEST_NAME}*.jar"))
+ push_files = [Path(runner.name)]
+ push_files += list(Path(".").glob(f"{TEST_NAME}*.jar"))
push_files += list(Path(".").glob(f"expected-*.txt"))
push_files += [p for p in [Path("profile"), Path("res")] if p.exists()]
- if push_files:
- run("adb push {} {}".format(" ".join(map(str, push_files)), chroot_dex_location))
- run(f"adb shell 'echo -n > {chroot_dex_location}/{test_stdout}'") # Empty file.
- run(f"adb shell 'echo -n > {chroot_dex_location}/{test_stderr}'") # Empty file.
+ run("adb push {} {}".format(" ".join(map(str, push_files)), chroot_dex_location))
- run_test_script()
+ chroot_prefix = f"chroot {chroot}" if chroot else ""
+ run(f"adb shell {chroot_prefix} sh {DEX_LOCATION}/run.sh",
+ fail_message=f"Runner {chroot_dex_location}/run.sh failed")
# Copy the on-device stdout/stderr to host.
pull_files = [test_stdout, test_stderr, "expected-stdout.txt", "expected-stderr.txt"]
run("adb pull {} .".format(" ".join(f"{chroot_dex_location}/{f}" for f in pull_files)))
else:
- run_test_script()
+ run(str(runner), fail_message=f"Runner {str(runner)} failed")
+
# NB: There is no exit code or return value.
# Failing tests just raise python exception.
os.chdir(tmp_dir)