Add ADB crash debugging for run-tests.
We see very flaky crashes on the build bots from adb commands.
Wrap the adb commands in such a way that allows us to distinguish
whether it is adb itself crashing or the target command crashing.
Test: ./art/test.py -r --target --optimizing --64
Change-Id: Ic4d3482f3d2d0ff5412df6ab51b2cc0b6d54e45a
diff --git a/test/etc/default_run.py b/test/etc/default_run.py
index 8203741..65e340b 100755
--- a/test/etc/default_run.py
+++ b/test/etc/default_run.py
@@ -158,6 +158,7 @@
stdout_file=None,
stderr_file=None,
check=True,
+ parse_exit_code_from_stdout=False,
expected_exit_code=0,
save_cmd=True) -> subprocess.CompletedProcess:
env.setdefault("PATH", PATH) # Ensure that PATH is always set.
@@ -181,6 +182,17 @@
encoding="utf8",
capture_output=True)
+ # ADB forwards exit code from the executed command, but if ADB process itself crashes,
+ # it will also return non-zero exit code, and we can not distinguish those two cases.
+ # As a work-around, we wrap the executed command so that it always returns 0 exit code
+ # and we also make it print the actual exit code as the last line of its stdout.
+ if parse_exit_code_from_stdout:
+ assert proc.returncode == 0, f"ADB crashed (out:'{proc.stdout}' err:'{proc.stderr}')"
+ found = re.search("exit_code=([0-9]+)$", proc.stdout)
+ assert found, "Expected exit code as the last line of stdout"
+ proc.stdout = proc.stdout[:found.start(0)] # Remove the exit code from stdout.
+ proc.returncode = int(found.group(1)) # Use it as if it was the process exit code.
+
# Save copy of the output on disk.
if stdout_file:
with open(stdout_file, "a") as f:
@@ -222,7 +234,8 @@
run("adb wait-for-device", self.env)
def shell(self, cmdline: str, **kwargs) -> subprocess.CompletedProcess:
- return run("adb shell " + cmdline, self.env, **kwargs)
+ return run(f"adb shell '{cmdline}; echo exit_code=$?'", self.env,
+ parse_exit_code_from_stdout=True, **kwargs)
def push(self, src: str, dst: str, **kwargs) -> None:
run(f"adb push {src} {dst}", self.env, **kwargs)