Add ADB crash debugging for run-tests.

We see very flaky crashes on the build bots from adb commands.

Wrap the adb commands in such a way that allows us to distinguish
whether it is adb itself crashing or the target command crashing.

Test: ./art/test.py -r --target --optimizing --64
Change-Id: Ic4d3482f3d2d0ff5412df6ab51b2cc0b6d54e45a

1 files changed, 14 insertions, 1 deletions

diff --git a/test/etc/default_run.py b/test/etc/default_run.py
index 82037416ee..65e340bae0 100755
--- a/test/etc/default_run.py
+++ b/test/etc/default_run.py
@@ -158,6 +158,7 @@ def default_run(ctx, args, **kwargs):
           stdout_file=None,
           stderr_file=None,
           check=True,
+          parse_exit_code_from_stdout=False,
           expected_exit_code=0,
           save_cmd=True) -> subprocess.CompletedProcess:
     env.setdefault("PATH", PATH)  # Ensure that PATH is always set.
@@ -181,6 +182,17 @@ def default_run(ctx, args, **kwargs):
                           encoding="utf8",
                           capture_output=True)
 
+    # ADB forwards exit code from the executed command, but if ADB process itself crashes,
+    # it will also return non-zero exit code, and we can not distinguish those two cases.
+    # As a work-around, we wrap the executed command so that it always returns 0 exit code
+    # and we also make it print the actual exit code as the last line of its stdout.
+    if parse_exit_code_from_stdout:
+      assert proc.returncode == 0, f"ADB crashed (out:'{proc.stdout}' err:'{proc.stderr}')"
+      found = re.search("exit_code=([0-9]+)$", proc.stdout)
+      assert found, "Expected exit code as the last line of stdout"
+      proc.stdout = proc.stdout[:found.start(0)]  # Remove the exit code from stdout.
+      proc.returncode = int(found.group(1))  # Use it as if it was the process exit code.
+
     # Save copy of the output on disk.
     if stdout_file:
       with open(stdout_file, "a") as f:
@@ -222,7 +234,8 @@ def default_run(ctx, args, **kwargs):
       run("adb wait-for-device", self.env)
 
     def shell(self, cmdline: str, **kwargs) -> subprocess.CompletedProcess:
-      return run("adb shell " + cmdline, self.env, **kwargs)
+      return run(f"adb shell '{cmdline}; echo exit_code=$?'", self.env,
+                 parse_exit_code_from_stdout=True, **kwargs)
 
     def push(self, src: str, dst: str, **kwargs) -> None:
       run(f"adb push {src} {dst}", self.env, **kwargs)