Preparing to switch away from the deprecated test runner

A lot of ravenwood tests (and other tests too) use the deprecated
test runner androidx.test.runner.AndroidJUnit4, but it actually has
problems. We should switch them to
androidx.test.ext.junit.runners.AndroidJUnit4.

However, simply updating all tests could be problematic because:
- we need to update the build rules and add "androidx.test.ext.junit"
as a static dependency, which is probably safe but it could
cause class conflicts.(?)
- More importantly, the subtle difference between the test runner
could make the device side test fail.

So I'm going to do it in two steps:

- Step 1 -- this CL:
Add an optional validator to RavenwoodTest, which can be enabled
with RAVENWOOD_OPTIONAL_VALIDATION=1.

This makes a test fail if it's using the deprecated one.

- Added a test for this

- Added a script to replace the test runner.

Test: run-ravenwood-tests
Test: RAVENWOOD_OPTIONAL_VALIDATION=1 atest RavenwoodCoreTest
Bug: 317131861

Change-Id: I5762bd097c3a529ae6e14460d9e326ed5c6086b4

9 files changed, 203 insertions, 6 deletions

diff --git a/ravenwood/bulk_enable.py b/ravenwood/bulk_enable.py
index 83fda9e72c3c..aafaaff7561a 100644
--- a/ravenwood/bulk_enable.py
+++ b/ravenwood/bulk_enable.py
@@ -21,7 +21,7 @@ Currently only offers to include classes which are fully passing; ignores
 classes that have partial success.
 
 Typical usage:
-$ ENABLE_PROBE_IGNORED=1 atest MyTestsRavenwood
+$ RAVENWOOD_RUN_DISABLED_TESTS=1 atest MyTestsRavenwood
 $ cd /path/to/tests/root
 $ python bulk_enable.py /path/to/atest/output/host_log.txt
 """
diff --git a/ravenwood/coretest/Android.bp b/ravenwood/coretest/Android.bp
new file mode 100644
index 000000000000..9b7f8f7d2171
--- /dev/null
+++ b/ravenwood/coretest/Android.bp
@@ -0,0 +1,23 @@
+package {
+    // See: http://go/android-license-faq
+    // A large-scale-change added 'default_applicable_licenses' to import
+    // all of the 'license_kinds' from "frameworks_base_license"
+    // to get the below license kinds:
+    //   SPDX-license-identifier-Apache-2.0
+    default_applicable_licenses: ["frameworks_base_license"],
+}
+
+android_ravenwood_test {
+    name: "RavenwoodCoreTest",
+
+    static_libs: [
+        "androidx.annotation_annotation",
+        "androidx.test.rules",
+    ],
+
+    srcs: [
+        "test/**/*.java",
+    ],
+    sdk_version: "test_current",
+    auto_gen_config: true,
+}
diff --git a/ravenwood/coretest/test/com/android/platform/test/ravenwood/coretest/RavenwoodTestRunnerValidationTest.java b/ravenwood/coretest/test/com/android/platform/test/ravenwood/coretest/RavenwoodTestRunnerValidationTest.java
new file mode 100644
index 000000000000..e58c282fb690
--- /dev/null
+++ b/ravenwood/coretest/test/com/android/platform/test/ravenwood/coretest/RavenwoodTestRunnerValidationTest.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.platform.test.ravenwood.coretest;
+
+import android.platform.test.ravenwood.RavenwoodRule;
+
+import androidx.test.runner.AndroidJUnit4;
+
+import org.junit.Assume;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.rules.RuleChain;
+import org.junit.runner.RunWith;
+
+/**
+ * Test for the test runner validator in RavenwoodRule.
+ */
+@RunWith(AndroidJUnit4.class)
+public class RavenwoodTestRunnerValidationTest {
+    // Note the following rules don't have a @Rule, because they need to be applied in a specific
+    // order. So we use a RuleChain instead.
+    private ExpectedException mThrown = ExpectedException.none();
+    private final RavenwoodRule mRavenwood = new RavenwoodRule();
+
+    @Rule
+    public final RuleChain chain = RuleChain.outerRule(mThrown).around(mRavenwood);
+
+    public RavenwoodTestRunnerValidationTest() {
+        Assume.assumeTrue(mRavenwood._ravenwood_private$isOptionalValidationEnabled());
+        // Because RavenwoodRule will throw this error before executing the test method,
+        // we can't do it in the test method itself.
+        // So instead, we initialize it here.
+        mThrown.expectMessage("Switch to androidx.test.ext.junit.runners.AndroidJUnit4");
+    }
+
+    @Test
+    public void testValidateTestRunner() {
+    }
+}
diff --git a/ravenwood/fix_test_runner.py b/ravenwood/fix_test_runner.py
new file mode 100755
index 000000000000..99b7a1ff226b
--- /dev/null
+++ b/ravenwood/fix_test_runner.py
@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2024 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Tool switch the deprecated jetpack test runner to the correct one.
+
+Typical usage:
+$ RAVENWOOD_OPTIONAL_VALIDATION=1 atest MyTestsRavenwood # Prepend RAVENWOOD_RUN_DISABLED_TESTS=1 as needed
+$ cd /path/to/tests/root
+$ python bulk_enable.py /path/to/atest/output/host_log.txt
+"""
+
+import collections
+import os
+import re
+import subprocess
+import sys
+
+re_result = re.compile("I/ModuleListener.+?null-device-0 (.+?)#(.+?) ([A-Z_]+)(.*)$")
+
+OLD_RUNNER = "androidx.test.runner.AndroidJUnit4"
+NEW_RUNNER = "androidx.test.ext.junit.runners.AndroidJUnit4"
+SED_ARG = r"s/%s/%s/g" % (OLD_RUNNER, NEW_RUNNER)
+
+target = collections.defaultdict()
+
+with open(sys.argv[1]) as f:
+    for line in f.readlines():
+        result = re_result.search(line)
+        if result:
+            clazz, method, state, msg = result.groups()
+            if NEW_RUNNER in msg:
+                target[clazz] = 1
+
+if len(target) == 0:
+    print("No tests need updating.")
+    sys.exit(0)
+
+num_fixed = 0
+for clazz in target.keys():
+    print("Fixing test runner", clazz)
+    clazz_match = re.compile("%s\.(kt|java)" % (clazz.split(".")[-1]))
+    found = False
+    for root, dirs, files in os.walk("."):
+        for f in files:
+            if clazz_match.match(f):
+                found = True
+                num_fixed += 1
+                path = os.path.join(root, f)
+                subprocess.run(["sed", "-i", "-E", SED_ARG, path])
+    if not found:
+        print(f"  Warining: tests {clazz} not found")
+
+
+print("Tests fixed", num_fixed)
diff --git a/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodRuleImpl.java b/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodRuleImpl.java
index 8af561fbd273..85fa65ac705c 100644
--- a/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodRuleImpl.java
+++ b/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodRuleImpl.java
@@ -28,7 +28,10 @@ import androidx.test.platform.app.InstrumentationRegistry;
 
 import com.android.internal.os.RuntimeInit;
 
+import org.junit.Assert;
 import org.junit.runner.Description;
+import org.junit.runner.RunWith;
+import org.junit.runners.model.Statement;
 
 import java.io.PrintStream;
 import java.util.Map;
@@ -167,4 +170,31 @@ public class RavenwoodRuleImpl {
             }
         }
     }
+
+    public static void validate(Statement base, Description description,
+            boolean enableOptionalValidation) {
+        validateTestRunner(base, description, enableOptionalValidation);
+    }
+
+    private static void validateTestRunner(Statement base, Description description,
+            boolean shouldFail) {
+        final var testClass = description.getTestClass();
+        final var runWith = testClass.getAnnotation(RunWith.class);
+        if (runWith == null) {
+            return;
+        }
+
+        // Due to build dependencies, we can't directly refer to androidx classes here,
+        // so just check the class name instead.
+        if (runWith.value().getCanonicalName().equals("androidx.test.runner.AndroidJUnit4")) {
+            var message = "Test " + testClass.getCanonicalName() + " uses deprecated"
+                    + " test runner androidx.test.runner.AndroidJUnit4."
+                    + " Switch to androidx.test.ext.junit.runners.AndroidJUnit4.";
+            if (shouldFail) {
+                Assert.fail(message);
+            } else {
+                System.err.println("Warning: " + message);
+            }
+        }
+    }
 }
diff --git a/ravenwood/junit-src/android/platform/test/ravenwood/RavenwoodRule.java b/ravenwood/junit-src/android/platform/test/ravenwood/RavenwoodRule.java
index daed457fb695..764573defd7a 100644
--- a/ravenwood/junit-src/android/platform/test/ravenwood/RavenwoodRule.java
+++ b/ravenwood/junit-src/android/platform/test/ravenwood/RavenwoodRule.java
@@ -89,6 +89,12 @@ public class RavenwoodRule implements TestRule {
     private static final boolean ENABLE_REALLY_DISABLE_PATTERN =
             !REALLY_DISABLE_PATTERN.pattern().isEmpty();
 
+    /**
+     * If true, enable optional validation on running tests.
+     */
+    private static final boolean ENABLE_OPTIONAL_VALIDATION = "1".equals(
+            System.getenv("RAVENWOOD_OPTIONAL_VALIDATION"));
+
     static {
         if (ENABLE_PROBE_IGNORED) {
             System.out.println("$RAVENWOOD_RUN_DISABLED_TESTS enabled: force running all tests");
@@ -275,6 +281,12 @@ public class RavenwoodRule implements TestRule {
         }
     }
 
+    private void commonPrologue(Statement base, Description description) {
+        RavenwoodRuleImpl.logTestRunner("started", description);
+        RavenwoodRuleImpl.validate(base, description, ENABLE_OPTIONAL_VALIDATION);
+        RavenwoodRuleImpl.init(RavenwoodRule.this);
+    }
+
     /**
      * Run the given {@link Statement} with no special treatment.
      */
@@ -284,8 +296,7 @@ public class RavenwoodRule implements TestRule {
             public void evaluate() throws Throwable {
                 Assume.assumeTrue(shouldEnableOnRavenwood(description));
 
-                RavenwoodRuleImpl.logTestRunner("started", description);
-                RavenwoodRuleImpl.init(RavenwoodRule.this);
+                commonPrologue(base, description);
                 try {
                     base.evaluate();
                     RavenwoodRuleImpl.logTestRunner("finished", description);
@@ -310,8 +321,7 @@ public class RavenwoodRule implements TestRule {
             public void evaluate() throws Throwable {
                 Assume.assumeFalse(shouldStillIgnoreInProbeIgnoreMode(description));
 
-                RavenwoodRuleImpl.logTestRunner("started", description);
-                RavenwoodRuleImpl.init(RavenwoodRule.this);
+                commonPrologue(base, description);
                 try {
                     base.evaluate();
                 } catch (Throwable t) {
@@ -331,4 +341,11 @@ public class RavenwoodRule implements TestRule {
             }
         };
     }
+
+    /**
+     * Do not use it outside ravenwood core classes.
+     */
+    public boolean _ravenwood_private$isOptionalValidationEnabled() {
+        return ENABLE_OPTIONAL_VALIDATION;
+    }
 }
diff --git a/ravenwood/junit-stub-src/android/platform/test/ravenwood/RavenwoodRuleImpl.java b/ravenwood/junit-stub-src/android/platform/test/ravenwood/RavenwoodRuleImpl.java
index 7d172f2a83c0..e951351bd829 100644
--- a/ravenwood/junit-stub-src/android/platform/test/ravenwood/RavenwoodRuleImpl.java
+++ b/ravenwood/junit-stub-src/android/platform/test/ravenwood/RavenwoodRuleImpl.java
@@ -17,6 +17,7 @@
 package android.platform.test.ravenwood;
 
 import org.junit.runner.Description;
+import org.junit.runners.model.Statement;
 
 public class RavenwoodRuleImpl {
     public static boolean isOnRavenwood() {
@@ -34,4 +35,8 @@ public class RavenwoodRuleImpl {
     public static void logTestRunner(String label, Description description) {
         // No-op when running on a real device
     }
+
+    public static void validate(Statement base, Description description,
+            boolean enableOptionalValidation) {
+    }
 }
diff --git a/ravenwood/minimum-test/Android.bp b/ravenwood/minimum-test/Android.bp
index bf3583cebd2c..e4ed3d5b0b73 100644
--- a/ravenwood/minimum-test/Android.bp
+++ b/ravenwood/minimum-test/Android.bp
@@ -13,6 +13,7 @@ android_ravenwood_test {
 
     static_libs: [
         "androidx.annotation_annotation",
+        "androidx.test.ext.junit",
         "androidx.test.rules",
     ],
 
diff --git a/ravenwood/minimum-test/test/com/android/ravenwood/RavenwoodMinimumTest.java b/ravenwood/minimum-test/test/com/android/ravenwood/RavenwoodMinimumTest.java
index 7abfecf0e424..03cfad5c7a70 100644
--- a/ravenwood/minimum-test/test/com/android/ravenwood/RavenwoodMinimumTest.java
+++ b/ravenwood/minimum-test/test/com/android/ravenwood/RavenwoodMinimumTest.java
@@ -18,7 +18,7 @@ package com.android.ravenwood;
 import android.platform.test.annotations.IgnoreUnderRavenwood;
 import android.platform.test.ravenwood.RavenwoodRule;
 
-import androidx.test.runner.AndroidJUnit4;
+import androidx.test.ext.junit.runners.AndroidJUnit4;
 
 import org.junit.Assert;
 import org.junit.Rule;