Move @DisabledOnRavenwood logic to the runner side

Now the whole logic about "@DisabledOnRavenwood" is moved to the runner,
with "RUN_DISABLED_TESTS" support too.
RavenwoodRule is no longer needed to handle @DisabledOnRavenwood.

Also create test result stats files, which contains number of tests
passed/failed/skipped.

Bug: 356918135
Test: $ANDROID_BUILD_TOP/frameworks/base/ravenwood/scripts/run-ravenwood-tests.sh
Test: atest RavenwoodBivalentTest_device
Flag: EXEMPT host test change only

Change-Id: I8e1ee027e7cac9014cdc2b67b8f748b0922a536e

14 files changed, 671 insertions, 206 deletions

diff --git a/ravenwood/Android.bp b/ravenwood/Android.bp
index be4cd761a4ec..9b0c8e554d64 100644
--- a/ravenwood/Android.bp
+++ b/ravenwood/Android.bp
@@ -160,6 +160,7 @@ java_library {
         "ravenwood-framework",
         "services.core.ravenwood",
         "junit",
+        "framework-annotations-lib",
     ],
     sdk_version: "core_current",
     visibility: ["//frameworks/base"],
diff --git a/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/RavenwoodClassRuleDeviceOnlyTest.java b/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/RavenwoodClassRuleDeviceOnlyTest.java
index 3a24c0e829a4..e8f59db86901 100644
--- a/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/RavenwoodClassRuleDeviceOnlyTest.java
+++ b/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/RavenwoodClassRuleDeviceOnlyTest.java
@@ -26,6 +26,10 @@ import org.junit.ClassRule;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 
+/**
+ * Test to ensure @DisabledOnRavenwood works. Note, now the DisabledOnRavenwood annotation
+ * is handled by the test runner, so it won't really need the class rule.
+ */
 @RunWith(AndroidJUnit4.class)
 @DisabledOnRavenwood
 public class RavenwoodClassRuleDeviceOnlyTest {
diff --git a/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/ravenizer/RavenwoodImplicitClassRuleDeviceOnlyTest.java b/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/ravenizer/RavenwoodImplicitClassRuleDeviceOnlyTest.java
index 0f8be0eeebeb..7ef672e80bee 100644
--- a/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/ravenizer/RavenwoodImplicitClassRuleDeviceOnlyTest.java
+++ b/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/ravenizer/RavenwoodImplicitClassRuleDeviceOnlyTest.java
@@ -34,7 +34,12 @@ public class RavenwoodImplicitClassRuleDeviceOnlyTest {
 
     @BeforeClass
     public static void beforeClass() {
-        Assert.assertFalse(RavenwoodRule.isOnRavenwood());
+        // This method shouldn't be called -- unless RUN_DISABLED_TESTS is enabled.
+
+        // If we're doing RUN_DISABLED_TESTS, don't throw here, because that'd confuse junit.
+        if (!RavenwoodRule.private$ravenwood().isRunningDisabledTests()) {
+            Assert.assertFalse(RavenwoodRule.isOnRavenwood());
+        }
     }
 
     @Test
@@ -46,7 +51,10 @@ public class RavenwoodImplicitClassRuleDeviceOnlyTest {
     public static void afterClass() {
         if (RavenwoodRule.isOnRavenwood()) {
             Log.e(TAG, "Even @AfterClass shouldn't be executed!");
-            System.exit(1);
+
+            if (!RavenwoodRule.private$ravenwood().isRunningDisabledTests()) {
+                System.exit(1);
+            }
         }
     }
 }
diff --git a/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/ravenizer/RavenwoodRunDisabledTestsReallyDisabledTest.java b/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/ravenizer/RavenwoodRunDisabledTestsReallyDisabledTest.java
new file mode 100644
index 000000000000..c77841b1b55a
--- /dev/null
+++ b/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/ravenizer/RavenwoodRunDisabledTestsReallyDisabledTest.java
@@ -0,0 +1,88 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.ravenwoodtest.bivalenttest.ravenizer;
+
+import static org.junit.Assert.fail;
+
+import android.platform.test.annotations.DisabledOnRavenwood;
+import android.platform.test.ravenwood.RavenwoodAwareTestRunner.RavenwoodTestRunnerInitializing;
+import android.platform.test.ravenwood.RavenwoodRule;
+import android.util.Log;
+
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+
+import org.junit.AfterClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+/**
+ * Test for "RAVENWOOD_RUN_DISABLED_TESTS" with "REALLY_DISABLED" set.
+ *
+ * This test is only executed on Ravenwood.
+ */
+@RunWith(AndroidJUnit4.class)
+public class RavenwoodRunDisabledTestsReallyDisabledTest {
+    private static final String TAG = "RavenwoodRunDisabledTestsTest";
+
+    private static final CallTracker sCallTracker = new CallTracker();
+
+    @RavenwoodTestRunnerInitializing
+    public static void ravenwoodRunnerInitializing() {
+        RavenwoodRule.private$ravenwood().overrideRunDisabledTest(true,
+                "\\#testReallyDisabled$");
+    }
+
+    /**
+     * This test gets to run with RAVENWOOD_RUN_DISABLED_TESTS set.
+     */
+    @Test
+    @DisabledOnRavenwood
+    public void testDisabledTestGetsToRun() {
+        if (!RavenwoodRule.isOnRavenwood()) {
+            return;
+        }
+        sCallTracker.incrementMethodCallCount();
+
+        fail("This test won't pass on Ravenwood.");
+    }
+
+    /**
+     * This will still not be executed due to the "really disabled" pattern.
+     */
+    @Test
+    @DisabledOnRavenwood
+    public void testReallyDisabled() {
+        if (!RavenwoodRule.isOnRavenwood()) {
+            return;
+        }
+        sCallTracker.incrementMethodCallCount();
+
+        fail("This test won't pass on Ravenwood.");
+    }
+
+    @AfterClass
+    public static void afterClass() {
+        if (!RavenwoodRule.isOnRavenwood()) {
+            return;
+        }
+        Log.i(TAG, "afterClass called");
+
+        sCallTracker.assertCallsOrDie(
+                "testDisabledTestGetsToRun", 1,
+                "testReallyDisabled", 0
+        );
+    }
+}
diff --git a/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/ravenizer/RavenwoodRunDisabledTestsTest.java b/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/ravenizer/RavenwoodRunDisabledTestsTest.java
new file mode 100644
index 000000000000..ea1a29d57482
--- /dev/null
+++ b/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/ravenizer/RavenwoodRunDisabledTestsTest.java
@@ -0,0 +1,87 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.ravenwoodtest.bivalenttest.ravenizer;
+
+import static org.junit.Assert.fail;
+
+import android.platform.test.annotations.DisabledOnRavenwood;
+import android.platform.test.ravenwood.RavenwoodAwareTestRunner.RavenwoodTestRunnerInitializing;
+import android.platform.test.ravenwood.RavenwoodRule;
+import android.util.Log;
+
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+
+import org.junit.AfterClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+
+/**
+ * Test for "RAVENWOOD_RUN_DISABLED_TESTS". (with no "REALLY_DISABLED" set.)
+ *
+ * This test is only executed on Ravenwood.
+ */
+@RunWith(AndroidJUnit4.class)
+public class RavenwoodRunDisabledTestsTest {
+    private static final String TAG = "RavenwoodRunDisabledTestsTest";
+
+    @Rule
+    public ExpectedException mExpectedException = ExpectedException.none();
+
+    private static final CallTracker sCallTracker = new CallTracker();
+
+    @RavenwoodTestRunnerInitializing
+    public static void ravenwoodRunnerInitializing() {
+        RavenwoodRule.private$ravenwood().overrideRunDisabledTest(true, null);
+    }
+
+    @Test
+    @DisabledOnRavenwood
+    public void testDisabledTestGetsToRun() {
+        if (!RavenwoodRule.isOnRavenwood()) {
+            return;
+        }
+        sCallTracker.incrementMethodCallCount();
+
+        fail("This test won't pass on Ravenwood.");
+    }
+
+    @Test
+    @DisabledOnRavenwood
+    public void testDisabledButPass() {
+        if (!RavenwoodRule.isOnRavenwood()) {
+            return;
+        }
+        sCallTracker.incrementMethodCallCount();
+
+        // When a @DisabledOnRavenwood actually passed, the runner should make fail().
+        mExpectedException.expectMessage("it actually passed under Ravenwood");
+    }
+
+    @AfterClass
+    public static void afterClass() {
+        if (!RavenwoodRule.isOnRavenwood()) {
+            return;
+        }
+        Log.i(TAG, "afterClass called");
+
+        sCallTracker.assertCallsOrDie(
+                "testDisabledTestGetsToRun", 1,
+                "testDisabledButPass", 1
+        );
+    }
+}
diff --git a/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodAwareTestRunnerHook.java b/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodAwareTestRunnerHook.java
index 03600ad5511f..1da93eba94f7 100644
--- a/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodAwareTestRunnerHook.java
+++ b/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodAwareTestRunnerHook.java
@@ -17,14 +17,16 @@ package android.platform.test.ravenwood;
 
 import static com.android.ravenwood.common.RavenwoodCommonUtils.RAVENWOOD_VERSION_JAVA_SYSPROP;
 
+import static org.junit.Assert.fail;
+
 import android.os.Bundle;
 import android.platform.test.ravenwood.RavenwoodAwareTestRunner.Order;
 import android.platform.test.ravenwood.RavenwoodAwareTestRunner.Scope;
+import android.platform.test.ravenwood.RavenwoodTestStats.Result;
+import android.util.Log;
 
 import androidx.test.platform.app.InstrumentationRegistry;
 
-import com.android.ravenwood.common.RavenwoodCommonUtils;
-
 import org.junit.runner.Description;
 import org.junit.runner.Runner;
 import org.junit.runners.model.TestClass;
@@ -38,12 +40,24 @@ public class RavenwoodAwareTestRunnerHook {
     private RavenwoodAwareTestRunnerHook() {
     }
 
-    private static void log(String message) {
-        RavenwoodCommonUtils.log(TAG, message);
+    private static RavenwoodTestStats sStats; // lazy initialization.
+    private static Description sCurrentClassDescription;
+
+    private static RavenwoodTestStats getStats() {
+        if (sStats == null) {
+            // We don't want to throw in the static initializer, because tradefed may not report
+            // it properly, so we initialize it here.
+            sStats = new RavenwoodTestStats();
+        }
+        return sStats;
     }
 
+    /**
+     * Called when a runner starts, before the inner runner gets a chance to run.
+     */
     public static void onRunnerInitializing(Runner runner, TestClass testClass) {
-        log("onRunnerStart: testClass=" + testClass + " runner=" + runner);
+        // This log call also ensures the framework JNI is loaded.
+        Log.i(TAG, "onRunnerInitializing: testClass=" + testClass + " runner=" + runner);
 
         // TODO: Move the initialization code to a better place.
 
@@ -52,26 +66,97 @@ public class RavenwoodAwareTestRunnerHook {
                 "androidx.test.internal.runner.junit4.AndroidJUnit4ClassRunner");
         System.setProperty(RAVENWOOD_VERSION_JAVA_SYSPROP, "1");
 
+
         // This is needed to make AndroidJUnit4ClassRunner happy.
         InstrumentationRegistry.registerInstance(null, Bundle.EMPTY);
     }
 
+    /**
+     * Called when a whole test class is skipped.
+     */
+    public static void onClassSkipped(Description description) {
+        Log.i(TAG, "onClassSkipped: description=" + description);
+        getStats().onClassSkipped(description);
+    }
+
+    /**
+     * Called before a test / class.
+     *
+     * Return false if it should be skipped.
+     */
     public static boolean onBefore(RavenwoodAwareTestRunner runner, Description description,
             Scope scope, Order order) {
-        log("onBefore: description=" + description + ", " + scope + ", " + order);
+        Log.i(TAG, "onBefore: description=" + description + ", " + scope + ", " + order);
+
+        if (scope == Scope.Class && order == Order.First) {
+            // Keep track of the current class.
+            sCurrentClassDescription = description;
+        }
 
         // Class-level annotations are checked by the runner already, so we only check
         // method-level annotations here.
         if (scope == Scope.Instance && order == Order.First) {
-            if (!RavenwoodRule.shouldEnableOnRavenwood(description)) {
+            if (!RavenwoodEnablementChecker.shouldEnableOnRavenwood(
+                    description, true)) {
+                getStats().onTestFinished(sCurrentClassDescription, description, Result.Skipped);
                 return false;
             }
         }
         return true;
     }
 
-    public static void onAfter(RavenwoodAwareTestRunner runner, Description description,
+    /**
+     * Called after a test / class.
+     *
+     * Return false if the exception should be ignored.
+     */
+    public static boolean onAfter(RavenwoodAwareTestRunner runner, Description description,
             Scope scope, Order order, Throwable th) {
-        log("onAfter: description=" + description + ", " + scope + ", " + order + ", " + th);
+        Log.i(TAG, "onAfter: description=" + description + ", " + scope + ", " + order + ", " + th);
+
+        if (scope == Scope.Instance && order == Order.First) {
+            getStats().onTestFinished(sCurrentClassDescription, description,
+                    th == null ? Result.Passed : Result.Failed);
+
+        } else if (scope == Scope.Class && order == Order.Last) {
+            getStats().onClassFinished(sCurrentClassDescription);
+        }
+
+        // If RUN_DISABLED_TESTS is set, and the method did _not_ throw, make it an error.
+        if (RavenwoodRule.private$ravenwood().isRunningDisabledTests()
+                && scope == Scope.Instance && order == Order.First) {
+
+            boolean isTestEnabled = RavenwoodEnablementChecker.shouldEnableOnRavenwood(
+                    description, false);
+            if (th == null) {
+                // Test passed. Is the test method supposed to be enabled?
+                if (isTestEnabled) {
+                    // Enabled and didn't throw, okay.
+                    return true;
+                } else {
+                    // Disabled and didn't throw. We should report it.
+                    fail("Test wasn't included under Ravenwood, but it actually "
+                            + "passed under Ravenwood; consider updating annotations");
+                    return true; // unreachable.
+                }
+            } else {
+                // Test failed.
+                if (isTestEnabled) {
+                    // Enabled but failed. We should throw the exception.
+                    return true;
+                } else {
+                    // Disabled and failed. Expected. Don't throw.
+                    return false;
+                }
+            }
+        }
+        return true;
+    }
+
+    /**
+     * Called by {@link RavenwoodAwareTestRunner} to see if it should run a test class or not.
+     */
+    public static boolean shouldRunClassOnRavenwood(Class<?> clazz) {
+        return RavenwoodEnablementChecker.shouldRunClassOnRavenwood(clazz, true);
     }
 }
diff --git a/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodEnablementChecker.java b/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodEnablementChecker.java
new file mode 100644
index 000000000000..77275c445dd9
--- /dev/null
+++ b/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodEnablementChecker.java
@@ -0,0 +1,117 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package android.platform.test.ravenwood;
+
+import android.annotation.NonNull;
+import android.annotation.Nullable;
+import android.platform.test.annotations.DisabledOnRavenwood;
+import android.platform.test.annotations.EnabledOnRavenwood;
+import android.platform.test.annotations.IgnoreUnderRavenwood;
+
+import org.junit.runner.Description;
+
+/**
+ * Calculates which tests need to be executed on Ravenwood.
+ */
+public class RavenwoodEnablementChecker {
+    private static final String TAG = "RavenwoodDisablementChecker";
+
+    private RavenwoodEnablementChecker() {
+    }
+
+    /**
+     * Determine if the given {@link Description} should be enabled when running on the
+     * Ravenwood test environment.
+     *
+     * A more specific method-level annotation always takes precedence over any class-level
+     * annotation, and an {@link EnabledOnRavenwood} annotation always takes precedence over
+     * an {@link DisabledOnRavenwood} annotation.
+     */
+    public static boolean shouldEnableOnRavenwood(Description description,
+            boolean takeIntoAccountRunDisabledTestsFlag) {
+        // First, consult any method-level annotations
+        if (description.isTest()) {
+            Boolean result = null;
+
+            // Stopgap for http://g/ravenwood/EPAD-N5ntxM
+            if (description.getMethodName().endsWith("$noRavenwood")) {
+                result = false;
+            } else if (description.getAnnotation(EnabledOnRavenwood.class) != null) {
+                result = true;
+            } else if (description.getAnnotation(DisabledOnRavenwood.class) != null) {
+                result = false;
+            } else if (description.getAnnotation(IgnoreUnderRavenwood.class) != null) {
+                result = false;
+            }
+            if (result != null) {
+                if (takeIntoAccountRunDisabledTestsFlag
+                        && RavenwoodRule.private$ravenwood().isRunningDisabledTests()) {
+                    result = !shouldStillIgnoreInProbeIgnoreMode(
+                            description.getTestClass(), description.getMethodName());
+                }
+            }
+            if (result != null) {
+                return result;
+            }
+        }
+
+        // Otherwise, consult any class-level annotations
+        return shouldRunClassOnRavenwood(description.getTestClass(),
+                takeIntoAccountRunDisabledTestsFlag);
+    }
+
+    public static boolean shouldRunClassOnRavenwood(@NonNull Class<?> testClass,
+            boolean takeIntoAccountRunDisabledTestsFlag) {
+        boolean result = true;
+        if (testClass.getAnnotation(EnabledOnRavenwood.class) != null) {
+            result = true;
+        } else if (testClass.getAnnotation(DisabledOnRavenwood.class) != null) {
+            result = false;
+        } else if (testClass.getAnnotation(IgnoreUnderRavenwood.class) != null) {
+            result = false;
+        }
+        if (!result) {
+            if (takeIntoAccountRunDisabledTestsFlag
+                    && RavenwoodRule.private$ravenwood().isRunningDisabledTests()) {
+                result = !shouldStillIgnoreInProbeIgnoreMode(testClass, null);
+            }
+        }
+        return result;
+    }
+
+    /**
+     * Check if a test should _still_ disabled even if {@code RUN_DISABLED_TESTS}
+     * is true, using {@code REALLY_DISABLED_PATTERN}.
+     *
+     * This only works on tests, not on classes.
+     */
+    static boolean shouldStillIgnoreInProbeIgnoreMode(
+            @NonNull Class<?> testClass, @Nullable String methodName) {
+        if (RavenwoodRule.private$ravenwood().getReallyDisabledPattern().pattern().isEmpty()) {
+            return false;
+        }
+
+        final var fullname = testClass.getName() + (methodName != null ? "#" + methodName : "");
+
+        System.out.println("XXX=" + fullname);
+
+        if (RavenwoodRule.private$ravenwood().getReallyDisabledPattern().matcher(fullname).find()) {
+            System.out.println("Still ignoring " + fullname);
+            return true;
+        }
+        return false;
+    }
+}
diff --git a/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodRuleImpl.java b/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodRuleImpl.java
index 7b4c17390942..a2088fd0b77f 100644
--- a/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodRuleImpl.java
+++ b/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodRuleImpl.java
@@ -40,7 +40,6 @@ import com.android.internal.os.RuntimeInit;
 import com.android.server.LocalServices;
 
 import org.junit.runner.Description;
-import org.junit.runners.model.Statement;
 
 import java.io.File;
 import java.io.IOException;
@@ -226,11 +225,6 @@ public class RavenwoodRuleImpl {
         }
     }
 
-    public static void validate(Statement base, Description description,
-            boolean enableOptionalValidation) {
-        // Nothing to check, for now.
-    }
-
     /**
      * Set the current configuration to the actual SystemProperties.
      */
diff --git a/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodTestStats.java b/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodTestStats.java
new file mode 100644
index 000000000000..631f68ff1dec
--- /dev/null
+++ b/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodTestStats.java
@@ -0,0 +1,163 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package android.platform.test.ravenwood;
+
+import android.util.Log;
+
+import org.junit.runner.Description;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Creats a "stats" CSV file containing the test results.
+ *
+ * The output file is created as `/tmp/Ravenwood-stats_[TEST-MODULE=NAME]_[TIMESTAMP].csv`.
+ * A symlink to the latest result will be created as
+ * `/tmp/Ravenwood-stats_[TEST-MODULE=NAME]_latest.csv`.
+ */
+public class RavenwoodTestStats {
+    private static final String TAG = "RavenwoodTestStats";
+    private static final String HEADER = "Module,Class,ClassDesc,Passed,Failed,Skipped";
+
+    public enum Result {
+        Passed,
+        Failed,
+        Skipped,
+    }
+
+    private final File mOutputFile;
+    private final PrintWriter mOutputWriter;
+    private final String mTestModuleName;
+
+    public final Map<Description, Map<Description, Result>> mStats = new HashMap<>();
+
+    /** Ctor */
+    public RavenwoodTestStats() {
+        mTestModuleName = guessTestModuleName();
+
+        var basename = "Ravenwood-stats_" + mTestModuleName + "_";
+
+        // Get the current time
+        LocalDateTime now = LocalDateTime.now();
+        DateTimeFormatter fmt = DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss");
+
+        var tmpdir = System.getProperty("java.io.tmpdir");
+        mOutputFile = new File(tmpdir, basename + now.format(fmt) + ".csv");
+
+        try {
+            mOutputWriter = new PrintWriter(mOutputFile);
+        } catch (IOException e) {
+            throw new RuntimeException("Failed to crete logfile. File=" + mOutputFile, e);
+        }
+
+        // Crete the "latest" symlink.
+        Path symlink = Paths.get(tmpdir, basename + "latest.csv");
+        try {
+            if (Files.exists(symlink)) {
+                Files.delete(symlink);
+            }
+            Files.createSymbolicLink(symlink, Paths.get(mOutputFile.getName()));
+
+        } catch (IOException e) {
+            throw new RuntimeException("Failed to crete logfile. File=" + mOutputFile, e);
+        }
+
+        Log.i(TAG, "Test result stats file: " + mOutputFile);
+
+        // Print the header.
+        mOutputWriter.println(HEADER);
+        mOutputWriter.flush();
+    }
+
+    private String guessTestModuleName() {
+        // Assume the current directory name is the test module name.
+        File cwd;
+        try {
+            cwd = new File(".").getCanonicalFile();
+        } catch (IOException e) {
+            throw new RuntimeException("Failed to get the current directory", e);
+        }
+        return cwd.getName();
+    }
+
+    private void addResult(Description classDescription, Description methodDescription,
+            Result result) {
+        mStats.compute(classDescription, (classDesc, value) -> {
+            if (value == null) {
+                value = new HashMap<>();
+            }
+            value.put(methodDescription, result);
+            return value;
+        });
+    }
+
+    public void onClassSkipped(Description classDescription) {
+        addResult(classDescription, Description.EMPTY, Result.Skipped);
+        onClassFinished(classDescription);
+    }
+
+    public void onTestFinished(Description classDescription, Description testDescription,
+            Result result) {
+        addResult(classDescription, testDescription, result);
+    }
+
+    public void onClassFinished(Description classDescription) {
+        int passed = 0;
+        int skipped = 0;
+        int failed = 0;
+        for (var e : mStats.get(classDescription).values()) {
+            switch (e) {
+                case Passed: passed++; break;
+                case Skipped: skipped++; break;
+                case Failed: failed++; break;
+            }
+        }
+
+        var testClass = extractTestClass(classDescription);
+
+        mOutputWriter.printf("%s,%s,%s,%d,%d,%d\n",
+                mTestModuleName, (testClass == null ? "?" : testClass.getCanonicalName()),
+                classDescription, passed, failed, skipped);
+        mOutputWriter.flush();
+    }
+
+    /**
+     * Try to extract the class from a description, which is needed because
+     * ParameterizedAndroidJunit4's description doesn't contain a class.
+     */
+    private Class<?> extractTestClass(Description desc) {
+        if (desc.getTestClass() != null) {
+            return desc.getTestClass();
+        }
+        // Look into the children.
+        for (var child : desc.getChildren()) {
+            var fromChild = extractTestClass(child);
+            if (fromChild != null) {
+                return fromChild;
+            }
+        }
+        return null;
+    }
+}
diff --git a/ravenwood/junit-src/android/platform/test/ravenwood/RavenwoodAwareTestRunner.java b/ravenwood/junit-src/android/platform/test/ravenwood/RavenwoodAwareTestRunner.java
index a4fa41af26e5..2b55ac52ab75 100644
--- a/ravenwood/junit-src/android/platform/test/ravenwood/RavenwoodAwareTestRunner.java
+++ b/ravenwood/junit-src/android/platform/test/ravenwood/RavenwoodAwareTestRunner.java
@@ -15,8 +15,6 @@
  */
 package android.platform.test.ravenwood;
 
-import static android.platform.test.ravenwood.RavenwoodRule.shouldRunCassOnRavenwood;
-
 import static com.android.ravenwood.common.RavenwoodCommonUtils.ensureIsPublicVoidMethod;
 import static com.android.ravenwood.common.RavenwoodCommonUtils.isOnRavenwood;
 
@@ -164,7 +162,8 @@ public class RavenwoodAwareTestRunner extends Runner implements Filterable, Orde
          * If the class has @DisabledOnRavenwood, then we'll delegate to ClassSkippingTestRunner,
          * which simply skips it.
          */
-        if (isOnRavenwood() && !shouldRunCassOnRavenwood(mTestClsas.getJavaClass())) {
+        if (isOnRavenwood() && !RavenwoodAwareTestRunnerHook.shouldRunClassOnRavenwood(
+                mTestClsas.getJavaClass())) {
             mRealRunner = new ClassSkippingTestRunner(mTestClsas);
             return;
         }
@@ -238,6 +237,7 @@ public class RavenwoodAwareTestRunner extends Runner implements Filterable, Orde
     public void run(RunNotifier notifier) {
         if (mRealRunner instanceof ClassSkippingTestRunner) {
             mRealRunner.run(notifier);
+            RavenwoodAwareTestRunnerHook.onClassSkipped(getDescription());
             return;
         }
 
@@ -294,19 +294,23 @@ public class RavenwoodAwareTestRunner extends Runner implements Filterable, Orde
     }
 
     private void runWithHooks(Description description, Scope scope, Order order, Statement s) {
-        Throwable th = null;
         if (isOnRavenwood()) {
             Assume.assumeTrue(
                     RavenwoodAwareTestRunnerHook.onBefore(this, description, scope, order));
         }
         try {
             s.evaluate();
+            if (isOnRavenwood()) {
+                RavenwoodAwareTestRunnerHook.onAfter(this, description, scope, order, null);
+            }
         } catch (Throwable t) {
-            th = t;
-            SneakyThrow.sneakyThrow(t);
-        } finally {
+            boolean shouldThrow = true;
             if (isOnRavenwood()) {
-                RavenwoodAwareTestRunnerHook.onAfter(this, description, scope, order, th);
+                shouldThrow = RavenwoodAwareTestRunnerHook.onAfter(
+                        this, description, scope, order, t);
+            }
+            if (shouldThrow) {
+                SneakyThrow.sneakyThrow(t);
             }
         }
     }
diff --git a/ravenwood/junit-src/android/platform/test/ravenwood/RavenwoodClassRule.java b/ravenwood/junit-src/android/platform/test/ravenwood/RavenwoodClassRule.java
index 6c8d96add4ca..85297fe96d6a 100644
--- a/ravenwood/junit-src/android/platform/test/ravenwood/RavenwoodClassRule.java
+++ b/ravenwood/junit-src/android/platform/test/ravenwood/RavenwoodClassRule.java
@@ -16,37 +16,20 @@
 
 package android.platform.test.ravenwood;
 
-import static android.platform.test.ravenwood.RavenwoodRule.ENABLE_PROBE_IGNORED;
-import static android.platform.test.ravenwood.RavenwoodRule.IS_ON_RAVENWOOD;
-import static android.platform.test.ravenwood.RavenwoodRule.shouldEnableOnRavenwood;
-import static android.platform.test.ravenwood.RavenwoodRule.shouldStillIgnoreInProbeIgnoreMode;
-
-import android.platform.test.annotations.DisabledOnRavenwood;
-import android.platform.test.annotations.EnabledOnRavenwood;
-
-import org.junit.Assume;
 import org.junit.rules.TestRule;
 import org.junit.runner.Description;
 import org.junit.runners.model.Statement;
 
 /**
- * {@code @ClassRule} that respects Ravenwood-specific class annotations. This rule has no effect
- * when tests are run on non-Ravenwood test environments.
+ * No longer needed.
  *
- * By default, all tests are executed on Ravenwood, but annotations such as
- * {@link DisabledOnRavenwood} and {@link EnabledOnRavenwood} can be used at both the method
- * and class level to "ignore" tests that may not be ready.
+ * @deprecated this class used to be used to handle the class level annotation, which
+ * is now done by the test runner, so this class is not needed.
  */
+@Deprecated
 public class RavenwoodClassRule implements TestRule {
     @Override
     public Statement apply(Statement base, Description description) {
-        if (!IS_ON_RAVENWOOD) {
-            // No check on a real device.
-        } else if (ENABLE_PROBE_IGNORED) {
-            Assume.assumeFalse(shouldStillIgnoreInProbeIgnoreMode(description));
-        } else {
-            Assume.assumeTrue(shouldEnableOnRavenwood(description));
-        }
         return base;
     }
 }
diff --git a/ravenwood/junit-src/android/platform/test/ravenwood/RavenwoodRule.java b/ravenwood/junit-src/android/platform/test/ravenwood/RavenwoodRule.java
index 75faafb7fe58..d569896421eb 100644
--- a/ravenwood/junit-src/android/platform/test/ravenwood/RavenwoodRule.java
+++ b/ravenwood/junit-src/android/platform/test/ravenwood/RavenwoodRule.java
@@ -20,22 +20,20 @@ import static android.os.Process.FIRST_APPLICATION_UID;
 import static android.os.Process.SYSTEM_UID;
 import static android.os.UserHandle.SYSTEM;
 
-import static org.junit.Assert.fail;
+import static com.android.ravenwood.common.RavenwoodCommonUtils.log;
 
+import android.annotation.Nullable;
 import android.app.Instrumentation;
 import android.content.Context;
 import android.platform.test.annotations.DisabledOnRavenwood;
 import android.platform.test.annotations.EnabledOnRavenwood;
-import android.platform.test.annotations.IgnoreUnderRavenwood;
 
 import com.android.ravenwood.common.RavenwoodCommonUtils;
 
-import org.junit.Assume;
 import org.junit.rules.TestRule;
 import org.junit.runner.Description;
 import org.junit.runners.model.Statement;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
@@ -56,18 +54,18 @@ import java.util.regex.Pattern;
  * before a test class is fully initialized.
  */
 public class RavenwoodRule implements TestRule {
+    private static final String TAG = "RavenwoodRule";
+
     static final boolean IS_ON_RAVENWOOD = RavenwoodCommonUtils.isOnRavenwood();
 
     /**
-     * When probing is enabled, all tests will be unconditionally run on Ravenwood to detect
+     * When this flag is enabled, all tests will be unconditionally run on Ravenwood to detect
      * cases where a test is able to pass despite being marked as {@link DisabledOnRavenwood}.
      *
      * This is typically helpful for internal maintainers discovering tests that had previously
      * been ignored, but now have enough Ravenwood-supported functionality to be enabled.
-     *
-     * TODO: Rename it to a more descriptive name.
      */
-    static final boolean ENABLE_PROBE_IGNORED = "1".equals(
+    private static final boolean RUN_DISABLED_TESTS = "1".equals(
             System.getenv("RAVENWOOD_RUN_DISABLED_TESTS"));
 
     /**
@@ -92,23 +90,17 @@ public class RavenwoodRule implements TestRule {
      *
      * Because we use a regex-find, setting "." would disable all tests.
      */
-    private static final Pattern REALLY_DISABLE_PATTERN = Pattern.compile(
-            Objects.requireNonNullElse(System.getenv("RAVENWOOD_REALLY_DISABLE"), ""));
-
-    private static final boolean ENABLE_REALLY_DISABLE_PATTERN =
-            !REALLY_DISABLE_PATTERN.pattern().isEmpty();
+    private static final Pattern REALLY_DISABLED_PATTERN = Pattern.compile(
+            Objects.requireNonNullElse(System.getenv("RAVENWOOD_REALLY_DISABLED"), ""));
 
-    /**
-     * If true, enable optional validation on running tests.
-     */
-    private static final boolean ENABLE_OPTIONAL_VALIDATION = "1".equals(
-            System.getenv("RAVENWOOD_OPTIONAL_VALIDATION"));
+    private static final boolean HAS_REALLY_DISABLE_PATTERN =
+            !REALLY_DISABLED_PATTERN.pattern().isEmpty();
 
     static {
-        if (ENABLE_PROBE_IGNORED) {
-            System.out.println("$RAVENWOOD_RUN_DISABLED_TESTS enabled: force running all tests");
-            if (ENABLE_REALLY_DISABLE_PATTERN) {
-                System.out.println("$RAVENWOOD_REALLY_DISABLE=" + REALLY_DISABLE_PATTERN.pattern());
+        if (RUN_DISABLED_TESTS) {
+            log(TAG, "$RAVENWOOD_RUN_DISABLED_TESTS enabled: force running all tests");
+            if (HAS_REALLY_DISABLE_PATTERN) {
+                log(TAG, "$RAVENWOOD_REALLY_DISABLED=" + REALLY_DISABLED_PATTERN.pattern());
             }
         }
     }
@@ -275,103 +267,18 @@ public class RavenwoodRule implements TestRule {
                 "Instrumentation is only available during @Test execution");
     }
 
-    /**
-     * Determine if the given {@link Description} should be enabled when running on the
-     * Ravenwood test environment.
-     *
-     * A more specific method-level annotation always takes precedence over any class-level
-     * annotation, and an {@link EnabledOnRavenwood} annotation always takes precedence over
-     * an {@link DisabledOnRavenwood} annotation.
-     */
-    public static boolean shouldEnableOnRavenwood(Description description) {
-        // First, consult any method-level annotations
-        if (description.isTest()) {
-            // Stopgap for http://g/ravenwood/EPAD-N5ntxM
-            if (description.getMethodName().endsWith("$noRavenwood")) {
-                return false;
-            }
-            if (description.getAnnotation(EnabledOnRavenwood.class) != null) {
-                return true;
-            }
-            if (description.getAnnotation(DisabledOnRavenwood.class) != null) {
-                return false;
-            }
-            if (description.getAnnotation(IgnoreUnderRavenwood.class) != null) {
-                return false;
-            }
-        }
-
-        // Otherwise, consult any class-level annotations
-        return shouldRunCassOnRavenwood(description.getTestClass());
-    }
-
-    public static boolean shouldRunCassOnRavenwood(Class<?> clazz) {
-        if (clazz != null) {
-            if (clazz.getAnnotation(EnabledOnRavenwood.class) != null) {
-                return true;
-            }
-            if (clazz.getAnnotation(DisabledOnRavenwood.class) != null) {
-                return false;
-            }
-            if (clazz.getAnnotation(IgnoreUnderRavenwood.class) != null) {
-                return false;
-            }
-        }
-        return true;
-    }
-
-    static boolean shouldStillIgnoreInProbeIgnoreMode(Description description) {
-        if (!ENABLE_REALLY_DISABLE_PATTERN) {
-            return false;
-        }
-
-        final var fullname = description.getTestClass().getName()
-                + (description.isTest() ? "#" + description.getMethodName() : "");
-
-        if (REALLY_DISABLE_PATTERN.matcher(fullname).find()) {
-            System.out.println("Still ignoring " + fullname);
-            return true;
-        }
-        return false;
-    }
 
     @Override
     public Statement apply(Statement base, Description description) {
-        // No special treatment when running outside Ravenwood; run tests as-is
-        if (!IS_ON_RAVENWOOD) {
-            return base;
-        }
-
-        if (ENABLE_PROBE_IGNORED) {
-            return applyProbeIgnored(base, description);
-        } else {
-            return applyDefault(base, description);
-        }
-    }
-
-    private void commonPrologue(Statement base, Description description) throws IOException {
-        RavenwoodRuleImpl.logTestRunner("started", description);
-        RavenwoodRuleImpl.validate(base, description, ENABLE_OPTIONAL_VALIDATION);
-        RavenwoodRuleImpl.init(RavenwoodRule.this);
-    }
-
-    /**
-     * Run the given {@link Statement} with no special treatment.
-     */
-    private Statement applyDefault(Statement base, Description description) {
+        // TODO: Here, we're calling init() / reset() once for each rule.
+        // That means if a test class has multiple rules -- even if they refer to the same
+        // rule instance -- we're calling them multiple times. We need to fix it.
         return new Statement() {
             @Override
             public void evaluate() throws Throwable {
-                Assume.assumeTrue(shouldEnableOnRavenwood(description));
-
-                commonPrologue(base, description);
+                RavenwoodRuleImpl.init(RavenwoodRule.this);
                 try {
                     base.evaluate();
-
-                    RavenwoodRuleImpl.logTestRunner("finished", description);
-                } catch (Throwable t) {
-                    RavenwoodRuleImpl.logTestRunner("failed", description);
-                    throw t;
                 } finally {
                     RavenwoodRuleImpl.reset(RavenwoodRule.this);
                 }
@@ -380,44 +287,6 @@ public class RavenwoodRule implements TestRule {
     }
 
     /**
-     * Run the given {@link Statement} with probing enabled. All tests will be unconditionally
-     * run on Ravenwood to detect cases where a test is able to pass despite being marked as
-     * {@code IgnoreUnderRavenwood}.
-     */
-    private Statement applyProbeIgnored(Statement base, Description description) {
-        return new Statement() {
-            @Override
-            public void evaluate() throws Throwable {
-                Assume.assumeFalse(shouldStillIgnoreInProbeIgnoreMode(description));
-
-                commonPrologue(base, description);
-                try {
-                    base.evaluate();
-                } catch (Throwable t) {
-                    // If the test isn't included, eat the exception and report the
-                    // assumption failure that test authors expect; otherwise throw
-                    Assume.assumeTrue(shouldEnableOnRavenwood(description));
-                    throw t;
-                } finally {
-                    RavenwoodRuleImpl.logTestRunner("finished", description);
-                    RavenwoodRuleImpl.reset(RavenwoodRule.this);
-                }
-
-                if (!shouldEnableOnRavenwood(description)) {
-                    fail("Test wasn't included under Ravenwood, but it actually "
-                            + "passed under Ravenwood; consider updating annotations");
-                }
-            }
-        };
-    }
-
-    public static class _$RavenwoodPrivate {
-        public static boolean isOptionalValidationEnabled() {
-            return ENABLE_OPTIONAL_VALIDATION;
-        }
-    }
-
-    /**
      * Returns the "real" result from {@link System#currentTimeMillis()}.
      *
      * Currently, it's the same thing as calling {@link System#currentTimeMillis()},
@@ -427,4 +296,47 @@ public class RavenwoodRule implements TestRule {
     public long realCurrentTimeMillis() {
         return System.currentTimeMillis();
     }
+
+    // Below are internal to ravenwood. Don't use them from normal tests...
+
+    public static class RavenwoodPrivate {
+        private RavenwoodPrivate() {
+        }
+
+        private volatile Boolean mRunDisabledTestsOverride = null;
+
+        private volatile Pattern mReallyDisabledPattern = null;
+
+        public boolean isRunningDisabledTests() {
+            if (mRunDisabledTestsOverride != null) {
+                return mRunDisabledTestsOverride;
+            }
+            return RUN_DISABLED_TESTS;
+        }
+
+        public Pattern getReallyDisabledPattern() {
+            if (mReallyDisabledPattern != null) {
+                return mReallyDisabledPattern;
+            }
+            return REALLY_DISABLED_PATTERN;
+        }
+
+        public void overrideRunDisabledTest(boolean runDisabledTests,
+                @Nullable String reallyDisabledPattern) {
+            mRunDisabledTestsOverride = runDisabledTests;
+            mReallyDisabledPattern =
+                    reallyDisabledPattern == null ? null : Pattern.compile(reallyDisabledPattern);
+        }
+
+        public void resetRunDisabledTest() {
+            mRunDisabledTestsOverride = null;
+            mReallyDisabledPattern = null;
+        }
+    }
+
+    private static final RavenwoodPrivate sRavenwoodPrivate = new  RavenwoodPrivate();
+
+    public static RavenwoodPrivate private$ravenwood() {
+        return sRavenwoodPrivate;
+    }
 }
diff --git a/ravenwood/junit-stub-src/android/platform/test/ravenwood/RavenwoodAwareTestRunnerHook.java b/ravenwood/junit-stub-src/android/platform/test/ravenwood/RavenwoodAwareTestRunnerHook.java
index 6b80e0cbf91e..1e4889ce0678 100644
--- a/ravenwood/junit-stub-src/android/platform/test/ravenwood/RavenwoodAwareTestRunnerHook.java
+++ b/ravenwood/junit-stub-src/android/platform/test/ravenwood/RavenwoodAwareTestRunnerHook.java
@@ -23,27 +23,51 @@ import org.junit.runner.Runner;
 import org.junit.runners.model.TestClass;
 
 /**
- * Provide hook points created by {@link RavenwoodAwareTestRunner}.
+ * Provide hook points created by {@link RavenwoodAwareTestRunner}. This is a version
+ * that's used on a device side test.
+ *
+ * All methods are no-op in real device tests.
+ *
+ * TODO: Use some kind of factory to provide different implementation for the device test
+ * and the ravenwood test.
  */
 public class RavenwoodAwareTestRunnerHook {
     private RavenwoodAwareTestRunnerHook() {
     }
 
     /**
-     * Called when a runner starts, befre the inner runner gets a chance to run.
+     * Called when a runner starts, before the inner runner gets a chance to run.
      */
     public static void onRunnerInitializing(Runner runner, TestClass testClass) {
-        // No-op on a real device.
     }
 
+    /**
+     * Called when a whole test class is skipped.
+     */
+    public static void onClassSkipped(Description description) {
+    }
+
+    /**
+     * Called before a test / class.
+     *
+     * Return false if it should be skipped.
+     */
     public static boolean onBefore(RavenwoodAwareTestRunner runner, Description description,
             Scope scope, Order order) {
-        // No-op on a real device.
         return true;
     }
 
-    public static void onAfter(RavenwoodAwareTestRunner runner, Description description,
+    /**
+     * Called after a test / class.
+     *
+     * Return false if the exception should be ignored.
+     */
+    public static boolean onAfter(RavenwoodAwareTestRunner runner, Description description,
             Scope scope, Order order, Throwable th) {
-        // No-op on a real device.
+        return true;
+    }
+
+    public static boolean shouldRunClassOnRavenwood(Class<?> clazz) {
+        return true;
     }
 }
diff --git a/ravenwood/junit-stub-src/android/platform/test/ravenwood/RavenwoodRuleImpl.java b/ravenwood/junit-stub-src/android/platform/test/ravenwood/RavenwoodRuleImpl.java
index 483b98a96034..a470626dcbe7 100644
--- a/ravenwood/junit-stub-src/android/platform/test/ravenwood/RavenwoodRuleImpl.java
+++ b/ravenwood/junit-stub-src/android/platform/test/ravenwood/RavenwoodRuleImpl.java
@@ -17,7 +17,6 @@
 package android.platform.test.ravenwood;
 
 import org.junit.runner.Description;
-import org.junit.runners.model.Statement;
 
 public class RavenwoodRuleImpl {
     public static void init(RavenwoodRule rule) {
@@ -32,10 +31,6 @@ public class RavenwoodRuleImpl {
         // No-op when running on a real device
     }
 
-    public static void validate(Statement base, Description description,
-            boolean enableOptionalValidation) {
-    }
-
     public static long realCurrentTimeMillis() {
         return System.currentTimeMillis();
     }