diff options
12 files changed, 686 insertions, 35 deletions
diff --git a/ravenwood/Android.bp b/ravenwood/Android.bp index 333fe4c8147f..eebe5e9fc054 100644 --- a/ravenwood/Android.bp +++ b/ravenwood/Android.bp @@ -94,6 +94,9 @@ java_library { libs: [ "ravenwood-runtime-common-ravenwood", ], + static_libs: [ + "framework-annotations-lib", // should it be "libs" instead? + ], visibility: ["//visibility:private"], } diff --git a/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/listenertests/RavenwoodAfterClassFailureTest.java b/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/listenertests/RavenwoodAfterClassFailureTest.java new file mode 100644 index 000000000000..f9794ad5941e --- /dev/null +++ b/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/listenertests/RavenwoodAfterClassFailureTest.java @@ -0,0 +1,70 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.android.ravenwoodtest.bivalenttest.listenertests; + +import static android.platform.test.ravenwood.RavenwoodRule.isOnRavenwood; + +import org.junit.AfterClass; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runner.RunWith; + +import java.util.ArrayList; +import java.util.List; + +import platform.test.runner.parameterized.ParameterizedAndroidJunit4; +import platform.test.runner.parameterized.Parameters; + +/** + * Test that throws from @AfterClass. + * + * Tradefed would ignore it, so instead RavenwoodAwareTestRunner would detect it and kill + * the self (test) process. + * + * Unfortunately, this behavior can't easily be tested from within this class, so for now + * it's only used for a manual test, which you can run by removing the @Ignore. + * + * TODO(b/364948126) Improve the tests and automate it. + */ +@Ignore +@RunWith(ParameterizedAndroidJunit4.class) +public class RavenwoodAfterClassFailureTest { + public RavenwoodAfterClassFailureTest(String param) { + } + + @AfterClass + public static void afterClass() { + if (!isOnRavenwood()) return; // Don't do anything on real device. + + throw new RuntimeException("FAILURE"); + } + + @Parameters + public static List<String> getParams() { + var params = new ArrayList<String>(); + params.add("foo"); + params.add("bar"); + return params; + } + + @Test + public void test1() { + } + + @Test + public void test2() { + } +} diff --git a/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/listenertests/RavenwoodBeforeClassAssumptionFailureTest.java b/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/listenertests/RavenwoodBeforeClassAssumptionFailureTest.java new file mode 100644 index 000000000000..61fb06865545 --- /dev/null +++ b/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/listenertests/RavenwoodBeforeClassAssumptionFailureTest.java @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.android.ravenwoodtest.bivalenttest.listenertests; + +import static android.platform.test.ravenwood.RavenwoodRule.isOnRavenwood; + +import org.junit.Assume; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; + +import java.util.ArrayList; +import java.util.List; + +import platform.test.runner.parameterized.ParameterizedAndroidJunit4; +import platform.test.runner.parameterized.Parameters; + +/** + * Test that fails in assumption in @BeforeClass. + * + * This is only used for manual tests. Make sure `atest` shows 4 test results with + * "ASSUMPTION_FAILED". + * + * TODO(b/364948126) Improve the tests and automate it. + */ +@RunWith(ParameterizedAndroidJunit4.class) +public class RavenwoodBeforeClassAssumptionFailureTest { + public RavenwoodBeforeClassAssumptionFailureTest(String param) { + } + + @BeforeClass + public static void beforeClass() { + if (!isOnRavenwood()) return; // Don't do anything on real device. + + Assume.assumeTrue(false); + } + + @Parameters + public static List<String> getParams() { + var params = new ArrayList<String>(); + params.add("foo"); + params.add("bar"); + return params; + } + + @Test + public void test1() { + } + + @Test + public void test2() { + } +} diff --git a/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/listenertests/RavenwoodBeforeClassFailureTest.java b/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/listenertests/RavenwoodBeforeClassFailureTest.java new file mode 100644 index 000000000000..626ce8198eeb --- /dev/null +++ b/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/listenertests/RavenwoodBeforeClassFailureTest.java @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.android.ravenwoodtest.bivalenttest.listenertests; + +import static android.platform.test.ravenwood.RavenwoodRule.isOnRavenwood; + +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runner.RunWith; + +import java.util.ArrayList; +import java.util.List; + +import platform.test.runner.parameterized.ParameterizedAndroidJunit4; +import platform.test.runner.parameterized.Parameters; + +/** + * Test that fails throws from @BeforeClass. + * + * This is only used for manual tests. Make sure `atest` shows 4 test results with + * a "FAILURE" runtime exception. + * + * In order to run the test, you'll need to remove the @Ignore. + * + * TODO(b/364948126) Improve the tests and automate it. + */ +@Ignore +@RunWith(ParameterizedAndroidJunit4.class) +public class RavenwoodBeforeClassFailureTest { + public static final String TAG = "RavenwoodBeforeClassFailureTest"; + + public RavenwoodBeforeClassFailureTest(String param) { + } + + @BeforeClass + public static void beforeClass() { + if (!isOnRavenwood()) return; // Don't do anything on real device. + + throw new RuntimeException("FAILURE"); + } + + @Parameters + public static List<String> getParams() { + var params = new ArrayList<String>(); + params.add("foo"); + params.add("bar"); + return params; + } + + @Test + public void test1() { + } + + @Test + public void test2() { + } +} diff --git a/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/listenertests/RavenwoodClassRuleAssumptionFailureTest.java b/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/listenertests/RavenwoodClassRuleAssumptionFailureTest.java new file mode 100644 index 000000000000..dc949c466110 --- /dev/null +++ b/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/listenertests/RavenwoodClassRuleAssumptionFailureTest.java @@ -0,0 +1,78 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.android.ravenwoodtest.bivalenttest.listenertests; + +import static android.platform.test.ravenwood.RavenwoodRule.isOnRavenwood; + +import static org.junit.Assume.assumeTrue; + +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.rules.TestRule; +import org.junit.runner.Description; +import org.junit.runner.RunWith; +import org.junit.runners.model.Statement; + +import java.util.ArrayList; +import java.util.List; + +import platform.test.runner.parameterized.ParameterizedAndroidJunit4; +import platform.test.runner.parameterized.Parameters; + +/** + * Test that fails in assumption from a class rule. + * + * This is only used for manual tests. Make sure `atest` shows 4 test results with + * "ASSUMPTION_FAILED". + * + * TODO(b/364948126) Improve the tests and automate it. + */ +@RunWith(ParameterizedAndroidJunit4.class) +public class RavenwoodClassRuleAssumptionFailureTest { + public static final String TAG = "RavenwoodClassRuleFailureTest"; + + @ClassRule + public static final TestRule sClassRule = new TestRule() { + @Override + public Statement apply(Statement base, Description description) { + if (!isOnRavenwood()) { + return base; // Just run the test as-is on a real device. + } + + assumeTrue(false); + return null; // unreachable + } + }; + + public RavenwoodClassRuleAssumptionFailureTest(String param) { + } + + @Parameters + public static List<String> getParams() { + var params = new ArrayList<String>(); + params.add("foo"); + params.add("bar"); + return params; + } + + @Test + public void test1() { + } + + @Test + public void test2() { + } +} diff --git a/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/listenertests/RavenwoodClassRuleFailureTest.java b/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/listenertests/RavenwoodClassRuleFailureTest.java new file mode 100644 index 000000000000..9996bec41525 --- /dev/null +++ b/ravenwood/bivalenttest/test/com/android/ravenwoodtest/bivalenttest/listenertests/RavenwoodClassRuleFailureTest.java @@ -0,0 +1,79 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.android.ravenwoodtest.bivalenttest.listenertests; + +import static android.platform.test.ravenwood.RavenwoodRule.isOnRavenwood; + +import org.junit.ClassRule; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.rules.TestRule; +import org.junit.runner.Description; +import org.junit.runner.RunWith; +import org.junit.runners.model.Statement; + +import java.util.ArrayList; +import java.util.List; + +import platform.test.runner.parameterized.ParameterizedAndroidJunit4; +import platform.test.runner.parameterized.Parameters; + +/** + * Test that fails throws from a class rule. + * + * This is only used for manual tests. Make sure `atest` shows 4 test results with + * a "FAILURE" runtime exception. + * + * In order to run the test, you'll need to remove the @Ignore. + * + * TODO(b/364948126) Improve the tests and automate it. + */ +@Ignore +@RunWith(ParameterizedAndroidJunit4.class) +public class RavenwoodClassRuleFailureTest { + public static final String TAG = "RavenwoodClassRuleFailureTest"; + + @ClassRule + public static final TestRule sClassRule = new TestRule() { + @Override + public Statement apply(Statement base, Description description) { + if (!isOnRavenwood()) { + return base; // Just run the test as-is on a real device. + } + + throw new RuntimeException("FAILURE"); + } + }; + + public RavenwoodClassRuleFailureTest(String param) { + } + + @Parameters + public static List<String> getParams() { + var params = new ArrayList<String>(); + params.add("foo"); + params.add("bar"); + return params; + } + + @Test + public void test1() { + } + + @Test + public void test2() { + } +} diff --git a/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodAwareTestRunnerHook.java b/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodAwareTestRunnerHook.java index 1d182da5e7fd..6d21e440e911 100644 --- a/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodAwareTestRunnerHook.java +++ b/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodAwareTestRunnerHook.java @@ -34,6 +34,8 @@ import org.junit.runner.Description; import org.junit.runner.Runner; import org.junit.runners.model.TestClass; +import java.util.Stack; + /** * Provide hook points created by {@link RavenwoodAwareTestRunner}. */ @@ -44,6 +46,11 @@ public class RavenwoodAwareTestRunnerHook { } private static RavenwoodTestStats sStats; // lazy initialization. + + // Keep track of the current class description. + + // Test classes can be nested because of "Suite", so we need a stack to keep track. + private static final Stack<Description> sClassDescriptions = new Stack<>(); private static Description sCurrentClassDescription; private static RavenwoodTestStats getStats() { @@ -108,14 +115,15 @@ public class RavenwoodAwareTestRunnerHook { Scope scope, Order order) { Log.v(TAG, "onBefore: description=" + description + ", " + scope + ", " + order); - if (scope == Scope.Class && order == Order.First) { + if (scope == Scope.Class && order == Order.Outer) { // Keep track of the current class. sCurrentClassDescription = description; + sClassDescriptions.push(description); } // Class-level annotations are checked by the runner already, so we only check // method-level annotations here. - if (scope == Scope.Instance && order == Order.First) { + if (scope == Scope.Instance && order == Order.Outer) { if (!RavenwoodEnablementChecker.shouldEnableOnRavenwood( description, true)) { getStats().onTestFinished(sCurrentClassDescription, description, Result.Skipped); @@ -134,17 +142,20 @@ public class RavenwoodAwareTestRunnerHook { Scope scope, Order order, Throwable th) { Log.v(TAG, "onAfter: description=" + description + ", " + scope + ", " + order + ", " + th); - if (scope == Scope.Instance && order == Order.First) { + if (scope == Scope.Instance && order == Order.Outer) { getStats().onTestFinished(sCurrentClassDescription, description, th == null ? Result.Passed : Result.Failed); - } else if (scope == Scope.Class && order == Order.Last) { + } else if (scope == Scope.Class && order == Order.Outer) { getStats().onClassFinished(sCurrentClassDescription); + sClassDescriptions.pop(); + sCurrentClassDescription = + sClassDescriptions.size() == 0 ? null : sClassDescriptions.peek(); } // If RUN_DISABLED_TESTS is set, and the method did _not_ throw, make it an error. if (RavenwoodRule.private$ravenwood().isRunningDisabledTests() - && scope == Scope.Instance && order == Order.First) { + && scope == Scope.Instance && order == Order.Outer) { boolean isTestEnabled = RavenwoodEnablementChecker.shouldEnableOnRavenwood( description, false); diff --git a/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodTestStats.java b/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodTestStats.java index 631f68ff1dec..3ffabefb7681 100644 --- a/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodTestStats.java +++ b/ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodTestStats.java @@ -127,7 +127,11 @@ public class RavenwoodTestStats { int passed = 0; int skipped = 0; int failed = 0; - for (var e : mStats.get(classDescription).values()) { + var stats = mStats.get(classDescription); + if (stats == null) { + return; + } + for (var e : stats.values()) { switch (e) { case Passed: passed++; break; case Skipped: skipped++; break; diff --git a/ravenwood/junit-src/android/platform/test/ravenwood/RavenwoodAwareTestRunner.java b/ravenwood/junit-src/android/platform/test/ravenwood/RavenwoodAwareTestRunner.java index bfde9cb7099e..dffb263e77cb 100644 --- a/ravenwood/junit-src/android/platform/test/ravenwood/RavenwoodAwareTestRunner.java +++ b/ravenwood/junit-src/android/platform/test/ravenwood/RavenwoodAwareTestRunner.java @@ -15,20 +15,25 @@ */ package android.platform.test.ravenwood; +import static com.android.ravenwood.common.RavenwoodCommonUtils.RAVENWOOD_VERBOSE_LOGGING; import static com.android.ravenwood.common.RavenwoodCommonUtils.ensureIsPublicVoidMethod; import static com.android.ravenwood.common.RavenwoodCommonUtils.isOnRavenwood; import static java.lang.annotation.ElementType.METHOD; import static java.lang.annotation.ElementType.TYPE; +import android.annotation.NonNull; +import android.annotation.Nullable; import android.util.Log; import com.android.ravenwood.common.SneakyThrow; import org.junit.Assume; +import org.junit.AssumptionViolatedException; import org.junit.internal.builders.AllDefaultPossibilitiesBuilder; import org.junit.rules.TestRule; import org.junit.runner.Description; +import org.junit.runner.Result; import org.junit.runner.Runner; import org.junit.runner.manipulation.Filter; import org.junit.runner.manipulation.Filterable; @@ -39,8 +44,11 @@ import org.junit.runner.manipulation.Orderer; import org.junit.runner.manipulation.Sortable; import org.junit.runner.manipulation.Sorter; import org.junit.runner.notification.Failure; +import org.junit.runner.notification.RunListener; import org.junit.runner.notification.RunNotifier; +import org.junit.runner.notification.StoppedByUserException; import org.junit.runners.BlockJUnit4ClassRunner; +import org.junit.runners.model.MultipleFailureException; import org.junit.runners.model.RunnerBuilder; import org.junit.runners.model.Statement; import org.junit.runners.model.TestClass; @@ -51,6 +59,8 @@ import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.Stack; /** * A test runner used for Ravenwood. @@ -61,7 +71,7 @@ import java.lang.reflect.InvocationTargetException; * the inner runner gets a chance to run. This can be used to initialize stuff used by the * inner runner. * - Add hook points, which are handed by RavenwoodAwareTestRunnerHook, with help from - * the four test rules such as {@link #sImplicitClassMinRule}, which are also injected by + * the four test rules such as {@link #sImplicitClassOuterRule}, which are also injected by * the ravenizer tool. * * We use this runner to: @@ -102,28 +112,50 @@ public class RavenwoodAwareTestRunner extends Runner implements Filterable, Orde /** Order of a hook. */ public enum Order { - First, - Last, + Outer, + Inner, } // The following four rule instances will be injected to tests by the Ravenizer tool. + private static class RavenwoodClassOuterRule implements TestRule { + @Override + public Statement apply(Statement base, Description description) { + return getCurrentRunner().updateStatement(base, description, Scope.Class, Order.Outer); + } + } - public static final TestRule sImplicitClassMinRule = (base, description) -> - getCurrentRunner().updateStatement(base, description, Scope.Class, Order.First); + private static class RavenwoodClassInnerRule implements TestRule { + @Override + public Statement apply(Statement base, Description description) { + return getCurrentRunner().updateStatement(base, description, Scope.Class, Order.Inner); + } + } - public static final TestRule sImplicitClassMaxRule = (base, description) -> - getCurrentRunner().updateStatement(base, description, Scope.Class, Order.Last); + private static class RavenwoodInstanceOuterRule implements TestRule { + @Override + public Statement apply(Statement base, Description description) { + return getCurrentRunner().updateStatement( + base, description, Scope.Instance, Order.Outer); + } + } - public static final TestRule sImplicitInstMinRule = (base, description) -> - getCurrentRunner().updateStatement(base, description, Scope.Instance, Order.First); + private static class RavenwoodInstanceInnerRule implements TestRule { + @Override + public Statement apply(Statement base, Description description) { + return getCurrentRunner().updateStatement( + base, description, Scope.Instance, Order.Inner); + } + } - public static final TestRule sImplicitInstMaxRule = (base, description) -> - getCurrentRunner().updateStatement(base, description, Scope.Instance, Order.Last); + public static final TestRule sImplicitClassOuterRule = new RavenwoodClassOuterRule(); + public static final TestRule sImplicitClassInnerRule = new RavenwoodClassInnerRule(); + public static final TestRule sImplicitInstOuterRule = new RavenwoodInstanceOuterRule(); + public static final TestRule sImplicitInstInnerRule = new RavenwoodInstanceOuterRule(); - public static final String IMPLICIT_CLASS_MIN_RULE_NAME = "sImplicitClassMinRule"; - public static final String IMPLICIT_CLASS_MAX_RULE_NAME = "sImplicitClassMaxRule"; - public static final String IMPLICIT_INST_MIN_RULE_NAME = "sImplicitInstMinRule"; - public static final String IMPLICIT_INST_MAX_RULE_NAME = "sImplicitInstMaxRule"; + public static final String IMPLICIT_CLASS_OUTER_RULE_NAME = "sImplicitClassOuterRule"; + public static final String IMPLICIT_CLASS_INNER_RULE_NAME = "sImplicitClassInnerRule"; + public static final String IMPLICIT_INST_OUTER_RULE_NAME = "sImplicitInstOuterRule"; + public static final String IMPLICIT_INST_INNER_RULE_NAME = "sImplicitInstInnerRule"; /** Keeps track of the runner on the current thread. */ private static final ThreadLocal<RavenwoodAwareTestRunner> sCurrentRunner = new ThreadLocal<>(); @@ -157,6 +189,8 @@ public class RavenwoodAwareTestRunner extends Runner implements Filterable, Orde try { mTestClass = new TestClass(testClass); + Log.v(TAG, "RavenwoodAwareTestRunner starting for " + testClass.getCanonicalName()); + onRunnerInitializing(); /* @@ -261,20 +295,27 @@ public class RavenwoodAwareTestRunner extends Runner implements Filterable, Orde } @Override - public void run(RunNotifier notifier) { + public void run(RunNotifier realNotifier) { + final RunNotifier notifier = new RavenwoodRunNotifier(realNotifier); + if (mRealRunner instanceof ClassSkippingTestRunner) { mRealRunner.run(notifier); RavenwoodAwareTestRunnerHook.onClassSkipped(getDescription()); return; } + Log.v(TAG, "Starting " + mTestClass.getJavaClass().getCanonicalName()); + if (RAVENWOOD_VERBOSE_LOGGING) { + dumpDescription(getDescription()); + } + if (maybeReportExceptionFromConstructor(notifier)) { return; } sCurrentRunner.set(this); try { - runWithHooks(getDescription(), Scope.Runner, Order.First, + runWithHooks(getDescription(), Scope.Runner, Order.Outer, () -> mRealRunner.run(notifier)); } finally { sCurrentRunner.remove(); @@ -399,4 +440,217 @@ public class RavenwoodAwareTestRunner extends Runner implements Filterable, Orde } } } + + private void dumpDescription(Description desc) { + dumpDescription(desc, "[TestDescription]=", " "); + } + + private void dumpDescription(Description desc, String header, String indent) { + Log.v(TAG, indent + header + desc); + + var children = desc.getChildren(); + var childrenIndent = " " + indent; + for (int i = 0; i < children.size(); i++) { + dumpDescription(children.get(i), "#" + i + ": ", childrenIndent); + } + } + + /** + * A run notifier that wraps another notifier and provides the following features: + * - Handle a failure that happened before testStarted and testEnded (typically that means + * it's from @BeforeClass or @AfterClass, or a @ClassRule) and deliver it as if + * individual tests in the class reported it. This is for b/364395552. + * + * - Logging. + */ + private class RavenwoodRunNotifier extends RunNotifier { + private final RunNotifier mRealNotifier; + + private final Stack<Description> mSuiteStack = new Stack<>(); + private Description mCurrentSuite = null; + private final ArrayList<Throwable> mOutOfTestFailures = new ArrayList<>(); + + private boolean mBeforeTest = true; + private boolean mAfterTest = false; + + private RavenwoodRunNotifier(RunNotifier realNotifier) { + mRealNotifier = realNotifier; + } + + private boolean isInTest() { + return !mBeforeTest && !mAfterTest; + } + + @Override + public void addListener(RunListener listener) { + mRealNotifier.addListener(listener); + } + + @Override + public void removeListener(RunListener listener) { + mRealNotifier.removeListener(listener); + } + + @Override + public void addFirstListener(RunListener listener) { + mRealNotifier.addFirstListener(listener); + } + + @Override + public void fireTestRunStarted(Description description) { + Log.i(TAG, "testRunStarted: " + description); + mRealNotifier.fireTestRunStarted(description); + } + + @Override + public void fireTestRunFinished(Result result) { + Log.i(TAG, "testRunFinished: " + + result.getRunCount() + "," + + result.getFailureCount() + "," + + result.getAssumptionFailureCount() + "," + + result.getIgnoreCount()); + mRealNotifier.fireTestRunFinished(result); + } + + @Override + public void fireTestSuiteStarted(Description description) { + Log.i(TAG, "testSuiteStarted: " + description); + mRealNotifier.fireTestSuiteStarted(description); + + mBeforeTest = true; + mAfterTest = false; + + // Keep track of the current suite, needed if the outer test is a Suite, + // in which case its children are test classes. (not test methods) + mCurrentSuite = description; + mSuiteStack.push(description); + + mOutOfTestFailures.clear(); + } + + @Override + public void fireTestSuiteFinished(Description description) { + Log.i(TAG, "testSuiteFinished: " + description); + mRealNotifier.fireTestSuiteFinished(description); + + maybeHandleOutOfTestFailures(); + + mBeforeTest = true; + mAfterTest = false; + + // Restore the upper suite. + mSuiteStack.pop(); + mCurrentSuite = mSuiteStack.size() == 0 ? null : mSuiteStack.peek(); + } + + @Override + public void fireTestStarted(Description description) throws StoppedByUserException { + Log.i(TAG, "testStarted: " + description); + mRealNotifier.fireTestStarted(description); + + mAfterTest = false; + mBeforeTest = false; + } + + @Override + public void fireTestFailure(Failure failure) { + Log.i(TAG, "testFailure: " + failure); + + if (isInTest()) { + mRealNotifier.fireTestFailure(failure); + } else { + mOutOfTestFailures.add(failure.getException()); + } + } + + @Override + public void fireTestAssumptionFailed(Failure failure) { + Log.i(TAG, "testAssumptionFailed: " + failure); + + if (isInTest()) { + mRealNotifier.fireTestAssumptionFailed(failure); + } else { + mOutOfTestFailures.add(failure.getException()); + } + } + + @Override + public void fireTestIgnored(Description description) { + Log.i(TAG, "testIgnored: " + description); + mRealNotifier.fireTestIgnored(description); + } + + @Override + public void fireTestFinished(Description description) { + Log.i(TAG, "testFinished: " + description); + mRealNotifier.fireTestFinished(description); + + mAfterTest = true; + } + + @Override + public void pleaseStop() { + Log.w(TAG, "pleaseStop:"); + mRealNotifier.pleaseStop(); + } + + /** + * At the end of each Suite, we handle failures happened out of test methods. + * (typically in @BeforeClass or @AfterClasses) + * + * This is to work around b/364395552. + */ + private boolean maybeHandleOutOfTestFailures() { + if (mOutOfTestFailures.size() == 0) { + return false; + } + Throwable th; + if (mOutOfTestFailures.size() == 1) { + th = mOutOfTestFailures.get(0); + } else { + th = new MultipleFailureException(mOutOfTestFailures); + } + if (mBeforeTest) { + reportBeforeTestFailure(mCurrentSuite, th); + return true; + } + if (mAfterTest) { + // Unfortunately, there's no good way to report it, so kill the own process. + onCriticalError( + "Failures detected in @AfterClass, which would be swalloed by tradefed", + th); + return true; // unreachable + } + return false; + } + + private void reportBeforeTestFailure(Description suiteDesc, Throwable th) { + // If a failure happens befere running any tests, we'll need to pretend + // as if each test in the suite reported the failure, to work around b/364395552. + for (var child : suiteDesc.getChildren()) { + if (child.isSuite()) { + // If the chiil is still a "parent" -- a test class or a test suite + // -- propagate to its children. + mRealNotifier.fireTestSuiteStarted(child); + reportBeforeTestFailure(child, th); + mRealNotifier.fireTestSuiteFinished(child); + } else { + mRealNotifier.fireTestStarted(child); + Failure f = new Failure(child, th); + if (th instanceof AssumptionViolatedException) { + mRealNotifier.fireTestAssumptionFailed(f); + } else { + mRealNotifier.fireTestFailure(f); + } + mRealNotifier.fireTestFinished(child); + } + } + } + } + + private void onCriticalError(@NonNull String message, @Nullable Throwable th) { + Log.e(TAG, "Critical error! Ravenwood cannot continue. Killing self process: " + + message, th); + System.exit(1); + } } diff --git a/ravenwood/runtime-common-src/com/android/ravenwood/common/RavenwoodCommonUtils.java b/ravenwood/runtime-common-src/com/android/ravenwood/common/RavenwoodCommonUtils.java index 7b5bc5aeb7b6..875ce71149cd 100644 --- a/ravenwood/runtime-common-src/com/android/ravenwood/common/RavenwoodCommonUtils.java +++ b/ravenwood/runtime-common-src/com/android/ravenwood/common/RavenwoodCommonUtils.java @@ -15,12 +15,17 @@ */ package com.android.ravenwood.common; +import android.annotation.NonNull; +import android.annotation.Nullable; + import com.android.ravenwood.common.divergence.RavenwoodDivergence; import java.io.File; import java.io.FileDescriptor; import java.io.FileInputStream; import java.io.PrintStream; +import java.io.PrintWriter; +import java.io.StringWriter; import java.lang.reflect.Method; import java.lang.reflect.Modifier; import java.util.Arrays; @@ -33,6 +38,14 @@ public class RavenwoodCommonUtils { private static final Object sLock = new Object(); + /** + * If set to "1", we enable the verbose logging. + * + * (See also InitLogging() in http://ac/system/libbase/logging.cpp) + */ + public static final boolean RAVENWOOD_VERBOSE_LOGGING = "1".equals(System.getenv( + "RAVENWOOD_VERBOSE")); + /** Name of `libravenwood_runtime` */ private static final String RAVENWOOD_NATIVE_RUNTIME_NAME = "ravenwood_runtime"; @@ -265,4 +278,12 @@ public class RavenwoodCommonUtils { method.getDeclaringClass().getName(), method.getName(), (isStatic ? "static " : ""))); } + + @NonNull + public static String getStackTraceString(@Nullable Throwable th) { + StringWriter stringWriter = new StringWriter(); + PrintWriter writer = new PrintWriter(stringWriter); + th.printStackTrace(writer); + return stringWriter.toString(); + } } diff --git a/ravenwood/runtime-helper-src/framework/com/android/platform/test/ravenwood/runtimehelper/ClassLoadHook.java b/ravenwood/runtime-helper-src/framework/com/android/platform/test/ravenwood/runtimehelper/ClassLoadHook.java index c519204d0586..01e19a70eece 100644 --- a/ravenwood/runtime-helper-src/framework/com/android/platform/test/ravenwood/runtimehelper/ClassLoadHook.java +++ b/ravenwood/runtime-helper-src/framework/com/android/platform/test/ravenwood/runtimehelper/ClassLoadHook.java @@ -15,6 +15,8 @@ */ package com.android.platform.test.ravenwood.runtimehelper; +import static com.android.ravenwood.common.RavenwoodCommonUtils.RAVENWOOD_VERBOSE_LOGGING; + import android.system.ErrnoException; import android.system.Os; @@ -40,14 +42,6 @@ public class ClassLoadHook { private static final boolean SKIP_LOADING_LIBANDROID = "1".equals(System.getenv( "RAVENWOOD_SKIP_LOADING_LIBANDROID")); - /** - * If set to 1, and if $ANDROID_LOG_TAGS isn't set, we enable the verbose logging. - * - * (See also InitLogging() in http://ac/system/libbase/logging.cpp) - */ - private static final boolean RAVENWOOD_VERBOSE_LOGGING = "1".equals(System.getenv( - "RAVENWOOD_VERBOSE")); - public static final String CORE_NATIVE_CLASSES = "core_native_classes"; public static final String ICU_DATA_PATH = "icu.data.path"; public static final String KEYBOARD_PATHS = "keyboard_paths"; diff --git a/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/adapter/RunnerRewritingAdapter.kt b/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/adapter/RunnerRewritingAdapter.kt index eaef2cf6a956..bd9d96d81604 100644 --- a/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/adapter/RunnerRewritingAdapter.kt +++ b/ravenwood/tools/ravenizer/src/com/android/platform/test/ravenwood/ravenizer/adapter/RunnerRewritingAdapter.kt @@ -302,7 +302,7 @@ class RunnerRewritingAdapter private constructor( override fun visitCode() { visitFieldInsn(Opcodes.GETSTATIC, ravenwoodTestRunnerType.internlName, - RavenwoodAwareTestRunner.IMPLICIT_CLASS_MIN_RULE_NAME, + RavenwoodAwareTestRunner.IMPLICIT_CLASS_OUTER_RULE_NAME, testRuleType.desc ) visitFieldInsn(Opcodes.PUTSTATIC, @@ -313,7 +313,7 @@ class RunnerRewritingAdapter private constructor( visitFieldInsn(Opcodes.GETSTATIC, ravenwoodTestRunnerType.internlName, - RavenwoodAwareTestRunner.IMPLICIT_CLASS_MAX_RULE_NAME, + RavenwoodAwareTestRunner.IMPLICIT_CLASS_INNER_RULE_NAME, testRuleType.desc ) visitFieldInsn(Opcodes.PUTSTATIC, @@ -361,7 +361,7 @@ class RunnerRewritingAdapter private constructor( visitVarInsn(ALOAD, 0) visitFieldInsn(Opcodes.GETSTATIC, ravenwoodTestRunnerType.internlName, - RavenwoodAwareTestRunner.IMPLICIT_INST_MIN_RULE_NAME, + RavenwoodAwareTestRunner.IMPLICIT_INST_OUTER_RULE_NAME, testRuleType.desc ) visitFieldInsn(Opcodes.PUTFIELD, @@ -373,7 +373,7 @@ class RunnerRewritingAdapter private constructor( visitVarInsn(ALOAD, 0) visitFieldInsn(Opcodes.GETSTATIC, ravenwoodTestRunnerType.internlName, - RavenwoodAwareTestRunner.IMPLICIT_INST_MAX_RULE_NAME, + RavenwoodAwareTestRunner.IMPLICIT_INST_INNER_RULE_NAME, testRuleType.desc ) visitFieldInsn(Opcodes.PUTFIELD, |