diff options
8 files changed, 273 insertions, 139 deletions
diff --git a/packages/CrashRecovery/services/module/java/com/android/server/PackageWatchdog.java b/packages/CrashRecovery/services/module/java/com/android/server/PackageWatchdog.java index 2ba93f15f7fc..80d17d459cc8 100644 --- a/packages/CrashRecovery/services/module/java/com/android/server/PackageWatchdog.java +++ b/packages/CrashRecovery/services/module/java/com/android/server/PackageWatchdog.java @@ -25,6 +25,7 @@ import static com.android.server.crashrecovery.CrashRecoveryUtils.dumpCrashRecov import static java.lang.annotation.RetentionPolicy.SOURCE; +import android.annotation.CallbackExecutor; import android.annotation.FlaggedApi; import android.annotation.IntDef; import android.annotation.NonNull; @@ -91,6 +92,7 @@ import java.util.List; import java.util.Map; import java.util.NoSuchElementException; import java.util.Set; +import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; /** @@ -336,20 +338,28 @@ public class PackageWatchdog { /** * Registers {@code observer} to listen for package failures. Add a new ObserverInternal for * this observer if it does not already exist. + * For executing mitigations observers will receive callback on the given executor. * * <p>Observers are expected to call this on boot. It does not specify any packages but * it will resume observing any packages requested from a previous boot. + * + * @param observer instance of {@link PackageHealthObserver} for observing package failures + * and boot loops. + * @param executor Executor for the thread on which observers would receive callbacks * @hide */ - public void registerHealthObserver(PackageHealthObserver observer) { + public void registerHealthObserver(@NonNull PackageHealthObserver observer, + @NonNull @CallbackExecutor Executor executor) { synchronized (sLock) { ObserverInternal internalObserver = mAllObservers.get(observer.getUniqueIdentifier()); if (internalObserver != null) { internalObserver.registeredObserver = observer; + internalObserver.observerExecutor = executor; } else { internalObserver = new ObserverInternal(observer.getUniqueIdentifier(), new ArrayList<>()); internalObserver.registeredObserver = observer; + internalObserver.observerExecutor = executor; mAllObservers.put(observer.getUniqueIdentifier(), internalObserver); syncState("added new observer"); } @@ -357,40 +367,54 @@ public class PackageWatchdog { } /** - * Starts observing the health of the {@code packages} for {@code observer} and notifies - * {@code observer} of any package failures within the monitoring duration. + * Starts observing the health of the {@code packages} for {@code observer}. + * Note: Observer needs to be registered with {@link #registerHealthObserver} before calling + * this API. * * <p>If monitoring a package supporting explicit health check, at the end of the monitoring * duration if {@link #onHealthCheckPassed} was never called, - * {@link PackageHealthObserver#onExecuteHealthCheckMitigation} will be called as if the package failed. + * {@link PackageHealthObserver#onExecuteHealthCheckMitigation} will be called as if the + * package failed. * * <p>If {@code observer} is already monitoring a package in {@code packageNames}, * the monitoring window of that package will be reset to {@code durationMs} and the health - * check state will be reset to a default depending on if the package is contained in - * {@link mPackagesWithExplicitHealthCheckEnabled}. + * check state will be reset to a default. + * + * <p>The {@code observer} must be registered with {@link #registerHealthObserver} before + * calling this method. * - * <p>If {@code packageNames} is empty, this will be a no-op. + * @param packageNames The list of packages to check. If this is empty, the call will be a + * no-op. * - * <p>If {@code durationMs} is less than 1, a default monitoring duration - * {@link #DEFAULT_OBSERVING_DURATION_MS} will be used. + * @param timeoutMs The timeout after which Explicit Health Checks would not run. If this is + * less than 1, a default monitoring duration 2 days will be used. + * + * @throws IllegalStateException if the observer was not previously registered * @hide */ - public void startObservingHealth(PackageHealthObserver observer, List<String> packageNames, - long durationMs) { + public void startExplicitHealthCheck(@NonNull PackageHealthObserver observer, + @NonNull List<String> packageNames, long timeoutMs) { + synchronized (sLock) { + if (!mAllObservers.containsKey(observer.getUniqueIdentifier())) { + Slog.wtf(TAG, "No observer found, need to register the observer: " + + observer.getUniqueIdentifier()); + throw new IllegalStateException("Observer not registered"); + } + } if (packageNames.isEmpty()) { Slog.wtf(TAG, "No packages to observe, " + observer.getUniqueIdentifier()); return; } - if (durationMs < 1) { - Slog.wtf(TAG, "Invalid duration " + durationMs + "ms for observer " + if (timeoutMs < 1) { + Slog.wtf(TAG, "Invalid duration " + timeoutMs + "ms for observer " + observer.getUniqueIdentifier() + ". Not observing packages " + packageNames); - durationMs = DEFAULT_OBSERVING_DURATION_MS; + timeoutMs = DEFAULT_OBSERVING_DURATION_MS; } List<MonitoredPackage> packages = new ArrayList<>(); for (int i = 0; i < packageNames.size(); i++) { // Health checks not available yet so health check state will start INACTIVE - MonitoredPackage pkg = newMonitoredPackage(packageNames.get(i), durationMs, false); + MonitoredPackage pkg = newMonitoredPackage(packageNames.get(i), timeoutMs, false); if (pkg != null) { packages.add(pkg); } else { @@ -423,9 +447,6 @@ public class PackageWatchdog { } } - // Register observer in case not already registered - registerHealthObserver(observer); - // Sync after we add the new packages to the observers. We may have received packges // requiring an earlier schedule than we are currently scheduled for. syncState("updated observers"); @@ -485,7 +506,7 @@ public class PackageWatchdog { for (int pIndex = 0; pIndex < packages.size(); pIndex++) { VersionedPackage versionedPackage = packages.get(pIndex); // Observer that will receive failure for versionedPackage - PackageHealthObserver currentObserverToNotify = null; + ObserverInternal currentObserverToNotify = null; int currentObserverImpact = Integer.MAX_VALUE; MonitoredPackage currentMonitoredPackage = null; @@ -506,7 +527,7 @@ public class PackageWatchdog { versionedPackage, failureReason, mitigationCount); if (impact != PackageHealthObserverImpact.USER_IMPACT_LEVEL_0 && impact < currentObserverImpact) { - currentObserverToNotify = registeredObserver; + currentObserverToNotify = observer; currentObserverImpact = impact; currentMonitoredPackage = p; } @@ -515,18 +536,23 @@ public class PackageWatchdog { // Execute action with least user impact if (currentObserverToNotify != null) { - int mitigationCount = 1; + int mitigationCount; if (currentMonitoredPackage != null) { currentMonitoredPackage.noteMitigationCallLocked(); mitigationCount = currentMonitoredPackage.getMitigationCountLocked(); + } else { + mitigationCount = 1; } if (Flags.recoverabilityDetection()) { maybeExecute(currentObserverToNotify, versionedPackage, failureReason, currentObserverImpact, mitigationCount); } else { - currentObserverToNotify.onExecuteHealthCheckMitigation( - versionedPackage, failureReason, mitigationCount); + PackageHealthObserver registeredObserver = + currentObserverToNotify.registeredObserver; + currentObserverToNotify.observerExecutor.execute(() -> + registeredObserver.onExecuteHealthCheckMitigation( + versionedPackage, failureReason, mitigationCount)); } } } @@ -539,10 +565,11 @@ public class PackageWatchdog { * For native crashes or explicit health check failures, call directly into each observer to * mitigate the error without going through failure threshold logic. */ + @GuardedBy("sLock") private void handleFailureImmediately(List<VersionedPackage> packages, @FailureReasons int failureReason) { VersionedPackage failingPackage = packages.size() > 0 ? packages.get(0) : null; - PackageHealthObserver currentObserverToNotify = null; + ObserverInternal currentObserverToNotify = null; int currentObserverImpact = Integer.MAX_VALUE; for (ObserverInternal observer: mAllObservers.values()) { PackageHealthObserver registeredObserver = observer.registeredObserver; @@ -551,7 +578,7 @@ public class PackageWatchdog { failingPackage, failureReason, 1); if (impact != PackageHealthObserverImpact.USER_IMPACT_LEVEL_0 && impact < currentObserverImpact) { - currentObserverToNotify = registeredObserver; + currentObserverToNotify = observer; currentObserverImpact = impact; } } @@ -561,23 +588,30 @@ public class PackageWatchdog { maybeExecute(currentObserverToNotify, failingPackage, failureReason, currentObserverImpact, /*mitigationCount=*/ 1); } else { - currentObserverToNotify.onExecuteHealthCheckMitigation(failingPackage, - failureReason, 1); + PackageHealthObserver registeredObserver = + currentObserverToNotify.registeredObserver; + currentObserverToNotify.observerExecutor.execute(() -> + registeredObserver.onExecuteHealthCheckMitigation(failingPackage, + failureReason, 1)); + } } } - private void maybeExecute(PackageHealthObserver currentObserverToNotify, + private void maybeExecute(ObserverInternal currentObserverToNotify, VersionedPackage versionedPackage, @FailureReasons int failureReason, int currentObserverImpact, int mitigationCount) { if (allowMitigations(currentObserverImpact, versionedPackage)) { + PackageHealthObserver registeredObserver; synchronized (sLock) { mLastMitigation = mSystemClock.uptimeMillis(); + registeredObserver = currentObserverToNotify.registeredObserver; } - currentObserverToNotify.onExecuteHealthCheckMitigation(versionedPackage, failureReason, - mitigationCount); + currentObserverToNotify.observerExecutor.execute(() -> + registeredObserver.onExecuteHealthCheckMitigation(versionedPackage, + failureReason, mitigationCount)); } } @@ -613,8 +647,7 @@ public class PackageWatchdog { mBootThreshold.reset(); } int mitigationCount = mBootThreshold.getMitigationCount() + 1; - PackageHealthObserver currentObserverToNotify = null; - ObserverInternal currentObserverInternal = null; + ObserverInternal currentObserverToNotify = null; int currentObserverImpact = Integer.MAX_VALUE; for (int i = 0; i < mAllObservers.size(); i++) { final ObserverInternal observer = mAllObservers.valueAt(i); @@ -626,25 +659,31 @@ public class PackageWatchdog { : registeredObserver.onBootLoop(mitigationCount); if (impact != PackageHealthObserverImpact.USER_IMPACT_LEVEL_0 && impact < currentObserverImpact) { - currentObserverToNotify = registeredObserver; - currentObserverInternal = observer; + currentObserverToNotify = observer; currentObserverImpact = impact; } } } + if (currentObserverToNotify != null) { + PackageHealthObserver registeredObserver = + currentObserverToNotify.registeredObserver; if (Flags.recoverabilityDetection()) { int currentObserverMitigationCount = - currentObserverInternal.getBootMitigationCount() + 1; - currentObserverInternal.setBootMitigationCount( + currentObserverToNotify.getBootMitigationCount() + 1; + currentObserverToNotify.setBootMitigationCount( currentObserverMitigationCount); saveAllObserversBootMitigationCountToMetadata(METADATA_FILE); - currentObserverToNotify.onExecuteBootLoopMitigation( - currentObserverMitigationCount); + currentObserverToNotify.observerExecutor + .execute(() -> registeredObserver.onExecuteBootLoopMitigation( + currentObserverMitigationCount)); } else { mBootThreshold.setMitigationCount(mitigationCount); mBootThreshold.saveMitigationCountToMetadata(); - currentObserverToNotify.onExecuteBootLoopMitigation(mitigationCount); + currentObserverToNotify.observerExecutor + .execute(() -> registeredObserver.onExecuteBootLoopMitigation( + mitigationCount)); + } } } @@ -879,7 +918,7 @@ public class PackageWatchdog { * passed to observers in these API. * * <p> A persistent observer may choose to start observing certain failing packages, even if - * it has not explicitly asked to watch the package with {@link #startObservingHealth}. + * it has not explicitly asked to watch the package with {@link #startExplicitHealthCheck}. */ default boolean mayObservePackage(@NonNull String packageName) { return false; @@ -1136,8 +1175,10 @@ public class PackageWatchdog { if (versionedPkg != null) { Slog.i(TAG, "Explicit health check failed for package " + versionedPkg); - registeredObserver.onExecuteHealthCheckMitigation(versionedPkg, - PackageWatchdog.FAILURE_REASON_EXPLICIT_HEALTH_CHECK, 1); + observer.observerExecutor.execute(() -> + registeredObserver.onExecuteHealthCheckMitigation(versionedPkg, + PackageWatchdog.FAILURE_REASON_EXPLICIT_HEALTH_CHECK, + 1)); } } } @@ -1395,6 +1436,7 @@ public class PackageWatchdog { @Nullable @GuardedBy("sLock") public PackageHealthObserver registeredObserver; + public Executor observerExecutor; private int mMitigationCount; ObserverInternal(String name, List<MonitoredPackage> packages) { diff --git a/packages/CrashRecovery/services/module/java/com/android/server/RescueParty.java b/packages/CrashRecovery/services/module/java/com/android/server/RescueParty.java index 992f581a8a70..bad6ab7c1dd4 100644 --- a/packages/CrashRecovery/services/module/java/com/android/server/RescueParty.java +++ b/packages/CrashRecovery/services/module/java/com/android/server/RescueParty.java @@ -160,7 +160,7 @@ public class RescueParty { /** Register the Rescue Party observer as a Package Watchdog health observer */ public static void registerHealthObserver(Context context) { PackageWatchdog.getInstance(context).registerHealthObserver( - RescuePartyObserver.getInstance(context)); + RescuePartyObserver.getInstance(context), context.getMainExecutor()); } private static boolean isDisabled() { @@ -313,7 +313,7 @@ public class RescueParty { callingPackageList.addAll(callingPackages); Slog.i(TAG, "Starting to observe: " + callingPackageList + ", updated namespace: " + updatedNamespace); - PackageWatchdog.getInstance(context).startObservingHealth( + PackageWatchdog.getInstance(context).startExplicitHealthCheck( rescuePartyObserver, callingPackageList, DEFAULT_OBSERVING_DURATION_MS); diff --git a/packages/CrashRecovery/services/module/java/com/android/server/rollback/RollbackPackageHealthObserver.java b/packages/CrashRecovery/services/module/java/com/android/server/rollback/RollbackPackageHealthObserver.java index 311def80f248..e1e5866e7b04 100644 --- a/packages/CrashRecovery/services/module/java/com/android/server/rollback/RollbackPackageHealthObserver.java +++ b/packages/CrashRecovery/services/module/java/com/android/server/rollback/RollbackPackageHealthObserver.java @@ -111,7 +111,8 @@ public final class RollbackPackageHealthObserver implements PackageHealthObserve dataDir.mkdirs(); mLastStagedRollbackIdsFile = new File(dataDir, "last-staged-rollback-ids"); mTwoPhaseRollbackEnabledFile = new File(dataDir, "two-phase-rollback-enabled"); - PackageWatchdog.getInstance(mContext).registerHealthObserver(this); + PackageWatchdog.getInstance(mContext).registerHealthObserver(this, + context.getMainExecutor()); if (SystemProperties.getBoolean("sys.boot_completed", false)) { // Load the value from the file if system server has crashed and restarted @@ -280,7 +281,7 @@ public final class RollbackPackageHealthObserver implements PackageHealthObserve @AnyThread @NonNull public void startObservingHealth(@NonNull List<String> packages, @NonNull long durationMs) { - PackageWatchdog.getInstance(mContext).startObservingHealth(this, packages, durationMs); + PackageWatchdog.getInstance(mContext).startExplicitHealthCheck(this, packages, durationMs); } @AnyThread diff --git a/packages/CrashRecovery/services/platform/java/com/android/server/PackageWatchdog.java b/packages/CrashRecovery/services/platform/java/com/android/server/PackageWatchdog.java index 88fe36cda395..4fea9372971d 100644 --- a/packages/CrashRecovery/services/platform/java/com/android/server/PackageWatchdog.java +++ b/packages/CrashRecovery/services/platform/java/com/android/server/PackageWatchdog.java @@ -87,6 +87,7 @@ import java.util.List; import java.util.Map; import java.util.NoSuchElementException; import java.util.Set; +import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; /** @@ -362,7 +363,7 @@ public class PackageWatchdog { * it will resume observing any packages requested from a previous boot. * @hide */ - public void registerHealthObserver(PackageHealthObserver observer) { + public void registerHealthObserver(PackageHealthObserver observer, Executor ignoredExecutor) { synchronized (mLock) { ObserverInternal internalObserver = mAllObservers.get(observer.getUniqueIdentifier()); if (internalObserver != null) { @@ -396,7 +397,7 @@ public class PackageWatchdog { * {@link #DEFAULT_OBSERVING_DURATION_MS} will be used. * @hide */ - public void startObservingHealth(PackageHealthObserver observer, List<String> packageNames, + public void startExplicitHealthCheck(PackageHealthObserver observer, List<String> packageNames, long durationMs) { if (packageNames.isEmpty()) { Slog.wtf(TAG, "No packages to observe, " + observer.getUniqueIdentifier()); @@ -445,7 +446,7 @@ public class PackageWatchdog { } // Register observer in case not already registered - registerHealthObserver(observer); + registerHealthObserver(observer, null); // Sync after we add the new packages to the observers. We may have received packges // requiring an earlier schedule than we are currently scheduled for. @@ -861,7 +862,7 @@ public class PackageWatchdog { * otherwise * * <p> A persistent observer may choose to start observing certain failing packages, even if - * it has not explicitly asked to watch the package with {@link #startObservingHealth}. + * it has not explicitly asked to watch the package with {@link #startExplicitHealthCheck}. */ default boolean mayObservePackage(@NonNull String packageName) { return false; diff --git a/packages/CrashRecovery/services/platform/java/com/android/server/RescueParty.java b/packages/CrashRecovery/services/platform/java/com/android/server/RescueParty.java index f757236613d1..2bb72fb43dff 100644 --- a/packages/CrashRecovery/services/platform/java/com/android/server/RescueParty.java +++ b/packages/CrashRecovery/services/platform/java/com/android/server/RescueParty.java @@ -166,7 +166,7 @@ public class RescueParty { /** Register the Rescue Party observer as a Package Watchdog health observer */ public static void registerHealthObserver(Context context) { PackageWatchdog.getInstance(context).registerHealthObserver( - RescuePartyObserver.getInstance(context)); + RescuePartyObserver.getInstance(context), null); } private static boolean isDisabled() { @@ -387,7 +387,7 @@ public class RescueParty { callingPackageList.addAll(callingPackages); Slog.i(TAG, "Starting to observe: " + callingPackageList + ", updated namespace: " + updatedNamespace); - PackageWatchdog.getInstance(context).startObservingHealth( + PackageWatchdog.getInstance(context).startExplicitHealthCheck( rescuePartyObserver, callingPackageList, DEFAULT_OBSERVING_DURATION_MS); diff --git a/packages/CrashRecovery/services/platform/java/com/android/server/rollback/RollbackPackageHealthObserver.java b/packages/CrashRecovery/services/platform/java/com/android/server/rollback/RollbackPackageHealthObserver.java index 7445534e95bf..0692cdbc5e40 100644 --- a/packages/CrashRecovery/services/platform/java/com/android/server/rollback/RollbackPackageHealthObserver.java +++ b/packages/CrashRecovery/services/platform/java/com/android/server/rollback/RollbackPackageHealthObserver.java @@ -110,7 +110,7 @@ public final class RollbackPackageHealthObserver implements PackageHealthObserve dataDir.mkdirs(); mLastStagedRollbackIdsFile = new File(dataDir, "last-staged-rollback-ids"); mTwoPhaseRollbackEnabledFile = new File(dataDir, "two-phase-rollback-enabled"); - PackageWatchdog.getInstance(mContext).registerHealthObserver(this); + PackageWatchdog.getInstance(mContext).registerHealthObserver(this, null); mApexManager = apexManager; if (SystemProperties.getBoolean("sys.boot_completed", false)) { @@ -284,7 +284,7 @@ public final class RollbackPackageHealthObserver implements PackageHealthObserve @AnyThread @NonNull public void startObservingHealth(@NonNull List<String> packages, @NonNull long durationMs) { - PackageWatchdog.getInstance(mContext).startObservingHealth(this, packages, durationMs); + PackageWatchdog.getInstance(mContext).startExplicitHealthCheck(this, packages, durationMs); } @AnyThread diff --git a/tests/PackageWatchdog/src/com/android/server/CrashRecoveryTest.java b/tests/PackageWatchdog/src/com/android/server/CrashRecoveryTest.java index af87bf724a30..49616c30b784 100644 --- a/tests/PackageWatchdog/src/com/android/server/CrashRecoveryTest.java +++ b/tests/PackageWatchdog/src/com/android/server/CrashRecoveryTest.java @@ -83,6 +83,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Set; +import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; @@ -112,6 +113,7 @@ public class CrashRecoveryTest { private final TestClock mTestClock = new TestClock(); private TestLooper mTestLooper; + private Executor mTestExecutor; private Context mSpyContext; // Keep track of all created watchdogs to apply device config changes private List<PackageWatchdog> mAllocatedWatchdogs; @@ -141,6 +143,7 @@ public class CrashRecoveryTest { Manifest.permission.WRITE_DEVICE_CONFIG, Manifest.permission.WRITE_ALLOWLISTED_DEVICE_CONFIG); mTestLooper = new TestLooper(); + mTestExecutor = mTestLooper.getNewExecutor(); mSpyContext = spy(InstrumentationRegistry.getContext()); when(mSpyContext.getPackageManager()).thenReturn(mMockPackageManager); when(mMockPackageManager.getPackageInfo(anyString(), anyInt())).then(inv -> { @@ -231,31 +234,37 @@ public class CrashRecoveryTest { watchdog.noteBoot(); } + mTestLooper.dispatchAll(); verify(rescuePartyObserver).onExecuteBootLoopMitigation(1); verify(rescuePartyObserver, never()).onExecuteBootLoopMitigation(2); watchdog.noteBoot(); + mTestLooper.dispatchAll(); verify(rescuePartyObserver).onExecuteBootLoopMitigation(2); verify(rescuePartyObserver, never()).onExecuteBootLoopMitigation(3); watchdog.noteBoot(); + mTestLooper.dispatchAll(); verify(rescuePartyObserver).onExecuteBootLoopMitigation(3); verify(rescuePartyObserver, never()).onExecuteBootLoopMitigation(4); watchdog.noteBoot(); + mTestLooper.dispatchAll(); verify(rescuePartyObserver).onExecuteBootLoopMitigation(4); verify(rescuePartyObserver, never()).onExecuteBootLoopMitigation(5); watchdog.noteBoot(); + mTestLooper.dispatchAll(); verify(rescuePartyObserver).onExecuteBootLoopMitigation(5); verify(rescuePartyObserver, never()).onExecuteBootLoopMitigation(6); watchdog.noteBoot(); + mTestLooper.dispatchAll(); verify(rescuePartyObserver).onExecuteBootLoopMitigation(6); verify(rescuePartyObserver, never()).onExecuteBootLoopMitigation(7); } @@ -272,6 +281,7 @@ public class CrashRecoveryTest { watchdog.noteBoot(); } + mTestLooper.dispatchAll(); verify(rollbackObserver).onExecuteBootLoopMitigation(1); verify(rollbackObserver, never()).onExecuteBootLoopMitigation(2); @@ -281,6 +291,7 @@ public class CrashRecoveryTest { watchdog.noteBoot(); + mTestLooper.dispatchAll(); verify(rollbackObserver).onExecuteBootLoopMitigation(2); verify(rollbackObserver, never()).onExecuteBootLoopMitigation(3); @@ -289,6 +300,7 @@ public class CrashRecoveryTest { watchdog.noteBoot(); + mTestLooper.dispatchAll(); verify(rollbackObserver, never()).onExecuteBootLoopMitigation(3); } @@ -305,18 +317,21 @@ public class CrashRecoveryTest { for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT; i++) { watchdog.noteBoot(); } + mTestLooper.dispatchAll(); verify(rescuePartyObserver).onExecuteBootLoopMitigation(1); verify(rescuePartyObserver, never()).onExecuteBootLoopMitigation(2); verify(rollbackObserver, never()).onExecuteBootLoopMitigation(1); watchdog.noteBoot(); + mTestLooper.dispatchAll(); verify(rescuePartyObserver).onExecuteBootLoopMitigation(2); verify(rescuePartyObserver, never()).onExecuteBootLoopMitigation(3); verify(rollbackObserver, never()).onExecuteBootLoopMitigation(2); watchdog.noteBoot(); + mTestLooper.dispatchAll(); verify(rescuePartyObserver, never()).onExecuteBootLoopMitigation(3); verify(rollbackObserver).onExecuteBootLoopMitigation(1); verify(rollbackObserver, never()).onExecuteBootLoopMitigation(2); @@ -326,24 +341,28 @@ public class CrashRecoveryTest { watchdog.noteBoot(); + mTestLooper.dispatchAll(); verify(rescuePartyObserver).onExecuteBootLoopMitigation(3); verify(rescuePartyObserver, never()).onExecuteBootLoopMitigation(4); verify(rollbackObserver, never()).onExecuteBootLoopMitigation(2); watchdog.noteBoot(); + mTestLooper.dispatchAll(); verify(rescuePartyObserver).onExecuteBootLoopMitigation(4); verify(rescuePartyObserver, never()).onExecuteBootLoopMitigation(5); verify(rollbackObserver, never()).onExecuteBootLoopMitigation(2); watchdog.noteBoot(); + mTestLooper.dispatchAll(); verify(rescuePartyObserver).onExecuteBootLoopMitigation(5); verify(rescuePartyObserver, never()).onExecuteBootLoopMitigation(6); verify(rollbackObserver, never()).onExecuteBootLoopMitigation(2); watchdog.noteBoot(); + mTestLooper.dispatchAll(); verify(rescuePartyObserver, never()).onExecuteBootLoopMitigation(6); verify(rollbackObserver).onExecuteBootLoopMitigation(2); verify(rollbackObserver, never()).onExecuteBootLoopMitigation(3); @@ -352,6 +371,7 @@ public class CrashRecoveryTest { watchdog.noteBoot(); + mTestLooper.dispatchAll(); verify(rescuePartyObserver).onExecuteBootLoopMitigation(6); verify(rescuePartyObserver, never()).onExecuteBootLoopMitigation(7); verify(rollbackObserver, never()).onExecuteBootLoopMitigation(3); @@ -362,6 +382,7 @@ public class CrashRecoveryTest { for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT; i++) { watchdog.noteBoot(); } + mTestLooper.dispatchAll(); verify(rescuePartyObserver).onExecuteBootLoopMitigation(1); verify(rescuePartyObserver, never()).onExecuteBootLoopMitigation(2); } @@ -379,12 +400,14 @@ public class CrashRecoveryTest { for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT; i++) { watchdog.noteBoot(); } + mTestLooper.dispatchAll(); verify(rescuePartyObserver).onExecuteBootLoopMitigation(1); verify(rescuePartyObserver, never()).onExecuteBootLoopMitigation(2); verify(rollbackObserver, never()).onExecuteBootLoopMitigation(1); watchdog.noteBoot(); + mTestLooper.dispatchAll(); verify(rescuePartyObserver, never()).onExecuteBootLoopMitigation(2); verify(rollbackObserver).onExecuteBootLoopMitigation(1); verify(rollbackObserver, never()).onExecuteBootLoopMitigation(2); @@ -394,6 +417,7 @@ public class CrashRecoveryTest { watchdog.noteBoot(); + mTestLooper.dispatchAll(); verify(rescuePartyObserver, never()).onExecuteBootLoopMitigation(2); verify(rollbackObserver).onExecuteBootLoopMitigation(2); verify(rollbackObserver, never()).onExecuteBootLoopMitigation(3); @@ -402,6 +426,7 @@ public class CrashRecoveryTest { watchdog.noteBoot(); + mTestLooper.dispatchAll(); verify(rescuePartyObserver).onExecuteBootLoopMitigation(2); verify(rescuePartyObserver, never()).onExecuteBootLoopMitigation(3); verify(rollbackObserver, never()).onExecuteBootLoopMitigation(3); @@ -412,6 +437,7 @@ public class CrashRecoveryTest { for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT; i++) { watchdog.noteBoot(); } + mTestLooper.dispatchAll(); verify(rescuePartyObserver).onExecuteBootLoopMitigation(1); verify(rescuePartyObserver, never()).onExecuteBootLoopMitigation(2); } @@ -739,14 +765,14 @@ public class CrashRecoveryTest { } catch (PackageManager.NameNotFoundException e) { throw new RuntimeException(e); } - watchdog.registerHealthObserver(rollbackObserver); + watchdog.registerHealthObserver(rollbackObserver, mTestExecutor); return rollbackObserver; } RescuePartyObserver setUpRescuePartyObserver(PackageWatchdog watchdog) { setCrashRecoveryPropRescueBootCount(0); RescuePartyObserver rescuePartyObserver = spy(RescuePartyObserver.getInstance(mSpyContext)); assertFalse(RescueParty.isRebootPropertySet()); - watchdog.registerHealthObserver(rescuePartyObserver); + watchdog.registerHealthObserver(rescuePartyObserver, mTestExecutor); return rescuePartyObserver; } diff --git a/tests/PackageWatchdog/src/com/android/server/PackageWatchdogTest.java b/tests/PackageWatchdog/src/com/android/server/PackageWatchdogTest.java index 5a8a6bedf9eb..c64dc7296f0a 100644 --- a/tests/PackageWatchdog/src/com/android/server/PackageWatchdogTest.java +++ b/tests/PackageWatchdog/src/com/android/server/PackageWatchdogTest.java @@ -53,6 +53,7 @@ import android.platform.test.flag.junit.SetFlagsRule; import android.provider.DeviceConfig; import android.util.AtomicFile; import android.util.LongArrayQueue; +import android.util.Slog; import android.util.Xml; import androidx.test.InstrumentationRegistry; @@ -88,6 +89,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Set; +import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import java.util.function.Supplier; @@ -119,6 +121,7 @@ public class PackageWatchdogTest { private final TestClock mTestClock = new TestClock(); private TestLooper mTestLooper; + private Executor mTestExecutor; private Context mSpyContext; // Keep track of all created watchdogs to apply device config changes private List<PackageWatchdog> mAllocatedWatchdogs; @@ -155,6 +158,7 @@ public class PackageWatchdogTest { Manifest.permission.WRITE_DEVICE_CONFIG, Manifest.permission.WRITE_ALLOWLISTED_DEVICE_CONFIG); mTestLooper = new TestLooper(); + mTestExecutor = mTestLooper.getNewExecutor(); mSpyContext = spy(InstrumentationRegistry.getContext()); when(mSpyContext.getPackageManager()).thenReturn(mMockPackageManager); when(mMockPackageManager.getPackageInfo(anyString(), anyInt())).then(inv -> { @@ -226,7 +230,8 @@ public class PackageWatchdogTest { PackageWatchdog watchdog = createWatchdog(); TestObserver observer = new TestObserver(OBSERVER_NAME_1); - watchdog.startObservingHealth(observer, Arrays.asList(APP_A), SHORT_DURATION); + watchdog.registerHealthObserver(observer, mTestExecutor); + watchdog.startExplicitHealthCheck(observer, Arrays.asList(APP_A), SHORT_DURATION); raiseFatalFailureAndDispatch(watchdog, Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)), PackageWatchdog.FAILURE_REASON_UNKNOWN); @@ -242,8 +247,10 @@ public class PackageWatchdogTest { TestObserver observer1 = new TestObserver(OBSERVER_NAME_1); TestObserver observer2 = new TestObserver(OBSERVER_NAME_2); - watchdog.startObservingHealth(observer1, Arrays.asList(APP_A), SHORT_DURATION); - watchdog.startObservingHealth(observer2, Arrays.asList(APP_A, APP_B), SHORT_DURATION); + watchdog.registerHealthObserver(observer1, mTestExecutor); + watchdog.startExplicitHealthCheck(observer1, Arrays.asList(APP_A), SHORT_DURATION); + watchdog.registerHealthObserver(observer2, mTestExecutor); + watchdog.startExplicitHealthCheck(observer2, Arrays.asList(APP_A, APP_B), SHORT_DURATION); raiseFatalFailureAndDispatch(watchdog, Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE), new VersionedPackage(APP_B, VERSION_CODE)), @@ -260,7 +267,8 @@ public class PackageWatchdogTest { PackageWatchdog watchdog = createWatchdog(); TestObserver observer = new TestObserver(OBSERVER_NAME_1); - watchdog.startObservingHealth(observer, Arrays.asList(APP_A), SHORT_DURATION); + watchdog.registerHealthObserver(observer, mTestExecutor); + watchdog.startExplicitHealthCheck(observer, Arrays.asList(APP_A), SHORT_DURATION); watchdog.unregisterHealthObserver(observer); raiseFatalFailureAndDispatch(watchdog, Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)), @@ -276,8 +284,10 @@ public class PackageWatchdogTest { TestObserver observer1 = new TestObserver(OBSERVER_NAME_1); TestObserver observer2 = new TestObserver(OBSERVER_NAME_2); - watchdog.startObservingHealth(observer1, Arrays.asList(APP_A), SHORT_DURATION); - watchdog.startObservingHealth(observer2, Arrays.asList(APP_A), SHORT_DURATION); + watchdog.registerHealthObserver(observer1, mTestExecutor); + watchdog.startExplicitHealthCheck(observer1, Arrays.asList(APP_A), SHORT_DURATION); + watchdog.registerHealthObserver(observer2, mTestExecutor); + watchdog.startExplicitHealthCheck(observer2, Arrays.asList(APP_A), SHORT_DURATION); watchdog.unregisterHealthObserver(observer2); raiseFatalFailureAndDispatch(watchdog, Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)), @@ -294,7 +304,8 @@ public class PackageWatchdogTest { PackageWatchdog watchdog = createWatchdog(); TestObserver observer = new TestObserver(OBSERVER_NAME_1); - watchdog.startObservingHealth(observer, Arrays.asList(APP_A), SHORT_DURATION); + watchdog.registerHealthObserver(observer, mTestExecutor); + watchdog.startExplicitHealthCheck(observer, Arrays.asList(APP_A), SHORT_DURATION); moveTimeForwardAndDispatch(SHORT_DURATION); raiseFatalFailureAndDispatch(watchdog, Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)), @@ -310,8 +321,10 @@ public class PackageWatchdogTest { TestObserver observer1 = new TestObserver(OBSERVER_NAME_1); TestObserver observer2 = new TestObserver(OBSERVER_NAME_2); - watchdog.startObservingHealth(observer1, Arrays.asList(APP_A), SHORT_DURATION); - watchdog.startObservingHealth(observer2, Arrays.asList(APP_A), LONG_DURATION); + watchdog.registerHealthObserver(observer1, mTestExecutor); + watchdog.startExplicitHealthCheck(observer1, Arrays.asList(APP_A), SHORT_DURATION); + watchdog.registerHealthObserver(observer2, mTestExecutor); + watchdog.startExplicitHealthCheck(observer2, Arrays.asList(APP_A), LONG_DURATION); moveTimeForwardAndDispatch(SHORT_DURATION); raiseFatalFailureAndDispatch(watchdog, Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)), @@ -330,13 +343,14 @@ public class PackageWatchdogTest { TestObserver observer = new TestObserver(OBSERVER_NAME_1); // Start observing APP_A - watchdog.startObservingHealth(observer, Arrays.asList(APP_A), SHORT_DURATION); + watchdog.registerHealthObserver(observer, mTestExecutor); + watchdog.startExplicitHealthCheck(observer, Arrays.asList(APP_A), SHORT_DURATION); // Then advance time half-way moveTimeForwardAndDispatch(SHORT_DURATION / 2); // Start observing APP_A again - watchdog.startObservingHealth(observer, Arrays.asList(APP_A), SHORT_DURATION); + watchdog.startExplicitHealthCheck(observer, Arrays.asList(APP_A), SHORT_DURATION); // Then advance time such that it should have expired were it not for the second observation moveTimeForwardAndDispatch((SHORT_DURATION / 2) + 1); @@ -358,15 +372,17 @@ public class PackageWatchdogTest { TestObserver observer1 = new TestObserver(OBSERVER_NAME_1); TestObserver observer2 = new TestObserver(OBSERVER_NAME_2); - watchdog1.startObservingHealth(observer1, Arrays.asList(APP_A), SHORT_DURATION); - watchdog1.startObservingHealth(observer2, Arrays.asList(APP_A, APP_B), SHORT_DURATION); + watchdog1.registerHealthObserver(observer1, mTestExecutor); + watchdog1.startExplicitHealthCheck(observer1, Arrays.asList(APP_A), SHORT_DURATION); + watchdog1.registerHealthObserver(observer2, mTestExecutor); + watchdog1.startExplicitHealthCheck(observer2, Arrays.asList(APP_A, APP_B), SHORT_DURATION); // Then advance time and run IO Handler so file is saved mTestLooper.dispatchAll(); // Then start a new watchdog PackageWatchdog watchdog2 = createWatchdog(); // Then resume observer1 and observer2 - watchdog2.registerHealthObserver(observer1); - watchdog2.registerHealthObserver(observer2); + watchdog2.registerHealthObserver(observer1, mTestExecutor); + watchdog2.registerHealthObserver(observer2, mTestExecutor); raiseFatalFailureAndDispatch(watchdog2, Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE), new VersionedPackage(APP_B, VERSION_CODE)), @@ -374,6 +390,7 @@ public class PackageWatchdogTest { // We should receive failed packages as expected to ensure observers are persisted and // resumed correctly + mTestLooper.dispatchAll(); assertThat(observer1.mHealthCheckFailedPackages).containsExactly(APP_A); assertThat(observer2.mHealthCheckFailedPackages).containsExactly(APP_A, APP_B); } @@ -387,8 +404,10 @@ public class PackageWatchdogTest { TestObserver observer1 = new TestObserver(OBSERVER_NAME_1); TestObserver observer2 = new TestObserver(OBSERVER_NAME_2); - watchdog.startObservingHealth(observer2, Arrays.asList(APP_A), SHORT_DURATION); - watchdog.startObservingHealth(observer1, Arrays.asList(APP_A), SHORT_DURATION); + watchdog.registerHealthObserver(observer2, mTestExecutor); + watchdog.startExplicitHealthCheck(observer2, Arrays.asList(APP_A), SHORT_DURATION); + watchdog.registerHealthObserver(observer1, mTestExecutor); + watchdog.startExplicitHealthCheck(observer1, Arrays.asList(APP_A), SHORT_DURATION); // Then fail APP_A below the threshold for (int i = 0; i < watchdog.getTriggerFailureCount() - 1; i++) { @@ -414,9 +433,10 @@ public class PackageWatchdogTest { TestObserver observer1 = new TestObserver(OBSERVER_NAME_1); TestObserver observer2 = new TestObserver(OBSERVER_NAME_2); - - watchdog.startObservingHealth(observer2, Arrays.asList(APP_A), SHORT_DURATION); - watchdog.startObservingHealth(observer1, Arrays.asList(APP_B), SHORT_DURATION); + watchdog.registerHealthObserver(observer2, mTestExecutor); + watchdog.startExplicitHealthCheck(observer2, Arrays.asList(APP_A), SHORT_DURATION); + watchdog.registerHealthObserver(observer1, mTestExecutor); + watchdog.startExplicitHealthCheck(observer1, Arrays.asList(APP_B), SHORT_DURATION); // Then fail APP_C (not observed) above the threshold raiseFatalFailureAndDispatch(watchdog, @@ -448,7 +468,8 @@ public class PackageWatchdogTest { } }; - watchdog.startObservingHealth(observer, Arrays.asList(APP_A), SHORT_DURATION); + watchdog.registerHealthObserver(observer, mTestExecutor); + watchdog.startExplicitHealthCheck(observer, Arrays.asList(APP_A), SHORT_DURATION); // Then fail APP_A (different version) above the threshold raiseFatalFailureAndDispatch(watchdog, @@ -477,13 +498,17 @@ public class PackageWatchdogTest { PackageHealthObserverImpact.USER_IMPACT_LEVEL_10); // Start observing for all impact observers - watchdog.startObservingHealth(observerNone, Arrays.asList(APP_A, APP_B, APP_C, APP_D), + watchdog.registerHealthObserver(observerNone, mTestExecutor); + watchdog.startExplicitHealthCheck(observerNone, Arrays.asList(APP_A, APP_B, APP_C, APP_D), SHORT_DURATION); - watchdog.startObservingHealth(observerHigh, Arrays.asList(APP_A, APP_B, APP_C), + watchdog.registerHealthObserver(observerHigh, mTestExecutor); + watchdog.startExplicitHealthCheck(observerHigh, Arrays.asList(APP_A, APP_B, APP_C), SHORT_DURATION); - watchdog.startObservingHealth(observerMid, Arrays.asList(APP_A, APP_B), + watchdog.registerHealthObserver(observerMid, mTestExecutor); + watchdog.startExplicitHealthCheck(observerMid, Arrays.asList(APP_A, APP_B), SHORT_DURATION); - watchdog.startObservingHealth(observerLow, Arrays.asList(APP_A), + watchdog.registerHealthObserver(observerLow, mTestExecutor); + watchdog.startExplicitHealthCheck(observerLow, Arrays.asList(APP_A), SHORT_DURATION); // Then fail all apps above the threshold @@ -523,13 +548,17 @@ public class PackageWatchdogTest { PackageHealthObserverImpact.USER_IMPACT_LEVEL_10); // Start observing for all impact observers - watchdog.startObservingHealth(observerNone, Arrays.asList(APP_A, APP_B, APP_C, APP_D), + watchdog.registerHealthObserver(observerNone, mTestExecutor); + watchdog.startExplicitHealthCheck(observerNone, Arrays.asList(APP_A, APP_B, APP_C, APP_D), SHORT_DURATION); - watchdog.startObservingHealth(observerHigh, Arrays.asList(APP_A, APP_B, APP_C), + watchdog.registerHealthObserver(observerHigh, mTestExecutor); + watchdog.startExplicitHealthCheck(observerHigh, Arrays.asList(APP_A, APP_B, APP_C), SHORT_DURATION); - watchdog.startObservingHealth(observerMid, Arrays.asList(APP_A, APP_B), + watchdog.registerHealthObserver(observerMid, mTestExecutor); + watchdog.startExplicitHealthCheck(observerMid, Arrays.asList(APP_A, APP_B), SHORT_DURATION); - watchdog.startObservingHealth(observerLow, Arrays.asList(APP_A), + watchdog.registerHealthObserver(observerLow, mTestExecutor); + watchdog.startExplicitHealthCheck(observerLow, Arrays.asList(APP_A), SHORT_DURATION); // Then fail all apps above the threshold @@ -577,8 +606,10 @@ public class PackageWatchdogTest { PackageHealthObserverImpact.USER_IMPACT_LEVEL_30); // Start observing for observerFirst and observerSecond with failure handling - watchdog.startObservingHealth(observerFirst, Arrays.asList(APP_A), LONG_DURATION); - watchdog.startObservingHealth(observerSecond, Arrays.asList(APP_A), LONG_DURATION); + watchdog.registerHealthObserver(observerFirst, mTestExecutor); + watchdog.startExplicitHealthCheck(observerFirst, Arrays.asList(APP_A), LONG_DURATION); + watchdog.registerHealthObserver(observerSecond, mTestExecutor); + watchdog.startExplicitHealthCheck(observerSecond, Arrays.asList(APP_A), LONG_DURATION); // Then fail APP_A above the threshold raiseFatalFailureAndDispatch(watchdog, @@ -641,8 +672,10 @@ public class PackageWatchdogTest { PackageHealthObserverImpact.USER_IMPACT_LEVEL_30); // Start observing for observerFirst and observerSecond with failure handling - watchdog.startObservingHealth(observerFirst, Arrays.asList(APP_A), LONG_DURATION); - watchdog.startObservingHealth(observerSecond, Arrays.asList(APP_A), LONG_DURATION); + watchdog.registerHealthObserver(observerFirst, mTestExecutor); + watchdog.startExplicitHealthCheck(observerFirst, Arrays.asList(APP_A), LONG_DURATION); + watchdog.registerHealthObserver(observerSecond, mTestExecutor); + watchdog.startExplicitHealthCheck(observerSecond, Arrays.asList(APP_A), LONG_DURATION); // Then fail APP_A above the threshold raiseFatalFailureAndDispatch(watchdog, @@ -709,8 +742,10 @@ public class PackageWatchdogTest { PackageHealthObserverImpact.USER_IMPACT_LEVEL_100); // Start observing for observer1 and observer2 with failure handling - watchdog.startObservingHealth(observer2, Arrays.asList(APP_A), SHORT_DURATION); - watchdog.startObservingHealth(observer1, Arrays.asList(APP_A), SHORT_DURATION); + watchdog.registerHealthObserver(observer2, mTestExecutor); + watchdog.startExplicitHealthCheck(observer2, Arrays.asList(APP_A), SHORT_DURATION); + watchdog.registerHealthObserver(observer1, mTestExecutor); + watchdog.startExplicitHealthCheck(observer1, Arrays.asList(APP_A), SHORT_DURATION); // Then fail APP_A above the threshold raiseFatalFailureAndDispatch(watchdog, @@ -731,8 +766,10 @@ public class PackageWatchdogTest { PackageHealthObserverImpact.USER_IMPACT_LEVEL_50); // Start observing for observer1 and observer2 with failure handling - watchdog.startObservingHealth(observer2, Arrays.asList(APP_A), SHORT_DURATION); - watchdog.startObservingHealth(observer1, Arrays.asList(APP_A), SHORT_DURATION); + watchdog.registerHealthObserver(observer2, mTestExecutor); + watchdog.startExplicitHealthCheck(observer2, Arrays.asList(APP_A), SHORT_DURATION); + watchdog.registerHealthObserver(observer1, mTestExecutor); + watchdog.startExplicitHealthCheck(observer1, Arrays.asList(APP_A), SHORT_DURATION); // Then fail APP_A above the threshold raiseFatalFailureAndDispatch(watchdog, @@ -762,8 +799,10 @@ public class PackageWatchdogTest { // Start observing with explicit health checks for APP_A and APP_B respectively // with observer1 and observer2 controller.setSupportedPackages(Arrays.asList(APP_A, APP_B)); - watchdog.startObservingHealth(observer1, Arrays.asList(APP_A), SHORT_DURATION); - watchdog.startObservingHealth(observer2, Arrays.asList(APP_B), SHORT_DURATION); + watchdog.registerHealthObserver(observer1, mTestExecutor); + watchdog.startExplicitHealthCheck(observer1, Arrays.asList(APP_A), SHORT_DURATION); + watchdog.registerHealthObserver(observer2, mTestExecutor); + watchdog.startExplicitHealthCheck(observer2, Arrays.asList(APP_B), SHORT_DURATION); // Run handler so requests are dispatched to the controller mTestLooper.dispatchAll(); @@ -779,7 +818,8 @@ public class PackageWatchdogTest { // Observer3 didn't exist when we got the explicit health check above, so // it starts out with a non-passing explicit health check and has to wait for a pass // otherwise it would be notified of APP_A failure on expiry - watchdog.startObservingHealth(observer3, Arrays.asList(APP_A), SHORT_DURATION); + watchdog.registerHealthObserver(observer3, mTestExecutor); + watchdog.startExplicitHealthCheck(observer3, Arrays.asList(APP_A), SHORT_DURATION); // Then expire observers moveTimeForwardAndDispatch(SHORT_DURATION); @@ -809,8 +849,9 @@ public class PackageWatchdogTest { // Start observing with explicit health checks for APP_A and APP_B controller.setSupportedPackages(Arrays.asList(APP_A, APP_B, APP_C)); - watchdog.startObservingHealth(observer, Arrays.asList(APP_A), SHORT_DURATION); - watchdog.startObservingHealth(observer, Arrays.asList(APP_B), LONG_DURATION); + watchdog.registerHealthObserver(observer, mTestExecutor); + watchdog.startExplicitHealthCheck(observer, Arrays.asList(APP_A), SHORT_DURATION); + watchdog.startExplicitHealthCheck(observer, Arrays.asList(APP_B), LONG_DURATION); // Run handler so requests are dispatched to the controller mTestLooper.dispatchAll(); @@ -846,7 +887,7 @@ public class PackageWatchdogTest { // Then set new supported packages controller.setSupportedPackages(Arrays.asList(APP_C)); // Start observing APP_A and APP_C; only APP_C has support for explicit health checks - watchdog.startObservingHealth(observer, Arrays.asList(APP_A, APP_C), SHORT_DURATION); + watchdog.startExplicitHealthCheck(observer, Arrays.asList(APP_A, APP_C), SHORT_DURATION); // Run handler so requests/cancellations are dispatched to the controller mTestLooper.dispatchAll(); @@ -877,7 +918,8 @@ public class PackageWatchdogTest { // package observation duration == LONG_DURATION // health check duration == SHORT_DURATION (set by default in the TestController) controller.setSupportedPackages(Arrays.asList(APP_A)); - watchdog.startObservingHealth(observer, Arrays.asList(APP_A), LONG_DURATION); + watchdog.registerHealthObserver(observer, mTestExecutor); + watchdog.startExplicitHealthCheck(observer, Arrays.asList(APP_A), LONG_DURATION); // Then APP_A has exceeded health check duration moveTimeForwardAndDispatch(SHORT_DURATION); @@ -908,7 +950,8 @@ public class PackageWatchdogTest { // package observation duration == SHORT_DURATION / 2 // health check duration == SHORT_DURATION (set by default in the TestController) controller.setSupportedPackages(Arrays.asList(APP_A)); - watchdog.startObservingHealth(observer, Arrays.asList(APP_A), SHORT_DURATION / 2); + watchdog.registerHealthObserver(observer, mTestExecutor); + watchdog.startExplicitHealthCheck(observer, Arrays.asList(APP_A), SHORT_DURATION / 2); // Forward time to expire the observation duration moveTimeForwardAndDispatch(SHORT_DURATION / 2); @@ -981,7 +1024,7 @@ public class PackageWatchdogTest { // Start observing with failure handling TestObserver observer = new TestObserver(OBSERVER_NAME_1, PackageHealthObserverImpact.USER_IMPACT_LEVEL_100); - wd.startObservingHealth(observer, Collections.singletonList(APP_A), SHORT_DURATION); + wd.startExplicitHealthCheck(observer, Collections.singletonList(APP_A), SHORT_DURATION); // Notify of NetworkStack failure mConnectivityModuleCallbackCaptor.getValue().onNetworkStackFailure(APP_A); @@ -1001,7 +1044,7 @@ public class PackageWatchdogTest { // Start observing with failure handling TestObserver observer = new TestObserver(OBSERVER_NAME_1, PackageHealthObserverImpact.USER_IMPACT_LEVEL_100); - wd.startObservingHealth(observer, Collections.singletonList(APP_A), SHORT_DURATION); + wd.startExplicitHealthCheck(observer, Collections.singletonList(APP_A), SHORT_DURATION); // Notify of NetworkStack failure mConnectivityModuleCallbackCaptor.getValue().onNetworkStackFailure(APP_A); @@ -1022,7 +1065,8 @@ public class PackageWatchdogTest { PackageWatchdog watchdog = createWatchdog(); TestObserver observer = new TestObserver(OBSERVER_NAME_1); - watchdog.startObservingHealth(observer, Arrays.asList(APP_A), SHORT_DURATION); + watchdog.registerHealthObserver(observer, mTestExecutor); + watchdog.startExplicitHealthCheck(observer, Arrays.asList(APP_A), SHORT_DURATION); // Fail APP_A below the threshold which should not trigger package failures for (int i = 0; i < PackageWatchdog.DEFAULT_TRIGGER_FAILURE_COUNT - 1; i++) { watchdog.notifyPackageFailure(Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)), @@ -1050,7 +1094,8 @@ public class PackageWatchdogTest { PackageWatchdog watchdog = createWatchdog(); TestObserver observer = new TestObserver(OBSERVER_NAME_1); - watchdog.startObservingHealth(observer, Arrays.asList(APP_A, APP_B), Long.MAX_VALUE); + watchdog.registerHealthObserver(observer, mTestExecutor); + watchdog.startExplicitHealthCheck(observer, Arrays.asList(APP_A, APP_B), Long.MAX_VALUE); watchdog.notifyPackageFailure(Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)), PackageWatchdog.FAILURE_REASON_UNKNOWN); moveTimeForwardAndDispatch(PackageWatchdog.DEFAULT_TRIGGER_FAILURE_DURATION_MS + 1); @@ -1075,15 +1120,16 @@ public class PackageWatchdogTest { } /** - * Test default monitoring duration is used when PackageWatchdog#startObservingHealth is offered - * an invalid durationMs. + * Test default monitoring duration is used when PackageWatchdog#startExplicitHealthCheck is + * offered an invalid durationMs. */ @Test public void testInvalidMonitoringDuration_beforeExpiry() { PackageWatchdog watchdog = createWatchdog(); TestObserver observer = new TestObserver(OBSERVER_NAME_1); - watchdog.startObservingHealth(observer, Arrays.asList(APP_A), -1); + watchdog.registerHealthObserver(observer, mTestExecutor); + watchdog.startExplicitHealthCheck(observer, Arrays.asList(APP_A), -1); // Note: Don't move too close to the expiration time otherwise the handler will be thrashed // by PackageWatchdog#scheduleNextSyncStateLocked which keeps posting runnables with very // small timeouts. @@ -1097,15 +1143,16 @@ public class PackageWatchdogTest { } /** - * Test default monitoring duration is used when PackageWatchdog#startObservingHealth is offered - * an invalid durationMs. + * Test default monitoring duration is used when PackageWatchdog#startExplicitHealthCheck is + * offered an invalid durationMs. */ @Test public void testInvalidMonitoringDuration_afterExpiry() { PackageWatchdog watchdog = createWatchdog(); TestObserver observer = new TestObserver(OBSERVER_NAME_1); - watchdog.startObservingHealth(observer, Arrays.asList(APP_A), -1); + watchdog.registerHealthObserver(observer, mTestExecutor); + watchdog.startExplicitHealthCheck(observer, Arrays.asList(APP_A), -1); moveTimeForwardAndDispatch(PackageWatchdog.DEFAULT_OBSERVING_DURATION_MS + 1); raiseFatalFailureAndDispatch(watchdog, Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)), @@ -1127,7 +1174,8 @@ public class PackageWatchdogTest { PackageWatchdog watchdog = createWatchdog(); TestObserver observer = new TestObserver(OBSERVER_NAME_1); - watchdog.startObservingHealth(observer, Arrays.asList(APP_A), Long.MAX_VALUE); + watchdog.registerHealthObserver(observer, mTestExecutor); + watchdog.startExplicitHealthCheck(observer, Arrays.asList(APP_A), Long.MAX_VALUE); // Raise 2 failures at t=0 and t=900 respectively watchdog.notifyPackageFailure(Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)), PackageWatchdog.FAILURE_REASON_UNKNOWN); @@ -1154,8 +1202,10 @@ public class PackageWatchdogTest { TestObserver observer1 = new TestObserver(OBSERVER_NAME_1); TestObserver observer2 = new TestObserver(OBSERVER_NAME_2); - watchdog.startObservingHealth(observer1, Arrays.asList(APP_A), SHORT_DURATION); - watchdog.startObservingHealth(observer2, Arrays.asList(APP_B), SHORT_DURATION); + watchdog.registerHealthObserver(observer1, mTestExecutor); + watchdog.startExplicitHealthCheck(observer1, Arrays.asList(APP_A), SHORT_DURATION); + watchdog.registerHealthObserver(observer2, mTestExecutor); + watchdog.startExplicitHealthCheck(observer2, Arrays.asList(APP_B), SHORT_DURATION); raiseFatalFailureAndDispatch(watchdog, Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)), PackageWatchdog.FAILURE_REASON_APP_CRASH); @@ -1174,7 +1224,8 @@ public class PackageWatchdogTest { PackageWatchdog watchdog = createWatchdog(); TestObserver observer1 = new TestObserver(OBSERVER_NAME_1); - watchdog.startObservingHealth(observer1, Arrays.asList(APP_A), SHORT_DURATION); + watchdog.registerHealthObserver(observer1, mTestExecutor); + watchdog.startExplicitHealthCheck(observer1, Arrays.asList(APP_A), SHORT_DURATION); raiseFatalFailureAndDispatch(watchdog, Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)), PackageWatchdog.FAILURE_REASON_NATIVE_CRASH); @@ -1194,7 +1245,8 @@ public class PackageWatchdogTest { persistentObserver.setPersistent(true); persistentObserver.setMayObservePackages(true); - watchdog.startObservingHealth(persistentObserver, Arrays.asList(APP_B), SHORT_DURATION); + watchdog.registerHealthObserver(persistentObserver, mTestExecutor); + watchdog.startExplicitHealthCheck(persistentObserver, Arrays.asList(APP_B), SHORT_DURATION); raiseFatalFailureAndDispatch(watchdog, Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)), PackageWatchdog.FAILURE_REASON_UNKNOWN); @@ -1212,7 +1264,8 @@ public class PackageWatchdogTest { persistentObserver.setPersistent(true); persistentObserver.setMayObservePackages(false); - watchdog.startObservingHealth(persistentObserver, Arrays.asList(APP_B), SHORT_DURATION); + watchdog.registerHealthObserver(persistentObserver, mTestExecutor); + watchdog.startExplicitHealthCheck(persistentObserver, Arrays.asList(APP_B), SHORT_DURATION); raiseFatalFailureAndDispatch(watchdog, Arrays.asList(new VersionedPackage(APP_A, VERSION_CODE)), PackageWatchdog.FAILURE_REASON_UNKNOWN); @@ -1223,13 +1276,15 @@ public class PackageWatchdogTest { /** Ensure that boot loop mitigation is done when the number of boots meets the threshold. */ @Test public void testBootLoopDetection_meetsThreshold() { + Slog.w("hrm1243", "I should definitely be here try 1 "); mSetFlagsRule.disableFlags(Flags.FLAG_RECOVERABILITY_DETECTION); PackageWatchdog watchdog = createWatchdog(); TestObserver bootObserver = new TestObserver(OBSERVER_NAME_1); - watchdog.registerHealthObserver(bootObserver); + watchdog.registerHealthObserver(bootObserver, mTestExecutor); for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT; i++) { watchdog.noteBoot(); } + mTestLooper.dispatchAll(); assertThat(bootObserver.mitigatedBootLoop()).isTrue(); } @@ -1237,10 +1292,11 @@ public class PackageWatchdogTest { public void testBootLoopDetection_meetsThresholdRecoverability() { PackageWatchdog watchdog = createWatchdog(); TestObserver bootObserver = new TestObserver(OBSERVER_NAME_1); - watchdog.registerHealthObserver(bootObserver); + watchdog.registerHealthObserver(bootObserver, mTestExecutor); for (int i = 0; i < 15; i++) { watchdog.noteBoot(); } + mTestLooper.dispatchAll(); assertThat(bootObserver.mitigatedBootLoop()).isTrue(); } @@ -1252,10 +1308,11 @@ public class PackageWatchdogTest { public void testBootLoopDetection_doesNotMeetThreshold() { PackageWatchdog watchdog = createWatchdog(); TestObserver bootObserver = new TestObserver(OBSERVER_NAME_1); - watchdog.registerHealthObserver(bootObserver); + watchdog.registerHealthObserver(bootObserver, mTestExecutor); for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT - 1; i++) { watchdog.noteBoot(); } + mTestLooper.dispatchAll(); assertThat(bootObserver.mitigatedBootLoop()).isFalse(); } @@ -1268,10 +1325,11 @@ public class PackageWatchdogTest { PackageWatchdog watchdog = createWatchdog(); TestObserver bootObserver = new TestObserver(OBSERVER_NAME_1, PackageHealthObserverImpact.USER_IMPACT_LEVEL_30); - watchdog.registerHealthObserver(bootObserver); + watchdog.registerHealthObserver(bootObserver, mTestExecutor); for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT - 1; i++) { watchdog.noteBoot(); } + mTestLooper.dispatchAll(); assertThat(bootObserver.mitigatedBootLoop()).isFalse(); } @@ -1286,11 +1344,12 @@ public class PackageWatchdogTest { bootObserver1.setImpact(PackageHealthObserverImpact.USER_IMPACT_LEVEL_10); TestObserver bootObserver2 = new TestObserver(OBSERVER_NAME_2); bootObserver2.setImpact(PackageHealthObserverImpact.USER_IMPACT_LEVEL_30); - watchdog.registerHealthObserver(bootObserver1); - watchdog.registerHealthObserver(bootObserver2); + watchdog.registerHealthObserver(bootObserver1, mTestExecutor); + watchdog.registerHealthObserver(bootObserver2, mTestExecutor); for (int i = 0; i < PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT; i++) { watchdog.noteBoot(); } + mTestLooper.dispatchAll(); assertThat(bootObserver1.mitigatedBootLoop()).isTrue(); assertThat(bootObserver2.mitigatedBootLoop()).isFalse(); } @@ -1302,11 +1361,12 @@ public class PackageWatchdogTest { bootObserver1.setImpact(PackageHealthObserverImpact.USER_IMPACT_LEVEL_10); TestObserver bootObserver2 = new TestObserver(OBSERVER_NAME_2); bootObserver2.setImpact(PackageHealthObserverImpact.USER_IMPACT_LEVEL_30); - watchdog.registerHealthObserver(bootObserver1); - watchdog.registerHealthObserver(bootObserver2); + watchdog.registerHealthObserver(bootObserver1, mTestExecutor); + watchdog.registerHealthObserver(bootObserver2, mTestExecutor); for (int i = 0; i < 15; i++) { watchdog.noteBoot(); } + mTestLooper.dispatchAll(); assertThat(bootObserver1.mitigatedBootLoop()).isTrue(); assertThat(bootObserver2.mitigatedBootLoop()).isFalse(); } @@ -1319,7 +1379,7 @@ public class PackageWatchdogTest { mSetFlagsRule.disableFlags(Flags.FLAG_RECOVERABILITY_DETECTION); PackageWatchdog watchdog = createWatchdog(); TestObserver bootObserver = new TestObserver(OBSERVER_NAME_1); - watchdog.registerHealthObserver(bootObserver); + watchdog.registerHealthObserver(bootObserver, mTestExecutor); for (int i = 0; i < 4; i++) { for (int j = 0; j < PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT; j++) { watchdog.noteBoot(); @@ -1333,7 +1393,7 @@ public class PackageWatchdogTest { watchdog.noteBoot(); } } - + mTestLooper.dispatchAll(); assertThat(bootObserver.mBootMitigationCounts).isEqualTo(List.of(1, 2, 3, 4, 1, 2, 3, 4)); } @@ -1342,7 +1402,7 @@ public class PackageWatchdogTest { PackageWatchdog watchdog = createWatchdog(); TestObserver bootObserver = new TestObserver(OBSERVER_NAME_1, PackageHealthObserverImpact.USER_IMPACT_LEVEL_30); - watchdog.registerHealthObserver(bootObserver); + watchdog.registerHealthObserver(bootObserver, mTestExecutor); for (int j = 0; j < PackageWatchdog.DEFAULT_BOOT_LOOP_TRIGGER_COUNT - 1; j++) { watchdog.noteBoot(); } @@ -1358,7 +1418,7 @@ public class PackageWatchdogTest { for (int i = 0; i < 4; i++) { watchdog.noteBoot(); } - + mTestLooper.dispatchAll(); assertThat(bootObserver.mBootMitigationCounts).isEqualTo(List.of(1, 2, 3, 4, 1, 2, 3, 4)); } @@ -1370,7 +1430,8 @@ public class PackageWatchdogTest { public void testNullFailedPackagesList() { PackageWatchdog watchdog = createWatchdog(); TestObserver observer1 = new TestObserver(OBSERVER_NAME_1); - watchdog.startObservingHealth(observer1, List.of(APP_A), LONG_DURATION); + watchdog.registerHealthObserver(observer1, mTestExecutor); + watchdog.startExplicitHealthCheck(observer1, List.of(APP_A), LONG_DURATION); raiseFatalFailureAndDispatch(watchdog, null, PackageWatchdog.FAILURE_REASON_APP_CRASH); assertThat(observer1.mMitigatedPackages).isEmpty(); @@ -1388,18 +1449,18 @@ public class PackageWatchdogTest { PackageWatchdog watchdog = createWatchdog(testController, true); TestObserver testObserver1 = new TestObserver(OBSERVER_NAME_1); - watchdog.registerHealthObserver(testObserver1); - watchdog.startObservingHealth(testObserver1, List.of(APP_A), LONG_DURATION); + watchdog.registerHealthObserver(testObserver1, mTestExecutor); + watchdog.startExplicitHealthCheck(testObserver1, List.of(APP_A), LONG_DURATION); mTestLooper.dispatchAll(); TestObserver testObserver2 = new TestObserver(OBSERVER_NAME_2); - watchdog.registerHealthObserver(testObserver2); - watchdog.startObservingHealth(testObserver2, List.of(APP_B), LONG_DURATION); + watchdog.registerHealthObserver(testObserver2, mTestExecutor); + watchdog.startExplicitHealthCheck(testObserver2, List.of(APP_B), LONG_DURATION); mTestLooper.dispatchAll(); TestObserver testObserver3 = new TestObserver(OBSERVER_NAME_3); - watchdog.registerHealthObserver(testObserver3); - watchdog.startObservingHealth(testObserver3, List.of(APP_C), LONG_DURATION); + watchdog.registerHealthObserver(testObserver3, mTestExecutor); + watchdog.startExplicitHealthCheck(testObserver3, List.of(APP_C), LONG_DURATION); mTestLooper.dispatchAll(); watchdog.unregisterHealthObserver(testObserver1); @@ -1431,14 +1492,15 @@ public class PackageWatchdogTest { public void testFailureHistoryIsPreserved() { PackageWatchdog watchdog = createWatchdog(); TestObserver observer = new TestObserver(OBSERVER_NAME_1); - watchdog.startObservingHealth(observer, List.of(APP_A), SHORT_DURATION); + watchdog.registerHealthObserver(observer, mTestExecutor); + watchdog.startExplicitHealthCheck(observer, List.of(APP_A), SHORT_DURATION); for (int i = 0; i < PackageWatchdog.DEFAULT_TRIGGER_FAILURE_COUNT - 1; i++) { watchdog.notifyPackageFailure(List.of(new VersionedPackage(APP_A, VERSION_CODE)), PackageWatchdog.FAILURE_REASON_UNKNOWN); } mTestLooper.dispatchAll(); assertThat(observer.mMitigatedPackages).isEmpty(); - watchdog.startObservingHealth(observer, List.of(APP_A), LONG_DURATION); + watchdog.startExplicitHealthCheck(observer, List.of(APP_A), LONG_DURATION); watchdog.notifyPackageFailure(List.of(new VersionedPackage(APP_A, VERSION_CODE)), PackageWatchdog.FAILURE_REASON_UNKNOWN); mTestLooper.dispatchAll(); @@ -1453,7 +1515,8 @@ public class PackageWatchdogTest { public void testMitigationSlidingWindow() { PackageWatchdog watchdog = createWatchdog(); TestObserver observer = new TestObserver(OBSERVER_NAME_1); - watchdog.startObservingHealth(observer, List.of(APP_A), + watchdog.registerHealthObserver(observer, mTestExecutor); + watchdog.startExplicitHealthCheck(observer, List.of(APP_A), PackageWatchdog.DEFAULT_OBSERVING_DURATION_MS * 2); @@ -1895,6 +1958,7 @@ public class PackageWatchdogTest { } public boolean onExecuteBootLoopMitigation(int level) { + Slog.w("hrm1243", "I'm here " + level); mMitigatedBootLoop = true; mBootMitigationCounts.add(level); return true; |