diff options
5 files changed, 224 insertions, 120 deletions
diff --git a/services/core/java/com/android/server/PackageWatchdog.java b/services/core/java/com/android/server/PackageWatchdog.java index e675d8d458c7..f07954609603 100644 --- a/services/core/java/com/android/server/PackageWatchdog.java +++ b/services/core/java/com/android/server/PackageWatchdog.java @@ -68,6 +68,7 @@ import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.NoSuchElementException; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -117,6 +118,9 @@ public class PackageWatchdog { static final int DEFAULT_TRIGGER_FAILURE_COUNT = 5; @VisibleForTesting static final long DEFAULT_OBSERVING_DURATION_MS = TimeUnit.DAYS.toMillis(2); + // Sliding window for tracking how many mitigation calls were made for a package. + @VisibleForTesting + static final long DEFAULT_DEESCALATION_WINDOW_MS = TimeUnit.HOURS.toMillis(1); // Whether explicit health checks are enabled or not private static final boolean DEFAULT_EXPLICIT_HEALTH_CHECK_ENABLED = true; @@ -388,6 +392,7 @@ public class PackageWatchdog { // Observer that will receive failure for versionedPackage PackageHealthObserver currentObserverToNotify = null; int currentObserverImpact = Integer.MAX_VALUE; + MonitoredPackage currentMonitoredPackage = null; // Find observer with least user impact for (int oIndex = 0; oIndex < mAllObservers.size(); oIndex++) { @@ -396,19 +401,33 @@ public class PackageWatchdog { if (registeredObserver != null && observer.onPackageFailureLocked( versionedPackage.getPackageName())) { + MonitoredPackage p = observer.getMonitoredPackage( + versionedPackage.getPackageName()); + int mitigationCount = 1; + if (p != null) { + mitigationCount = p.getMitigationCountLocked() + 1; + } int impact = registeredObserver.onHealthCheckFailed( - versionedPackage, failureReason); + versionedPackage, failureReason, mitigationCount); if (impact != PackageHealthObserverImpact.USER_IMPACT_NONE && impact < currentObserverImpact) { currentObserverToNotify = registeredObserver; currentObserverImpact = impact; + currentMonitoredPackage = p; } } } // Execute action with least user impact if (currentObserverToNotify != null) { - currentObserverToNotify.execute(versionedPackage, failureReason); + int mitigationCount = 1; + if (currentMonitoredPackage != null) { + currentMonitoredPackage.noteMitigationCallLocked(); + mitigationCount = + currentMonitoredPackage.getMitigationCountLocked(); + } + currentObserverToNotify.execute(versionedPackage, + failureReason, mitigationCount); } } } @@ -429,7 +448,7 @@ public class PackageWatchdog { PackageHealthObserver registeredObserver = observer.registeredObserver; if (registeredObserver != null) { int impact = registeredObserver.onHealthCheckFailed( - failingPackage, failureReason); + failingPackage, failureReason, 1); if (impact != PackageHealthObserverImpact.USER_IMPACT_NONE && impact < currentObserverImpact) { currentObserverToNotify = registeredObserver; @@ -438,7 +457,7 @@ public class PackageWatchdog { } } if (currentObserverToNotify != null) { - currentObserverToNotify.execute(failingPackage, failureReason); + currentObserverToNotify.execute(failingPackage, failureReason, 1); } } @@ -559,6 +578,8 @@ public class PackageWatchdog { * @param versionedPackage the package that is failing. This may be null if a native * service is crashing. * @param failureReason the type of failure that is occurring. + * @param mitigationCount the number of times mitigation has been called for this package + * (including this time). * * * @return any one of {@link PackageHealthObserverImpact} to express the impact @@ -566,7 +587,8 @@ public class PackageWatchdog { */ @PackageHealthObserverImpact int onHealthCheckFailed( @Nullable VersionedPackage versionedPackage, - @FailureReasons int failureReason); + @FailureReasons int failureReason, + int mitigationCount); /** * Executes mitigation for {@link #onHealthCheckFailed}. @@ -574,10 +596,12 @@ public class PackageWatchdog { * @param versionedPackage the package that is failing. This may be null if a native * service is crashing. * @param failureReason the type of failure that is occurring. + * @param mitigationCount the number of times mitigation has been called for this package + * (including this time). * @return {@code true} if action was executed successfully, {@code false} otherwise */ boolean execute(@Nullable VersionedPackage versionedPackage, - @FailureReasons int failureReason); + @FailureReasons int failureReason, int mitigationCount); /** @@ -684,7 +708,7 @@ public class PackageWatchdog { synchronized (mLock) { for (int observerIdx = 0; observerIdx < mAllObservers.size(); observerIdx++) { ObserverInternal observer = mAllObservers.valueAt(observerIdx); - MonitoredPackage monitoredPackage = observer.packages.get(packageName); + MonitoredPackage monitoredPackage = observer.getMonitoredPackage(packageName); if (monitoredPackage != null) { int oldState = monitoredPackage.getHealthCheckStateLocked(); @@ -713,7 +737,8 @@ public class PackageWatchdog { Slog.d(TAG, "Received supported packages " + supportedPackages); Iterator<ObserverInternal> oit = mAllObservers.values().iterator(); while (oit.hasNext()) { - Iterator<MonitoredPackage> pit = oit.next().packages.values().iterator(); + Iterator<MonitoredPackage> pit = oit.next().getMonitoredPackages() + .values().iterator(); while (pit.hasNext()) { MonitoredPackage monitoredPackage = pit.next(); String packageName = monitoredPackage.getName(); @@ -746,7 +771,7 @@ public class PackageWatchdog { while (oit.hasNext()) { ObserverInternal observer = oit.next(); Iterator<MonitoredPackage> pit = - observer.packages.values().iterator(); + observer.getMonitoredPackages().values().iterator(); while (pit.hasNext()) { MonitoredPackage monitoredPackage = pit.next(); String packageName = monitoredPackage.getName(); @@ -804,7 +829,8 @@ public class PackageWatchdog { private long getNextStateSyncMillisLocked() { long shortestDurationMs = Long.MAX_VALUE; for (int oIndex = 0; oIndex < mAllObservers.size(); oIndex++) { - ArrayMap<String, MonitoredPackage> packages = mAllObservers.valueAt(oIndex).packages; + ArrayMap<String, MonitoredPackage> packages = mAllObservers.valueAt(oIndex) + .getMonitoredPackages(); for (int pIndex = 0; pIndex < packages.size(); pIndex++) { MonitoredPackage mp = packages.valueAt(pIndex); long duration = mp.getShortestScheduleDurationMsLocked(); @@ -838,7 +864,7 @@ public class PackageWatchdog { if (!failedPackages.isEmpty()) { onHealthCheckFailed(observer, failedPackages); } - if (observer.packages.isEmpty() && (observer.registeredObserver == null + if (observer.getMonitoredPackages().isEmpty() && (observer.registeredObserver == null || !observer.registeredObserver.isPersistent())) { Slog.i(TAG, "Discarding observer " + observer.name + ". All packages expired"); it.remove(); @@ -857,7 +883,7 @@ public class PackageWatchdog { VersionedPackage versionedPkg = it.next().mPackage; Slog.i(TAG, "Explicit health check failed for package " + versionedPkg); registeredObserver.execute(versionedPkg, - PackageWatchdog.FAILURE_REASON_EXPLICIT_HEALTH_CHECK); + PackageWatchdog.FAILURE_REASON_EXPLICIT_HEALTH_CHECK, 1); } } } @@ -1054,7 +1080,7 @@ public class PackageWatchdog { private static class ObserverInternal { public final String name; @GuardedBy("mLock") - public final ArrayMap<String, MonitoredPackage> packages = new ArrayMap<>(); + private final ArrayMap<String, MonitoredPackage> mPackages = new ArrayMap<>(); @Nullable @GuardedBy("mLock") public PackageHealthObserver registeredObserver; @@ -1073,8 +1099,8 @@ public class PackageWatchdog { try { out.startTag(null, TAG_OBSERVER); out.attribute(null, ATTR_NAME, name); - for (int i = 0; i < packages.size(); i++) { - MonitoredPackage p = packages.valueAt(i); + for (int i = 0; i < mPackages.size(); i++) { + MonitoredPackage p = mPackages.valueAt(i); p.writeLocked(out); } out.endTag(null, TAG_OBSERVER); @@ -1089,11 +1115,11 @@ public class PackageWatchdog { public void updatePackagesLocked(List<MonitoredPackage> packages) { for (int pIndex = 0; pIndex < packages.size(); pIndex++) { MonitoredPackage p = packages.get(pIndex); - MonitoredPackage existingPackage = this.packages.get(p.getName()); + MonitoredPackage existingPackage = getMonitoredPackage(p.getName()); if (existingPackage != null) { existingPackage.updateHealthCheckDuration(p.mDurationMs); } else { - this.packages.put(p.getName(), p); + putMonitoredPackage(p); } } } @@ -1111,7 +1137,7 @@ public class PackageWatchdog { @GuardedBy("mLock") private Set<MonitoredPackage> prunePackagesLocked(long elapsedMs) { Set<MonitoredPackage> failedPackages = new ArraySet<>(); - Iterator<MonitoredPackage> it = packages.values().iterator(); + Iterator<MonitoredPackage> it = mPackages.values().iterator(); while (it.hasNext()) { MonitoredPackage p = it.next(); int oldState = p.getHealthCheckStateLocked(); @@ -1134,12 +1160,12 @@ public class PackageWatchdog { */ @GuardedBy("mLock") public boolean onPackageFailureLocked(String packageName) { - if (packages.get(packageName) == null && registeredObserver.isPersistent() + if (getMonitoredPackage(packageName) == null && registeredObserver.isPersistent() && registeredObserver.mayObservePackage(packageName)) { - packages.put(packageName, sPackageWatchdog.newMonitoredPackage( + putMonitoredPackage(sPackageWatchdog.newMonitoredPackage( packageName, DEFAULT_OBSERVING_DURATION_MS, false)); } - MonitoredPackage p = packages.get(packageName); + MonitoredPackage p = getMonitoredPackage(packageName); if (p != null) { return p.onFailureLocked(); } @@ -1147,6 +1173,40 @@ public class PackageWatchdog { } /** + * Returns the map of packages monitored by this observer. + * + * @return a mapping of package names to {@link MonitoredPackage} objects. + */ + @GuardedBy("mLock") + public ArrayMap<String, MonitoredPackage> getMonitoredPackages() { + return mPackages; + } + + /** + * Returns the {@link MonitoredPackage} associated with a given package name if the + * package is being monitored by this observer. + * + * @param packageName: the name of the package. + * @return the {@link MonitoredPackage} object associated with the package name if one + * exists, {@code null} otherwise. + */ + @GuardedBy("mLock") + @Nullable + public MonitoredPackage getMonitoredPackage(String packageName) { + return mPackages.get(packageName); + } + + /** + * Associates a {@link MonitoredPackage} with the observer. + * + * @param p: the {@link MonitoredPackage} to store. + */ + @GuardedBy("mLock") + public void putMonitoredPackage(MonitoredPackage p) { + mPackages.put(p.getName(), p); + } + + /** * Returns one ObserverInternal from the {@code parser} and advances its state. * * <p>Note that this method is <b>not</b> thread safe. It should only be called from @@ -1201,8 +1261,8 @@ public class PackageWatchdog { public void dump(IndentingPrintWriter pw) { boolean isPersistent = registeredObserver != null && registeredObserver.isPersistent(); pw.println("Persistent: " + isPersistent); - for (String packageName : packages.keySet()) { - MonitoredPackage p = packages.get(packageName); + for (String packageName : mPackages.keySet()) { + MonitoredPackage p = getMonitoredPackage(packageName); pw.println(packageName + ": "); pw.increaseIndent(); pw.println("# Failures: " + p.mFailureHistory.size()); @@ -1257,6 +1317,10 @@ public class PackageWatchdog { // Times when package failures happen sorted in ascending order @GuardedBy("mLock") private final LongArrayQueue mFailureHistory = new LongArrayQueue(); + // Times when an observer was called to mitigate this package's failure. Sorted in + // ascending order. + @GuardedBy("mLock") + private final LongArrayQueue mMitigationCalls = new LongArrayQueue(); // One of STATE_[ACTIVE|INACTIVE|PASSED|FAILED]. Updated on construction and after // methods that could change the health check state: handleElapsedTimeLocked and // tryPassHealthCheckLocked @@ -1322,6 +1386,33 @@ public class PackageWatchdog { } /** + * Notes the timestamp of a mitigation call into the observer. + */ + @GuardedBy("mLock") + public void noteMitigationCallLocked() { + mMitigationCalls.addLast(mSystemClock.uptimeMillis()); + } + + /** + * Prunes any mitigation calls outside of the de-escalation window, and returns the + * number of calls that are in the window afterwards. + * + * @return the number of mitigation calls made in the de-escalation window. + */ + @GuardedBy("mLock") + public int getMitigationCountLocked() { + try { + final long now = mSystemClock.uptimeMillis(); + while (now - mMitigationCalls.peekFirst() > DEFAULT_DEESCALATION_WINDOW_MS) { + mMitigationCalls.removeFirst(); + } + } catch (NoSuchElementException ignore) { + } + + return mMitigationCalls.size(); + } + + /** * Sets the initial health check duration. * * @return the new health check state diff --git a/services/core/java/com/android/server/RescueParty.java b/services/core/java/com/android/server/RescueParty.java index ef6dab5a45fb..6206f7a583d1 100644 --- a/services/core/java/com/android/server/RescueParty.java +++ b/services/core/java/com/android/server/RescueParty.java @@ -77,6 +77,7 @@ public class RescueParty { static final String PROP_ENABLE_RESCUE = "persist.sys.enable_rescue"; @VisibleForTesting static final String PROP_RESCUE_LEVEL = "sys.rescue_level"; + static final String PROP_ATTEMPTING_FACTORY_RESET = "sys.attempting_factory_reset"; @VisibleForTesting static final int LEVEL_NONE = 0; @VisibleForTesting @@ -155,7 +156,7 @@ public class RescueParty { * Check if we're currently attempting to reboot for a factory reset. */ public static boolean isAttemptingFactoryReset() { - return SystemProperties.getInt(PROP_RESCUE_LEVEL, LEVEL_NONE) == LEVEL_FACTORY_RESET; + return SystemProperties.getBoolean(PROP_ATTEMPTING_FACTORY_RESET, false); } /** @@ -230,14 +231,38 @@ public class RescueParty { } } + private static int getMaxRescueLevel() { + return SystemProperties.getBoolean(PROP_DISABLE_FACTORY_RESET_FLAG, false) + ? LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS : LEVEL_FACTORY_RESET; + } + + /** + * Get the rescue level to perform if this is the n-th attempt at mitigating failure. + * + * @param mitigationCount: the mitigation attempt number (1 = first attempt etc.) + * @return the rescue level for the n-th mitigation attempt. + */ + private static int getRescueLevel(int mitigationCount) { + if (mitigationCount == 1) { + return LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS; + } else if (mitigationCount == 2) { + return LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES; + } else if (mitigationCount == 3) { + return LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS; + } else if (mitigationCount >= 4) { + return getMaxRescueLevel(); + } else { + Slog.w(TAG, "Expected positive mitigation count, was " + mitigationCount); + return LEVEL_NONE; + } + } + /** * Get the next rescue level. This indicates the next level of mitigation that may be taken. */ private static int getNextRescueLevel() { - int maxRescueLevel = SystemProperties.getBoolean(PROP_DISABLE_FACTORY_RESET_FLAG, false) - ? LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS : LEVEL_FACTORY_RESET; return MathUtils.constrain(SystemProperties.getInt(PROP_RESCUE_LEVEL, LEVEL_NONE) + 1, - LEVEL_NONE, maxRescueLevel); + LEVEL_NONE, getMaxRescueLevel()); } /** @@ -256,7 +281,11 @@ public class RescueParty { private static void executeRescueLevel(Context context, @Nullable String failedPackage) { final int level = SystemProperties.getInt(PROP_RESCUE_LEVEL, LEVEL_NONE); if (level == LEVEL_NONE) return; + executeRescueLevel(context, failedPackage, level); + } + private static void executeRescueLevel(Context context, @Nullable String failedPackage, + int level) { Slog.w(TAG, "Attempting rescue level " + levelToString(level)); try { executeRescueLevelInternal(context, level, failedPackage); @@ -284,6 +313,7 @@ public class RescueParty { case LEVEL_FACTORY_RESET: // Request the reboot from a separate thread to avoid deadlock on PackageWatchdog // when device shutting down. + SystemProperties.set(PROP_ATTEMPTING_FACTORY_RESET, "true"); Runnable runnable = new Runnable() { @Override public void run() { @@ -320,15 +350,6 @@ public class RescueParty { } } - private static int getPackageUid(Context context, String packageName) { - try { - return context.getPackageManager().getPackageUid(packageName, 0); - } catch (PackageManager.NameNotFoundException e) { - // Since UIDs are always >= 0, this value means the UID could not be determined. - return -1; - } - } - private static void resetAllSettings(Context context, int mode, @Nullable String failedPackage) throws Exception { // Try our best to reset all settings possible, and once finished @@ -359,7 +380,7 @@ public class RescueParty { private static void resetDeviceConfig(Context context, int resetMode, @Nullable String failedPackage) { - if (!shouldPerformScopedResets() || failedPackage == null) { + if (!shouldPerformScopedResets(resetMode) || failedPackage == null) { resetAllAffectedNamespaces(context, resetMode); } else { performScopedReset(context, resetMode, failedPackage); @@ -384,11 +405,8 @@ public class RescueParty { } } - private static boolean shouldPerformScopedResets() { - int rescueLevel = MathUtils.constrain( - SystemProperties.getInt(PROP_RESCUE_LEVEL, LEVEL_NONE), - LEVEL_NONE, LEVEL_FACTORY_RESET); - return rescueLevel <= LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES; + private static boolean shouldPerformScopedResets(int resetMode) { + return resetMode <= Settings.RESET_MODE_UNTRUSTED_CHANGES; } private static void performScopedReset(Context context, int resetMode, @@ -452,10 +470,10 @@ public class RescueParty { @Override public int onHealthCheckFailed(@Nullable VersionedPackage failedPackage, - @FailureReasons int failureReason) { + @FailureReasons int failureReason, int mitigationCount) { if (!isDisabled() && (failureReason == PackageWatchdog.FAILURE_REASON_APP_CRASH || failureReason == PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING)) { - return mapRescueLevelToUserImpact(getNextRescueLevel()); + return mapRescueLevelToUserImpact(getRescueLevel(mitigationCount)); } else { return PackageHealthObserverImpact.USER_IMPACT_NONE; } @@ -463,16 +481,15 @@ public class RescueParty { @Override public boolean execute(@Nullable VersionedPackage failedPackage, - @FailureReasons int failureReason) { + @FailureReasons int failureReason, int mitigationCount) { if (isDisabled()) { return false; } if (failureReason == PackageWatchdog.FAILURE_REASON_APP_CRASH || failureReason == PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING) { - int triggerUid = getPackageUid(mContext, failedPackage.getPackageName()); - incrementRescueLevel(triggerUid); + final int level = getRescueLevel(mitigationCount); executeRescueLevel(mContext, - failedPackage == null ? null : failedPackage.getPackageName()); + failedPackage == null ? null : failedPackage.getPackageName(), level); return true; } else { return false; diff --git a/services/core/java/com/android/server/rollback/RollbackPackageHealthObserver.java b/services/core/java/com/android/server/rollback/RollbackPackageHealthObserver.java index ba1401d7469e..1295b7008e67 100644 --- a/services/core/java/com/android/server/rollback/RollbackPackageHealthObserver.java +++ b/services/core/java/com/android/server/rollback/RollbackPackageHealthObserver.java @@ -92,7 +92,7 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver { @Override public int onHealthCheckFailed(@Nullable VersionedPackage failedPackage, - @FailureReasons int failureReason) { + @FailureReasons int failureReason, int mitigationCount) { // For native crashes, we will roll back any available rollbacks if (failureReason == PackageWatchdog.FAILURE_REASON_NATIVE_CRASH && !mContext.getSystemService(RollbackManager.class) @@ -110,7 +110,7 @@ final class RollbackPackageHealthObserver implements PackageHealthObserver { @Override public boolean execute(@Nullable VersionedPackage failedPackage, - @FailureReasons int rollbackReason) { + @FailureReasons int rollbackReason, int mitigationCount) { if (rollbackReason == PackageWatchdog.FAILURE_REASON_NATIVE_CRASH) { mHandler.post(() -> rollbackAll()); return true; diff --git a/services/tests/mockingservicestests/src/com/android/server/RescuePartyTest.java b/services/tests/mockingservicestests/src/com/android/server/RescuePartyTest.java index 2c92ae44d63a..da4071b6b9de 100644 --- a/services/tests/mockingservicestests/src/com/android/server/RescuePartyTest.java +++ b/services/tests/mockingservicestests/src/com/android/server/RescuePartyTest.java @@ -226,28 +226,20 @@ public class RescuePartyTest { @Test public void testPersistentAppCrashDetectionWithExecutionForAllRescueLevels() { - notePersistentAppCrash(); + notePersistentAppCrash(1); verifySettingsResets(Settings.RESET_MODE_UNTRUSTED_DEFAULTS, /*resetNamespaces=*/ null); - assertEquals(RescueParty.LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS, - SystemProperties.getInt(RescueParty.PROP_RESCUE_LEVEL, RescueParty.LEVEL_NONE)); - notePersistentAppCrash(); + notePersistentAppCrash(2); verifySettingsResets(Settings.RESET_MODE_UNTRUSTED_CHANGES, /*resetNamespaces=*/ null); - assertEquals(RescueParty.LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES, - SystemProperties.getInt(RescueParty.PROP_RESCUE_LEVEL, RescueParty.LEVEL_NONE)); - notePersistentAppCrash(); + notePersistentAppCrash(3); verifySettingsResets(Settings.RESET_MODE_TRUSTED_DEFAULTS, /*resetNamespaces=*/ null); - assertEquals(RescueParty.LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS, - SystemProperties.getInt(RescueParty.PROP_RESCUE_LEVEL, RescueParty.LEVEL_NONE)); - notePersistentAppCrash(); - - assertEquals(LEVEL_FACTORY_RESET, - SystemProperties.getInt(RescueParty.PROP_RESCUE_LEVEL, RescueParty.LEVEL_NONE)); + notePersistentAppCrash(4); + assertTrue(RescueParty.isAttemptingFactoryReset()); } @Test @@ -281,25 +273,19 @@ public class RescuePartyTest { final String[] expectedAllResetNamespaces = new String[]{NAMESPACE1, NAMESPACE2, NAMESPACE3}; observer.execute(new VersionedPackage( - CALLING_PACKAGE1, 1), PackageWatchdog.FAILURE_REASON_APP_CRASH); + CALLING_PACKAGE1, 1), PackageWatchdog.FAILURE_REASON_APP_CRASH, 1); verifySettingsResets(Settings.RESET_MODE_UNTRUSTED_DEFAULTS, expectedResetNamespaces); - assertEquals(RescueParty.LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS, - SystemProperties.getInt(RescueParty.PROP_RESCUE_LEVEL, RescueParty.LEVEL_NONE)); observer.execute(new VersionedPackage( - CALLING_PACKAGE1, 1), PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING); + CALLING_PACKAGE1, 1), PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 2); verifySettingsResets(Settings.RESET_MODE_UNTRUSTED_CHANGES, expectedResetNamespaces); - assertEquals(RescueParty.LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES, - SystemProperties.getInt(RescueParty.PROP_RESCUE_LEVEL, RescueParty.LEVEL_NONE)); observer.execute(new VersionedPackage( - CALLING_PACKAGE1, 1), PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING); + CALLING_PACKAGE1, 1), PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 3); verifySettingsResets(Settings.RESET_MODE_TRUSTED_DEFAULTS, expectedAllResetNamespaces); - assertEquals(RescueParty.LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS, - SystemProperties.getInt(RescueParty.PROP_RESCUE_LEVEL, RescueParty.LEVEL_NONE)); observer.execute(new VersionedPackage( - CALLING_PACKAGE1, 1), PackageWatchdog.FAILURE_REASON_APP_CRASH); + CALLING_PACKAGE1, 1), PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 4); assertTrue(RescueParty.isAttemptingFactoryReset()); } @@ -341,11 +327,11 @@ public class RescuePartyTest { SystemProperties.set(RescueParty.PROP_ENABLE_RESCUE, Boolean.toString(false)); SystemProperties.set(PROP_DISABLE_RESCUE, Boolean.toString(true)); assertEquals(RescuePartyObserver.getInstance(mMockContext).execute(sFailingPackage, - PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING), false); + PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 1), false); SystemProperties.set(RescueParty.PROP_ENABLE_RESCUE, Boolean.toString(true)); assertTrue(RescuePartyObserver.getInstance(mMockContext).execute(sFailingPackage, - PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING)); + PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 1)); } @Test @@ -354,7 +340,7 @@ public class RescuePartyTest { SystemProperties.set(PROP_DEVICE_CONFIG_DISABLE_FLAG, Boolean.toString(true)); assertEquals(RescuePartyObserver.getInstance(mMockContext).execute(sFailingPackage, - PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING), false); + PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 1), false); // Restore the property value initialized in SetUp() SystemProperties.set(RescueParty.PROP_ENABLE_RESCUE, Boolean.toString(true)); @@ -379,46 +365,24 @@ public class RescuePartyTest { RescuePartyObserver observer = RescuePartyObserver.getInstance(mMockContext); // Ensure that no action is taken for cases where the failure reason is unknown - SystemProperties.set(RescueParty.PROP_RESCUE_LEVEL, Integer.toString( - LEVEL_FACTORY_RESET)); - assertEquals(observer.onHealthCheckFailed(null, PackageWatchdog.FAILURE_REASON_UNKNOWN), + assertEquals(observer.onHealthCheckFailed(null, PackageWatchdog.FAILURE_REASON_UNKNOWN, 1), PackageHealthObserverImpact.USER_IMPACT_NONE); - /* - For the following cases, ensure that the returned user impact corresponds with the user - impact of the next available rescue level, not the current one. - */ - SystemProperties.set(RescueParty.PROP_RESCUE_LEVEL, Integer.toString( - RescueParty.LEVEL_NONE)); + // Ensure the correct user impact is returned for each mitigation count. assertEquals(observer.onHealthCheckFailed(null, - PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING), + PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 1), PackageHealthObserverImpact.USER_IMPACT_LOW); - SystemProperties.set(RescueParty.PROP_RESCUE_LEVEL, Integer.toString( - RescueParty.LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS)); assertEquals(observer.onHealthCheckFailed(null, - PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING), + PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 2), PackageHealthObserverImpact.USER_IMPACT_LOW); - - SystemProperties.set(RescueParty.PROP_RESCUE_LEVEL, Integer.toString( - RescueParty.LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES)); assertEquals(observer.onHealthCheckFailed(null, - PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING), + PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 3), PackageHealthObserverImpact.USER_IMPACT_HIGH); - - SystemProperties.set(RescueParty.PROP_RESCUE_LEVEL, Integer.toString( - RescueParty.LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS)); assertEquals(observer.onHealthCheckFailed(null, - PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING), - PackageHealthObserverImpact.USER_IMPACT_HIGH); - - - SystemProperties.set(RescueParty.PROP_RESCUE_LEVEL, Integer.toString( - LEVEL_FACTORY_RESET)); - assertEquals(observer.onHealthCheckFailed(null, - PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING), + PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING, 4), PackageHealthObserverImpact.USER_IMPACT_HIGH); } @@ -451,17 +415,6 @@ public class RescuePartyTest { assertEquals(observer.onBootLoop(), PackageHealthObserverImpact.USER_IMPACT_HIGH); } - @Test - public void testRescueLevelIncrementsWhenExecuted() { - RescuePartyObserver observer = RescuePartyObserver.getInstance(mMockContext); - SystemProperties.set(RescueParty.PROP_RESCUE_LEVEL, Integer.toString( - RescueParty.LEVEL_NONE)); - observer.execute(sFailingPackage, - PackageWatchdog.FAILURE_REASON_APP_CRASH); - assertEquals(SystemProperties.getInt(RescueParty.PROP_RESCUE_LEVEL, -1), - RescueParty.LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS); - } - private void verifySettingsResets(int resetMode, String[] resetNamespaces) { verify(() -> Settings.Global.resetToDefaultsAsUser(mMockContentResolver, null, resetMode, UserHandle.USER_SYSTEM)); @@ -481,9 +434,9 @@ public class RescuePartyTest { RescuePartyObserver.getInstance(mMockContext).executeBootLoopMitigation(); } - private void notePersistentAppCrash() { + private void notePersistentAppCrash(int mitigationCount) { RescuePartyObserver.getInstance(mMockContext).execute(new VersionedPackage( - "com.package.name", 1), PackageWatchdog.FAILURE_REASON_APP_CRASH); + "com.package.name", 1), PackageWatchdog.FAILURE_REASON_APP_CRASH, mitigationCount); } private Bundle getConfigAccessBundle(String callingPackage, String namespace) { diff --git a/tests/PackageWatchdog/src/com/android/server/PackageWatchdogTest.java b/tests/PackageWatchdog/src/com/android/server/PackageWatchdogTest.java index ae93a81f274e..fa0574a503f1 100644 --- a/tests/PackageWatchdog/src/com/android/server/PackageWatchdogTest.java +++ b/tests/PackageWatchdog/src/com/android/server/PackageWatchdogTest.java @@ -376,7 +376,7 @@ public class PackageWatchdogTest { TestObserver observer = new TestObserver(OBSERVER_NAME_1) { @Override public int onHealthCheckFailed(VersionedPackage versionedPackage, - int failureReason) { + int failureReason, int mitigationCount) { if (versionedPackage.getVersionCode() == VERSION_CODE) { // Only rollback for specific versionCode return PackageHealthObserverImpact.USER_IMPACT_MEDIUM; @@ -1146,6 +1146,45 @@ public class PackageWatchdogTest { assertThat(observer.mMitigatedPackages).isEqualTo(List.of(APP_A)); } + /** + * Ensure that the sliding window logic results in the correct mitigation count being sent to + * an observer. + */ + @Test + public void testMitigationSlidingWindow() { + PackageWatchdog watchdog = createWatchdog(); + TestObserver observer = new TestObserver(OBSERVER_NAME_1); + watchdog.startObservingHealth(observer, List.of(APP_A), + PackageWatchdog.DEFAULT_OBSERVING_DURATION_MS * 2); + + + raiseFatalFailureAndDispatch(watchdog, Arrays.asList(new VersionedPackage(APP_A, + VERSION_CODE)), PackageWatchdog.FAILURE_REASON_UNKNOWN); + + moveTimeForwardAndDispatch(TimeUnit.MINUTES.toMillis(10)); + + raiseFatalFailureAndDispatch(watchdog, Arrays.asList(new VersionedPackage(APP_A, + VERSION_CODE)), PackageWatchdog.FAILURE_REASON_UNKNOWN); + raiseFatalFailureAndDispatch(watchdog, Arrays.asList(new VersionedPackage(APP_A, + VERSION_CODE)), PackageWatchdog.FAILURE_REASON_UNKNOWN); + + moveTimeForwardAndDispatch(PackageWatchdog.DEFAULT_DEESCALATION_WINDOW_MS); + + // The first failure will be outside the threshold. + raiseFatalFailureAndDispatch(watchdog, Arrays.asList(new VersionedPackage(APP_A, + VERSION_CODE)), PackageWatchdog.FAILURE_REASON_UNKNOWN); + + moveTimeForwardAndDispatch(TimeUnit.MINUTES.toMillis(20)); + + // The next 2 failures will also be outside the threshold. + raiseFatalFailureAndDispatch(watchdog, Arrays.asList(new VersionedPackage(APP_A, + VERSION_CODE)), PackageWatchdog.FAILURE_REASON_UNKNOWN); + raiseFatalFailureAndDispatch(watchdog, Arrays.asList(new VersionedPackage(APP_A, + VERSION_CODE)), PackageWatchdog.FAILURE_REASON_UNKNOWN); + + assertThat(observer.mMitigationCounts).isEqualTo(List.of(1, 2, 3, 3, 2, 3)); + } + private void adoptShellPermissions(String... permissions) { InstrumentationRegistry .getInstrumentation() @@ -1227,6 +1266,7 @@ public class PackageWatchdogTest { private boolean mMitigatedBootLoop = false; final List<String> mHealthCheckFailedPackages = new ArrayList<>(); final List<String> mMitigatedPackages = new ArrayList<>(); + final List<Integer> mMitigationCounts = new ArrayList<>(); TestObserver(String name) { mName = name; @@ -1238,13 +1278,16 @@ public class PackageWatchdogTest { mImpact = impact; } - public int onHealthCheckFailed(VersionedPackage versionedPackage, int failureReason) { + public int onHealthCheckFailed(VersionedPackage versionedPackage, int failureReason, + int mitigationCount) { mHealthCheckFailedPackages.add(versionedPackage.getPackageName()); return mImpact; } - public boolean execute(VersionedPackage versionedPackage, int failureReason) { + public boolean execute(VersionedPackage versionedPackage, int failureReason, + int mitigationCount) { mMitigatedPackages.add(versionedPackage.getPackageName()); + mMitigationCounts.add(mitigationCount); mLastFailureReason = failureReason; return true; } |