Change backoff policy for Abandoned Job failures

Jobs that are abandoned due to a certain number of
STOP_REASON_TIMEOUT_ABANDONED stop reasons will be rescheduled with an
exponential backoff policy. This is because these jobs are likely to be
empty and not perform any related work until the system times out the
job.

bug: 372529068
Test: atest CtsJobSchedulerTestCases
Test: atest FrameworksMockingServicesTests
Flag: android.app.job.handle_abandoned_jobs
Change-Id: I2f5e7e3ba3d9a3b2c229ff322eeaaa62c090947b

3 files changed, 81 insertions, 9 deletions

diff --git a/apex/jobscheduler/service/java/com/android/server/job/JobSchedulerService.java b/apex/jobscheduler/service/java/com/android/server/job/JobSchedulerService.java
index f569388ef3c1..197de37f8ac4 100644
--- a/apex/jobscheduler/service/java/com/android/server/job/JobSchedulerService.java
+++ b/apex/jobscheduler/service/java/com/android/server/job/JobSchedulerService.java
@@ -572,6 +572,7 @@ public class JobSchedulerService extends com.android.server.SystemService
                         case Constants.KEY_MIN_LINEAR_BACKOFF_TIME_MS:
                         case Constants.KEY_MIN_EXP_BACKOFF_TIME_MS:
                         case Constants.KEY_SYSTEM_STOP_TO_FAILURE_RATIO:
+                        case Constants.KEY_ABANDONED_JOB_TIMEOUTS_BEFORE_AGGRESSIVE_BACKOFF:
                             mConstants.updateBackoffConstantsLocked();
                             break;
                         case Constants.KEY_CONN_CONGESTION_DELAY_FRAC:
@@ -678,6 +679,8 @@ public class JobSchedulerService extends com.android.server.SystemService
         private static final String KEY_MIN_EXP_BACKOFF_TIME_MS = "min_exp_backoff_time_ms";
         private static final String KEY_SYSTEM_STOP_TO_FAILURE_RATIO =
                 "system_stop_to_failure_ratio";
+        private static final String KEY_ABANDONED_JOB_TIMEOUTS_BEFORE_AGGRESSIVE_BACKOFF =
+                "abandoned_job_timeouts_before_aggressive_backoff";
         private static final String KEY_CONN_CONGESTION_DELAY_FRAC = "conn_congestion_delay_frac";
         private static final String KEY_CONN_PREFETCH_RELAX_FRAC = "conn_prefetch_relax_frac";
         private static final String KEY_CONN_USE_CELL_SIGNAL_STRENGTH =
@@ -749,6 +752,7 @@ public class JobSchedulerService extends com.android.server.SystemService
         private static final long DEFAULT_MIN_LINEAR_BACKOFF_TIME_MS = JobInfo.MIN_BACKOFF_MILLIS;
         private static final long DEFAULT_MIN_EXP_BACKOFF_TIME_MS = JobInfo.MIN_BACKOFF_MILLIS;
         private static final int DEFAULT_SYSTEM_STOP_TO_FAILURE_RATIO = 3;
+        private static final int DEFAULT_ABANDONED_JOB_TIMEOUTS_BEFORE_AGGRESSIVE_BACKOFF = 3;
         private static final float DEFAULT_CONN_CONGESTION_DELAY_FRAC = 0.5f;
         private static final float DEFAULT_CONN_PREFETCH_RELAX_FRAC = 0.5f;
         private static final boolean DEFAULT_CONN_USE_CELL_SIGNAL_STRENGTH = true;
@@ -844,7 +848,12 @@ public class JobSchedulerService extends com.android.server.SystemService
          * incremental failure in the backoff policy calculation.
          */
         int SYSTEM_STOP_TO_FAILURE_RATIO = DEFAULT_SYSTEM_STOP_TO_FAILURE_RATIO;
-
+        /**
+         * Number of consecutive timeouts by abandoned jobs before we change to aggressive backoff
+         * policy.
+         */
+        int ABANDONED_JOB_TIMEOUTS_BEFORE_AGGRESSIVE_BACKOFF =
+                DEFAULT_ABANDONED_JOB_TIMEOUTS_BEFORE_AGGRESSIVE_BACKOFF;
         /**
          * The fraction of a job's running window that must pass before we
          * consider running it when the network is congested.
@@ -1077,6 +1086,10 @@ public class JobSchedulerService extends com.android.server.SystemService
             SYSTEM_STOP_TO_FAILURE_RATIO = DeviceConfig.getInt(DeviceConfig.NAMESPACE_JOB_SCHEDULER,
                     KEY_SYSTEM_STOP_TO_FAILURE_RATIO,
                     DEFAULT_SYSTEM_STOP_TO_FAILURE_RATIO);
+            ABANDONED_JOB_TIMEOUTS_BEFORE_AGGRESSIVE_BACKOFF = DeviceConfig.getInt(
+                    DeviceConfig.NAMESPACE_JOB_SCHEDULER,
+                    KEY_ABANDONED_JOB_TIMEOUTS_BEFORE_AGGRESSIVE_BACKOFF,
+                    DEFAULT_ABANDONED_JOB_TIMEOUTS_BEFORE_AGGRESSIVE_BACKOFF);
         }
 
         // TODO(141645789): move into ConnectivityController.CcConfig
@@ -1286,6 +1299,8 @@ public class JobSchedulerService extends com.android.server.SystemService
             pw.print(KEY_MIN_LINEAR_BACKOFF_TIME_MS, MIN_LINEAR_BACKOFF_TIME_MS).println();
             pw.print(KEY_MIN_EXP_BACKOFF_TIME_MS, MIN_EXP_BACKOFF_TIME_MS).println();
             pw.print(KEY_SYSTEM_STOP_TO_FAILURE_RATIO, SYSTEM_STOP_TO_FAILURE_RATIO).println();
+            pw.print(KEY_ABANDONED_JOB_TIMEOUTS_BEFORE_AGGRESSIVE_BACKOFF,
+                    ABANDONED_JOB_TIMEOUTS_BEFORE_AGGRESSIVE_BACKOFF).println();
             pw.print(KEY_CONN_CONGESTION_DELAY_FRAC, CONN_CONGESTION_DELAY_FRAC).println();
             pw.print(KEY_CONN_PREFETCH_RELAX_FRAC, CONN_PREFETCH_RELAX_FRAC).println();
             pw.print(KEY_CONN_USE_CELL_SIGNAL_STRENGTH, CONN_USE_CELL_SIGNAL_STRENGTH).println();
@@ -2990,6 +3005,7 @@ public class JobSchedulerService extends com.android.server.SystemService
 
         final long initialBackoffMillis = job.getInitialBackoffMillis();
         int numFailures = failureToReschedule.getNumFailures();
+        int numAbandonedFailures = failureToReschedule.getNumAbandonedFailures();
         int numSystemStops = failureToReschedule.getNumSystemStops();
         // We should back off slowly if JobScheduler keeps stopping the job,
         // but back off immediately if the issue appeared to be the app's fault
@@ -2999,9 +3015,19 @@ public class JobSchedulerService extends com.android.server.SystemService
                 || internalStopReason == JobParameters.INTERNAL_STOP_REASON_ANR
                 || stopReason == JobParameters.STOP_REASON_USER) {
             numFailures++;
+        } else if (android.app.job.Flags.handleAbandonedJobs()
+                && internalStopReason == JobParameters.INTERNAL_STOP_REASON_TIMEOUT_ABANDONED) {
+            numAbandonedFailures++;
+            numFailures++;
         } else {
             numSystemStops++;
         }
+
+        int backoffPolicy = job.getBackoffPolicy();
+        if (shouldUseAggressiveBackoff(numAbandonedFailures)) {
+            backoffPolicy = JobInfo.BACKOFF_POLICY_EXPONENTIAL;
+        }
+
         final int backoffAttempts =
                 numFailures + numSystemStops / mConstants.SYSTEM_STOP_TO_FAILURE_RATIO;
         final long earliestRuntimeMs;
@@ -3010,7 +3036,7 @@ public class JobSchedulerService extends com.android.server.SystemService
             earliestRuntimeMs = JobStatus.NO_EARLIEST_RUNTIME;
         } else {
             long delayMillis;
-            switch (job.getBackoffPolicy()) {
+            switch (backoffPolicy) {
                 case JobInfo.BACKOFF_POLICY_LINEAR: {
                     long backoff = initialBackoffMillis;
                     if (backoff < mConstants.MIN_LINEAR_BACKOFF_TIME_MS) {
@@ -3039,7 +3065,7 @@ public class JobSchedulerService extends com.android.server.SystemService
         }
         JobStatus newJob = new JobStatus(failureToReschedule,
                 earliestRuntimeMs,
-                JobStatus.NO_LATEST_RUNTIME, numFailures, numSystemStops,
+                JobStatus.NO_LATEST_RUNTIME, numFailures, numAbandonedFailures, numSystemStops,
                 failureToReschedule.getLastSuccessfulRunTime(), sSystemClock.millis(),
                 failureToReschedule.getCumulativeExecutionTimeMs());
         if (stopReason == JobParameters.STOP_REASON_USER) {
@@ -3062,6 +3088,20 @@ public class JobSchedulerService extends com.android.server.SystemService
     }
 
     /**
+     * Returns {@code true} if the given number of abandoned failures indicates that JobScheduler
+     * should use an aggressive backoff policy.
+     *
+     * @param numAbandonedFailures The number of abandoned failures.
+     * @return {@code true} if the given number of abandoned failures indicates that JobScheduler
+     *     should use an aggressive backoff policy.
+     */
+    public boolean shouldUseAggressiveBackoff(int numAbandonedFailures) {
+        return android.app.job.Flags.handleAbandonedJobs()
+                && numAbandonedFailures
+                        > mConstants.ABANDONED_JOB_TIMEOUTS_BEFORE_AGGRESSIVE_BACKOFF;
+    }
+
+    /**
      * Maximum time buffer in which JobScheduler will try to optimize periodic job scheduling. This
      * does not cause a job's period to be larger than requested (eg: if the requested period is
      * shorter than this buffer). This is used to put a limit on when JobScheduler will intervene
@@ -3140,6 +3180,7 @@ public class JobSchedulerService extends com.android.server.SystemService
             return new JobStatus(periodicToReschedule,
                     elapsedNow + period - flex, elapsedNow + period,
                     0 /* numFailures */, 0 /* numSystemStops */,
+                    0 /* numAbandonedFailures */,
                     sSystemClock.millis() /* lastSuccessfulRunTime */,
                     periodicToReschedule.getLastFailedRunTime(),
                     0 /* Reset cumulativeExecutionTime because of successful execution */);
@@ -3156,6 +3197,7 @@ public class JobSchedulerService extends com.android.server.SystemService
         return new JobStatus(periodicToReschedule,
                 newEarliestRunTimeElapsed, newLatestRuntimeElapsed,
                 0 /* numFailures */, 0 /* numSystemStops */,
+                0 /* numAbandonedFailures */,
                 sSystemClock.millis() /* lastSuccessfulRunTime */,
                 periodicToReschedule.getLastFailedRunTime(),
                 0 /* Reset cumulativeExecutionTime because of successful execution */);
@@ -3164,6 +3206,10 @@ public class JobSchedulerService extends com.android.server.SystemService
     @VisibleForTesting
     void maybeProcessBuggyJob(@NonNull JobStatus jobStatus, int debugStopReason) {
         boolean jobTimedOut = debugStopReason == JobParameters.INTERNAL_STOP_REASON_TIMEOUT;
+        if (android.app.job.Flags.handleAbandonedJobs()) {
+            jobTimedOut |= (debugStopReason
+                == JobParameters.INTERNAL_STOP_REASON_TIMEOUT_ABANDONED);
+        }
         // If madeActive = 0, the job never actually started.
         if (!jobTimedOut && jobStatus.madeActive > 0) {
             final long executionDurationMs = sUptimeMillisClock.millis() - jobStatus.madeActive;
@@ -3245,9 +3291,12 @@ public class JobSchedulerService extends com.android.server.SystemService
         // we stop it.
         final JobStatus rescheduledJob = needsReschedule
                 ? getRescheduleJobForFailureLocked(jobStatus, stopReason, debugStopReason) : null;
+        final boolean isStopReasonAbandoned = android.app.job.Flags.handleAbandonedJobs()
+                && (debugStopReason == JobParameters.INTERNAL_STOP_REASON_TIMEOUT_ABANDONED);
         if (rescheduledJob != null
                 && !rescheduledJob.shouldTreatAsUserInitiatedJob()
                 && (debugStopReason == JobParameters.INTERNAL_STOP_REASON_TIMEOUT
+                || isStopReasonAbandoned
                 || debugStopReason == JobParameters.INTERNAL_STOP_REASON_PREEMPT)) {
             rescheduledJob.disallowRunInBatterySaverAndDoze();
         }
diff --git a/apex/jobscheduler/service/java/com/android/server/job/JobStore.java b/apex/jobscheduler/service/java/com/android/server/job/JobStore.java
index d8934d8f83b8..dfb36818c818 100644
--- a/apex/jobscheduler/service/java/com/android/server/job/JobStore.java
+++ b/apex/jobscheduler/service/java/com/android/server/job/JobStore.java
@@ -269,7 +269,9 @@ public final class JobStore {
                         convertRtcBoundsToElapsed(utcTimes, elapsedNow);
                 JobStatus newJob = new JobStatus(job,
                         elapsedRuntimes.first, elapsedRuntimes.second,
-                        0, 0, job.getLastSuccessfulRunTime(), job.getLastFailedRunTime(),
+                        0 /* numFailures */, 0 /* numAbandonedFailures */,
+                        0 /* numSystemStops */, job.getLastSuccessfulRunTime(),
+                        job.getLastFailedRunTime(),
                         job.getCumulativeExecutionTimeMs());
                 newJob.prepareLocked();
                 toAdd.add(newJob);
diff --git a/apex/jobscheduler/service/java/com/android/server/job/controllers/JobStatus.java b/apex/jobscheduler/service/java/com/android/server/job/controllers/JobStatus.java
index 58579eb0db47..0c5be2185c2b 100644
--- a/apex/jobscheduler/service/java/com/android/server/job/controllers/JobStatus.java
+++ b/apex/jobscheduler/service/java/com/android/server/job/controllers/JobStatus.java
@@ -312,6 +312,12 @@ public final class JobStatus {
     private final int numFailures;
 
     /**
+     * How many times this job has stopped due to {@link
+     * JobParameters#STOP_REASON_TIMEOUT_ABANDONED}.
+     */
+    private final int mNumAbandonedFailures;
+
+    /**
      * The number of times JobScheduler has forced this job to stop due to reasons mostly outside
      * of the app's control.
      */
@@ -597,6 +603,8 @@ public final class JobStatus {
      * @param tag A string associated with the job for debugging/logging purposes.
      * @param numFailures Count of how many times this job has requested a reschedule because
      *     its work was not yet finished.
+     * @param mNumAbandonedFailures Count of how many times this job has requested a reschedule
+     *     because it was abandoned.
      * @param numSystemStops Count of how many times JobScheduler has forced this job to stop due to
      *     factors mostly out of the app's control.
      * @param earliestRunTimeElapsedMillis Milestone: earliest point in time at which the job
@@ -609,7 +617,7 @@ public final class JobStatus {
      */
     private JobStatus(JobInfo job, int callingUid, String sourcePackageName,
             int sourceUserId, int standbyBucket, @Nullable String namespace, String tag,
-            int numFailures, int numSystemStops,
+            int numFailures, int mNumAbandonedFailures, int numSystemStops,
             long earliestRunTimeElapsedMillis, long latestRunTimeElapsedMillis,
             long lastSuccessfulRunTime, long lastFailedRunTime, long cumulativeExecutionTimeMs,
             int internalFlags,
@@ -669,6 +677,7 @@ public final class JobStatus {
         this.latestRunTimeElapsedMillis = latestRunTimeElapsedMillis;
         this.mOriginalLatestRunTimeElapsedMillis = latestRunTimeElapsedMillis;
         this.numFailures = numFailures;
+        this.mNumAbandonedFailures = mNumAbandonedFailures;
         mNumSystemStops = numSystemStops;
 
         int requiredConstraints = job.getConstraintFlags();
@@ -742,7 +751,8 @@ public final class JobStatus {
         this(jobStatus.getJob(), jobStatus.getUid(),
                 jobStatus.getSourcePackageName(), jobStatus.getSourceUserId(),
                 jobStatus.getStandbyBucket(), jobStatus.getNamespace(),
-                jobStatus.getSourceTag(), jobStatus.getNumFailures(), jobStatus.getNumSystemStops(),
+                jobStatus.getSourceTag(), jobStatus.getNumFailures(),
+                jobStatus.getNumAbandonedFailures(), jobStatus.getNumSystemStops(),
                 jobStatus.getEarliestRunTime(), jobStatus.getLatestRunTimeElapsed(),
                 jobStatus.getLastSuccessfulRunTime(), jobStatus.getLastFailedRunTime(),
                 jobStatus.getCumulativeExecutionTimeMs(),
@@ -779,6 +789,7 @@ public final class JobStatus {
         this(job, callingUid, sourcePkgName, sourceUserId,
                 standbyBucket, namespace,
                 sourceTag, /* numFailures */ 0, /* numSystemStops */ 0,
+                /* mNumAbandonedFailures */ 0,
                 earliestRunTimeElapsedMillis, latestRunTimeElapsedMillis,
                 lastSuccessfulRunTime, lastFailedRunTime, cumulativeExecutionTimeMs,
                 innerFlags, dynamicConstraints);
@@ -798,13 +809,15 @@ public final class JobStatus {
     /** Create a new job to be rescheduled with the provided parameters. */
     public JobStatus(JobStatus rescheduling,
             long newEarliestRuntimeElapsedMillis,
-            long newLatestRuntimeElapsedMillis, int numFailures, int numSystemStops,
+            long newLatestRuntimeElapsedMillis, int numFailures,
+            int mNumAbandonedFailures, int numSystemStops,
             long lastSuccessfulRunTime, long lastFailedRunTime,
             long cumulativeExecutionTimeMs) {
         this(rescheduling.job, rescheduling.getUid(),
                 rescheduling.getSourcePackageName(), rescheduling.getSourceUserId(),
                 rescheduling.getStandbyBucket(), rescheduling.getNamespace(),
-                rescheduling.getSourceTag(), numFailures, numSystemStops,
+                rescheduling.getSourceTag(), numFailures,
+                mNumAbandonedFailures, numSystemStops,
                 newEarliestRuntimeElapsedMillis,
                 newLatestRuntimeElapsedMillis,
                 lastSuccessfulRunTime, lastFailedRunTime, cumulativeExecutionTimeMs,
@@ -843,7 +856,8 @@ public final class JobStatus {
         int standbyBucket = JobSchedulerService.standbyBucketForPackage(jobPackage,
                 sourceUserId, elapsedNow);
         return new JobStatus(job, callingUid, sourcePkg, sourceUserId,
-                standbyBucket, namespace, tag, /* numFailures */ 0, /* numSystemStops */ 0,
+                standbyBucket, namespace, tag, /* numFailures */ 0,
+                /* mNumAbandonedFailures */ 0, /* numSystemStops */ 0,
                 earliestRunTimeElapsedMillis, latestRunTimeElapsedMillis,
                 0 /* lastSuccessfulRunTime */, 0 /* lastFailedRunTime */,
                 /* cumulativeExecutionTime */ 0,
@@ -1138,6 +1152,13 @@ public final class JobStatus {
     }
 
     /**
+     * Returns the number of times the job stopped previously for STOP_REASON_TIMEOUT_ABANDONED.
+     */
+    public int getNumAbandonedFailures() {
+        return mNumAbandonedFailures;
+    }
+
+    /**
      * Returns the number of times the system stopped a previous execution of this job for reasons
      * that were likely outside the app's control.
      */