Merge "Enhance watchdog debug log + extend sleep from 2s to 5s"

3 files changed, 38 insertions, 32 deletions

diff --git a/services/core/java/com/android/server/Watchdog.java b/services/core/java/com/android/server/Watchdog.java
index aaca85b85943..6d69fcd3e453 100644
--- a/services/core/java/com/android/server/Watchdog.java
+++ b/services/core/java/com/android/server/Watchdog.java
@@ -61,7 +61,8 @@ import java.util.List;
 public class Watchdog extends Thread {
     static final String TAG = "Watchdog";
 
-    private static final boolean DEBUG = true; // STOPSHIP disable it (b/113252928)
+    /** Debug flag. */
+    public static final boolean DEBUG = true; // STOPSHIP disable it (b/113252928)
 
     // Set this to true to use debug default values.
     static final boolean DB = false;
@@ -141,7 +142,7 @@ public class Watchdog extends Thread {
             mCompleted = true;
         }
 
-        public void addMonitor(Monitor monitor) {
+        void addMonitorLocked(Monitor monitor) {
             mMonitors.add(monitor);
         }
 
@@ -168,7 +169,7 @@ public class Watchdog extends Thread {
             mHandler.postAtFrontOfQueue(this);
         }
 
-        public boolean isOverdueLocked() {
+        boolean isOverdueLocked() {
             return (!mCompleted) && (SystemClock.uptimeMillis() > mStartTime + mWaitMax);
         }
 
@@ -194,7 +195,7 @@ public class Watchdog extends Thread {
             return mName;
         }
 
-        public String describeBlockedStateLocked() {
+        String describeBlockedStateLocked() {
             if (mCurrentMonitor == null) {
                 return "Blocked in handler on " + mName + " (" + getThread().getName() + ")";
             } else {
@@ -324,7 +325,7 @@ public class Watchdog extends Thread {
             if (isAlive()) {
                 throw new RuntimeException("Monitors can't be added once the Watchdog is running");
             }
-            mMonitorChecker.addMonitor(monitor);
+            mMonitorChecker.addMonitorLocked(monitor);
         }
     }
 
@@ -484,7 +485,7 @@ public class Watchdog extends Thread {
                             // trace and wait another half.
                             ArrayList<Integer> pids = new ArrayList<Integer>();
                             pids.add(Process.myPid());
-                            ActivityManagerService.dumpStackTraces(true, pids, null, null,
+                            ActivityManagerService.dumpStackTraces(pids, null, null,
                                 getInterestingNativePids());
                             waitedHalf = true;
                         }
@@ -509,14 +510,13 @@ public class Watchdog extends Thread {
             ArrayList<Integer> pids = new ArrayList<>();
             pids.add(Process.myPid());
             if (mPhonePid > 0) pids.add(mPhonePid);
-            // Pass !waitedHalf so that just in case we somehow wind up here without having
-            // dumped the halfway stacks, we properly re-initialize the trace file.
+
             final File stack = ActivityManagerService.dumpStackTraces(
-                    !waitedHalf, pids, null, null, getInterestingNativePids());
+                    pids, null, null, getInterestingNativePids());
 
             // Give some extra time to make sure the stack traces get written.
             // The system's been hanging for a minute, another second or two won't hurt much.
-            SystemClock.sleep(2000);
+            SystemClock.sleep(5000);
 
             // Trigger the kernel to dump all blocked threads, and backtraces on all CPUs to the kernel log
             doSysRq('w');
diff --git a/services/core/java/com/android/server/am/ActivityManagerService.java b/services/core/java/com/android/server/am/ActivityManagerService.java
index fb0d9adf34b2..b91a449aab89 100644
--- a/services/core/java/com/android/server/am/ActivityManagerService.java
+++ b/services/core/java/com/android/server/am/ActivityManagerService.java
@@ -4570,18 +4570,19 @@ public class ActivityManagerService extends IActivityManager.Stub
 
     /**
      * If a stack trace dump file is configured, dump process stack traces.
-     * @param clearTraces causes the dump file to be erased prior to the new
-     *    traces being written, if true; when false, the new traces will be
-     *    appended to any existing file content.
      * @param firstPids of dalvik VM processes to dump stack traces for first
      * @param lastPids of dalvik VM processes to dump stack traces for last
      * @param nativePids optional list of native pids to dump stack crawls
      */
-    public static File dumpStackTraces(boolean clearTraces, ArrayList<Integer> firstPids,
+    public static File dumpStackTraces(ArrayList<Integer> firstPids,
             ProcessCpuTracker processCpuTracker, SparseArray<Boolean> lastPids,
             ArrayList<Integer> nativePids) {
         ArrayList<Integer> extraPids = null;
 
+        if (DEBUG_ANR) {
+            Slog.d(TAG, "dumpStackTraces pids=" + lastPids + " nativepids=" + nativePids);
+        }
+
         // Measure CPU usage as soon as we're called in order to get a realistic sampling
         // of the top users at the time of the request.
         if (processCpuTracker != null) {
@@ -21561,11 +21562,16 @@ public class ActivityManagerService extends IActivityManager.Stub
      * cause a watchdog kill.
      */
     void maybeTriggerWatchdog() {
-        if (SystemProperties.getInt("debug.trigger.watchdog", 0) == 1) {
-            Slog.w(TAG, "TRIGGERING WATCHDOG");
+        final String key = "debug.trigger.watchdog";
+        if (Watchdog.DEBUG && SystemProperties.getInt(key, 0) == 1) {
+            Slog.w(TAG, "!!! TRIGGERING WATCHDOG !!!");
+
+            // Clear the property; otherwise the system would hang again after a watchdog restart.
+            SystemProperties.set(key, "");
             synchronized (ActivityManagerService.this) {
                 try {
-                    Thread.sleep(600 * 1000);
+                    // Arbitrary long sleep for watchdog to catch.
+                    Thread.sleep(60 * 60 * 1000);
                 } catch (InterruptedException e) {
                 }
             }
diff --git a/services/core/java/com/android/server/am/AppErrors.java b/services/core/java/com/android/server/am/AppErrors.java
index 162f344d012f..6a9c8877ce5b 100644
--- a/services/core/java/com/android/server/am/AppErrors.java
+++ b/services/core/java/com/android/server/am/AppErrors.java
@@ -16,12 +16,12 @@
 
 package com.android.server.am;
 
-import com.android.internal.app.ProcessMap;
-import com.android.internal.logging.MetricsLogger;
-import com.android.internal.logging.nano.MetricsProto;
-import com.android.internal.os.ProcessCpuTracker;
-import com.android.server.RescueParty;
-import com.android.server.Watchdog;
+import static com.android.server.Watchdog.NATIVE_STACKS_OF_INTEREST;
+import static com.android.server.am.ActivityManagerDebugConfig.DEBUG_ANR;
+import static com.android.server.am.ActivityManagerDebugConfig.TAG_AM;
+import static com.android.server.am.ActivityManagerDebugConfig.TAG_WITH_CLASS_NAME;
+import static com.android.server.am.ActivityManagerService.MY_PID;
+import static com.android.server.am.ActivityManagerService.SYSTEM_DEBUGGABLE;
 
 import android.app.ActivityManager;
 import android.app.ActivityOptions;
@@ -45,11 +45,18 @@ import android.util.ArrayMap;
 import android.util.ArraySet;
 import android.util.EventLog;
 import android.util.Slog;
-import android.util.StatsLog;
 import android.util.SparseArray;
+import android.util.StatsLog;
 import android.util.TimeUtils;
 import android.util.proto.ProtoOutputStream;
 
+import com.android.internal.app.ProcessMap;
+import com.android.internal.logging.MetricsLogger;
+import com.android.internal.logging.nano.MetricsProto;
+import com.android.internal.os.ProcessCpuTracker;
+import com.android.server.RescueParty;
+import com.android.server.Watchdog;
+
 import java.io.File;
 import java.io.FileDescriptor;
 import java.io.PrintWriter;
@@ -57,13 +64,6 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Set;
 
-import static com.android.server.Watchdog.NATIVE_STACKS_OF_INTEREST;
-import static com.android.server.am.ActivityManagerDebugConfig.DEBUG_ANR;
-import static com.android.server.am.ActivityManagerDebugConfig.TAG_AM;
-import static com.android.server.am.ActivityManagerDebugConfig.TAG_WITH_CLASS_NAME;
-import static com.android.server.am.ActivityManagerService.MY_PID;
-import static com.android.server.am.ActivityManagerService.SYSTEM_DEBUGGABLE;
-
 /**
  * Controls error conditions in applications.
  */
@@ -1017,7 +1017,7 @@ class AppErrors {
         // For background ANRs, don't pass the ProcessCpuTracker to
         // avoid spending 1/2 second collecting stats to rank lastPids.
         File tracesFile = ActivityManagerService.dumpStackTraces(
-                true, firstPids,
+                firstPids,
                 (isSilentANR) ? null : processCpuTracker,
                 (isSilentANR) ? null : lastPids,
                 nativePids);