Fix a deadlock between debugger and GC.

Avoid a deadlock between GC and debugger's SuspendAllForDebugger where
GC gets suspended during GC.

Fix the jdwp tests with the CC collector, but the deadlock is NOT
specific to the CC collector.

Bug: 25800335
Bug: 12687968
Change-Id: I8f2869872bce4692d943020072e35ebf5a4e68a5

4 files changed, 11 insertions, 0 deletions

diff --git a/runtime/debugger.cc b/runtime/debugger.cc
index d8325525ac..55f68d3f2c 100644
--- a/runtime/debugger.cc
+++ b/runtime/debugger.cc
@@ -2362,6 +2362,10 @@ JDWP::ObjectId Dbg::GetThreadId(Thread* thread) {
 }
 
 void Dbg::SuspendVM() {
+  // Avoid a deadlock between GC and debugger where GC gets suspended during GC. b/25800335.
+  gc::ScopedGCCriticalSection gcs(Thread::Current(),
+                                  gc::kGcCauseDebugger,
+                                  gc::kCollectorTypeDebugger);
   Runtime::Current()->GetThreadList()->SuspendAllForDebugger();
 }
 
@@ -4101,6 +4105,8 @@ void Dbg::ExecuteMethodWithoutPendingException(ScopedObjectAccess& soa, DebugInv
   // Suspend other threads if the invoke is not single-threaded.
   if ((pReq->options & JDWP::INVOKE_SINGLE_THREADED) == 0) {
     ScopedThreadSuspension sts(soa.Self(), kWaitingForDebuggerSuspension);
+    // Avoid a deadlock between GC and debugger where GC gets suspended during GC. b/25800335.
+    gc::ScopedGCCriticalSection gcs(soa.Self(), gc::kGcCauseDebugger, gc::kCollectorTypeDebugger);
     VLOG(jdwp) << "      Suspending all threads";
     Runtime::Current()->GetThreadList()->SuspendAllForDebugger();
   }
diff --git a/runtime/gc/collector_type.h b/runtime/gc/collector_type.h
index ae412262fc..4ffc8afd09 100644
--- a/runtime/gc/collector_type.h
+++ b/runtime/gc/collector_type.h
@@ -44,6 +44,8 @@ enum CollectorType {
   kCollectorTypeInstrumentation,
   // Fake collector for adding or removing application image spaces.
   kCollectorTypeAddRemoveAppImageSpace,
+  // Fake collector used to implement exclusion between GC and debugger.
+  kCollectorTypeDebugger,
   // A homogeneous space compaction collector used in background transition
   // when both foreground and background collector are CMS.
   kCollectorTypeHomogeneousSpaceCompact,
diff --git a/runtime/gc/gc_cause.cc b/runtime/gc/gc_cause.cc
index 679432ba89..18e5703fcf 100644
--- a/runtime/gc/gc_cause.cc
+++ b/runtime/gc/gc_cause.cc
@@ -35,6 +35,7 @@ const char* PrettyCause(GcCause cause) {
     case kGcCauseTrim: return "HeapTrim";
     case kGcCauseInstrumentation: return "Instrumentation";
     case kGcCauseAddRemoveAppImageSpace: return "AddRemoveAppImageSpace";
+    case kGcCauseDebugger: return "Debugger";
     default:
       LOG(FATAL) << "Unreachable";
       UNREACHABLE();
diff --git a/runtime/gc/gc_cause.h b/runtime/gc/gc_cause.h
index c6b505c481..ad67eb7ebc 100644
--- a/runtime/gc/gc_cause.h
+++ b/runtime/gc/gc_cause.h
@@ -43,6 +43,8 @@ enum GcCause {
   kGcCauseInstrumentation,
   // Not a real GC cause, used to add or remove app image spaces.
   kGcCauseAddRemoveAppImageSpace,
+  // Not a real GC cause, used to implement exclusion between GC and debugger.
+  kGcCauseDebugger,
   // GC triggered for background transition when both foreground and background collector are CMS.
   kGcCauseHomogeneousSpaceCompact,
 };