Thread.join, GC daemons, suspend daemons on shutdown, and wait for non-daemon threads to exit.

(I've been testing with a modified SystemMethods test that does various thread
operations.)

Change-Id: I3087087546f90c43da7a0e63fae918ff0a6e7005
diff --git a/src/runtime.cc b/src/runtime.cc
index de94b3d..99f5e3a 100644
--- a/src/runtime.cc
+++ b/src/runtime.cc
@@ -40,8 +40,9 @@
 Runtime::~Runtime() {
   // Make sure our internal threads are dead before we start tearing down things they're using.
   delete signal_catcher_;
+  // TODO: GC thread.
 
-  // Make sure all other threads have terminated too.
+  // Make sure all other non-daemon threads have terminated, and all daemon threads are suspended.
   delete thread_list_;
 
   delete class_linker_;
@@ -355,11 +356,9 @@
 void Runtime::Start() {
   started_ = true;
 
-  // Initialize both the built-in and libcore native methods.
-  InitLibraries();
+  InitNativeMethods();
 
-  // Finish attaching the main thread.
-  Thread::Current()->CreatePeer("main", false);
+  Thread::FinishStartup();
 
   RunImageClinits();
 
@@ -382,7 +381,7 @@
   CHECK(c != NULL);
   Method* m = c->FindDirectMethod("start", "()V");
   CHECK(m != NULL);
-//  m->Invoke(Thread::Current(), NULL, NULL, NULL);
+  m->Invoke(Thread::Current(), NULL, NULL, NULL);
 }
 
 bool Runtime::IsStarted() {
@@ -432,7 +431,7 @@
   return true;
 }
 
-void Runtime::InitLibraries() {
+void Runtime::InitNativeMethods() {
   Thread* self = Thread::Current();
   JNIEnv* env = self->GetJniEnv();
 
@@ -516,6 +515,8 @@
 }
 
 void Runtime::DetachCurrentThread() {
+  // TODO: check we're not calling DetachCurrentThread from a call stack that
+  // includes managed frames. (It's only valid if the stack is all-native.)
   thread_list_->Unregister();
 }
 
diff --git a/src/runtime.h b/src/runtime.h
index 6d11cf3..70bdc99 100644
--- a/src/runtime.h
+++ b/src/runtime.h
@@ -161,7 +161,7 @@
   void BlockSignals();
 
   bool Init(const Options& options, bool ignore_unrecognized);
-  void InitLibraries();
+  void InitNativeMethods();
   void RegisterRuntimeNativeMethods(JNIEnv*);
   void RunImageClinits();
   void StartDaemonThreads();
diff --git a/src/thread.cc b/src/thread.cc
index e1207d0..dde0f66 100644
--- a/src/thread.cc
+++ b/src/thread.cc
@@ -41,6 +41,15 @@
 
 pthread_key_t Thread::pthread_key_self_;
 
+static Field* gThread_daemon = NULL;
+static Field* gThread_group = NULL;
+static Field* gThread_lock = NULL;
+static Field* gThread_name = NULL;
+static Field* gThread_priority = NULL;
+static Field* gThread_vmData = NULL;
+static Field* gThreadGroup_name = NULL;
+static Method* gThread_run = NULL;
+
 // Temporary debugging hook for compiler.
 void DebugMe(Method* method, uint32_t info) {
     LOG(INFO) << "DebugMe";
@@ -398,13 +407,7 @@
 
   self->Attach(runtime);
 
-  ClassLinker* class_linker = runtime->GetClassLinker();
-
-  Class* thread_class = class_linker->FindSystemClass("Ljava/lang/Thread;");
-  Class* string_class = class_linker->FindSystemClass("Ljava/lang/String;");
-
-  Field* name_field = thread_class->FindDeclaredInstanceField("name", string_class);
-  String* thread_name = reinterpret_cast<String*>(name_field->GetObject(self->peer_));
+  String* thread_name = reinterpret_cast<String*>(gThread_name->GetObject(self->peer_));
   if (thread_name != NULL) {
     SetThreadName(thread_name->ToModifiedUtf8().c_str());
   }
@@ -421,8 +424,7 @@
   // Invoke the 'run' method of our java.lang.Thread.
   CHECK(self->peer_ != NULL);
   Object* receiver = self->peer_;
-  Method* Thread_run = thread_class->FindVirtualMethod("run", "()V");
-  Method* m = receiver->GetClass()->FindVirtualMethodForVirtualOrInterface(Thread_run);
+  Method* m = receiver->GetClass()->FindVirtualMethodForVirtualOrInterface(gThread_run);
   m->Invoke(self, receiver, NULL, NULL);
 
   // Detach.
@@ -432,14 +434,7 @@
 }
 
 void SetVmData(Object* managed_thread, Thread* native_thread) {
-  ClassLinker* class_linker = Runtime::Current()->GetClassLinker();
-
-  Class* thread_class = class_linker->FindSystemClass("Ljava/lang/Thread;");
-  Class* int_class = class_linker->FindPrimitiveClass('I');
-
-  Field* vmData_field = thread_class->FindDeclaredInstanceField("vmData", int_class);
-
-  vmData_field->SetInt(managed_thread, reinterpret_cast<uintptr_t>(native_thread));
+  gThread_vmData->SetInt(managed_thread, reinterpret_cast<uintptr_t>(native_thread));
 }
 
 void Thread::Create(Object* peer, size_t stack_size) {
@@ -604,28 +599,14 @@
   bool is_daemon = false;
 
   if (peer_ != NULL) {
-    ClassLinker* class_linker = Runtime::Current()->GetClassLinker();
-
-    Class* boolean_class = class_linker->FindPrimitiveClass('Z');
-    Class* int_class = class_linker->FindPrimitiveClass('I');
-    Class* string_class = class_linker->FindSystemClass("Ljava/lang/String;");
-    Class* thread_class = class_linker->FindSystemClass("Ljava/lang/Thread;");
-    Class* thread_group_class = class_linker->FindSystemClass("Ljava/lang/ThreadGroup;");
-
-    Field* name_field = thread_class->FindDeclaredInstanceField("name", string_class);
-    Field* priority_field = thread_class->FindDeclaredInstanceField("priority", int_class);
-    Field* daemon_field = thread_class->FindDeclaredInstanceField("daemon", boolean_class);
-    Field* thread_group_field = thread_class->FindDeclaredInstanceField("group", thread_group_class);
-
-    String* thread_name_string = reinterpret_cast<String*>(name_field->GetObject(peer_));
+    String* thread_name_string = reinterpret_cast<String*>(gThread_name->GetObject(peer_));
     thread_name = (thread_name_string != NULL) ? thread_name_string->ToModifiedUtf8() : "<null>";
-    priority = priority_field->GetInt(peer_);
-    is_daemon = daemon_field->GetBoolean(peer_);
+    priority = gThread_priority->GetInt(peer_);
+    is_daemon = gThread_daemon->GetBoolean(peer_);
 
-    Object* thread_group = thread_group_field->GetObject(peer_);
+    Object* thread_group = gThread_group->GetObject(peer_);
     if (thread_group != NULL) {
-      Field* name_field = thread_group_class->FindDeclaredInstanceField("name", string_class);
-      String* group_name_string = reinterpret_cast<String*>(name_field->GetObject(thread_group));
+      String* group_name_string = reinterpret_cast<String*>(gThreadGroup_name->GetObject(thread_group));
       group_name = (group_name_string != NULL) ? group_name_string->ToModifiedUtf8() : "<null>";
     }
   } else {
@@ -838,8 +819,28 @@
   if (pthread_getspecific(pthread_key_self_) != NULL) {
     LOG(FATAL) << "newly-created pthread TLS slot is not NULL";
   }
+}
 
-  // TODO: initialize other locks and condition variables
+void Thread::FinishStartup() {
+  // Finish attaching the main thread.
+  Thread::Current()->CreatePeer("main", false);
+
+  // Now the ClassLinker is ready, we can find the various Class*, Field*, and Method*s we need.
+  ClassLinker* class_linker = Runtime::Current()->GetClassLinker();
+  Class* boolean_class = class_linker->FindPrimitiveClass('Z');
+  Class* int_class = class_linker->FindPrimitiveClass('I');
+  Class* String_class = class_linker->FindSystemClass("Ljava/lang/String;");
+  Class* Thread_class = class_linker->FindSystemClass("Ljava/lang/Thread;");
+  Class* ThreadGroup_class = class_linker->FindSystemClass("Ljava/lang/ThreadGroup;");
+  Class* ThreadLock_class = class_linker->FindSystemClass("Ljava/lang/ThreadLock;");
+  gThread_daemon = Thread_class->FindDeclaredInstanceField("daemon", boolean_class);
+  gThread_group = Thread_class->FindDeclaredInstanceField("group", ThreadGroup_class);
+  gThread_lock = Thread_class->FindDeclaredInstanceField("lock", ThreadLock_class);
+  gThread_name = Thread_class->FindDeclaredInstanceField("name", String_class);
+  gThread_priority = Thread_class->FindDeclaredInstanceField("priority", int_class);
+  gThread_run = Thread_class->FindVirtualMethod("run", "()V");
+  gThread_vmData = Thread_class->FindDeclaredInstanceField("vmData", int_class);
+  gThreadGroup_name = ThreadGroup_class->FindDeclaredInstanceField("name", String_class);
 }
 
 void Thread::Shutdown() {
@@ -875,9 +876,6 @@
 }
 
 Thread::~Thread() {
-  // TODO: check we're not calling the JNI DetachCurrentThread function from
-  // a call stack that includes managed frames. (It's only valid if the stack is all-native.)
-
   // On thread detach, all monitors entered with JNI MonitorEnter are automatically exited.
   if (jni_env_ != NULL) {
     jni_env_->monitors.VisitRoots(MonitorExitVisitor, NULL);
@@ -901,14 +899,9 @@
   // Thread.join() is implemented as an Object.wait() on the Thread.lock
   // object. Signal anyone who is waiting.
   if (peer_ != NULL) {
-    ClassLinker* class_linker = Runtime::Current()->GetClassLinker();
-    Class* java_lang_Thread_class = class_linker->FindSystemClass("Ljava/lang/Thread;");
-    Class* java_lang_ThreadLock_class = class_linker->FindSystemClass("Ljava/lang/ThreadLock;");
-    Field* lock_field = java_lang_Thread_class->FindDeclaredInstanceField("lock", java_lang_ThreadLock_class);
-
     Thread* self = Thread::Current();
-    Object* lock = lock_field->GetObject(peer_);
-    // This conditional is only needed for tests, where Thread.lock won't have been set.
+    Object* lock = gThread_lock->GetObject(peer_);
+    // (This conditional is only needed for tests, where Thread.lock won't have been set.)
     if (lock != NULL) {
       lock->MonitorEnter(self);
       lock->NotifyAll();
@@ -1330,6 +1323,10 @@
   return object->GetLockOwner() == thin_lock_id_;
 }
 
+bool Thread::IsDaemon() {
+  return gThread_daemon->GetBoolean(peer_);
+}
+
 void Thread::VisitRoots(Heap::RootVisitor* visitor, void* arg) const {
   if (exception_ != NULL) {
     visitor(exception_, arg);
diff --git a/src/thread.h b/src/thread.h
index f4cc747..de65d42 100644
--- a/src/thread.h
+++ b/src/thread.h
@@ -265,6 +265,8 @@
 
   State SetState(State new_state);
 
+  bool IsDaemon();
+
   void WaitUntilSuspended();
 
   bool HoldsLock(Object*);
@@ -372,6 +374,7 @@
   void SetName(const char* name);
 
   static void Startup();
+  static void FinishStartup();
   static void Shutdown();
 
   // JNI methods
diff --git a/src/thread_list.cc b/src/thread_list.cc
index 7b743ee..f17414c 100644
--- a/src/thread_list.cc
+++ b/src/thread_list.cc
@@ -16,25 +16,26 @@
 
 #include "thread_list.h"
 
+#include <unistd.h>
+
 namespace art {
 
 ThreadList::ThreadList()
     : thread_list_lock_("thread list lock"),
       thread_start_cond_("thread_start_cond_"),
+      thread_exit_cond_("thread_exit_cond_"),
       thread_suspend_count_lock_("thread suspend count lock"),
       thread_suspend_count_cond_("thread_suspend_count_cond_") {
 }
 
 ThreadList::~ThreadList() {
+  // Detach the current thread if necessary.
   if (Contains(Thread::Current())) {
     Runtime::Current()->DetachCurrentThread();
   }
 
-  // All threads should have exited and unregistered when we
-  // reach this point. This means that all daemon threads had been
-  // shutdown cleanly.
-  // TODO: dump ThreadList if non-empty.
-  CHECK_EQ(list_.size(), 0U);
+  WaitForNonDaemonThreadsToExit();
+  SuspendAllDaemonThreads();
 }
 
 bool ThreadList::Contains(Thread* thread) {
@@ -105,7 +106,7 @@
    *
    * It's also okay if the thread transitions to a non-kRunnable state.
    *
-   * Note we released the threadSuspendCountLock before getting here,
+   * Note we released the thread_suspend_count_lock_ before getting here,
    * so if another thread is fiddling with its suspend count (perhaps
    * self-suspending for the debugger) it won't block while we're waiting
    * in here.
@@ -180,6 +181,9 @@
   // Clear the TLS data, so that thread is recognizably detached.
   // (It may wish to reattach later.)
   CHECK_PTHREAD_CALL(pthread_setspecific, (Thread::pthread_key_self_, NULL), "detach self");
+
+  // Signal that a thread just detached.
+  thread_exit_cond_.Signal();
 }
 
 void ThreadList::VisitRoots(Heap::RootVisitor* visitor, void* arg) const {
@@ -242,6 +246,56 @@
   self->SetState(Thread::kRunnable);
 }
 
+bool ThreadList::AllThreadsAreDaemons() {
+  for (It it = list_.begin(), end = list_.end(); it != end; ++it) {
+    if (!(*it)->IsDaemon()) {
+      return false;
+    }
+  }
+  return true;
+}
+
+void ThreadList::WaitForNonDaemonThreadsToExit() {
+  MutexLock mu(thread_list_lock_);
+  while (!AllThreadsAreDaemons()) {
+    thread_exit_cond_.Wait(thread_list_lock_);
+  }
+}
+
+void ThreadList::SuspendAllDaemonThreads() {
+  MutexLock mu(thread_list_lock_);
+
+  // Tell all the daemons it's time to suspend. (At this point, we know
+  // all threads are daemons.)
+  {
+    MutexLock mu(thread_suspend_count_lock_);
+    for (It it = list_.begin(), end = list_.end(); it != end; ++it) {
+      Thread* thread = *it;
+      ++thread->suspend_count_;
+    }
+  }
+
+  // Give the threads a chance to suspend, complaining if they're slow.
+  bool have_complained = false;
+  for (int i = 0; i < 10; ++i) {
+    usleep(200 * 1000);
+    bool all_suspended = true;
+    for (It it = list_.begin(), end = list_.end(); it != end; ++it) {
+      Thread* thread = *it;
+      if (thread->GetState() == Thread::kRunnable) {
+        if (!have_complained) {
+          LOG(WARNING) << "daemon thread not yet suspended: " << *thread;
+          have_complained = true;
+        }
+        all_suspended = false;
+      }
+    }
+    if (all_suspended) {
+      return;
+    }
+  }
+}
+
 uint32_t ThreadList::AllocThreadId() {
   MutexLock mu(thread_list_lock_);
   for (size_t i = 0; i < allocated_ids_.size(); ++i) {
diff --git a/src/thread_list.h b/src/thread_list.h
index aa1415a..32bcf20 100644
--- a/src/thread_list.h
+++ b/src/thread_list.h
@@ -52,14 +52,18 @@
  private:
   typedef std::list<Thread*>::const_iterator It; // TODO: C++0x auto
 
+  bool AllThreadsAreDaemons();
   uint32_t AllocThreadId();
   void ReleaseThreadId(uint32_t id);
+  void SuspendAllDaemonThreads();
+  void WaitForNonDaemonThreadsToExit();
 
   mutable Mutex thread_list_lock_;
   std::bitset<kMaxThreadId> allocated_ids_;
   std::list<Thread*> list_;
 
   ConditionVariable thread_start_cond_;
+  ConditionVariable thread_exit_cond_;
 
   // This lock guards every thread's suspend_count_ field...
   mutable Mutex thread_suspend_count_lock_;