Add dex2oat watch dog thread for host builds

Change-Id: I90b4b6b3a1aebb82955b4aa84d3f2d599af1d13b

8 files changed, 154 insertions, 48 deletions

diff --git a/src/dex2oat.cc b/src/dex2oat.cc
index bbc07a63b9..40218a8712 100644
--- a/src/dex2oat.cc
+++ b/src/dex2oat.cc
@@ -473,7 +473,108 @@ static size_t OpenDexFiles(const std::vector<const char*>& dex_filenames,
   return failure_count;
 }
 
+// The primary goal of the watchdog is to prevent stuck build servers
+// during development when fatal aborts lead to a cascade of failures
+// that result in a deadlock.
+class WatchDog {
+
+// WatchDog defines its own CHECK_PTHREAD_CALL to avoid using Log which uses locks
+#undef CHECK_PTHREAD_CALL
+#define CHECK_WATCH_DOG_PTHREAD_CALL(call, args, what) \
+  do { \
+    int rc = call args; \
+    if (rc != 0) { \
+      errno = rc; \
+      std::string message(# call); \
+      message += " failed for "; \
+      message += reason; \
+      Die(message); \
+    } \
+  } while (false)
+
+ public:
+  WatchDog() {
+    if (!kIsWatchDogEnabled) {
+      return;
+    }
+    shutting_down_ = false;
+    const char* reason = "dex2oat watch dog thread startup";
+    CHECK_WATCH_DOG_PTHREAD_CALL(pthread_mutex_init, (&mutex_, NULL), reason);
+    CHECK_WATCH_DOG_PTHREAD_CALL(pthread_cond_init, (&cond_, NULL), reason);
+    CHECK_WATCH_DOG_PTHREAD_CALL(pthread_attr_init, (&attr_), reason);
+    CHECK_WATCH_DOG_PTHREAD_CALL(pthread_create, (&pthread_, &attr_, &CallBack, this), reason);
+    CHECK_WATCH_DOG_PTHREAD_CALL(pthread_attr_destroy, (&attr_), reason);
+  }
+  ~WatchDog() {
+    if (!kIsWatchDogEnabled) {
+      return;
+    }
+    const char* reason = "dex2oat watch dog thread shutdown";
+    CHECK_WATCH_DOG_PTHREAD_CALL(pthread_mutex_lock, (&mutex_), reason);
+    shutting_down_ = true;
+    CHECK_WATCH_DOG_PTHREAD_CALL(pthread_cond_signal, (&cond_), reason);
+    CHECK_WATCH_DOG_PTHREAD_CALL(pthread_mutex_unlock, (&mutex_), reason);
+
+    CHECK_WATCH_DOG_PTHREAD_CALL(pthread_join, (pthread_, NULL), reason);
+
+    CHECK_WATCH_DOG_PTHREAD_CALL(pthread_cond_destroy, (&cond_), reason);
+    CHECK_WATCH_DOG_PTHREAD_CALL(pthread_mutex_destroy, (&mutex_), reason);
+  }
+
+ private:
+  static void* CallBack(void* arg) {
+    WatchDog* self = reinterpret_cast<WatchDog*>(arg);
+    self->Wait();
+    return NULL;
+  }
+
+  static void Die(const std::string& message) {
+    // TODO: Switch to LOG(FATAL) when we can guarantee it won't prevent shutdown in error cases
+    fprintf(stderr, "%s\n", message.c_str());
+    exit(1);
+  }
+
+  void Wait() {
+    int64_t ms = kWatchDogTimeoutSeconds * 1000;
+    int32_t ns = 0;
+    timespec ts;
+    InitTimeSpec(true, CLOCK_REALTIME, ms, ns, &ts);
+    const char* reason = "dex2oat watch dog thread waiting";
+    CHECK_WATCH_DOG_PTHREAD_CALL(pthread_mutex_lock, (&mutex_), reason);
+    while (!shutting_down_) {
+      int rc = TEMP_FAILURE_RETRY(pthread_cond_timedwait(&cond_, &mutex_, &ts));
+      if (rc == ETIMEDOUT) {
+        std::string message(StringPrintf("dex2oat did not finish after %d seconds",
+                                         kWatchDogTimeoutSeconds));
+        Die(message.c_str());
+      }
+      if (rc != 0) {
+        std::string message(StringPrintf("pthread_cond_timedwait failed: %s",
+                                         strerror(errno)));
+        Die(message.c_str());
+      }
+    }
+    CHECK_WATCH_DOG_PTHREAD_CALL(pthread_mutex_unlock, (&mutex_), reason);
+  }
+
+  static const bool kIsWatchDogEnabled = !kIsTargetBuild;
+#ifdef ART_USE_LLVM_COMPILER
+  static const unsigned int kWatchDogTimeoutSeconds = 20 * 60; // 15 minutes + buffer
+#else
+  static const unsigned int kWatchDogTimeoutSeconds = 2 * 60;  // 1 minute + buffer
+#endif
+
+  bool shutting_down_;
+  // TODO: Switch to Mutex when we can guarantee it won't prevent shutdown in error cases
+  pthread_mutex_t mutex_;
+  pthread_cond_t cond_;
+  pthread_attr_t attr_;
+  pthread_t pthread_;
+};
+
 static int dex2oat(int argc, char** argv) {
+  WatchDog watch_dog;
+
   InitLogging(argv);
 
   // Skip over argv[0].
diff --git a/src/globals.h b/src/globals.h
index 8577b43969..dc9341ae0f 100644
--- a/src/globals.h
+++ b/src/globals.h
@@ -66,6 +66,13 @@ const bool kIsDebugBuild = false;
 const bool kIsDebugBuild = true;
 #endif
 
+// Whether or not this is a target (vs host) build. Useful in conditionals where ART_TARGET isn't.
+#if defined(ART_TARGET)
+const bool kIsTargetBuild = true;
+#else
+const bool kIsTargetBuild = false;
+#endif
+
 }  // namespace art
 
 #endif  // ART_SRC_GLOBALS_H_
diff --git a/src/mutex.cc b/src/mutex.cc
index 0e33f3cc48..e2044d6395 100644
--- a/src/mutex.cc
+++ b/src/mutex.cc
@@ -27,12 +27,6 @@
 #include "thread.h"
 #include "utils.h"
 
-#if defined(__APPLE__)
-#include "AvailabilityMacros.h" // For MAC_OS_X_VERSION_MAX_ALLOWED
-// No clocks to specify on OS/X, fake value to pass to routines that require a clock.
-#define CLOCK_REALTIME 0xebadf00d
-#endif
-
 #define CHECK_MUTEX_CALL(call, args) CHECK_PTHREAD_CALL(call, args, name_)
 
 extern int pthread_mutex_lock(pthread_mutex_t* mutex) EXCLUSIVE_LOCK_FUNCTION(mutex);
@@ -100,43 +94,6 @@ static uint64_t SafeGetTid(const Thread* self) {
   }
 }
 
-// Initialize a timespec to either an absolute or relative time.
-static void InitTimeSpec(Thread* self, bool absolute, int clock, int64_t ms, int32_t ns,
-                         timespec* ts) {
-  int64_t endSec;
-
-  if (absolute) {
-#if !defined(__APPLE__)
-    clock_gettime(clock, ts);
-#else
-    UNUSED(clock);
-    timeval tv;
-    gettimeofday(&tv, NULL);
-    ts->tv_sec = tv.tv_sec;
-    ts->tv_nsec = tv.tv_usec * 1000;
-#endif
-  } else {
-    ts->tv_sec = 0;
-    ts->tv_nsec = 0;
-  }
-  endSec = ts->tv_sec + ms / 1000;
-  if (UNLIKELY(endSec >= 0x7fffffff)) {
-    std::ostringstream ss;
-    ScopedObjectAccess soa(self);
-    self->Dump(ss);
-    LOG(INFO) << "Note: end time exceeds epoch: " << ss.str();
-    endSec = 0x7ffffffe;
-  }
-  ts->tv_sec = endSec;
-  ts->tv_nsec = (ts->tv_nsec + (ms % 1000) * 1000000) + ns;
-
-  // Catch rollover.
-  if (ts->tv_nsec >= 1000000000L) {
-    ts->tv_sec++;
-    ts->tv_nsec -= 1000000000L;
-  }
-}
-
 #if ART_USE_FUTEXES
 static bool ComputeRelativeTimeSpec(timespec* result_ts, const timespec& lhs, const timespec& rhs) {
   const long int one_sec = 1000 * 1000 * 1000;  // one second in nanoseconds.
@@ -563,7 +520,7 @@ bool ReaderWriterMutex::ExclusiveLockWithTimeout(Thread* self, int64_t ms, int32
   exclusive_owner_ = SafeGetTid(self);
 #else
   timespec ts;
-  InitTimeSpec(self, true, CLOCK_REALTIME, ms, ns, &ts);
+  InitTimeSpec(true, CLOCK_REALTIME, ms, ns, &ts);
   int result = pthread_rwlock_timedwrlock(&rwlock_, &ts);
   if (result == ETIMEDOUT) {
     return false;
@@ -891,8 +848,8 @@ void ConditionVariable::TimedWait(Thread* self, int64_t ms, int32_t ns) {
 #endif
   guard_.recursion_count_ = 0;
   timespec ts;
-  InitTimeSpec(self, true, clock, ms, ns, &ts);
-  int rc = TIMEDWAIT(&cond_, &guard_.mutex_, &ts);
+  InitTimeSpec(true, clock, ms, ns, &ts);
+  int rc = TEMP_FAILURE_RETRY(TIMEDWAIT(&cond_, &guard_.mutex_, &ts));
   if (rc != 0 && rc != ETIMEDOUT) {
     errno = rc;
     PLOG(FATAL) << "TimedWait failed for " << name_;
diff --git a/src/thread_list.cc b/src/thread_list.cc
index 43a0cee150..1a2fd47dfa 100644
--- a/src/thread_list.cc
+++ b/src/thread_list.cc
@@ -207,7 +207,7 @@ size_t ThreadList::RunCheckpoint(Closure* checkpoint_function) {
       // Shouldn't need to wait for longer than 1 millisecond.
       const uint64_t threshold = 1;
       if (NsToMs(end - start) > threshold) {
-        LOG(INFO) << "Warning: waited longer than " << threshold << " ms for thrad suspend"
+        LOG(INFO) << "Warning: waited longer than " << threshold << " ms for thread suspend"
                   << std::endl;
       }
     }
diff --git a/src/thread_pool.cc b/src/thread_pool.cc
index f3319e2d40..ba531132f9 100644
--- a/src/thread_pool.cc
+++ b/src/thread_pool.cc
@@ -11,6 +11,7 @@ ThreadPoolWorker::ThreadPoolWorker(ThreadPool* thread_pool, const std::string& n
       name_(name),
       stack_size_(stack_size) {
   const char* reason = "new thread pool worker thread";
+  pthread_attr_t attr;
   CHECK_PTHREAD_CALL(pthread_attr_init, (&attr), reason);
   CHECK_PTHREAD_CALL(pthread_attr_setstacksize, (&attr, stack_size), reason);
   CHECK_PTHREAD_CALL(pthread_create, (&pthread_, &attr, &Callback, this), reason);
diff --git a/src/thread_pool.h b/src/thread_pool.h
index 22e30b71fe..1d0f85deea 100644
--- a/src/thread_pool.h
+++ b/src/thread_pool.h
@@ -47,7 +47,6 @@ class ThreadPoolWorker {
   const std::string name_;
   const size_t stack_size_;
   pthread_t pthread_;
-  pthread_attr_t attr;
 
   friend class ThreadPool;
   DISALLOW_COPY_AND_ASSIGN(ThreadPoolWorker);
diff --git a/src/utils.cc b/src/utils.cc
index cbe07a2fa6..3cf69148b3 100644
--- a/src/utils.cc
+++ b/src/utils.cc
@@ -180,6 +180,39 @@ uint64_t ThreadCpuNanoTime() {
 #endif
 }
 
+void InitTimeSpec(bool absolute, int clock, int64_t ms, int32_t ns, timespec* ts) {
+  int64_t endSec;
+
+  if (absolute) {
+#if !defined(__APPLE__)
+    clock_gettime(clock, ts);
+#else
+    UNUSED(clock);
+    timeval tv;
+    gettimeofday(&tv, NULL);
+    ts->tv_sec = tv.tv_sec;
+    ts->tv_nsec = tv.tv_usec * 1000;
+#endif
+  } else {
+    ts->tv_sec = 0;
+    ts->tv_nsec = 0;
+  }
+  endSec = ts->tv_sec + ms / 1000;
+  if (UNLIKELY(endSec >= 0x7fffffff)) {
+    std::ostringstream ss;
+    LOG(INFO) << "Note: end time exceeds epoch: " << ss.str();
+    endSec = 0x7ffffffe;
+  }
+  ts->tv_sec = endSec;
+  ts->tv_nsec = (ts->tv_nsec + (ms % 1000) * 1000000) + ns;
+
+  // Catch rollover.
+  if (ts->tv_nsec >= 1000000000L) {
+    ts->tv_sec++;
+    ts->tv_nsec -= 1000000000L;
+  }
+}
+
 std::string PrettyDescriptor(const String* java_descriptor) {
   if (java_descriptor == NULL) {
     return "null";
diff --git a/src/utils.h b/src/utils.h
index 719ce572a7..d33cc4bf37 100644
--- a/src/utils.h
+++ b/src/utils.h
@@ -285,6 +285,14 @@ static inline uint64_t MsToNs(uint64_t ns) {
   return ns * 1000 * 1000;
 }
 
+#if defined(__APPLE__)
+// No clocks to specify on OS/X, fake value to pass to routines that require a clock.
+#define CLOCK_REALTIME 0xebadf00d
+#endif
+
+// Initialize a timespec to either an absolute or relative time.
+void InitTimeSpec(bool absolute, int clock, int64_t ms, int32_t ns, timespec* ts);
+
 // Splits a string using the given separator character into a vector of
 // strings. Empty strings will be omitted.
 void Split(const std::string& s, char separator, std::vector<std::string>& result);