Allow JNI AttachCurrentThread to fail if not enough stack.

Add unit tests and move JavaVM JNI tests into there own set of gtests.
Bug: 18330119

Change-Id: I0e93dff783b1f5d787b3084d24122883e14951a1
diff --git a/runtime/base/logging.cc b/runtime/base/logging.cc
index d3a2655..b781d60 100644
--- a/runtime/base/logging.cc
+++ b/runtime/base/logging.cc
@@ -236,4 +236,28 @@
 #endif
 }
 
+void LogMessage::LogLineLowStack(const char* file, unsigned int line, LogSeverity log_severity,
+                                 const char* message) {
+#ifdef HAVE_ANDROID_OS
+  // TODO: be more conservative on stack usage here.
+  LogLine(file, line, log_severity, message);
+#else
+  static const char* log_characters = "VDIWEFF";
+  CHECK_EQ(strlen(log_characters), INTERNAL_FATAL + 1U);
+
+  const char* program_name = ProgramInvocationShortName();
+  write(STDERR_FILENO, program_name, strlen(program_name));
+  write(STDERR_FILENO, " ", 1);
+  write(STDERR_FILENO, &log_characters[log_severity], 1);
+  write(STDERR_FILENO, " ", 1);
+  // TODO: pid and tid.
+  write(STDERR_FILENO, file, strlen(file));
+  // TODO: line.
+  UNUSED(line);
+  write(STDERR_FILENO, "] ", 2);
+  write(STDERR_FILENO, message, strlen(message));
+  write(STDERR_FILENO, "\n", 1);
+#endif
+}
+
 }  // namespace art
diff --git a/runtime/base/logging.h b/runtime/base/logging.h
index baa83e3..ae83e33 100644
--- a/runtime/base/logging.h
+++ b/runtime/base/logging.h
@@ -244,6 +244,10 @@
   // The routine that performs the actual logging.
   static void LogLine(const char* file, unsigned int line, LogSeverity severity, const char* msg);
 
+  // A variant of the above for use with little stack.
+  static void LogLineLowStack(const char* file, unsigned int line, LogSeverity severity,
+                              const char* msg);
+
  private:
   const std::unique_ptr<LogMessageData> data_;
 
diff --git a/runtime/java_vm_ext.cc b/runtime/java_vm_ext.cc
index 19e03d8..a5abce6 100644
--- a/runtime/java_vm_ext.cc
+++ b/runtime/java_vm_ext.cc
@@ -795,13 +795,13 @@
   return JNI_OK;
 }
 
-extern "C" jint JNI_GetCreatedJavaVMs(JavaVM** vms, jsize, jsize* vm_count) {
+extern "C" jint JNI_GetCreatedJavaVMs(JavaVM** vms_buf, jsize buf_len, jsize* vm_count) {
   Runtime* runtime = Runtime::Current();
-  if (runtime == nullptr) {
+  if (runtime == nullptr || buf_len == 0) {
     *vm_count = 0;
   } else {
     *vm_count = 1;
-    vms[0] = runtime->GetJavaVM();
+    vms_buf[0] = runtime->GetJavaVM();
   }
   return JNI_OK;
 }
diff --git a/runtime/java_vm_ext_test.cc b/runtime/java_vm_ext_test.cc
new file mode 100644
index 0000000..60c6a5c
--- /dev/null
+++ b/runtime/java_vm_ext_test.cc
@@ -0,0 +1,132 @@
+/*
+ * Copyright (C) 2011 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "jni_internal.h"
+
+#include <pthread.h>
+
+#include "common_runtime_test.h"
+#include "java_vm_ext.h"
+#include "runtime.h"
+
+namespace art {
+
+class JavaVmExtTest : public CommonRuntimeTest {
+ protected:
+  virtual void SetUp() {
+    CommonRuntimeTest::SetUp();
+
+    vm_ = Runtime::Current()->GetJavaVM();
+  }
+
+
+  virtual void TearDown() OVERRIDE {
+    CommonRuntimeTest::TearDown();
+  }
+
+  JavaVMExt* vm_;
+};
+
+TEST_F(JavaVmExtTest, JNI_GetDefaultJavaVMInitArgs) {
+  jint err = JNI_GetDefaultJavaVMInitArgs(nullptr);
+  EXPECT_EQ(JNI_ERR, err);
+}
+
+TEST_F(JavaVmExtTest, JNI_GetCreatedJavaVMs) {
+  JavaVM* vms_buf[1];
+  jsize num_vms;
+  jint ok = JNI_GetCreatedJavaVMs(vms_buf, arraysize(vms_buf), &num_vms);
+  EXPECT_EQ(JNI_OK, ok);
+  EXPECT_EQ(1, num_vms);
+  EXPECT_EQ(vms_buf[0], vm_);
+}
+
+static bool gSmallStack = false;
+static bool gAsDaemon = false;
+
+static void* attach_current_thread_callback(void* arg ATTRIBUTE_UNUSED) {
+  JavaVM* vms_buf[1];
+  jsize num_vms;
+  JNIEnv* env;
+  jint ok = JNI_GetCreatedJavaVMs(vms_buf, arraysize(vms_buf), &num_vms);
+  EXPECT_EQ(JNI_OK, ok);
+  if (ok == JNI_OK) {
+    if (!gAsDaemon) {
+      ok = vms_buf[0]->AttachCurrentThread(&env, nullptr);
+    } else {
+      ok = vms_buf[0]->AttachCurrentThreadAsDaemon(&env, nullptr);
+    }
+    EXPECT_EQ(gSmallStack ? JNI_ERR : JNI_OK, ok);
+    if (ok == JNI_OK) {
+      ok = vms_buf[0]->DetachCurrentThread();
+      EXPECT_EQ(JNI_OK, ok);
+    }
+  }
+  return nullptr;
+}
+
+TEST_F(JavaVmExtTest, AttachCurrentThread) {
+  pthread_t pthread;
+  const char* reason = __PRETTY_FUNCTION__;
+  gSmallStack = false;
+  gAsDaemon = false;
+  CHECK_PTHREAD_CALL(pthread_create, (&pthread, nullptr, attach_current_thread_callback,
+      nullptr), reason);
+  void* ret_val;
+  CHECK_PTHREAD_CALL(pthread_join, (pthread, &ret_val), reason);
+  EXPECT_EQ(ret_val, nullptr);
+}
+
+TEST_F(JavaVmExtTest, AttachCurrentThreadAsDaemon) {
+  pthread_t pthread;
+  const char* reason = __PRETTY_FUNCTION__;
+  gSmallStack = false;
+  gAsDaemon = true;
+  CHECK_PTHREAD_CALL(pthread_create, (&pthread, nullptr, attach_current_thread_callback,
+      nullptr), reason);
+  void* ret_val;
+  CHECK_PTHREAD_CALL(pthread_join, (pthread, &ret_val), reason);
+  EXPECT_EQ(ret_val, nullptr);
+}
+
+TEST_F(JavaVmExtTest, AttachCurrentThread_SmallStack) {
+  pthread_t pthread;
+  pthread_attr_t attr;
+  const char* reason = __PRETTY_FUNCTION__;
+  gSmallStack = true;
+  gAsDaemon = false;
+  CHECK_PTHREAD_CALL(pthread_attr_init, (&attr), reason);
+  CHECK_PTHREAD_CALL(pthread_attr_setstacksize, (&attr, PTHREAD_STACK_MIN), reason);
+  CHECK_PTHREAD_CALL(pthread_create, (&pthread, &attr, attach_current_thread_callback,
+      nullptr), reason);
+  CHECK_PTHREAD_CALL(pthread_attr_destroy, (&attr), reason);
+  void* ret_val;
+  CHECK_PTHREAD_CALL(pthread_join, (pthread, &ret_val), reason);
+  EXPECT_EQ(ret_val, nullptr);
+}
+
+TEST_F(JavaVmExtTest, DetachCurrentThread) {
+  JNIEnv* env;
+  jint ok = vm_->AttachCurrentThread(&env, nullptr);
+  ASSERT_EQ(JNI_OK, ok);
+  ok = vm_->DetachCurrentThread();
+  EXPECT_EQ(JNI_OK, ok);
+
+  jint err = vm_->DetachCurrentThread();
+  EXPECT_EQ(JNI_ERR, err);
+}
+
+}  // namespace art
diff --git a/runtime/jni_internal_test.cc b/runtime/jni_internal_test.cc
index b57cc17..ccad137 100644
--- a/runtime/jni_internal_test.cc
+++ b/runtime/jni_internal_test.cc
@@ -2019,14 +2019,4 @@
   }
 }
 
-TEST_F(JniInternalTest, DetachCurrentThread) {
-  CleanUpJniEnv();  // cleanup now so TearDown won't have junk from wrong JNIEnv
-  jint ok = vm_->DetachCurrentThread();
-  EXPECT_EQ(JNI_OK, ok);
-
-  jint err = vm_->DetachCurrentThread();
-  EXPECT_EQ(JNI_ERR, err);
-  vm_->AttachCurrentThread(&env_, nullptr);  // need attached thread for CommonRuntimeTest::TearDown
-}
-
 }  // namespace art
diff --git a/runtime/thread.cc b/runtime/thread.cc
index 2c44f27..8af8e65 100644
--- a/runtime/thread.cc
+++ b/runtime/thread.cc
@@ -158,7 +158,7 @@
     // Check that if we got here we cannot be shutting down (as shutdown should never have started
     // while threads are being born).
     CHECK(!runtime->IsShuttingDownLocked());
-    self->Init(runtime->GetThreadList(), runtime->GetJavaVM());
+    CHECK(self->Init(runtime->GetThreadList(), runtime->GetJavaVM()));
     Runtime::Current()->EndThreadBirth();
   }
   {
@@ -348,40 +348,46 @@
   }
 }
 
-void Thread::Init(ThreadList* thread_list, JavaVMExt* java_vm) {
+bool Thread::Init(ThreadList* thread_list, JavaVMExt* java_vm) {
   // This function does all the initialization that must be run by the native thread it applies to.
   // (When we create a new thread from managed code, we allocate the Thread* in Thread::Create so
   // we can handshake with the corresponding native thread when it's ready.) Check this native
   // thread hasn't been through here already...
   CHECK(Thread::Current() == nullptr);
+
+  // Set pthread_self_ ahead of pthread_setspecific, that makes Thread::Current function, this
+  // avoids pthread_self_ ever being invalid when discovered from Thread::Current().
+  tlsPtr_.pthread_self = pthread_self();
+  CHECK(is_started_);
+
   SetUpAlternateSignalStack();
+  if (!InitStackHwm()) {
+    return false;
+  }
   InitCpu();
   InitTlsEntryPoints();
   RemoveSuspendTrigger();
   InitCardTable();
   InitTid();
-  // Set pthread_self_ ahead of pthread_setspecific, that makes Thread::Current function, this
-  // avoids pthread_self_ ever being invalid when discovered from Thread::Current().
-  tlsPtr_.pthread_self = pthread_self();
-  CHECK(is_started_);
+
   CHECK_PTHREAD_CALL(pthread_setspecific, (Thread::pthread_key_self_, this), "attach self");
   DCHECK_EQ(Thread::Current(), this);
 
   tls32_.thin_lock_thread_id = thread_list->AllocThreadId(this);
-  InitStackHwm();
 
   tlsPtr_.jni_env = new JNIEnvExt(this, java_vm);
   thread_list->Register(this);
+  return true;
 }
 
 Thread* Thread::Attach(const char* thread_name, bool as_daemon, jobject thread_group,
                        bool create_peer) {
-  Thread* self;
   Runtime* runtime = Runtime::Current();
   if (runtime == nullptr) {
     LOG(ERROR) << "Thread attaching to non-existent runtime: " << thread_name;
     return nullptr;
   }
+  Thread* self;
   {
     MutexLock mu(nullptr, *Locks::runtime_shutdown_lock_);
     if (runtime->IsShuttingDownLocked()) {
@@ -390,8 +396,12 @@
     } else {
       Runtime::Current()->StartThreadBirth();
       self = new Thread(as_daemon);
-      self->Init(runtime->GetThreadList(), runtime->GetJavaVM());
+      bool init_success = self->Init(runtime->GetThreadList(), runtime->GetJavaVM());
       Runtime::Current()->EndThreadBirth();
+      if (!init_success) {
+        delete self;
+        return nullptr;
+      }
     }
   }
 
@@ -494,7 +504,7 @@
   Dbg::DdmSendThreadNotification(this, CHUNK_TYPE("THNM"));
 }
 
-void Thread::InitStackHwm() {
+bool Thread::InitStackHwm() {
   void* read_stack_base;
   size_t read_stack_size;
   size_t read_guard_size;
@@ -516,8 +526,10 @@
   uint32_t min_stack = GetStackOverflowReservedBytes(kRuntimeISA) + kStackOverflowProtectedSize
     + 4 * KB;
   if (read_stack_size <= min_stack) {
-    LOG(FATAL) << "Attempt to attach a thread with a too-small stack (" << read_stack_size
-               << " bytes)";
+    // Note, as we know the stack is small, avoid operations that could use a lot of stack.
+    LogMessage::LogLineLowStack(__PRETTY_FUNCTION__, __LINE__, ERROR,
+                                "Attempt to attach a thread with a too-small stack");
+    return false;
   }
 
   // Set stack_end_ to the bottom of the stack saving space of stack overflows
@@ -542,6 +554,8 @@
   // Sanity check.
   int stack_variable;
   CHECK_GT(&stack_variable, reinterpret_cast<void*>(tlsPtr_.stack_end));
+
+  return true;
 }
 
 void Thread::ShortDump(std::ostream& os) const {
@@ -1042,7 +1056,8 @@
   }
 
   // Allocate a TLS slot.
-  CHECK_PTHREAD_CALL(pthread_key_create, (&Thread::pthread_key_self_, Thread::ThreadExitCallback), "self key");
+  CHECK_PTHREAD_CALL(pthread_key_create, (&Thread::pthread_key_self_, Thread::ThreadExitCallback),
+                     "self key");
 
   // Double-check the TLS slot allocation.
   if (pthread_getspecific(pthread_key_self_) != nullptr) {
diff --git a/runtime/thread.h b/runtime/thread.h
index 89aee04..7e567fb 100644
--- a/runtime/thread.h
+++ b/runtime/thread.h
@@ -893,14 +893,14 @@
       SHARED_LOCKS_REQUIRED(Locks::mutator_lock_);
   void RemoveFromThreadGroup(ScopedObjectAccess& soa) SHARED_LOCKS_REQUIRED(Locks::mutator_lock_);
 
-  void Init(ThreadList*, JavaVMExt*) EXCLUSIVE_LOCKS_REQUIRED(Locks::runtime_shutdown_lock_);
+  bool Init(ThreadList*, JavaVMExt*) EXCLUSIVE_LOCKS_REQUIRED(Locks::runtime_shutdown_lock_);
   void InitCardTable();
   void InitCpu();
   void CleanupCpu();
   void InitTlsEntryPoints();
   void InitTid();
   void InitPthreadKeySelf();
-  void InitStackHwm();
+  bool InitStackHwm();
 
   void SetUpAlternateSignalStack();
   void TearDownAlternateSignalStack();