ART: Add IRT table size limit and overflow checks

Ensure that we don't waste too much space on tables (currently allow
128MB). Also be defensive and check for overflow when creating or
resizing.

Bug: 62223672
Test: m test-art-host
Change-Id: I60468a79d7d9dcb54767900323c2c50e79df35f3

2 files changed, 35 insertions, 0 deletions

diff --git a/runtime/indirect_reference_table.cc b/runtime/indirect_reference_table.cc
index c852d5af3a..cd8f84ada1 100644
--- a/runtime/indirect_reference_table.cc
+++ b/runtime/indirect_reference_table.cc
@@ -34,6 +34,9 @@ namespace art {
 static constexpr bool kDumpStackOnNonLocalReference = false;
 static constexpr bool kDebugIRT = false;
 
+// Maximum table size we allow.
+static constexpr size_t kMaxTableSizeInBytes = 128 * MB;
+
 const char* GetIndirectRefKindString(const IndirectRefKind& kind) {
   switch (kind) {
     case kHandleScopeOrInvalid:
@@ -71,6 +74,9 @@ IndirectReferenceTable::IndirectReferenceTable(size_t max_count,
   CHECK(error_msg != nullptr);
   CHECK_NE(desired_kind, kHandleScopeOrInvalid);
 
+  // Overflow and maximum check.
+  CHECK_LE(max_count, kMaxTableSizeInBytes / sizeof(IrtEntry));
+
   const size_t table_bytes = max_count * sizeof(IrtEntry);
   table_mem_map_.reset(MemMap::MapAnonymous("indirect ref table", nullptr, table_bytes,
                                             PROT_READ | PROT_WRITE, false, false, error_msg));
@@ -203,6 +209,13 @@ static inline void CheckHoleCount(IrtEntry* table,
 bool IndirectReferenceTable::Resize(size_t new_size, std::string* error_msg) {
   CHECK_GT(new_size, max_entries_);
 
+  constexpr size_t kMaxEntries = kMaxTableSizeInBytes / sizeof(IrtEntry);
+  if (new_size > kMaxEntries) {
+    *error_msg = android::base::StringPrintf("Requested size exceeds maximum: %zu", new_size);
+    return false;
+  }
+  // Note: the above check also ensures that there is no overflow below.
+
   const size_t table_bytes = new_size * sizeof(IrtEntry);
   std::unique_ptr<MemMap> new_map(MemMap::MapAnonymous("indirect ref table",
                                                        nullptr,
@@ -247,6 +260,14 @@ IndirectRef IndirectReferenceTable::Add(IRTSegmentState previous_state,
     }
 
     // Try to double space.
+    if (std::numeric_limits<size_t>::max() / 2 < max_entries_) {
+      LOG(FATAL) << "JNI ERROR (app bug): " << kind_ << " table overflow "
+                 << "(max=" << max_entries_ << ")" << std::endl
+                 << MutatorLockedDumpable<IndirectReferenceTable>(*this)
+                << " Resizing failed: exceeds size_t";
+      UNREACHABLE();
+    }
+
     std::string error_msg;
     if (!Resize(max_entries_ * 2, &error_msg)) {
       LOG(FATAL) << "JNI ERROR (app bug): " << kind_ << " table overflow "
diff --git a/runtime/jni_internal_test.cc b/runtime/jni_internal_test.cc
index 08d1eeb95d..3e9907dc73 100644
--- a/runtime/jni_internal_test.cc
+++ b/runtime/jni_internal_test.cc
@@ -1962,6 +1962,20 @@ TEST_F(JniInternalTest, PushLocalFrame_PopLocalFrame) {
   check_jni_abort_catcher.Check("use of deleted local reference");
 }
 
+TEST_F(JniInternalTest, PushLocalFrame_LimitAndOverflow) {
+  // Try a very large value that should fail.
+  ASSERT_NE(JNI_OK, env_->PushLocalFrame(std::numeric_limits<jint>::max()));
+
+  // On 32-bit, also check for some overflow conditions.
+#ifndef __LP64__
+  ASSERT_EQ(JNI_OK, env_->PushLocalFrame(10));
+  ASSERT_NE(JNI_OK, env_->PushLocalFrame(std::numeric_limits<jint>::max() - 10));
+  ASSERT_TRUE(env_->ExceptionCheck());
+  env_->ExceptionClear();
+  EXPECT_EQ(env_->PopLocalFrame(nullptr), nullptr);
+#endif
+}
+
 TEST_F(JniInternalTest, NewGlobalRef_nullptr) {
   EXPECT_EQ(env_->NewGlobalRef(nullptr), nullptr);
 }