commit a35510de5aa1ec5d39c6870da556e2d60666cdc3
author Vladimir Marko <vmarko@google.com> Mon Jan 16 22:42:09 2017 +0000
committer Vladimir Marko <vmarko@google.com> Mon Jan 16 23:03:24 2017 +0000
tree 65e2c9f63cbe409b2fe7a48194a69ac4ecca6dde
parent 0d478f289f0e33f19693d135f1d562b57427ed32

Fix missing OOME check in reflection.

Test: 080-oom-throw
Change-Id: I5f18ea756e815978bf8682555470505959a8fad5

runtime/native/java_lang_Class.cc

diff --git a/runtime/native/java_lang_Class.cc b/runtime/native/java_lang_Class.cc
index 3341f53..5438a6d 100644
--- a/runtime/native/java_lang_Class.cc
+++ b/runtime/native/java_lang_Class.cc
@@ -428,6 +428,10 @@
   }
   auto ret = hs.NewHandle(mirror::ObjectArray<mirror::Method>::Alloc(
       soa.Self(), mirror::Method::ArrayClass(), num_methods));
+  if (ret.Get() == nullptr) {
+    soa.Self()->AssertPendingOOMException();
+    return nullptr;
+  }
   num_methods = 0;
   for (auto& m : klass->GetDeclaredMethods(kRuntimePointerSize)) {
     auto modifiers = m.GetAccessFlags();

test/080-oom-throw/expected.txt

diff --git a/test/080-oom-throw/expected.txt b/test/080-oom-throw/expected.txt
index 904393b..0967278 100644
--- a/test/080-oom-throw/expected.txt
+++ b/test/080-oom-throw/expected.txt
@@ -1,3 +1,4 @@
 Test reflection correctly threw
+Test reflection2 correctly threw
 NEW_ARRAY correctly threw OOME
 NEW_INSTANCE correctly threw OOME

test/080-oom-throw/src/Main.java

diff --git a/test/080-oom-throw/src/Main.java b/test/080-oom-throw/src/Main.java
index 0ae92a9..a6c18b7 100644
--- a/test/080-oom-throw/src/Main.java
+++ b/test/080-oom-throw/src/Main.java
@@ -53,6 +53,30 @@
         }
     }
 
+    public static Object eatAllMemory() {
+        Object[] result = null;
+        int size = 1000000;
+        while (result == null && size != 0) {
+            try {
+                result = new Object[size];
+            } catch (OutOfMemoryError oome) {
+                size /= 2;
+            }
+        }
+        if (result != null) {
+            int index = 0;
+            while (index != result.length && size != 0) {
+                try {
+                    result[index] = new byte[size];
+                    ++index;
+                } catch (OutOfMemoryError oome) {
+                    size /= 2;
+                }
+            }
+        }
+        return result;
+    }
+
     static boolean triggerArrayOOM() {
         ArrayMemEater.blowup(new char[128 * 1024][]);
         return ArrayMemEater.sawOome;
@@ -74,6 +98,9 @@
         if (triggerReflectionOOM()) {
             System.out.println("Test reflection correctly threw");
         }
+        if (triggerReflectionOOM2()) {
+            System.out.println("Test reflection2 correctly threw");
+        }
 
         if (triggerArrayOOM()) {
             System.out.println("NEW_ARRAY correctly threw OOME");
@@ -125,4 +152,20 @@
         }
         return true;
     }
+
+    static boolean triggerReflectionOOM2() {
+        Object memory = eatAllMemory();
+        boolean result = false;
+        try {
+            Main.class.getDeclaredMethods();
+        } catch (OutOfMemoryError e) {
+            result = true;
+        }
+        if (!result) {
+            boolean memoryWasAllocated = (memory != null);
+            memory = null;
+            System.out.println("memoryWasAllocated = " + memoryWasAllocated);
+        }
+        return result;
+    }
 }