Fix calling instance methods of erroneous classes.

Fix the resolution trampoline to correctly handle the edge
case of calling instance methods on instances of erroneous
classes. The new test case would have previously failed the
`CHECK_EQ(code == nullptr, self->IsExceptionPending())`
because the trampoline would not retrieve the code pointer
for the direct method of an erroneous class.

Test: Added test to 174-escaping-instance-of-bad-class.
Test: m test-art-host-gtest
Test: testrunner.py --host --optimizing
Test: aosp_taimen-userdebug boots.
Bug: 62478025
Change-Id: Idf54a010f362c6f581a4c4aa27c33b6dc3ce6f69

3 files changed, 20 insertions, 3 deletions

diff --git a/runtime/entrypoints/quick/quick_trampoline_entrypoints.cc b/runtime/entrypoints/quick/quick_trampoline_entrypoints.cc
index d237586797..b597518474 100644
--- a/runtime/entrypoints/quick/quick_trampoline_entrypoints.cc
+++ b/runtime/entrypoints/quick/quick_trampoline_entrypoints.cc
@@ -1374,15 +1374,18 @@ extern "C" const void* artQuickResolutionTrampoline(
                                << invoke_type << " " << orig_called->GetVtableIndex();
     }
 
+    // Static invokes need class initialization check but instance invokes can proceed even if
+    // the class is erroneous, i.e. in the edge case of escaping instances of erroneous classes.
+    bool success = true;
     ObjPtr<mirror::Class> called_class = called->GetDeclaringClass();
     if (NeedsClinitCheckBeforeCall(called) && !called_class->IsVisiblyInitialized()) {
       // Ensure that the called method's class is initialized.
       StackHandleScope<1> hs(soa.Self());
       HandleWrapperObjPtr<mirror::Class> h_called_class(hs.NewHandleWrapper(&called_class));
-      linker->EnsureInitialized(soa.Self(), h_called_class, true, true);
+      success = linker->EnsureInitialized(soa.Self(), h_called_class, true, true);
     }
-    bool force_interpreter = self->IsForceInterpreter() && !called->IsNative();
-    if (called_class->IsInitialized() || called_class->IsInitializing()) {
+    if (success) {
+      bool force_interpreter = self->IsForceInterpreter() && !called->IsNative();
       if (UNLIKELY(force_interpreter)) {
         // If we are single-stepping or the called method is deoptimized (by a
         // breakpoint, for example), then we have to execute the called method
@@ -1398,6 +1401,7 @@ extern "C" const void* artQuickResolutionTrampoline(
       }
     } else {
       DCHECK(called_class->IsErroneous());
+      DCHECK(self->IsExceptionPending());
     }
   }
   CHECK_EQ(code == nullptr, self->IsExceptionPending());
diff --git a/test/174-escaping-instance-of-bad-class/expected.txt b/test/174-escaping-instance-of-bad-class/expected.txt
index 611d698f1a..4da6a09999 100644
--- a/test/174-escaping-instance-of-bad-class/expected.txt
+++ b/test/174-escaping-instance-of-bad-class/expected.txt
@@ -3,6 +3,7 @@ Bad.instanceValue = 33
 Caught NoClassDefFoundError.
 Bad.bar()
 Caught NoClassDefFoundError.
+Bad.$noinline$testResolutionTrampolineCallee()
 BadSuper.foo()
 BadSuper.superInstanceValue = 1
 Caught NoClassDefFoundError.
diff --git a/test/174-escaping-instance-of-bad-class/src/Main.java b/test/174-escaping-instance-of-bad-class/src/Main.java
index 4f66a31837..cee978ad85 100644
--- a/test/174-escaping-instance-of-bad-class/src/Main.java
+++ b/test/174-escaping-instance-of-bad-class/src/Main.java
@@ -50,6 +50,9 @@ public class Main {
         ncdfe.printStackTrace();
       }
     }
+
+    // Test that we handle bad instance correctly in the resolution trampoline.
+    bad.$noinline$testResolutionTrampoline();
   }
 
   public static void hierarchyTest() {
@@ -141,6 +144,15 @@ class Bad {
       return Bad.staticValue;
     }
   }
+
+  public void $noinline$testResolutionTrampoline() {
+    // The first call to private method uses the resolution trampoline when AOT-compiled.
+    $noinline$testResolutionTrampolineCallee();
+  }
+
+  private void $noinline$testResolutionTrampolineCallee() {
+    System.out.println("Bad.$noinline$testResolutionTrampolineCallee()");
+  }
 }
 
 class BadSuper {