Fix boot image extension class exclusion.

The code in CompilerDriver was too permissive for arrays and
the code in Transaction was not even checking interfaces.
Move the check to AotClassLinker and fix it.

Test: m test-art-host-gtest
Test: testrunner.py --host --optimizing
Change-Id: I3400b6c0e212e25acf17e3740ba19a8b407e03d3

4 files changed, 84 insertions, 79 deletions

diff --git a/dex2oat/driver/compiler_driver.cc b/dex2oat/driver/compiler_driver.cc
index c3801f4a96..99895b14a3 100644
--- a/dex2oat/driver/compiler_driver.cc
+++ b/dex2oat/driver/compiler_driver.cc
@@ -29,6 +29,7 @@
 #include "android-base/logging.h"
 #include "android-base/strings.h"
 
+#include "aot_class_linker.h"
 #include "art_field-inl.h"
 #include "art_method-inl.h"
 #include "base/arena_allocator.h"
@@ -1283,66 +1284,7 @@ class ClinitImageUpdate {
     if (heap->ObjectIsInBootImageSpace(klass)) {
       return false;  // Already included in the boot image we're compiling against.
     }
-
-    // When compiling a boot image extension, we must not include classes
-    // that are defined in dex files belonging to the boot image we're
-    // compiling against but not actually present in that boot image.
-
-    // Treat arrays and primitive types specially because they do not have a DexCache that we
-    // can use to check whether the dex file belongs to the boot image we're compiling against.
-    DCHECK(!klass->IsPrimitive());  // Primitive classes must be in the primary boot image.
-    if (klass->IsArrayClass()) {
-      DCHECK(heap->ObjectIsInBootImageSpace(klass->GetIfTable()));  // IfTable is OK.
-      // Arrays of all dimensions are tied to the dex file of the non-array component type.
-      ObjPtr<mirror::Class> component_type = klass;
-      do {
-        component_type = component_type->GetComponentType();
-      } while (component_type->IsArrayClass());
-      return !component_type->IsPrimitive() &&
-             !heap->ObjectIsInBootImageSpace(component_type->GetDexCache());
-    }
-
-    // Check the class itself.
-    if (heap->ObjectIsInBootImageSpace(klass->GetDexCache())) {
-      return false;
-    }
-
-    // Check superclasses.
-    ObjPtr<mirror::Class> superclass = klass->GetSuperClass();
-    while (!heap->ObjectIsInBootImageSpace(superclass)) {
-      DCHECK(superclass != nullptr);  // Cannot skip Object which is in the primary boot image.
-      if (heap->ObjectIsInBootImageSpace(superclass->GetDexCache())) {
-        return false;
-      }
-      superclass = superclass->GetSuperClass();
-    }
-
-    // Check IfTable. This includes direct and indirect interfaces.
-    ObjPtr<mirror::IfTable> if_table = klass->GetIfTable();
-    for (size_t i = 0, num_interfaces = klass->GetIfTableCount(); i < num_interfaces; ++i) {
-      ObjPtr<mirror::Class> interface = if_table->GetInterface(i);
-      DCHECK(interface != nullptr);
-      if (!heap->ObjectIsInBootImageSpace(interface) &&
-          heap->ObjectIsInBootImageSpace(interface->GetDexCache())) {
-        return false;
-      }
-    }
-
-    if (kIsDebugBuild) {
-      // All virtual methods must come from classes we have already checked above.
-      PointerSize pointer_size = Runtime::Current()->GetClassLinker()->GetImagePointerSize();
-      ObjPtr<mirror::Class> k = klass;
-      while (!heap->ObjectIsInBootImageSpace(k)) {
-        for (auto& m : k->GetVirtualMethods(pointer_size)) {
-          ObjPtr<mirror::Class> declaring_class = m.GetDeclaringClass();
-          CHECK(heap->ObjectIsInBootImageSpace(declaring_class) ||
-                !heap->ObjectIsInBootImageSpace(declaring_class->GetDexCache()));
-        }
-        k = k->GetSuperClass();
-      }
-    }
-
-    return true;
+    return AotClassLinker::CanReferenceInBootImageExtension(klass, heap);
   }
 
   mutable VariableSizedHandleScope hs_;
diff --git a/runtime/aot_class_linker.cc b/runtime/aot_class_linker.cc
index d0ceaf77ef..77faa2d2b9 100644
--- a/runtime/aot_class_linker.cc
+++ b/runtime/aot_class_linker.cc
@@ -19,6 +19,7 @@
 #include "class_status.h"
 #include "compiler_callbacks.h"
 #include "dex/class_reference.h"
+#include "gc/heap.h"
 #include "handle_scope-inl.h"
 #include "mirror/class-inl.h"
 #include "runtime.h"
@@ -124,4 +125,75 @@ verifier::FailureKind AotClassLinker::PerformClassVerification(Thread* self,
   return ClassLinker::PerformClassVerification(self, klass, log_level, error_msg);
 }
 
+bool AotClassLinker::CanReferenceInBootImageExtension(ObjPtr<mirror::Class> klass, gc::Heap* heap) {
+  // Do not allow referencing a class or instance of a class defined in a dex file
+  // belonging to the boot image we're compiling against but not itself in the boot image;
+  // or a class referencing such classes as component type, superclass or interface.
+  // Allowing this could yield duplicate class objects from multiple extensions.
+
+  if (heap->ObjectIsInBootImageSpace(klass)) {
+    return true;  // Already included in the boot image we're compiling against.
+  }
+
+  // Treat arrays and primitive types specially because they do not have a DexCache that we
+  // can use to check whether the dex file belongs to the boot image we're compiling against.
+  DCHECK(!klass->IsPrimitive());  // Primitive classes must be in the primary boot image.
+  if (klass->IsArrayClass()) {
+    DCHECK(heap->ObjectIsInBootImageSpace(klass->GetIfTable()));  // IfTable is OK.
+    // Arrays of all dimensions are tied to the dex file of the non-array component type.
+    do {
+      klass = klass->GetComponentType();
+    } while (klass->IsArrayClass());
+    if (klass->IsPrimitive()) {
+      return false;
+    }
+    // Do not allow arrays of erroneous classes (the array class is not itself erroneous).
+    if (klass->IsErroneous()) {
+      return false;
+    }
+  }
+
+  // Check the class itself.
+  if (heap->ObjectIsInBootImageSpace(klass->GetDexCache())) {
+    return false;
+  }
+
+  // Check superclasses.
+  ObjPtr<mirror::Class> superclass = klass->GetSuperClass();
+  while (!heap->ObjectIsInBootImageSpace(superclass)) {
+    DCHECK(superclass != nullptr);  // Cannot skip Object which is in the primary boot image.
+    if (heap->ObjectIsInBootImageSpace(superclass->GetDexCache())) {
+      return false;
+    }
+    superclass = superclass->GetSuperClass();
+  }
+
+  // Check IfTable. This includes direct and indirect interfaces.
+  ObjPtr<mirror::IfTable> if_table = klass->GetIfTable();
+  for (size_t i = 0, num_interfaces = klass->GetIfTableCount(); i < num_interfaces; ++i) {
+    ObjPtr<mirror::Class> interface = if_table->GetInterface(i);
+    DCHECK(interface != nullptr);
+    if (!heap->ObjectIsInBootImageSpace(interface) &&
+        heap->ObjectIsInBootImageSpace(interface->GetDexCache())) {
+      return false;
+    }
+  }
+
+  if (kIsDebugBuild) {
+    // All virtual methods must come from classes we have already checked above.
+    PointerSize pointer_size = Runtime::Current()->GetClassLinker()->GetImagePointerSize();
+    ObjPtr<mirror::Class> k = klass;
+    while (!heap->ObjectIsInBootImageSpace(k)) {
+      for (auto& m : k->GetVirtualMethods(pointer_size)) {
+        ObjPtr<mirror::Class> declaring_class = m.GetDeclaringClass();
+        CHECK(heap->ObjectIsInBootImageSpace(declaring_class) ||
+              !heap->ObjectIsInBootImageSpace(declaring_class->GetDexCache()));
+      }
+      k = k->GetSuperClass();
+    }
+  }
+
+  return true;
+}
+
 }  // namespace art
diff --git a/runtime/aot_class_linker.h b/runtime/aot_class_linker.h
index 74da33d777..2b50c46b8c 100644
--- a/runtime/aot_class_linker.h
+++ b/runtime/aot_class_linker.h
@@ -20,6 +20,11 @@
 #include "class_linker.h"
 
 namespace art {
+
+namespace gc {
+class Heap;
+}  // namespace gc
+
 // AotClassLinker is only used for AOT compiler, which includes some logic for class initialization
 // which will only be used in pre-compilation.
 class AotClassLinker : public ClassLinker {
@@ -27,6 +32,9 @@ class AotClassLinker : public ClassLinker {
   explicit AotClassLinker(InternTable *intern_table);
   ~AotClassLinker();
 
+  static bool CanReferenceInBootImageExtension(ObjPtr<mirror::Class> klass, gc::Heap* heap)
+      REQUIRES_SHARED(Locks::mutator_lock_);
+
  protected:
   // Overridden version of PerformClassVerification allows skipping verification if the class was
   // previously verified but unloaded.
diff --git a/runtime/transaction.cc b/runtime/transaction.cc
index 47e59a9677..46bbea33e0 100644
--- a/runtime/transaction.cc
+++ b/runtime/transaction.cc
@@ -18,6 +18,7 @@
 
 #include <android-base/logging.h>
 
+#include "aot_class_linker.h"
 #include "base/mutex-inl.h"
 #include "base/stl_util.h"
 #include "gc/accounting/card_table-inl.h"
@@ -141,26 +142,8 @@ bool Transaction::WriteValueConstraint(Thread* self, ObjPtr<mirror::Object> valu
     return false;  // No constraints for boot image.
   } else {
     // Boot image extension.
-    // Do not allow storing references to a class or instances of a class defined in a dex file
-    // belonging to the boot image we're compiling against but not itself in the boot image.
-    // Allowing this could yield duplicate class objects from multiple extensions.
-    if (heap->ObjectIsInBootImageSpace(value)) {
-      return false;  // References to boot image objects are OK.
-    }
     ObjPtr<mirror::Class> klass = value->IsClass() ? value->AsClass() : value->GetClass();
-    if (!value->IsClass() && heap->ObjectIsInBootImageSpace(klass)) {
-      // Instances of boot image classes are OK.
-      DCHECK(klass->IsInitialized());
-      return false;
-    }
-    // For arrays we need to determine the dex file based on the element type.
-    while (klass->IsArrayClass()) {
-      klass = klass->GetComponentType();
-    }
-    if (klass->IsPrimitive() || heap->ObjectIsInBootImageSpace(klass->GetDexCache())) {
-      return true;  // Boot image dex file but not boot image `klass`.
-    }
-    return false;
+    return !AotClassLinker::CanReferenceInBootImageExtension(klass, heap);
   }
 }