Use class unloading in dex2oat for verify and extract

Unload the main classloader in between each dex file compilation to
reduce RAM. This frees the whole java heap and associated linear
allocs. This is only used for quickening since filters that do
compilation may require loaded classes in the compiler and oat
writer.

This reduces dex2oat peak PSS for compiling a large app from 196MB
to 135MB.

Only works for verify and extract since the current approach is
incompatible with oat writer patching. b/63911263

Added a verification override that reads the compiled class status
to avoid ever verifying classes that were quickened (since this
is not supported and causes failures).

There is still some duplicated verification for some class with
superclasses in other dex files.

Support for quicken will be added in a follow up CL.

Bug: 63467744
Test: test-art-host
Test: test/testrunner/testrunner.py --interpreter --host -j40
Change-Id: Id0e4f84eb5db91d6143f752b498f4832a5b25b6e

15 files changed, 224 insertions, 92 deletions

diff --git a/compiler/dex/dex_to_dex_decompiler_test.cc b/compiler/dex/dex_to_dex_decompiler_test.cc
index 7b56f3ec1a..1ef3ba7c00 100644
--- a/compiler/dex/dex_to_dex_decompiler_test.cc
+++ b/compiler/dex/dex_to_dex_decompiler_test.cc
@@ -29,6 +29,7 @@
 #include "scoped_thread_state_change-inl.h"
 #include "thread.h"
 #include "verifier/method_verifier-inl.h"
+#include "verifier/verifier_deps.h"
 
 namespace art {
 
@@ -39,6 +40,11 @@ class DexToDexDecompilerTest : public CommonCompilerTest {
     TimingLogger::ScopedTiming t(__FUNCTION__, &timings);
     compiler_options_->boot_image_ = false;
     compiler_options_->SetCompilerFilter(CompilerFilter::kQuicken);
+    // Create the main VerifierDeps, here instead of in the compiler since we want to aggregate
+    // the results for all the dex files, not just the results for the current dex file.
+    Runtime::Current()->GetCompilerCallbacks()->SetVerifierDeps(
+        new verifier::VerifierDeps(GetDexFiles(class_loader)));
+    compiler_driver_->SetDexFilesForOatFile(GetDexFiles(class_loader));
     compiler_driver_->CompileAll(class_loader, GetDexFiles(class_loader), &timings);
   }
 
diff --git a/compiler/driver/compiler_driver.cc b/compiler/driver/compiler_driver.cc
index 756481df54..0b1bce62c9 100644
--- a/compiler/driver/compiler_driver.cc
+++ b/compiler/driver/compiler_driver.cc
@@ -303,7 +303,6 @@ CompilerDriver::CompilerDriver(
       timings_logger_(timer),
       compiler_context_(nullptr),
       support_boot_image_fixup_(true),
-      dex_files_for_oat_file_(nullptr),
       compiled_method_storage_(swap_fd),
       profile_compilation_info_(profile_compilation_info),
       max_arena_alloc_(0),
@@ -1915,8 +1914,8 @@ bool CompilerDriver::FastVerify(jobject jclass_loader,
                                 TimingLogger* timings) {
   verifier::VerifierDeps* verifier_deps =
       Runtime::Current()->GetCompilerCallbacks()->GetVerifierDeps();
-  // If there is an existing `VerifierDeps`, try to use it for fast verification.
-  if (verifier_deps == nullptr) {
+  // If there exist VerifierDeps that aren't the ones we just created to output, use them to verify.
+  if (verifier_deps == nullptr || verifier_deps->OutputOnly()) {
     return false;
   }
   TimingLogger::ScopedTiming t("Fast Verify", timings);
@@ -1983,13 +1982,6 @@ bool CompilerDriver::FastVerify(jobject jclass_loader,
 void CompilerDriver::Verify(jobject jclass_loader,
                             const std::vector<const DexFile*>& dex_files,
                             TimingLogger* timings) {
-  // Always add the dex files to compiled_classes_. This happens for all compiler filters.
-  for (const DexFile* dex_file : dex_files) {
-    if (!compiled_classes_.HaveDexFile(dex_file)) {
-      compiled_classes_.AddDexFile(dex_file, dex_file->NumClassDefs());
-    }
-  }
-
   if (FastVerify(jclass_loader, dex_files, timings)) {
     return;
   }
@@ -1999,14 +1991,16 @@ void CompilerDriver::Verify(jobject jclass_loader,
   // non boot image compilation. The verifier will need it to record the new dependencies.
   // Then dex2oat can update the vdex file with these new dependencies.
   if (!GetCompilerOptions().IsBootImage()) {
+    // Dex2oat creates the verifier deps.
     // Create the main VerifierDeps, and set it to this thread.
-    verifier::VerifierDeps* verifier_deps = new verifier::VerifierDeps(dex_files);
-    Runtime::Current()->GetCompilerCallbacks()->SetVerifierDeps(verifier_deps);
+    verifier::VerifierDeps* verifier_deps =
+        Runtime::Current()->GetCompilerCallbacks()->GetVerifierDeps();
+    CHECK(verifier_deps != nullptr);
     Thread::Current()->SetVerifierDeps(verifier_deps);
     // Create per-thread VerifierDeps to avoid contention on the main one.
     // We will merge them after verification.
     for (ThreadPoolWorker* worker : parallel_thread_pool_->GetWorkers()) {
-      worker->GetThread()->SetVerifierDeps(new verifier::VerifierDeps(dex_files));
+      worker->GetThread()->SetVerifierDeps(new verifier::VerifierDeps(dex_files_for_oat_file_));
     }
   }
 
@@ -2031,7 +2025,7 @@ void CompilerDriver::Verify(jobject jclass_loader,
     for (ThreadPoolWorker* worker : parallel_thread_pool_->GetWorkers()) {
       verifier::VerifierDeps* thread_deps = worker->GetThread()->GetVerifierDeps();
       worker->GetThread()->SetVerifierDeps(nullptr);
-      verifier_deps->MergeWith(*thread_deps, dex_files);;
+      verifier_deps->MergeWith(*thread_deps, dex_files_for_oat_file_);
       delete thread_deps;
     }
     Thread::Current()->SetVerifierDeps(nullptr);
@@ -2702,7 +2696,14 @@ void CompilerDriver::Compile(jobject class_loader,
             : profile_compilation_info_->DumpInfo(&dex_files));
   }
 
-  DCHECK(current_dex_to_dex_methods_ == nullptr);
+  current_dex_to_dex_methods_ = nullptr;
+  Thread* const self = Thread::Current();
+  {
+    // Clear in case we aren't the first call to Compile.
+    MutexLock mu(self, dex_to_dex_references_lock_);
+    dex_to_dex_references_.clear();
+  }
+
   for (const DexFile* dex_file : dex_files) {
     CHECK(dex_file != nullptr);
     CompileDexFile(class_loader,
@@ -2721,7 +2722,7 @@ void CompilerDriver::Compile(jobject class_loader,
   {
     // From this point on, we shall not modify dex_to_dex_references_, so
     // just grab a reference to it that we use without holding the mutex.
-    MutexLock lock(Thread::Current(), dex_to_dex_references_lock_);
+    MutexLock lock(self, dex_to_dex_references_lock_);
     dex_to_dex_references = ArrayRef<DexFileMethodSet>(dex_to_dex_references_);
   }
   for (const auto& method_set : dex_to_dex_references) {
@@ -2914,7 +2915,7 @@ void CompilerDriver::RecordClassStatus(ClassReference ref, mirror::Class::Status
       if (kIsDebugBuild) {
         // Check to make sure it's not a dex file for an oat file we are compiling since these
         // should always succeed. These do not include classes in for used libraries.
-        for (const DexFile* dex_file : *dex_files_for_oat_file_) {
+        for (const DexFile* dex_file : GetDexFilesForOatFile()) {
           CHECK_NE(dex_ref.dex_file, dex_file) << dex_ref.dex_file->GetLocation();
         }
       }
@@ -3032,4 +3033,13 @@ void CompilerDriver::FreeThreadPools() {
   single_thread_pool_.reset();
 }
 
+void CompilerDriver::SetDexFilesForOatFile(const std::vector<const DexFile*>& dex_files) {
+  dex_files_for_oat_file_ = dex_files;
+  for (const DexFile* dex_file : dex_files) {
+    if (!compiled_classes_.HaveDexFile(dex_file)) {
+      compiled_classes_.AddDexFile(dex_file, dex_file->NumClassDefs());
+    }
+  }
+}
+
 }  // namespace art
diff --git a/compiler/driver/compiler_driver.h b/compiler/driver/compiler_driver.h
index ecaed83e57..d9886a2fba 100644
--- a/compiler/driver/compiler_driver.h
+++ b/compiler/driver/compiler_driver.h
@@ -103,15 +103,11 @@ class CompilerDriver {
   ~CompilerDriver();
 
   // Set dex files that will be stored in the oat file after being compiled.
-  void SetDexFilesForOatFile(const std::vector<const DexFile*>& dex_files) {
-    dex_files_for_oat_file_ = &dex_files;
-  }
+  void SetDexFilesForOatFile(const std::vector<const DexFile*>& dex_files);
 
   // Get dex file that will be stored in the oat file after being compiled.
   ArrayRef<const DexFile* const> GetDexFilesForOatFile() const {
-    return (dex_files_for_oat_file_ != nullptr)
-        ? ArrayRef<const DexFile* const>(*dex_files_for_oat_file_)
-        : ArrayRef<const DexFile* const>();
+    return ArrayRef<const DexFile* const>(dex_files_for_oat_file_);
   }
 
   void CompileAll(jobject class_loader,
@@ -532,7 +528,7 @@ class CompilerDriver {
   bool support_boot_image_fixup_;
 
   // List of dex files that will be stored in the oat file.
-  const std::vector<const DexFile*>* dex_files_for_oat_file_;
+  std::vector<const DexFile*> dex_files_for_oat_file_;
 
   CompiledMethodStorage compiled_method_storage_;
 
diff --git a/compiler/driver/compiler_driver_test.cc b/compiler/driver/compiler_driver_test.cc
index 10bfd972f0..fee6afb91f 100644
--- a/compiler/driver/compiler_driver_test.cc
+++ b/compiler/driver/compiler_driver_test.cc
@@ -42,7 +42,9 @@ class CompilerDriverTest : public CommonCompilerTest {
   void CompileAll(jobject class_loader) REQUIRES(!Locks::mutator_lock_) {
     TimingLogger timings("CompilerDriverTest::CompileAll", false, false);
     TimingLogger::ScopedTiming t(__FUNCTION__, &timings);
-    compiler_driver_->CompileAll(class_loader, GetDexFiles(class_loader), &timings);
+    dex_files_ = GetDexFiles(class_loader);
+    compiler_driver_->SetDexFilesForOatFile(dex_files_);;
+    compiler_driver_->CompileAll(class_loader, dex_files_, &timings);
     t.NewTiming("MakeAllExecutable");
     MakeAllExecutable(class_loader);
   }
@@ -95,6 +97,7 @@ class CompilerDriverTest : public CommonCompilerTest {
   JNIEnv* env_;
   jclass class_;
   jmethodID mid_;
+  std::vector<const DexFile*> dex_files_;
 };
 
 // Disabled due to 10 second runtime on host
diff --git a/compiler/image_writer.cc b/compiler/image_writer.cc
index 318009c606..fc7cd016b2 100644
--- a/compiler/image_writer.cc
+++ b/compiler/image_writer.cc
@@ -894,7 +894,7 @@ bool ImageWriter::PruneAppImageClassInternal(
                                                 &my_early_exit,
                                                 visited);
   // Remove the class if the dex file is not in the set of dex files. This happens for classes that
-  // are from uses library if there is no profile. b/30688277
+  // are from uses-library if there is no profile. b/30688277
   mirror::DexCache* dex_cache = klass->GetDexCache();
   if (dex_cache != nullptr) {
     result = result ||
@@ -1153,9 +1153,22 @@ void ImageWriter::PruneNonImageClasses() {
   Thread* self = Thread::Current();
   ScopedAssertNoThreadSuspension sa(__FUNCTION__);
 
-  // Clear class table strong roots so that dex caches can get pruned. We require pruning the class
-  // path dex caches.
-  class_linker->ClearClassTableStrongRoots();
+  // Prune uses-library dex caches. Only prune the uses-library dex caches since we want to make
+  // sure the other ones don't get unloaded before the OatWriter runs.
+  class_linker->VisitClassTables(
+      [&](ClassTable* table) REQUIRES_SHARED(Locks::mutator_lock_) {
+    table->RemoveStrongRoots(
+        [&](GcRoot<mirror::Object> root) REQUIRES_SHARED(Locks::mutator_lock_) {
+      ObjPtr<mirror::Object> obj = root.Read();
+      if (obj->IsDexCache()) {
+        // Return true if the dex file is not one of the ones in the map.
+        return dex_file_oat_index_map_.find(obj->AsDexCache()->GetDexFile()) ==
+            dex_file_oat_index_map_.end();
+      }
+      // Return false to avoid removing.
+      return false;
+    });
+  });
 
   // Remove the undesired classes from the class roots.
   ObjPtr<mirror::ClassLoader> class_loader;
diff --git a/compiler/oat_writer.cc b/compiler/oat_writer.cc
index 4d258af843..d7e3a28777 100644
--- a/compiler/oat_writer.cc
+++ b/compiler/oat_writer.cc
@@ -1282,9 +1282,12 @@ class OatWriter::WriteCodeMethodVisitor : public OatDexMethodVisitor {
   bool StartClass(const DexFile* dex_file, size_t class_def_index) OVERRIDE
       REQUIRES_SHARED(Locks::mutator_lock_) {
     OatDexMethodVisitor::StartClass(dex_file, class_def_index);
-    if (dex_cache_ == nullptr || dex_cache_->GetDexFile() != dex_file) {
-      dex_cache_ = class_linker_->FindDexCache(Thread::Current(), *dex_file);
-      DCHECK(dex_cache_ != nullptr);
+    if (writer_->GetCompilerDriver()->GetCompilerOptions().IsAotCompilationEnabled()) {
+      // Only need to set the dex cache if we have compilation. Other modes might have unloaded it.
+      if (dex_cache_ == nullptr || dex_cache_->GetDexFile() != dex_file) {
+        dex_cache_ = class_linker_->FindDexCache(Thread::Current(), *dex_file);
+        DCHECK(dex_cache_ != nullptr);
+      }
     }
     return true;
   }
diff --git a/compiler/verifier_deps_test.cc b/compiler/verifier_deps_test.cc
index 72e2a6ce9f..e9f3f8022d 100644
--- a/compiler/verifier_deps_test.cc
+++ b/compiler/verifier_deps_test.cc
@@ -87,13 +87,13 @@ class VerifierDepsTest : public CommonCompilerTest {
     TimingLogger timings("Verify", false, false);
     // The compiler driver handles the verifier deps in the callbacks, so
     // remove what this class did for unit testing.
-    verifier_deps_.reset(nullptr);
+    if (deps == nullptr) {
+      // Create some verifier deps by default if they are not already specified.
+      deps = new verifier::VerifierDeps(dex_files_);
+      verifier_deps_.reset(deps);
+    }
     callbacks_->SetVerifierDeps(deps);
     compiler_driver_->Verify(class_loader_, dex_files_, &timings);
-    // The compiler driver may have updated the VerifierDeps in the callback object.
-    if (callbacks_->GetVerifierDeps() != deps) {
-      verifier_deps_.reset(callbacks_->GetVerifierDeps());
-    }
     callbacks_->SetVerifierDeps(nullptr);
     // Clear entries in the verification results to avoid hitting a DCHECK that
     // we always succeed inserting a new entry after verifying.
@@ -128,6 +128,7 @@ class VerifierDepsTest : public CommonCompilerTest {
     for (const DexFile* dex_file : dex_files_) {
       compiler_driver_->GetVerificationResults()->AddDexFile(dex_file);
     }
+    compiler_driver_->SetDexFilesForOatFile(dex_files_);
   }
 
   void LoadDexFile(ScopedObjectAccess* soa) REQUIRES_SHARED(Locks::mutator_lock_) {
@@ -1441,7 +1442,6 @@ TEST_F(VerifierDepsTest, CompilerDriver) {
         ASSERT_FALSE(verifier_deps_ == nullptr);
         ASSERT_FALSE(verifier_deps_->Equals(decoded_deps));
       } else {
-        ASSERT_TRUE(verifier_deps_ == nullptr);
         VerifyClassStatus(decoded_deps);
       }
     }
diff --git a/dex2oat/dex2oat.cc b/dex2oat/dex2oat.cc
index 3cc41a6b29..0826fa1488 100644
--- a/dex2oat/dex2oat.cc
+++ b/dex2oat/dex2oat.cc
@@ -65,8 +65,10 @@
 #include "elf_writer_quick.h"
 #include "gc/space/image_space.h"
 #include "gc/space/space-inl.h"
+#include "gc/verification.h"
 #include "image_writer.h"
 #include "interpreter/unstarted_runtime.h"
+#include "java_vm_ext.h"
 #include "jit/profile_compilation_info.h"
 #include "leb128.h"
 #include "linker/buffered_output_stream.h"
@@ -593,7 +595,6 @@ class Dex2Oat FINAL {
       passes_to_run_filename_(nullptr),
       multi_image_(false),
       is_host_(false),
-      class_loader_(nullptr),
       elf_writers_(),
       oat_writers_(),
       rodata_(),
@@ -1484,14 +1485,6 @@ class Dex2Oat FINAL {
     }
   }
 
-  void Shutdown() {
-    ScopedObjectAccess soa(Thread::Current());
-    for (jobject dex_cache : dex_caches_) {
-      soa.Env()->DeleteLocalRef(dex_cache);
-    }
-    dex_caches_.clear();
-  }
-
   void LoadClassProfileDescriptors() {
     if (profile_compilation_info_ != nullptr && IsImage()) {
       Runtime* runtime = Runtime::Current();
@@ -1660,6 +1653,8 @@ class Dex2Oat FINAL {
 
     // If we need to downgrade the compiler-filter for size reasons.
     if (!IsBootImage() && IsVeryLarge(dex_files_)) {
+      // If we need to downgrade the compiler-filter for size reasons, do that early before we read
+      // it below for creating verification callbacks.
       if (!CompilerFilter::IsAsGoodAs(kLargeAppFilter, compiler_options_->GetCompilerFilter())) {
         LOG(INFO) << "Very large app, downgrading to verify.";
         // Note: this change won't be reflected in the key-value store, as that had to be
@@ -1712,13 +1707,11 @@ class Dex2Oat FINAL {
     Thread* self = Thread::Current();
     WellKnownClasses::Init(self->GetJniEnv());
 
-    ClassLinker* const class_linker = Runtime::Current()->GetClassLinker();
     if (!IsBootImage()) {
       constexpr bool kSaveDexInput = false;
       if (kSaveDexInput) {
         SaveDexInput();
       }
-      class_loader_ = class_loader_context_->CreateClassLoader(dex_files_);
     }
 
     // Ensure opened dex files are writable for dex-to-dex transformations.
@@ -1729,24 +1722,12 @@ class Dex2Oat FINAL {
       }
     }
 
-    // Ensure that the dex caches stay live since we don't want class unloading
-    // to occur during compilation.
-    for (const auto& dex_file : dex_files_) {
-      ScopedObjectAccess soa(self);
-      dex_caches_.push_back(soa.AddLocalReference<jobject>(
-          class_linker->RegisterDexFile(*dex_file,
-                                        soa.Decode<mirror::ClassLoader>(class_loader_).Ptr())));
-      if (dex_caches_.back() == nullptr) {
-        soa.Self()->AssertPendingException();
-        soa.Self()->ClearException();
-        PLOG(ERROR) << "Failed to register dex file.";
-        return dex2oat::ReturnCode::kOther;
-      }
-      // Pre-register dex files so that we can access verification results without locks during
-      // compilation and verification.
-      if (verification_results_ != nullptr) {
-        // Verification results are only required for modes that have any compilation. Avoid
-        // adding the dex files if possible to prevent allocating large arrays.
+    // Verification results are only required for modes that have any compilation. Avoid
+    // adding the dex files if possible to prevent allocating large arrays.
+    if (verification_results_ != nullptr) {
+      for (const auto& dex_file : dex_files_) {
+        // Pre-register dex files so that we can access verification results without locks during
+        // compilation and verification.
         verification_results_->AddDexFile(dex_file);
       }
     }
@@ -1759,13 +1740,50 @@ class Dex2Oat FINAL {
     return IsImage() && oat_fd_ != kInvalidFd;
   }
 
-  // Create and invoke the compiler driver. This will compile all the dex files.
-  void Compile() {
+  // Doesn't return the class loader since it's not meant to be used for image compilation.
+  void CompileDexFilesIndividually() {
+    CHECK(!IsImage()) << "Not supported with image";
+    for (const DexFile* dex_file : dex_files_) {
+      std::vector<const DexFile*> dex_files(1u, dex_file);
+      VLOG(compiler) << "Compiling " << dex_file->GetLocation();
+      jobject class_loader = CompileDexFiles(dex_files);
+      CHECK(class_loader != nullptr);
+      ScopedObjectAccess soa(Thread::Current());
+      // Unload class loader to free RAM.
+      jweak weak_class_loader = soa.Env()->vm->AddWeakGlobalRef(
+          soa.Self(),
+          soa.Decode<mirror::ClassLoader>(class_loader));
+      soa.Env()->vm->DeleteGlobalRef(soa.Self(), class_loader);
+      runtime_->GetHeap()->CollectGarbage(/*clear_soft_references*/ true);
+      ObjPtr<mirror::ClassLoader> decoded_weak = soa.Decode<mirror::ClassLoader>(weak_class_loader);
+      if (decoded_weak != nullptr) {
+        LOG(FATAL) << "Failed to unload class loader, path from root set: "
+                   << runtime_->GetHeap()->GetVerification()->FirstPathFromRootSet(decoded_weak);
+      }
+      VLOG(compiler) << "Unloaded classloader";
+    }
+  }
+
+  bool ShouldCompileDexFilesIndividually() const {
+    // Compile individually if we are not building an image, not using any compilation, and are
+    // using multidex.
+    // This means extract, verify, and quicken will use the individual compilation mode (to reduce
+    // RAM used by the compiler).
+    // TODO: Still do it for app images to get testing coverage. Note that this will generate empty
+    // app images.
+    return !IsImage() &&
+        dex_files_.size() > 1 &&
+        !CompilerFilter::IsAnyCompilationEnabled(compiler_options_->GetCompilerFilter());
+  }
+
+  // Set up and create the compiler driver and then invoke it to compile all the dex files.
+  jobject Compile() {
+    ClassLinker* const class_linker = Runtime::Current()->GetClassLinker();
+
     TimingLogger::ScopedTiming t("dex2oat Compile", timings_);
     compiler_phases_timings_.reset(new CumulativeLogger("compilation times"));
 
     // Find the dex files we should not inline from.
-
     std::vector<std::string> no_inline_filters;
     Split(no_inline_from_string_, ',', &no_inline_filters);
 
@@ -1776,7 +1794,6 @@ class Dex2Oat FINAL {
     }
 
     if (!no_inline_filters.empty()) {
-      ClassLinker* class_linker = Runtime::Current()->GetClassLinker();
       std::vector<const DexFile*> class_path_files;
       if (!IsBootImage()) {
         // The class loader context is used only for apps.
@@ -1842,8 +1859,46 @@ class Dex2Oat FINAL {
       // experimentation.
       TimingLogger::ScopedTiming time_unquicken("Unquicken", timings_);
       VdexFile::Unquicken(dex_files_, input_vdex_file_->GetQuickeningInfo());
+    } else {
+      // Create the main VerifierDeps, here instead of in the compiler since we want to aggregate
+      // the results for all the dex files, not just the results for the current dex file.
+      callbacks_->SetVerifierDeps(new verifier::VerifierDeps(dex_files_));
+    }
+    // Invoke the compilation.
+    if (ShouldCompileDexFilesIndividually()) {
+      CompileDexFilesIndividually();
+      // Return a null classloader since we already freed released it.
+      return nullptr;
     }
-    driver_->CompileAll(class_loader_, dex_files_, timings_);
+    return CompileDexFiles(dex_files_);
+  }
+
+  // Create the class loader, use it to compile, and return.
+  jobject CompileDexFiles(const std::vector<const DexFile*>& dex_files) {
+    ClassLinker* const class_linker = Runtime::Current()->GetClassLinker();
+
+    jobject class_loader = nullptr;
+    if (!IsBootImage()) {
+      class_loader = class_loader_context_->CreateClassLoader(dex_files_);
+    }
+
+    // Register dex caches and key them to the class loader so that they only unload when the
+    // class loader unloads.
+    for (const auto& dex_file : dex_files) {
+      ScopedObjectAccess soa(Thread::Current());
+      // Registering the dex cache adds a strong root in the class loader that prevents the dex
+      // cache from being unloaded early.
+      ObjPtr<mirror::DexCache> dex_cache = class_linker->RegisterDexFile(
+          *dex_file,
+          soa.Decode<mirror::ClassLoader>(class_loader));
+      if (dex_cache == nullptr) {
+        soa.Self()->AssertPendingException();
+        LOG(FATAL) << "Failed to register dex file " << dex_file->GetLocation() << " "
+                   << soa.Self()->GetException()->Dump();
+      }
+    }
+    driver_->CompileAll(class_loader, dex_files, timings_);
+    return class_loader;
   }
 
   // Notes on the interleaving of creating the images and oat files to
@@ -2800,8 +2855,6 @@ class Dex2Oat FINAL {
   // Dex files we are compiling, does not include the class path dex files.
   std::vector<const DexFile*> dex_files_;
   std::string no_inline_from_string_;
-  std::vector<jobject> dex_caches_;
-  jobject class_loader_;
 
   std::vector<std::unique_ptr<ElfWriter>> elf_writers_;
   std::vector<std::unique_ptr<OatWriter>> oat_writers_;
@@ -2870,9 +2923,23 @@ static void b13564922() {
 #endif
 }
 
+class ScopedGlobalRef {
+ public:
+  explicit ScopedGlobalRef(jobject obj) : obj_(obj) {}
+  ~ScopedGlobalRef() {
+    if (obj_ != nullptr) {
+      ScopedObjectAccess soa(Thread::Current());
+      soa.Env()->vm->DeleteGlobalRef(soa.Self(), obj_);
+    }
+  }
+
+ private:
+  jobject obj_;
+};
+
 static dex2oat::ReturnCode CompileImage(Dex2Oat& dex2oat) {
   dex2oat.LoadClassProfileDescriptors();
-  dex2oat.Compile();
+  ScopedGlobalRef class_loader(dex2oat.Compile());
 
   if (!dex2oat.WriteOutputFiles()) {
     dex2oat.EraseOutputFiles();
@@ -2920,7 +2987,7 @@ static dex2oat::ReturnCode CompileImage(Dex2Oat& dex2oat) {
 }
 
 static dex2oat::ReturnCode CompileApp(Dex2Oat& dex2oat) {
-  dex2oat.Compile();
+  ScopedGlobalRef class_loader(dex2oat.Compile());
 
   if (!dex2oat.WriteOutputFiles()) {
     dex2oat.EraseOutputFiles();
@@ -3014,7 +3081,6 @@ static dex2oat::ReturnCode Dex2oat(int argc, char** argv) {
     result = CompileApp(*dex2oat);
   }
 
-  dex2oat->Shutdown();
   return result;
 }
 }  // namespace art
diff --git a/runtime/class_linker-inl.h b/runtime/class_linker-inl.h
index 0096c37b33..439ecaf28e 100644
--- a/runtime/class_linker-inl.h
+++ b/runtime/class_linker-inl.h
@@ -312,6 +312,17 @@ inline mirror::Class* ClassLinker::GetClassRoot(ClassRoot class_root) {
   return klass.Ptr();
 }
 
+template <class Visitor>
+inline void ClassLinker::VisitClassTables(const Visitor& visitor) {
+  Thread* const self = Thread::Current();
+  WriterMutexLock mu(self, *Locks::classlinker_classes_lock_);
+  for (const ClassLoaderData& data : class_loaders_) {
+    if (data.class_table != nullptr) {
+      visitor(data.class_table);
+    }
+  }
+}
+
 }  // namespace art
 
 #endif  // ART_RUNTIME_CLASS_LINKER_INL_H_
diff --git a/runtime/class_linker.cc b/runtime/class_linker.cc
index a26486768a..1219f6f39f 100644
--- a/runtime/class_linker.cc
+++ b/runtime/class_linker.cc
@@ -3555,6 +3555,14 @@ void ClassLinker::RegisterDexFileLocked(const DexFile& dex_file,
   data.resolved_methods = dex_cache->GetResolvedMethods();
   data.class_table = ClassTableForClassLoader(class_loader);
   DCHECK(data.class_table != nullptr);
+  // Make sure to hold the dex cache live in the class table. This case happens for the boot class
+  // path dex caches without an image.
+  data.class_table->InsertStrongRoot(dex_cache);
+  if (class_loader != nullptr) {
+    // Since we added a strong root to the class table, do the write barrier as required for
+    // remembered sets and generational GCs.
+    Runtime::Current()->GetHeap()->WriteBarrierEveryFieldOf(class_loader);
+  }
   dex_caches_.push_back(data);
 }
 
@@ -8839,16 +8847,6 @@ void ClassLinker::DropFindArrayClassCache() {
   find_array_class_cache_next_victim_ = 0;
 }
 
-void ClassLinker::ClearClassTableStrongRoots() const {
-  Thread* const self = Thread::Current();
-  WriterMutexLock mu(self, *Locks::classlinker_classes_lock_);
-  for (const ClassLoaderData& data : class_loaders_) {
-    if (data.class_table != nullptr) {
-      data.class_table->ClearStrongRoots();
-    }
-  }
-}
-
 void ClassLinker::VisitClassLoaders(ClassLoaderVisitor* visitor) const {
   Thread* const self = Thread::Current();
   for (const ClassLoaderData& data : class_loaders_) {
diff --git a/runtime/class_linker.h b/runtime/class_linker.h
index 324ed0c9bc..bf14aebb52 100644
--- a/runtime/class_linker.h
+++ b/runtime/class_linker.h
@@ -635,9 +635,9 @@ class ClassLinker {
   // Create the IMT and conflict tables for a class.
   void FillIMTAndConflictTables(ObjPtr<mirror::Class> klass) REQUIRES_SHARED(Locks::mutator_lock_);
 
-  // Clear class table strong roots (other than classes themselves). This is done by dex2oat to
-  // allow pruning dex caches.
-  void ClearClassTableStrongRoots() const
+  // Visit all of the class tables. This is used by dex2oat to allow pruning dex caches.
+  template <class Visitor>
+  void VisitClassTables(const Visitor& visitor)
       REQUIRES(!Locks::classlinker_classes_lock_)
       REQUIRES_SHARED(Locks::mutator_lock_);
 
diff --git a/runtime/class_table-inl.h b/runtime/class_table-inl.h
index b15d82f5e4..1280466a91 100644
--- a/runtime/class_table-inl.h
+++ b/runtime/class_table-inl.h
@@ -132,6 +132,13 @@ inline ClassTable::TableSlot::TableSlot(ObjPtr<mirror::Class> klass, uint32_t de
   }
 }
 
+template <typename Filter>
+inline void ClassTable::RemoveStrongRoots(const Filter& filter) {
+  WriterMutexLock mu(Thread::Current(), lock_);
+  strong_roots_.erase(std::remove_if(strong_roots_.begin(), strong_roots_.end(), filter),
+                      strong_roots_.end());
+}
+
 }  // namespace art
 
 #endif  // ART_RUNTIME_CLASS_TABLE_INL_H_
diff --git a/runtime/class_table.h b/runtime/class_table.h
index 8616dfba93..a259725399 100644
--- a/runtime/class_table.h
+++ b/runtime/class_table.h
@@ -250,6 +250,12 @@ class ClassTable {
       REQUIRES(!lock_)
       REQUIRES_SHARED(Locks::mutator_lock_);
 
+  // Filter strong roots (other than classes themselves).
+  template <typename Filter>
+  void RemoveStrongRoots(const Filter& filter)
+      REQUIRES(!lock_)
+      REQUIRES_SHARED(Locks::mutator_lock_);
+
   ReaderWriterMutex& GetLock() {
     return lock_;
   }
diff --git a/runtime/verifier/verifier_deps.cc b/runtime/verifier/verifier_deps.cc
index 112eec847d..470b0b3d58 100644
--- a/runtime/verifier/verifier_deps.cc
+++ b/runtime/verifier/verifier_deps.cc
@@ -33,7 +33,8 @@
 namespace art {
 namespace verifier {
 
-VerifierDeps::VerifierDeps(const std::vector<const DexFile*>& dex_files) {
+VerifierDeps::VerifierDeps(const std::vector<const DexFile*>& dex_files, bool output_only)
+    : output_only_(output_only) {
   for (const DexFile* dex_file : dex_files) {
     DCHECK(GetDexFileDeps(*dex_file) == nullptr);
     std::unique_ptr<DexFileDeps> deps(new DexFileDeps());
@@ -41,6 +42,9 @@ VerifierDeps::VerifierDeps(const std::vector<const DexFile*>& dex_files) {
   }
 }
 
+VerifierDeps::VerifierDeps(const std::vector<const DexFile*>& dex_files)
+    : VerifierDeps(dex_files, /*output_only*/ true) {}
+
 void VerifierDeps::MergeWith(const VerifierDeps& other,
                              const std::vector<const DexFile*>& dex_files) {
   DCHECK(dex_deps_.size() == other.dex_deps_.size());
@@ -694,7 +698,7 @@ void VerifierDeps::Encode(const std::vector<const DexFile*>& dex_files,
 
 VerifierDeps::VerifierDeps(const std::vector<const DexFile*>& dex_files,
                            ArrayRef<const uint8_t> data)
-    : VerifierDeps(dex_files) {
+    : VerifierDeps(dex_files, /*output_only*/ false) {
   if (data.empty()) {
     // Return eagerly, as the first thing we expect from VerifierDeps data is
     // the number of created strings, even if there is no dependency.
diff --git a/runtime/verifier/verifier_deps.h b/runtime/verifier/verifier_deps.h
index b883a9e642..2d452f6d6b 100644
--- a/runtime/verifier/verifier_deps.h
+++ b/runtime/verifier/verifier_deps.h
@@ -121,6 +121,10 @@ class VerifierDeps {
     return GetDexFileDeps(dex_file)->unverified_classes_;
   }
 
+  bool OutputOnly() const {
+    return output_only_;
+  }
+
  private:
   static constexpr uint16_t kUnresolvedMarker = static_cast<uint16_t>(-1);
 
@@ -198,6 +202,8 @@ class VerifierDeps {
     bool Equals(const DexFileDeps& rhs) const;
   };
 
+  VerifierDeps(const std::vector<const DexFile*>& dex_files, bool output_only);
+
   // Finds the DexFileDep instance associated with `dex_file`, or nullptr if
   // `dex_file` is not reported as being compiled.
   DexFileDeps* GetDexFileDeps(const DexFile& dex_file);
@@ -321,6 +327,9 @@ class VerifierDeps {
   // Map from DexFiles into dependencies collected from verification of their methods.
   std::map<const DexFile*, std::unique_ptr<DexFileDeps>> dex_deps_;
 
+  // Output only signifies if we are using the verifier deps to verify or just to generate them.
+  const bool output_only_;
+
   friend class VerifierDepsTest;
   ART_FRIEND_TEST(VerifierDepsTest, StringToId);
   ART_FRIEND_TEST(VerifierDepsTest, EncodeDecode);