Use class unloading in dex2oat for verify and extract
Unload the main classloader in between each dex file compilation to
reduce RAM. This frees the whole java heap and associated linear
allocs. This is only used for quickening since filters that do
compilation may require loaded classes in the compiler and oat
writer.
This reduces dex2oat peak PSS for compiling a large app from 196MB
to 135MB.
Only works for verify and extract since the current approach is
incompatible with oat writer patching. b/63911263
Added a verification override that reads the compiled class status
to avoid ever verifying classes that were quickened (since this
is not supported and causes failures).
There is still some duplicated verification for some class with
superclasses in other dex files.
Support for quicken will be added in a follow up CL.
Bug: 63467744
Test: test-art-host
Test: test/testrunner/testrunner.py --interpreter --host -j40
Change-Id: Id0e4f84eb5db91d6143f752b498f4832a5b25b6e
diff --git a/compiler/dex/dex_to_dex_decompiler_test.cc b/compiler/dex/dex_to_dex_decompiler_test.cc
index 7b56f3e..1ef3ba7 100644
--- a/compiler/dex/dex_to_dex_decompiler_test.cc
+++ b/compiler/dex/dex_to_dex_decompiler_test.cc
@@ -29,6 +29,7 @@
#include "scoped_thread_state_change-inl.h"
#include "thread.h"
#include "verifier/method_verifier-inl.h"
+#include "verifier/verifier_deps.h"
namespace art {
@@ -39,6 +40,11 @@
TimingLogger::ScopedTiming t(__FUNCTION__, &timings);
compiler_options_->boot_image_ = false;
compiler_options_->SetCompilerFilter(CompilerFilter::kQuicken);
+ // Create the main VerifierDeps, here instead of in the compiler since we want to aggregate
+ // the results for all the dex files, not just the results for the current dex file.
+ Runtime::Current()->GetCompilerCallbacks()->SetVerifierDeps(
+ new verifier::VerifierDeps(GetDexFiles(class_loader)));
+ compiler_driver_->SetDexFilesForOatFile(GetDexFiles(class_loader));
compiler_driver_->CompileAll(class_loader, GetDexFiles(class_loader), &timings);
}
diff --git a/compiler/driver/compiler_driver.cc b/compiler/driver/compiler_driver.cc
index 756481d..0b1bce6 100644
--- a/compiler/driver/compiler_driver.cc
+++ b/compiler/driver/compiler_driver.cc
@@ -303,7 +303,6 @@
timings_logger_(timer),
compiler_context_(nullptr),
support_boot_image_fixup_(true),
- dex_files_for_oat_file_(nullptr),
compiled_method_storage_(swap_fd),
profile_compilation_info_(profile_compilation_info),
max_arena_alloc_(0),
@@ -1915,8 +1914,8 @@
TimingLogger* timings) {
verifier::VerifierDeps* verifier_deps =
Runtime::Current()->GetCompilerCallbacks()->GetVerifierDeps();
- // If there is an existing `VerifierDeps`, try to use it for fast verification.
- if (verifier_deps == nullptr) {
+ // If there exist VerifierDeps that aren't the ones we just created to output, use them to verify.
+ if (verifier_deps == nullptr || verifier_deps->OutputOnly()) {
return false;
}
TimingLogger::ScopedTiming t("Fast Verify", timings);
@@ -1983,13 +1982,6 @@
void CompilerDriver::Verify(jobject jclass_loader,
const std::vector<const DexFile*>& dex_files,
TimingLogger* timings) {
- // Always add the dex files to compiled_classes_. This happens for all compiler filters.
- for (const DexFile* dex_file : dex_files) {
- if (!compiled_classes_.HaveDexFile(dex_file)) {
- compiled_classes_.AddDexFile(dex_file, dex_file->NumClassDefs());
- }
- }
-
if (FastVerify(jclass_loader, dex_files, timings)) {
return;
}
@@ -1999,14 +1991,16 @@
// non boot image compilation. The verifier will need it to record the new dependencies.
// Then dex2oat can update the vdex file with these new dependencies.
if (!GetCompilerOptions().IsBootImage()) {
+ // Dex2oat creates the verifier deps.
// Create the main VerifierDeps, and set it to this thread.
- verifier::VerifierDeps* verifier_deps = new verifier::VerifierDeps(dex_files);
- Runtime::Current()->GetCompilerCallbacks()->SetVerifierDeps(verifier_deps);
+ verifier::VerifierDeps* verifier_deps =
+ Runtime::Current()->GetCompilerCallbacks()->GetVerifierDeps();
+ CHECK(verifier_deps != nullptr);
Thread::Current()->SetVerifierDeps(verifier_deps);
// Create per-thread VerifierDeps to avoid contention on the main one.
// We will merge them after verification.
for (ThreadPoolWorker* worker : parallel_thread_pool_->GetWorkers()) {
- worker->GetThread()->SetVerifierDeps(new verifier::VerifierDeps(dex_files));
+ worker->GetThread()->SetVerifierDeps(new verifier::VerifierDeps(dex_files_for_oat_file_));
}
}
@@ -2031,7 +2025,7 @@
for (ThreadPoolWorker* worker : parallel_thread_pool_->GetWorkers()) {
verifier::VerifierDeps* thread_deps = worker->GetThread()->GetVerifierDeps();
worker->GetThread()->SetVerifierDeps(nullptr);
- verifier_deps->MergeWith(*thread_deps, dex_files);;
+ verifier_deps->MergeWith(*thread_deps, dex_files_for_oat_file_);
delete thread_deps;
}
Thread::Current()->SetVerifierDeps(nullptr);
@@ -2702,7 +2696,14 @@
: profile_compilation_info_->DumpInfo(&dex_files));
}
- DCHECK(current_dex_to_dex_methods_ == nullptr);
+ current_dex_to_dex_methods_ = nullptr;
+ Thread* const self = Thread::Current();
+ {
+ // Clear in case we aren't the first call to Compile.
+ MutexLock mu(self, dex_to_dex_references_lock_);
+ dex_to_dex_references_.clear();
+ }
+
for (const DexFile* dex_file : dex_files) {
CHECK(dex_file != nullptr);
CompileDexFile(class_loader,
@@ -2721,7 +2722,7 @@
{
// From this point on, we shall not modify dex_to_dex_references_, so
// just grab a reference to it that we use without holding the mutex.
- MutexLock lock(Thread::Current(), dex_to_dex_references_lock_);
+ MutexLock lock(self, dex_to_dex_references_lock_);
dex_to_dex_references = ArrayRef<DexFileMethodSet>(dex_to_dex_references_);
}
for (const auto& method_set : dex_to_dex_references) {
@@ -2914,7 +2915,7 @@
if (kIsDebugBuild) {
// Check to make sure it's not a dex file for an oat file we are compiling since these
// should always succeed. These do not include classes in for used libraries.
- for (const DexFile* dex_file : *dex_files_for_oat_file_) {
+ for (const DexFile* dex_file : GetDexFilesForOatFile()) {
CHECK_NE(dex_ref.dex_file, dex_file) << dex_ref.dex_file->GetLocation();
}
}
@@ -3032,4 +3033,13 @@
single_thread_pool_.reset();
}
+void CompilerDriver::SetDexFilesForOatFile(const std::vector<const DexFile*>& dex_files) {
+ dex_files_for_oat_file_ = dex_files;
+ for (const DexFile* dex_file : dex_files) {
+ if (!compiled_classes_.HaveDexFile(dex_file)) {
+ compiled_classes_.AddDexFile(dex_file, dex_file->NumClassDefs());
+ }
+ }
+}
+
} // namespace art
diff --git a/compiler/driver/compiler_driver.h b/compiler/driver/compiler_driver.h
index ecaed83..d9886a2 100644
--- a/compiler/driver/compiler_driver.h
+++ b/compiler/driver/compiler_driver.h
@@ -103,15 +103,11 @@
~CompilerDriver();
// Set dex files that will be stored in the oat file after being compiled.
- void SetDexFilesForOatFile(const std::vector<const DexFile*>& dex_files) {
- dex_files_for_oat_file_ = &dex_files;
- }
+ void SetDexFilesForOatFile(const std::vector<const DexFile*>& dex_files);
// Get dex file that will be stored in the oat file after being compiled.
ArrayRef<const DexFile* const> GetDexFilesForOatFile() const {
- return (dex_files_for_oat_file_ != nullptr)
- ? ArrayRef<const DexFile* const>(*dex_files_for_oat_file_)
- : ArrayRef<const DexFile* const>();
+ return ArrayRef<const DexFile* const>(dex_files_for_oat_file_);
}
void CompileAll(jobject class_loader,
@@ -532,7 +528,7 @@
bool support_boot_image_fixup_;
// List of dex files that will be stored in the oat file.
- const std::vector<const DexFile*>* dex_files_for_oat_file_;
+ std::vector<const DexFile*> dex_files_for_oat_file_;
CompiledMethodStorage compiled_method_storage_;
diff --git a/compiler/driver/compiler_driver_test.cc b/compiler/driver/compiler_driver_test.cc
index 10bfd97..fee6afb 100644
--- a/compiler/driver/compiler_driver_test.cc
+++ b/compiler/driver/compiler_driver_test.cc
@@ -42,7 +42,9 @@
void CompileAll(jobject class_loader) REQUIRES(!Locks::mutator_lock_) {
TimingLogger timings("CompilerDriverTest::CompileAll", false, false);
TimingLogger::ScopedTiming t(__FUNCTION__, &timings);
- compiler_driver_->CompileAll(class_loader, GetDexFiles(class_loader), &timings);
+ dex_files_ = GetDexFiles(class_loader);
+ compiler_driver_->SetDexFilesForOatFile(dex_files_);;
+ compiler_driver_->CompileAll(class_loader, dex_files_, &timings);
t.NewTiming("MakeAllExecutable");
MakeAllExecutable(class_loader);
}
@@ -95,6 +97,7 @@
JNIEnv* env_;
jclass class_;
jmethodID mid_;
+ std::vector<const DexFile*> dex_files_;
};
// Disabled due to 10 second runtime on host
diff --git a/compiler/image_writer.cc b/compiler/image_writer.cc
index 318009c..fc7cd01 100644
--- a/compiler/image_writer.cc
+++ b/compiler/image_writer.cc
@@ -894,7 +894,7 @@
&my_early_exit,
visited);
// Remove the class if the dex file is not in the set of dex files. This happens for classes that
- // are from uses library if there is no profile. b/30688277
+ // are from uses-library if there is no profile. b/30688277
mirror::DexCache* dex_cache = klass->GetDexCache();
if (dex_cache != nullptr) {
result = result ||
@@ -1153,9 +1153,22 @@
Thread* self = Thread::Current();
ScopedAssertNoThreadSuspension sa(__FUNCTION__);
- // Clear class table strong roots so that dex caches can get pruned. We require pruning the class
- // path dex caches.
- class_linker->ClearClassTableStrongRoots();
+ // Prune uses-library dex caches. Only prune the uses-library dex caches since we want to make
+ // sure the other ones don't get unloaded before the OatWriter runs.
+ class_linker->VisitClassTables(
+ [&](ClassTable* table) REQUIRES_SHARED(Locks::mutator_lock_) {
+ table->RemoveStrongRoots(
+ [&](GcRoot<mirror::Object> root) REQUIRES_SHARED(Locks::mutator_lock_) {
+ ObjPtr<mirror::Object> obj = root.Read();
+ if (obj->IsDexCache()) {
+ // Return true if the dex file is not one of the ones in the map.
+ return dex_file_oat_index_map_.find(obj->AsDexCache()->GetDexFile()) ==
+ dex_file_oat_index_map_.end();
+ }
+ // Return false to avoid removing.
+ return false;
+ });
+ });
// Remove the undesired classes from the class roots.
ObjPtr<mirror::ClassLoader> class_loader;
diff --git a/compiler/oat_writer.cc b/compiler/oat_writer.cc
index 4d258af..d7e3a28 100644
--- a/compiler/oat_writer.cc
+++ b/compiler/oat_writer.cc
@@ -1282,9 +1282,12 @@
bool StartClass(const DexFile* dex_file, size_t class_def_index) OVERRIDE
REQUIRES_SHARED(Locks::mutator_lock_) {
OatDexMethodVisitor::StartClass(dex_file, class_def_index);
- if (dex_cache_ == nullptr || dex_cache_->GetDexFile() != dex_file) {
- dex_cache_ = class_linker_->FindDexCache(Thread::Current(), *dex_file);
- DCHECK(dex_cache_ != nullptr);
+ if (writer_->GetCompilerDriver()->GetCompilerOptions().IsAotCompilationEnabled()) {
+ // Only need to set the dex cache if we have compilation. Other modes might have unloaded it.
+ if (dex_cache_ == nullptr || dex_cache_->GetDexFile() != dex_file) {
+ dex_cache_ = class_linker_->FindDexCache(Thread::Current(), *dex_file);
+ DCHECK(dex_cache_ != nullptr);
+ }
}
return true;
}
diff --git a/compiler/verifier_deps_test.cc b/compiler/verifier_deps_test.cc
index 72e2a6c..e9f3f80 100644
--- a/compiler/verifier_deps_test.cc
+++ b/compiler/verifier_deps_test.cc
@@ -87,13 +87,13 @@
TimingLogger timings("Verify", false, false);
// The compiler driver handles the verifier deps in the callbacks, so
// remove what this class did for unit testing.
- verifier_deps_.reset(nullptr);
+ if (deps == nullptr) {
+ // Create some verifier deps by default if they are not already specified.
+ deps = new verifier::VerifierDeps(dex_files_);
+ verifier_deps_.reset(deps);
+ }
callbacks_->SetVerifierDeps(deps);
compiler_driver_->Verify(class_loader_, dex_files_, &timings);
- // The compiler driver may have updated the VerifierDeps in the callback object.
- if (callbacks_->GetVerifierDeps() != deps) {
- verifier_deps_.reset(callbacks_->GetVerifierDeps());
- }
callbacks_->SetVerifierDeps(nullptr);
// Clear entries in the verification results to avoid hitting a DCHECK that
// we always succeed inserting a new entry after verifying.
@@ -128,6 +128,7 @@
for (const DexFile* dex_file : dex_files_) {
compiler_driver_->GetVerificationResults()->AddDexFile(dex_file);
}
+ compiler_driver_->SetDexFilesForOatFile(dex_files_);
}
void LoadDexFile(ScopedObjectAccess* soa) REQUIRES_SHARED(Locks::mutator_lock_) {
@@ -1441,7 +1442,6 @@
ASSERT_FALSE(verifier_deps_ == nullptr);
ASSERT_FALSE(verifier_deps_->Equals(decoded_deps));
} else {
- ASSERT_TRUE(verifier_deps_ == nullptr);
VerifyClassStatus(decoded_deps);
}
}