Support class unloading in dex2oat for quicken multidex
Support class unloading for the quicken compilation filter. This will
be enabled in a follow up CL.
Added a test that compares with and without unloading. The way that
it tests this is by adding an output app image. Having an app image
disables the unloading. This test also covers that app images don't
change the odex (currently).
Added a test for the assumed verified logic.
Bug: 63467744
Test: test-art-host
Test: test/testrunner/testrunner.py --interpreter --host -j40
Change-Id: I1e8a862c6f089c06c58aa0c846797e4c24bd072c
diff --git a/compiler/dex/quick_compiler_callbacks.cc b/compiler/dex/quick_compiler_callbacks.cc
index 872f7ea..c7e9f4f 100644
--- a/compiler/dex/quick_compiler_callbacks.cc
+++ b/compiler/dex/quick_compiler_callbacks.cc
@@ -16,6 +16,7 @@
#include "quick_compiler_callbacks.h"
+#include "driver/compiler_driver.h"
#include "verification_results.h"
#include "verifier/method_verifier-inl.h"
@@ -33,4 +34,17 @@
}
}
+bool QuickCompilerCallbacks::CanAssumeVerified(ClassReference ref) {
+ // If we don't have class unloading enabled in the compiler, we will never see class that were
+ // previously verified. Return false to avoid overhead from the lookup in the compiler driver.
+ if (!does_class_unloading_) {
+ return false;
+ }
+ DCHECK(compiler_driver_ != nullptr);
+ // In the case of the quicken filter: avoiding verification of quickened instructions, which the
+ // verifier doesn't currently support.
+ // In the case of the verify filter, avoiding verifiying twice.
+ return compiler_driver_->CanAssumeVerified(ref);
+}
+
} // namespace art
diff --git a/compiler/dex/quick_compiler_callbacks.h b/compiler/dex/quick_compiler_callbacks.h
index a3a6c09..578aff4 100644
--- a/compiler/dex/quick_compiler_callbacks.h
+++ b/compiler/dex/quick_compiler_callbacks.h
@@ -22,6 +22,7 @@
namespace art {
+class CompilerDriver;
class VerificationResults;
class QuickCompilerCallbacks FINAL : public CompilerCallbacks {
@@ -53,8 +54,19 @@
verification_results_ = verification_results;
}
+ bool CanAssumeVerified(ClassReference ref) OVERRIDE;
+
+ void SetDoesClassUnloading(bool does_class_unloading, CompilerDriver* compiler_driver)
+ OVERRIDE {
+ does_class_unloading_ = does_class_unloading;
+ compiler_driver_ = compiler_driver;
+ DCHECK(!does_class_unloading || compiler_driver_ != nullptr);
+ }
+
private:
VerificationResults* verification_results_ = nullptr;
+ bool does_class_unloading_ = false;
+ CompilerDriver* compiler_driver_ = nullptr;
std::unique_ptr<verifier::VerifierDeps> verifier_deps_;
};
diff --git a/compiler/driver/compiler_driver.cc b/compiler/driver/compiler_driver.cc
index 6e087c5..037e458 100644
--- a/compiler/driver/compiler_driver.cc
+++ b/compiler/driver/compiler_driver.cc
@@ -3040,4 +3040,10 @@
}
}
+bool CompilerDriver::CanAssumeVerified(ClassReference ref) const {
+ mirror::Class::Status existing = mirror::Class::kStatusNotReady;
+ compiled_classes_.Get(DexFileReference(ref.first, ref.second), &existing);
+ return existing >= mirror::Class::kStatusVerified;
+}
+
} // namespace art
diff --git a/compiler/driver/compiler_driver.h b/compiler/driver/compiler_driver.h
index d9886a2..ba4581c 100644
--- a/compiler/driver/compiler_driver.h
+++ b/compiler/driver/compiler_driver.h
@@ -377,6 +377,8 @@
return profile_compilation_info_;
}
+ bool CanAssumeVerified(ClassReference ref) const;
+
private:
void PreCompile(jobject class_loader,
const std::vector<const DexFile*>& dex_files,
diff --git a/compiler/driver/compiler_driver_test.cc b/compiler/driver/compiler_driver_test.cc
index 4da3e0d..392d57c 100644
--- a/compiler/driver/compiler_driver_test.cc
+++ b/compiler/driver/compiler_driver_test.cc
@@ -23,6 +23,7 @@
#include "art_method-inl.h"
#include "class_linker-inl.h"
#include "common_compiler_test.h"
+#include "compiler_callbacks.h"
#include "dex_file.h"
#include "dex_file_types.h"
#include "gc/heap.h"
@@ -368,7 +369,9 @@
// Test that a class of status kStatusRetryVerificationAtRuntime is indeed recorded that way in the
// driver.
-TEST_F(CompilerDriverVerifyTest, RetryVerifcationStatus) {
+// Test that checks that classes can be assumed as verified if unloading mode is enabled and
+// the class status is at least verified.
+TEST_F(CompilerDriverVerifyTest, RetryVerifcationStatusCheckVerified) {
Thread* const self = Thread::Current();
jobject class_loader;
std::vector<const DexFile*> dex_files;
@@ -382,6 +385,7 @@
dex_file = dex_files.front();
}
compiler_driver_->SetDexFilesForOatFile(dex_files);
+ callbacks_->SetDoesClassUnloading(true, compiler_driver_.get());
ClassReference ref(dex_file, 0u);
// Test that the status is read from the compiler driver as expected.
for (size_t i = mirror::Class::kStatusRetryVerificationAtRuntime;
@@ -397,6 +401,12 @@
mirror::Class::Status status = {};
ASSERT_TRUE(compiler_driver_->GetCompiledClass(ref, &status));
EXPECT_EQ(status, expected_status);
+
+ // Check that we can assume verified if we are a status that is at least verified.
+ if (status >= mirror::Class::kStatusVerified) {
+ // Check that the class can be assumed as verified in the compiler driver.
+ EXPECT_TRUE(callbacks_->CanAssumeVerified(ref)) << status;
+ }
}
}
diff --git a/dex2oat/dex2oat.cc b/dex2oat/dex2oat.cc
index ff193e9..f50e88f 100644
--- a/dex2oat/dex2oat.cc
+++ b/dex2oat/dex2oat.cc
@@ -1774,10 +1774,8 @@
bool ShouldCompileDexFilesIndividually() const {
// Compile individually if we are not building an image, not using any compilation, and are
// using multidex.
- // This means extract, verify, and quicken will use the individual compilation mode (to reduce
- // RAM used by the compiler).
- // TODO: Still do it for app images to get testing coverage. Note that this will generate empty
- // app images.
+ // This means extract, and verify, will use the individual compilation mode (to reduce RAM used
+ // by the compiler).
return !IsImage() &&
dex_files_.size() > 1 &&
!CompilerFilter::IsAnyCompilationEnabled(compiler_options_->GetCompilerFilter());
@@ -1856,6 +1854,16 @@
profile_compilation_info_.get()));
driver_->SetDexFilesForOatFile(dex_files_);
+ const bool compile_individually = ShouldCompileDexFilesIndividually();
+ if (compile_individually) {
+ // Set the compiler driver in the callbacks so that we can avoid re-verification. This not
+ // only helps performance but also prevents reverifying quickened bytecodes. Attempting
+ // verify quickened bytecode causes verification failures.
+ // Only set the compiler filter if we are doing separate compilation since there is a bit
+ // of overhead when checking if a class was previously verified.
+ callbacks_->SetDoesClassUnloading(true, driver_.get());
+ }
+
// Setup vdex for compilation.
if (!DoEagerUnquickeningOfVdex() && input_vdex_file_ != nullptr) {
callbacks_->SetVerifierDeps(
@@ -1872,7 +1880,7 @@
callbacks_->SetVerifierDeps(new verifier::VerifierDeps(dex_files_));
}
// Invoke the compilation.
- if (ShouldCompileDexFilesIndividually()) {
+ if (compile_individually) {
CompileDexFilesIndividually();
// Return a null classloader since we already freed released it.
return nullptr;
@@ -2966,6 +2974,8 @@
static dex2oat::ReturnCode CompileImage(Dex2Oat& dex2oat) {
dex2oat.LoadClassProfileDescriptors();
+ // Keep the class loader that was used for compilation live for the rest of the compilation
+ // process.
ScopedGlobalRef class_loader(dex2oat.Compile());
if (!dex2oat.WriteOutputFiles()) {
@@ -3014,6 +3024,8 @@
}
static dex2oat::ReturnCode CompileApp(Dex2Oat& dex2oat) {
+ // Keep the class loader that was used for compilation live for the rest of the compilation
+ // process.
ScopedGlobalRef class_loader(dex2oat.Compile());
if (!dex2oat.WriteOutputFiles()) {
diff --git a/dex2oat/dex2oat_test.cc b/dex2oat/dex2oat_test.cc
index 12bceb3..1673fc1 100644
--- a/dex2oat/dex2oat_test.cc
+++ b/dex2oat/dex2oat_test.cc
@@ -41,6 +41,7 @@
namespace art {
static constexpr size_t kMaxMethodIds = 65535;
+static constexpr bool kDebugArgs = false;
using android::base::StringPrintf;
@@ -55,7 +56,7 @@
}
protected:
- int GenerateOdexForTestWithStatus(const std::string& dex_location,
+ int GenerateOdexForTestWithStatus(const std::vector<std::string>& dex_locations,
const std::string& odex_location,
CompilerFilter::Filter filter,
std::string* error_msg,
@@ -63,7 +64,10 @@
bool use_fd = false) {
std::unique_ptr<File> oat_file;
std::vector<std::string> args;
- args.push_back("--dex-file=" + dex_location);
+ // Add dex file args.
+ for (const std::string& dex_location : dex_locations) {
+ args.push_back("--dex-file=" + dex_location);
+ }
if (use_fd) {
oat_file.reset(OS::CreateEmptyFile(odex_location.c_str()));
CHECK(oat_file != nullptr) << odex_location;
@@ -93,7 +97,7 @@
bool use_fd = false,
std::function<void(const OatFile&)> check_oat = [](const OatFile&) {}) {
std::string error_msg;
- int status = GenerateOdexForTestWithStatus(dex_location,
+ int status = GenerateOdexForTestWithStatus({dex_location},
odex_location,
filter,
&error_msg,
@@ -187,6 +191,14 @@
CHECK(android_root != nullptr);
argv.push_back("--android-root=" + std::string(android_root));
+ if (kDebugArgs) {
+ std::string all_args;
+ for (const std::string& arg : argv) {
+ all_args += arg + " ";
+ }
+ LOG(ERROR) << all_args;
+ }
+
int link[2];
if (pipe(link) == -1) {
@@ -951,7 +963,7 @@
Copy(GetTestDexFileName(), dex_location);
std::string error_msg;
- return GenerateOdexForTestWithStatus(dex_location,
+ return GenerateOdexForTestWithStatus({dex_location},
odex_location,
CompilerFilter::kSpeed,
&error_msg,
@@ -1107,4 +1119,66 @@
RunTest(context.c_str(), expected_classpath_key.c_str(), true);
}
+class Dex2oatDeterminism : public Dex2oatTest {};
+
+TEST_F(Dex2oatDeterminism, UnloadCompile) {
+ Runtime* const runtime = Runtime::Current();
+ std::string out_dir = GetScratchDir();
+ const std::string base_oat_name = out_dir + "/base.oat";
+ const std::string base_vdex_name = out_dir + "/base.vdex";
+ const std::string unload_oat_name = out_dir + "/unload.oat";
+ const std::string unload_vdex_name = out_dir + "/unload.vdex";
+ const std::string no_unload_oat_name = out_dir + "/nounload.oat";
+ const std::string no_unload_vdex_name = out_dir + "/nounload.vdex";
+ const std::string app_image_name = out_dir + "/unload.art";
+ std::string error_msg;
+ const std::vector<gc::space::ImageSpace*>& spaces = runtime->GetHeap()->GetBootImageSpaces();
+ ASSERT_GT(spaces.size(), 0u);
+ const std::string image_location = spaces[0]->GetImageLocation();
+ // Without passing in an app image, it will unload in between compilations.
+ const int res = GenerateOdexForTestWithStatus(
+ GetLibCoreDexFileNames(),
+ base_oat_name,
+ CompilerFilter::Filter::kQuicken,
+ &error_msg,
+ {"--force-determinism", "--avoid-storing-invocation"});
+ EXPECT_EQ(res, 0);
+ Copy(base_oat_name, unload_oat_name);
+ Copy(base_vdex_name, unload_vdex_name);
+ std::unique_ptr<File> unload_oat(OS::OpenFileForReading(unload_oat_name.c_str()));
+ std::unique_ptr<File> unload_vdex(OS::OpenFileForReading(unload_vdex_name.c_str()));
+ ASSERT_TRUE(unload_oat != nullptr);
+ ASSERT_TRUE(unload_vdex != nullptr);
+ EXPECT_GT(unload_oat->GetLength(), 0u);
+ EXPECT_GT(unload_vdex->GetLength(), 0u);
+ // Regenerate with an app image to disable the dex2oat unloading and verify that the output is
+ // the same.
+ const int res2 = GenerateOdexForTestWithStatus(
+ GetLibCoreDexFileNames(),
+ base_oat_name,
+ CompilerFilter::Filter::kQuicken,
+ &error_msg,
+ {"--force-determinism", "--avoid-storing-invocation", "--app-image-file=" + app_image_name});
+ EXPECT_EQ(res2, 0);
+ Copy(base_oat_name, no_unload_oat_name);
+ Copy(base_vdex_name, no_unload_vdex_name);
+ std::unique_ptr<File> no_unload_oat(OS::OpenFileForReading(no_unload_oat_name.c_str()));
+ std::unique_ptr<File> no_unload_vdex(OS::OpenFileForReading(no_unload_vdex_name.c_str()));
+ ASSERT_TRUE(no_unload_oat != nullptr);
+ ASSERT_TRUE(no_unload_vdex != nullptr);
+ EXPECT_GT(no_unload_oat->GetLength(), 0u);
+ EXPECT_GT(no_unload_vdex->GetLength(), 0u);
+ // Verify that both of the files are the same (odex and vdex).
+ EXPECT_EQ(unload_oat->GetLength(), no_unload_oat->GetLength());
+ EXPECT_EQ(unload_vdex->GetLength(), no_unload_vdex->GetLength());
+ EXPECT_EQ(unload_oat->Compare(no_unload_oat.get()), 0)
+ << unload_oat_name << " " << no_unload_oat_name;
+ EXPECT_EQ(unload_vdex->Compare(no_unload_vdex.get()), 0)
+ << unload_vdex_name << " " << no_unload_vdex_name;
+ // App image file.
+ std::unique_ptr<File> app_image_file(OS::OpenFileForReading(app_image_name.c_str()));
+ ASSERT_TRUE(app_image_file != nullptr);
+ EXPECT_GT(app_image_file->GetLength(), 0u);
+}
+
} // namespace art
diff --git a/runtime/aot_class_linker.cc b/runtime/aot_class_linker.cc
index b1bc3f8..9396565 100644
--- a/runtime/aot_class_linker.cc
+++ b/runtime/aot_class_linker.cc
@@ -16,10 +16,12 @@
#include "aot_class_linker.h"
+#include "class_reference.h"
+#include "compiler_callbacks.h"
#include "handle_scope-inl.h"
-#include "mirror/class.h"
-#include "mirror/object-inl.h"
+#include "mirror/class-inl.h"
#include "runtime.h"
+#include "verifier/verifier_enums.h"
namespace art {
@@ -66,4 +68,18 @@
}
return success;
}
+
+verifier::FailureKind AotClassLinker::PerformClassVerification(Thread* self,
+ Handle<mirror::Class> klass,
+ verifier::HardFailLogMode log_level,
+ std::string* error_msg) {
+ Runtime* const runtime = Runtime::Current();
+ CompilerCallbacks* callbacks = runtime->GetCompilerCallbacks();
+ if (callbacks->CanAssumeVerified(ClassReference(&klass->GetDexFile(),
+ klass->GetDexClassDefIndex()))) {
+ return verifier::FailureKind::kNoFailure;
+ }
+ return ClassLinker::PerformClassVerification(self, klass, log_level, error_msg);
+}
+
} // namespace art
diff --git a/runtime/aot_class_linker.h b/runtime/aot_class_linker.h
index 11bea86..927b533 100644
--- a/runtime/aot_class_linker.h
+++ b/runtime/aot_class_linker.h
@@ -27,6 +27,16 @@
explicit AotClassLinker(InternTable *intern_table);
~AotClassLinker();
+ protected:
+ // Overridden version of PerformClassVerification allows skipping verification if the class was
+ // previously verified but unloaded.
+ verifier::FailureKind PerformClassVerification(Thread* self,
+ Handle<mirror::Class> klass,
+ verifier::HardFailLogMode log_level,
+ std::string* error_msg)
+ OVERRIDE
+ REQUIRES_SHARED(Locks::mutator_lock_);
+
bool InitializeClass(Thread *self,
Handle<mirror::Class> klass,
bool can_run_clinit,
diff --git a/runtime/class_linker.cc b/runtime/class_linker.cc
index 3ac87c5..ed673d4 100644
--- a/runtime/class_linker.cc
+++ b/runtime/class_linker.cc
@@ -3694,7 +3694,8 @@
ObjPtr<mirror::DexCache> ClassLinker::FindDexCache(Thread* self, const DexFile& dex_file) {
ReaderMutexLock mu(self, *Locks::dex_lock_);
- ObjPtr<mirror::DexCache> dex_cache = DecodeDexCache(self, FindDexCacheDataLocked(dex_file));
+ DexCacheData dex_cache_data = FindDexCacheDataLocked(dex_file);
+ ObjPtr<mirror::DexCache> dex_cache = DecodeDexCache(self, dex_cache_data);
if (dex_cache != nullptr) {
return dex_cache;
}
@@ -3704,7 +3705,8 @@
LOG(FATAL_WITHOUT_ABORT) << "Registered dex file " << data.dex_file->GetLocation();
}
}
- LOG(FATAL) << "Failed to find DexCache for DexFile " << dex_file.GetLocation();
+ LOG(FATAL) << "Failed to find DexCache for DexFile " << dex_file.GetLocation()
+ << " " << &dex_file << " " << dex_cache_data.dex_file;
UNREACHABLE();
}
@@ -4280,13 +4282,7 @@
std::string error_msg;
verifier::FailureKind verifier_failure = verifier::FailureKind::kNoFailure;
if (!preverified) {
- Runtime* runtime = Runtime::Current();
- verifier_failure = verifier::MethodVerifier::VerifyClass(self,
- klass.Get(),
- runtime->GetCompilerCallbacks(),
- runtime->IsAotCompiler(),
- log_level,
- &error_msg);
+ verifier_failure = PerformClassVerification(self, klass, log_level, &error_msg);
}
// Verification is done, grab the lock again.
@@ -4354,6 +4350,19 @@
return verifier_failure;
}
+verifier::FailureKind ClassLinker::PerformClassVerification(Thread* self,
+ Handle<mirror::Class> klass,
+ verifier::HardFailLogMode log_level,
+ std::string* error_msg) {
+ Runtime* const runtime = Runtime::Current();
+ return verifier::MethodVerifier::VerifyClass(self,
+ klass.Get(),
+ runtime->GetCompilerCallbacks(),
+ runtime->IsAotCompiler(),
+ log_level,
+ error_msg);
+}
+
bool ClassLinker::VerifyClassUsingOatFile(const DexFile& dex_file,
ObjPtr<mirror::Class> klass,
mirror::Class::Status& oat_file_class_status) {
diff --git a/runtime/class_linker.h b/runtime/class_linker.h
index bf14aeb..2fbbe79 100644
--- a/runtime/class_linker.h
+++ b/runtime/class_linker.h
@@ -710,6 +710,12 @@
REQUIRES_SHARED(Locks::mutator_lock_)
REQUIRES(!Locks::dex_lock_);
+ virtual verifier::FailureKind PerformClassVerification(Thread* self,
+ Handle<mirror::Class> klass,
+ verifier::HardFailLogMode log_level,
+ std::string* error_msg)
+ REQUIRES_SHARED(Locks::mutator_lock_);
+
private:
class LinkInterfaceMethodsHelper;
diff --git a/runtime/compiler_callbacks.h b/runtime/compiler_callbacks.h
index 806653a..c51bb5e 100644
--- a/runtime/compiler_callbacks.h
+++ b/runtime/compiler_callbacks.h
@@ -22,6 +22,8 @@
namespace art {
+class CompilerDriver;
+
namespace verifier {
class MethodVerifier;
@@ -49,6 +51,13 @@
virtual verifier::VerifierDeps* GetVerifierDeps() const = 0;
virtual void SetVerifierDeps(verifier::VerifierDeps* deps ATTRIBUTE_UNUSED) {}
+ virtual bool CanAssumeVerified(ClassReference ref ATTRIBUTE_UNUSED) {
+ return false;
+ }
+
+ virtual void SetDoesClassUnloading(bool does_class_unloading ATTRIBUTE_UNUSED,
+ CompilerDriver* compiler_driver ATTRIBUTE_UNUSED) {}
+
bool IsBootImage() {
return mode_ == CallbackMode::kCompileBootImage;
}