Add option for not copying dex
If specified --avoid-copying-dex does not write the dex file into the
output vdex.
Added test to dex2oat_test.
Motivation: Use this for preopt on low end devices to speed
up first boot.
Bug: 70934104
Test: test-art-host
Change-Id: I181d35a923dccb0d08bb855c0a7f74af3ed1f48d
diff --git a/dex2oat/Android.bp b/dex2oat/Android.bp
index dd16ba4..d1272c9 100644
--- a/dex2oat/Android.bp
+++ b/dex2oat/Android.bp
@@ -248,6 +248,7 @@
"libbase",
"liblz4",
"libsigchain",
+ "libziparchive",
],
static_libs: [
"libartd-dex2oat",
diff --git a/dex2oat/dex2oat.cc b/dex2oat/dex2oat.cc
index 64db7be..b5c5e45 100644
--- a/dex2oat/dex2oat.cc
+++ b/dex2oat/dex2oat.cc
@@ -456,6 +456,9 @@
UsageError(" --deduplicate-code=true|false: enable|disable code deduplication. Deduplicated");
UsageError(" code will have an arbitrary symbol tagged with [DEDUPED].");
UsageError("");
+ UsageError(" --copying-dex-files=true|false: enable|disable copying the dex files into the");
+ UsageError(" output vdex.");
+ UsageError("");
UsageError(" --compilation-reason=<string>: optional metadata specifying the reason for");
UsageError(" compiling the apk. If specified, the string will be embedded verbatim in");
UsageError(" the key value store of the oat file.");
@@ -1232,6 +1235,7 @@
AssignTrueIfExists(args, M::Host, &is_host_);
AssignTrueIfExists(args, M::AvoidStoringInvocation, &avoid_storing_invocation_);
AssignTrueIfExists(args, M::MultiImage, &multi_image_);
+ AssignIfExists(args, M::CopyDexFiles, ©_dex_files_);
if (args.Exists(M::ForceDeterminism)) {
if (!SupportsDeterministicCompilation()) {
@@ -1425,13 +1429,10 @@
LOG(INFO) << "No " << VdexFile::kVdexNameInDmFile << " file in DexMetadata archive. "
<< "Not doing fast verification.";
} else {
- std::unique_ptr<MemMap> input_file;
- if (zip_entry->IsUncompressed()) {
- input_file.reset(zip_entry->MapDirectlyFromFile(VdexFile::kVdexNameInDmFile, &error_msg));
- } else {
- input_file.reset(zip_entry->ExtractToMemMap(
- kDexMetadata, VdexFile::kVdexNameInDmFile, &error_msg));
- }
+ std::unique_ptr<MemMap> input_file(zip_entry->MapDirectlyOrExtract(
+ VdexFile::kVdexNameInDmFile,
+ kDexMetadata,
+ &error_msg));
if (input_file == nullptr) {
LOG(WARNING) << "Could not open vdex file in DexMetadata archive: " << error_msg;
} else {
@@ -1618,6 +1619,7 @@
key_value_store_.get(),
verify,
update_input_vdex_,
+ copy_dex_files_,
&opened_dex_files_map,
&opened_dex_files)) {
return dex2oat::ReturnCode::kOther;
@@ -2918,6 +2920,9 @@
// Whether the given input vdex is also the output.
bool update_input_vdex_ = false;
+ // By default, copy the dex to the vdex file.
+ bool copy_dex_files_ = true;
+
// The reason for invoking the compiler.
std::string compilation_reason_;
diff --git a/dex2oat/dex2oat_options.cc b/dex2oat/dex2oat_options.cc
index 4b6f8a4..1ef100d 100644
--- a/dex2oat/dex2oat_options.cc
+++ b/dex2oat/dex2oat_options.cc
@@ -233,6 +233,11 @@
.IntoKey(M::VeryLargeAppThreshold)
.Define("--force-determinism")
.IntoKey(M::ForceDeterminism)
+ .Define("--copy-dex-files=_")
+ .WithType<bool>()
+ .WithValueMap({{"true", false},
+ {"false", false}})
+ .IntoKey(M::CopyDexFiles)
.Define("--classpath-dir=_")
.WithType<std::string>()
.IntoKey(M::ClasspathDir)
diff --git a/dex2oat/dex2oat_options.def b/dex2oat/dex2oat_options.def
index a1646aa..a9d349b 100644
--- a/dex2oat/dex2oat_options.def
+++ b/dex2oat/dex2oat_options.def
@@ -74,6 +74,7 @@
DEX2OAT_OPTIONS_KEY (Unit, DumpTiming)
DEX2OAT_OPTIONS_KEY (Unit, DumpPasses)
DEX2OAT_OPTIONS_KEY (Unit, DumpStats)
+DEX2OAT_OPTIONS_KEY (bool, CopyDexFiles)
DEX2OAT_OPTIONS_KEY (Unit, AvoidStoringInvocation)
DEX2OAT_OPTIONS_KEY (std::string, SwapFile)
DEX2OAT_OPTIONS_KEY (int, SwapFileFd)
diff --git a/dex2oat/dex2oat_test.cc b/dex2oat/dex2oat_test.cc
index 4ac8e6a..a627dbd 100644
--- a/dex2oat/dex2oat_test.cc
+++ b/dex2oat/dex2oat_test.cc
@@ -41,6 +41,8 @@
#include "oat.h"
#include "oat_file.h"
#include "utils.h"
+#include "vdex_file.h"
+#include "ziparchive/zip_writer.h"
namespace art {
@@ -1781,4 +1783,87 @@
ASSERT_EQ(nullptr, odex_file->GetCompilationReason());
}
+TEST_F(Dex2oatTest, DontExtract) {
+ std::unique_ptr<const DexFile> dex(OpenTestDexFile("ManyMethods"));
+ std::string error_msg;
+ const std::string out_dir = GetScratchDir();
+ const std::string dex_location = dex->GetLocation();
+ const std::string odex_location = out_dir + "/base.oat";
+ const std::string vdex_location = out_dir + "/base.vdex";
+ GenerateOdexForTest(dex_location,
+ odex_location,
+ CompilerFilter::Filter::kVerify,
+ { "--copy-dex-files=false" },
+ true, // expect_success
+ false, // use_fd
+ [](const OatFile&) {
+ });
+ {
+ // Check the vdex doesn't have dex.
+ std::unique_ptr<VdexFile> vdex(VdexFile::Open(vdex_location.c_str(),
+ /*writable*/ false,
+ /*low_4gb*/ false,
+ /*unquicken*/ false,
+ &error_msg));
+ ASSERT_TRUE(vdex != nullptr);
+ EXPECT_EQ(vdex->GetHeader().GetDexSize(), 0u) << output_;
+ }
+ std::unique_ptr<OatFile> odex_file(OatFile::Open(odex_location.c_str(),
+ odex_location.c_str(),
+ nullptr,
+ nullptr,
+ false,
+ /*low_4gb*/ false,
+ dex_location.c_str(),
+ &error_msg));
+ ASSERT_TRUE(odex_file != nullptr) << dex_location;
+ std::vector<const OatDexFile*> oat_dex_files = odex_file->GetOatDexFiles();
+ ASSERT_EQ(oat_dex_files.size(), 1u);
+ // Verify that the oat file can still open the dex files.
+ for (const OatDexFile* oat_dex : oat_dex_files) {
+ std::unique_ptr<const DexFile> dex_file(oat_dex->OpenDexFile(&error_msg));
+ ASSERT_TRUE(dex_file != nullptr) << error_msg;
+ }
+ // Create a dm file and use it to verify.
+ // Add produced artifacts to a zip file that doesn't contain the classes.dex.
+ ScratchFile dm_file;
+ {
+ std::unique_ptr<File> vdex_file(OS::OpenFileForReading(vdex_location.c_str()));
+ ASSERT_TRUE(vdex_file != nullptr);
+ ASSERT_GT(vdex_file->GetLength(), 0u);
+ FILE* file = fdopen(dm_file.GetFd(), "w+b");
+ ZipWriter writer(file);
+ auto write_all_bytes = [&](File* file) {
+ std::unique_ptr<uint8_t[]> bytes(new uint8_t[file->GetLength()]);
+ ASSERT_TRUE(file->ReadFully(&bytes[0], file->GetLength()));
+ ASSERT_GE(writer.WriteBytes(&bytes[0], file->GetLength()), 0);
+ };
+ // Add vdex to zip.
+ writer.StartEntry(VdexFile::kVdexNameInDmFile, ZipWriter::kCompress);
+ write_all_bytes(vdex_file.get());
+ writer.FinishEntry();
+ writer.Finish();
+ ASSERT_EQ(dm_file.GetFile()->Flush(), 0);
+ }
+
+ // Generate a quickened dex by using the input dm file to verify.
+ GenerateOdexForTest(dex_location,
+ odex_location,
+ CompilerFilter::Filter::kQuicken,
+ { "--dump-timings", "--dm-file=" + dm_file.GetFilename() },
+ true, // expect_success
+ false, // use_fd
+ [](const OatFile& o) {
+ CHECK(o.ContainsDexCode());
+ });
+ std::istringstream iss(output_);
+ std::string line;
+ bool found_fast_verify = false;
+ const std::string kFastVerifyString = "Fast Verify";
+ while (std::getline(iss, line) && !found_fast_verify) {
+ found_fast_verify = found_fast_verify || line.find(kFastVerifyString) != std::string::npos;
+ }
+ EXPECT_TRUE(found_fast_verify) << "Expected to find " << kFastVerifyString << "\n" << output_;
+}
+
} // namespace art
diff --git a/dex2oat/linker/image_test.h b/dex2oat/linker/image_test.h
index 62519fc..303c873 100644
--- a/dex2oat/linker/image_test.h
+++ b/dex2oat/linker/image_test.h
@@ -276,6 +276,7 @@
&key_value_store,
/* verify */ false, // Dex files may be dex-to-dex-ed, don't verify.
/* update_input_vdex */ false,
+ /* extract_dex_files */ true,
&cur_opened_dex_files_maps,
&cur_opened_dex_files);
ASSERT_TRUE(dex_files_ok);
diff --git a/dex2oat/linker/oat_writer.cc b/dex2oat/linker/oat_writer.cc
index 7d8065e..09f9010 100644
--- a/dex2oat/linker/oat_writer.cc
+++ b/dex2oat/linker/oat_writer.cc
@@ -368,7 +368,7 @@
compiler_driver_(nullptr),
image_writer_(nullptr),
compiling_boot_image_(compiling_boot_image),
- only_contains_uncompressed_zip_entries_(false),
+ extract_dex_files_into_vdex_(true),
dex_files_(nullptr),
vdex_size_(0u),
vdex_dex_files_offset_(0u),
@@ -643,6 +643,7 @@
SafeMap<std::string, std::string>* key_value_store,
bool verify,
bool update_input_vdex,
+ bool copy_dex_files,
/*out*/ std::vector<std::unique_ptr<MemMap>>* opened_dex_files_map,
/*out*/ std::vector<std::unique_ptr<const DexFile>>* opened_dex_files) {
CHECK(write_state_ == WriteState::kAddingDexFileSources);
@@ -669,7 +670,7 @@
std::unique_ptr<BufferedOutputStream> vdex_out =
std::make_unique<BufferedOutputStream>(std::make_unique<FileOutputStream>(vdex_file));
// Write DEX files into VDEX, mmap and open them.
- if (!WriteDexFiles(vdex_out.get(), vdex_file, update_input_vdex) ||
+ if (!WriteDexFiles(vdex_out.get(), vdex_file, update_input_vdex, copy_dex_files) ||
!OpenDexFiles(vdex_file, verify, &dex_files_map, &dex_files)) {
return false;
}
@@ -2749,7 +2750,7 @@
};
bool OatWriter::WriteQuickeningInfo(OutputStream* vdex_out) {
- if (only_contains_uncompressed_zip_entries_) {
+ if (!extract_dex_files_into_vdex_) {
// Nothing to write. Leave `vdex_size_` untouched and unaligned.
vdex_quickening_info_offset_ = vdex_size_;
size_quickening_info_alignment_ = 0;
@@ -3339,25 +3340,32 @@
return true;
}
-bool OatWriter::WriteDexFiles(OutputStream* out, File* file, bool update_input_vdex) {
+bool OatWriter::WriteDexFiles(OutputStream* out,
+ File* file,
+ bool update_input_vdex,
+ bool copy_dex_files) {
TimingLogger::ScopedTiming split("Write Dex files", timings_);
vdex_dex_files_offset_ = vdex_size_;
- only_contains_uncompressed_zip_entries_ = true;
- for (OatDexFile& oat_dex_file : oat_dex_files_) {
- if (!oat_dex_file.source_.IsZipEntry()) {
- only_contains_uncompressed_zip_entries_ = false;
- break;
- }
- ZipEntry* entry = oat_dex_file.source_.GetZipEntry();
- if (!entry->IsUncompressed() || !entry->IsAlignedToDexHeader()) {
- only_contains_uncompressed_zip_entries_ = false;
- break;
+ extract_dex_files_into_vdex_ = copy_dex_files;
+ // If extraction is enabled, only do it if not all the dex files are aligned and uncompressed.
+ if (extract_dex_files_into_vdex_) {
+ extract_dex_files_into_vdex_ = false;
+ for (OatDexFile& oat_dex_file : oat_dex_files_) {
+ if (!oat_dex_file.source_.IsZipEntry()) {
+ extract_dex_files_into_vdex_ = true;
+ break;
+ }
+ ZipEntry* entry = oat_dex_file.source_.GetZipEntry();
+ if (!entry->IsUncompressed() || !entry->IsAlignedToDexHeader()) {
+ extract_dex_files_into_vdex_ = true;
+ break;
+ }
}
}
- if (!only_contains_uncompressed_zip_entries_) {
+ if (extract_dex_files_into_vdex_) {
// Write dex files.
for (OatDexFile& oat_dex_file : oat_dex_files_) {
if (!WriteDexFile(out, file, &oat_dex_file, update_input_vdex)) {
@@ -3823,13 +3831,13 @@
return true;
}
- if (only_contains_uncompressed_zip_entries_) {
+ if (!extract_dex_files_into_vdex_) {
std::vector<std::unique_ptr<const DexFile>> dex_files;
std::vector<std::unique_ptr<MemMap>> maps;
for (OatDexFile& oat_dex_file : oat_dex_files_) {
std::string error_msg;
- MemMap* map = oat_dex_file.source_.GetZipEntry()->MapDirectlyFromFile(
- oat_dex_file.dex_file_location_data_, &error_msg);
+ MemMap* map = oat_dex_file.source_.GetZipEntry()->MapDirectlyOrExtract(
+ oat_dex_file.dex_file_location_data_, "zipped dex", &error_msg);
if (map == nullptr) {
LOG(ERROR) << error_msg;
return false;
diff --git a/dex2oat/linker/oat_writer.h b/dex2oat/linker/oat_writer.h
index 7edb032..6a7e1e4 100644
--- a/dex2oat/linker/oat_writer.h
+++ b/dex2oat/linker/oat_writer.h
@@ -168,6 +168,7 @@
// This is generally the case, and should only be false for tests.
// If `update_input_vdex` is true, then this method won't actually write the dex files,
// and the compiler will just re-use the existing vdex file.
+ // If `copy_dex_files` is true, copy the input dex files into the vdex file.
bool WriteAndOpenDexFiles(File* vdex_file,
OutputStream* oat_rodata,
InstructionSet instruction_set,
@@ -175,6 +176,7 @@
SafeMap<std::string, std::string>* key_value_store,
bool verify,
bool update_input_vdex,
+ bool copy_dex_files,
/*out*/ std::vector<std::unique_ptr<MemMap>>* opened_dex_files_map,
/*out*/ std::vector<std::unique_ptr<const DexFile>>* opened_dex_files);
bool WriteQuickeningInfo(OutputStream* vdex_out);
@@ -283,7 +285,10 @@
// If `update_input_vdex` is true, then this method won't actually write the dex files,
// and the compiler will just re-use the existing vdex file.
- bool WriteDexFiles(OutputStream* out, File* file, bool update_input_vdex);
+ bool WriteDexFiles(OutputStream* out,
+ File* file,
+ bool update_input_vdex,
+ bool copy_dex_files);
bool WriteDexFile(OutputStream* out,
File* file,
OatDexFile* oat_dex_file,
@@ -341,7 +346,7 @@
bool MayHaveCompiledMethods() const;
bool VdexWillContainDexFiles() const {
- return dex_files_ != nullptr && !only_contains_uncompressed_zip_entries_;
+ return dex_files_ != nullptr && extract_dex_files_into_vdex_;
}
// Find the address of the GcRoot<String> in the InternTable for a boot image string.
@@ -375,8 +380,8 @@
const CompilerDriver* compiler_driver_;
ImageWriter* image_writer_;
const bool compiling_boot_image_;
- // Whether the dex files being compiled are all uncompressed in the APK.
- bool only_contains_uncompressed_zip_entries_;
+ // Whether the dex files being compiled are going to be extracted to the vdex.
+ bool extract_dex_files_into_vdex_;
// note OatFile does not take ownership of the DexFiles
const std::vector<const DexFile*>* dex_files_;
diff --git a/dex2oat/linker/oat_writer_test.cc b/dex2oat/linker/oat_writer_test.cc
index cd6ca51..6dd8e19 100644
--- a/dex2oat/linker/oat_writer_test.cc
+++ b/dex2oat/linker/oat_writer_test.cc
@@ -190,15 +190,17 @@
OutputStream* oat_rodata = elf_writer->StartRoData();
std::vector<std::unique_ptr<MemMap>> opened_dex_files_maps;
std::vector<std::unique_ptr<const DexFile>> opened_dex_files;
- if (!oat_writer.WriteAndOpenDexFiles(vdex_file,
- oat_rodata,
- compiler_driver_->GetInstructionSet(),
- compiler_driver_->GetInstructionSetFeatures(),
- &key_value_store,
- verify,
- /* update_input_vdex */ false,
- &opened_dex_files_maps,
- &opened_dex_files)) {
+ if (!oat_writer.WriteAndOpenDexFiles(
+ vdex_file,
+ oat_rodata,
+ compiler_driver_->GetInstructionSet(),
+ compiler_driver_->GetInstructionSetFeatures(),
+ &key_value_store,
+ verify,
+ /* update_input_vdex */ false,
+ compiler_driver_->GetCompilerOptions().GetCompilerFilter(),
+ &opened_dex_files_maps,
+ &opened_dex_files)) {
return false;
}
diff --git a/runtime/jit/profile_compilation_info.cc b/runtime/jit/profile_compilation_info.cc
index dcb4a20..21caa42 100644
--- a/runtime/jit/profile_compilation_info.cc
+++ b/runtime/jit/profile_compilation_info.cc
@@ -1195,22 +1195,10 @@
return kProfileLoadBadData;
}
- std::unique_ptr<MemMap> map;
- if (zip_entry->IsUncompressed()) {
- // Map uncompressed files within zip as file-backed to avoid a dirty copy.
- map.reset(zip_entry->MapDirectlyFromFile(kDexMetadataProfileEntry, error));
- if (map == nullptr) {
- LOG(WARNING) << "Can't mmap profile directly; "
- << "is your ZIP file corrupted? Falling back to extraction.";
- // Try again with Extraction which still has a chance of recovery.
- }
- }
-
- if (map == nullptr) {
- // Default path for compressed ZIP entries, and fallback for stored ZIP entries.
- // TODO(calin) pass along file names to assist with debugging.
- map.reset(zip_entry->ExtractToMemMap("profile file", kDexMetadataProfileEntry, error));
- }
+ // TODO(calin) pass along file names to assist with debugging.
+ std::unique_ptr<MemMap> map(zip_entry->MapDirectlyOrExtract(kDexMetadataProfileEntry,
+ "profile file",
+ error));
if (map != nullptr) {
source->reset(ProfileSource::Create(std::move(map)));
diff --git a/runtime/zip_archive.cc b/runtime/zip_archive.cc
index 2caed4b..5b3cab4 100644
--- a/runtime/zip_archive.cc
+++ b/runtime/zip_archive.cc
@@ -193,6 +193,19 @@
return map.release();
}
+MemMap* ZipEntry::MapDirectlyOrExtract(const char* zip_filename,
+ const char* entry_filename,
+ std::string* error_msg) {
+ if (IsUncompressed() && GetFileDescriptor(handle_) >= 0) {
+ MemMap* ret = MapDirectlyFromFile(zip_filename, error_msg);
+ if (ret != nullptr) {
+ return ret;
+ }
+ }
+ // Fall back to extraction for the failure case.
+ return ExtractToMemMap(zip_filename, entry_filename, error_msg);
+}
+
static void SetCloseOnExec(int fd) {
// This dance is more portable than Linux's O_CLOEXEC open(2) flag.
int flags = fcntl(fd, F_GETFD);
diff --git a/runtime/zip_archive.h b/runtime/zip_archive.h
index 70518e1..2ca4aa2 100644
--- a/runtime/zip_archive.h
+++ b/runtime/zip_archive.h
@@ -55,6 +55,10 @@
MemMap* MapDirectlyFromFile(const char* zip_filename, /*out*/std::string* error_msg);
virtual ~ZipEntry();
+ MemMap* MapDirectlyOrExtract(const char* zip_filename,
+ const char* entry_filename,
+ std::string* error_msg);
+
uint32_t GetUncompressedLength();
uint32_t GetCrc32();