diff options
author | 2023-02-27 19:15:07 +0000 | |
---|---|---|
committer | 2023-11-13 18:27:34 +0000 | |
commit | 854b363b284592d0bbfdbe31eb487fa64a4b9471 (patch) | |
tree | a5674430183437769a8a390bd1b4b2e532d2223a | |
parent | 0d3cf53ae0d400f7850d96798116dd3d4b963530 (diff) |
Add support for dex containers (DEX v41).
Allow multiple dex files within single "container"
(either a zip entry or a plain on-disk dex file).
This allows sharing of string (and other) dex data,
since the offsets can point to shared data payload.
Bug: 266950186
Test: test.py -b --host
Change-Id: I4f5901fd2f26a5a9dba427eb48c0fa5ddb6243d8
26 files changed, 392 insertions, 135 deletions
diff --git a/dex2oat/dex2oat.cc b/dex2oat/dex2oat.cc index 68a7d02fb9..774a108679 100644 --- a/dex2oat/dex2oat.cc +++ b/dex2oat/dex2oat.cc @@ -1584,7 +1584,7 @@ class Dex2Oat final { CHECK(!DexFileLoader::IsMultiDexLocation(bcp_dex_files[bcp_df_pos]->GetLocation())); ++bcp_df_pos; while (bcp_df_pos != bcp_df_end && - DexFileLoader::IsMultiDexLocation(bcp_dex_files[bcp_df_pos]->GetLocation().c_str())) { + DexFileLoader::IsMultiDexLocation(bcp_dex_files[bcp_df_pos]->GetLocation())) { ++bcp_df_pos; } } diff --git a/dex2oat/dex2oat_test.cc b/dex2oat/dex2oat_test.cc index ed57dc316c..0effe9148e 100644 --- a/dex2oat/dex2oat_test.cc +++ b/dex2oat/dex2oat_test.cc @@ -1320,6 +1320,10 @@ TEST_F(Dex2oatTest, LayoutSections) { ASSERT_EQ(oat_dex_files.size(), 1u); // Check that the code sections match what we expect. for (const OatDexFile* oat_dex : oat_dex_files) { + if (oat_dex->GetDexVersion() >= DexFile::kDexContainerVersion) { + continue; // Compact dex isn't supported together with dex container. + } + const DexLayoutSections* const sections = oat_dex->GetDexLayoutSections(); // Testing of logging the sections. ASSERT_TRUE(sections != nullptr); @@ -1433,6 +1437,10 @@ TEST_F(Dex2oatTest, GenerateCompactDex) { std::vector<std::unique_ptr<const CompactDexFile>> compact_dex_files; for (const OatDexFile* oat_dex : oat_dex_files) { std::unique_ptr<const DexFile> dex_file(oat_dex->OpenDexFile(&error_msg)); + if (dex_file->HasDexContainer()) { + ASSERT_FALSE(dex_file->IsCompactDexFile()); + continue; // Compact dex isn't supported together with dex container. + } ASSERT_TRUE(dex_file != nullptr) << error_msg; ASSERT_TRUE(dex_file->IsCompactDexFile()); compact_dex_files.push_back( @@ -2011,7 +2019,7 @@ TEST_F(Dex2oatWithExpectedFilterTest, AppImageEmptyDex) { ASSERT_GT(header->file_size_, sizeof(*header) + sizeof(dex::MapList) + sizeof(dex::MapItem) * 2); // Move map list to be right after the header. - header->map_off_ = sizeof(DexFile::Header); + header->map_off_ = header->header_size_; dex::MapList* map_list = const_cast<dex::MapList*>(dex->GetMapList()); map_list->list_[0].type_ = DexFile::kDexTypeHeaderItem; map_list->list_[0].size_ = 1u; @@ -2022,6 +2030,7 @@ TEST_F(Dex2oatWithExpectedFilterTest, AppImageEmptyDex) { map_list->size_ = 2; header->data_off_ = header->map_off_; header->data_size_ = map_list->Size(); + header->SetDexContainer(0, header->file_size_); }); } std::unique_ptr<const DexFile> dex_file(OpenDexFile(temp_dex.GetFilename().c_str())); diff --git a/dex2oat/linker/oat_writer.cc b/dex2oat/linker/oat_writer.cc index 5dc070ffdd..c5fcbc9636 100644 --- a/dex2oat/linker/oat_writer.cc +++ b/dex2oat/linker/oat_writer.cc @@ -3145,6 +3145,15 @@ bool OatWriter::WriteDexFiles(File* file, } } + // Compact dex reader/writer does not understand dex containers, + // which is ok since dex containers replace compat-dex. + for (OatDexFile& oat_dex_file : oat_dex_files_) { + const DexFile* dex_file = oat_dex_file.GetDexFile(); + if (dex_file->HasDexContainer()) { + compact_dex_level_ = CompactDexLevel::kCompactDexLevelNone; + } + } + if (extract_dex_files_into_vdex_) { vdex_dex_files_offset_ = vdex_size_; diff --git a/dexlayout/dex_writer.cc b/dexlayout/dex_writer.cc index 96fa9f2110..c1cf6cdfb6 100644 --- a/dexlayout/dex_writer.cc +++ b/dexlayout/dex_writer.cc @@ -789,7 +789,7 @@ void DexWriter::GenerateAndWriteMapItems(Stream* stream) { } void DexWriter::WriteHeader(Stream* stream) { - StandardDexFile::Header header; + StandardDexFile::HeaderV41 header{}; if (CompactDexFile::IsMagicValid(header_->Magic())) { StandardDexFile::WriteMagic(header.magic_.data()); if (header_->SupportDefaultMethods()) { @@ -823,15 +823,17 @@ void DexWriter::WriteHeader(Stream* stream) { header.class_defs_off_ = header_->ClassDefs().GetOffset(); header.data_size_ = header_->DataSize(); header.data_off_ = header_->DataOffset(); + header.SetDexContainer(0, header_->FileSize()); - CHECK_EQ(sizeof(header), GetHeaderSize()); - static_assert(sizeof(header) == 0x70, "Size doesn't match dex spec"); + static_assert(sizeof(header) == 0x78, "Size doesn't match dex spec"); stream->Seek(0); - stream->Overwrite(reinterpret_cast<uint8_t*>(&header), sizeof(header)); + stream->Overwrite(reinterpret_cast<uint8_t*>(&header), GetHeaderSize()); } size_t DexWriter::GetHeaderSize() const { - return sizeof(StandardDexFile::Header); + return header_->Magic() == DexFile::Magic{'d', 'e', 'x', '\n', '0', '4', '1', '\0'} ? + sizeof(StandardDexFile::HeaderV41) : + sizeof(StandardDexFile::Header); } bool DexWriter::Write(DexContainer* output, std::string* error_msg) { diff --git a/dexlayout/dexlayout.cc b/dexlayout/dexlayout.cc index 3db9cf3c54..e8d0a07e23 100644 --- a/dexlayout/dexlayout.cc +++ b/dexlayout/dexlayout.cc @@ -2197,6 +2197,14 @@ bool DexLayout::ProcessDexFile(const char* file_name, GetOptions())); SetHeader(header.get()); + // Dexlayout does not support containers, but allow it if it has just single dex file. + const DexFile::Header& hdr = dex_file->GetHeader(); + if (hdr.HeaderOffset() != 0u || hdr.ContainerSize() != hdr.file_size_) { + *error_msg = "DEX containers are not supported in dexlayout"; + DCHECK(false) << *error_msg; + return false; + } + if (options_.verbose_) { fprintf(out_file_, "Opened '%s', DEX version '%.3s'\n", diff --git a/dexlayout/dexlayout_test.cc b/dexlayout/dexlayout_test.cc index 4e4811ffa5..97c19ee147 100644 --- a/dexlayout/dexlayout_test.cc +++ b/dexlayout/dexlayout_test.cc @@ -746,6 +746,7 @@ TEST_F(DexLayoutTest, LinkData) { header.link_off_ = header.file_size_; header.link_size_ = 16 * KB; header.file_size_ += header.link_size_; + header.SetDexContainer(0, header.file_size_); file_size = header.file_size_; }); TEMP_FAILURE_RETRY(temp_dex.GetFile()->SetLength(file_size)); diff --git a/libdexfile/dex/code_item_accessors_test.cc b/libdexfile/dex/code_item_accessors_test.cc index a923d042b5..036036141b 100644 --- a/libdexfile/dex/code_item_accessors_test.cc +++ b/libdexfile/dex/code_item_accessors_test.cc @@ -41,6 +41,7 @@ std::unique_ptr<const DexFile> CreateFakeDex(bool compact_dex, std::vector<uint8 auto* header = reinterpret_cast<DexFile::Header*>(data->data()); StandardDexFile::WriteMagic(data->data()); StandardDexFile::WriteCurrentVersion(data->data()); + header->header_size_ = sizeof(*header); header->file_size_ = data->size(); } DexFileLoader dex_file_loader(data->data(), data->size(), "location"); diff --git a/libdexfile/dex/dex_file.cc b/libdexfile/dex/dex_file.cc index b27855e5bb..39addb5df9 100644 --- a/libdexfile/dex/dex_file.cc +++ b/libdexfile/dex/dex_file.cc @@ -34,6 +34,7 @@ #include "base/leb128.h" #include "base/stl_util.h" #include "class_accessor-inl.h" +#include "compact_dex_file.h" #include "descriptors_names.h" #include "dex_file-inl.h" #include "standard_dex_file.h" @@ -96,13 +97,51 @@ bool DexFile::DisableWrite() const { return container_->DisableWrite(); } +bool DexFile::Header::HasDexContainer() const { + if (CompactDexFile::IsMagicValid(magic_.data())) { + return false; + } + DCHECK_EQ(header_size_, GetVersion() >= 41 ? sizeof(HeaderV41) : sizeof(Header)); + return header_size_ >= sizeof(HeaderV41); +} + +uint32_t DexFile::Header::HeaderOffset() const { + return HasDexContainer() ? reinterpret_cast<const HeaderV41*>(this)->header_offset_ : 0; +} + +uint32_t DexFile::Header::ContainerSize() const { + return HasDexContainer() ? reinterpret_cast<const HeaderV41*>(this)->container_size_ : file_size_; +} + +void DexFile::Header::SetDexContainer(size_t header_offset, size_t container_size) { + if (HasDexContainer()) { + DCHECK_LE(header_offset, container_size); + DCHECK_LE(file_size_, container_size - header_offset); + data_off_ = 0; + data_size_ = 0; + auto* headerV41 = reinterpret_cast<HeaderV41*>(this); + DCHECK_GE(header_size_, sizeof(*headerV41)); + headerV41->header_offset_ = header_offset; + headerV41->container_size_ = container_size; + } else { + DCHECK_EQ(header_offset, 0u); + DCHECK_EQ(container_size, file_size_); + } +} + template <typename T> ALWAYS_INLINE const T* DexFile::GetSection(const uint32_t* offset, DexFileContainer* container) { size_t size = container->End() - begin_; if (size < sizeof(Header)) { return nullptr; // Invalid dex file. } - return reinterpret_cast<const T*>(begin_ + *offset); + // Compact dex is inconsistent: section offsets are relative to the + // header as opposed to the data section like all other its offsets. + if (CompactDexFile::IsMagicValid(begin_)) { + const uint8_t* data = reinterpret_cast<const uint8_t*>(header_); + return reinterpret_cast<const T*>(data + *offset); + } + return reinterpret_cast<const T*>(data_.data() + *offset); } DexFile::DexFile(const uint8_t* base, @@ -205,7 +244,14 @@ ArrayRef<const uint8_t> DexFile::GetDataRange(const uint8_t* data, DexFileContai size_t size = container->End() - data; if (size >= sizeof(StandardDexFile::Header) && StandardDexFile::IsMagicValid(data)) { auto header = reinterpret_cast<const DexFile::Header*>(data); - size = header->file_size_; + CHECK_EQ(container->Data().size(), 0u) << "Unsupported for standard dex"; + if (size >= sizeof(HeaderV41) && header->header_size_ >= sizeof(HeaderV41)) { + auto headerV41 = reinterpret_cast<const DexFile::HeaderV41*>(data); + data -= headerV41->header_offset_; // Allow underflow and later overflow. + size = headerV41->container_size_; + } else { + size = header->file_size_; + } } else if (size >= sizeof(CompactDexFile::Header) && CompactDexFile::IsMagicValid(data)) { auto header = reinterpret_cast<const CompactDexFile::Header*>(data); // TODO: Remove. This is a hack. See comment of the Data method. @@ -222,13 +268,18 @@ ArrayRef<const uint8_t> DexFile::GetDataRange(const uint8_t* data, DexFileContai } void DexFile::InitializeSectionsFromMapList() { + // NB: This function must survive random data to pass fuzzing and testing. static_assert(sizeof(MapList) <= sizeof(Header)); DCHECK_GE(DataSize(), sizeof(MapList)); if (header_->map_off_ == 0 || header_->map_off_ > DataSize() - sizeof(MapList)) { // Bad offset. The dex file verifier runs after this method and will reject the file. return; } - const MapList* map_list = reinterpret_cast<const MapList*>(DataBegin() + header_->map_off_); + const uint8_t* map_list_raw = DataBegin() + header_->map_off_; + if (map_list_raw < Begin()) { + return; + } + const MapList* map_list = reinterpret_cast<const MapList*>(map_list_raw); const size_t count = map_list->size_; size_t map_limit = @@ -244,10 +295,10 @@ void DexFile::InitializeSectionsFromMapList() { for (size_t i = 0; i < count; ++i) { const MapItem& map_item = map_list->list_[i]; if (map_item.type_ == kDexTypeMethodHandleItem) { - method_handles_ = reinterpret_cast<const MethodHandleItem*>(Begin() + map_item.offset_); + method_handles_ = GetSection<MethodHandleItem>(&map_item.offset_, container_.get()); num_method_handles_ = map_item.size_; } else if (map_item.type_ == kDexTypeCallSiteIdItem) { - call_site_ids_ = reinterpret_cast<const CallSiteIdItem*>(Begin() + map_item.offset_); + call_site_ids_ = GetSection<CallSiteIdItem>(&map_item.offset_, container_.get()); num_call_site_ids_ = map_item.size_; } else if (map_item.type_ == kDexTypeHiddenapiClassData) { hiddenapi_class_data_ = diff --git a/libdexfile/dex/dex_file.h b/libdexfile/dex/dex_file.h index 5068480001..5f4f98496d 100644 --- a/libdexfile/dex/dex_file.h +++ b/libdexfile/dex/dex_file.h @@ -116,6 +116,8 @@ class DexFile { static constexpr size_t kDexMagicSize = 4; static constexpr size_t kDexVersionLen = 4; + static constexpr uint32_t kDexContainerVersion = 41; + // First Dex format version enforcing class definition ordering rules. static constexpr uint32_t kClassDefinitionOrderEnforcedVersion = 37; @@ -162,6 +164,26 @@ class DexFile { // Decode the dex magic version uint32_t GetVersion() const; + + // Returns true for standard DEX version 41 or newer. + bool HasDexContainer() const; + + // Returns offset of this header within the container. + // Returns 0 for older dex versions without container. + uint32_t HeaderOffset() const; + + // Returns size of the whole container. + // Returns file_size_ for older dex versions without container. + uint32_t ContainerSize() const; + + // Set the DEX container fields to the given values. + // Must be [0, file_size_) for older dex versions. + void SetDexContainer(size_t header_offset, size_t container_size); + }; + + struct HeaderV41 : public Header { + uint32_t container_size_ = 0; // total size of all dex files in the container. + uint32_t header_offset_ = 0; // offset of this dex's header in the container. }; // Map item type codes. @@ -267,6 +289,20 @@ class DexFile { return GetHeader().GetVersion(); } + // Returns true if this is DEX V41 or later (i.e. supports container). + // Returns true even if the container contains just a single DEX file. + bool HasDexContainer() const { return GetHeader().HasDexContainer(); } + + // Returns the whole memory range of the DEX V41 container. + // Returns just the range of the DEX file for V40 or older. + ArrayRef<const uint8_t> GetDexContainerRange() const { + return {Begin() - header_->HeaderOffset(), header_->ContainerSize()}; + } + + bool IsDexContainerFirstEntry() const { return Begin() == GetDexContainerRange().begin(); } + + bool IsDexContainerLastEntry() const { return End() == GetDexContainerRange().end(); } + // Returns true if the byte string points to the magic value. virtual bool IsMagicValid() const = 0; @@ -765,12 +801,14 @@ class DexFile { bool DisableWrite() const; - const uint8_t* Begin() const { - return begin_; - } + const uint8_t* Begin() const { return begin_; } + + const uint8_t* End() const { return Begin() + Size(); } size_t Size() const { return header_->file_size_; } + size_t SizeIncludingSharedData() const { return GetDexContainerRange().end() - Begin(); } + static ArrayRef<const uint8_t> GetDataRange(const uint8_t* data, DexFileContainer* container); const uint8_t* DataBegin() const { return data_.data(); } @@ -888,6 +926,7 @@ class DexFile { // Data memory range: Most dex offsets are relative to this memory range. // Standard dex: same as (begin_, size_). + // Dex container: all dex files (starting from the first header). // Compact: shared data which is located after all non-shared data. // // This is different to the "data section" in the standard dex header. diff --git a/libdexfile/dex/dex_file_loader.cc b/libdexfile/dex/dex_file_loader.cc index 0266c41726..2fdb16eada 100644 --- a/libdexfile/dex/dex_file_loader.cc +++ b/libdexfile/dex/dex_file_loader.cc @@ -89,7 +89,11 @@ class MemMapContainer : public DexFileContainer { bool IsReadOnly() const override { return GetPermissions() == PROT_READ; } bool EnableWrite() override { - CHECK(IsReadOnly()); + if (!IsReadOnly()) { + // We can already write to the container. + // This method may be called multiple times by tests if DexFiles share container. + return true; + } if (!mem_map_.IsValid()) { return false; } else { @@ -150,10 +154,10 @@ std::string DexFileLoader::GetMultiDexClassesDexName(size_t index) { } std::string DexFileLoader::GetMultiDexLocation(size_t index, const char* dex_location) { + DCHECK(!IsMultiDexLocation(dex_location)); if (index == 0) { return dex_location; } - DCHECK(!IsMultiDexLocation(dex_location)); return StringPrintf("%s%cclasses%zu.dex", dex_location, kMultiDexSeparator, index + 1); } @@ -367,14 +371,15 @@ bool DexFileLoader::Open(bool verify, DCHECK(!error_msg->empty()); return false; } + size_t multidex_count = 0; for (size_t i = 0;; ++i) { std::string name = GetMultiDexClassesDexName(i); - std::string multidex_location = GetMultiDexLocation(i, location_.c_str()); bool ok = OpenFromZipEntry(*zip_archive, name.c_str(), - multidex_location, + location_, verify, verify_checksum, + &multidex_count, error_code, error_msg, dex_files); @@ -398,23 +403,32 @@ bool DexFileLoader::Open(bool verify, return false; } DCHECK(root_container_ != nullptr); - std::unique_ptr<const DexFile> dex_file = - OpenCommon(root_container_, - root_container_->Begin(), - root_container_->Size(), - location_, - /*location_checksum*/ {}, // Use default checksum from dex header. - /*oat_dex_file=*/nullptr, - verify, - verify_checksum, - error_msg, - nullptr); - if (dex_file.get() != nullptr) { + size_t header_offset = 0; + for (size_t i = 0;; i++) { + std::string multidex_location = GetMultiDexLocation(i, location_.c_str()); + std::unique_ptr<const DexFile> dex_file = + OpenCommon(root_container_, + root_container_->Begin() + header_offset, + root_container_->Size() - header_offset, + multidex_location, + /*location_checksum*/ {}, // Use default checksum from dex header. + /*oat_dex_file=*/nullptr, + verify, + verify_checksum, + error_msg, + error_code); + if (dex_file == nullptr) { + return false; + } dex_files->push_back(std::move(dex_file)); - return true; - } else { - return false; + size_t file_size = dex_files->back()->GetHeader().file_size_; + CHECK_LE(file_size, root_container_->Size() - header_offset); + header_offset += file_size; + if (dex_files->back()->IsDexContainerLastEntry()) { + break; + } } + return true; } *error_msg = StringPrintf("Expected valid zip or dex file"); return false; @@ -481,6 +495,7 @@ bool DexFileLoader::OpenFromZipEntry(const ZipArchive& zip_archive, const std::string& location, bool verify, bool verify_checksum, + size_t* multidex_count, DexFileLoaderErrorCode* error_code, std::string* error_msg, std::vector<std::unique_ptr<const DexFile>>* dex_files) const { @@ -538,21 +553,37 @@ bool DexFileLoader::OpenFromZipEntry(const ZipArchive& zip_archive, return false; } - std::unique_ptr<const DexFile> dex_file = OpenCommon(container, - container->Begin(), - container->Size(), - location, - zip_entry->GetCrc32(), - /*oat_dex_file=*/nullptr, - verify, - verify_checksum, - error_msg, - error_code); - if (dex_file == nullptr) { - return false; + size_t header_offset = 0; + for (size_t i = 0;; i++) { + std::string multidex_location = GetMultiDexLocation(*multidex_count, location.c_str()); + ++(*multidex_count); + uint32_t multidex_checksum = zip_entry->GetCrc32() + i; + std::unique_ptr<const DexFile> dex_file = OpenCommon(container, + container->Begin() + header_offset, + container->Size() - header_offset, + multidex_location, + multidex_checksum, + /*oat_dex_file=*/nullptr, + verify, + verify_checksum, + error_msg, + error_code); + if (dex_file == nullptr) { + return false; + } + if (dex_file->IsCompactDexFile()) { + *error_msg = StringPrintf("Can not open compact dex file from zip '%s'", location.c_str()); + return false; + } + CHECK(dex_file->IsReadOnly()) << multidex_location; + dex_files->push_back(std::move(dex_file)); + size_t file_size = dex_files->back()->GetHeader().file_size_; + CHECK_LE(file_size, container->Size() - header_offset); + header_offset += file_size; + if (dex_files->back()->IsDexContainerLastEntry()) { + break; + } } - CHECK(dex_file->IsReadOnly()) << location; - dex_files->push_back(std::move(dex_file)); return true; } diff --git a/libdexfile/dex/dex_file_loader.h b/libdexfile/dex/dex_file_loader.h index 1cc7d4a87a..f912053d8b 100644 --- a/libdexfile/dex/dex_file_loader.h +++ b/libdexfile/dex/dex_file_loader.h @@ -93,13 +93,19 @@ class DexFileLoader { CHECK_LT(*i, dex_files.size()) << "No dex files"; std::optional<uint32_t> checksum; for (; *i < dex_files.size(); ++(*i)) { - const char* location = dex_files[*i]->GetLocation().c_str(); + const auto* dex_file = &*dex_files[*i]; + bool is_primary_dex = !IsMultiDexLocation(dex_file->GetLocation().c_str()); if (!checksum.has_value()) { // First dex file. - CHECK(!IsMultiDexLocation(location)) << location; // Expect primary dex. - } else if (!IsMultiDexLocation(location)) { // Later dex file. + CHECK(is_primary_dex) << dex_file->GetLocation(); // Expect primary dex. + } else if (is_primary_dex) { // Later dex file. break; // Found another primary dex file, terminate iteration. } - checksum = checksum.value_or(kEmptyMultiDexChecksum) ^ dex_files[*i]->GetLocationChecksum(); + if (!is_primary_dex && dex_file->GetDexVersion() >= DexFile::kDexContainerVersion) { + if (dex_file->GetLocationChecksum() == dex_files[*i - 1]->GetLocationChecksum() + 1) { + continue; + } + } + checksum = checksum.value_or(kEmptyMultiDexChecksum) ^ dex_file->GetLocationChecksum(); } CHECK(checksum.has_value()); return checksum.value(); @@ -193,16 +199,18 @@ class DexFileLoader { bool verify, bool verify_checksum, std::string* error_msg) { - return Open( + std::unique_ptr<const DexFile> dex_file = Open( /*header_offset=*/0, location_checksum, oat_dex_file, verify, verify_checksum, error_msg); + // This API returns only singe DEX file, so check there is just single dex in the container. + CHECK(dex_file == nullptr || dex_file->IsDexContainerLastEntry()) << location_; + return dex_file; } std::unique_ptr<const DexFile> Open(uint32_t location_checksum, bool verify, bool verify_checksum, std::string* error_msg) { - return Open(/*header_offset=*/0, - location_checksum, + return Open(location_checksum, /*oat_dex_file=*/nullptr, verify, verify_checksum, @@ -304,9 +312,10 @@ class DexFileLoader { const std::string& location, bool verify, bool verify_checksum, - DexFileLoaderErrorCode* error_code, - std::string* error_msg, - std::vector<std::unique_ptr<const DexFile>>* dex_files) const; + /*inout*/ size_t* multidex_count, + /*out*/ DexFileLoaderErrorCode* error_code, + /*out*/ std::string* error_msg, + /*out*/ std::vector<std::unique_ptr<const DexFile>>* dex_files) const; // The DexFileLoader can be backed either by file or by memory (i.e. DexFileContainer). // We can not just mmap the file since APKs might be unreasonably large for 32-bit system. diff --git a/libdexfile/dex/dex_file_loader_test.cc b/libdexfile/dex/dex_file_loader_test.cc index 279411a02d..ecfd23afb9 100644 --- a/libdexfile/dex/dex_file_loader_test.cc +++ b/libdexfile/dex/dex_file_loader_test.cc @@ -111,17 +111,37 @@ static const char kRawDex40[] = "uAAAAAYAAAABAAAA0AAAAAEgAAACAAAA8AAAAAEQAAABAAAAHAEAAAIgAAAIAAAAIgEAAAMgAAAC" "AAAAcwEAAAAgAAABAAAAfgEAAAAQAAABAAAAjAEAAA=="; +// Taken from 001-Main. static const char kRawDex41[] = - "ZGV4CjA0MQC4OovJlJ1089ikzK6asMf/f8qp3Kve5VsgAgAAcAAAAHhWNBIAAAAAAAAAAIwBAAAI" - "AAAAcAAAAAQAAACQAAAAAgAAAKAAAAAAAAAAAAAAAAMAAAC4AAAAAQAAANAAAAAwAQAA8AAAACIB" - "AAAqAQAAMgEAAEYBAABRAQAAVAEAAFgBAABtAQAAAQAAAAIAAAAEAAAABgAAAAQAAAACAAAAAAAA" - "AAUAAAACAAAAHAEAAAAAAAAAAAAAAAABAAcAAAABAAAAAAAAAAAAAAABAAAAAQAAAAAAAAADAAAA" - "AAAAAH4BAAAAAAAAAQABAAEAAABzAQAABAAAAHAQAgAAAA4AAQABAAAAAAB4AQAAAQAAAA4AAAAB" - "AAAAAwAGPGluaXQ+AAZMTWFpbjsAEkxqYXZhL2xhbmcvT2JqZWN0OwAJTWFpbi5qYXZhAAFWAAJW" - "TAATW0xqYXZhL2xhbmcvU3RyaW5nOwAEbWFpbgABAAcOAAMBAAcOAAAAAgAAgYAE8AEBCYgCDAAA" - "AAAAAAABAAAAAAAAAAEAAAAIAAAAcAAAAAIAAAAEAAAAkAAAAAMAAAACAAAAoAAAAAUAAAADAAAA" - "uAAAAAYAAAABAAAA0AAAAAEgAAACAAAA8AAAAAEQAAABAAAAHAEAAAIgAAAIAAAAIgEAAAMgAAAC" - "AAAAcwEAAAAgAAABAAAAfgEAAAAQAAABAAAAjAEAAA=="; + "ZGV4CjA0MQBBaEGw/8clTiOn3IafJ++m20gViy5Peh7UAgAAeAAAAHhWNBIAAAAAAAAAAEACAAAK" + "AAAAeAAAAAQAAACgAAAAAgAAALAAAAAAAAAAAAAAAAMAAADIAAAAAQAAAOAAAAAAAAAAAAAAANQC" + "AAAAAAAAOgEAAEIBAABKAQAAXgEAAGkBAABsAQAAcAEAAIUBAACLAQAAkQEAAAEAAAACAAAABAAA" + "AAYAAAAEAAAAAgAAAAAAAAAFAAAAAgAAADQBAAAAAAAAAAAAAAAAAQAIAAAAAQAAAAAAAAAAAAAA" + "AQAAAAEAAAAAAAAAAwAAAAAAAAAxAgAAAAAAAAEAAQABAAAAKgEAAAQAAABwEAIAAAAOAAEAAQAA" + "AAAALgEAAAEAAAAOABEADgATAQgOAAABAAAAAwAGPGluaXQ+AAZMTWFpbjsAEkxqYXZhL2xhbmcv" + "T2JqZWN0OwAJTWFpbi5qYXZhAAFWAAJWTAATW0xqYXZhL2xhbmcvU3RyaW5nOwAEYXJncwAEbWFp" + "bgCdAX5+RDh7ImJhY2tlbmQiOiJkZXgiLCJjb21waWxhdGlvbi1tb2RlIjoiZGVidWciLCJoYXMt" + "Y2hlY2tzdW1zIjpmYWxzZSwibWluLWFwaSI6MjYsInNoYS0xIjoiNTRjYmIzMTZlNGI3OWFhMDM1" + "ZDUwMTM4ZTI3NjY4OGJiOTM5ZGIwNCIsInZlcnNpb24iOiI4LjMuMTQtZGV2In0AAAACAACBgASA" + "AgEJmAIADAAAAAAAAAABAAAAAAAAAAEAAAAKAAAAeAAAAAIAAAAEAAAAoAAAAAMAAAACAAAAsAAA" + "AAUAAAADAAAAyAAAAAYAAAABAAAA4AAAAAEgAAACAAAAAAEAAAMgAAACAAAAKgEAAAEQAAABAAAA" + "NAEAAAIgAAAKAAAAOgEAAAAgAAABAAAAMQIAAAAQAAABAAAAQAIAAA=="; + +// Taken from 001-Main and modified. +static const char kRawDex42[] = + "ZGV4CjA0MgBBaEGw/8clTiOn3IafJ++m20gViy5Peh7UAgAAeAAAAHhWNBIAAAAAAAAAAEACAAAK" + "AAAAeAAAAAQAAACgAAAAAgAAALAAAAAAAAAAAAAAAAMAAADIAAAAAQAAAOAAAAAAAAAAAAAAANQC" + "AAAAAAAAOgEAAEIBAABKAQAAXgEAAGkBAABsAQAAcAEAAIUBAACLAQAAkQEAAAEAAAACAAAABAAA" + "AAYAAAAEAAAAAgAAAAAAAAAFAAAAAgAAADQBAAAAAAAAAAAAAAAAAQAIAAAAAQAAAAAAAAAAAAAA" + "AQAAAAEAAAAAAAAAAwAAAAAAAAAxAgAAAAAAAAEAAQABAAAAKgEAAAQAAABwEAIAAAAOAAEAAQAA" + "AAAALgEAAAEAAAAOABEADgATAQgOAAABAAAAAwAGPGluaXQ+AAZMTWFpbjsAEkxqYXZhL2xhbmcv" + "T2JqZWN0OwAJTWFpbi5qYXZhAAFWAAJWTAATW0xqYXZhL2xhbmcvU3RyaW5nOwAEYXJncwAEbWFp" + "bgCdAX5+RDh7ImJhY2tlbmQiOiJkZXgiLCJjb21waWxhdGlvbi1tb2RlIjoiZGVidWciLCJoYXMt" + "Y2hlY2tzdW1zIjpmYWxzZSwibWluLWFwaSI6MjYsInNoYS0xIjoiNTRjYmIzMTZlNGI3OWFhMDM1" + "ZDUwMTM4ZTI3NjY4OGJiOTM5ZGIwNCIsInZlcnNpb24iOiI4LjMuMTQtZGV2In0AAAACAACBgASA" + "AgEJmAIADAAAAAAAAAABAAAAAAAAAAEAAAAKAAAAeAAAAAIAAAAEAAAAoAAAAAMAAAACAAAAsAAA" + "AAUAAAADAAAAyAAAAAYAAAABAAAA4AAAAAEgAAACAAAAAAEAAAMgAAACAAAAKgEAAAEQAAABAAAA" + "NAEAAAIgAAAKAAAAOgEAAAAgAAABAAAAMQIAAAAQAAABAAAAQAIAAA=="; static const char kRawDexZeroLength[] = "UEsDBAoAAAAAAOhxAkkAAAAAAAAAAAAAAAALABwAY2xhc3Nlcy5kZXhVVAkAA2QNoVdnDaFXdXgL" @@ -324,7 +344,8 @@ static bool OpenDexFilesBase64(const char* base64, static std::unique_ptr<const DexFile> OpenDexFileBase64(const char* base64, const char* location, - std::vector<uint8_t>* dex_bytes) { + std::vector<uint8_t>* dex_bytes, + size_t expected_dex_files = 1) { // read dex files. DexFileLoaderErrorCode error_code; std::string error_msg; @@ -332,7 +353,7 @@ static std::unique_ptr<const DexFile> OpenDexFileBase64(const char* base64, bool success = OpenDexFilesBase64(base64, location, dex_bytes, &dex_files, &error_code, &error_msg); CHECK(success) << error_msg; - EXPECT_EQ(1U, dex_files.size()); + EXPECT_EQ(expected_dex_files, dex_files.size()); return std::move(dex_files[0]); } @@ -433,9 +454,18 @@ TEST_F(DexFileLoaderTest, Version40Accepted) { EXPECT_EQ(40u, header.GetVersion()); } -TEST_F(DexFileLoaderTest, Version41Rejected) { +TEST_F(DexFileLoaderTest, Version41Accepted) { + std::vector<uint8_t> dex_bytes; + std::unique_ptr<const DexFile> raw(OpenDexFileBase64(kRawDex41, kLocationString, &dex_bytes, 1)); + ASSERT_TRUE(raw.get() != nullptr); + + const DexFile::Header& header = raw->GetHeader(); + EXPECT_EQ(41u, header.GetVersion()); +} + +TEST_F(DexFileLoaderTest, Version42Rejected) { std::vector<uint8_t> dex_bytes; - DecodeDexFile(kRawDex41, &dex_bytes); + DecodeDexFile(kRawDex42, &dex_bytes); static constexpr bool kVerifyChecksum = true; DexFileLoaderErrorCode error_code; diff --git a/libdexfile/dex/dex_file_verifier.cc b/libdexfile/dex/dex_file_verifier.cc index b94d7ce6b2..1a6ff81c68 100644 --- a/libdexfile/dex/dex_file_verifier.cc +++ b/libdexfile/dex/dex_file_verifier.cc @@ -127,8 +127,8 @@ class DexFileVerifier { public: DexFileVerifier(const DexFile* dex_file, const char* location, bool verify_checksum) : dex_file_(dex_file), - offset_base_address_(dex_file->Begin()), - size_(0), // Initialized after we verify the header. + offset_base_address_(dex_file->DataBegin()), + size_(dex_file->DataSize()), location_(location), verify_checksum_(verify_checksum), header_(&dex_file->GetHeader()), @@ -150,6 +150,7 @@ class DexFileVerifier { private: template <class T = uint8_t> ALWAYS_INLINE const T* OffsetToPtr(size_t offset) { + DCHECK_GE(offset, static_cast<size_t>(dex_file_->Begin() - offset_base_address_)); DCHECK_LE(offset, size_); return reinterpret_cast<const T*>(offset_base_address_ + offset); } @@ -385,11 +386,12 @@ class DexFileVerifier { const DexFile* const dex_file_; const uint8_t* const offset_base_address_; - size_t size_; + const size_t size_; ArrayRef<const uint8_t> data_; // The "data" section of the dex file. const char* const location_; const bool verify_checksum_; const DexFile::Header* const header_; + uint32_t dex_version_ = 0; struct OffsetTypeMapEmptyFn { // Make a hash map slot empty by making the offset 0. Offset 0 is a valid dex file offset that @@ -570,6 +572,11 @@ bool DexFileVerifier::CheckValidOffsetAndSize(uint32_t offset, return false; } } + size_t hdr_offset = PtrToOffset(header_); + if (offset < hdr_offset) { + ErrorStringPrintf("Offset(%d) should be after header(%zu) for %s.", offset, hdr_offset, label); + return false; + } if (size_ <= offset) { ErrorStringPrintf("Offset(%d) should be within file size(%zu) for %s.", offset, size_, label); return false; @@ -603,10 +610,11 @@ bool DexFileVerifier::CheckHeader() { ErrorStringPrintf("Unknown dex version"); return false; } + dex_version_ = header_->GetVersion(); // Check file size from the header. size_t file_size = header_->file_size_; - size_t header_size = sizeof(DexFile::Header); + size_t header_size = (dex_version_ >= 41) ? sizeof(DexFile::HeaderV41) : sizeof(DexFile::Header); if (file_size < header_size) { ErrorStringPrintf("Bad file size (%zu, expected at least %zu)", file_size, header_size); return false; @@ -616,7 +624,6 @@ bool DexFileVerifier::CheckHeader() { return false; } CHECK_GE(size, header_size); // Implied by the two checks above. - size_ = file_size; // Check header size. if (header_->header_size_ != header_size) { @@ -642,6 +649,22 @@ bool DexFileVerifier::CheckHeader() { } } + if (dex_version_ >= 41) { + auto headerV41 = reinterpret_cast<const DexFile::HeaderV41*>(header_); + if (headerV41->container_size_ <= headerV41->header_offset_) { + ErrorStringPrintf("Dex container is too small: size=%ud header_offset=%ud", + headerV41->container_size_, + headerV41->header_offset_); + return false; + } + uint32_t remainder = headerV41->container_size_ - headerV41->header_offset_; + if (headerV41->file_size_ > remainder) { + ErrorStringPrintf( + "Header file_size(%ud) is past multi-dex size(%ud)", headerV41->file_size_, remainder); + return false; + } + } + // Check that all offsets are inside the file. bool ok = CheckValidOffsetAndSize(header_->link_off_, @@ -686,7 +709,9 @@ bool DexFileVerifier::CheckHeader() { "data"); if (ok) { - data_ = ArrayRef<const uint8_t>(OffsetToPtr(header_->data_off_), header_->data_size_); + data_ = (dex_version_ >= 41) + ? ArrayRef<const uint8_t>(dex_file_->Begin(), EndOfFile() - dex_file_->Begin()) + : ArrayRef<const uint8_t>(OffsetToPtr(header_->data_off_), header_->data_size_); } return ok; } @@ -722,9 +747,8 @@ bool DexFileVerifier::CheckMap() { last_type); return false; } - if (UNLIKELY(item->offset_ >= header_->file_size_)) { - ErrorStringPrintf("Map item after end of file: %x, size %x", - item->offset_, header_->file_size_); + if (UNLIKELY(item->offset_ >= size_)) { + ErrorStringPrintf("Map item after end of file: %x, size %zx", item->offset_, size_); return false; } @@ -961,6 +985,10 @@ bool DexFileVerifier::CheckPadding(uint32_t aligned_offset, if (!CheckListSize(OffsetToPtr(offset), aligned_offset - offset, sizeof(uint8_t), "section")) { return false; } + if (dex_version_ >= 41) { + ptr_ += aligned_offset - offset; + return true; + } while (offset < aligned_offset) { if (UNLIKELY(*ptr_ != '\0')) { ErrorStringPrintf("Non-zero padding %x before section of type %zu at offset 0x%zx", @@ -2290,17 +2318,19 @@ bool DexFileVerifier::CheckIntraSection() { // Check each item based on its type. switch (type) { - case DexFile::kDexTypeHeaderItem: + case DexFile::kDexTypeHeaderItem: { if (UNLIKELY(section_count != 1)) { ErrorStringPrintf("Multiple header items"); return false; } - if (UNLIKELY(section_offset != 0)) { - ErrorStringPrintf("Header at %x, not at start of file", section_offset); + uint32_t expected = dex_version_ >= 41 ? PtrToOffset(dex_file_->Begin()) : 0; + if (UNLIKELY(section_offset != expected)) { + ErrorStringPrintf("Header at %x, expected %x", section_offset, expected); return false; } ptr_ = OffsetToPtr(header_->header_size_); break; + } #define CHECK_INTRA_ID_SECTION_CASE(type) \ case type: \ diff --git a/libdexfile/dex/standard_dex_file.cc b/libdexfile/dex/standard_dex_file.cc index 912cff6084..4dc96d487d 100644 --- a/libdexfile/dex/standard_dex_file.cc +++ b/libdexfile/dex/standard_dex_file.cc @@ -24,18 +24,20 @@ namespace art { const uint8_t StandardDexFile::kDexMagic[] = { 'd', 'e', 'x', '\n' }; -const uint8_t StandardDexFile::kDexMagicVersions[StandardDexFile::kNumDexVersions] - [StandardDexFile::kDexVersionLen] = { - {'0', '3', '5', '\0'}, - // Dex version 036 skipped because of an old dalvik bug on some versions of android where dex - // files with that version number would erroneously be accepted and run. - {'0', '3', '7', '\0'}, - // Dex version 038: Android "O" and beyond. - {'0', '3', '8', '\0'}, - // Dex version 039: Android "P" and beyond. - {'0', '3', '9', '\0'}, - // Dex version 040: beyond Android "10" (previously known as Android "Q"). - {'0', '4', '0', '\0'}, +const uint8_t StandardDexFile::kDexMagicVersions + [StandardDexFile::kNumDexVersions][StandardDexFile::kDexVersionLen] = { + {'0', '3', '5', '\0'}, + // Dex version 036 skipped because of an old dalvik bug on some versions of android where + // dex files with that version number would erroneously be accepted and run. + {'0', '3', '7', '\0'}, + // Dex version 038: Android "O" and beyond. + {'0', '3', '8', '\0'}, + // Dex version 039: Android "P" and beyond. + {'0', '3', '9', '\0'}, + // Dex version 040: Android "Q" and beyond (aka Android 10). + {'0', '4', '0', '\0'}, + // Dex version 041: Android "V" and beyond (aka Android 15). + {'0', '4', '1', '\0'}, }; void StandardDexFile::WriteMagic(uint8_t* magic) { diff --git a/libdexfile/dex/standard_dex_file.h b/libdexfile/dex/standard_dex_file.h index 05f7d41c15..0b12c186f6 100644 --- a/libdexfile/dex/standard_dex_file.h +++ b/libdexfile/dex/standard_dex_file.h @@ -91,7 +91,7 @@ class StandardDexFile : public DexFile { static void WriteVersionBeforeDefaultMethods(uint8_t* magic); static const uint8_t kDexMagic[kDexMagicSize]; - static constexpr size_t kNumDexVersions = 5; + static constexpr size_t kNumDexVersions = 6; static const uint8_t kDexMagicVersions[kNumDexVersions][kDexVersionLen]; // Returns true if the byte string points to the magic value. diff --git a/libdexfile/external/dex_file_ext.cc b/libdexfile/external/dex_file_ext.cc index e71233c1cf..774755f9a1 100644 --- a/libdexfile/external/dex_file_ext.cc +++ b/libdexfile/external/dex_file_ext.cc @@ -155,6 +155,13 @@ ADexFile_Error ADexFile_create(const void* _Nonnull address, } const art::DexFile::Header* header = reinterpret_cast<const art::DexFile::Header*>(address); + if (size < header->header_size_) { + if (new_size != nullptr) { + *new_size = header->header_size_; + } + return ADEXFILE_ERROR_NOT_ENOUGH_DATA; + } + uint32_t dex_size = header->file_size_; // Size of "one dex file" excluding any shared data. uint32_t full_size = dex_size; // Includes referenced shared data past the end of dex. if (art::CompactDexFile::IsMagicValid(header->magic_)) { @@ -169,7 +176,9 @@ ADexFile_Error ADexFile_create(const void* _Nonnull address, if (computed_file_size > full_size) { full_size = computed_file_size; } - } else if (!art::StandardDexFile::IsMagicValid(header->magic_)) { + } else if (art::StandardDexFile::IsMagicValid(header->magic_)) { + full_size = header->ContainerSize() - header->HeaderOffset(); + } else { return ADEXFILE_ERROR_INVALID_HEADER; } diff --git a/oatdump/oatdump.cc b/oatdump/oatdump.cc index 512fe335fe..2f23cab13f 100644 --- a/oatdump/oatdump.cc +++ b/oatdump/oatdump.cc @@ -647,6 +647,11 @@ class OatDumper { CHECK(oat_dex_file != nullptr); CHECK(vdex_dex_file != nullptr); + if (!vdex_dex_file->IsDexContainerFirstEntry()) { + // All the data was already exported together with the primary dex file. + continue; + } + // If a CompactDex file is detected within a Vdex container, DexLayout is used to convert // back to a StandardDex file. Since the converted DexFile will most likely not reproduce // the original input Dex file, the `update_checksum_` option is used to recompute the @@ -993,6 +998,9 @@ class OatDumper { return false; } } + // Extend the data range to export all the dex files in the container. + CHECK(dex_file->IsDexContainerFirstEntry()) << dex_file_location; + fsize = dex_file->GetHeader().ContainerSize(); } // Verify output directory exists diff --git a/openjdkjvmti/ti_class_definition.cc b/openjdkjvmti/ti_class_definition.cc index a9e7b3191a..72fe7c7193 100644 --- a/openjdkjvmti/ti_class_definition.cc +++ b/openjdkjvmti/ti_class_definition.cc @@ -106,7 +106,8 @@ jvmtiError ArtClassDefinition::Init(art::Thread* self, jclass klass) { dex_data_ = art::ArrayRef<const unsigned char>(dex_data_memory_); const art::DexFile& cur_dex = m_klass->GetDexFile(); - current_dex_file_ = art::ArrayRef<const unsigned char>(cur_dex.Begin(), cur_dex.Size()); + current_dex_file_ = + art::ArrayRef<const unsigned char>(cur_dex.Begin(), cur_dex.SizeIncludingSharedData()); return OK; } @@ -132,7 +133,8 @@ jvmtiError ArtClassDefinition::Init(art::Thread* self, jclass klass) { } } const art::DexFile& cur_dex = m_klass->GetDexFile(); - current_dex_file_ = art::ArrayRef<const unsigned char>(cur_dex.Begin(), cur_dex.Size()); + current_dex_file_ = + art::ArrayRef<const unsigned char>(cur_dex.Begin(), cur_dex.SizeIncludingSharedData()); return OK; } } @@ -189,8 +191,8 @@ jvmtiError ArtClassDefinition::Init(const art::DexFile& dex_file) { } } // Keep the dex_data alive. - dex_data_memory_.resize(original_dex_file->Size()); - memcpy(dex_data_memory_.data(), original_dex_file->Begin(), original_dex_file->Size()); + dex_data_memory_.resize(original_dex_file->SizeIncludingSharedData()); + memcpy(dex_data_memory_.data(), original_dex_file->Begin(), dex_data_memory_.size()); dex_data_ = art::ArrayRef<const unsigned char>(dex_data_memory_); // In case dex_data gets re-used for redefinition, keep the dex file live @@ -200,7 +202,8 @@ jvmtiError ArtClassDefinition::Init(const art::DexFile& dex_file) { current_dex_file_ = art::ArrayRef<const unsigned char>(current_dex_memory_); } else { // Dex file will always stay live, use it directly. - dex_data_ = art::ArrayRef<const unsigned char>(dex_file.Begin(), dex_file.Size()); + dex_data_ = + art::ArrayRef<const unsigned char>(dex_file.Begin(), dex_file.SizeIncludingSharedData()); current_dex_file_ = dex_data_; } return OK; diff --git a/runtime/dex2oat_environment_test.h b/runtime/dex2oat_environment_test.h index 43e1bf816e..54a007f891 100644 --- a/runtime/dex2oat_environment_test.h +++ b/runtime/dex2oat_environment_test.h @@ -131,8 +131,14 @@ class Dex2oatEnvironmentTest : public Dex2oatScratchDirs, public CommonRuntimeTe ASSERT_EQ(multi1[0]->GetHeader().checksum_, multi2[0]->GetHeader().checksum_); ASSERT_NE(multi1[1]->GetHeader().checksum_, multi2[1]->GetHeader().checksum_); - ASSERT_EQ(multi1[0]->GetLocationChecksum(), multi2[0]->GetLocationChecksum()); - ASSERT_NE(multi1[1]->GetLocationChecksum(), multi2[1]->GetLocationChecksum()); + if (multi1[0]->HasDexContainer()) { + // Checksum is the CRC of the whole container, so both of them should differ. + ASSERT_NE(multi1[0]->GetLocationChecksum(), multi2[0]->GetLocationChecksum()); + ASSERT_NE(multi1[1]->GetLocationChecksum(), multi2[1]->GetLocationChecksum()); + } else { + ASSERT_EQ(multi1[0]->GetLocationChecksum(), multi2[0]->GetLocationChecksum()); + ASSERT_NE(multi1[1]->GetLocationChecksum(), multi2[1]->GetLocationChecksum()); + } } void SetUpRuntimeOptions(RuntimeOptions* options) override { diff --git a/runtime/oat_file.cc b/runtime/oat_file.cc index 4cee0d4f71..97164f725e 100644 --- a/runtime/oat_file.cc +++ b/runtime/oat_file.cc @@ -17,6 +17,7 @@ #include "oat_file.h" #include <dlfcn.h> + #ifndef __APPLE__ #include <link.h> // for dl_iterate_phdr. #endif @@ -2211,7 +2212,7 @@ void OatDexFile::InitializeTypeLookupTable() { // Initialize TypeLookupTable. if (lookup_table_data_ != nullptr) { // Peek the number of classes from the DexFile. - const DexFile::Header* dex_header = reinterpret_cast<const DexFile::Header*>(dex_file_pointer_); + auto* dex_header = reinterpret_cast<const DexFile::Header*>(dex_file_pointer_); const uint32_t num_class_defs = dex_header->class_defs_size_; if (lookup_table_data_ + TypeLookupTable::RawDataLength(num_class_defs) > GetOatFile()->DexEnd()) { @@ -2219,6 +2220,9 @@ void OatDexFile::InitializeTypeLookupTable() { } else { const uint8_t* dex_data = dex_file_pointer_; // TODO: Clean this up to create the type lookup table after the dex file has been created? + if (StandardDexFile::IsMagicValid(dex_header->magic_)) { + dex_data -= dex_header->HeaderOffset(); + } if (CompactDexFile::IsMagicValid(dex_header->magic_)) { dex_data += dex_header->data_off_; } @@ -2564,6 +2568,10 @@ void OatDexFile::AssertAotCompiler() { CHECK(Runtime::Current()->IsAotCompiler()); } +uint32_t OatDexFile::GetDexVersion() const { + return atoi(reinterpret_cast<const char*>(&dex_file_magic_[4])); +} + bool OatFile::IsBackedByVdexOnly() const { return oat_dex_files_storage_.size() >= 1 && oat_dex_files_storage_[0]->IsBackedByVdexOnly(); } diff --git a/runtime/oat_file.h b/runtime/oat_file.h index 806e616770..2e1bd2e1ff 100644 --- a/runtime/oat_file.h +++ b/runtime/oat_file.h @@ -528,6 +528,8 @@ class OatDexFile final { DexFile::Magic GetMagic() const { return dex_file_magic_; } + uint32_t GetDexVersion() const; + // Returns checksum of original DexFile that was the source of this OatDexFile; uint32_t GetDexFileLocationChecksum() const { return dex_file_location_checksum_; diff --git a/test/180-native-default-method/build.py b/test/180-native-default-method/build.py index 62aac82103..d88008f070 100644 --- a/test/180-native-default-method/build.py +++ b/test/180-native-default-method/build.py @@ -14,7 +14,7 @@ # limitations under the License. def build(ctx): - ctx.default_build() + ctx.default_build(d8_dex_container=False) if ctx.jvm: return # Change the generated dex file to have a v35 magic number if it is version 38 diff --git a/test/MultiDex/Second.java b/test/MultiDex/Second.java index 5067bcc2e8..ec8849a402 100644 --- a/test/MultiDex/Second.java +++ b/test/MultiDex/Second.java @@ -16,12 +16,6 @@ class Second { public String getSecond() { - return "I Second That."; - } - - // This method makes sure the second dex file has quickening - // instructions. - public String callSecond() { - return getSecond(); + return "Original"; } } diff --git a/test/MultiDexModifiedSecondary/Second.java b/test/MultiDexModifiedSecondary/Second.java index 3555a7ff36..08ad625bf0 100644 --- a/test/MultiDexModifiedSecondary/Second.java +++ b/test/MultiDexModifiedSecondary/Second.java @@ -15,11 +15,7 @@ */ class Second { - public String getThird() { - return "I Third That."; - } - public String getSecond() { - return "I Second That."; + return "Modified"; // Must have the same length as original so that dex size is same. } } diff --git a/test/run_test_build.py b/test/run_test_build.py index e1f6e731c2..7dfec8a0ac 100755 --- a/test/run_test_build.py +++ b/test/run_test_build.py @@ -216,6 +216,7 @@ class BuildTestContext: javac_args=[], javac_classpath: List[Path]=[], d8_flags=[], + d8_dex_container=True, smali_args=[], use_smali=True, use_jasmin=True, @@ -305,7 +306,10 @@ class BuildTestContext: # packaged in a jar file. def make_dex(src_dir: Path): dst_jar = Path(src_dir.name + ".jar") - args = d8_flags + ["--min-api", str(api_level), "--output", dst_jar] + args = [] + if d8_dex_container: + args += ["-JDcom.android.tools.r8.dexContainerExperiment"] + args += d8_flags + ["--min-api", str(api_level), "--output", dst_jar] args += ["--lib", self.bootclasspath] if use_desugar else ["--no-desugaring"] args += sorted(src_dir.glob("**/*.class")) self.d8(args) @@ -328,7 +332,11 @@ class BuildTestContext: # It is useful to normalize non-deterministic smali output. tmp_dir = self.test_dir / "dexmerge" tmp_dir.mkdir() - self.d8(["--min-api", str(api_level), "--output", tmp_dir] + srcs) + flags = [] + if d8_dex_container: + flags += ["-JDcom.android.tools.r8.dexContainerExperiment"] + flags += ["--min-api", str(api_level), "--output", tmp_dir] + self.d8(flags + srcs) assert not (tmp_dir / "classes2.dex").exists() for src_file in srcs: src_file.unlink() diff --git a/tools/hiddenapi/hiddenapi.cc b/tools/hiddenapi/hiddenapi.cc index 4c42e1317d..a1fea05f82 100644 --- a/tools/hiddenapi/hiddenapi.cc +++ b/tools/hiddenapi/hiddenapi.cc @@ -299,14 +299,14 @@ class ClassPath final { for (const std::string& filename : dex_paths) { DexFileLoader dex_file_loader(filename); + DexFileLoaderErrorCode error_code; bool success = dex_file_loader.Open(/* verify= */ true, /* verify_checksum= */ true, + /*allow_no_dex_files=*/ ignore_empty, + &error_code, &error_msg, &dex_files_); - // If requested ignore a jar with no classes.dex files. - if (!success && ignore_empty && error_msg != "Entry not found") { - CHECK(success) << "Open failed for '" << filename << "' " << error_msg; - } + CHECK(success) << "Open failed for '" << filename << "' " << error_msg; } } @@ -660,16 +660,15 @@ class DexFileEditor final { // Writes the edited dex file into a file. void WriteTo(const std::string& path) { + CHECK_GT(inputs_.size(), 0u); std::vector<uint8_t> output; // Copy the old dex files into the backing data vector. - size_t truncated_size = 0; std::vector<size_t> header_offset; for (size_t i = 0; i < inputs_.size(); i++) { const DexFile* dex = inputs_[i].first; header_offset.push_back(output.size()); - std::copy( - dex->Begin(), dex->Begin() + dex->GetHeader().file_size_, std::back_inserter(output)); + std::copy(dex->Begin(), dex->End(), std::back_inserter(output)); // Clear the old map list (make it into padding). const dex::MapList* map = dex->GetMapList(); @@ -678,9 +677,7 @@ class DexFileEditor final { CHECK_LE(map_off, output.size()) << "Map list past the end of file"; CHECK_EQ(map_size, output.size() - map_off) << "Map list expected at the end of file"; std::fill_n(output.data() + map_off, map_size, 0); - truncated_size = output.size() - map_size; } - output.resize(truncated_size); // Truncate last map list. // Append the hidden api data into the backing data vector. std::vector<size_t> hiddenapi_offset; @@ -691,7 +688,8 @@ class DexFileEditor final { std::copy(hiddenapi_data.begin(), hiddenapi_data.end(), std::back_inserter(output)); } - // Update the dex headers and map lists. + // Append modified map lists. + std::vector<uint32_t> map_list_offset; for (size_t i = 0; i < inputs_.size(); i++) { output.resize(RoundUp(output.size(), kMapListAlignment)); // Align. @@ -716,16 +714,18 @@ class DexFileEditor final { uint32_t payload_offset = hiddenapi_offset[i]; items.push_back(dex::MapItem{DexFile::kDexTypeHiddenapiClassData, 0, 1u, payload_offset}); } - uint32_t map_offset = output.size(); - items.push_back(dex::MapItem{DexFile::kDexTypeMapList, 0, 1u, map_offset}); + map_list_offset.push_back(output.size()); + items.push_back(dex::MapItem{DexFile::kDexTypeMapList, 0, 1u, map_list_offset.back()}); uint32_t item_count = items.size(); Append(&output, &item_count, 1); Append(&output, items.data(), items.size()); + } - // Update header. + // Update headers. + for (size_t i = 0; i < inputs_.size(); i++) { uint8_t* begin = output.data() + header_offset[i]; auto* header = reinterpret_cast<DexFile::Header*>(begin); - header->map_off_ = map_offset; + header->map_off_ = map_list_offset[i]; if (i + 1 < inputs_.size()) { CHECK_EQ(header->file_size_, header_offset[i + 1] - header_offset[i]); } else { @@ -733,6 +733,7 @@ class DexFileEditor final { header->data_size_ = output.size() - header->data_off_; header->file_size_ = output.size() - header_offset[i]; } + header->SetDexContainer(header_offset[i], output.size()); size_t sha1_start = offsetof(DexFile::Header, file_size_); SHA1(begin + sha1_start, header->file_size_ - sha1_start, header->signature_.data()); header->checksum_ = DexFile::CalculateChecksum(begin, header->file_size_); |