Reject compact dex files on load.

We don't want to bump the vdex version, so check all dex files inside a
vdex directly when it's loaded.

Test: atest art_standalone_runtime_tests art_standalone_libdexfile_tests
Bug: 325430813
Bug: 399472993
Change-Id: Icf63cba35a30c609b369e80781e7de732b3a0605

5 files changed, 24 insertions, 36 deletions

diff --git a/libdexfile/dex/code_item_accessors_test.cc b/libdexfile/dex/code_item_accessors_test.cc
index a815d06cb8..9f20fc2495 100644
--- a/libdexfile/dex/code_item_accessors_test.cc
+++ b/libdexfile/dex/code_item_accessors_test.cc
@@ -61,10 +61,6 @@ TEST(CodeItemAccessorsTest, TestDexInstructionsAccessor) {
   std::unique_ptr<const DexFile> standard_dex(CreateFakeDex(/*compact_dex=*/false,
                                                             &standard_dex_data));
   ASSERT_TRUE(standard_dex != nullptr);
-  std::vector<uint8_t> compact_dex_data;
-  std::unique_ptr<const DexFile> compact_dex(CreateFakeDex(/*compact_dex=*/true,
-                                                           &compact_dex_data));
-  ASSERT_TRUE(compact_dex != nullptr);
   static constexpr uint16_t kRegisterSize = 2;
   static constexpr uint16_t kInsSize = 1;
   static constexpr uint16_t kOutsSize = 3;
@@ -98,19 +94,6 @@ TEST(CodeItemAccessorsTest, TestDexInstructionsAccessor) {
   dex_code_item->tries_size_ = kTriesSize;
   dex_code_item->insns_size_in_code_units_ = kInsnsSizeInCodeUnits;
   verify_code_item(standard_dex.get(), dex_code_item, dex_code_item->insns_);
-
-  CompactDexFile::CodeItem* cdex_code_item =
-      reinterpret_cast<CompactDexFile::CodeItem*>(const_cast<uint8_t*>(compact_dex->Begin() +
-          CompactDexFile::CodeItem::kMaxPreHeaderSize * sizeof(uint16_t)));
-  std::vector<uint16_t> preheader;
-  cdex_code_item->Create(kRegisterSize,
-                         kInsSize,
-                         kOutsSize,
-                         kTriesSize,
-                         kInsnsSizeInCodeUnits,
-                         cdex_code_item->GetPreHeader());
-
-  verify_code_item(compact_dex.get(), cdex_code_item, cdex_code_item->insns_);
 }
 
 }  // namespace art
diff --git a/libdexfile/dex/compact_dex_file_test.cc b/libdexfile/dex/compact_dex_file_test.cc
index 799967e255..345e66b1d5 100644
--- a/libdexfile/dex/compact_dex_file_test.cc
+++ b/libdexfile/dex/compact_dex_file_test.cc
@@ -38,8 +38,8 @@ TEST(CompactDexFileTest, MagicAndVersion) {
       }
       EXPECT_EQ(valid_magic, CompactDexFile::IsMagicValid(header));
       EXPECT_EQ(valid_version, CompactDexFile::IsVersionValid(header));
-      EXPECT_EQ(valid_magic, DexFileLoader::IsMagicValid(header));
-      EXPECT_EQ(valid_magic && valid_version, DexFileLoader::IsVersionAndMagicValid(header));
+      EXPECT_FALSE(DexFileLoader::IsMagicValid(header));
+      EXPECT_FALSE(DexFileLoader::IsVersionAndMagicValid(header));
     }
   }
 }
diff --git a/libdexfile/dex/dex_file_loader.cc b/libdexfile/dex/dex_file_loader.cc
index e92a5ac813..df9c9c11cb 100644
--- a/libdexfile/dex/dex_file_loader.cc
+++ b/libdexfile/dex/dex_file_loader.cc
@@ -131,18 +131,11 @@ bool DexFileLoader::IsMagicValid(uint32_t magic) {
 }
 
 bool DexFileLoader::IsMagicValid(const uint8_t* magic) {
-  return StandardDexFile::IsMagicValid(magic) ||
-      CompactDexFile::IsMagicValid(magic);
+  return StandardDexFile::IsMagicValid(magic);
 }
 
 bool DexFileLoader::IsVersionAndMagicValid(const uint8_t* magic) {
-  if (StandardDexFile::IsMagicValid(magic)) {
-    return StandardDexFile::IsVersionValid(magic);
-  }
-  if (CompactDexFile::IsMagicValid(magic)) {
-    return CompactDexFile::IsVersionValid(magic);
-  }
-  return false;
+  return StandardDexFile::IsMagicValid(magic) && StandardDexFile::IsVersionValid(magic);
 }
 
 bool DexFileLoader::IsMultiDexLocation(std::string_view location) {
@@ -474,9 +467,6 @@ std::unique_ptr<DexFile> DexFileLoader::OpenCommon(std::shared_ptr<DexFileContai
   if (size >= sizeof(StandardDexFile::Header) && StandardDexFile::IsMagicValid(base)) {
     uint32_t checksum = location_checksum.value_or(header->checksum_);
     dex_file.reset(new StandardDexFile(base, location, checksum, oat_dex_file, container));
-  } else if (size >= sizeof(CompactDexFile::Header) && CompactDexFile::IsMagicValid(base)) {
-    uint32_t checksum = location_checksum.value_or(header->checksum_);
-    dex_file.reset(new CompactDexFile(base, location, checksum, oat_dex_file, container));
   } else {
     *error_msg = StringPrintf("Invalid or truncated dex file '%s'", location.c_str());
   }
@@ -489,8 +479,7 @@ std::unique_ptr<DexFile> DexFileLoader::OpenCommon(std::shared_ptr<DexFileContai
     dex_file.reset();
     return nullptr;
   }
-  // NB: Dex verifier does not understand the compact dex format.
-  if (verify && !dex_file->IsCompactDexFile()) {
+  if (verify) {
     DEXFILE_SCOPED_TRACE(std::string("Verify dex file ") + location);
     if (!dex::Verify(dex_file.get(), location.c_str(), verify_checksum, error_msg)) {
       if (error_code != nullptr) {
diff --git a/runtime/vdex_file.cc b/runtime/vdex_file.cc
index 1d3a9fbe66..c8f1bfd810 100644
--- a/runtime/vdex_file.cc
+++ b/runtime/vdex_file.cc
@@ -204,6 +204,23 @@ std::unique_ptr<VdexFile> VdexFile::OpenFromDm(const std::string& filename,
   return vdex_file;
 }
 
+bool VdexFile::IsValid() const {
+  if (mmap_.Size() < sizeof(VdexFileHeader) || !GetVdexFileHeader().IsValid()) {
+    return false;
+  }
+
+  // Invalidate vdex files that contain dex files in the no longer supported
+  // compact dex format. Revert this whenever the vdex version is bumped.
+  size_t i = 0;
+  for (const uint8_t* dex_file_start = GetNextDexFileData(nullptr, i); dex_file_start != nullptr;
+       dex_file_start = GetNextDexFileData(dex_file_start, ++i)) {
+    if (!DexFileLoader::IsMagicValid(dex_file_start)) {
+      return false;
+    }
+  }
+  return true;
+}
+
 const uint8_t* VdexFile::GetNextDexFileData(const uint8_t* cursor, uint32_t dex_file_index) const {
   DCHECK(cursor == nullptr || (cursor > Begin() && cursor <= End()));
   if (cursor == nullptr) {
diff --git a/runtime/vdex_file.h b/runtime/vdex_file.h
index 4ccc402b32..a85c015202 100644
--- a/runtime/vdex_file.h
+++ b/runtime/vdex_file.h
@@ -124,6 +124,7 @@ class VdexFile {
     static constexpr uint8_t kVdexMagic[] = { 'v', 'd', 'e', 'x' };
 
     // The format version of the verifier deps header and the verifier deps.
+    // TODO: Revert the dex header checks in VdexFile::IsValid when this is bumped.
     // Last update: Introduce vdex sections.
     static constexpr uint8_t kVdexVersion[] = { '0', '2', '7', '\0' };
 
@@ -251,9 +252,7 @@ class VdexFile {
         GetSectionHeader(VdexSection::kVerifierDepsSection).section_size);
   }
 
-  bool IsValid() const {
-    return mmap_.Size() >= sizeof(VdexFileHeader) && GetVdexFileHeader().IsValid();
-  }
+  EXPORT bool IsValid() const;
 
   // This method is for iterating over the dex files in the vdex. If `cursor` is null,
   // the first dex file is returned. If `cursor` is not null, it must point to a dex