Fix bug where the output vdex could get compact dex data even when

--compact-dex-level is none.

If an input vdex containing cdex was given (which is typically the
already existing vdex), then it was used as dex files source even when
the resulting vdex shouldn't contain cdex.

Correct the file magic check to detect that, and ignore the input vdex
in that case.

Test: m AppUsedByOtherApp
      adb install-multiple out/target/product/vsoc_x86_64/testcases/AppUsedByOtherApp/x86_64/AppUsedByOtherApp.apk AppUsedByOtherApp.dm
      # where AppUsedByOtherApp.dm is created from a non-empty profile
      adb shell pm bg-dexopt-job
      adb root && adb wait-for-device
      adb pull `adb shell ls /data/app/\*/android.compilation.cts.appusedbyotherapp-\*/oat/x86_64/base.vdex`
      grep cdex001 base.vdex
      # -> check that a match is found
      adb shell setprop persist.device_config.runtime_native_boot.disable_compact_dex true
      adb reboot && adb wait-for-device
      adb shell pm bg-dexopt-job
      adb root && adb wait-for-device
      adb pull `adb shell ls /data/app/\*/android.compilation.cts.appusedbyotherapp-\*/oat/x86_64/base.vdex`
      grep cdex001 base.vdex
      # -> check that no match is found
Bug: 256664509
Change-Id: I222c48c2e29d545e799f0b2faa7f6a7d952e21f6

4 files changed, 26 insertions, 3 deletions

diff --git a/dex2oat/dex2oat.cc b/dex2oat/dex2oat.cc
index f4c3e2a094..4b083030ed 100644
--- a/dex2oat/dex2oat.cc
+++ b/dex2oat/dex2oat.cc
@@ -2614,7 +2614,12 @@ class Dex2Oat final {
 
   bool AddDexFileSources() {
     TimingLogger::ScopedTiming t2("AddDexFileSources", timings_);
-    if (input_vdex_file_ != nullptr && input_vdex_file_->HasDexSection()) {
+    if (input_vdex_file_ != nullptr && input_vdex_file_->HasDexSection() &&
+        // If we don't want compact dex in the vdex then we have to make sure
+        // the input vdex only has standard dex files, because we can convert
+        // dex to cdex but not the other way around.
+        (compact_dex_level_ == CompactDexLevel::kCompactDexLevelFast ||
+         input_vdex_file_->HasOnlyStandardDexFiles())) {
       DCHECK_EQ(oat_writers_.size(), 1u);
       const std::string& name = zip_location_.empty() ? dex_locations_[0] : zip_location_;
       DCHECK(!name.empty());
diff --git a/dex2oat/linker/oat_writer.cc b/dex2oat/linker/oat_writer.cc
index 222a5f4a5f..23c209b721 100644
--- a/dex2oat/linker/oat_writer.cc
+++ b/dex2oat/linker/oat_writer.cc
@@ -490,8 +490,16 @@ bool OatWriter::AddVdexDexFilesSource(const VdexFile& vdex_file, const char* loc
       return false;
     }
 
-    if (!DexFileLoader::IsMagicValid(current_dex_data)) {
-      LOG(ERROR) << "Invalid magic in vdex file created from " << location;
+    if (StandardDexFile::IsMagicValid(current_dex_data)) {
+      // Standard dex is always ok - we'll run dexlayout to convert it to cdex if needed.
+    } else if (compact_dex_level_ == CompactDexLevel::kCompactDexLevelFast &&
+               CompactDexFile::IsMagicValid(current_dex_data)) {
+      // Compact dex is ok if we want compact dex in the output, but not
+      // otherwise since we cannot convert it to standard dex.
+    } else {
+      LOG(ERROR) << "Invalid magic in vdex file created from " << location << " - want "
+                 << (compact_dex_level_ == CompactDexLevel::kCompactDexLevelFast ? "dex or cdex" :
+                                                                                   "dex");
       return false;
     }
     // We used `zipped_dex_file_locations_` to keep the strings in memory.
diff --git a/runtime/vdex_file.cc b/runtime/vdex_file.cc
index bdead55390..9bcde00531 100644
--- a/runtime/vdex_file.cc
+++ b/runtime/vdex_file.cc
@@ -384,6 +384,12 @@ bool VdexFile::MatchesDexFileChecksums(const std::vector<const DexFile::Header*>
   return true;
 }
 
+bool VdexFile::HasOnlyStandardDexFiles() const {
+  // All are the same so it's enough to check the first one.
+  const uint8_t* dex_file_start = GetNextDexFileData(nullptr, 0);
+  return dex_file_start == nullptr || StandardDexFile::IsMagicValid(dex_file_start);
+}
+
 static ObjPtr<mirror::Class> FindClassAndClearException(ClassLinker* class_linker,
                                                         Thread* self,
                                                         const char* name,
diff --git a/runtime/vdex_file.h b/runtime/vdex_file.h
index a35d720f71..fe65c07bac 100644
--- a/runtime/vdex_file.h
+++ b/runtime/vdex_file.h
@@ -296,6 +296,10 @@ class VdexFile {
   // order must match too.
   bool MatchesDexFileChecksums(const std::vector<const DexFile::Header*>& dex_headers) const;
 
+  // Returns true if all dex files are standard dex rather than compact dex.
+  // Also returns true if there are no dex files at all.
+  bool HasOnlyStandardDexFiles() const;
+
   ClassStatus ComputeClassStatus(Thread* self, Handle<mirror::Class> cls) const
       REQUIRES_SHARED(Locks::mutator_lock_);