Add basic support for writing cdex container in dexlayout

Pass compact dex level option to dexlayout and generate a file with
a compact dex header if it is enabled.

Added test in dex2oat_test.

Added logic to check for CompactDexFile header in
DexFileLoader::OpenCommon.

Bug: 63756964
Test: test-art-host-gtest-dex2oat_test

Change-Id: Icfec03cdb64f71de900b92754ab4be3789c71cc3

16 files changed, 128 insertions, 22 deletions

diff --git a/dex2oat/dex2oat.cc b/dex2oat/dex2oat.cc
index 49fee1757e..ebd4afe20e 100644
--- a/dex2oat/dex2oat.cc
+++ b/dex2oat/dex2oat.cc
@@ -449,7 +449,7 @@ NO_RETURN static void Usage(const char* fmt, ...) {
   UsageError("      The image writer will group them together.");
   UsageError("");
   UsageError("  --compact-dex-level=none|fast: None avoids generating compact dex, fast");
-  UsageError("      generates compact dex with fast optimiations.");
+  UsageError("      generates compact dex with fast optimizations.");
   UsageError("");
   std::cerr << "See log for usage error information\n";
   exit(EXIT_FAILURE);
@@ -2458,7 +2458,8 @@ class Dex2Oat FINAL {
       oat_writers_.emplace_back(new linker::OatWriter(
           IsBootImage(),
           timings_,
-          do_oat_writer_layout ? profile_compilation_info_.get() : nullptr));
+          do_oat_writer_layout ? profile_compilation_info_.get() : nullptr,
+          compact_dex_level_));
     }
   }
 
diff --git a/dex2oat/dex2oat_test.cc b/dex2oat/dex2oat_test.cc
index 568be7dd0d..4500ff902c 100644
--- a/dex2oat/dex2oat_test.cc
+++ b/dex2oat/dex2oat_test.cc
@@ -1395,6 +1395,41 @@ TEST_F(Dex2oatTest, LayoutSections) {
   }
 }
 
+// Test that generating compact dex works.
+TEST_F(Dex2oatTest, GenerateCompactDex) {
+  std::unique_ptr<const DexFile> dex(OpenTestDexFile("ManyMethods"));
+  // Generate a compact dex based odex.
+  const std::string dir = GetScratchDir();
+  const std::string oat_filename = dir + "/base.oat";
+  const std::string vdex_filename = dir + "/base.vdex";
+  std::string error_msg;
+  const int res = GenerateOdexForTestWithStatus(
+      {dex->GetLocation()},
+      oat_filename,
+      CompilerFilter::Filter::kQuicken,
+      &error_msg,
+      {"--compact-dex-level=fast"});
+  EXPECT_EQ(res, 0);
+  // Open our generated oat file.
+  std::unique_ptr<OatFile> odex_file(OatFile::Open(oat_filename.c_str(),
+                                                   oat_filename.c_str(),
+                                                   nullptr,
+                                                   nullptr,
+                                                   false,
+                                                   /*low_4gb*/false,
+                                                   dex->GetLocation().c_str(),
+                                                   &error_msg));
+  ASSERT_TRUE(odex_file != nullptr);
+  std::vector<const OatDexFile*> oat_dex_files = odex_file->GetOatDexFiles();
+  ASSERT_EQ(oat_dex_files.size(), 1u);
+  // Check that each dex is a compact dex.
+  for (const OatDexFile* oat_dex : oat_dex_files) {
+    std::unique_ptr<const DexFile> dex_file(oat_dex->OpenDexFile(&error_msg));
+    ASSERT_TRUE(dex_file != nullptr) << error_msg;
+    ASSERT_TRUE(dex_file->IsCompactDexFile());
+  }
+}
+
 class Dex2oatVerifierAbort : public Dex2oatTest {};
 
 TEST_F(Dex2oatVerifierAbort, HardFail) {
diff --git a/dex2oat/linker/image_test.h b/dex2oat/linker/image_test.h
index 492c76bc54..dc570da832 100644
--- a/dex2oat/linker/image_test.h
+++ b/dex2oat/linker/image_test.h
@@ -247,7 +247,8 @@ inline void CompilationHelper::Compile(CompilerDriver* driver,
         elf_writers.back()->Start();
         oat_writers.emplace_back(new OatWriter(/*compiling_boot_image*/true,
                                                &timings,
-                                               /*profile_compilation_info*/nullptr));
+                                               /*profile_compilation_info*/nullptr,
+                                               CompactDexLevel::kCompactDexLevelNone));
       }
 
       std::vector<OutputStream*> rodata;
diff --git a/dex2oat/linker/oat_writer.cc b/dex2oat/linker/oat_writer.cc
index 17ceca3662..4f169600b2 100644
--- a/dex2oat/linker/oat_writer.cc
+++ b/dex2oat/linker/oat_writer.cc
@@ -330,7 +330,10 @@ class OatWriter::OatDexFile {
   DCHECK_EQ(static_cast<off_t>(file_offset + offset_), out->Seek(0, kSeekCurrent)) \
     << "file_offset=" << file_offset << " offset_=" << offset_
 
-OatWriter::OatWriter(bool compiling_boot_image, TimingLogger* timings, ProfileCompilationInfo* info)
+OatWriter::OatWriter(bool compiling_boot_image,
+                     TimingLogger* timings,
+                     ProfileCompilationInfo* info,
+                     CompactDexLevel compact_dex_level)
   : write_state_(WriteState::kAddingDexFileSources),
     timings_(timings),
     raw_dex_files_(),
@@ -404,7 +407,8 @@ OatWriter::OatWriter(bool compiling_boot_image, TimingLogger* timings, ProfileCo
     size_method_bss_mappings_(0u),
     relative_patcher_(nullptr),
     absolute_patch_locations_(),
-    profile_compilation_info_(info) {
+    profile_compilation_info_(info),
+    compact_dex_level_(compact_dex_level) {
 }
 
 bool OatWriter::AddDexFileSource(const char* filename,
@@ -3160,7 +3164,8 @@ bool OatWriter::WriteDexFile(OutputStream* out,
   if (!SeekToDexFile(out, file, oat_dex_file)) {
     return false;
   }
-  if (profile_compilation_info_ != nullptr) {
+  if (profile_compilation_info_ != nullptr ||
+          compact_dex_level_ != CompactDexLevel::kCompactDexLevelNone) {
     CHECK(!update_input_vdex) << "We should never update the input vdex when doing dexlayout";
     if (!LayoutAndWriteDexFile(out, oat_dex_file)) {
       return false;
@@ -3286,6 +3291,7 @@ bool OatWriter::LayoutAndWriteDexFile(OutputStream* out, OatDexFile* oat_dex_fil
   }
   Options options;
   options.output_to_memmap_ = true;
+  options.compact_dex_level_ = compact_dex_level_;
   DexLayout dex_layout(options, profile_compilation_info_, nullptr);
   dex_layout.ProcessDexFile(location.c_str(), dex_file.get(), 0);
   std::unique_ptr<MemMap> mem_map(dex_layout.GetAndReleaseMemMap());
diff --git a/dex2oat/linker/oat_writer.h b/dex2oat/linker/oat_writer.h
index c742fd4441..6a82fd1d59 100644
--- a/dex2oat/linker/oat_writer.h
+++ b/dex2oat/linker/oat_writer.h
@@ -24,6 +24,7 @@
 
 #include "base/array_ref.h"
 #include "base/dchecked_vector.h"
+#include "cdex/compact_dex_level.h"
 #include "linker/relative_patcher.h"  // For RelativePatcherTargetProvider.
 #include "mem_map.h"
 #include "method_reference.h"
@@ -114,7 +115,10 @@ class OatWriter {
     kDefault = kCreate
   };
 
-  OatWriter(bool compiling_boot_image, TimingLogger* timings, ProfileCompilationInfo* info);
+  OatWriter(bool compiling_boot_image,
+            TimingLogger* timings,
+            ProfileCompilationInfo* info,
+            CompactDexLevel compact_dex_level);
 
   // To produce a valid oat file, the user must first add sources with any combination of
   //   - AddDexFileSource(),
@@ -491,6 +495,9 @@ class OatWriter {
   // Profile info used to generate new layout of files.
   ProfileCompilationInfo* profile_compilation_info_;
 
+  // Compact dex level that is generated.
+  CompactDexLevel compact_dex_level_;
+
   using OrderedMethodList = std::vector<OrderedMethodData>;
 
   // List of compiled methods, sorted by the order defined in OrderedMethodData.
diff --git a/dex2oat/linker/oat_writer_test.cc b/dex2oat/linker/oat_writer_test.cc
index a19057a0ed..3efebfd45f 100644
--- a/dex2oat/linker/oat_writer_test.cc
+++ b/dex2oat/linker/oat_writer_test.cc
@@ -127,7 +127,8 @@ class OatTest : public CommonCompilerTest {
     TimingLogger timings("WriteElf", false, false);
     OatWriter oat_writer(/*compiling_boot_image*/false,
                          &timings,
-                         /*profile_compilation_info*/nullptr);
+                         /*profile_compilation_info*/nullptr,
+                         CompactDexLevel::kCompactDexLevelNone);
     for (const DexFile* dex_file : dex_files) {
       ArrayRef<const uint8_t> raw_dex_file(
           reinterpret_cast<const uint8_t*>(&dex_file->GetHeader()),
@@ -148,7 +149,10 @@ class OatTest : public CommonCompilerTest {
                 bool verify,
                 ProfileCompilationInfo* profile_compilation_info) {
     TimingLogger timings("WriteElf", false, false);
-    OatWriter oat_writer(/*compiling_boot_image*/false, &timings, profile_compilation_info);
+    OatWriter oat_writer(/*compiling_boot_image*/false,
+                         &timings,
+                         profile_compilation_info,
+                         CompactDexLevel::kCompactDexLevelNone);
     for (const char* dex_filename : dex_filenames) {
       if (!oat_writer.AddDexFileSource(dex_filename, dex_filename)) {
         return false;
@@ -166,7 +170,8 @@ class OatTest : public CommonCompilerTest {
     TimingLogger timings("WriteElf", false, false);
     OatWriter oat_writer(/*compiling_boot_image*/false,
                          &timings,
-                         /*profile_compilation_info*/nullptr);
+                         /*profile_compilation_info*/nullptr,
+                         CompactDexLevel::kCompactDexLevelNone);
     if (!oat_writer.AddZippedDexFilesSource(std::move(zip_fd), location)) {
       return false;
     }
diff --git a/dexlayout/dex_writer.cc b/dexlayout/dex_writer.cc
index 11ba2a6357..8c821066d2 100644
--- a/dexlayout/dex_writer.cc
+++ b/dexlayout/dex_writer.cc
@@ -23,7 +23,9 @@
 #include <queue>
 #include <vector>
 
+#include "cdex/compact_dex_file.h"
 #include "dex_file_types.h"
+#include "standard_dex_file.h"
 #include "utf.h"
 
 namespace art {
@@ -630,7 +632,18 @@ void DexWriter::WriteHeader() {
   uint32_t buffer[20];
   dex_ir::Collections& collections = header_->GetCollections();
   size_t offset = 0;
-  offset += Write(header_->Magic(), 8 * sizeof(uint8_t), offset);
+  if (compact_dex_level_ != CompactDexLevel::kCompactDexLevelNone) {
+    static constexpr size_t kMagicAndVersionLen =
+        CompactDexFile::kDexMagicSize + CompactDexFile::kDexVersionLen;
+    uint8_t magic_and_version[kMagicAndVersionLen] = {};
+    CompactDexFile::WriteMagic(&magic_and_version[0]);
+    CompactDexFile::WriteCurrentVersion(&magic_and_version[0]);
+    offset += Write(magic_and_version, kMagicAndVersionLen * sizeof(uint8_t), offset);
+  } else {
+    static constexpr size_t kMagicAndVersionLen =
+        StandardDexFile::kDexMagicSize + StandardDexFile::kDexVersionLen;
+    offset += Write(header_->Magic(), kMagicAndVersionLen * sizeof(uint8_t), offset);
+  }
   buffer[0] = header_->Checksum();
   offset += Write(buffer, sizeof(uint32_t), offset);
   offset += Write(header_->Signature(), 20 * sizeof(uint8_t), offset);
@@ -681,8 +694,8 @@ void DexWriter::WriteMemMap() {
   WriteHeader();
 }
 
-void DexWriter::Output(dex_ir::Header* header, MemMap* mem_map) {
-  DexWriter dex_writer(header, mem_map);
+void DexWriter::Output(dex_ir::Header* header, MemMap* mem_map, CompactDexLevel compact_dex_level) {
+  DexWriter dex_writer(header, mem_map, compact_dex_level);
   dex_writer.WriteMemMap();
 }
 
diff --git a/dexlayout/dex_writer.h b/dexlayout/dex_writer.h
index b396adf126..d932b9f006 100644
--- a/dexlayout/dex_writer.h
+++ b/dexlayout/dex_writer.h
@@ -20,6 +20,7 @@
 #define ART_DEXLAYOUT_DEX_WRITER_H_
 
 #include "base/unix_file/fd_file.h"
+#include "cdex/compact_dex_level.h"
 #include "dex_ir.h"
 #include "mem_map.h"
 #include "os.h"
@@ -28,9 +29,14 @@ namespace art {
 
 class DexWriter {
  public:
-  DexWriter(dex_ir::Header* header, MemMap* mem_map) : header_(header), mem_map_(mem_map) { }
+  DexWriter(dex_ir::Header* header,
+            MemMap* mem_map,
+            CompactDexLevel compact_dex_level)
+      : header_(header),
+        mem_map_(mem_map),
+        compact_dex_level_(compact_dex_level) { }
 
-  static void Output(dex_ir::Header* header, MemMap* mem_map);
+  static void Output(dex_ir::Header* header, MemMap* mem_map, CompactDexLevel compact_dex_level);
 
  private:
   void WriteMemMap();
@@ -66,6 +72,7 @@ class DexWriter {
 
   dex_ir::Header* const header_;
   MemMap* const mem_map_;
+  const CompactDexLevel compact_dex_level_;
 
   DISALLOW_COPY_AND_ASSIGN(DexWriter);
 };
diff --git a/dexlayout/dexlayout.cc b/dexlayout/dexlayout.cc
index 40449ae8bd..4a5b64d53d 100644
--- a/dexlayout/dexlayout.cc
+++ b/dexlayout/dexlayout.cc
@@ -1986,7 +1986,7 @@ void DexLayout::OutputDexFile(const DexFile* dex_file) {
     }
     return;
   }
-  DexWriter::Output(header_, mem_map_.get());
+  DexWriter::Output(header_, mem_map_.get(), options_.compact_dex_level_);
   if (new_file != nullptr) {
     UNUSED(new_file->FlushCloseOrErase());
   }
diff --git a/dexlayout/dexlayout.h b/dexlayout/dexlayout.h
index c5818c90ff..eaa7cac32b 100644
--- a/dexlayout/dexlayout.h
+++ b/dexlayout/dexlayout.h
@@ -26,6 +26,7 @@
 #include <stdint.h>
 #include <stdio.h>
 
+#include "cdex/compact_dex_level.h"
 #include "dex_file_layout.h"
 #include "dex_ir.h"
 #include "mem_map.h"
@@ -61,6 +62,7 @@ class Options {
   bool verbose_ = false;
   bool verify_output_ = false;
   bool visualize_pattern_ = false;
+  CompactDexLevel compact_dex_level_ = CompactDexLevel::kCompactDexLevelNone;
   OutputFormat output_format_ = kOutputPlain;
   const char* output_dex_directory_ = nullptr;
   const char* output_file_name_ = nullptr;
diff --git a/runtime/cdex/compact_dex_file.cc b/runtime/cdex/compact_dex_file.cc
index dbe2c66ecc..82ffdb0adb 100644
--- a/runtime/cdex/compact_dex_file.cc
+++ b/runtime/cdex/compact_dex_file.cc
@@ -21,6 +21,14 @@ namespace art {
 constexpr uint8_t CompactDexFile::kDexMagic[kDexMagicSize];
 constexpr uint8_t CompactDexFile::kDexMagicVersion[];
 
+void CompactDexFile::WriteMagic(uint8_t* magic) {
+  std::copy_n(kDexMagic, kDexMagicSize, magic);
+}
+
+void CompactDexFile::WriteCurrentVersion(uint8_t* magic) {
+  std::copy_n(kDexMagicVersion, kDexVersionLen, magic + kDexMagicSize);
+}
+
 bool CompactDexFile::IsMagicValid(const uint8_t* magic) {
   return (memcmp(magic, kDexMagic, sizeof(kDexMagic)) == 0);
 }
diff --git a/runtime/cdex/compact_dex_file.h b/runtime/cdex/compact_dex_file.h
index fa6eab2d76..910473be3f 100644
--- a/runtime/cdex/compact_dex_file.h
+++ b/runtime/cdex/compact_dex_file.h
@@ -27,6 +27,12 @@ class CompactDexFile : public DexFile {
   static constexpr uint8_t kDexMagic[kDexMagicSize] = { 'c', 'd', 'e', 'x' };
   static constexpr uint8_t kDexMagicVersion[] = {'0', '0', '1', '\0'};
 
+  // Write the compact dex specific magic.
+  static void WriteMagic(uint8_t* magic);
+
+  // Write the current version, note that the input is the address of the magic.
+  static void WriteCurrentVersion(uint8_t* magic);
+
   // Returns true if the byte string points to the magic value.
   static bool IsMagicValid(const uint8_t* magic);
   virtual bool IsMagicValid() const OVERRIDE;
@@ -35,6 +41,10 @@ class CompactDexFile : public DexFile {
   static bool IsVersionValid(const uint8_t* magic);
   virtual bool IsVersionValid() const OVERRIDE;
 
+  bool IsCompactDexFile() const OVERRIDE {
+    return true;
+  }
+
  private:
   // Not supported yet.
   CompactDexFile(const uint8_t* base,
@@ -45,6 +55,7 @@ class CompactDexFile : public DexFile {
       : DexFile(base, size, location, location_checksum, oat_dex_file) {}
 
   friend class DexFile;
+  friend class DexFileLoader;
 
   DISALLOW_COPY_AND_ASSIGN(CompactDexFile);
 };
diff --git a/runtime/cdex/compact_dex_file_test.cc b/runtime/cdex/compact_dex_file_test.cc
index 6fe4bccd16..b43b35d69a 100644
--- a/runtime/cdex/compact_dex_file_test.cc
+++ b/runtime/cdex/compact_dex_file_test.cc
@@ -32,12 +32,10 @@ TEST_F(CompactDexFileTest, MagicAndVersion) {
       const bool valid_magic = (i & 1) == 0;
       const bool valid_version = (j & 1) == 0;
       if (valid_magic) {
-        std::copy_n(CompactDexFile::kDexMagic, CompactDexFile::kDexMagicSize, header);
+        CompactDexFile::WriteMagic(header);
       }
       if (valid_version) {
-        std::copy_n(CompactDexFile::kDexMagicVersion,
-                    CompactDexFile::kDexVersionLen,
-                    header + CompactDexFile::kDexMagicSize);
+        CompactDexFile::WriteCurrentVersion(header);
       }
       EXPECT_EQ(valid_magic, CompactDexFile::IsMagicValid(header));
       EXPECT_EQ(valid_version, CompactDexFile::IsVersionValid(header));
diff --git a/runtime/dex_file.h b/runtime/dex_file.h
index 6868d525e2..d7ea7459f1 100644
--- a/runtime/dex_file.h
+++ b/runtime/dex_file.h
@@ -976,6 +976,14 @@ class DexFile {
   // Returns a human-readable form of the type at an index.
   std::string PrettyType(dex::TypeIndex type_idx) const;
 
+  // Helper functions.
+  virtual bool IsCompactDexFile() const {
+    return false;
+  }
+  virtual bool IsStandardDexFile() const {
+    return false;
+  }
+
  protected:
   DexFile(const uint8_t* base,
           size_t size,
diff --git a/runtime/dex_file_loader.cc b/runtime/dex_file_loader.cc
index e300e0e58f..54fd15df98 100644
--- a/runtime/dex_file_loader.cc
+++ b/runtime/dex_file_loader.cc
@@ -464,8 +464,8 @@ std::unique_ptr<DexFile> DexFileLoader::OpenCommon(const uint8_t* base,
   std::unique_ptr<DexFile> dex_file;
   if (StandardDexFile::IsMagicValid(base)) {
     dex_file.reset(new StandardDexFile(base, size, location, location_checksum, oat_dex_file));
-  } else {
-    return nullptr;
+  } else if (CompactDexFile::IsMagicValid(base)) {
+    dex_file.reset(new CompactDexFile(base, size, location, location_checksum, oat_dex_file));
   }
   if (dex_file == nullptr) {
     *error_msg = StringPrintf("Failed to open dex file '%s' from memory: %s", location.c_str(),
diff --git a/runtime/standard_dex_file.h b/runtime/standard_dex_file.h
index 1ec06edef3..4fa3efe343 100644
--- a/runtime/standard_dex_file.h
+++ b/runtime/standard_dex_file.h
@@ -40,6 +40,10 @@ class StandardDexFile : public DexFile {
   static bool IsVersionValid(const uint8_t* magic);
   virtual bool IsVersionValid() const OVERRIDE;
 
+  bool IsStandardDexFile() const OVERRIDE {
+    return true;
+  }
+
  private:
   StandardDexFile(const uint8_t* base,
                   size_t size,