Compress mini-debug-info in small chunks.

Compress the mini-debug-info in small independent parts.
This still results in standard compressed data stream.
However, by dividing into chunks, the compressed data
can be random accessed - that is, it makes it possible
to decompressed only the data that is needed.

This increases size of .oat files by 1.2%.

Bug: 110133331
Test: test.py --host -b -t 137
Change-Id: I9d1d4f1971b3c072d45ca4289f8c15f44da4036a
diff --git a/compiler/Android.bp b/compiler/Android.bp
index eff4955..c365537 100644
--- a/compiler/Android.bp
+++ b/compiler/Android.bp
@@ -26,6 +26,7 @@
     srcs: [
         "compiled_method.cc",
         "debug/elf_debug_writer.cc",
+        "debug/xz_utils.cc",
         "dex/dex_to_dex_compiler.cc",
         "dex/inline_method_analyser.cc",
         "dex/verified_method.cc",
diff --git a/compiler/debug/elf_debug_writer.cc b/compiler/debug/elf_debug_writer.cc
index 59a080f..71422d4 100644
--- a/compiler/debug/elf_debug_writer.cc
+++ b/compiler/debug/elf_debug_writer.cc
@@ -26,9 +26,9 @@
 #include "debug/elf_debug_info_writer.h"
 #include "debug/elf_debug_line_writer.h"
 #include "debug/elf_debug_loc_writer.h"
-#include "debug/elf_gnu_debugdata_writer.h"
 #include "debug/elf_symtab_writer.h"
 #include "debug/method_debug_info.h"
+#include "debug/xz_utils.h"
 #include "linker/elf_builder.h"
 #include "linker/vector_output_stream.h"
 #include "oat.h"
@@ -105,6 +105,39 @@
   }
 }
 
+template <typename ElfTypes>
+static std::vector<uint8_t> MakeMiniDebugInfoInternal(
+    InstructionSet isa,
+    const InstructionSetFeatures* features,
+    typename ElfTypes::Addr text_section_address,
+    size_t text_section_size,
+    typename ElfTypes::Addr dex_section_address,
+    size_t dex_section_size,
+    const DebugInfo& debug_info) {
+  std::vector<uint8_t> buffer;
+  buffer.reserve(KB);
+  linker::VectorOutputStream out("Mini-debug-info ELF file", &buffer);
+  std::unique_ptr<linker::ElfBuilder<ElfTypes>> builder(
+      new linker::ElfBuilder<ElfTypes>(isa, features, &out));
+  builder->Start(false /* write_program_headers */);
+  // Mirror ELF sections as NOBITS since the added symbols will reference them.
+  builder->GetText()->AllocateVirtualMemory(text_section_address, text_section_size);
+  if (dex_section_size != 0) {
+    builder->GetDex()->AllocateVirtualMemory(dex_section_address, dex_section_size);
+  }
+  WriteDebugSymbols(builder.get(), true /* mini-debug-info */, debug_info);
+  WriteCFISection(builder.get(),
+                  debug_info.compiled_methods,
+                  dwarf::DW_DEBUG_FRAME_FORMAT,
+                  false /* write_oat_paches */);
+  builder->End();
+  CHECK(builder->Good());
+  std::vector<uint8_t> compressed_buffer;
+  compressed_buffer.reserve(buffer.size() / 4);
+  XzCompress(ArrayRef<uint8_t>(buffer), &compressed_buffer);
+  return compressed_buffer;
+}
+
 std::vector<uint8_t> MakeMiniDebugInfo(
     InstructionSet isa,
     const InstructionSetFeatures* features,
diff --git a/compiler/debug/elf_gnu_debugdata_writer.h b/compiler/debug/elf_gnu_debugdata_writer.h
deleted file mode 100644
index fd132f4..0000000
--- a/compiler/debug/elf_gnu_debugdata_writer.h
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Copyright (C) 2016 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef ART_COMPILER_DEBUG_ELF_GNU_DEBUGDATA_WRITER_H_
-#define ART_COMPILER_DEBUG_ELF_GNU_DEBUGDATA_WRITER_H_
-
-#include <vector>
-
-#include "arch/instruction_set.h"
-#include "linker/elf_builder.h"
-#include "linker/vector_output_stream.h"
-
-// liblzma.
-#include "7zCrc.h"
-#include "XzCrc64.h"
-#include "XzEnc.h"
-
-namespace art {
-namespace debug {
-
-static void XzCompress(const std::vector<uint8_t>* src, std::vector<uint8_t>* dst) {
-  // Configure the compression library.
-  CrcGenerateTable();
-  Crc64GenerateTable();
-  CLzma2EncProps lzma2Props;
-  Lzma2EncProps_Init(&lzma2Props);
-  lzma2Props.lzmaProps.level = 1;  // Fast compression.
-  Lzma2EncProps_Normalize(&lzma2Props);
-  CXzProps props;
-  XzProps_Init(&props);
-  props.lzma2Props = lzma2Props;
-  // Implement the required interface for communication (written in C so no virtual methods).
-  struct XzCallbacks : public ISeqInStream, public ISeqOutStream, public ICompressProgress {
-    static SRes ReadImpl(const ISeqInStream* p, void* buf, size_t* size) {
-      auto* ctx = static_cast<XzCallbacks*>(const_cast<ISeqInStream*>(p));
-      *size = std::min(*size, ctx->src_->size() - ctx->src_pos_);
-      memcpy(buf, ctx->src_->data() + ctx->src_pos_, *size);
-      ctx->src_pos_ += *size;
-      return SZ_OK;
-    }
-    static size_t WriteImpl(const ISeqOutStream* p, const void* buf, size_t size) {
-      auto* ctx = static_cast<const XzCallbacks*>(p);
-      const uint8_t* buffer = reinterpret_cast<const uint8_t*>(buf);
-      ctx->dst_->insert(ctx->dst_->end(), buffer, buffer + size);
-      return size;
-    }
-    static SRes ProgressImpl(const ICompressProgress* , UInt64, UInt64) {
-      return SZ_OK;
-    }
-    size_t src_pos_;
-    const std::vector<uint8_t>* src_;
-    std::vector<uint8_t>* dst_;
-  };
-  XzCallbacks callbacks;
-  callbacks.Read = XzCallbacks::ReadImpl;
-  callbacks.Write = XzCallbacks::WriteImpl;
-  callbacks.Progress = XzCallbacks::ProgressImpl;
-  callbacks.src_pos_ = 0;
-  callbacks.src_ = src;
-  callbacks.dst_ = dst;
-  // Compress.
-  SRes res = Xz_Encode(&callbacks, &callbacks, &props, &callbacks);
-  CHECK_EQ(res, SZ_OK);
-}
-
-template <typename ElfTypes>
-static std::vector<uint8_t> MakeMiniDebugInfoInternal(
-    InstructionSet isa,
-    const InstructionSetFeatures* features,
-    typename ElfTypes::Addr text_section_address,
-    size_t text_section_size,
-    typename ElfTypes::Addr dex_section_address,
-    size_t dex_section_size,
-    const DebugInfo& debug_info) {
-  std::vector<uint8_t> buffer;
-  buffer.reserve(KB);
-  linker::VectorOutputStream out("Mini-debug-info ELF file", &buffer);
-  std::unique_ptr<linker::ElfBuilder<ElfTypes>> builder(
-      new linker::ElfBuilder<ElfTypes>(isa, features, &out));
-  builder->Start(false /* write_program_headers */);
-  // Mirror ELF sections as NOBITS since the added symbols will reference them.
-  builder->GetText()->AllocateVirtualMemory(text_section_address, text_section_size);
-  if (dex_section_size != 0) {
-    builder->GetDex()->AllocateVirtualMemory(dex_section_address, dex_section_size);
-  }
-  WriteDebugSymbols(builder.get(), true /* mini-debug-info */, debug_info);
-  WriteCFISection(builder.get(),
-                  debug_info.compiled_methods,
-                  dwarf::DW_DEBUG_FRAME_FORMAT,
-                  false /* write_oat_paches */);
-  builder->End();
-  CHECK(builder->Good());
-  std::vector<uint8_t> compressed_buffer;
-  compressed_buffer.reserve(buffer.size() / 4);
-  XzCompress(&buffer, &compressed_buffer);
-  return compressed_buffer;
-}
-
-}  // namespace debug
-}  // namespace art
-
-#endif  // ART_COMPILER_DEBUG_ELF_GNU_DEBUGDATA_WRITER_H_
diff --git a/compiler/debug/xz_utils.cc b/compiler/debug/xz_utils.cc
new file mode 100644
index 0000000..a9e30a6
--- /dev/null
+++ b/compiler/debug/xz_utils.cc
@@ -0,0 +1,144 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "xz_utils.h"
+
+#include <vector>
+
+#include "base/array_ref.h"
+#include "dwarf/writer.h"
+#include "base/leb128.h"
+
+// liblzma.
+#include "7zCrc.h"
+#include "XzCrc64.h"
+#include "XzEnc.h"
+
+namespace art {
+namespace debug {
+
+constexpr size_t kChunkSize = kPageSize;
+
+static void XzCompressChunk(ArrayRef<uint8_t> src, std::vector<uint8_t>* dst) {
+  // Configure the compression library.
+  CrcGenerateTable();
+  Crc64GenerateTable();
+  CLzma2EncProps lzma2Props;
+  Lzma2EncProps_Init(&lzma2Props);
+  lzma2Props.lzmaProps.level = 1;  // Fast compression.
+  Lzma2EncProps_Normalize(&lzma2Props);
+  CXzProps props;
+  XzProps_Init(&props);
+  props.lzma2Props = lzma2Props;
+  // Implement the required interface for communication (written in C so no virtual methods).
+  struct XzCallbacks : public ISeqInStream, public ISeqOutStream, public ICompressProgress {
+    static SRes ReadImpl(const ISeqInStream* p, void* buf, size_t* size) {
+      auto* ctx = static_cast<XzCallbacks*>(const_cast<ISeqInStream*>(p));
+      *size = std::min(*size, ctx->src_.size() - ctx->src_pos_);
+      memcpy(buf, ctx->src_.data() + ctx->src_pos_, *size);
+      ctx->src_pos_ += *size;
+      return SZ_OK;
+    }
+    static size_t WriteImpl(const ISeqOutStream* p, const void* buf, size_t size) {
+      auto* ctx = static_cast<const XzCallbacks*>(p);
+      const uint8_t* buffer = reinterpret_cast<const uint8_t*>(buf);
+      ctx->dst_->insert(ctx->dst_->end(), buffer, buffer + size);
+      return size;
+    }
+    static SRes ProgressImpl(const ICompressProgress* , UInt64, UInt64) {
+      return SZ_OK;
+    }
+    size_t src_pos_;
+    ArrayRef<uint8_t> src_;
+    std::vector<uint8_t>* dst_;
+  };
+  XzCallbacks callbacks;
+  callbacks.Read = XzCallbacks::ReadImpl;
+  callbacks.Write = XzCallbacks::WriteImpl;
+  callbacks.Progress = XzCallbacks::ProgressImpl;
+  callbacks.src_pos_ = 0;
+  callbacks.src_ = src;
+  callbacks.dst_ = dst;
+  // Compress.
+  SRes res = Xz_Encode(&callbacks, &callbacks, &props, &callbacks);
+  CHECK_EQ(res, SZ_OK);
+}
+
+// Compress data while splitting it to smaller chunks to enable random-access reads.
+// The XZ file format supports this well, but the compression library does not.
+// Therefore compress the chunks separately and then glue them together manually.
+//
+// The XZ file format is described here: https://tukaani.org/xz/xz-file-format.txt
+// In short, the file format is: [header] [compressed_block]* [index] [footer]
+// Where [index] is: [num_records] ([compressed_size] [uncompressed_size])* [crc32]
+//
+void XzCompress(ArrayRef<uint8_t> src, std::vector<uint8_t>* dst) {
+  uint8_t header[] = { 0xFD, '7', 'z', 'X', 'Z', 0, 0, 1, 0x69, 0x22, 0xDE, 0x36 };
+  uint8_t footer[] = { 0, 1, 'Y', 'Z' };
+  dst->insert(dst->end(), header, header + sizeof(header));
+  std::vector<uint8_t> tmp;
+  std::vector<uint32_t> index;
+  for (size_t offset = 0; offset < src.size(); offset += kChunkSize) {
+    size_t size = std::min(src.size() - offset, kChunkSize);
+    tmp.clear();
+    XzCompressChunk(src.SubArray(offset, size), &tmp);
+    DCHECK_EQ(memcmp(tmp.data(), header, sizeof(header)), 0);
+    DCHECK_EQ(memcmp(tmp.data() + tmp.size() - sizeof(footer), footer, sizeof(footer)), 0);
+    uint32_t* index_size = reinterpret_cast<uint32_t*>(tmp.data() + tmp.size() - 8);
+    DCHECK_ALIGNED(index_size, sizeof(uint32_t));
+    size_t index_offset = tmp.size() - 16 - *index_size * 4;
+    const uint8_t* index_ptr = tmp.data() + index_offset;
+    uint8_t index_indicator = *(index_ptr++);
+    CHECK_EQ(index_indicator, 0);  // Mark the start of index (as opposed to compressed block).
+    uint32_t num_records = DecodeUnsignedLeb128(&index_ptr);
+    for (uint32_t i = 0; i < num_records; i++) {
+      index.push_back(DecodeUnsignedLeb128(&index_ptr));  // Compressed size.
+      index.push_back(DecodeUnsignedLeb128(&index_ptr));  // Uncompressed size.
+    }
+    // Copy the raw compressed block(s) located between the header and index.
+    dst->insert(dst->end(), tmp.data() + sizeof(header), tmp.data() + index_offset);
+  }
+
+  // Write the index.
+  uint32_t index_size_in_words;
+  {
+    tmp.clear();
+    dwarf::Writer<> writer(&tmp);
+    writer.PushUint8(0);  // Index indicator.
+    writer.PushUleb128(static_cast<uint32_t>(index.size()) / 2);  // Record count.
+    for (uint32_t i : index) {
+      writer.PushUleb128(i);
+    }
+    writer.Pad(4);
+    index_size_in_words = writer.size() / sizeof(uint32_t);
+    writer.PushUint32(CrcCalc(tmp.data(), tmp.size()));
+    dst->insert(dst->end(), tmp.begin(), tmp.end());
+  }
+
+  // Write the footer.
+  {
+    tmp.clear();
+    dwarf::Writer<> writer(&tmp);
+    writer.PushUint32(0);  // CRC placeholder.
+    writer.PushUint32(index_size_in_words);
+    writer.PushData(footer, sizeof(footer));
+    writer.UpdateUint32(0, CrcCalc(tmp.data() + 4, 6));
+    dst->insert(dst->end(), tmp.begin(), tmp.end());
+  }
+}
+
+}  // namespace debug
+}  // namespace art
diff --git a/compiler/debug/xz_utils.h b/compiler/debug/xz_utils.h
new file mode 100644
index 0000000..c4076c6
--- /dev/null
+++ b/compiler/debug/xz_utils.h
@@ -0,0 +1,32 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ART_COMPILER_DEBUG_XZ_UTILS_H_
+#define ART_COMPILER_DEBUG_XZ_UTILS_H_
+
+#include <vector>
+
+#include "base/array_ref.h"
+
+namespace art {
+namespace debug {
+
+void XzCompress(ArrayRef<uint8_t> src, std::vector<uint8_t>* dst);
+
+}  // namespace debug
+}  // namespace art
+
+#endif  // ART_COMPILER_DEBUG_XZ_UTILS_H_