Dex2oat support for multiple oat file and image file outputs.

Multiple changes to dex2oat and the runtime to support a --multi-image
option. This generates a separate oat file and image file output for
each dex file input.

Change-Id: Ie1d6f0b8afa8aed5790065b8c2eb177990c60129
diff --git a/runtime/gc/accounting/mod_union_table.cc b/runtime/gc/accounting/mod_union_table.cc
index 8f7bb94..d16afd9 100644
--- a/runtime/gc/accounting/mod_union_table.cc
+++ b/runtime/gc/accounting/mod_union_table.cc
@@ -487,7 +487,9 @@
 
 // Mark all references to the alloc space(s).
 void ModUnionTableCardCache::UpdateAndMarkReferences(MarkObjectVisitor* visitor) {
-  auto* image_space = heap_->GetBootImageSpace();
+  // TODO: Needs better support for multi-images? b/26317072
+  space::ImageSpace* image_space =
+      heap_->GetBootImageSpaces().empty() ? nullptr : heap_->GetBootImageSpaces()[0];
   // If we don't have an image space, just pass in space_ as the immune space. Pass in the same
   // space_ instead of image_space to avoid a null check in ModUnionUpdateObjectReferencesVisitor.
   CardBitVisitor bit_visitor(visitor, space_, image_space != nullptr ? image_space : space_,
diff --git a/runtime/gc/accounting/space_bitmap-inl.h b/runtime/gc/accounting/space_bitmap-inl.h
index 3be7181..61c67f8 100644
--- a/runtime/gc/accounting/space_bitmap-inl.h
+++ b/runtime/gc/accounting/space_bitmap-inl.h
@@ -167,7 +167,10 @@
   uintptr_t* address = &bitmap_begin_[index];
   uintptr_t old_word = *address;
   if (kSetBit) {
-    *address = old_word | mask;
+    if ((old_word & mask) == 0) {
+      // Avoid dirtying the page if possible.
+      *address = old_word | mask;
+    }
   } else {
     *address = old_word & ~mask;
   }
diff --git a/runtime/gc/heap.cc b/runtime/gc/heap.cc
index 6d72f31..3e432c7 100644
--- a/runtime/gc/heap.cc
+++ b/runtime/gc/heap.cc
@@ -233,8 +233,7 @@
       backtrace_lock_(nullptr),
       seen_backtrace_count_(0u),
       unique_backtrace_count_(0u),
-      gc_disabled_for_shutdown_(false),
-      boot_image_space_(nullptr) {
+      gc_disabled_for_shutdown_(false) {
   if (VLOG_IS_ON(heap) || VLOG_IS_ON(startup)) {
     LOG(INFO) << "Heap() entering";
   }
@@ -260,23 +259,107 @@
     CHECK_GE(300 * MB, non_moving_space_capacity);
     requested_alloc_space_begin = reinterpret_cast<uint8_t*>(300 * MB) - non_moving_space_capacity;
   }
+
+  // Load image space(s).
   if (!image_file_name.empty()) {
-    ATRACE_BEGIN("ImageSpace::Create");
-    std::string error_msg;
-    boot_image_space_ = space::ImageSpace::Create(image_file_name.c_str(),
-                                                  image_instruction_set,
-                                                  &error_msg);
-    ATRACE_END();
-    if (boot_image_space_ != nullptr) {
-      AddSpace(boot_image_space_);
-      // Oat files referenced by image files immediately follow them in memory, ensure alloc space
-      // isn't going to get in the middle
-      uint8_t* oat_file_end_addr = boot_image_space_->GetImageHeader().GetOatFileEnd();
-      CHECK_GT(oat_file_end_addr, boot_image_space_->End());
-      requested_alloc_space_begin = AlignUp(oat_file_end_addr, kPageSize);
-    } else {
-      LOG(ERROR) << "Could not create image space with image file '" << image_file_name << "'. "
-                   << "Attempting to fall back to imageless running. Error was: " << error_msg;
+    // For code reuse, handle this like a work queue.
+    std::vector<std::string> image_file_names;
+    image_file_names.push_back(image_file_name);
+
+    for (size_t index = 0; index < image_file_names.size(); ++index) {
+      std::string& image_name = image_file_names[index];
+      ATRACE_BEGIN("ImageSpace::Create");
+      std::string error_msg;
+      space::ImageSpace* boot_image_space = space::ImageSpace::Create(image_name.c_str(),
+                                                                      image_instruction_set,
+                                                                      index > 0,
+                                                                      &error_msg);
+      ATRACE_END();
+      if (boot_image_space != nullptr) {
+        AddSpace(boot_image_space);
+        // Oat files referenced by image files immediately follow them in memory, ensure alloc space
+        // isn't going to get in the middle
+        uint8_t* oat_file_end_addr = boot_image_space->GetImageHeader().GetOatFileEnd();
+        CHECK_GT(oat_file_end_addr, boot_image_space->End());
+        requested_alloc_space_begin = AlignUp(oat_file_end_addr, kPageSize);
+        boot_image_spaces_.push_back(boot_image_space);
+
+        if (index == 0) {
+          // If this was the first space, check whether there are more images to load.
+          const OatFile* boot_oat_file = boot_image_space->GetOatFile();
+          if (boot_oat_file == nullptr) {
+            continue;
+          }
+
+          const OatHeader& boot_oat_header = boot_oat_file->GetOatHeader();
+          const char* boot_classpath =
+              boot_oat_header.GetStoreValueByKey(OatHeader::kBootClassPath);
+          if (boot_classpath == nullptr) {
+            continue;
+          }
+
+          std::vector<std::string> images;
+          Split(boot_classpath, ':', &images);
+
+          // Add the rest into the list. We have to adjust locations, possibly:
+          //
+          // For example, image_file_name is /a/b/c/d/e.art
+          //              images[0] is          f/c/d/e.art
+          // ----------------------------------------------
+          //              images[1] is          g/h/i/j.art  -> /a/b/h/i/j.art
+
+          // Derive pattern.
+          std::vector<std::string> left;
+          Split(image_file_name, '/', &left);
+          std::vector<std::string> right;
+          Split(images[0], '/', &right);
+
+          size_t common = 1;
+          while (common < left.size() && common < right.size()) {
+            if (left[left.size() - common - 1] != right[right.size() - common - 1]) {
+              break;
+            }
+            common++;
+          }
+
+          std::vector<std::string> prefix_vector(left.begin(), left.end() - common);
+          std::string common_prefix = Join(prefix_vector, '/');
+          if (!common_prefix.empty() && common_prefix[0] != '/' && image_file_name[0] == '/') {
+            common_prefix = "/" + common_prefix;
+          }
+
+          // Apply pattern to images[1] .. images[n].
+          for (size_t i = 1; i < images.size(); ++i) {
+            std::string image = images[i];
+
+            size_t rslash = std::string::npos;
+            for (size_t j = 0; j < common; ++j) {
+              if (rslash != std::string::npos) {
+                rslash--;
+              }
+
+              rslash = image.rfind('/', rslash);
+              if (rslash == std::string::npos) {
+                rslash = 0;
+              }
+              if (rslash == 0) {
+                break;
+              }
+            }
+            std::string image_part = image.substr(rslash);
+
+            std::string new_image = common_prefix + (StartsWith(image_part, "/") ? "" : "/") +
+                image_part;
+            image_file_names.push_back(new_image);
+          }
+        }
+      } else {
+        LOG(ERROR) << "Could not create image space with image file '" << image_file_name << "'. "
+            << "Attempting to fall back to imageless running. Error was: " << error_msg
+            << "\nAttempted image: " << image_name;
+        // TODO: Remove already loaded spaces.
+        break;
+      }
     }
   }
   /*
@@ -456,13 +539,15 @@
     rb_table_.reset(new accounting::ReadBarrierTable());
     DCHECK(rb_table_->IsAllCleared());
   }
-  if (GetBootImageSpace() != nullptr) {
+  if (HasBootImageSpace()) {
     // Don't add the image mod union table if we are running without an image, this can crash if
     // we use the CardCache implementation.
-    accounting::ModUnionTable* mod_union_table = new accounting::ModUnionTableToZygoteAllocspace(
-        "Image mod-union table", this, GetBootImageSpace());
-    CHECK(mod_union_table != nullptr) << "Failed to create image mod-union table";
-    AddModUnionTable(mod_union_table);
+    for (space::ImageSpace* image_space : GetBootImageSpaces()) {
+      accounting::ModUnionTable* mod_union_table = new accounting::ModUnionTableToZygoteAllocspace(
+          "Image mod-union table", this, image_space);
+      CHECK(mod_union_table != nullptr) << "Failed to create image mod-union table";
+      AddModUnionTable(mod_union_table);
+    }
   }
   if (collector::SemiSpace::kUseRememberedSet && non_moving_space_ != main_space_) {
     accounting::RememberedSet* non_moving_space_rem_set =
@@ -525,11 +610,12 @@
       garbage_collectors_.push_back(mark_compact_collector_);
     }
   }
-  if (GetBootImageSpace() != nullptr && non_moving_space_ != nullptr &&
+  if (!GetBootImageSpaces().empty() && non_moving_space_ != nullptr &&
       (is_zygote || separate_non_moving_space || foreground_collector_type_ == kCollectorTypeGSS)) {
     // Check that there's no gap between the image space and the non moving space so that the
     // immune region won't break (eg. due to a large object allocated in the gap). This is only
     // required when we're the zygote or using GSS.
+    /* TODO: Modify this check to support multi-images. b/26317072
     bool no_gap = MemMap::CheckNoGaps(GetBootImageSpace()->GetMemMap(),
                                       non_moving_space_->GetMemMap());
     if (!no_gap) {
@@ -537,6 +623,7 @@
       MemMap::DumpMaps(LOG(ERROR), true);
       LOG(FATAL) << "There's a gap between the image space and the non-moving space";
     }
+    */
   }
   instrumentation::Instrumentation* const instrumentation = runtime->GetInstrumentation();
   if (gc_stress_mode_) {
@@ -1202,8 +1289,8 @@
   return FindDiscontinuousSpaceFromObject(obj, fail_ok);
 }
 
-space::ImageSpace* Heap::GetBootImageSpace() const {
-  return boot_image_space_;
+std::vector<space::ImageSpace*> Heap::GetBootImageSpaces() const {
+  return boot_image_spaces_;
 }
 
 void Heap::ThrowOutOfMemoryError(Thread* self, size_t byte_count, AllocatorType allocator_type) {
diff --git a/runtime/gc/heap.h b/runtime/gc/heap.h
index e23b1a3..e7ea983 100644
--- a/runtime/gc/heap.h
+++ b/runtime/gc/heap.h
@@ -580,9 +580,8 @@
   // Unbind any bound bitmaps.
   void UnBindBitmaps() REQUIRES(Locks::heap_bitmap_lock_);
 
-  // Returns the boot image space. There may be multiple image spaces, but there is only one boot
-  // image space.
-  space::ImageSpace* GetBootImageSpace() const;
+  // Returns the boot image spaces. There may be multiple boot image spaces.
+  std::vector<space::ImageSpace*> GetBootImageSpaces() const;
 
   // Permenantly disable moving garbage collection.
   void DisableMovingGc() REQUIRES(!*gc_complete_lock_);
@@ -660,8 +659,8 @@
   void RemoveRememberedSet(space::Space* space);
 
   bool IsCompilingBoot() const;
-  bool HasImageSpace() const {
-    return boot_image_space_ != nullptr;
+  bool HasBootImageSpace() const {
+    return !boot_image_spaces_.empty();
   }
 
   ReferenceProcessor* GetReferenceProcessor() {
@@ -1322,8 +1321,8 @@
   // allocating.
   bool gc_disabled_for_shutdown_ GUARDED_BY(gc_complete_lock_);
 
-  // Boot image space.
-  space::ImageSpace* boot_image_space_;
+  // Boot image spaces.
+  std::vector<space::ImageSpace*> boot_image_spaces_;
 
   friend class CollectorTransitionTask;
   friend class collector::GarbageCollector;
diff --git a/runtime/gc/space/image_space.cc b/runtime/gc/space/image_space.cc
index 8f67c21..952759c 100644
--- a/runtime/gc/space/image_space.cc
+++ b/runtime/gc/space/image_space.cc
@@ -43,12 +43,17 @@
 
 Atomic<uint32_t> ImageSpace::bitmap_index_(0);
 
-ImageSpace::ImageSpace(const std::string& image_filename, const char* image_location,
-                       MemMap* mem_map, accounting::ContinuousSpaceBitmap* live_bitmap,
-                       uint8_t* end)
+ImageSpace::ImageSpace(const std::string& image_filename,
+                       const char* image_location,
+                       MemMap* mem_map,
+                       accounting::ContinuousSpaceBitmap* live_bitmap,
+                       uint8_t* end,
+                       MemMap* shadow_map)
     : MemMapSpace(image_filename, mem_map, mem_map->Begin(), end, end,
                   kGcRetentionPolicyNeverCollect),
-      image_location_(image_location) {
+      oat_file_non_owned_(nullptr),
+      image_location_(image_location),
+      shadow_map_(shadow_map) {
   DCHECK(live_bitmap != nullptr);
   live_bitmap_.reset(live_bitmap);
 }
@@ -470,6 +475,7 @@
 
 ImageSpace* ImageSpace::Create(const char* image_location,
                                const InstructionSet image_isa,
+                               bool secondary_image,
                                std::string* error_msg) {
   std::string system_filename;
   bool has_system = false;
@@ -481,7 +487,7 @@
                                              &has_system, &cache_filename, &dalvik_cache_exists,
                                              &has_cache, &is_global_cache);
 
-  if (Runtime::Current()->IsZygote()) {
+  if (Runtime::Current()->IsZygote() && !secondary_image) {
     MarkZygoteStart(image_isa, Runtime::Current()->GetZygoteMaxFailedBoots());
   }
 
@@ -686,7 +692,7 @@
     return nullptr;
   }
 
-  if (kIsDebugBuild) {
+  if (VLOG_IS_ON(startup)) {
     LOG(INFO) << "Dumping image sections";
     for (size_t i = 0; i < ImageHeader::kSectionCount; ++i) {
       const auto section_idx = static_cast<ImageHeader::ImageSections>(i);
@@ -799,11 +805,52 @@
     return nullptr;
   }
 
+  // In case of multi-images, the images are spaced apart so that the bitmaps don't overlap. We
+  // need to reserve the slack, as otherwise the large object space might allocate in there.
+  // TODO: Reconsider the multi-image layout. b/26317072
+  std::unique_ptr<MemMap> shadow_map;
+  {
+    uintptr_t image_begin = reinterpret_cast<uintptr_t>(image_header.GetImageBegin());
+    uintptr_t image_end = RoundUp(image_begin + image_header.GetImageSize(), kPageSize);
+    uintptr_t oat_begin = reinterpret_cast<uintptr_t>(image_header.GetOatFileBegin());
+    if (image_end < oat_begin) {
+      // There's a gap. Could be multi-image, could be the oat file spaced apart. Go ahead and
+      // dummy-reserve the space covered by the bitmap (which will be a shadow that introduces
+      // a gap to the next image).
+      uintptr_t heap_size = bitmap->HeapSize();
+      uintptr_t bitmap_coverage_end = RoundUp(image_begin + heap_size, kPageSize);
+      if (bitmap_coverage_end > image_end) {
+        VLOG(startup) << "Reserving bitmap shadow ["
+                      << std::hex << image_end << ";"
+                      << std::hex << bitmap_coverage_end << ";] (oat file begins at "
+                      << std::hex << oat_begin;
+        // Note: we cannot use MemMap::Dummy here, as that won't reserve the space in 32-bit mode.
+        shadow_map.reset(MemMap::MapAnonymous("Image bitmap shadow",
+                                              reinterpret_cast<uint8_t*>(image_end),
+                                              bitmap_coverage_end - image_end,
+                                              PROT_NONE,
+                                              false,
+                                              false,
+                                              error_msg));
+        if (shadow_map == nullptr) {
+          return nullptr;
+        }
+        // madvise it away, we don't really want it, just reserve the address space.
+        // TODO: Should we use MadviseDontNeedAndZero? b/26317072
+        madvise(shadow_map->BaseBegin(), shadow_map->BaseSize(), MADV_DONTNEED);
+      }
+    }
+  }
+
   // We only want the mirror object, not the ArtFields and ArtMethods.
   uint8_t* const image_end =
       map->Begin() + image_header.GetImageSection(ImageHeader::kSectionObjects).End();
-  std::unique_ptr<ImageSpace> space(new ImageSpace(image_filename, image_location,
-                                                   map.release(), bitmap.release(), image_end));
+  std::unique_ptr<ImageSpace> space(new ImageSpace(image_filename,
+                                                   image_location,
+                                                   map.release(),
+                                                   bitmap.release(),
+                                                   image_end,
+                                                   shadow_map.release()));
 
   // VerifyImageAllocations() will be called later in Runtime::Init()
   // as some class roots like ArtMethod::java_lang_reflect_ArtMethod_
@@ -826,16 +873,18 @@
   Runtime* runtime = Runtime::Current();
   runtime->SetInstructionSet(space->oat_file_->GetOatHeader().GetInstructionSet());
 
-  runtime->SetResolutionMethod(image_header.GetImageMethod(ImageHeader::kResolutionMethod));
-  runtime->SetImtConflictMethod(image_header.GetImageMethod(ImageHeader::kImtConflictMethod));
-  runtime->SetImtUnimplementedMethod(
-      image_header.GetImageMethod(ImageHeader::kImtUnimplementedMethod));
-  runtime->SetCalleeSaveMethod(
-      image_header.GetImageMethod(ImageHeader::kCalleeSaveMethod), Runtime::kSaveAll);
-  runtime->SetCalleeSaveMethod(
-      image_header.GetImageMethod(ImageHeader::kRefsOnlySaveMethod), Runtime::kRefsOnly);
-  runtime->SetCalleeSaveMethod(
-      image_header.GetImageMethod(ImageHeader::kRefsAndArgsSaveMethod), Runtime::kRefsAndArgs);
+  if (!runtime->HasResolutionMethod()) {
+    runtime->SetResolutionMethod(image_header.GetImageMethod(ImageHeader::kResolutionMethod));
+    runtime->SetImtConflictMethod(image_header.GetImageMethod(ImageHeader::kImtConflictMethod));
+    runtime->SetImtUnimplementedMethod(
+        image_header.GetImageMethod(ImageHeader::kImtUnimplementedMethod));
+    runtime->SetCalleeSaveMethod(
+        image_header.GetImageMethod(ImageHeader::kCalleeSaveMethod), Runtime::kSaveAll);
+    runtime->SetCalleeSaveMethod(
+        image_header.GetImageMethod(ImageHeader::kRefsOnlySaveMethod), Runtime::kRefsOnly);
+    runtime->SetCalleeSaveMethod(
+        image_header.GetImageMethod(ImageHeader::kRefsAndArgsSaveMethod), Runtime::kRefsAndArgs);
+  }
 
   if (VLOG_IS_ON(heap) || VLOG_IS_ON(startup)) {
     LOG(INFO) << "ImageSpace::Init exiting (" << PrettyDuration(NanoTime() - start_time)
diff --git a/runtime/gc/space/image_space.h b/runtime/gc/space/image_space.h
index babd672..a54358a 100644
--- a/runtime/gc/space/image_space.h
+++ b/runtime/gc/space/image_space.h
@@ -43,7 +43,10 @@
   // creation of the alloc space. The ReleaseOatFile will later be
   // used to transfer ownership of the OatFile to the ClassLinker when
   // it is initialized.
-  static ImageSpace* Create(const char* image, InstructionSet image_isa, std::string* error_msg)
+  static ImageSpace* Create(const char* image,
+                            InstructionSet image_isa,
+                            bool secondary_image,
+                            std::string* error_msg)
       SHARED_REQUIRES(Locks::mutator_lock_);
 
   // Reads the image header from the specified image location for the
@@ -158,8 +161,12 @@
 
   std::unique_ptr<accounting::ContinuousSpaceBitmap> live_bitmap_;
 
-  ImageSpace(const std::string& name, const char* image_location,
-             MemMap* mem_map, accounting::ContinuousSpaceBitmap* live_bitmap, uint8_t* end);
+  ImageSpace(const std::string& name,
+             const char* image_location,
+             MemMap* mem_map,
+             accounting::ContinuousSpaceBitmap* live_bitmap,
+             uint8_t* end,
+             MemMap* shadow_map = nullptr);
 
   // The OatFile associated with the image during early startup to
   // reserve space contiguous to the image. It is later released to
@@ -172,6 +179,10 @@
 
   const std::string image_location_;
 
+  // A MemMap reserving the space of the bitmap "shadow," so that we don't allocate into it. Only
+  // used in the multi-image case.
+  std::unique_ptr<MemMap> shadow_map_;
+
  private:
   DISALLOW_COPY_AND_ASSIGN(ImageSpace);
 };