runtime/gc/space/rosalloc_space.cc - LeafOS-Project/android_art - Gitiles


 /*
  * Copyright (C) 2013 The Android Open Source Project
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
  *      http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */

 #include "rosalloc_space-inl.h"

 #include "base/logging.h"  // For VLOG.
 #include "base/time_utils.h"
 #include "base/utils.h"
 #include "gc/accounting/card_table.h"
 #include "gc/accounting/space_bitmap-inl.h"
 #include "gc/heap.h"
 #include "memory_tool_malloc_space-inl.h"
 #include "mirror/class-inl.h"
 #include "mirror/object-inl.h"
 #include "runtime.h"
 #include "scoped_thread_state_change-inl.h"
 #include "thread.h"
 #include "thread_list.h"

 namespace art {
 namespace gc {
 namespace space {

 static constexpr bool kPrefetchDuringRosAllocFreeList = false;
 static constexpr size_t kPrefetchLookAhead = 8;
 // Use this only for verification, it is not safe to use since the class of the object may have
 // been freed.
 static constexpr bool kVerifyFreedBytes = false;

 // TODO: Fix
 // template class MemoryToolMallocSpace<RosAllocSpace, allocator::RosAlloc*>;

 RosAllocSpace::RosAllocSpace(MemMap&& mem_map,
                              size_t initial_size,
                              const std::string& name,
                              art::gc::allocator::RosAlloc* rosalloc,
                              uint8_t* begin,
                              uint8_t* end,
                              uint8_t* limit,
                              size_t growth_limit,
                              bool can_move_objects,
                              size_t starting_size,
                              bool low_memory_mode)
     : MallocSpace(name,
                   std::move(mem_map),
                   begin,
                   end,
                   limit,
                   growth_limit,
                   true,
                   can_move_objects,
                   starting_size, initial_size),
       rosalloc_(rosalloc), low_memory_mode_(low_memory_mode) {
   CHECK(rosalloc != nullptr);
 }

 RosAllocSpace* RosAllocSpace::CreateFromMemMap(MemMap&& mem_map,
                                                const std::string& name,
                                                size_t starting_size,
                                                size_t initial_size,
                                                size_t growth_limit,
                                                size_t capacity,
                                                bool low_memory_mode,
                                                bool can_move_objects) {
   DCHECK(mem_map.IsValid());

   bool running_on_memory_tool = Runtime::Current()->IsRunningOnMemoryTool();

   allocator::RosAlloc* rosalloc = CreateRosAlloc(mem_map.Begin(),
                                                  starting_size,
                                                  initial_size,
                                                  capacity,
                                                  low_memory_mode,
                                                  running_on_memory_tool);
   if (rosalloc == nullptr) {
     LOG(ERROR) << "Failed to initialize rosalloc for alloc space (" << name << ")";
     return nullptr;
   }

   // Protect memory beyond the starting size. MoreCore will add r/w permissions when necessory
   uint8_t* end = mem_map.Begin() + starting_size;
   if (capacity - starting_size > 0) {
     CheckedCall(mprotect, name.c_str(), end, capacity - starting_size, PROT_NONE);
   }

   // Everything is set so record in immutable structure and leave
   uint8_t* begin = mem_map.Begin();
   // TODO: Fix RosAllocSpace to support ASan. There is currently some issues with
   // AllocationSize caused by redzones. b/12944686
   if (running_on_memory_tool) {
     return new MemoryToolMallocSpace<RosAllocSpace, kDefaultMemoryToolRedZoneBytes, false, true>(
         std::move(mem_map),
         initial_size,
         name,
         rosalloc,
         begin,
         end,
         begin + capacity,
         growth_limit,
         can_move_objects,
         starting_size,
         low_memory_mode);
   } else {
     return new RosAllocSpace(std::move(mem_map),
                              initial_size,
                              name,
                              rosalloc,
                              begin,
                              end,
                              begin + capacity,
                              growth_limit,
                              can_move_objects,
                              starting_size,
                              low_memory_mode);
   }
 }

 RosAllocSpace::~RosAllocSpace() {
   delete rosalloc_;
 }

 RosAllocSpace* RosAllocSpace::Create(const std::string& name,
                                      size_t initial_size,
                                      size_t growth_limit,
                                      size_t capacity,
                                      bool low_memory_mode,
                                      bool can_move_objects) {
   uint64_t start_time = 0;
   if (VLOG_IS_ON(heap) || VLOG_IS_ON(startup)) {
     start_time = NanoTime();
     VLOG(startup) << "RosAllocSpace::Create entering " << name
                   << " initial_size=" << PrettySize(initial_size)
                   << " growth_limit=" << PrettySize(growth_limit)
                   << " capacity=" << PrettySize(capacity);
   }

   // Memory we promise to rosalloc before it asks for morecore.
   // Note: making this value large means that large allocations are unlikely to succeed as rosalloc
   // will ask for this memory from sys_alloc which will fail as the footprint (this value plus the
   // size of the large allocation) will be greater than the footprint limit.
   size_t starting_size = Heap::kDefaultStartingSize;
   MemMap mem_map = CreateMemMap(name, starting_size, &initial_size, &growth_limit, &capacity);
   if (!mem_map.IsValid()) {
     LOG(ERROR) << "Failed to create mem map for alloc space (" << name << ") of size "
                << PrettySize(capacity);
     return nullptr;
   }

   RosAllocSpace* space = CreateFromMemMap(std::move(mem_map),
                                           name,
                                           starting_size,
                                           initial_size,
                                           growth_limit,
                                           capacity,
                                           low_memory_mode,
                                           can_move_objects);
   // We start out with only the initial size possibly containing objects.
   if (VLOG_IS_ON(heap) || VLOG_IS_ON(startup)) {
     LOG(INFO) << "RosAllocSpace::Create exiting (" << PrettyDuration(NanoTime() - start_time)
         << " ) " << *space;
   }
   return space;
 }

 allocator::RosAlloc* RosAllocSpace::CreateRosAlloc(void* begin, size_t morecore_start,
                                                    size_t initial_size,
                                                    size_t maximum_size, bool low_memory_mode,
                                                    bool running_on_memory_tool) {
   // clear errno to allow PLOG on error
   errno = 0;
   // create rosalloc using our backing storage starting at begin and
   // with a footprint of morecore_start. When morecore_start bytes of
   // memory is exhaused morecore will be called.
   allocator::RosAlloc* rosalloc = new art::gc::allocator::RosAlloc(
       begin, morecore_start, maximum_size,
       low_memory_mode ?
           art::gc::allocator::RosAlloc::kPageReleaseModeAll :
           art::gc::allocator::RosAlloc::kPageReleaseModeSizeAndEnd,
       running_on_memory_tool);
   if (rosalloc != nullptr) {
     rosalloc->SetFootprintLimit(initial_size);
   } else {
     PLOG(ERROR) << "RosAlloc::Create failed";
   }
   return rosalloc;
 }

 mirror::Object* RosAllocSpace::AllocWithGrowth(Thread* self, size_t num_bytes,
                                                size_t* bytes_allocated, size_t* usable_size,
                                                size_t* bytes_tl_bulk_allocated) {
   mirror::Object* result;
   {
     MutexLock mu(self, lock_);
     // Grow as much as possible within the space.
     size_t max_allowed = Capacity();
     rosalloc_->SetFootprintLimit(max_allowed);
     // Try the allocation.
     result = AllocCommon(self, num_bytes, bytes_allocated, usable_size,
                          bytes_tl_bulk_allocated);
     // Shrink back down as small as possible.
     size_t footprint = rosalloc_->Footprint();
     rosalloc_->SetFootprintLimit(footprint);
   }
   // Note RosAlloc zeroes memory internally.
   // Return the new allocation or null.
   CHECK(!kDebugSpaces || result == nullptr || Contains(result));
   return result;
 }

 MallocSpace* RosAllocSpace::CreateInstance(MemMap&& mem_map,
                                            const std::string& name,
                                            void* allocator,
                                            uint8_t* begin,
                                            uint8_t* end,
                                            uint8_t* limit,
                                            size_t growth_limit,
                                            bool can_move_objects) {
   if (Runtime::Current()->IsRunningOnMemoryTool()) {
     return new MemoryToolMallocSpace<RosAllocSpace, kDefaultMemoryToolRedZoneBytes, false, true>(
         std::move(mem_map),
         initial_size_,
         name,
         reinterpret_cast<allocator::RosAlloc*>(allocator),
         begin,
         end,
         limit,
         growth_limit,
         can_move_objects,
         starting_size_,
         low_memory_mode_);
   } else {
     return new RosAllocSpace(std::move(mem_map),
                              initial_size_,
                              name,
                              reinterpret_cast<allocator::RosAlloc*>(allocator),
                              begin,
                              end,
                              limit,
                              growth_limit,
                              can_move_objects,
                              starting_size_,
                              low_memory_mode_);
   }
 }

 size_t RosAllocSpace::Free(Thread* self, mirror::Object* ptr) {
   if (kDebugSpaces) {
     CHECK(ptr != nullptr);
     CHECK(Contains(ptr)) << "Free (" << ptr << ") not in bounds of heap " << *this;
   }
   if (kRecentFreeCount > 0) {
     MutexLock mu(self, lock_);
     RegisterRecentFree(ptr);
   }
   return rosalloc_->Free(self, ptr);
 }

 size_t RosAllocSpace::FreeList(Thread* self, size_t num_ptrs, mirror::Object** ptrs) {
   DCHECK(ptrs != nullptr);

   size_t verify_bytes = 0;
   for (size_t i = 0; i < num_ptrs; i++) {
     if (kPrefetchDuringRosAllocFreeList && i + kPrefetchLookAhead < num_ptrs) {
       __builtin_prefetch(reinterpret_cast<char*>(ptrs[i + kPrefetchLookAhead]));
     }
     if (kVerifyFreedBytes) {
       verify_bytes += AllocationSizeNonvirtual<true>(ptrs[i], nullptr);
     }
   }

   if (kRecentFreeCount > 0) {
     MutexLock mu(self, lock_);
     for (size_t i = 0; i < num_ptrs; i++) {
       RegisterRecentFree(ptrs[i]);
     }
   }

   if (kDebugSpaces) {
     size_t num_broken_ptrs = 0;
     for (size_t i = 0; i < num_ptrs; i++) {
       if (!Contains(ptrs[i])) {
         num_broken_ptrs++;
         LOG(ERROR) << "FreeList[" << i << "] (" << ptrs[i] << ") not in bounds of heap " << *this;
       } else {
         size_t size = rosalloc_->UsableSize(ptrs[i]);
         memset(ptrs[i], 0xEF, size);
       }
     }
     CHECK_EQ(num_broken_ptrs, 0u);
   }

   const size_t bytes_freed = rosalloc_->BulkFree(self, reinterpret_cast<void**>(ptrs), num_ptrs);
   if (kVerifyFreedBytes) {
     CHECK_EQ(verify_bytes, bytes_freed);
   }
   return bytes_freed;
 }

 size_t RosAllocSpace::Trim() {
   VLOG(heap) << "RosAllocSpace::Trim() ";
   {
     Thread* const self = Thread::Current();
     // SOA required for Rosalloc::Trim() -> ArtRosAllocMoreCore() -> Heap::GetRosAllocSpace.
     ScopedObjectAccess soa(self);
     MutexLock mu(self, lock_);
     // Trim to release memory at the end of the space.
     rosalloc_->Trim();
   }
   // Attempt to release pages if it does not release all empty pages.
   if (!rosalloc_->DoesReleaseAllPages()) {
     return rosalloc_->ReleasePages();
   }
   return 0;
 }

 void RosAllocSpace::Walk(void(*callback)(void *start, void *end, size_t num_bytes, void* callback_arg),
                          void* arg) {
   InspectAllRosAlloc(callback, arg, true);
 }

 size_t RosAllocSpace::GetFootprint() {
   MutexLock mu(Thread::Current(), lock_);
   return rosalloc_->Footprint();
 }

 size_t RosAllocSpace::GetFootprintLimit() {
   MutexLock mu(Thread::Current(), lock_);
   return rosalloc_->FootprintLimit();
 }

 void RosAllocSpace::SetFootprintLimit(size_t new_size) {
   MutexLock mu(Thread::Current(), lock_);
   VLOG(heap) << "RosAllocSpace::SetFootprintLimit " << PrettySize(new_size);
   // Compare against the actual footprint, rather than the Size(), because the heap may not have
   // grown all the way to the allowed size yet.
   size_t current_space_size = rosalloc_->Footprint();
   if (new_size < current_space_size) {
     // Don't let the space grow any more.
     new_size = current_space_size;
   }
   rosalloc_->SetFootprintLimit(new_size);
 }

 uint64_t RosAllocSpace::GetBytesAllocated() {
   size_t bytes_allocated = 0;
   InspectAllRosAlloc(art::gc::allocator::RosAlloc::BytesAllocatedCallback, &bytes_allocated, false);
   return bytes_allocated;
 }

 uint64_t RosAllocSpace::GetObjectsAllocated() {
   size_t objects_allocated = 0;
   InspectAllRosAlloc(art::gc::allocator::RosAlloc::ObjectsAllocatedCallback, &objects_allocated, false);
   return objects_allocated;
 }

 void RosAllocSpace::InspectAllRosAllocWithSuspendAll(
     void (*callback)(void *start, void *end, size_t num_bytes, void* callback_arg),
     void* arg, bool do_null_callback_at_end) NO_THREAD_SAFETY_ANALYSIS {
   // TODO: NO_THREAD_SAFETY_ANALYSIS.
   Thread* self = Thread::Current();
   ScopedSuspendAll ssa(__FUNCTION__);
   MutexLock mu(self, *Locks::runtime_shutdown_lock_);
   MutexLock mu2(self, *Locks::thread_list_lock_);
   rosalloc_->InspectAll(callback, arg);
   if (do_null_callback_at_end) {
     callback(nullptr, nullptr, 0, arg);  // Indicate end of a space.
   }
 }

 void RosAllocSpace::InspectAllRosAlloc(void (*callback)(void *start, void *end, size_t num_bytes, void* callback_arg),
                                        void* arg, bool do_null_callback_at_end) NO_THREAD_SAFETY_ANALYSIS {
   // TODO: NO_THREAD_SAFETY_ANALYSIS.
   Thread* self = Thread::Current();
   if (Locks::mutator_lock_->IsExclusiveHeld(self)) {
     // The mutators are already suspended. For example, a call path
     // from SignalCatcher::HandleSigQuit().
     rosalloc_->InspectAll(callback, arg);
     if (do_null_callback_at_end) {
       callback(nullptr, nullptr, 0, arg);  // Indicate end of a space.
     }
   } else if (Locks::mutator_lock_->IsSharedHeld(self)) {
     // The mutators are not suspended yet and we have a shared access
     // to the mutator lock. Temporarily release the shared access by
     // transitioning to the suspend state, and suspend the mutators.
     ScopedThreadSuspension sts(self, ThreadState::kSuspended);
     InspectAllRosAllocWithSuspendAll(callback, arg, do_null_callback_at_end);
   } else {
     // The mutators are not suspended yet. Suspend the mutators.
     InspectAllRosAllocWithSuspendAll(callback, arg, do_null_callback_at_end);
   }
 }

 size_t RosAllocSpace::RevokeThreadLocalBuffers(Thread* thread) {
   return rosalloc_->RevokeThreadLocalRuns(thread);
 }

 size_t RosAllocSpace::RevokeAllThreadLocalBuffers() {
   return rosalloc_->RevokeAllThreadLocalRuns();
 }

 void RosAllocSpace::AssertThreadLocalBuffersAreRevoked(Thread* thread) {
   if (kIsDebugBuild) {
     rosalloc_->AssertThreadLocalRunsAreRevoked(thread);
   }
 }

 void RosAllocSpace::AssertAllThreadLocalBuffersAreRevoked() {
   if (kIsDebugBuild) {
     rosalloc_->AssertAllThreadLocalRunsAreRevoked();
   }
 }

 void RosAllocSpace::Clear() {
   size_t footprint_limit = GetFootprintLimit();
   madvise(GetMemMap()->Begin(), GetMemMap()->Size(), MADV_DONTNEED);
   live_bitmap_.Clear();
   mark_bitmap_.Clear();
   SetEnd(begin_ + starting_size_);
   delete rosalloc_;
   rosalloc_ = CreateRosAlloc(mem_map_.Begin(),
                              starting_size_,
                              initial_size_,
                              NonGrowthLimitCapacity(),
                              low_memory_mode_,
                              Runtime::Current()->IsRunningOnMemoryTool());
   SetFootprintLimit(footprint_limit);
 }

 void RosAllocSpace::DumpStats(std::ostream& os) {
   ScopedSuspendAll ssa(__FUNCTION__);
   rosalloc_->DumpStats(os);
 }

 template<bool kMaybeIsRunningOnMemoryTool>
 size_t RosAllocSpace::AllocationSizeNonvirtual(mirror::Object* obj, size_t* usable_size) {
   // obj is a valid object. Use its class in the header to get the size.
   // Don't use verification since the object may be dead if we are sweeping.
   size_t size = obj->SizeOf<kVerifyNone>();
   bool add_redzones = false;
   if (kMaybeIsRunningOnMemoryTool) {
     add_redzones = kRunningOnMemoryTool && kMemoryToolAddsRedzones;
     if (add_redzones) {
       size += 2 * kDefaultMemoryToolRedZoneBytes;
     }
   } else {
     DCHECK(!kRunningOnMemoryTool);
   }
   size_t size_by_size = rosalloc_->UsableSize(size);
   if (kIsDebugBuild) {
     // On memory tool, the red zone has an impact...
     const uint8_t* obj_ptr = reinterpret_cast<const uint8_t*>(obj);
     size_t size_by_ptr = rosalloc_->UsableSize(
         obj_ptr - (add_redzones ? kDefaultMemoryToolRedZoneBytes : 0));
     if (size_by_size != size_by_ptr) {
       LOG(INFO) << "Found a bad sized obj of size " << size
                 << " at " << std::hex << reinterpret_cast<intptr_t>(obj_ptr) << std::dec
                 << " size_by_size=" << size_by_size << " size_by_ptr=" << size_by_ptr;
     }
     DCHECK_EQ(size_by_size, size_by_ptr);
   }
   if (usable_size != nullptr) {
     *usable_size = size_by_size;
   }
   return size_by_size;
 }

 }  // namespace space

 namespace allocator {

 // Callback from rosalloc when it needs to increase the footprint.
 void* ArtRosAllocMoreCore(allocator::RosAlloc* rosalloc, intptr_t increment)
     REQUIRES_SHARED(Locks::mutator_lock_) {
   Heap* heap = Runtime::Current()->GetHeap();
   art::gc::space::RosAllocSpace* rosalloc_space = heap->GetRosAllocSpace(rosalloc);
   DCHECK(rosalloc_space != nullptr);
   DCHECK_EQ(rosalloc_space->GetRosAlloc(), rosalloc);
   return rosalloc_space->MoreCore(increment);
 }

 }  // namespace allocator

 }  // namespace gc
 }  // namespace art

	/*
	* Copyright (C) 2013 The Android Open Source Project
	*
	* Licensed under the Apache License, Version 2.0 (the "License");
	* you may not use this file except in compliance with the License.
	* You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/

	#include "rosalloc_space-inl.h"

	#include "base/logging.h" // For VLOG.
	#include "base/time_utils.h"
	#include "base/utils.h"
	#include "gc/accounting/card_table.h"
	#include "gc/accounting/space_bitmap-inl.h"
	#include "gc/heap.h"
	#include "memory_tool_malloc_space-inl.h"
	#include "mirror/class-inl.h"
	#include "mirror/object-inl.h"
	#include "runtime.h"
	#include "scoped_thread_state_change-inl.h"
	#include "thread.h"
	#include "thread_list.h"

	namespace art {
	namespace gc {
	namespace space {

	static constexpr bool kPrefetchDuringRosAllocFreeList = false;
	static constexpr size_t kPrefetchLookAhead = 8;
	// Use this only for verification, it is not safe to use since the class of the object may have
	// been freed.
	static constexpr bool kVerifyFreedBytes = false;

	// TODO: Fix
	// template class MemoryToolMallocSpace<RosAllocSpace, allocator::RosAlloc*>;

	RosAllocSpace::RosAllocSpace(MemMap&& mem_map,
	size_t initial_size,
	const std::string& name,
	art::gc::allocator::RosAlloc* rosalloc,
	uint8_t* begin,
	uint8_t* end,
	uint8_t* limit,
	size_t growth_limit,
	bool can_move_objects,
	size_t starting_size,
	bool low_memory_mode)
	: MallocSpace(name,
	std::move(mem_map),
	begin,
	end,
	limit,
	growth_limit,
	true,
	can_move_objects,
	starting_size, initial_size),
	rosalloc_(rosalloc), low_memory_mode_(low_memory_mode) {
	CHECK(rosalloc != nullptr);
	}

	RosAllocSpace* RosAllocSpace::CreateFromMemMap(MemMap&& mem_map,
	const std::string& name,
	size_t starting_size,
	size_t initial_size,
	size_t growth_limit,
	size_t capacity,
	bool low_memory_mode,
	bool can_move_objects) {
	DCHECK(mem_map.IsValid());

	bool running_on_memory_tool = Runtime::Current()->IsRunningOnMemoryTool();

	allocator::RosAlloc* rosalloc = CreateRosAlloc(mem_map.Begin(),
	starting_size,
	initial_size,
	capacity,
	low_memory_mode,
	running_on_memory_tool);
	if (rosalloc == nullptr) {
	LOG(ERROR) << "Failed to initialize rosalloc for alloc space (" << name << ")";
	return nullptr;
	}

	// Protect memory beyond the starting size. MoreCore will add r/w permissions when necessory
	uint8_t* end = mem_map.Begin() + starting_size;
	if (capacity - starting_size > 0) {
	CheckedCall(mprotect, name.c_str(), end, capacity - starting_size, PROT_NONE);
	}

	// Everything is set so record in immutable structure and leave
	uint8_t* begin = mem_map.Begin();
	// TODO: Fix RosAllocSpace to support ASan. There is currently some issues with
	// AllocationSize caused by redzones. b/12944686
	if (running_on_memory_tool) {
	return new MemoryToolMallocSpace<RosAllocSpace, kDefaultMemoryToolRedZoneBytes, false, true>(
	std::move(mem_map),
	initial_size,
	name,
	rosalloc,
	begin,
	end,
	begin + capacity,
	growth_limit,
	can_move_objects,
	starting_size,
	low_memory_mode);
	} else {
	return new RosAllocSpace(std::move(mem_map),
	initial_size,
	name,
	rosalloc,
	begin,
	end,
	begin + capacity,
	growth_limit,
	can_move_objects,
	starting_size,
	low_memory_mode);
	}
	}

	RosAllocSpace::~RosAllocSpace() {
	delete rosalloc_;
	}

	RosAllocSpace* RosAllocSpace::Create(const std::string& name,
	size_t initial_size,
	size_t growth_limit,
	size_t capacity,
	bool low_memory_mode,
	bool can_move_objects) {
	uint64_t start_time = 0;
	if (VLOG_IS_ON(heap) \|\| VLOG_IS_ON(startup)) {
	start_time = NanoTime();
	VLOG(startup) << "RosAllocSpace::Create entering " << name
	<< " initial_size=" << PrettySize(initial_size)
	<< " growth_limit=" << PrettySize(growth_limit)
	<< " capacity=" << PrettySize(capacity);
	}

	// Memory we promise to rosalloc before it asks for morecore.
	// Note: making this value large means that large allocations are unlikely to succeed as rosalloc
	// will ask for this memory from sys_alloc which will fail as the footprint (this value plus the
	// size of the large allocation) will be greater than the footprint limit.
	size_t starting_size = Heap::kDefaultStartingSize;
	MemMap mem_map = CreateMemMap(name, starting_size, &initial_size, &growth_limit, &capacity);
	if (!mem_map.IsValid()) {
	LOG(ERROR) << "Failed to create mem map for alloc space (" << name << ") of size "
	<< PrettySize(capacity);
	return nullptr;
	}

	RosAllocSpace* space = CreateFromMemMap(std::move(mem_map),
	name,
	starting_size,
	initial_size,
	growth_limit,
	capacity,
	low_memory_mode,
	can_move_objects);
	// We start out with only the initial size possibly containing objects.
	if (VLOG_IS_ON(heap) \|\| VLOG_IS_ON(startup)) {
	LOG(INFO) << "RosAllocSpace::Create exiting (" << PrettyDuration(NanoTime() - start_time)
	<< " ) " << *space;
	}
	return space;
	}

	allocator::RosAlloc* RosAllocSpace::CreateRosAlloc(void* begin, size_t morecore_start,
	size_t initial_size,
	size_t maximum_size, bool low_memory_mode,
	bool running_on_memory_tool) {
	// clear errno to allow PLOG on error
	errno = 0;
	// create rosalloc using our backing storage starting at begin and
	// with a footprint of morecore_start. When morecore_start bytes of
	// memory is exhaused morecore will be called.
	allocator::RosAlloc* rosalloc = new art::gc::allocator::RosAlloc(
	begin, morecore_start, maximum_size,
	low_memory_mode ?
	art::gc::allocator::RosAlloc::kPageReleaseModeAll :
	art::gc::allocator::RosAlloc::kPageReleaseModeSizeAndEnd,
	running_on_memory_tool);
	if (rosalloc != nullptr) {
	rosalloc->SetFootprintLimit(initial_size);
	} else {
	PLOG(ERROR) << "RosAlloc::Create failed";
	}
	return rosalloc;
	}

	mirror::Object* RosAllocSpace::AllocWithGrowth(Thread* self, size_t num_bytes,
	size_t* bytes_allocated, size_t* usable_size,
	size_t* bytes_tl_bulk_allocated) {
	mirror::Object* result;
	{
	MutexLock mu(self, lock_);
	// Grow as much as possible within the space.
	size_t max_allowed = Capacity();
	rosalloc_->SetFootprintLimit(max_allowed);
	// Try the allocation.
	result = AllocCommon(self, num_bytes, bytes_allocated, usable_size,
	bytes_tl_bulk_allocated);
	// Shrink back down as small as possible.
	size_t footprint = rosalloc_->Footprint();
	rosalloc_->SetFootprintLimit(footprint);
	}
	// Note RosAlloc zeroes memory internally.
	// Return the new allocation or null.
	CHECK(!kDebugSpaces \|\| result == nullptr \|\| Contains(result));
	return result;
	}

	MallocSpace* RosAllocSpace::CreateInstance(MemMap&& mem_map,
	const std::string& name,
	void* allocator,
	uint8_t* begin,
	uint8_t* end,
	uint8_t* limit,
	size_t growth_limit,
	bool can_move_objects) {
	if (Runtime::Current()->IsRunningOnMemoryTool()) {
	return new MemoryToolMallocSpace<RosAllocSpace, kDefaultMemoryToolRedZoneBytes, false, true>(
	std::move(mem_map),
	initial_size_,
	name,
	reinterpret_cast<allocator::RosAlloc*>(allocator),
	begin,
	end,
	limit,
	growth_limit,
	can_move_objects,
	starting_size_,
	low_memory_mode_);
	} else {
	return new RosAllocSpace(std::move(mem_map),
	initial_size_,
	name,
	reinterpret_cast<allocator::RosAlloc*>(allocator),
	begin,
	end,
	limit,
	growth_limit,
	can_move_objects,
	starting_size_,
	low_memory_mode_);
	}
	}

	size_t RosAllocSpace::Free(Thread* self, mirror::Object* ptr) {
	if (kDebugSpaces) {
	CHECK(ptr != nullptr);
	CHECK(Contains(ptr)) << "Free (" << ptr << ") not in bounds of heap " << *this;
	}
	if (kRecentFreeCount > 0) {
	MutexLock mu(self, lock_);
	RegisterRecentFree(ptr);
	}
	return rosalloc_->Free(self, ptr);
	}

	size_t RosAllocSpace::FreeList(Thread* self, size_t num_ptrs, mirror::Object** ptrs) {
	DCHECK(ptrs != nullptr);

	size_t verify_bytes = 0;
	for (size_t i = 0; i < num_ptrs; i++) {
	if (kPrefetchDuringRosAllocFreeList && i + kPrefetchLookAhead < num_ptrs) {
	__builtin_prefetch(reinterpret_cast<char*>(ptrs[i + kPrefetchLookAhead]));
	}
	if (kVerifyFreedBytes) {
	verify_bytes += AllocationSizeNonvirtual<true>(ptrs[i], nullptr);
	}
	}

	if (kRecentFreeCount > 0) {
	MutexLock mu(self, lock_);
	for (size_t i = 0; i < num_ptrs; i++) {
	RegisterRecentFree(ptrs[i]);
	}
	}

	if (kDebugSpaces) {
	size_t num_broken_ptrs = 0;
	for (size_t i = 0; i < num_ptrs; i++) {
	if (!Contains(ptrs[i])) {
	num_broken_ptrs++;
	LOG(ERROR) << "FreeList[" << i << "] (" << ptrs[i] << ") not in bounds of heap " << *this;
	} else {
	size_t size = rosalloc_->UsableSize(ptrs[i]);
	memset(ptrs[i], 0xEF, size);
	}
	}
	CHECK_EQ(num_broken_ptrs, 0u);
	}

	const size_t bytes_freed = rosalloc_->BulkFree(self, reinterpret_cast<void**>(ptrs), num_ptrs);
	if (kVerifyFreedBytes) {
	CHECK_EQ(verify_bytes, bytes_freed);
	}
	return bytes_freed;
	}

	size_t RosAllocSpace::Trim() {
	VLOG(heap) << "RosAllocSpace::Trim() ";
	{
	Thread* const self = Thread::Current();
	// SOA required for Rosalloc::Trim() -> ArtRosAllocMoreCore() -> Heap::GetRosAllocSpace.
	ScopedObjectAccess soa(self);
	MutexLock mu(self, lock_);
	// Trim to release memory at the end of the space.
	rosalloc_->Trim();
	}
	// Attempt to release pages if it does not release all empty pages.
	if (!rosalloc_->DoesReleaseAllPages()) {
	return rosalloc_->ReleasePages();
	}
	return 0;
	}

	void RosAllocSpace::Walk(void(callback)(void start, void end, size_t num_bytes, void callback_arg),
	void* arg) {
	InspectAllRosAlloc(callback, arg, true);
	}

	size_t RosAllocSpace::GetFootprint() {
	MutexLock mu(Thread::Current(), lock_);
	return rosalloc_->Footprint();
	}

	size_t RosAllocSpace::GetFootprintLimit() {
	MutexLock mu(Thread::Current(), lock_);
	return rosalloc_->FootprintLimit();
	}

	void RosAllocSpace::SetFootprintLimit(size_t new_size) {
	MutexLock mu(Thread::Current(), lock_);
	VLOG(heap) << "RosAllocSpace::SetFootprintLimit " << PrettySize(new_size);
	// Compare against the actual footprint, rather than the Size(), because the heap may not have
	// grown all the way to the allowed size yet.
	size_t current_space_size = rosalloc_->Footprint();
	if (new_size < current_space_size) {
	// Don't let the space grow any more.
	new_size = current_space_size;
	}
	rosalloc_->SetFootprintLimit(new_size);
	}

	uint64_t RosAllocSpace::GetBytesAllocated() {
	size_t bytes_allocated = 0;
	InspectAllRosAlloc(art::gc::allocator::RosAlloc::BytesAllocatedCallback, &bytes_allocated, false);
	return bytes_allocated;
	}

	uint64_t RosAllocSpace::GetObjectsAllocated() {
	size_t objects_allocated = 0;
	InspectAllRosAlloc(art::gc::allocator::RosAlloc::ObjectsAllocatedCallback, &objects_allocated, false);
	return objects_allocated;
	}

	void RosAllocSpace::InspectAllRosAllocWithSuspendAll(
	void (callback)(void start, void end, size_t num_bytes, void callback_arg),
	void* arg, bool do_null_callback_at_end) NO_THREAD_SAFETY_ANALYSIS {
	// TODO: NO_THREAD_SAFETY_ANALYSIS.
	Thread* self = Thread::Current();
	ScopedSuspendAll ssa(__FUNCTION__);
	MutexLock mu(self, *Locks::runtime_shutdown_lock_);
	MutexLock mu2(self, *Locks::thread_list_lock_);
	rosalloc_->InspectAll(callback, arg);
	if (do_null_callback_at_end) {
	callback(nullptr, nullptr, 0, arg); // Indicate end of a space.
	}
	}

	void RosAllocSpace::InspectAllRosAlloc(void (callback)(void start, void end, size_t num_bytes, void callback_arg),
	void* arg, bool do_null_callback_at_end) NO_THREAD_SAFETY_ANALYSIS {
	// TODO: NO_THREAD_SAFETY_ANALYSIS.
	Thread* self = Thread::Current();
	if (Locks::mutator_lock_->IsExclusiveHeld(self)) {
	// The mutators are already suspended. For example, a call path
	// from SignalCatcher::HandleSigQuit().
	rosalloc_->InspectAll(callback, arg);
	if (do_null_callback_at_end) {
	callback(nullptr, nullptr, 0, arg); // Indicate end of a space.
	}
	} else if (Locks::mutator_lock_->IsSharedHeld(self)) {
	// The mutators are not suspended yet and we have a shared access
	// to the mutator lock. Temporarily release the shared access by
	// transitioning to the suspend state, and suspend the mutators.
	ScopedThreadSuspension sts(self, ThreadState::kSuspended);
	InspectAllRosAllocWithSuspendAll(callback, arg, do_null_callback_at_end);
	} else {
	// The mutators are not suspended yet. Suspend the mutators.
	InspectAllRosAllocWithSuspendAll(callback, arg, do_null_callback_at_end);
	}
	}

	size_t RosAllocSpace::RevokeThreadLocalBuffers(Thread* thread) {
	return rosalloc_->RevokeThreadLocalRuns(thread);
	}

	size_t RosAllocSpace::RevokeAllThreadLocalBuffers() {
	return rosalloc_->RevokeAllThreadLocalRuns();
	}

	void RosAllocSpace::AssertThreadLocalBuffersAreRevoked(Thread* thread) {
	if (kIsDebugBuild) {
	rosalloc_->AssertThreadLocalRunsAreRevoked(thread);
	}
	}

	void RosAllocSpace::AssertAllThreadLocalBuffersAreRevoked() {
	if (kIsDebugBuild) {
	rosalloc_->AssertAllThreadLocalRunsAreRevoked();
	}
	}

	void RosAllocSpace::Clear() {
	size_t footprint_limit = GetFootprintLimit();
	madvise(GetMemMap()->Begin(), GetMemMap()->Size(), MADV_DONTNEED);
	live_bitmap_.Clear();
	mark_bitmap_.Clear();
	SetEnd(begin_ + starting_size_);
	delete rosalloc_;
	rosalloc_ = CreateRosAlloc(mem_map_.Begin(),
	starting_size_,
	initial_size_,
	NonGrowthLimitCapacity(),
	low_memory_mode_,
	Runtime::Current()->IsRunningOnMemoryTool());
	SetFootprintLimit(footprint_limit);
	}

	void RosAllocSpace::DumpStats(std::ostream& os) {
	ScopedSuspendAll ssa(__FUNCTION__);
	rosalloc_->DumpStats(os);
	}

	template<bool kMaybeIsRunningOnMemoryTool>
	size_t RosAllocSpace::AllocationSizeNonvirtual(mirror::Object* obj, size_t* usable_size) {
	// obj is a valid object. Use its class in the header to get the size.
	// Don't use verification since the object may be dead if we are sweeping.
	size_t size = obj->SizeOf<kVerifyNone>();
	bool add_redzones = false;
	if (kMaybeIsRunningOnMemoryTool) {
	add_redzones = kRunningOnMemoryTool && kMemoryToolAddsRedzones;
	if (add_redzones) {
	size += 2 * kDefaultMemoryToolRedZoneBytes;
	}
	} else {
	DCHECK(!kRunningOnMemoryTool);
	}
	size_t size_by_size = rosalloc_->UsableSize(size);
	if (kIsDebugBuild) {
	// On memory tool, the red zone has an impact...
	const uint8_t* obj_ptr = reinterpret_cast<const uint8_t*>(obj);
	size_t size_by_ptr = rosalloc_->UsableSize(
	obj_ptr - (add_redzones ? kDefaultMemoryToolRedZoneBytes : 0));
	if (size_by_size != size_by_ptr) {
	LOG(INFO) << "Found a bad sized obj of size " << size
	<< " at " << std::hex << reinterpret_cast<intptr_t>(obj_ptr) << std::dec
	<< " size_by_size=" << size_by_size << " size_by_ptr=" << size_by_ptr;
	}
	DCHECK_EQ(size_by_size, size_by_ptr);
	}
	if (usable_size != nullptr) {
	*usable_size = size_by_size;
	}
	return size_by_size;
	}

	} // namespace space

	namespace allocator {

	// Callback from rosalloc when it needs to increase the footprint.
	void* ArtRosAllocMoreCore(allocator::RosAlloc* rosalloc, intptr_t increment)
	REQUIRES_SHARED(Locks::mutator_lock_) {
	Heap* heap = Runtime::Current()->GetHeap();
	art::gc::space::RosAllocSpace* rosalloc_space = heap->GetRosAllocSpace(rosalloc);
	DCHECK(rosalloc_space != nullptr);
	DCHECK_EQ(rosalloc_space->GetRosAlloc(), rosalloc);
	return rosalloc_space->MoreCore(increment);
	}

	} // namespace allocator

	} // namespace gc
	} // namespace art