Large object space
The large object space helps prevent fragmentation by putting large objects in mem maps insead of the alloc space.
Instead of mark and live bitmaps it uses mark and live sets.
Change-Id: Iada5db70b88a1572007d8af921fa353681a55dc7
diff --git a/src/class_linker.cc b/src/class_linker.cc
index 26af575..1c87747 100644
--- a/src/class_linker.cc
+++ b/src/class_linker.cc
@@ -979,11 +979,7 @@
{
ReaderMutexLock mu(*Locks::heap_bitmap_lock_);
heap->FlushAllocStack();
- const Spaces& vec = heap->GetSpaces();
- // TODO: C++0x auto
- for (Spaces::const_iterator it = vec.begin(); it != vec.end(); ++it) {
- (*it)->GetLiveBitmap()->Walk(InitFromImageCallback, this);
- }
+ heap->GetLiveBitmap()->Walk(InitFromImageCallback, this);
}
// reinit class_roots_
diff --git a/src/debugger.cc b/src/debugger.cc
index 8b87945..a25f5dc 100644
--- a/src/debugger.cc
+++ b/src/debugger.cc
@@ -3093,6 +3093,8 @@
(*cur)->AsAllocSpace()->Walk(HeapChunkContext::HeapChunkCallback, &context);
}
}
+ // Walk the large objects, these are not in the AllocSpace.
+ heap->GetLargeObjectsSpace()->Walk(HeapChunkContext::HeapChunkCallback, &context);
}
// Finally, send a heap end chunk.
diff --git a/src/heap.cc b/src/heap.cc
index 1b77d2e..74e91f5 100644
--- a/src/heap.cc
+++ b/src/heap.cc
@@ -43,19 +43,6 @@
namespace art {
-static void UpdateFirstAndLastSpace(Space** first_space, Space** last_space, Space* space) {
- if (*first_space == NULL) {
- *first_space = space;
- *last_space = space;
- } else {
- if ((*first_space)->Begin() > space->Begin()) {
- *first_space = space;
- } else if (space->Begin() > (*last_space)->Begin()) {
- *last_space = space;
- }
- }
-}
-
static bool GenerateImage(const std::string& image_file_name) {
const std::string boot_class_path_string(Runtime::Current()->GetBootClassPathString());
std::vector<std::string> boot_class_path;
@@ -146,7 +133,10 @@
concurrent_start_bytes_(std::numeric_limits<size_t>::max()),
concurrent_start_size_(128 * KB),
concurrent_min_free_(256 * KB),
+ bytes_since_last_gc_(0),
+ concurrent_gc_start_rate_(3 * MB / 2),
sticky_gc_count_(0),
+ large_object_threshold_(12 * KB),
num_bytes_allocated_(0),
num_objects_allocated_(0),
verify_missing_card_marks_(false),
@@ -171,11 +161,6 @@
LOG(INFO) << "Heap() entering";
}
- // Compute the bounds of all spaces for allocating live and mark bitmaps
- // there will be at least one space (the alloc space)
- Space* first_space = NULL;
- Space* last_space = NULL;
-
live_bitmap_.reset(new HeapBitmap(this));
mark_bitmap_.reset(new HeapBitmap(this));
@@ -208,11 +193,10 @@
}
AddSpace(image_space);
- UpdateFirstAndLastSpace(&first_space, &last_space, image_space);
// Oat files referenced by image files immediately follow them in memory, ensure alloc space
// isn't going to get in the middle
byte* oat_end_addr = GetImageSpace()->GetImageHeader().GetOatEnd();
- CHECK(oat_end_addr > GetImageSpace()->End());
+ CHECK_GT(oat_end_addr, GetImageSpace()->End());
if (oat_end_addr > requested_begin) {
requested_begin = reinterpret_cast<byte*>(RoundUp(reinterpret_cast<uintptr_t>(oat_end_addr),
kPageSize));
@@ -225,9 +209,9 @@
CHECK(alloc_space_ != NULL) << "Failed to create alloc space";
AddSpace(alloc_space_);
- UpdateFirstAndLastSpace(&first_space, &last_space, alloc_space_);
- byte* heap_begin = first_space->Begin();
- size_t heap_capacity = (last_space->Begin() - first_space->Begin()) + last_space->NonGrowthLimitCapacity();
+ // Spaces are sorted in order of Begin().
+ byte* heap_begin = spaces_.front()->Begin();
+ size_t heap_capacity = (spaces_.back()->Begin() - spaces_.front()->Begin()) + spaces_.back()->NonGrowthLimitCapacity();
// Mark image objects in the live bitmap
for (size_t i = 0; i < spaces_.size(); ++i) {
@@ -237,6 +221,11 @@
}
}
+ // Allocate the large object space.
+ large_object_space_.reset(Space::CreateLargeObjectSpace("large object space"));
+ live_bitmap_->SetLargeObjects(large_object_space_->GetLiveObjects());
+ mark_bitmap_->SetLargeObjects(large_object_space_->GetMarkObjects());
+
// Allocate the card table.
card_table_.reset(CardTable::Create(heap_begin, heap_capacity));
CHECK(card_table_.get() != NULL) << "Failed to create card table";
@@ -331,7 +320,6 @@
// those threads can't resume. We're the only running thread, and we can do whatever we like...
STLDeleteElements(&spaces_);
delete gc_complete_lock_;
-
STLDeleteValues(&cumulative_timings_);
}
@@ -374,33 +362,69 @@
}
}
+bool Heap::CanAllocateBytes(size_t bytes) const {
+ return num_bytes_allocated_ + large_object_space_->GetNumBytesAllocated() + bytes <=
+ alloc_space_->Capacity();
+}
+
Object* Heap::AllocObject(Class* c, size_t byte_count) {
DCHECK(c == NULL || (c->IsClassClass() && byte_count >= sizeof(Class)) ||
(c->IsVariableSize() || c->GetObjectSize() == byte_count) ||
strlen(ClassHelper(c).GetDescriptor()) == 0);
DCHECK_GE(byte_count, sizeof(Object));
- Object* obj = Allocate(alloc_space_, byte_count);
+
+ Object* obj = NULL;
+ size_t size = 0;
+
+ // We need to have a zygote space or else our newly allocated large object can end up in the
+ // Zygote resulting in it being prematurely freed.
+ // We can only do this for primive objects since large objects will not be within the card table
+ // range. This also means that we rely on SetClass not dirtying the object's card.
+ if (byte_count >= large_object_threshold_ && have_zygote_space_ && c->IsPrimitiveArray()) {
+ obj = Allocate(NULL, RoundUp(byte_count, kPageSize));
+ size = 0;
+
+ if (obj != NULL) {
+ // Make sure that our large object didn't get placed anywhere within the space interval or else
+ // it breaks the immune range.
+ DCHECK(reinterpret_cast<byte*>(obj) < spaces_.front()->Begin() ||
+ reinterpret_cast<byte*>(obj) >= spaces_.back()->End());
+ }
+ } else {
+ obj = Allocate(alloc_space_, byte_count);
+ size = alloc_space_->AllocationSize(obj);
+
+ if (obj != NULL) {
+ // Additional verification to ensure that we did not allocate into a zygote space.
+ DCHECK(!have_zygote_space_ || !FindSpaceFromObject(obj)->IsZygoteSpace());
+ }
+ }
+
if (LIKELY(obj != NULL)) {
+ android_atomic_add(byte_count, reinterpret_cast<volatile int32_t*>(
+ reinterpret_cast<size_t>(&bytes_since_last_gc_)));
+
obj->SetClass(c);
// Record allocation after since we want to use the atomic add for the atomic fence to guard
// the SetClass since we do not want the class to appear NULL in another thread.
- RecordAllocation(alloc_space_, obj);
+ RecordAllocation(size, obj);
if (Dbg::IsAllocTrackingEnabled()) {
Dbg::RecordAllocation(c, byte_count);
}
- const bool request_concurrent_gc = num_bytes_allocated_ >= concurrent_start_bytes_;
- if (request_concurrent_gc) {
+ if (bytes_since_last_gc_ >= concurrent_gc_start_rate_ ||
+ num_bytes_allocated_ >= concurrent_start_bytes_) {
+ // We already have a request pending, no reason to start more until we update
+ // concurrent_start_bytes_.
+ concurrent_start_bytes_ = std::numeric_limits<size_t>::max();
+ bytes_since_last_gc_ = 0;
// The SirtRef is necessary since the calls in RequestConcurrentGC are a safepoint.
SirtRef<Object> ref(obj);
RequestConcurrentGC();
}
VerifyObject(obj);
- // Additional verification to ensure that we did not allocate into a zygote space.
- DCHECK(!have_zygote_space_ || !FindSpaceFromObject(obj)->IsZygoteSpace());
-
return obj;
}
int64_t total_bytes_free = GetFreeMemory();
@@ -432,7 +456,8 @@
return true;
}
}
- return false;
+ // TODO: Find a way to check large object space without using a lock.
+ return true;
}
bool Heap::IsLiveObjectLocked(const Object* obj) {
@@ -459,9 +484,9 @@
<< *space->GetLiveBitmap() << "\n"
<< *space->GetMarkBitmap();
}
+ // TODO: Dump large object space?
}
-// We want to avoid bit rotting.
void Heap::VerifyObjectBody(const Object* obj) {
if (!IsAligned<kObjectAlignment>(obj)) {
LOG(FATAL) << "Object isn't aligned: " << obj;
@@ -472,8 +497,11 @@
if (!std::binary_search(live_stack_->Begin(), live_stack_->End(), obj) &&
std::find(allocation_stack_->Begin(), allocation_stack_->End(), obj) ==
allocation_stack_->End()) {
- DumpSpaces();
- LOG(FATAL) << "Object is dead: " << obj;
+ ReaderMutexLock mu(*Locks::heap_bitmap_lock_);
+ if (large_object_space_->GetLiveObjects()->Test(obj)) {
+ DumpSpaces();
+ LOG(FATAL) << "Object is dead: " << obj;
+ }
}
}
@@ -510,17 +538,15 @@
GetLiveBitmap()->Walk(Heap::VerificationCallback, this);
}
-void Heap::RecordAllocation(AllocSpace* space, const Object* obj) {
+void Heap::RecordAllocation(size_t size, const Object* obj) {
DCHECK(obj != NULL);
-
- size_t size = space->AllocationSize(obj);
DCHECK_GT(size, 0u);
COMPILE_ASSERT(sizeof(size_t) == sizeof(int32_t),
int32_t_must_be_same_size_as_size_t_for_used_atomic_operations);
- android_atomic_add(
- size, reinterpret_cast<volatile int32_t*>(reinterpret_cast<size_t>(&num_bytes_allocated_)));
- android_atomic_add(
- 1, reinterpret_cast<volatile int32_t*>(reinterpret_cast<size_t>(&num_objects_allocated_)));
+ android_atomic_add(size, reinterpret_cast<volatile int32_t*>(
+ reinterpret_cast<size_t>(&num_bytes_allocated_)));
+ android_atomic_add(1, reinterpret_cast<volatile int32_t*>(
+ reinterpret_cast<size_t>(&num_objects_allocated_)));
if (Runtime::Current()->HasStatsEnabled()) {
RuntimeStats* global_stats = Runtime::Current()->GetStats();
@@ -557,11 +583,21 @@
}
}
+Object* Heap::TryToAllocate(AllocSpace* space, size_t alloc_size, bool grow) {
+ if (UNLIKELY(space == NULL) && CanAllocateBytes(alloc_size)) {
+ return large_object_space_->Alloc(alloc_size);
+ } else if (grow) {
+ return space->AllocWithGrowth(alloc_size);
+ } else {
+ return space->AllocWithoutGrowth(alloc_size);
+ }
+}
+
Object* Heap::Allocate(AllocSpace* space, size_t alloc_size) {
- Thread* self = Thread::Current();
// Since allocation can cause a GC which will need to SuspendAll, make sure all allocations are
// done in the runnable state where suspension is expected.
#ifndef NDEBUG
+ Thread* self = Thread::Current();
{
MutexLock mu(*Locks::thread_suspend_count_lock_);
CHECK_EQ(self->GetState(), kRunnable);
@@ -569,7 +605,7 @@
self->AssertThreadSuspensionIsAllowable();
#endif
- Object* ptr = space->AllocWithoutGrowth(alloc_size);
+ Object* ptr = TryToAllocate(space, alloc_size, false);
if (ptr != NULL) {
return ptr;
}
@@ -579,13 +615,16 @@
GcType last_gc = WaitForConcurrentGcToComplete();
if (last_gc != kGcTypeNone) {
// A GC was in progress and we blocked, retry allocation now that memory has been freed.
- Object* ptr = space->AllocWithoutGrowth(alloc_size);
+ ptr = TryToAllocate(space, alloc_size, false);
if (ptr != NULL) {
return ptr;
}
}
// Loop through our different Gc types and try to Gc until we get enough free memory.
+#ifdef NDEBUG
+ Thread* self = Thread::Current();
+#endif
for (size_t i = static_cast<size_t>(last_gc) + 1; i < static_cast<size_t>(kGcTypeMax); ++i) {
bool run_gc = false;
GcType gc_type = static_cast<GcType>(i);
@@ -620,7 +659,7 @@
self->TransitionFromSuspendedToRunnable();
// Did we free sufficient memory for the allocation to succeed?
- ptr = space->AllocWithoutGrowth(alloc_size);
+ ptr = TryToAllocate(space, alloc_size, false);
if (ptr != NULL) {
return ptr;
}
@@ -629,14 +668,16 @@
// Allocations have failed after GCs; this is an exceptional state.
// Try harder, growing the heap if necessary.
- ptr = space->AllocWithGrowth(alloc_size);
+ ptr = TryToAllocate(space, alloc_size, true);
if (ptr != NULL) {
- size_t new_footprint = space->GetFootprintLimit();
- // OLD-TODO: may want to grow a little bit more so that the amount of
- // free space is equal to the old free space + the
- // utilization slop for the new allocation.
- VLOG(gc) << "Grow heap (frag case) to " << PrettySize(new_footprint)
- << " for a " << PrettySize(alloc_size) << " allocation";
+ if (space != NULL) {
+ size_t new_footprint = space->GetFootprintLimit();
+ // OLD-TODO: may want to grow a little bit more so that the amount of
+ // free space is equal to the old free space + the
+ // utilization slop for the new allocation.
+ VLOG(gc) << "Grow alloc space (frag case) to " << PrettySize(new_footprint)
+ << " for a " << PrettySize(alloc_size) << " allocation";
+ }
return ptr;
}
@@ -656,7 +697,7 @@
self->TransitionFromRunnableToSuspended(kWaitingPerformingGc);
CollectGarbageInternal(kGcTypeFull, true);
self->TransitionFromSuspendedToRunnable();
- return space->AllocWithGrowth(alloc_size);
+ return TryToAllocate(space, alloc_size, true);
}
int64_t Heap::GetMaxMemory() {
@@ -777,12 +818,13 @@
}
void Heap::FlushAllocStack() {
- MarkAllocStack(alloc_space_->GetLiveBitmap(), allocation_stack_.get());
+ MarkAllocStack(alloc_space_->GetLiveBitmap(), large_object_space_->GetLiveObjects(),
+ allocation_stack_.get());
allocation_stack_->Reset();
}
size_t Heap::GetUsedMemorySize() const {
- size_t total = num_bytes_allocated_;
+ size_t total = num_bytes_allocated_ + large_object_space_->GetNumBytesAllocated();
for (Spaces::const_iterator it = spaces_.begin(); it != spaces_.end(); ++it) {
if ((*it)->IsZygoteSpace()) {
total += (*it)->AsAllocSpace()->Size();
@@ -791,23 +833,29 @@
return total;
}
-void Heap::MarkAllocStack(SpaceBitmap* bitmap, MarkStack* stack) {
- // Empty the allocation stack.
+void Heap::MarkAllocStack(SpaceBitmap* bitmap, SpaceSetMap* large_objects, MarkStack* stack) {
const size_t count = stack->Size();
for (size_t i = 0; i < count; ++i) {
const Object* obj = stack->Get(i);
DCHECK(obj != NULL);
- bitmap->Set(obj);
+ if (LIKELY(bitmap->HasAddress(obj))) {
+ bitmap->Set(obj);
+ } else {
+ large_objects->Set(obj);
+ }
}
}
-void Heap::UnMarkAllocStack(SpaceBitmap* bitmap, MarkStack* stack) {
- // Clear all of the things in the AllocStack.
+void Heap::UnMarkAllocStack(SpaceBitmap* bitmap, SpaceSetMap* large_objects, MarkStack* stack) {
size_t count = stack->Size();
for (size_t i = 0; i < count; ++i) {
const Object* obj = stack->Get(i);
DCHECK(obj != NULL);
- bitmap->Clear(obj);
+ if (LIKELY(bitmap->HasAddress(obj))) {
+ bitmap->Clear(obj);
+ } else {
+ large_objects->Clear(obj);
+ }
}
}
@@ -852,6 +900,7 @@
} else {
CollectGarbageMarkSweepPlan(gc_type, clear_soft_references);
}
+ bytes_since_last_gc_ = 0;
{
MutexLock mu(*gc_complete_lock_);
@@ -937,20 +986,24 @@
}
timings.AddSplit("CopyMarkBits");
- // We can assume that everything < alloc_space_ start is marked at this point.
- mark_sweep.SetCondemned(reinterpret_cast<Object*>(alloc_space_->Begin()));
+ // We can assume that everything from the start of the first space to the alloc space is marked.
+ mark_sweep.SetImmuneRange(reinterpret_cast<Object*>(spaces_[0]->Begin()),
+ reinterpret_cast<Object*>(alloc_space_->Begin()));
} else if (gc_type == kGcTypeSticky) {
- for (Spaces::iterator it = spaces_.begin(); it != spaces_.end(); ++it) {
+ for (Spaces::iterator it = spaces_.begin();it != spaces_.end(); ++it) {
if ((*it)->GetGcRetentionPolicy() != GCRP_NEVER_COLLECT) {
mark_sweep.CopyMarkBits(*it);
}
}
+ large_object_space_->CopyLiveToMarked();
timings.AddSplit("CopyMarkBits");
- mark_sweep.SetCondemned(reinterpret_cast<Object*>(alloc_space_->Begin()));
+ mark_sweep.SetImmuneRange(reinterpret_cast<Object*>(spaces_[0]->Begin()),
+ reinterpret_cast<Object*>(alloc_space_->Begin()));
}
- MarkAllocStack(alloc_space_->GetLiveBitmap(), live_stack_.get());
+ MarkAllocStack(alloc_space_->GetLiveBitmap(), large_object_space_->GetLiveObjects(),
+ live_stack_.get());
if (gc_type != kGcTypeSticky) {
live_stack_->Reset();
@@ -988,8 +1041,6 @@
mark_sweep.SweepSystemWeaks(false);
timings.AddSplit("SweepSystemWeaks");
- // Need to swap for VERIFY_OBJECT_ENABLED since we put things in the live bitmap after they
- // have been allocated.
const bool swap = true;
if (swap) {
SwapBitmaps();
@@ -1002,11 +1053,13 @@
#endif
if (gc_type != kGcTypeSticky) {
+ mark_sweep.SweepLargeObjects(swap);
+ timings.AddSplit("SweepLargeObjects");
mark_sweep.Sweep(gc_type == kGcTypePartial, swap);
} else {
mark_sweep.SweepArray(timings, live_stack_.get(), swap);
+ timings.AddSplit("SweepArray");
}
- timings.AddSplit("Sweep");
if (verify_system_weaks_) {
mark_sweep.VerifySystemWeaks();
@@ -1182,6 +1235,8 @@
if (bitmap->Test(obj)) {
return true;
}
+ } else if (heap_->GetLargeObjectsSpace()->GetLiveObjects()->Test(obj)) {
+ return true;
} else {
heap_->DumpSpaces();
LOG(ERROR) << "Object " << obj << " not found in any spaces";
@@ -1360,6 +1415,10 @@
space->AsAllocSpace()->SwapBitmaps();
}
}
+
+ large_object_space_->SwapBitmaps();
+ live_bitmap_->SetLargeObjects(large_object_space_->GetLiveObjects());
+ mark_bitmap_->SetLargeObjects(large_object_space_->GetMarkObjects());
}
void Heap::SwapStacks() {
@@ -1446,6 +1505,10 @@
{
WriterMutexLock mu(*Locks::heap_bitmap_lock_);
+ for (Object** it = live_stack_->Begin(); it != live_stack_->End(); ++it) {
+ CHECK(!GetLiveBitmap()->Test(*it));
+ }
+
if (gc_type == kGcTypePartial) {
// Copy the mark bits over from the live bits, do this as early as possible or else we can
// accidentally un-mark roots.
@@ -1456,15 +1519,18 @@
}
}
timings.AddSplit("CopyMarkBits");
- mark_sweep.SetCondemned(reinterpret_cast<Object*>(alloc_space_->Begin()));
+ mark_sweep.SetImmuneRange(reinterpret_cast<Object*>(spaces_.front()->Begin()),
+ reinterpret_cast<Object*>(alloc_space_->Begin()));
} else if (gc_type == kGcTypeSticky) {
for (Spaces::iterator it = spaces_.begin(); it != spaces_.end(); ++it) {
if ((*it)->GetGcRetentionPolicy() != GCRP_NEVER_COLLECT) {
mark_sweep.CopyMarkBits(*it);
}
}
+ large_object_space_->CopyLiveToMarked();
timings.AddSplit("CopyMarkBits");
- mark_sweep.SetCondemned(reinterpret_cast<Object*>(alloc_space_->Begin()));
+ mark_sweep.SetImmuneRange(reinterpret_cast<Object*>(spaces_.front()->Begin()),
+ reinterpret_cast<Object*>(alloc_space_->Begin()));
}
// Marking roots is not necessary for sticky mark bits since we only actually require the
@@ -1497,7 +1563,8 @@
// Mark everything as live so that sweeping system weak works correctly for sticky mark bit
// GCs.
- MarkAllocStack(alloc_space_->GetLiveBitmap(), live_stack_.get());
+ MarkAllocStack(alloc_space_->GetLiveBitmap(), large_object_space_->GetLiveObjects(),
+ live_stack_.get());
timings.AddSplit("MarkStackAsLive");
if (gc_type != kGcTypeSticky) {
@@ -1524,7 +1591,8 @@
if (verify_missing_card_marks_) {
// Since verify missing card marks uses a sweep array to empty the allocation stack, we
// need to make sure that we don't free weaks which wont get swept by SweepSystemWeaks.
- MarkAllocStack(alloc_space_->GetLiveBitmap(), allocation_stack_.get());
+ MarkAllocStack(alloc_space_->GetLiveBitmap(), large_object_space_->GetLiveObjects(),
+ allocation_stack_.get());
}
// Scan dirty objects, this is only required if we are not doing concurrent GC.
@@ -1560,7 +1628,8 @@
WriterMutexLock mu(*Locks::heap_bitmap_lock_);
// We only sweep over the live stack, and the live stack should not intersect with the
// allocation stack, so it should be safe to UnMark anything in the allocation stack as live.
- UnMarkAllocStack(alloc_space_->GetLiveBitmap(), allocation_stack_.get());
+ UnMarkAllocStack(alloc_space_->GetLiveBitmap(), large_object_space_->GetLiveObjects(),
+ allocation_stack_.get());
timings.AddSplit("UnMarkAllocStack");
}
@@ -1587,9 +1656,12 @@
// TODO: this lock shouldn't be necessary (it's why we did the bitmap flip above).
WriterMutexLock mu(*Locks::heap_bitmap_lock_);
if (gc_type != kGcTypeSticky) {
+ mark_sweep.SweepLargeObjects(swap);
+ timings.AddSplit("SweepLargeObjects");
mark_sweep.Sweep(gc_type == kGcTypePartial, swap);
} else {
mark_sweep.SweepArray(timings, live_stack_.get(), swap);
+ timings.AddSplit("SweepArray");
}
live_stack_->Reset();
timings.AddSplit("Sweep");
@@ -1625,7 +1697,6 @@
<< "% free, " << PrettySize(current_heap_size) << "/"
<< PrettySize(total_memory) << ", " << "paused " << PrettyDuration(pause_roots)
<< "+" << PrettyDuration(pause_dirty) << " total " << PrettyDuration(duration);
-
if (VLOG_IS_ON(heap)) {
timings.Dump();
}
@@ -1684,8 +1755,8 @@
void Heap::SetIdealFootprint(size_t max_allowed_footprint) {
AllocSpace* alloc_space = alloc_space_;
- // TODO: Behavior for multiple alloc spaces?
size_t alloc_space_capacity = alloc_space->Capacity();
+ alloc_space_capacity -= large_object_space_->GetNumBytesAllocated();
if (max_allowed_footprint > alloc_space_capacity) {
VLOG(gc) << "Clamp target GC heap from " << PrettySize(max_allowed_footprint) << " to "
<< PrettySize(alloc_space_capacity);
diff --git a/src/heap.h b/src/heap.h
index d3127de..b905952 100644
--- a/src/heap.h
+++ b/src/heap.h
@@ -41,8 +41,10 @@
class Class;
class HeapBitmap;
class ImageSpace;
+class LargeObjectSpace;
class MarkStack;
class ModUnionTable;
+
class Object;
class Space;
class SpaceTest;
@@ -262,11 +264,11 @@
EXCLUSIVE_LOCKS_REQUIRED(Locks::heap_bitmap_lock_);
// Mark all the objects in the allocation stack in the specified bitmap.
- void MarkAllocStack(SpaceBitmap* bitmap, MarkStack* stack)
+ void MarkAllocStack(SpaceBitmap* bitmap, SpaceSetMap* large_objects, MarkStack* stack)
EXCLUSIVE_LOCKS_REQUIRED(Locks::heap_bitmap_lock_);
// Unmark all the objects in the allocation stack in the specified bitmap.
- void UnMarkAllocStack(SpaceBitmap* bitmap, MarkStack* stack)
+ void UnMarkAllocStack(SpaceBitmap* bitmap, SpaceSetMap* large_objects, MarkStack* stack)
EXCLUSIVE_LOCKS_REQUIRED(Locks::heap_bitmap_lock_);
// Update and mark mod union table based on gc type.
@@ -277,22 +279,34 @@
// Assumes there is only one image space.
ImageSpace* GetImageSpace();
AllocSpace* GetAllocSpace();
+ LargeObjectSpace* GetLargeObjectsSpace() {
+ return large_object_space_.get();
+ }
void DumpSpaces();
private:
- // Allocates uninitialized storage.
+ // Allocates uninitialized storage. Passing in a null space tries to place the object in the
+ // large object space.
Object* Allocate(AllocSpace* space, size_t num_bytes)
LOCKS_EXCLUDED(Locks::thread_suspend_count_lock_)
SHARED_LOCKS_REQUIRED(Locks::mutator_lock_);
+ // Try to allocate a number of bytes, this function never does any GCs.
+ Object* TryToAllocate(AllocSpace* space, size_t alloc_size, bool grow)
+ LOCKS_EXCLUDED(Locks::thread_suspend_count_lock_)
+ SHARED_LOCKS_REQUIRED(Locks::mutator_lock_);
+
// Pushes a list of cleared references out to the managed heap.
void EnqueueClearedReferences(Object** cleared_references);
void RequestHeapTrim();
void RequestConcurrentGC();
- void RecordAllocation(AllocSpace* space, const Object* object)
- LOCKS_EXCLUDED(Locks::heap_bitmap_lock_);
+ // Swap bitmaps (if we are a full Gc then we swap the zygote bitmap too).
+ void SwapBitmaps() EXCLUSIVE_LOCKS_REQUIRED(GlobalSynchronization::heap_bitmap_lock_);
+
+ void RecordAllocation(size_t size, const Object* object)
+ LOCKS_EXCLUDED(GlobalSynchronization::heap_bitmap_lock_);
// Sometimes CollectGarbageInternal decides to run a different Gc than you requested. Returns
// which type of Gc was actually ran.
@@ -317,6 +331,9 @@
void AddSpace(Space* space) LOCKS_EXCLUDED(Locks::heap_bitmap_lock_);
+ // Returns true if we "should" be able to allocate a number of bytes.
+ bool CanAllocateBytes(size_t bytes) const;
+
// No thread saftey analysis since we call this everywhere and it is impossible to find a proper
// lock ordering for it.
void VerifyObjectBody(const Object *obj)
@@ -325,8 +342,7 @@
static void VerificationCallback(Object* obj, void* arg)
SHARED_LOCKS_REQUIRED(GlobalSychronization::heap_bitmap_lock_);
- // Swpa bitmaps (if we are a full Gc then we swap the zygote bitmap too).
- void SwapBitmaps();
+ // Swap the allocation stack with the live stack.
void SwapStacks();
Spaces spaces_;
@@ -372,8 +388,18 @@
volatile size_t concurrent_start_bytes_;
size_t concurrent_start_size_;
size_t concurrent_min_free_;
+ // Number of bytes allocated since the last Gc, we use this to help determine when to schedule concurrent GCs.
+ size_t bytes_since_last_gc_;
+ // Start a concurrent GC if we have allocated concurrent_gc_start_rate_ bytes and not done a GCs.
+ size_t concurrent_gc_start_rate_;
size_t sticky_gc_count_;
+ // Primitive objects larger than this size are put in the large object space.
+ size_t large_object_threshold_;
+
+ // Large object space.
+ UniquePtr<LargeObjectSpace> large_object_space_;
+
// Number of bytes allocated. Adjusted after each allocation and free.
volatile size_t num_bytes_allocated_;
@@ -412,7 +438,7 @@
// Used to ensure that we don't ever recursively request GC.
volatile bool requesting_gc_;
- // Mark stack that we reuse to avoid re-allocating the mark stack
+ // Mark stack that we reuse to avoid re-allocating the mark stack.
UniquePtr<MarkStack> mark_stack_;
// Allocation stack, new allocations go here so that we can do sticky mark bits. This enables us
diff --git a/src/heap_bitmap.cc b/src/heap_bitmap.cc
index 9cab8c2..cef6884 100644
--- a/src/heap_bitmap.cc
+++ b/src/heap_bitmap.cc
@@ -28,4 +28,22 @@
bitmaps_.push_back(bitmap);
}
+void HeapBitmap::SetLargeObjects(SpaceSetMap* large_objects) {
+ DCHECK(large_objects != NULL);
+ large_objects_ = large_objects;
+}
+
+HeapBitmap::HeapBitmap(Heap* heap) : heap_(heap), large_objects_(NULL) {
+
+}
+
+void HeapBitmap::Walk(SpaceBitmap::Callback* callback, void* arg) {
+ // TODO: C++0x auto
+ for (Bitmaps::iterator it = bitmaps_.begin(); it != bitmaps_.end(); ++it) {
+ (*it)->Walk(callback, arg);
+ }
+
+ large_objects_->Walk(callback, arg);
+}
+
} // namespace art
diff --git a/src/heap_bitmap.h b/src/heap_bitmap.h
index 50ecc7a..23dcd47 100644
--- a/src/heap_bitmap.h
+++ b/src/heap_bitmap.h
@@ -28,28 +28,31 @@
bool Test(const Object* obj)
SHARED_LOCKS_REQUIRED(Locks::heap_bitmap_lock_) {
SpaceBitmap* bitmap = GetSpaceBitmap(obj);
- DCHECK(bitmap != NULL);
- return bitmap->Test(obj);
+ if (LIKELY(bitmap != NULL)) {
+ return bitmap->Test(obj);
+ } else {
+ return large_objects_->Test(obj);
+ }
}
void Clear(const Object* obj)
EXCLUSIVE_LOCKS_REQUIRED(Locks::heap_bitmap_lock_) {
SpaceBitmap* bitmap = GetSpaceBitmap(obj);
- DCHECK(bitmap != NULL)
- << "tried to clear object "
- << reinterpret_cast<const void*>(obj)
- << " which did not belong to any bitmaps";
- return bitmap->Clear(obj);
+ if (LIKELY(bitmap != NULL)) {
+ return bitmap->Clear(obj);
+ } else {
+ return large_objects_->Clear(obj);
+ }
}
void Set(const Object* obj)
EXCLUSIVE_LOCKS_REQUIRED(Locks::heap_bitmap_lock_) {
SpaceBitmap* bitmap = GetSpaceBitmap(obj);
- DCHECK(bitmap != NULL)
- << "tried to mark object "
- << reinterpret_cast<const void*>(obj)
- << " which did not belong to any bitmaps";
- bitmap->Set(obj);
+ if (LIKELY(bitmap != NULL)) {
+ bitmap->Set(obj);
+ } else {
+ large_objects_->Set(obj);
+ }
}
SpaceBitmap* GetSpaceBitmap(const Object* obj) {
@@ -63,12 +66,7 @@
}
void Walk(SpaceBitmap::Callback* callback, void* arg)
- SHARED_LOCKS_REQUIRED(Locks::heap_bitmap_lock_) {
- // TODO: C++0x auto
- for (Bitmaps::iterator it = bitmaps_.begin(); it!= bitmaps_.end(); ++it) {
- (*it)->Walk(callback, arg);
- }
- }
+ SHARED_LOCKS_REQUIRED(Locks::heap_bitmap_lock_);
template <typename Visitor>
void Visit(const Visitor& visitor)
@@ -79,16 +77,21 @@
bitmap->VisitMarkedRange(bitmap->HeapBegin(), bitmap->HeapLimit(), visitor,
IdentityFunctor());
}
+ large_objects_->Visit(visitor);
}
// Find and replace a bitmap pointer, this is used by for the bitmap swapping in the GC.
void ReplaceBitmap(SpaceBitmap* old_bitmap, SpaceBitmap* new_bitmap)
EXCLUSIVE_LOCKS_REQUIRED(Locks::heap_bitmap_lock_);
- HeapBitmap(Heap* heap) : heap_(heap) {
+ HeapBitmap(Heap* heap);
+ inline SpaceSetMap* GetLargeObjects() const {
+ return large_objects_;
}
+ void SetLargeObjects(SpaceSetMap* large_objects);
+
private:
const Heap* const heap_;
@@ -98,6 +101,9 @@
typedef std::vector<SpaceBitmap*> Bitmaps;
Bitmaps bitmaps_;
+ // Large object sets.
+ SpaceSetMap* large_objects_;
+
friend class Heap;
};
} // namespace art
diff --git a/src/image_writer.cc b/src/image_writer.cc
index 2ec47ec..e883599 100644
--- a/src/image_writer.cc
+++ b/src/image_writer.cc
@@ -383,8 +383,7 @@
SirtRef<ObjectArray<Object> > image_roots(CreateImageRoots());
Heap* heap = Runtime::Current()->GetHeap();
- typedef std::vector<Space*> SpaceVec;
- const SpaceVec& spaces = heap->GetSpaces();
+ const Spaces& spaces = heap->GetSpaces();
DCHECK(!spaces.empty());
DCHECK_EQ(0U, image_end_);
@@ -393,7 +392,6 @@
image_end_ += RoundUp(sizeof(ImageHeader), 8); // 64-bit-alignment
{
- Heap* heap = Runtime::Current()->GetHeap();
ReaderMutexLock mu(*Locks::heap_bitmap_lock_);
heap->FlushAllocStack();
}
@@ -402,7 +400,8 @@
// TODO: Image spaces only?
// TODO: Add InOrderWalk to heap bitmap.
const char* old = Thread::Current()->StartAssertNoThreadSuspension("ImageWriter");
- for (SpaceVec::const_iterator it = spaces.begin(); it != spaces.end(); ++it) {
+ DCHECK(heap->GetLargeObjectsSpace()->GetLiveObjects()->IsEmpty());
+ for (Spaces::const_iterator it = spaces.begin(); it != spaces.end(); ++it) {
(*it)->GetLiveBitmap()->InOrderWalk(CalculateNewObjectOffsetsCallback, this);
DCHECK_LT(image_end_, image_->Size());
}
diff --git a/src/mark_sweep.cc b/src/mark_sweep.cc
index cf49605..a2aa1c9 100644
--- a/src/mark_sweep.cc
+++ b/src/mark_sweep.cc
@@ -58,7 +58,8 @@
mark_stack_(mark_stack),
heap_(NULL),
finger_(NULL),
- condemned_(NULL),
+ immune_begin_(NULL),
+ immune_end_(NULL),
soft_reference_list_(NULL),
weak_reference_list_(NULL),
finalizer_reference_list_(NULL),
@@ -92,26 +93,31 @@
inline void MarkSweep::MarkObject0(const Object* obj, bool check_finger) {
DCHECK(obj != NULL);
- if (obj < condemned_) {
- // May later use condemned for the NULL check.
- DCHECK(obj == NULL || IsMarked(obj));
+ if (obj >= immune_begin_ && obj < immune_end_) {
+ DCHECK(IsMarked(obj));
return;
}
// Try to take advantage of locality of references within a space, failing this find the space
// the hard way.
if (UNLIKELY(!current_mark_bitmap_->HasAddress(obj))) {
- current_mark_bitmap_ = heap_->GetMarkBitmap()->GetSpaceBitmap(obj);
- if (UNLIKELY(current_mark_bitmap_ == NULL)) {
- LOG(ERROR) << "Failed to find space bitmap for object: " << obj;
- GetHeap()->DumpSpaces();
- LOG(FATAL) << "space_bitmap == NULL";
+ SpaceBitmap* new_bitmap = heap_->GetMarkBitmap()->GetSpaceBitmap(obj);
+ if (new_bitmap != NULL) {
+ current_mark_bitmap_ = new_bitmap;
+ } else {
+ LargeObjectSpace* large_object_space = GetHeap()->GetLargeObjectsSpace();
+ SpaceSetMap* large_objects = large_object_space->GetMarkObjects();
+ if (!large_objects->Test(obj)) {
+ large_objects->Set(obj);
+ // Don't need to check finger since large objects never have any object references.
+ }
+ // TODO: Improve clarity of control flow in this function?
+ return;
}
}
- bool is_marked = current_mark_bitmap_->Test(obj);
// This object was not previously marked.
- if (!is_marked) {
+ if (!current_mark_bitmap_->Test(obj)) {
current_mark_bitmap_->Set(obj);
if (check_finger && obj < finger_) {
// The object must be pushed on to the mark stack.
@@ -208,6 +214,7 @@
const Spaces& spaces = heap_->GetSpaces();
CardTable* card_table = heap_->GetCardTable();
ScanImageRootVisitor image_root_visitor(this);
+ // TODO: C++ 0x auto
for (Spaces::const_iterator it = spaces.begin(); it != spaces.end(); ++it) {
Space* space = *it;
if (space->IsImageSpace()) {
@@ -222,6 +229,7 @@
CardTable* card_table = heap_->GetCardTable();
ScanImageRootVisitor image_root_visitor(this);
SetFingerVisitor finger_visitor(this);
+ // TODO: C++ 0x auto
for (Spaces::const_iterator it = spaces.begin(); it != spaces.end(); ++it) {
Space* space = *it;
byte* begin = space->Begin();
@@ -310,7 +318,7 @@
current_mark_bitmap_ = space->GetMarkBitmap();
if (current_mark_bitmap_ == NULL) {
GetHeap()->DumpSpaces();
- DCHECK(false) << "invalid bitmap";
+ LOG(FATAL) << "invalid bitmap";
}
// This function does not handle heap end increasing, so we must use the space end.
uintptr_t begin = reinterpret_cast<uintptr_t>(space->Begin());
@@ -341,7 +349,7 @@
#ifndef NDEBUG
if (current_mark_bitmap_ == NULL) {
GetHeap()->DumpSpaces();
- DCHECK(false) << "Object " << reinterpret_cast<const void*>(start_obj);
+ LOG(FATAL) << "Object " << reinterpret_cast<const void*>(start_obj);
}
#endif
}
@@ -506,10 +514,15 @@
// going to free.
SpaceBitmap* live_bitmap = space->GetLiveBitmap();
SpaceBitmap* mark_bitmap = space->GetMarkBitmap();
+ LargeObjectSpace* large_object_space = GetHeap()->GetLargeObjectsSpace();
+ SpaceSetMap* large_live_objects = large_object_space->GetLiveObjects();
+ SpaceSetMap* large_mark_objects = large_object_space->GetMarkObjects();
if (swap_bitmaps) {
std::swap(live_bitmap, mark_bitmap);
+ std::swap(large_live_objects, large_mark_objects);
}
+ size_t freed_large_objects = 0;
size_t count = allocations->Size();
Object** objects = allocations->Begin();
Object** out = objects;
@@ -517,32 +530,31 @@
// Empty the allocation stack.
for (size_t i = 0;i < count;++i) {
Object* obj = objects[i];
- // There should only be objects in the AllocSpace in the allocation stack.
- DCHECK(space->Contains(obj));
- DCHECK(mark_bitmap->HasAddress(obj));
- // Mark as live in the live bitmap if and only if we are marked in the mark bitmap.
- if (mark_bitmap->Test(obj)) {
- live_bitmap->Set(obj);
- } else {
- // Due to bitmap swapping, a young object can end up being marked as live.
- live_bitmap->Clear(obj);
- // Don't bother un-marking since we clear the mark bitmap anyways.
- *(out++) = obj;
- size_t size = space->AllocationSize(obj);
- freed_bytes += size;
+ // There should only be objects in the AllocSpace/LargeObjectSpace in the allocation stack.
+ if (LIKELY(mark_bitmap->HasAddress(obj))) {
+ if (!mark_bitmap->Test(obj)) {
+ // Don't bother un-marking since we clear the mark bitmap anyways.
+ *(out++) = obj;
+ size_t size = space->AllocationSize(obj);
+ freed_bytes += size;
+ }
+ } else if (!large_mark_objects->Test(obj)) {
+ ++freed_large_objects;
+ size_t size = large_object_space->AllocationSize(obj);
+ freed_bytes_ += size;
+ large_object_space->Free(obj);
}
}
logger.AddSplit("Process allocation stack");
size_t freed_objects = out - objects;
VLOG(heap) << "Freed " << freed_objects << "/" << count
- << " objects with size " << PrettySize(freed_bytes);
+ << " objects with size " << PrettySize(freed_bytes);
space->FreeList(freed_objects, objects);
- heap_->RecordFree(freed_objects, freed_bytes);
+ heap_->RecordFree(freed_objects + freed_large_objects, freed_bytes);
freed_objects_ += freed_objects;
freed_bytes_ += freed_bytes;
logger.AddSplit("FreeList");
-
allocations->Reset();
logger.AddSplit("Reset stack");
}
@@ -585,6 +597,31 @@
}
}
+void MarkSweep::SweepLargeObjects(bool swap_bitmaps) {
+ // Sweep large objects
+ LargeObjectSpace* large_object_space = GetHeap()->GetLargeObjectsSpace();
+ MutexLock mu(large_object_space->GetLock());
+ SpaceSetMap* large_live_objects = large_object_space->GetLiveObjects();
+ SpaceSetMap* large_mark_objects = large_object_space->GetMarkObjects();
+ if (swap_bitmaps) {
+ std::swap(large_live_objects, large_mark_objects);
+ }
+ SpaceSetMap::Objects& live_objects = large_live_objects->GetObjects();
+ // O(n*log(n)) but hopefully there are not too many large objects.
+ size_t freed_objects = 0;
+ // TODO: C++0x
+ for (SpaceSetMap::Objects::iterator it = live_objects.begin(); it != live_objects.end(); ++it) {
+ if (!large_mark_objects->Test(*it)) {
+ freed_bytes_ += large_object_space->AllocationSize(*it);
+ large_object_space->Free(const_cast<Object*>(*it));
+ ++freed_objects;
+ }
+ }
+ freed_objects_ += freed_objects;
+ // Large objects don't count towards bytes_allocated.
+ GetHeap()->RecordFree(freed_objects, 0);
+}
+
// Scans instance fields.
inline void MarkSweep::ScanInstanceFields(const Object* obj) {
DCHECK(obj != NULL);
@@ -943,6 +980,11 @@
}
}
mark_stack_->Reset();
+
+ // Reset the marked large objects.
+ LargeObjectSpace* large_objects = GetHeap()->GetLargeObjectsSpace();
+ MutexLock mu(large_objects->GetLock());
+ large_objects->GetMarkObjects()->Clear();
}
} // namespace art
diff --git a/src/mark_sweep.h b/src/mark_sweep.h
index d6218dc..0de75c5 100644
--- a/src/mark_sweep.h
+++ b/src/mark_sweep.h
@@ -101,6 +101,10 @@
void Sweep(bool partial, bool swap_bitmaps)
EXCLUSIVE_LOCKS_REQUIRED(Locks::heap_bitmap_lock_);
+ // Sweeps unmarked objects to complete the garbage collection.
+ void SweepLargeObjects(bool swap_bitmaps)
+ EXCLUSIVE_LOCKS_REQUIRED(GlobalSynchronization::heap_bitmap_lock_);
+
// Sweep only pointers within an array. WARNING: Trashes objects.
void SweepArray(TimingLogger& logger, MarkStack* allocation_stack_, bool swap_bitmaps)
EXCLUSIVE_LOCKS_REQUIRED(Locks::heap_bitmap_lock_);
@@ -135,8 +139,10 @@
return freed_objects_;
}
- void SetCondemned(Object* condemned) {
- condemned_ = condemned;
+ // Everything inside the immune range is marked.
+ void SetImmuneRange(Object* begin, Object* end) {
+ immune_begin_ = begin;
+ immune_end_ = end;
}
void SweepSystemWeaks(bool swap_bitmaps)
@@ -376,7 +382,9 @@
Object* finger_;
- Object* condemned_;
+ // Immune range, every object inside the immune range is assumed to be marked.
+ Object* immune_begin_;
+ Object* immune_end_;
Object* soft_reference_list_;
diff --git a/src/mod_union_table.cc b/src/mod_union_table.cc
index d62128d..83f67db 100644
--- a/src/mod_union_table.cc
+++ b/src/mod_union_table.cc
@@ -360,13 +360,12 @@
}
void ModUnionTableReferenceCache::MarkReferences() {
- Heap* heap = GetMarkSweep()->GetHeap();
- HeapBitmap* mark_bitmap = heap->GetMarkBitmap();
+ MarkSweep* mark_sweep = GetMarkSweep();
// TODO: C++0x auto
size_t count = 0;
for (ReferenceMap::const_iterator it = references_.begin(); it != references_.end(); ++it) {
for (ReferenceArray::const_iterator it_ref = it->second.begin(); it_ref != it->second.end(); ++it_ref ) {
- mark_bitmap->Set(*it_ref);
+ mark_sweep->MarkObject(*it_ref);
++count;
}
}
diff --git a/src/mod_union_table.h b/src/mod_union_table.h
index 17ca240..2e8bd0e 100644
--- a/src/mod_union_table.h
+++ b/src/mod_union_table.h
@@ -176,11 +176,9 @@
return (*it)->IsAllocSpace();
}
}
- if (ref != NULL) {
- Implementation::GetHeap()->DumpSpaces();
- LOG(FATAL) << "Reference " << ref << " not in any space!";
- }
- return false;
+ // Assume it points to a large object.
+ // TODO: Check.
+ return true;
}
};
diff --git a/src/oatdump.cc b/src/oatdump.cc
index 88d6976..f08a498 100644
--- a/src/oatdump.cc
+++ b/src/oatdump.cc
@@ -579,6 +579,9 @@
(*cur)->GetLiveBitmap()->Walk(ImageDumper::Callback, this);
os_ << "\n";
}
+ // Dump the large objects separately.
+ heap->GetLargeObjectsSpace()->GetLiveObjects()->Walk(ImageDumper::Callback, this);
+ os_ << "\n";
os_ << "STATS:\n" << std::flush;
UniquePtr<File> file(OS::OpenFile(image_filename_.c_str(), false));
diff --git a/src/object.h b/src/object.h
index a1e1032..9da725c 100644
--- a/src/object.h
+++ b/src/object.h
@@ -1376,6 +1376,10 @@
return GetPrimitiveType() == Primitive::kPrimVoid;
}
+ bool IsPrimitiveArray() const {
+ return IsArrayClass() && GetComponentType()->IsPrimitive();
+ }
+
// Depth of class from java.lang.Object
size_t Depth() {
size_t depth = 0;
diff --git a/src/space.cc b/src/space.cc
index f61f872..428c122 100644
--- a/src/space.cc
+++ b/src/space.cc
@@ -135,6 +135,13 @@
return space;
}
+LargeObjectSpace* Space::CreateLargeObjectSpace(const std::string& name) {
+ if (VLOG_IS_ON(heap) || VLOG_IS_ON(startup)) {
+ VLOG(startup) << "Space::CreateLargeObjectSpace entering " << name;
+ }
+ return new LargeObjectSpace(name);
+}
+
void* AllocSpace::CreateMallocSpace(void* begin, size_t morecore_start, size_t initial_size) {
// clear errno to allow PLOG on error
errno = 0;
@@ -479,4 +486,67 @@
return os;
}
+LargeObjectSpace::LargeObjectSpace(const std::string& name)
+ : Space(name, NULL, NULL, NULL, GCRP_ALWAYS_COLLECT),
+ lock_("large object space lock", kAllocSpaceLock),
+ num_bytes_allocated_(0) {
+ live_objects_.reset(new SpaceSetMap("large live objects"));
+ mark_objects_.reset(new SpaceSetMap("large marked objects"));
+}
+
+void LargeObjectSpace::SwapBitmaps() {
+ MutexLock mu(lock_);
+ SpaceSetMap* temp_live_objects = live_objects_.release();
+ live_objects_.reset(mark_objects_.release());
+ mark_objects_.reset(temp_live_objects);
+ // Swap names to get more descriptive diagnostics.
+ std::string temp_name = live_objects_->GetName();
+ live_objects_->SetName(mark_objects_->GetName());
+ mark_objects_->SetName(temp_name);
+}
+
+Object* LargeObjectSpace::Alloc(size_t num_bytes) {
+ MutexLock mu(lock_);
+ MemMap* mem_map = MemMap::MapAnonymous("allocation", NULL, num_bytes, PROT_READ | PROT_WRITE);
+ if (mem_map == NULL) {
+ return NULL;
+ }
+ Object* obj = reinterpret_cast<Object*>(mem_map->Begin());
+ large_objects_.push_back(obj);
+ mem_maps_.Put(obj, mem_map);
+ num_bytes_allocated_ += mem_map->Size();
+ return obj;
+}
+
+void LargeObjectSpace::CopyLiveToMarked() {
+ MutexLock mu(lock_);
+ mark_objects_->CopyFrom(*live_objects_.get());
+}
+
+void LargeObjectSpace::Free(Object* ptr) {
+ MutexLock mu(lock_);
+ MemMaps::iterator found = mem_maps_.find(ptr);
+ CHECK(found != mem_maps_.end()) << "Attempted to free large object which was not live";
+ DCHECK_GE(num_bytes_allocated_, found->second->Size());
+ num_bytes_allocated_ -= found->second->Size();
+ delete found->second;
+ mem_maps_.erase(found);
+}
+
+size_t LargeObjectSpace::AllocationSize(const Object* obj) {
+ MutexLock mu(lock_);
+ MemMaps::iterator found = mem_maps_.find(const_cast<Object*>(obj));
+ CHECK(found != mem_maps_.end()) << "Attempted to get size of a large object which is not live";
+ return found->second->Size();
+}
+
+void LargeObjectSpace::Walk(AllocSpace::WalkCallback callback, void* arg) {
+ MutexLock mu(lock_);
+ for (MemMaps::iterator it = mem_maps_.begin(); it != mem_maps_.end(); ++it) {
+ MemMap* mem_map = it->second;
+ callback(mem_map->Begin(), mem_map->End(), mem_map->Size(), arg);
+ callback(NULL, NULL, 0, arg);
+ }
+}
+
} // namespace art
diff --git a/src/space.h b/src/space.h
index 3132911..cf10ce4 100644
--- a/src/space.h
+++ b/src/space.h
@@ -30,6 +30,7 @@
class AllocSpace;
class ImageSpace;
+class LargeObjectSpace;
class Object;
class SpaceBitmap;
@@ -51,6 +52,10 @@
size_t growth_limit, size_t capacity,
byte* requested_begin);
+ // Creates a large object space. Allocations into the large object space use mem maps instead of
+ // malloc.
+ static LargeObjectSpace* CreateLargeObjectSpace(const std::string& name);
+
// create a Space from an image file. cannot be used for future allocation or collected.
static ImageSpace* CreateImageSpace(const std::string& image)
SHARED_LOCKS_REQUIRED(Locks::mutator_lock_);
@@ -111,9 +116,15 @@
return down_cast<AllocSpace*>(this);
}
+ LargeObjectSpace* AsLargeObjectSpace() {
+ DCHECK(IsLargeObjectSpace());
+ return down_cast<LargeObjectSpace*>(this);
+ }
+
virtual bool IsAllocSpace() const = 0;
virtual bool IsImageSpace() const = 0;
virtual bool IsZygoteSpace() const = 0;
+ virtual bool IsLargeObjectSpace() const = 0;
virtual SpaceBitmap* GetLiveBitmap() const = 0;
virtual SpaceBitmap* GetMarkBitmap() const = 0;
@@ -153,6 +164,8 @@
// An alloc space is a space where objects may be allocated and garbage collected.
class AllocSpace : public Space {
public:
+ typedef void(*WalkCallback)(void *start, void *end, size_t num_bytes, void* callback_arg);
+
// Allocate num_bytes without allowing the underlying mspace to grow.
Object* AllocWithGrowth(size_t num_bytes);
@@ -177,8 +190,7 @@
// Perform a mspace_inspect_all which calls back for each allocation chunk. The chunk may not be
// in use, indicated by num_bytes equaling zero.
- void Walk(void(*callback)(void *start, void *end, size_t num_bytes, void* callback_arg),
- void* arg);
+ void Walk(WalkCallback callback, void* arg);
// Returns the number of bytes that the heap is allowed to obtain from the system via MoreCore.
size_t GetFootprintLimit();
@@ -212,6 +224,10 @@
return false;
}
+ virtual bool IsLargeObjectSpace() const {
+ return false;
+ }
+
virtual bool IsZygoteSpace() const {
return gc_retention_policy_ == GCRP_FULL_COLLECT;
}
@@ -227,7 +243,7 @@
void SetGrowthLimit(size_t growth_limit);
// Swap the live and mark bitmaps of this space. This is used by the GC for concurrent sweeping.
- void SwapBitmaps();
+ virtual void SwapBitmaps();
// Turn ourself into a zygote space and return a new alloc space which has our unused memory.
AllocSpace* CreateZygoteSpace();
@@ -296,15 +312,19 @@
return false;
}
- virtual SpaceBitmap* GetLiveBitmap() const {
- return live_bitmap_.get();
- }
+ virtual bool IsLargeObjectSpace() const {
+ return false;
+ }
- virtual SpaceBitmap* GetMarkBitmap() const {
- // ImageSpaces have the same bitmap for both live and marked. This helps reduce the number of
- // special cases to test against.
- return live_bitmap_.get();
- }
+ virtual SpaceBitmap* GetLiveBitmap() const {
+ return live_bitmap_.get();
+ }
+
+ virtual SpaceBitmap* GetMarkBitmap() const {
+ // ImageSpaces have the same bitmap for both live and marked. This helps reduce the number of
+ // special cases to test against.
+ return live_bitmap_.get();
+ }
private:
friend class Space;
@@ -317,6 +337,76 @@
DISALLOW_COPY_AND_ASSIGN(ImageSpace);
};
+class LargeObjectSpace : public Space {
+ public:
+ virtual bool IsAllocSpace() const {
+ return false;
+ }
+
+ virtual bool IsImageSpace() const {
+ return false;
+ }
+
+ virtual bool IsZygoteSpace() const {
+ return false;
+ }
+
+ virtual bool IsLargeObjectSpace() const {
+ return true;
+ }
+
+ virtual SpaceBitmap* GetLiveBitmap() const {
+ LOG(FATAL) << "Unimplemented";
+ return NULL;
+ }
+
+ virtual SpaceBitmap* GetMarkBitmap() const {
+ LOG(FATAL) << "Unimplemented";
+ return NULL;
+ }
+
+ virtual SpaceSetMap* GetLiveObjects() const {
+ return live_objects_.get();
+ }
+
+ virtual SpaceSetMap* GetMarkObjects() const {
+ return mark_objects_.get();
+ }
+
+ // Return the storage space required by obj.
+ size_t AllocationSize(const Object* obj);
+
+ virtual void SwapBitmaps();
+ virtual void CopyLiveToMarked();
+
+ Object* Alloc(size_t num_bytes);
+ void Free(Object* ptr);
+ void Walk(AllocSpace::WalkCallback, void* arg);
+
+ size_t GetNumBytesAllocated() const {
+ return num_bytes_allocated_;
+ }
+
+ Mutex& GetLock() {
+ return lock_;
+ }
+
+ private:
+ LargeObjectSpace(const std::string& name);
+
+ // Used to ensure mutual exclusion when the allocation spaces data structures are being modified.
+ Mutex lock_;
+
+ size_t num_bytes_allocated_;
+ UniquePtr<SpaceSetMap> live_objects_;
+ UniquePtr<SpaceSetMap> mark_objects_;
+ std::vector<Object*> large_objects_;
+ typedef SafeMap<Object*, MemMap*> MemMaps;
+ MemMaps mem_maps_;
+
+ friend class Space;
+};
+
// Callback for dlmalloc_inspect_all or mspace_inspect_all that will madvise(2) unused
// pages back to the kernel.
void MspaceMadviseCallback(void* start, void* end, size_t used_bytes, void* /*arg*/);
diff --git a/src/space_bitmap.cc b/src/space_bitmap.cc
index d117c21..b0b9d90 100644
--- a/src/space_bitmap.cc
+++ b/src/space_bitmap.cc
@@ -30,6 +30,12 @@
name_ = name;
}
+void SpaceSetMap::Walk(SpaceBitmap::Callback* callback, void* arg) {
+ for (Objects::iterator it = contained_.begin(); it != contained_.end(); ++it) {
+ callback(const_cast<Object*>(*it), arg);
+ }
+}
+
SpaceBitmap* SpaceBitmap::Create(const std::string& name, byte* heap_begin, size_t heap_capacity) {
CHECK(heap_begin != NULL);
// Round up since heap_capacity is not necessarily a multiple of kAlignment * kBitsPerWord.
@@ -251,6 +257,22 @@
}
}
+std::string SpaceSetMap::GetName() const {
+ return name_;
+}
+
+void SpaceSetMap::SetName(const std::string& name) {
+ name_ = name;
+}
+
+SpaceSetMap::SpaceSetMap(const std::string& name) : name_(name) {
+
+}
+
+void SpaceSetMap::CopyFrom(const SpaceSetMap& space_set) {
+ contained_ = space_set.contained_;
+}
+
std::ostream& operator << (std::ostream& stream, const SpaceBitmap& bitmap) {
return stream
<< bitmap.GetName() << "["
diff --git a/src/space_bitmap.h b/src/space_bitmap.h
index 68a014b..885491f 100644
--- a/src/space_bitmap.h
+++ b/src/space_bitmap.h
@@ -18,6 +18,7 @@
#define ART_SRC_SPACE_BITMAP_H_
#include <limits.h>
+#include <set>
#include <stdint.h>
#include <vector>
@@ -267,6 +268,60 @@
std::string name_;
};
+// Like a bitmap except it keeps track of objects using sets.
+class SpaceSetMap {
+ public:
+ typedef std::set<const Object*> Objects;
+
+ bool IsEmpty() const {
+ return contained_.empty();
+ }
+
+ inline void Set(const Object* obj) {
+ contained_.insert(obj);
+ }
+
+ inline void Clear(const Object* obj) {
+ Objects::iterator found = contained_.find(obj);
+ if (found != contained_.end()) {
+ contained_.erase(found);
+ }
+ }
+
+ void Clear() {
+ contained_.clear();
+ }
+
+ inline bool Test(const Object* obj) const {
+ return contained_.find(obj) != contained_.end();
+ }
+
+ std::string GetName() const;
+ void SetName(const std::string& name);
+
+ void Walk(SpaceBitmap::Callback* callback, void* arg)
+ SHARED_LOCKS_REQUIRED(GlobalSynchronization::heap_bitmap_lock_);
+
+ void CopyFrom(const SpaceSetMap& space_set);
+
+ template <typename Visitor>
+ void Visit(const Visitor& visitor) NO_THREAD_SAFETY_ANALYSIS {
+ for (Objects::iterator it = contained_.begin(); it != contained_.end(); ++it) {
+ visitor(*it);
+ }
+ }
+
+ SpaceSetMap(const std::string& name);
+
+ Objects& GetObjects() {
+ return contained_;
+ }
+
+ private:
+ std::string name_;
+ Objects contained_;
+};
+
std::ostream& operator << (std::ostream& stream, const SpaceBitmap& bitmap);
} // namespace art
diff --git a/src/thread.h b/src/thread.h
index ad861e0..cad06ed 100644
--- a/src/thread.h
+++ b/src/thread.h
@@ -246,7 +246,7 @@
void AssertThreadSuspensionIsAllowable(bool check_locks = true) const;
#else
void AssertThreadSuspensionIsAllowable(bool check_locks = true) const {
- check_locks = !check_locks; // Keep GCC happy about unused parameters.
+ UNUSED(check_locks); // Keep GCC happy about unused parameters.
}
#endif
diff --git a/src/timing_logger.h b/src/timing_logger.h
index a399a28..5bc9d71 100644
--- a/src/timing_logger.h
+++ b/src/timing_logger.h
@@ -57,9 +57,9 @@
uint64_t divisor = GetNsToTimeUnitDivisor(tu);
for (size_t i = 1; i < times_.size(); ++i) {
uint64_t delta_time = times_[i] - times_[i - 1];
- if (!precise_) {
+ if (!precise_ && divisor >= 1000) {
// Make the fraction 0.
- delta_time -= delta_time % divisor;
+ delta_time -= delta_time % (divisor / 1000);
}
os << name_ << ": " << std::setw(8) << FormatDuration(delta_time, tu) << " " << labels_[i]
<< "\n";
@@ -160,10 +160,10 @@
uint64_t mean = times_[i] / iterations_;
uint64_t variance = mean_x2 - (mean * mean);
uint64_t std_dev = static_cast<uint64_t>(std::sqrt(static_cast<double>(variance)));
- if (!precise_) {
+ if (!precise_ && divisor >= 1000) {
// Make the fraction 0.
- mean -= mean % divisor;
- std_dev -= std_dev % divisor;
+ mean -= mean % (divisor / 1000);
+ std_dev -= std_dev % (divisor / 1000);
}
os << name_ << ": " << std::setw(8)
<< FormatDuration(mean * kAdjust, tu) << " std_dev "