Enable concurrent sweeping
Now we unlock the heap during the sweeping process. When sweeping, the heap is only locked when dealing with dlmalloc.
Change-Id: I705ac499adbf0039a3e57d2c5d354b1087317032
diff --git a/src/heap.cc b/src/heap.cc
index acb80e7..144f766 100644
--- a/src/heap.cc
+++ b/src/heap.cc
@@ -733,10 +733,18 @@
mark_sweep.ProcessReferences(clear_soft_references);
timings.AddSplit("ProcessReferences");
- // TODO: swap live and marked bitmaps
- // Note: Need to be careful about image spaces if we do this since not
- // everything image space will be marked, resulting in things not being
- // marked as live anymore.
+ // Swap the live and mark bitmaps for each alloc space. This is needed since sweep re-swaps
+ // these bitmaps. Doing this enables us to sweep with the heap unlocked since new allocations
+ // set the live bit, but since we have the bitmaps reversed at this point, this sets the mark bit
+ // instead, resulting in no new allocated objects being incorrectly freed by sweep.
+ for (Spaces::iterator it = spaces_.begin(); it != spaces_.end(); ++it) {
+ Space* space = *it;
+ if (space->IsAllocSpace()) {
+ live_bitmap_->ReplaceBitmap(space->GetLiveBitmap(), space->GetMarkBitmap());
+ mark_bitmap_->ReplaceBitmap(space->GetMarkBitmap(), space->GetLiveBitmap());
+ space->AsAllocSpace()->SwapBitmaps();
+ }
+ }
// Verify that we only reach marked objects from the image space
mark_sweep.VerifyImageRoots();
@@ -745,6 +753,7 @@
if (concurrent) {
thread_list->ResumeAll();
dirty_end = NanoTime();
+ Unlock();
}
mark_sweep.Sweep();
@@ -753,6 +762,12 @@
cleared_references = mark_sweep.GetClearedReferences();
}
+ if (concurrent) {
+ // Relock since we unlocked earlier.
+ // TODO: We probably don't need to have the heap locked for all remainder of the function, except for GrowForUtilization.
+ Lock();
+ }
+
GrowForUtilization();
timings.AddSplit("GrowForUtilization");
diff --git a/src/heap.h b/src/heap.h
index 132d5e5..e383665 100644
--- a/src/heap.h
+++ b/src/heap.h
@@ -136,7 +136,7 @@
lock_->AssertNotHeld();
}
- const std::vector<Space*>& GetSpaces() {
+ const Spaces& GetSpaces() {
return spaces_;
}
diff --git a/src/heap_bitmap.cc b/src/heap_bitmap.cc
index c7b2f6c..7d81a5d 100644
--- a/src/heap_bitmap.cc
+++ b/src/heap_bitmap.cc
@@ -1,2 +1,16 @@
#include "heap_bitmap.h"
+namespace art {
+
+void HeapBitmap::ReplaceBitmap(SpaceBitmap* old_bitmap, SpaceBitmap* new_bitmap) {
+ // TODO: C++0x auto
+ for (Bitmaps::iterator cur = bitmaps_.begin(); cur != bitmaps_.end(); ++cur) {
+ if (*cur == old_bitmap) {
+ *cur = new_bitmap;
+ return;
+ }
+ }
+ LOG(FATAL) << "bitmap " << static_cast<const void*>(old_bitmap) << " not found";
+}
+
+} // namespace art
diff --git a/src/heap_bitmap.h b/src/heap_bitmap.h
index e2109d6..29a7b1f 100644
--- a/src/heap_bitmap.h
+++ b/src/heap_bitmap.h
@@ -51,7 +51,7 @@
SpaceBitmap* GetSpaceBitmap(const Object* obj) {
// TODO: C++0x auto
- for (BitmapVec::iterator cur = bitmaps_.begin(); cur != bitmaps_.end(); ++cur) {
+ for (Bitmaps::iterator cur = bitmaps_.begin(); cur != bitmaps_.end(); ++cur) {
if ((*cur)->HasAddress(obj)) {
return *cur;
}
@@ -61,18 +61,21 @@
void Walk(SpaceBitmap::Callback* callback, void* arg) {
// TODO: C++0x auto
- for (BitmapVec::iterator cur = bitmaps_.begin(); cur != bitmaps_.end(); ++cur) {
+ for (Bitmaps::iterator cur = bitmaps_.begin(); cur != bitmaps_.end(); ++cur) {
(*cur)->Walk(callback, arg);
}
}
+ // Find and replace a bitmap pointer, this is used by for the bitmap swapping in the GC.
+ void ReplaceBitmap(SpaceBitmap* old_bitmap, SpaceBitmap* new_bitmap);
+
private:
void AddSpaceBitmap(SpaceBitmap* space) {
bitmaps_.push_back(space);
}
- typedef std::vector<SpaceBitmap*> BitmapVec;
- BitmapVec bitmaps_;
+ typedef std::vector<SpaceBitmap*> Bitmaps;
+ Bitmaps bitmaps_;
friend class Heap;
};
diff --git a/src/mark_sweep.cc b/src/mark_sweep.cc
index 1a7bc83..5155e30 100644
--- a/src/mark_sweep.cc
+++ b/src/mark_sweep.cc
@@ -30,6 +30,7 @@
#include "monitor.h"
#include "object.h"
#include "runtime.h"
+#include "scoped_heap_lock.h"
#include "space.h"
#include "timing_logger.h"
#include "thread.h"
@@ -267,7 +268,7 @@
JavaVMExt* vm = Runtime::Current()->GetJavaVM();
MutexLock mu(vm->weak_globals_lock);
IndirectReferenceTable* table = &vm->weak_globals;
- typedef IndirectReferenceTable::iterator It; // TODO: C++0x auto
+ typedef IndirectReferenceTable::iterator It; // TODO: C++0x auto
for (It it = table->begin(), end = table->end(); it != end; ++it) {
const Object** entry = *it;
if (!heap_->GetMarkBitmap()->Test(*entry)) {
@@ -288,7 +289,8 @@
};
void MarkSweep::SweepCallback(size_t num_ptrs, Object** ptrs, void* arg) {
- // TODO: lock heap if concurrent
+ ScopedHeapLock lock;
+
size_t freed_objects = num_ptrs;
size_t freed_bytes = 0;
SweepCallbackContext* context = static_cast<SweepCallbackContext*>(arg);
@@ -316,7 +318,6 @@
}
}
heap->RecordFreeLocked(freed_objects, freed_bytes);
- // TODO: unlock heap if concurrent
}
void MarkSweep::Sweep() {
@@ -332,8 +333,9 @@
uintptr_t begin = reinterpret_cast<uintptr_t>(spaces[i]->Begin());
uintptr_t end = reinterpret_cast<uintptr_t>(spaces[i]->End());
scc.space = spaces[i]->AsAllocSpace();
- SpaceBitmap* live_bitmap = scc.space->GetLiveBitmap();
- SpaceBitmap* mark_bitmap = scc.space->GetMarkBitmap();
+ // Bitmaps are pre-swapped for optimization which enables sweeping with the heap unlocked.
+ SpaceBitmap* live_bitmap = scc.space->GetMarkBitmap();
+ SpaceBitmap* mark_bitmap = scc.space->GetLiveBitmap();
SpaceBitmap::SweepWalk(*live_bitmap, *mark_bitmap, begin, end,
&MarkSweep::SweepCallback, reinterpret_cast<void*>(&scc));
}
diff --git a/src/space.cc b/src/space.cc
index 569a0c9..3d8c5e0 100644
--- a/src/space.cc
+++ b/src/space.cc
@@ -144,6 +144,12 @@
return msp;
}
+void AllocSpace::SwapBitmaps() {
+ SpaceBitmap* temp_live_bitmap = live_bitmap_.release();
+ live_bitmap_.reset(mark_bitmap_.release());
+ mark_bitmap_.reset(temp_live_bitmap);
+}
+
Object* AllocSpace::AllocWithoutGrowth(size_t num_bytes) {
Object* result = reinterpret_cast<Object*>(mspace_calloc(mspace_, 1, num_bytes));
#if DEBUG_SPACES
diff --git a/src/space.h b/src/space.h
index ff16a51..be0fb61 100644
--- a/src/space.h
+++ b/src/space.h
@@ -195,6 +195,9 @@
return mark_bitmap_.get();
}
+ // Swap the live and mark bitmaps of this space. This is used by the GC for concurrent sweeping.
+ void SwapBitmaps();
+
private:
friend class Space;
@@ -256,6 +259,7 @@
virtual SpaceBitmap* GetMarkBitmap() const {
return live_bitmap_.get();
}
+
private:
friend class Space;