Upgrade to latest dlmalloc. Refactor Heap and related APIs to use STL like naming.
We fail assertions in the existing heap code, as does Dalvik. This refactoring
is to clean the heap and space APIs and to reduce duplication of data
and thereby solve a failing assertion in the card table.
This change also wires up clearing of soft references including before
out-of-memory errors are reported.
In doing this change it was made clear that mspaces are buggy (and
violating invariants with the garbage collector). This
change upgrades to an un-Android molested version of dlmalloc-2.8.5 and
implements a version of the mspace morecore routine under ART control.
run-test 061-out-of-memory is updated for current heap sizes.
Change-Id: I377e83ab2a8c78afb9b1881f03356929e2c9dc64
diff --git a/src/heap_bitmap.cc b/src/heap_bitmap.cc
index 2adeedc..57c60ba 100644
--- a/src/heap_bitmap.cc
+++ b/src/heap_bitmap.cc
@@ -22,30 +22,16 @@
namespace art {
-HeapBitmap* HeapBitmap::Create(const char* name, byte* base, size_t length) {
- UniquePtr<HeapBitmap> bitmap(new HeapBitmap(base, length));
- if (!bitmap->Init(name, base, length)) {
+HeapBitmap* HeapBitmap::Create(const char* name, byte* heap_begin, size_t heap_capacity) {
+ CHECK(heap_begin != NULL);
+ size_t bitmap_size = HB_OFFSET_TO_INDEX(heap_capacity) * kWordSize;
+ UniquePtr<MemMap> mem_map(MemMap::MapAnonymous(name, NULL, bitmap_size, PROT_READ | PROT_WRITE));
+ if (mem_map.get() == NULL) {
+ LOG(ERROR) << "Failed to allocate bitmap " << name;
return NULL;
- } else {
- return bitmap.release();
}
-}
-
-// Initialize a HeapBitmap so that it points to a bitmap large enough
-// to cover a heap at <base> of <max_size> bytes, where objects are
-// guaranteed to be kAlignment-aligned.
-bool HeapBitmap::Init(const char* name, const byte* base, size_t max_size) {
- CHECK(base != NULL);
- size_t length = HB_OFFSET_TO_INDEX(max_size) * kWordSize;
- mem_map_.reset(MemMap::MapAnonymous(name, NULL, length, PROT_READ | PROT_WRITE));
- if (mem_map_.get() == NULL) {
- return false;
- }
- words_ = reinterpret_cast<word*>(mem_map_->GetAddress());
- num_bytes_ = length;
- base_ = reinterpret_cast<uintptr_t>(base);
- max_ = base_ - 1;
- return true;
+ word* bitmap_begin = reinterpret_cast<word*>(mem_map->Begin());
+ return new HeapBitmap(name, mem_map.release(), bitmap_begin, bitmap_size, heap_begin);
}
// Clean up any resources associated with the bitmap.
@@ -54,37 +40,36 @@
// Fill the bitmap with zeroes. Returns the bitmap's memory to the
// system as a side-effect.
void HeapBitmap::Clear() {
- if (words_ != NULL) {
+ if (bitmap_begin_ != NULL) {
// This returns the memory to the system. Successive page faults
// will return zeroed memory.
- int result = madvise(words_, num_bytes_, MADV_DONTNEED);
+ int result = madvise(bitmap_begin_, bitmap_size_, MADV_DONTNEED);
if (result == -1) {
PLOG(WARNING) << "madvise failed";
}
- max_ = base_ - 1;
+ heap_end_ = heap_begin_ - 1;
}
}
-// Return true iff <obj> is within the range of pointers that this
-// bitmap could potentially cover, even if a bit has not been set for
-// it.
+// Return true iff <obj> is within the range of pointers that this bitmap could potentially cover,
+// even if a bit has not been set for it.
bool HeapBitmap::HasAddress(const void* obj) const {
if (obj != NULL) {
- const uintptr_t offset = (uintptr_t)obj - base_;
+ const uintptr_t offset = (uintptr_t)obj - heap_begin_;
const size_t index = HB_OFFSET_TO_INDEX(offset);
- return index < num_bytes_ / kWordSize;
+ return index < bitmap_size_ / kWordSize;
}
return false;
}
-void HeapBitmap::VisitRange(uintptr_t base, uintptr_t max, Callback* visitor, void* arg) const {
- size_t start = HB_OFFSET_TO_INDEX(base - base_);
- size_t end = HB_OFFSET_TO_INDEX(max - base_ - 1);
+void HeapBitmap::VisitRange(uintptr_t visit_begin, uintptr_t visit_end, Callback* visitor, void* arg) const {
+ size_t start = HB_OFFSET_TO_INDEX(visit_begin - heap_begin_);
+ size_t end = HB_OFFSET_TO_INDEX(visit_end - heap_begin_ - 1);
for (size_t i = start; i <= end; i++) {
- word w = words_[i];
+ word w = bitmap_begin_[i];
if (w != 0) {
word high_bit = 1 << (kBitsPerWord - 1);
- uintptr_t ptr_base = HB_INDEX_TO_OFFSET(i) + base_;
+ uintptr_t ptr_base = HB_INDEX_TO_OFFSET(i) + heap_begin_;
while (w != 0) {
const int shift = CLZ(w);
Object* obj = reinterpret_cast<Object*>(ptr_base + shift * kAlignment);
@@ -98,14 +83,14 @@
// Visits set bits in address order. The callback is not permitted to
// change the bitmap bits or max during the traversal.
void HeapBitmap::Walk(HeapBitmap::Callback* callback, void* arg) {
- CHECK(words_ != NULL);
+ CHECK(bitmap_begin_ != NULL);
CHECK(callback != NULL);
- uintptr_t end = HB_OFFSET_TO_INDEX(max_ - base_);
+ uintptr_t end = HB_OFFSET_TO_INDEX(heap_end_ - heap_begin_);
for (uintptr_t i = 0; i <= end; ++i) {
- word w = words_[i];
+ word w = bitmap_begin_[i];
if (UNLIKELY(w != 0)) {
word high_bit = 1 << (kBitsPerWord - 1);
- uintptr_t ptr_base = HB_INDEX_TO_OFFSET(i) + base_;
+ uintptr_t ptr_base = HB_INDEX_TO_OFFSET(i) + heap_begin_;
while (w != 0) {
const int shift = CLZ(w);
Object* obj = reinterpret_cast<Object*>(ptr_base + shift * kAlignment);
@@ -116,32 +101,30 @@
}
}
-// Similar to Walk but the callback routine is permitted to change the
-// bitmap bits and max during traversal. Used by the the root marking
-// scan exclusively.
+// Similar to Walk but the callback routine is permitted to change the bitmap bits and end during
+// traversal. Used by the the root marking scan exclusively.
//
-// The callback is invoked with a finger argument. The finger is a
-// pointer to an address not yet visited by the traversal. If the
-// callback sets a bit for an address at or above the finger, this
-// address will be visited by the traversal. If the callback sets a
-// bit for an address below the finger, this address will not be
-// visited.
-void HeapBitmap::ScanWalk(uintptr_t base, uintptr_t max, ScanCallback* callback, void* arg) {
- CHECK(words_ != NULL);
+// The callback is invoked with a finger argument. The finger is a pointer to an address not yet
+// visited by the traversal. If the callback sets a bit for an address at or above the finger, this
+// address will be visited by the traversal. If the callback sets a bit for an address below the
+// finger, this address will not be visited (typiscally such an address would be placed on the
+// marking stack).
+void HeapBitmap::ScanWalk(uintptr_t scan_begin, uintptr_t scan_end, ScanCallback* callback, void* arg) {
+ CHECK(bitmap_begin_ != NULL);
CHECK(callback != NULL);
- CHECK_LE(base, max);
- CHECK_GE(base, base_);
- size_t start = HB_OFFSET_TO_INDEX(base - base_);
- if (max < max_) {
+ CHECK_LE(scan_begin, scan_end);
+ CHECK_GE(scan_begin, heap_begin_);
+ size_t start = HB_OFFSET_TO_INDEX(scan_begin - heap_begin_);
+ if (scan_end < heap_end_) {
// The end of the space we're looking at is before the current maximum bitmap PC, scan to that
// and don't recompute end on each iteration
- size_t end = HB_OFFSET_TO_INDEX(max - base_ - 1);
+ size_t end = HB_OFFSET_TO_INDEX(scan_end - heap_begin_ - 1);
for (size_t i = start; i <= end; i++) {
- word w = words_[i];
+ word w = bitmap_begin_[i];
if (UNLIKELY(w != 0)) {
word high_bit = 1 << (kBitsPerWord - 1);
- uintptr_t ptr_base = HB_INDEX_TO_OFFSET(i) + base_;
- void* finger = reinterpret_cast<void*>(HB_INDEX_TO_OFFSET(i + 1) + base_);
+ uintptr_t ptr_base = HB_INDEX_TO_OFFSET(i) + heap_begin_;
+ void* finger = reinterpret_cast<void*>(HB_INDEX_TO_OFFSET(i + 1) + heap_begin_);
while (w != 0) {
const int shift = CLZ(w);
Object* obj = reinterpret_cast<Object*>(ptr_base + shift * kAlignment);
@@ -151,13 +134,13 @@
}
}
} else {
- size_t end = HB_OFFSET_TO_INDEX(max_ - base_);
+ size_t end = HB_OFFSET_TO_INDEX(heap_end_ - heap_begin_);
for (size_t i = start; i <= end; i++) {
- word w = words_[i];
+ word w = bitmap_begin_[i];
if (UNLIKELY(w != 0)) {
word high_bit = 1 << (kBitsPerWord - 1);
- uintptr_t ptr_base = HB_INDEX_TO_OFFSET(i) + base_;
- void* finger = reinterpret_cast<void*>(HB_INDEX_TO_OFFSET(i + 1) + base_);
+ uintptr_t ptr_base = HB_INDEX_TO_OFFSET(i) + heap_begin_;
+ void* finger = reinterpret_cast<void*>(HB_INDEX_TO_OFFSET(i + 1) + heap_begin_);
while (w != 0) {
const int shift = CLZ(w);
Object* obj = reinterpret_cast<Object*>(ptr_base + shift * kAlignment);
@@ -165,7 +148,8 @@
w &= ~(high_bit >> shift);
}
}
- end = HB_OFFSET_TO_INDEX(max_ - base_);
+ // update 'end' in case callback modified bitmap
+ end = HB_OFFSET_TO_INDEX(heap_end_ - heap_begin_);
}
}
}
@@ -177,37 +161,37 @@
// The callback is not permitted to increase the max of either bitmap.
void HeapBitmap::SweepWalk(const HeapBitmap& live_bitmap,
const HeapBitmap& mark_bitmap,
- uintptr_t base, uintptr_t max,
+ uintptr_t sweep_begin, uintptr_t sweep_end,
HeapBitmap::SweepCallback* callback, void* arg) {
- CHECK(live_bitmap.words_ != NULL);
- CHECK(mark_bitmap.words_ != NULL);
- CHECK_EQ(live_bitmap.base_, mark_bitmap.base_);
- CHECK_EQ(live_bitmap.num_bytes_, mark_bitmap.num_bytes_);
+ CHECK(live_bitmap.bitmap_begin_ != NULL);
+ CHECK(mark_bitmap.bitmap_begin_ != NULL);
+ CHECK_EQ(live_bitmap.heap_begin_, mark_bitmap.heap_begin_);
+ CHECK_EQ(live_bitmap.bitmap_size_, mark_bitmap.bitmap_size_);
CHECK(callback != NULL);
- CHECK_LE(base, max);
- CHECK_GE(base, live_bitmap.base_);
- max = std::min(max - 1, live_bitmap.max_);
- if (live_bitmap.max_ < live_bitmap.base_) {
+ CHECK_LE(sweep_begin, sweep_end);
+ CHECK_GE(sweep_begin, live_bitmap.heap_begin_);
+ sweep_end = std::min(sweep_end - 1, live_bitmap.heap_end_);
+ if (live_bitmap.heap_end_ < live_bitmap.heap_begin_) {
// Easy case; both are obviously empty.
// TODO: this should never happen
return;
}
- // TODO: rewrite the callbacks to accept a std::vector<void*> rather than a void**?
- std::vector<void*> pointer_buf(4 * kBitsPerWord);
- void** pb = &pointer_buf[0];
- size_t start = HB_OFFSET_TO_INDEX(base - live_bitmap.base_);
- size_t end = HB_OFFSET_TO_INDEX(max - live_bitmap.base_);
- word* live = live_bitmap.words_;
- word* mark = mark_bitmap.words_;
+ // TODO: rewrite the callbacks to accept a std::vector<Object*> rather than a Object**?
+ std::vector<Object*> pointer_buf(4 * kBitsPerWord);
+ Object** pb = &pointer_buf[0];
+ size_t start = HB_OFFSET_TO_INDEX(sweep_begin - live_bitmap.heap_begin_);
+ size_t end = HB_OFFSET_TO_INDEX(sweep_end - live_bitmap.heap_begin_);
+ word* live = live_bitmap.bitmap_begin_;
+ word* mark = mark_bitmap.bitmap_begin_;
for (size_t i = start; i <= end; i++) {
word garbage = live[i] & ~mark[i];
if (UNLIKELY(garbage != 0)) {
word high_bit = 1 << (kBitsPerWord - 1);
- uintptr_t ptr_base = HB_INDEX_TO_OFFSET(i) + live_bitmap.base_;
+ uintptr_t ptr_base = HB_INDEX_TO_OFFSET(i) + live_bitmap.heap_begin_;
while (garbage != 0) {
int shift = CLZ(garbage);
garbage &= ~(high_bit >> shift);
- *pb++ = reinterpret_cast<void*>(ptr_base + shift * kAlignment);
+ *pb++ = reinterpret_cast<Object*>(ptr_base + shift * kAlignment);
}
// Make sure that there are always enough slots available for an
// entire word of one bits.