Some low-hanging GC performance fruit.
Halves the GC pause in my test benchmark (using libart rather than libartd).
Change-Id: I57e5f9e312fd59ea62eea9da37a116e1ad21d4ec
diff --git a/src/mark_sweep.cc b/src/mark_sweep.cc
index 2e8041d..71983fe 100644
--- a/src/mark_sweep.cc
+++ b/src/mark_sweep.cc
@@ -40,7 +40,7 @@
return true;
}
-void MarkSweep::MarkObject0(const Object* obj, bool check_finger) {
+inline void MarkSweep::MarkObject0(const Object* obj, bool check_finger) {
DCHECK(obj != NULL);
if (obj < condemned_) {
DCHECK(IsMarked(obj));
@@ -62,7 +62,7 @@
// objects. Any newly-marked objects whose addresses are lower than
// the finger won't be visited by the bitmap scan, so those objects
// need to be added to the mark stack.
-void MarkSweep::MarkObject(const Object* obj) {
+inline void MarkSweep::MarkObject(const Object* obj) {
if (obj != NULL) {
MarkObject0(obj, true);
}
@@ -167,7 +167,7 @@
}
// Scans instance fields.
-void MarkSweep::ScanInstanceFields(const Object* obj) {
+inline void MarkSweep::ScanInstanceFields(const Object* obj) {
DCHECK(obj != NULL);
Class* klass = obj->GetClass();
DCHECK(klass != NULL);
@@ -175,12 +175,12 @@
}
// Scans static storage on a Class.
-void MarkSweep::ScanStaticFields(const Class* klass) {
+inline void MarkSweep::ScanStaticFields(const Class* klass) {
DCHECK(klass != NULL);
ScanFields(klass, klass->GetReferenceStaticOffsets(), true);
}
-void MarkSweep::ScanFields(const Object* obj, uint32_t ref_offsets, bool is_static) {
+inline void MarkSweep::ScanFields(const Object* obj, uint32_t ref_offsets, bool is_static) {
if (ref_offsets != CLASS_WALK_SUPER) {
// Found a reference offset bitmap. Mark the specified offsets.
while (ref_offsets != 0) {
@@ -215,7 +215,7 @@
// Scans the header, static field references, and interface pointers
// of a class object.
-void MarkSweep::ScanClass(const Object* obj) {
+inline void MarkSweep::ScanClass(const Object* obj) {
DCHECK(obj != NULL);
DCHECK(obj->IsClass());
const Class* klass = obj->AsClass();
@@ -243,7 +243,7 @@
// Scans the header of all array objects. If the array object is
// specialized to a reference type, scans the array data as well.
-void MarkSweep::ScanArray(const Object* obj) {
+inline void MarkSweep::ScanArray(const Object* obj) {
DCHECK(obj != NULL);
DCHECK(obj->GetClass() != NULL);
MarkObject(obj->GetClass());
@@ -285,7 +285,7 @@
// Scans the header and field references of a data object. If the
// scanned object is a reference subclass, it is scheduled for later
// processing.
-void MarkSweep::ScanOther(const Object* obj) {
+inline void MarkSweep::ScanOther(const Object* obj) {
DCHECK(obj != NULL);
Class* klass = obj->GetClass();
DCHECK(klass != NULL);
@@ -298,7 +298,7 @@
// Scans an object reference. Determines the type of the reference
// and dispatches to a specialized scanning routine.
-void MarkSweep::ScanObject(const Object* obj) {
+inline void MarkSweep::ScanObject(const Object* obj) {
DCHECK(obj != NULL);
DCHECK(obj->GetClass() != NULL);
DCHECK(IsMarked(obj));
diff --git a/src/object.h b/src/object.h
index a102386..882c025 100644
--- a/src/object.h
+++ b/src/object.h
@@ -1450,8 +1450,7 @@
}
Class* GetComponentType() const {
- return GetFieldObject<Class*>(
- OFFSET_OF_OBJECT_MEMBER(Class, component_type_), false);
+ return GetFieldObject<Class*>(OFFSET_OF_OBJECT_MEMBER(Class, component_type_), false);
}
void SetComponentType(Class* new_component_type) {
diff --git a/src/object_bitmap.cc b/src/object_bitmap.cc
index 1a3d600..bacfd3d 100644
--- a/src/object_bitmap.cc
+++ b/src/object_bitmap.cc
@@ -84,15 +84,15 @@
CHECK(callback != NULL);
uintptr_t end = HB_OFFSET_TO_INDEX(max_ - base_);
for (uintptr_t i = 0; i <= end; ++i) {
- unsigned long word = words_[i];
- if (word != 0) {
- unsigned long high_bit = 1 << (kBitsPerWord - 1);
+ word w = words_[i];
+ if (UNLIKELY(w != 0)) {
+ word high_bit = 1 << (kBitsPerWord - 1);
uintptr_t ptr_base = HB_INDEX_TO_OFFSET(i) + base_;
- while (word != 0) {
- const int shift = CLZ(word);
- Object* obj = (Object*) (ptr_base + shift * kAlignment);
+ while (w != 0) {
+ const int shift = CLZ(w);
+ Object* obj = reinterpret_cast<Object*>(ptr_base + shift * kAlignment);
(*callback)(obj, arg);
- word &= ~(high_bit >> shift);
+ w &= ~(high_bit >> shift);
}
}
}
@@ -109,21 +109,21 @@
// bit for an address below the finger, this address will not be
// visited.
void HeapBitmap::ScanWalk(uintptr_t base, ScanCallback* callback, void* arg) {
- CHECK(words_ != NULL);
- CHECK(callback != NULL);
- CHECK_GE(base, base_);
+ DCHECK(words_ != NULL);
+ DCHECK(callback != NULL);
+ DCHECK_GE(base, base_);
uintptr_t end = HB_OFFSET_TO_INDEX(max_ - base);
for (uintptr_t i = 0; i <= end; ++i) {
- unsigned long word = words_[i];
- if (word != 0) {
- unsigned long high_bit = 1 << (kBitsPerWord - 1);
+ word w = words_[i];
+ if (UNLIKELY(w != 0)) {
+ word high_bit = 1 << (kBitsPerWord - 1);
uintptr_t ptr_base = HB_INDEX_TO_OFFSET(i) + base_;
- void* finger = (void*)(HB_INDEX_TO_OFFSET(i + 1) + base_);
- while (word != 0) {
- const int shift = CLZ(word);
- Object* obj = (Object*)(ptr_base + shift * kAlignment);
+ void* finger = reinterpret_cast<void*>(HB_INDEX_TO_OFFSET(i + 1) + base_);
+ while (w != 0) {
+ const int shift = CLZ(w);
+ Object* obj = reinterpret_cast<Object*>(ptr_base + shift * kAlignment);
(*callback)(obj, finger, arg);
- word &= ~(high_bit >> shift);
+ w &= ~(high_bit >> shift);
}
end = HB_OFFSET_TO_INDEX(max_ - base_);
}
@@ -160,14 +160,14 @@
word* live = live_bitmap.words_;
word* mark = mark_bitmap.words_;
for (size_t i = start; i <= end; i++) {
- unsigned long garbage = live[i] & ~mark[i];
- if (garbage != 0) {
- unsigned long high_bit = 1 << (kBitsPerWord - 1);
+ word garbage = live[i] & ~mark[i];
+ if (UNLIKELY(garbage != 0)) {
+ word high_bit = 1 << (kBitsPerWord - 1);
uintptr_t ptr_base = HB_INDEX_TO_OFFSET(i) + live_bitmap.base_;
while (garbage != 0) {
int shift = CLZ(garbage);
garbage &= ~(high_bit >> shift);
- *pb++ = (void*)(ptr_base + shift * kAlignment);
+ *pb++ = reinterpret_cast<void*>(ptr_base + shift * kAlignment);
}
// Make sure that there are always enough slots available for an
// entire word of one bits.
diff --git a/src/object_bitmap.h b/src/object_bitmap.h
index 79b4027..1374563 100644
--- a/src/object_bitmap.h
+++ b/src/object_bitmap.h
@@ -36,13 +36,12 @@
((index_) * kAlignment * kBitsPerWord)
#define HB_OFFSET_TO_BYTE_INDEX(offset_) \
- (HB_OFFSET_TO_INDEX(offset_) * sizeof(*((HeapBitmap *)0)->words_))
+ (HB_OFFSET_TO_INDEX(offset_) * sizeof(*(reinterpret_cast<HeapBitmap*>(0))->words_))
// Pack the bits in backwards so they come out in address order
// when using CLZ.
#define HB_OFFSET_TO_MASK(offset_) \
- (1 << \
- (31-(((uintptr_t)(offset_) / kAlignment) % kBitsPerWord)))
+ (1 << (31-(((uintptr_t)(offset_) / kAlignment) % kBitsPerWord)))
class HeapBitmap {
public:
@@ -58,26 +57,26 @@
~HeapBitmap();
- void Set(const Object* obj) {
+ inline void Set(const Object* obj) {
Modify(obj, true);
}
- void Clear(const Object* obj) {
+ inline void Clear(const Object* obj) {
Modify(obj, false);
}
void Clear();
- bool Test(const Object* obj) {
- CHECK(HasAddress(obj)) << obj;
- CHECK(words_ != NULL);
- CHECK_GE((uintptr_t)obj, base_);
- if ((uintptr_t)obj <= max_) {
- const uintptr_t offset = (uintptr_t)obj - base_;
- unsigned long word = words_[HB_OFFSET_TO_INDEX(offset)];
- return (word & HB_OFFSET_TO_MASK(offset)) != 0;
+ inline bool Test(const Object* obj) {
+ uintptr_t addr = reinterpret_cast<uintptr_t>(obj);
+ DCHECK(HasAddress(obj)) << obj;
+ DCHECK(words_ != NULL);
+ DCHECK_GE(addr, base_);
+ if (addr <= max_) {
+ const uintptr_t offset = addr - base_;
+ return (words_[HB_OFFSET_TO_INDEX(offset)] & HB_OFFSET_TO_MASK(offset)) != 0;
} else {
- return false;
+ return false;
}
}
@@ -99,13 +98,13 @@
base_(reinterpret_cast<uintptr_t>(base)) {
};
- void Modify(const Object* obj, bool do_set) {
+ inline void Modify(const Object* obj, bool do_set) {
uintptr_t addr = reinterpret_cast<uintptr_t>(obj);
- CHECK_GE(addr, base_);
+ DCHECK_GE(addr, base_);
const uintptr_t offset = addr - base_;
const size_t index = HB_OFFSET_TO_INDEX(offset);
- const unsigned long mask = HB_OFFSET_TO_MASK(offset);
- CHECK_LT(index, num_bytes_ / kWordSize);
+ const word mask = HB_OFFSET_TO_MASK(offset);
+ DCHECK_LT(index, num_bytes_ / kWordSize);
if (do_set) {
if (addr > max_) {
max_ = addr;