Some low-hanging GC performance fruit.

Halves the GC pause in my test benchmark (using libart rather than libartd).

Change-Id: I57e5f9e312fd59ea62eea9da37a116e1ad21d4ec
diff --git a/src/mark_sweep.cc b/src/mark_sweep.cc
index 2e8041d..71983fe 100644
--- a/src/mark_sweep.cc
+++ b/src/mark_sweep.cc
@@ -40,7 +40,7 @@
   return true;
 }
 
-void MarkSweep::MarkObject0(const Object* obj, bool check_finger) {
+inline void MarkSweep::MarkObject0(const Object* obj, bool check_finger) {
   DCHECK(obj != NULL);
   if (obj < condemned_) {
     DCHECK(IsMarked(obj));
@@ -62,7 +62,7 @@
 // objects.  Any newly-marked objects whose addresses are lower than
 // the finger won't be visited by the bitmap scan, so those objects
 // need to be added to the mark stack.
-void MarkSweep::MarkObject(const Object* obj) {
+inline void MarkSweep::MarkObject(const Object* obj) {
   if (obj != NULL) {
     MarkObject0(obj, true);
   }
@@ -167,7 +167,7 @@
 }
 
 // Scans instance fields.
-void MarkSweep::ScanInstanceFields(const Object* obj) {
+inline void MarkSweep::ScanInstanceFields(const Object* obj) {
   DCHECK(obj != NULL);
   Class* klass = obj->GetClass();
   DCHECK(klass != NULL);
@@ -175,12 +175,12 @@
 }
 
 // Scans static storage on a Class.
-void MarkSweep::ScanStaticFields(const Class* klass) {
+inline void MarkSweep::ScanStaticFields(const Class* klass) {
   DCHECK(klass != NULL);
   ScanFields(klass, klass->GetReferenceStaticOffsets(), true);
 }
 
-void MarkSweep::ScanFields(const Object* obj, uint32_t ref_offsets, bool is_static) {
+inline void MarkSweep::ScanFields(const Object* obj, uint32_t ref_offsets, bool is_static) {
   if (ref_offsets != CLASS_WALK_SUPER) {
     // Found a reference offset bitmap.  Mark the specified offsets.
     while (ref_offsets != 0) {
@@ -215,7 +215,7 @@
 
 // Scans the header, static field references, and interface pointers
 // of a class object.
-void MarkSweep::ScanClass(const Object* obj) {
+inline void MarkSweep::ScanClass(const Object* obj) {
   DCHECK(obj != NULL);
   DCHECK(obj->IsClass());
   const Class* klass = obj->AsClass();
@@ -243,7 +243,7 @@
 
 // Scans the header of all array objects.  If the array object is
 // specialized to a reference type, scans the array data as well.
-void MarkSweep::ScanArray(const Object* obj) {
+inline void MarkSweep::ScanArray(const Object* obj) {
   DCHECK(obj != NULL);
   DCHECK(obj->GetClass() != NULL);
   MarkObject(obj->GetClass());
@@ -285,7 +285,7 @@
 // Scans the header and field references of a data object.  If the
 // scanned object is a reference subclass, it is scheduled for later
 // processing.
-void MarkSweep::ScanOther(const Object* obj) {
+inline void MarkSweep::ScanOther(const Object* obj) {
   DCHECK(obj != NULL);
   Class* klass = obj->GetClass();
   DCHECK(klass != NULL);
@@ -298,7 +298,7 @@
 
 // Scans an object reference.  Determines the type of the reference
 // and dispatches to a specialized scanning routine.
-void MarkSweep::ScanObject(const Object* obj) {
+inline void MarkSweep::ScanObject(const Object* obj) {
   DCHECK(obj != NULL);
   DCHECK(obj->GetClass() != NULL);
   DCHECK(IsMarked(obj));
diff --git a/src/object.h b/src/object.h
index a102386..882c025 100644
--- a/src/object.h
+++ b/src/object.h
@@ -1450,8 +1450,7 @@
   }
 
   Class* GetComponentType() const {
-    return GetFieldObject<Class*>(
-        OFFSET_OF_OBJECT_MEMBER(Class, component_type_), false);
+    return GetFieldObject<Class*>(OFFSET_OF_OBJECT_MEMBER(Class, component_type_), false);
   }
 
   void SetComponentType(Class* new_component_type) {
diff --git a/src/object_bitmap.cc b/src/object_bitmap.cc
index 1a3d600..bacfd3d 100644
--- a/src/object_bitmap.cc
+++ b/src/object_bitmap.cc
@@ -84,15 +84,15 @@
   CHECK(callback != NULL);
   uintptr_t end = HB_OFFSET_TO_INDEX(max_ - base_);
   for (uintptr_t i = 0; i <= end; ++i) {
-    unsigned long word = words_[i];
-    if (word != 0) {
-      unsigned long high_bit = 1 << (kBitsPerWord - 1);
+    word w = words_[i];
+    if (UNLIKELY(w != 0)) {
+      word high_bit = 1 << (kBitsPerWord - 1);
       uintptr_t ptr_base = HB_INDEX_TO_OFFSET(i) + base_;
-      while (word != 0) {
-        const int shift = CLZ(word);
-        Object* obj = (Object*) (ptr_base + shift * kAlignment);
+      while (w != 0) {
+        const int shift = CLZ(w);
+        Object* obj = reinterpret_cast<Object*>(ptr_base + shift * kAlignment);
         (*callback)(obj, arg);
-        word &= ~(high_bit >> shift);
+        w &= ~(high_bit >> shift);
       }
     }
   }
@@ -109,21 +109,21 @@
 // bit for an address below the finger, this address will not be
 // visited.
 void HeapBitmap::ScanWalk(uintptr_t base, ScanCallback* callback, void* arg) {
-  CHECK(words_ != NULL);
-  CHECK(callback != NULL);
-  CHECK_GE(base, base_);
+  DCHECK(words_ != NULL);
+  DCHECK(callback != NULL);
+  DCHECK_GE(base, base_);
   uintptr_t end = HB_OFFSET_TO_INDEX(max_ - base);
   for (uintptr_t i = 0; i <= end; ++i) {
-    unsigned long word = words_[i];
-    if (word != 0) {
-      unsigned long high_bit = 1 << (kBitsPerWord - 1);
+    word w = words_[i];
+    if (UNLIKELY(w != 0)) {
+      word high_bit = 1 << (kBitsPerWord - 1);
       uintptr_t ptr_base = HB_INDEX_TO_OFFSET(i) + base_;
-      void* finger = (void*)(HB_INDEX_TO_OFFSET(i + 1) + base_);
-      while (word != 0) {
-        const int shift = CLZ(word);
-        Object* obj = (Object*)(ptr_base + shift * kAlignment);
+      void* finger = reinterpret_cast<void*>(HB_INDEX_TO_OFFSET(i + 1) + base_);
+      while (w != 0) {
+        const int shift = CLZ(w);
+        Object* obj = reinterpret_cast<Object*>(ptr_base + shift * kAlignment);
         (*callback)(obj, finger, arg);
-        word &= ~(high_bit >> shift);
+        w &= ~(high_bit >> shift);
       }
       end = HB_OFFSET_TO_INDEX(max_ - base_);
     }
@@ -160,14 +160,14 @@
   word* live = live_bitmap.words_;
   word* mark = mark_bitmap.words_;
   for (size_t i = start; i <= end; i++) {
-    unsigned long garbage = live[i] & ~mark[i];
-    if (garbage != 0) {
-      unsigned long high_bit = 1 << (kBitsPerWord - 1);
+    word garbage = live[i] & ~mark[i];
+    if (UNLIKELY(garbage != 0)) {
+      word high_bit = 1 << (kBitsPerWord - 1);
       uintptr_t ptr_base = HB_INDEX_TO_OFFSET(i) + live_bitmap.base_;
       while (garbage != 0) {
         int shift = CLZ(garbage);
         garbage &= ~(high_bit >> shift);
-        *pb++ = (void*)(ptr_base + shift * kAlignment);
+        *pb++ = reinterpret_cast<void*>(ptr_base + shift * kAlignment);
       }
       // Make sure that there are always enough slots available for an
       // entire word of one bits.
diff --git a/src/object_bitmap.h b/src/object_bitmap.h
index 79b4027..1374563 100644
--- a/src/object_bitmap.h
+++ b/src/object_bitmap.h
@@ -36,13 +36,12 @@
     ((index_) * kAlignment * kBitsPerWord)
 
 #define HB_OFFSET_TO_BYTE_INDEX(offset_) \
-  (HB_OFFSET_TO_INDEX(offset_) * sizeof(*((HeapBitmap *)0)->words_))
+  (HB_OFFSET_TO_INDEX(offset_) * sizeof(*(reinterpret_cast<HeapBitmap*>(0))->words_))
 
 // Pack the bits in backwards so they come out in address order
 // when using CLZ.
 #define HB_OFFSET_TO_MASK(offset_) \
-    (1 << \
-        (31-(((uintptr_t)(offset_) / kAlignment) % kBitsPerWord)))
+    (1 << (31-(((uintptr_t)(offset_) / kAlignment) % kBitsPerWord)))
 
 class HeapBitmap {
  public:
@@ -58,26 +57,26 @@
 
   ~HeapBitmap();
 
-  void Set(const Object* obj) {
+  inline void Set(const Object* obj) {
     Modify(obj, true);
   }
 
-  void Clear(const Object* obj) {
+  inline void Clear(const Object* obj) {
     Modify(obj, false);
   }
 
   void Clear();
 
-  bool Test(const Object* obj) {
-    CHECK(HasAddress(obj)) << obj;
-    CHECK(words_ != NULL);
-    CHECK_GE((uintptr_t)obj, base_);
-    if ((uintptr_t)obj <= max_) {
-        const uintptr_t offset = (uintptr_t)obj - base_;
-        unsigned long word = words_[HB_OFFSET_TO_INDEX(offset)];
-        return (word & HB_OFFSET_TO_MASK(offset)) != 0;
+  inline bool Test(const Object* obj) {
+    uintptr_t addr = reinterpret_cast<uintptr_t>(obj);
+    DCHECK(HasAddress(obj)) << obj;
+    DCHECK(words_ != NULL);
+    DCHECK_GE(addr, base_);
+    if (addr <= max_) {
+      const uintptr_t offset = addr - base_;
+      return (words_[HB_OFFSET_TO_INDEX(offset)] & HB_OFFSET_TO_MASK(offset)) != 0;
     } else {
-        return false;
+      return false;
     }
   }
 
@@ -99,13 +98,13 @@
         base_(reinterpret_cast<uintptr_t>(base)) {
   };
 
-  void Modify(const Object* obj, bool do_set) {
+  inline void Modify(const Object* obj, bool do_set) {
     uintptr_t addr = reinterpret_cast<uintptr_t>(obj);
-    CHECK_GE(addr, base_);
+    DCHECK_GE(addr, base_);
     const uintptr_t offset = addr - base_;
     const size_t index = HB_OFFSET_TO_INDEX(offset);
-    const unsigned long mask = HB_OFFSET_TO_MASK(offset);
-    CHECK_LT(index, num_bytes_ / kWordSize);
+    const word mask = HB_OFFSET_TO_MASK(offset);
+    DCHECK_LT(index, num_bytes_ / kWordSize);
     if (do_set) {
       if (addr > max_) {
         max_ = addr;