Remove four counter increments out of the allocation path.
- This change removes the four allocation counters (the number of
bytes allocated, the number of objects allocated, the number of bytes
allocated ever, the number of objects allocated ever) from the
dlmalloc space allocation path. Now those counter values are computed
on the fly based on a combination of the two new counters that are
incremented by the GC side (the number of bytes freed ever and the
number of objects freed ever) and mspace_inspect_all() calls.
- This results in a 1-2% speedup (though with some noise) in Ritz
MemAllocTest on Nexus 4.
Bug: 9986565
Change-Id: Id9a8e05a745ac1e5ea7a2b5fd9319814a9d4af13
diff --git a/runtime/gc/allocator/dlmalloc.cc b/runtime/gc/allocator/dlmalloc.cc
index 3cc64e9..a6a3ee7 100644
--- a/runtime/gc/allocator/dlmalloc.cc
+++ b/runtime/gc/allocator/dlmalloc.cc
@@ -69,3 +69,19 @@
*reclaimed += length;
}
}
+
+extern "C" void DlmallocBytesAllocatedCallback(void* start, void* end, size_t used_bytes, void* arg) {
+ if (used_bytes == 0) {
+ return;
+ }
+ size_t* bytes_allocated = reinterpret_cast<size_t*>(arg);
+ *bytes_allocated += used_bytes + sizeof(size_t);
+}
+
+extern "C" void DlmallocObjectsAllocatedCallback(void* start, void* end, size_t used_bytes, void* arg) {
+ if (used_bytes == 0) {
+ return;
+ }
+ size_t* objects_allocated = reinterpret_cast<size_t*>(arg);
+ ++(*objects_allocated);
+}
diff --git a/runtime/gc/allocator/dlmalloc.h b/runtime/gc/allocator/dlmalloc.h
index 19159b1..c820b19 100644
--- a/runtime/gc/allocator/dlmalloc.h
+++ b/runtime/gc/allocator/dlmalloc.h
@@ -39,4 +39,10 @@
// pages back to the kernel.
extern "C" void DlmallocMadviseCallback(void* start, void* end, size_t used_bytes, void* /*arg*/);
+// Callbacks for dlmalloc_inspect_all or mspace_inspect_all that will
+// count the number of bytes allocated and objects allocated,
+// respectively.
+extern "C" void DlmallocBytesAllocatedCallback(void* start, void* end, size_t used_bytes, void* arg);
+extern "C" void DlmallocObjectsAllocatedCallback(void* start, void* end, size_t used_bytes, void* arg);
+
#endif // ART_RUNTIME_GC_ALLOCATOR_DLMALLOC_H_
diff --git a/runtime/gc/heap.cc b/runtime/gc/heap.cc
index c0e46ac..ed90242 100644
--- a/runtime/gc/heap.cc
+++ b/runtime/gc/heap.cc
@@ -1999,8 +1999,10 @@
// We could try mincore(2) but that's only a measure of how many pages we haven't given away,
// not how much use we're making of those pages.
uint64_t ms_time = MilliTime();
- float utilization =
- static_cast<float>(alloc_space_->GetBytesAllocated()) / alloc_space_->Size();
+ // Note the large object space's bytes allocated is equal to its capacity.
+ uint64_t los_bytes_allocated = large_object_space_->GetBytesAllocated();
+ float utilization = static_cast<float>(GetBytesAllocated() - los_bytes_allocated) /
+ (GetTotalMemory() - los_bytes_allocated);
if ((utilization > 0.75f && !IsLowMemoryMode()) || ((ms_time - last_trim_time_ms_) < 2 * 1000)) {
// Don't bother trimming the alloc space if it's more than 75% utilized and low memory mode is
// not enabled, or if a heap trim occurred in the last two seconds.
diff --git a/runtime/gc/space/dlmalloc_space-inl.h b/runtime/gc/space/dlmalloc_space-inl.h
index 242ef68..fb2c66b 100644
--- a/runtime/gc/space/dlmalloc_space-inl.h
+++ b/runtime/gc/space/dlmalloc_space-inl.h
@@ -47,10 +47,6 @@
size_t allocation_size = AllocationSizeNonvirtual(result);
DCHECK(bytes_allocated != NULL);
*bytes_allocated = allocation_size;
- num_bytes_allocated_ += allocation_size;
- total_bytes_allocated_ += allocation_size;
- ++total_objects_allocated_;
- ++num_objects_allocated_;
}
return result;
}
diff --git a/runtime/gc/space/dlmalloc_space.cc b/runtime/gc/space/dlmalloc_space.cc
index 1b6fca7..468d1d2 100644
--- a/runtime/gc/space/dlmalloc_space.cc
+++ b/runtime/gc/space/dlmalloc_space.cc
@@ -119,8 +119,7 @@
DlMallocSpace::DlMallocSpace(const std::string& name, MemMap* mem_map, void* mspace, byte* begin,
byte* end, size_t growth_limit)
: MemMapSpace(name, mem_map, end - begin, kGcRetentionPolicyAlwaysCollect),
- recent_free_pos_(0), num_bytes_allocated_(0), num_objects_allocated_(0),
- total_bytes_allocated_(0), total_objects_allocated_(0),
+ recent_free_pos_(0), total_bytes_freed_(0), total_objects_freed_(0),
lock_("allocation space lock", kAllocSpaceLock), mspace_(mspace),
growth_limit_(growth_limit) {
CHECK(mspace != NULL);
@@ -353,8 +352,8 @@
CHECK(Contains(ptr)) << "Free (" << ptr << ") not in bounds of heap " << *this;
}
const size_t bytes_freed = InternalAllocationSize(ptr);
- num_bytes_allocated_ -= bytes_freed;
- --num_objects_allocated_;
+ total_bytes_freed_ += bytes_freed;
+ ++total_objects_freed_;
if (kRecentFreeCount > 0) {
RegisterRecentFree(ptr);
}
@@ -400,8 +399,8 @@
{
MutexLock mu(self, lock_);
- num_bytes_allocated_ -= bytes_freed;
- num_objects_allocated_ -= num_ptrs;
+ total_bytes_freed_ += bytes_freed;
+ total_objects_freed_ += num_ptrs;
mspace_bulk_free(mspace_, reinterpret_cast<void**>(ptrs), num_ptrs);
return bytes_freed;
}
@@ -501,6 +500,20 @@
<< ",name=\"" << GetName() << "\"]";
}
+uint64_t DlMallocSpace::GetBytesAllocated() {
+ MutexLock mu(Thread::Current(), lock_);
+ size_t bytes_allocated = 0;
+ mspace_inspect_all(mspace_, DlmallocBytesAllocatedCallback, &bytes_allocated);
+ return bytes_allocated;
+}
+
+uint64_t DlMallocSpace::GetObjectsAllocated() {
+ MutexLock mu(Thread::Current(), lock_);
+ size_t objects_allocated = 0;
+ mspace_inspect_all(mspace_, DlmallocObjectsAllocatedCallback, &objects_allocated);
+ return objects_allocated;
+}
+
} // namespace space
} // namespace gc
} // namespace art
diff --git a/runtime/gc/space/dlmalloc_space.h b/runtime/gc/space/dlmalloc_space.h
index b08da01..522535e 100644
--- a/runtime/gc/space/dlmalloc_space.h
+++ b/runtime/gc/space/dlmalloc_space.h
@@ -127,20 +127,13 @@
// Turn ourself into a zygote space and return a new alloc space which has our unused memory.
DlMallocSpace* CreateZygoteSpace(const char* alloc_space_name);
- uint64_t GetBytesAllocated() const {
- return num_bytes_allocated_;
+ uint64_t GetBytesAllocated();
+ uint64_t GetObjectsAllocated();
+ uint64_t GetTotalBytesAllocated() {
+ return GetBytesAllocated() + total_bytes_freed_;
}
-
- uint64_t GetObjectsAllocated() const {
- return num_objects_allocated_;
- }
-
- uint64_t GetTotalBytesAllocated() const {
- return total_bytes_allocated_;
- }
-
- uint64_t GetTotalObjectsAllocated() const {
- return total_objects_allocated_;
+ uint64_t GetTotalObjectsAllocated() {
+ return GetObjectsAllocated() + total_objects_freed_;
}
// Returns the class of a recently freed object.
@@ -168,11 +161,9 @@
std::pair<const mirror::Object*, mirror::Class*> recent_freed_objects_[kRecentFreeCount];
size_t recent_free_pos_;
- // Approximate number of bytes which have been allocated into the space.
- size_t num_bytes_allocated_;
- size_t num_objects_allocated_;
- size_t total_bytes_allocated_;
- size_t total_objects_allocated_;
+ // Approximate number of bytes and objects which have been deallocated in the space.
+ size_t total_bytes_freed_;
+ size_t total_objects_freed_;
static size_t bitmap_index_;
diff --git a/runtime/gc/space/large_object_space.h b/runtime/gc/space/large_object_space.h
index a703e86..3f2e848 100644
--- a/runtime/gc/space/large_object_space.h
+++ b/runtime/gc/space/large_object_space.h
@@ -41,19 +41,19 @@
virtual void Walk(DlMallocSpace::WalkCallback, void* arg) = 0;
virtual ~LargeObjectSpace() {}
- uint64_t GetBytesAllocated() const {
+ uint64_t GetBytesAllocated() {
return num_bytes_allocated_;
}
- uint64_t GetObjectsAllocated() const {
+ uint64_t GetObjectsAllocated() {
return num_objects_allocated_;
}
- uint64_t GetTotalBytesAllocated() const {
+ uint64_t GetTotalBytesAllocated() {
return total_bytes_allocated_;
}
- uint64_t GetTotalObjectsAllocated() const {
+ uint64_t GetTotalObjectsAllocated() {
return total_objects_allocated_;
}
diff --git a/runtime/gc/space/space.h b/runtime/gc/space/space.h
index 68df563..6dd7952 100644
--- a/runtime/gc/space/space.h
+++ b/runtime/gc/space/space.h
@@ -146,13 +146,13 @@
class AllocSpace {
public:
// Number of bytes currently allocated.
- virtual uint64_t GetBytesAllocated() const = 0;
+ virtual uint64_t GetBytesAllocated() = 0;
// Number of objects currently allocated.
- virtual uint64_t GetObjectsAllocated() const = 0;
+ virtual uint64_t GetObjectsAllocated() = 0;
// Number of bytes allocated since the space was created.
- virtual uint64_t GetTotalBytesAllocated() const = 0;
+ virtual uint64_t GetTotalBytesAllocated() = 0;
// Number of objects allocated since the space was created.
- virtual uint64_t GetTotalObjectsAllocated() const = 0;
+ virtual uint64_t GetTotalObjectsAllocated() = 0;
// Allocate num_bytes without allowing growth. If the allocation
// succeeds, the output parameter bytes_allocated will be set to the
diff --git a/runtime/native/dalvik_system_VMRuntime.cc b/runtime/native/dalvik_system_VMRuntime.cc
index 5fc8bd5..dad6eff 100644
--- a/runtime/native/dalvik_system_VMRuntime.cc
+++ b/runtime/native/dalvik_system_VMRuntime.cc
@@ -169,10 +169,7 @@
// Trim the managed heap.
gc::Heap* heap = Runtime::Current()->GetHeap();
- gc::space::DlMallocSpace* alloc_space = heap->GetAllocSpace();
- size_t alloc_space_size = alloc_space->Size();
- float managed_utilization =
- static_cast<float>(alloc_space->GetBytesAllocated()) / alloc_space_size;
+ float managed_utilization = static_cast<float>(heap->GetBytesAllocated()) / heap->GetTotalMemory();
size_t managed_reclaimed = heap->Trim();
uint64_t gc_heap_end_ns = NanoTime();