Add reserve area to allocation stacks.
This fixes an issue with heap verification which was caused when
the allocation stack overflowed. This resulted in heap verification
failures since we were storing the newly allocated object in a
handle scope without having it be live either in the live bitmap
or allocation stack. We now push the object in the reserve area
before we do a GC due to allocation stack overflow.
Change-Id: I83b42c4b3250d7eaab1b49e53066e21c8656a740
diff --git a/runtime/gc/heap.cc b/runtime/gc/heap.cc
index fdc4367..a6093ca 100644
--- a/runtime/gc/heap.cc
+++ b/runtime/gc/heap.cc
@@ -84,9 +84,14 @@
static constexpr double kStickyGcThroughputAdjustment = 1.0;
// Whether or not we use the free list large object space.
static constexpr bool kUseFreeListSpaceForLOS = false;
-// Whtehr or not we compact the zygote in PreZygoteFork.
+// Whether or not we compact the zygote in PreZygoteFork.
static constexpr bool kCompactZygote = kMovingCollector;
static constexpr size_t kNonMovingSpaceCapacity = 64 * MB;
+// How many reserve entries are at the end of the allocation stack, these are only needed if the
+// allocation stack overflows.
+static constexpr size_t kAllocationStackReserveSize = 1024;
+// Default mark stack size in bytes.
+static const size_t kDefaultMarkStackSize = 64 * KB;
Heap::Heap(size_t initial_size, size_t growth_limit, size_t min_free, size_t max_free,
double target_utilization, double foreground_heap_growth_multiplier, size_t capacity,
@@ -295,13 +300,13 @@
// TODO: Count objects in the image space here.
num_bytes_allocated_.StoreRelaxed(0);
- // Default mark stack size in bytes.
- static const size_t default_mark_stack_size = 64 * KB;
- mark_stack_.reset(accounting::ObjectStack::Create("mark stack", default_mark_stack_size));
- allocation_stack_.reset(accounting::ObjectStack::Create("allocation stack",
- max_allocation_stack_size_));
- live_stack_.reset(accounting::ObjectStack::Create("live stack",
- max_allocation_stack_size_));
+ mark_stack_.reset(accounting::ObjectStack::Create("mark stack", kDefaultMarkStackSize,
+ kDefaultMarkStackSize));
+ const size_t alloc_stack_capacity = max_allocation_stack_size_ + kAllocationStackReserveSize;
+ allocation_stack_.reset(accounting::ObjectStack::Create(
+ "allocation stack", max_allocation_stack_size_, alloc_stack_capacity));
+ live_stack_.reset(accounting::ObjectStack::Create(
+ "live stack", max_allocation_stack_size_, alloc_stack_capacity));
// It's still too early to take a lock because there are no threads yet, but we can create locks
// now. We don't create it earlier to make it clear that you can't use locks during heap
@@ -2035,6 +2040,43 @@
const bool verify_referent_;
};
+void Heap::PushOnAllocationStackWithInternalGC(Thread* self, mirror::Object** obj) {
+ // Slow path, the allocation stack push back must have already failed.
+ DCHECK(!allocation_stack_->AtomicPushBack(*obj));
+ do {
+ // TODO: Add handle VerifyObject.
+ StackHandleScope<1> hs(self);
+ HandleWrapper<mirror::Object> wrapper(hs.NewHandleWrapper(obj));
+ // Push our object into the reserve region of the allocaiton stack. This is only required due
+ // to heap verification requiring that roots are live (either in the live bitmap or in the
+ // allocation stack).
+ CHECK(allocation_stack_->AtomicPushBackIgnoreGrowthLimit(*obj));
+ CollectGarbageInternal(collector::kGcTypeSticky, kGcCauseForAlloc, false);
+ } while (!allocation_stack_->AtomicPushBack(*obj));
+}
+
+void Heap::PushOnThreadLocalAllocationStackWithInternalGC(Thread* self, mirror::Object** obj) {
+ // Slow path, the allocation stack push back must have already failed.
+ DCHECK(!self->PushOnThreadLocalAllocationStack(*obj));
+ mirror::Object** start_address;
+ mirror::Object** end_address;
+ while (!allocation_stack_->AtomicBumpBack(kThreadLocalAllocationStackSize, &start_address,
+ &end_address)) {
+ // TODO: Add handle VerifyObject.
+ StackHandleScope<1> hs(self);
+ HandleWrapper<mirror::Object> wrapper(hs.NewHandleWrapper(obj));
+ // Push our object into the reserve region of the allocaiton stack. This is only required due
+ // to heap verification requiring that roots are live (either in the live bitmap or in the
+ // allocation stack).
+ CHECK(allocation_stack_->AtomicPushBackIgnoreGrowthLimit(*obj));
+ // Push into the reserve allocation stack.
+ CollectGarbageInternal(collector::kGcTypeSticky, kGcCauseForAlloc, false);
+ }
+ self->SetThreadLocalAllocationStack(start_address, end_address);
+ // Retry on the new thread-local allocation stack.
+ CHECK(self->PushOnThreadLocalAllocationStack(*obj)); // Must succeed.
+}
+
// Must do this with mutators suspended since we are directly accessing the allocation stacks.
size_t Heap::VerifyHeapReferences(bool verify_referents) {
Thread* self = Thread::Current();