diff options
| -rw-r--r-- | runtime/gc/heap-inl.h | 231 | ||||
| -rw-r--r-- | runtime/gc/heap.cc | 33 | ||||
| -rw-r--r-- | runtime/gc/heap.h | 5 | ||||
| -rw-r--r-- | runtime/thread.h | 39 |
4 files changed, 166 insertions, 142 deletions
diff --git a/runtime/gc/heap-inl.h b/runtime/gc/heap-inl.h index 04632ef96b..4499342ddb 100644 --- a/runtime/gc/heap-inl.h +++ b/runtime/gc/heap-inl.h @@ -65,141 +65,136 @@ inline mirror::Object* Heap::AllocObjectWithAllocator(Thread* self, HandleWrapperObjPtr<mirror::Class> h = hs.NewHandleWrapper(&klass); self->PoisonObjectPointers(); } - auto send_pre_object_allocated = [&]() REQUIRES_SHARED(Locks::mutator_lock_) - ACQUIRE(Roles::uninterruptible_) { + auto pre_object_allocated = [&]() REQUIRES_SHARED(Locks::mutator_lock_) + REQUIRES(!Roles::uninterruptible_) { if constexpr (kInstrumented) { - AllocationListener* l = nullptr; - l = alloc_listener_.load(std::memory_order_seq_cst); + AllocationListener* l = alloc_listener_.load(std::memory_order_seq_cst); if (UNLIKELY(l != nullptr) && UNLIKELY(l->HasPreAlloc())) { StackHandleScope<1> hs(self); HandleWrapperObjPtr<mirror::Class> h_klass(hs.NewHandleWrapper(&klass)); l->PreObjectAllocated(self, h_klass, &byte_count); } } - return self->StartAssertNoThreadSuspension("Called PreObjectAllocated, no suspend until alloc"); }; - // Do the initial pre-alloc - const char* old_cause = send_pre_object_allocated(); - // We shouldn't have any NoThreadSuspension here! - DCHECK(old_cause == nullptr) << old_cause; - - // Need to check that we aren't the large object allocator since the large object allocation code - // path includes this function. If we didn't check we would have an infinite loop. ObjPtr<mirror::Object> obj; - if (kCheckLargeObject && UNLIKELY(ShouldAllocLargeObject(klass, byte_count))) { - // AllocLargeObject can suspend and will recall PreObjectAllocated if needed. - self->EndAssertNoThreadSuspension(old_cause); - obj = AllocLargeObject<kInstrumented, PreFenceVisitor>(self, &klass, byte_count, - pre_fence_visitor); - if (obj != nullptr) { - return obj.Ptr(); - } else { - // There should be an OOM exception, since we are retrying, clear it. - self->ClearException(); - } - // If the large object allocation failed, try to use the normal spaces (main space, - // non moving space). This can happen if there is significant virtual address space - // fragmentation. - // We need to send the PreObjectAllocated again, we might have suspended during our failure. - old_cause = send_pre_object_allocated(); - } // bytes allocated for the (individual) object. size_t bytes_allocated; size_t usable_size; size_t new_num_bytes_allocated = 0; - if (IsTLABAllocator(allocator)) { - byte_count = RoundUp(byte_count, space::BumpPointerSpace::kAlignment); - } - // If we have a thread local allocation we don't need to update bytes allocated. - if (IsTLABAllocator(allocator) && byte_count <= self->TlabSize()) { - obj = self->AllocTlab(byte_count); - DCHECK(obj != nullptr) << "AllocTlab can't fail"; - obj->SetClass(klass); - if (kUseBakerReadBarrier) { - obj->AssertReadBarrierState(); + { + // Do the initial pre-alloc + pre_object_allocated(); + ScopedAssertNoThreadSuspension ants("Called PreObjectAllocated, no suspend until alloc"); + + // Need to check that we aren't the large object allocator since the large object allocation + // code path includes this function. If we didn't check we would have an infinite loop. + if (kCheckLargeObject && UNLIKELY(ShouldAllocLargeObject(klass, byte_count))) { + // AllocLargeObject can suspend and will recall PreObjectAllocated if needed. + ScopedAllowThreadSuspension ats; + obj = AllocLargeObject<kInstrumented, PreFenceVisitor>(self, &klass, byte_count, + pre_fence_visitor); + if (obj != nullptr) { + return obj.Ptr(); + } + // There should be an OOM exception, since we are retrying, clear it. + self->ClearException(); + + // If the large object allocation failed, try to use the normal spaces (main space, + // non moving space). This can happen if there is significant virtual address space + // fragmentation. + pre_object_allocated(); } - bytes_allocated = byte_count; - usable_size = bytes_allocated; - no_suspend_pre_fence_visitor(obj, usable_size); - QuasiAtomic::ThreadFenceForConstructor(); - self->EndAssertNoThreadSuspension(old_cause); - } else if ( - !kInstrumented && allocator == kAllocatorTypeRosAlloc && - (obj = rosalloc_space_->AllocThreadLocal(self, byte_count, &bytes_allocated)) != nullptr && - LIKELY(obj != nullptr)) { - DCHECK(!is_running_on_memory_tool_); - obj->SetClass(klass); - if (kUseBakerReadBarrier) { - obj->AssertReadBarrierState(); + if (IsTLABAllocator(allocator)) { + byte_count = RoundUp(byte_count, space::BumpPointerSpace::kAlignment); } - usable_size = bytes_allocated; - no_suspend_pre_fence_visitor(obj, usable_size); - QuasiAtomic::ThreadFenceForConstructor(); - self->EndAssertNoThreadSuspension(old_cause); - } else { - // Bytes allocated that includes bulk thread-local buffer allocations in addition to direct - // non-TLAB object allocations. - size_t bytes_tl_bulk_allocated = 0u; - obj = TryToAllocate<kInstrumented, false>(self, allocator, byte_count, &bytes_allocated, - &usable_size, &bytes_tl_bulk_allocated); - if (UNLIKELY(obj == nullptr)) { - // AllocateInternalWithGc can cause thread suspension, if someone instruments the entrypoints - // or changes the allocator in a suspend point here, we need to retry the allocation. - // It will send the pre-alloc event again. - self->EndAssertNoThreadSuspension(old_cause); - obj = AllocateInternalWithGc(self, - allocator, - kInstrumented, - byte_count, - &bytes_allocated, - &usable_size, - &bytes_tl_bulk_allocated, - &klass, - &old_cause); - if (obj == nullptr) { - self->EndAssertNoThreadSuspension(old_cause); - // The only way that we can get a null return if there is no pending exception is if the - // allocator or instrumentation changed. - if (!self->IsExceptionPending()) { - // AllocObject will pick up the new allocator type, and instrumented as true is the safe - // default. - return AllocObject</*kInstrumented=*/true>(self, - klass, - byte_count, - pre_fence_visitor); + // If we have a thread local allocation we don't need to update bytes allocated. + if (IsTLABAllocator(allocator) && byte_count <= self->TlabSize()) { + obj = self->AllocTlab(byte_count); + DCHECK(obj != nullptr) << "AllocTlab can't fail"; + obj->SetClass(klass); + if (kUseBakerReadBarrier) { + obj->AssertReadBarrierState(); + } + bytes_allocated = byte_count; + usable_size = bytes_allocated; + no_suspend_pre_fence_visitor(obj, usable_size); + QuasiAtomic::ThreadFenceForConstructor(); + } else if ( + !kInstrumented && allocator == kAllocatorTypeRosAlloc && + (obj = rosalloc_space_->AllocThreadLocal(self, byte_count, &bytes_allocated)) != nullptr && + LIKELY(obj != nullptr)) { + DCHECK(!is_running_on_memory_tool_); + obj->SetClass(klass); + if (kUseBakerReadBarrier) { + obj->AssertReadBarrierState(); + } + usable_size = bytes_allocated; + no_suspend_pre_fence_visitor(obj, usable_size); + QuasiAtomic::ThreadFenceForConstructor(); + } else { + // Bytes allocated that includes bulk thread-local buffer allocations in addition to direct + // non-TLAB object allocations. + size_t bytes_tl_bulk_allocated = 0u; + obj = TryToAllocate<kInstrumented, false>(self, allocator, byte_count, &bytes_allocated, + &usable_size, &bytes_tl_bulk_allocated); + if (UNLIKELY(obj == nullptr)) { + // AllocateInternalWithGc can cause thread suspension, if someone instruments the + // entrypoints or changes the allocator in a suspend point here, we need to retry the + // allocation. It will send the pre-alloc event again. + obj = AllocateInternalWithGc(self, + allocator, + kInstrumented, + byte_count, + &bytes_allocated, + &usable_size, + &bytes_tl_bulk_allocated, + &klass); + if (obj == nullptr) { + // The only way that we can get a null return if there is no pending exception is if the + // allocator or instrumentation changed. + if (!self->IsExceptionPending()) { + // Since we are restarting, allow thread suspension. + ScopedAllowThreadSuspension ats; + // AllocObject will pick up the new allocator type, and instrumented as true is the safe + // default. + return AllocObject</*kInstrumented=*/true>(self, + klass, + byte_count, + pre_fence_visitor); + } + return nullptr; } - return nullptr; } - } - DCHECK_GT(bytes_allocated, 0u); - DCHECK_GT(usable_size, 0u); - obj->SetClass(klass); - if (kUseBakerReadBarrier) { - obj->AssertReadBarrierState(); - } - if (collector::SemiSpace::kUseRememberedSet && UNLIKELY(allocator == kAllocatorTypeNonMoving)) { - // (Note this if statement will be constant folded away for the fast-path quick entry - // points.) Because SetClass() has no write barrier, the GC may need a write barrier in the - // case the object is non movable and points to a recently allocated movable class. - WriteBarrier::ForFieldWrite(obj, mirror::Object::ClassOffset(), klass); - } - no_suspend_pre_fence_visitor(obj, usable_size); - QuasiAtomic::ThreadFenceForConstructor(); - self->EndAssertNoThreadSuspension(old_cause); - if (bytes_tl_bulk_allocated > 0) { - size_t num_bytes_allocated_before = - num_bytes_allocated_.fetch_add(bytes_tl_bulk_allocated, std::memory_order_relaxed); - new_num_bytes_allocated = num_bytes_allocated_before + bytes_tl_bulk_allocated; - // Only trace when we get an increase in the number of bytes allocated. This happens when - // obtaining a new TLAB and isn't often enough to hurt performance according to golem. - if (region_space_) { - // With CC collector, during a GC cycle, the heap usage increases as - // there are two copies of evacuated objects. Therefore, add evac-bytes - // to the heap size. When the GC cycle is not running, evac-bytes - // are 0, as required. - TraceHeapSize(new_num_bytes_allocated + region_space_->EvacBytes()); - } else { - TraceHeapSize(new_num_bytes_allocated); + DCHECK_GT(bytes_allocated, 0u); + DCHECK_GT(usable_size, 0u); + obj->SetClass(klass); + if (kUseBakerReadBarrier) { + obj->AssertReadBarrierState(); + } + if (collector::SemiSpace::kUseRememberedSet && + UNLIKELY(allocator == kAllocatorTypeNonMoving)) { + // (Note this if statement will be constant folded away for the fast-path quick entry + // points.) Because SetClass() has no write barrier, the GC may need a write barrier in the + // case the object is non movable and points to a recently allocated movable class. + WriteBarrier::ForFieldWrite(obj, mirror::Object::ClassOffset(), klass); + } + no_suspend_pre_fence_visitor(obj, usable_size); + QuasiAtomic::ThreadFenceForConstructor(); + if (bytes_tl_bulk_allocated > 0) { + size_t num_bytes_allocated_before = + num_bytes_allocated_.fetch_add(bytes_tl_bulk_allocated, std::memory_order_relaxed); + new_num_bytes_allocated = num_bytes_allocated_before + bytes_tl_bulk_allocated; + // Only trace when we get an increase in the number of bytes allocated. This happens when + // obtaining a new TLAB and isn't often enough to hurt performance according to golem. + if (region_space_) { + // With CC collector, during a GC cycle, the heap usage increases as + // there are two copies of evacuated objects. Therefore, add evac-bytes + // to the heap size. When the GC cycle is not running, evac-bytes + // are 0, as required. + TraceHeapSize(new_num_bytes_allocated + region_space_->EvacBytes()); + } else { + TraceHeapSize(new_num_bytes_allocated); + } } } } diff --git a/runtime/gc/heap.cc b/runtime/gc/heap.cc index b462e294ba..4c069b3fa3 100644 --- a/runtime/gc/heap.cc +++ b/runtime/gc/heap.cc @@ -1724,8 +1724,7 @@ mirror::Object* Heap::AllocateInternalWithGc(Thread* self, size_t* bytes_allocated, size_t* usable_size, size_t* bytes_tl_bulk_allocated, - ObjPtr<mirror::Class>* klass, - /*out*/const char** old_no_thread_suspend_cause) { + ObjPtr<mirror::Class>* klass) { bool was_default_allocator = allocator == GetCurrentAllocator(); // Make sure there is no pending exception since we may need to throw an OOME. self->AssertNoPendingException(); @@ -1734,24 +1733,17 @@ mirror::Object* Heap::AllocateInternalWithGc(Thread* self, StackHandleScope<1> hs(self); HandleWrapperObjPtr<mirror::Class> h_klass(hs.NewHandleWrapper(klass)); - auto release_no_suspend = [&]() RELEASE(Roles::uninterruptible_) { - self->EndAssertNoThreadSuspension(*old_no_thread_suspend_cause); - }; - auto send_object_pre_alloc = [&]() ACQUIRE(Roles::uninterruptible_) - REQUIRES_SHARED(Locks::mutator_lock_) { - if (UNLIKELY(instrumented)) { - AllocationListener* l = nullptr; - l = alloc_listener_.load(std::memory_order_seq_cst); - if (UNLIKELY(l != nullptr) && UNLIKELY(l->HasPreAlloc())) { - l->PreObjectAllocated(self, h_klass, &alloc_size); + auto send_object_pre_alloc = [&]() REQUIRES_SHARED(Locks::mutator_lock_) { + if (UNLIKELY(instrumented)) { + AllocationListener* l = alloc_listener_.load(std::memory_order_seq_cst); + if (UNLIKELY(l != nullptr) && UNLIKELY(l->HasPreAlloc())) { + l->PreObjectAllocated(self, h_klass, &alloc_size); + } } - } - *old_no_thread_suspend_cause = - self->StartAssertNoThreadSuspension("Called PreObjectAllocated, no suspend until alloc"); -}; + }; #define PERFORM_SUSPENDING_OPERATION(op) \ [&]() REQUIRES(Roles::uninterruptible_) REQUIRES_SHARED(Locks::mutator_lock_) { \ - release_no_suspend(); \ + ScopedAllowThreadSuspension ats; \ auto res = (op); \ send_object_pre_alloc(); \ return res; \ @@ -1759,7 +1751,8 @@ mirror::Object* Heap::AllocateInternalWithGc(Thread* self, // The allocation failed. If the GC is running, block until it completes, and then retry the // allocation. - collector::GcType last_gc = WaitForGcToComplete(kGcCauseForAlloc, self); + collector::GcType last_gc = + PERFORM_SUSPENDING_OPERATION(WaitForGcToComplete(kGcCauseForAlloc, self)); // If we were the default allocator but the allocator changed while we were suspended, // abort the allocation. // We just waited, call the pre-alloc again. @@ -1896,10 +1889,8 @@ mirror::Object* Heap::AllocateInternalWithGc(Thread* self, #undef PERFORM_SUSPENDING_OPERATION // If the allocation hasn't succeeded by this point, throw an OOM error. if (ptr == nullptr) { - release_no_suspend(); + ScopedAllowThreadSuspension ats; ThrowOutOfMemoryError(self, alloc_size, allocator); - *old_no_thread_suspend_cause = - self->StartAssertNoThreadSuspension("Failed allocation fallback"); } return ptr; } diff --git a/runtime/gc/heap.h b/runtime/gc/heap.h index 6f6cfd1785..f8bbe10a72 100644 --- a/runtime/gc/heap.h +++ b/runtime/gc/heap.h @@ -1011,10 +1011,9 @@ class Heap { size_t* bytes_allocated, size_t* usable_size, size_t* bytes_tl_bulk_allocated, - ObjPtr<mirror::Class>* klass, - /*out*/const char** old_no_thread_suspend_cause) + ObjPtr<mirror::Class>* klass) REQUIRES(!Locks::thread_suspend_count_lock_, !*gc_complete_lock_, !*pending_task_lock_) - ACQUIRE(Roles::uninterruptible_) + REQUIRES(Roles::uninterruptible_) REQUIRES_SHARED(Locks::mutator_lock_); // Allocate into a specific space. diff --git a/runtime/thread.h b/runtime/thread.h index 16ca2aa4f6..29375e51f4 100644 --- a/runtime/thread.h +++ b/runtime/thread.h @@ -355,6 +355,21 @@ class Thread { Roles::uninterruptible_.Release(); // No-op. } + // End region where no thread suspension is expected. Returns the current open region in case we + // want to reopen it. Used for ScopedAllowThreadSuspension. Not supported if no_thread_suspension + // is larger than one. + const char* EndAssertNoThreadSuspension() RELEASE(Roles::uninterruptible_) WARN_UNUSED { + const char* ret = nullptr; + if (kIsDebugBuild) { + CHECK_EQ(tls32_.no_thread_suspension, 1u); + tls32_.no_thread_suspension--; + ret = tlsPtr_.last_no_thread_suspension_cause; + tlsPtr_.last_no_thread_suspension_cause = nullptr; + } + Roles::uninterruptible_.Release(); // No-op. + return ret; + } + void AssertThreadSuspensionIsAllowable(bool check_locks = true) const; // Return true if thread suspension is allowable. @@ -1910,6 +1925,30 @@ class SCOPED_CAPABILITY ScopedAssertNoThreadSuspension { const char* old_cause_; }; +class ScopedAllowThreadSuspension { + public: + ALWAYS_INLINE ScopedAllowThreadSuspension() RELEASE(Roles::uninterruptible_) { + if (kIsDebugBuild) { + self_ = Thread::Current(); + old_cause_ = self_->EndAssertNoThreadSuspension(); + } else { + Roles::uninterruptible_.Release(); // No-op. + } + } + ALWAYS_INLINE ~ScopedAllowThreadSuspension() ACQUIRE(Roles::uninterruptible_) { + if (kIsDebugBuild) { + CHECK(self_->StartAssertNoThreadSuspension(old_cause_) == nullptr); + } else { + Roles::uninterruptible_.Acquire(); // No-op. + } + } + + private: + Thread* self_; + const char* old_cause_; +}; + + class ScopedStackedShadowFramePusher { public: ScopedStackedShadowFramePusher(Thread* self, ShadowFrame* sf, StackedShadowFrameType type) |