summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
author Lokesh Gidra <lokeshgidra@google.com> 2024-07-19 19:31:10 +0000
committer Lokesh Gidra <lokeshgidra@google.com> 2024-07-22 18:20:21 +0000
commit20f6c15aed1933a7a936b51b16ae6a630ba4ef44 (patch)
treec8a2ee047a708872d3ff4371227aba92050c7307
parentb2f1766aeb7b442a1cc821f8dcda44a06fab7f85 (diff)
Remove minor-fault related code from CMC GC
No chance of using it in the future. Test: art/test/testrunner/testrunner.py --host Change-Id: I6be552cb44de60032cbff3522bd60b8144de46db
-rw-r--r--runtime/base/gc_visited_arena_pool.cc22
-rw-r--r--runtime/base/gc_visited_arena_pool.h5
-rw-r--r--runtime/gc/collector/mark_compact.cc782
-rw-r--r--runtime/gc/collector/mark_compact.h58
4 files changed, 104 insertions, 763 deletions
diff --git a/runtime/base/gc_visited_arena_pool.cc b/runtime/base/gc_visited_arena_pool.cc
index 57606d11a6..89e4d7c65a 100644
--- a/runtime/base/gc_visited_arena_pool.cc
+++ b/runtime/base/gc_visited_arena_pool.cc
@@ -49,25 +49,9 @@ TrackedArena::TrackedArena(uint8_t* start, size_t size, bool pre_zygote_fork, bo
}
}
-void TrackedArena::ReleasePages(uint8_t* begin, size_t size, bool pre_zygote_fork) {
- DCHECK_ALIGNED_PARAM(begin, gPageSize);
- // Userfaultfd GC uses MAP_SHARED mappings for linear-alloc and therefore
- // MADV_DONTNEED will not free the pages from page cache. Therefore use
- // MADV_REMOVE instead, which is meant for this purpose.
- // Arenas allocated pre-zygote fork are private anonymous and hence must be
- // released using MADV_DONTNEED.
- if (!gUseUserfaultfd || pre_zygote_fork ||
- (madvise(begin, size, MADV_REMOVE) == -1 && errno == EINVAL)) {
- // MADV_REMOVE fails if invoked on anonymous mapping, which could happen
- // if the arena is released before userfaultfd-GC starts using memfd. So
- // use MADV_DONTNEED.
- ZeroAndReleaseMemory(begin, size);
- }
-}
-
void TrackedArena::Release() {
if (bytes_allocated_ > 0) {
- ReleasePages(Begin(), Size(), pre_zygote_fork_);
+ ZeroAndReleaseMemory(Begin(), Size());
if (first_obj_array_.get() != nullptr) {
std::fill_n(first_obj_array_.get(), DivideByPageSize(Size()), nullptr);
}
@@ -210,7 +194,7 @@ void GcVisitedArenaPool::FreeSingleObjArena(uint8_t* addr) {
if (zygote_arena) {
free(addr);
} else {
- TrackedArena::ReleasePages(addr, size, /*pre_zygote_fork=*/false);
+ ZeroAndReleaseMemory(addr, size);
WriterMutexLock wmu(self, lock_);
FreeRangeLocked(addr, size);
}
@@ -383,7 +367,7 @@ void GcVisitedArenaPool::FreeArenaChain(Arena* first) {
for (auto& iter : free_ranges) {
// No need to madvise pre-zygote-fork arenas as they will munmapped below.
if (!std::get<2>(iter)) {
- TrackedArena::ReleasePages(std::get<0>(iter), std::get<1>(iter), /*pre_zygote_fork=*/false);
+ ZeroAndReleaseMemory(std::get<0>(iter), std::get<1>(iter));
}
}
diff --git a/runtime/base/gc_visited_arena_pool.h b/runtime/base/gc_visited_arena_pool.h
index 4caaaf4b8f..ac98563d67 100644
--- a/runtime/base/gc_visited_arena_pool.h
+++ b/runtime/base/gc_visited_arena_pool.h
@@ -112,11 +112,6 @@ class TrackedArena final : public Arena {
}
bool IsWaitingForDeletion() const { return waiting_for_deletion_; }
- // Madvise the pages in the given range. 'begin' is expected to be page
- // aligned.
- // TODO: Remove this once we remove the shmem (minor-fault) code in
- // userfaultfd GC and directly use ZeroAndReleaseMemory().
- static void ReleasePages(uint8_t* begin, size_t size, bool pre_zygote_fork);
void Release() override;
bool IsPreZygoteForkArena() const { return pre_zygote_fork_; }
bool IsSingleObjectArena() const { return first_obj_array_.get() == nullptr; }
diff --git a/runtime/gc/collector/mark_compact.cc b/runtime/gc/collector/mark_compact.cc
index b8e24c550a..86a6d8864b 100644
--- a/runtime/gc/collector/mark_compact.cc
+++ b/runtime/gc/collector/mark_compact.cc
@@ -397,11 +397,6 @@ bool MarkCompact::CreateUserfaultfd(bool post_fork) {
CHECK_EQ(gUffdFeatures & kUffdFeaturesForSigbus, kUffdFeaturesForSigbus);
api.features |= kUffdFeaturesForSigbus;
}
- if (uffd_minor_fault_supported_) {
- // NOTE: This option is currently disabled.
- CHECK_EQ(gUffdFeatures & kUffdFeaturesForMinorFault, kUffdFeaturesForMinorFault);
- api.features |= kUffdFeaturesForMinorFault;
- }
CHECK_EQ(ioctl(uffd_, UFFDIO_API, &api), 0)
<< "ioctl_userfaultfd: API: " << strerror(errno);
}
@@ -464,30 +459,17 @@ MarkCompact::MarkCompact(Heap* heap)
moving_space_bitmap_(bump_pointer_space_->GetMarkBitmap()),
moving_space_begin_(bump_pointer_space_->Begin()),
moving_space_end_(bump_pointer_space_->Limit()),
- moving_to_space_fd_(kFdUnused),
- moving_from_space_fd_(kFdUnused),
uffd_(kFdUnused),
sigbus_in_progress_count_(kSigbusCounterCompactionDoneMask),
compaction_in_progress_count_(0),
thread_pool_counter_(0),
compacting_(false),
uffd_initialized_(false),
- uffd_minor_fault_supported_(false),
use_uffd_sigbus_(IsSigbusFeatureAvailable()),
- minor_fault_initialized_(false),
- map_linear_alloc_shared_(false),
clamp_info_map_status_(ClampInfoStatus::kClampInfoNotDone) {
if (kIsDebugBuild) {
updated_roots_.reset(new std::unordered_set<void*>());
}
- // TODO: When using minor-fault feature, the first GC after zygote-fork
- // requires mapping the linear-alloc again with MAP_SHARED. This leaves a
- // gap for suspended threads to access linear-alloc when it's empty (after
- // mremap) and not yet userfaultfd registered. This cannot be fixed by merely
- // doing uffd registration first. For now, just assert that we are not using
- // minor-fault. Eventually, a cleanup of linear-alloc update logic to only
- // use private anonymous would be ideal.
- CHECK(!uffd_minor_fault_supported_);
uint8_t* moving_space_begin = bump_pointer_space_->Begin();
// TODO: Depending on how the bump-pointer space move is implemented. If we
@@ -535,26 +517,6 @@ MarkCompact::MarkCompact(Heap* heap)
from_space_begin_ = from_space_map_.Begin();
}
- // In some cases (32-bit or kObjPtrPoisoning) it's too much to ask for 3
- // heap-sized mappings in low-4GB. So tolerate failure here by attempting to
- // mmap again right before the compaction pause. And if even that fails, then
- // running the GC cycle in copy-mode rather than minor-fault.
- //
- // This map doesn't have to be aligned to 2MB as we don't mremap on it.
- if (!kObjPtrPoisoning && uffd_minor_fault_supported_) {
- // We need this map only if minor-fault feature is supported. But in that case
- // don't create the mapping if obj-ptr poisoning is enabled as then the mapping
- // has to be created in low_4gb. Doing this here rather than later causes the
- // Dex2oatImageTest.TestExtension gtest to fail in 64-bit platforms.
- shadow_to_space_map_ = MemMap::MapAnonymous("Concurrent mark-compact moving-space shadow",
- moving_space_size,
- PROT_NONE,
- /*low_4gb=*/false,
- &err_msg);
- if (!shadow_to_space_map_.IsValid()) {
- LOG(WARNING) << "Failed to allocate concurrent mark-compact moving-space shadow: " << err_msg;
- }
- }
const size_t num_pages =
1 + (use_uffd_sigbus_ ? kMutatorCompactionBufferCount :
std::min(heap_->GetParallelGCThreadCount(), kMaxNumUffdWorkers));
@@ -599,18 +561,6 @@ void MarkCompact::AddLinearAllocSpaceData(uint8_t* begin, size_t len) {
DCHECK_ALIGNED_PARAM(len, gPageSize);
DCHECK_GE(len, Heap::GetPMDSize());
size_t alignment = Heap::BestPageTableAlignment(len);
- bool is_shared = false;
- // We use MAP_SHARED on non-zygote processes for leveraging userfaultfd's minor-fault feature.
- if (map_linear_alloc_shared_) {
- void* ret = mmap(begin,
- len,
- PROT_READ | PROT_WRITE,
- MAP_ANONYMOUS | MAP_SHARED | MAP_FIXED,
- /*fd=*/-1,
- /*offset=*/0);
- CHECK_EQ(ret, begin) << "mmap failed: " << strerror(errno);
- is_shared = true;
- }
std::string err_msg;
MemMap shadow(MemMap::MapAnonymousAligned("linear-alloc shadow map",
len,
@@ -632,11 +582,8 @@ void MarkCompact::AddLinearAllocSpaceData(uint8_t* begin, size_t len) {
LOG(FATAL) << "Failed to allocate linear-alloc page-status shadow map: " << err_msg;
UNREACHABLE();
}
- linear_alloc_spaces_data_.emplace_back(std::forward<MemMap>(shadow),
- std::forward<MemMap>(page_status_map),
- begin,
- begin + len,
- is_shared);
+ linear_alloc_spaces_data_.emplace_back(
+ std::forward<MemMap>(shadow), std::forward<MemMap>(page_status_map), begin, begin + len);
}
void MarkCompact::ClampGrowthLimit(size_t new_capacity) {
@@ -655,11 +602,6 @@ void MarkCompact::ClampGrowthLimit(size_t new_capacity) {
if (kClampFromSpace) {
from_space_map_.SetSize(new_capacity);
}
- // NOTE: We usually don't use shadow_to_space_map_ and therefore the condition will
- // mostly be false.
- if (shadow_to_space_map_.IsValid() && shadow_to_space_map_.Size() > new_capacity) {
- shadow_to_space_map_.SetSize(new_capacity);
- }
clamp_info_map_status_ = ClampInfoStatus::kClampInfoPending;
}
CHECK_EQ(moving_space_begin_, bump_pointer_space_->Begin());
@@ -1017,27 +959,17 @@ void MarkCompact::InitNonMovingSpaceFirstObjects() {
non_moving_first_objs_count_ = page_idx;
}
-bool MarkCompact::CanCompactMovingSpaceWithMinorFault() {
- size_t min_size = (moving_first_objs_count_ + black_page_count_) * gPageSize;
- return minor_fault_initialized_ && shadow_to_space_map_.IsValid() &&
- shadow_to_space_map_.Size() >= min_size;
-}
-
class MarkCompact::ConcurrentCompactionGcTask : public SelfDeletingTask {
public:
explicit ConcurrentCompactionGcTask(MarkCompact* collector, size_t idx)
: collector_(collector), index_(idx) {}
void Run([[maybe_unused]] Thread* self) override REQUIRES_SHARED(Locks::mutator_lock_) {
- if (collector_->CanCompactMovingSpaceWithMinorFault()) {
- collector_->ConcurrentCompaction<MarkCompact::kMinorFaultMode>(/*buf=*/nullptr);
- } else {
- // The passed page/buf to ConcurrentCompaction is used by the thread as a
- // gPageSize buffer for compacting and updating objects into and then
- // passing the buf to uffd ioctls.
- uint8_t* buf = collector_->compaction_buffers_map_.Begin() + index_ * gPageSize;
- collector_->ConcurrentCompaction<MarkCompact::kCopyMode>(buf);
- }
+ // The passed page/buf to ConcurrentCompaction is used by the thread as a
+ // gPageSize buffer for compacting and updating objects into and then
+ // passing the buf to uffd ioctls.
+ uint8_t* buf = collector_->compaction_buffers_map_.Begin() + index_ * gPageSize;
+ collector_->ConcurrentCompaction(buf);
}
private:
@@ -1124,208 +1056,24 @@ void MarkCompact::PrepareForCompaction() {
CHECK_EQ(ioctl(uffd_, UFFDIO_REGISTER, &uffd_register), 0)
<< "ioctl_userfaultfd: register compaction termination page: " << strerror(errno);
}
- if (!uffd_minor_fault_supported_ && shadow_to_space_map_.IsValid()) {
- // A valid shadow-map for moving space is only possible if we
- // were able to map it in the constructor. That also means that its size
- // matches the moving-space.
- CHECK_EQ(shadow_to_space_map_.Size(), bump_pointer_space_->Capacity());
- // Release the shadow map for moving-space if we don't support minor-fault
- // as it's not required.
- shadow_to_space_map_.Reset();
- }
}
// For zygote we create the thread pool each time before starting compaction,
// and get rid of it when finished. This is expected to happen rarely as
// zygote spends most of the time in native fork loop.
- if (uffd_ != kFallbackMode) {
- if (!use_uffd_sigbus_) {
- ThreadPool* pool = heap_->GetThreadPool();
- if (UNLIKELY(pool == nullptr)) {
- // On devices with 2 cores, GetParallelGCThreadCount() will return 1,
- // which is desired number of workers on such devices.
- heap_->CreateThreadPool(std::min(heap_->GetParallelGCThreadCount(), kMaxNumUffdWorkers));
- pool = heap_->GetThreadPool();
- }
- size_t num_threads = pool->GetThreadCount();
- thread_pool_counter_ = num_threads;
- for (size_t i = 0; i < num_threads; i++) {
- pool->AddTask(thread_running_gc_, new ConcurrentCompactionGcTask(this, i + 1));
- }
- CHECK_EQ(pool->GetTaskCount(thread_running_gc_), num_threads);
+ if (uffd_ != kFallbackMode && !use_uffd_sigbus_) {
+ ThreadPool* pool = heap_->GetThreadPool();
+ if (UNLIKELY(pool == nullptr)) {
+ // On devices with 2 cores, GetParallelGCThreadCount() will return 1,
+ // which is desired number of workers on such devices.
+ heap_->CreateThreadPool(std::min(heap_->GetParallelGCThreadCount(), kMaxNumUffdWorkers));
+ pool = heap_->GetThreadPool();
}
- /*
- * Possible scenarios for mappings:
- * A) All zygote GCs (or if minor-fault feature isn't available): uses
- * uffd's copy mode
- * 1) For moving-space ('to' space is same as the moving-space):
- * a) Private-anonymous mappings for 'to' and 'from' space are created in
- * the constructor.
- * b) In the compaction pause, we mremap(dontunmap) from 'to' space to
- * 'from' space. This results in moving all pages to 'from' space and
- * emptying the 'to' space, thereby preparing it for userfaultfd
- * registration.
- *
- * 2) For linear-alloc space:
- * a) Private-anonymous mappings for the linear-alloc and its 'shadow'
- * are created by the arena-pool.
- * b) In the compaction pause, we mremap(dontumap) with similar effect as
- * (A.1.b) above.
- *
- * B) First GC after zygote: uses uffd's copy-mode
- * 1) For moving-space:
- * a) If the mmap for shadow-map has been successful in the constructor,
- * then we remap it (mmap with MAP_FIXED) to get a shared-anonymous
- * mapping.
- * b) Else, we create two memfd and ftruncate them to the moving-space
- * size.
- * c) Same as (A.1.b)
- * d) If (B.1.a), then mremap(dontunmap) from shadow-map to
- * 'to' space. This will make both of them map to the same pages
- * e) If (B.1.b), then mmap with the first memfd in shared mode on the
- * 'to' space.
- * f) At the end of compaction, we will have moved the moving-space
- * objects to a MAP_SHARED mapping, readying it for minor-fault from next
- * GC cycle.
- *
- * 2) For linear-alloc space:
- * a) Same as (A.2.b)
- * b) mmap a shared-anonymous mapping onto the linear-alloc space.
- * c) Same as (B.1.f)
- *
- * C) All subsequent GCs: preferable minor-fault mode. But may also require
- * using copy-mode.
- * 1) For moving-space:
- * a) If the shadow-map is created and no memfd was used, then that means
- * we are using shared-anonymous. Therefore, mmap a shared-anonymous on
- * the shadow-space.
- * b) If the shadow-map is not mapped yet, then mmap one with a size
- * big enough to hold the compacted moving space. This may fail, in which
- * case we will use uffd's copy-mode.
- * c) If (b) is successful, then mmap the free memfd onto shadow-map.
- * d) Same as (A.1.b)
- * e) In compaction pause, if the shadow-map was not created, then use
- * copy-mode.
- * f) Else, if the created map is smaller than the required-size, then
- * use mremap (without dontunmap) to expand the size. If failed, then use
- * copy-mode.
- * g) Otherwise, same as (B.1.d) and use minor-fault mode.
- *
- * 2) For linear-alloc space:
- * a) Same as (A.2.b)
- * b) Use minor-fault mode
- */
- auto mmap_shadow_map = [this](int flags, int fd) {
- void* ret = mmap(shadow_to_space_map_.Begin(),
- shadow_to_space_map_.Size(),
- PROT_READ | PROT_WRITE,
- flags,
- fd,
- /*offset=*/0);
- DCHECK_NE(ret, MAP_FAILED) << "mmap for moving-space shadow failed:" << strerror(errno);
- };
- // Setup all the virtual memory ranges required for concurrent compaction.
- if (minor_fault_initialized_) {
- DCHECK(!is_zygote);
- if (UNLIKELY(!shadow_to_space_map_.IsValid())) {
- // This case happens only once on the first GC in minor-fault mode, if
- // we were unable to reserve shadow-map for moving-space in the
- // beginning.
- DCHECK_GE(moving_to_space_fd_, 0);
- // Take extra 4MB to reduce the likelihood of requiring resizing this
- // map in the pause due to black allocations.
- size_t reqd_size = std::min(moving_first_objs_count_ * gPageSize + 4 * MB,
- bump_pointer_space_->Capacity());
- // We cannot support memory-tool with shadow-map (as it requires
- // appending a redzone) in this case because the mapping may have to be expanded
- // using mremap (in KernelPreparation()), which would ignore the redzone.
- // MemMap::MapFile() appends a redzone, but MemMap::MapAnonymous() doesn't.
- std::string err_msg;
- shadow_to_space_map_ = MemMap::MapAnonymous("moving-space-shadow",
- reqd_size,
- PROT_NONE,
- /*low_4gb=*/kObjPtrPoisoning,
- &err_msg);
-
- if (shadow_to_space_map_.IsValid()) {
- CHECK(!kMemoryToolAddsRedzones || shadow_to_space_map_.GetRedzoneSize() == 0u);
- // We want to use MemMap to get low-4GB mapping, if required, but then also
- // want to have its ownership as we may grow it (in
- // KernelPreparation()). If the ownership is not taken and we try to
- // resize MemMap, then it unmaps the virtual range.
- MemMap temp = shadow_to_space_map_.TakeReservedMemory(shadow_to_space_map_.Size(),
- /*reuse*/ true);
- std::swap(temp, shadow_to_space_map_);
- DCHECK(!temp.IsValid());
- } else {
- LOG(WARNING) << "Failed to create moving space's shadow map of " << PrettySize(reqd_size)
- << " size. " << err_msg;
- }
- }
-
- if (LIKELY(shadow_to_space_map_.IsValid())) {
- int fd = moving_to_space_fd_;
- int mmap_flags = MAP_SHARED | MAP_FIXED;
- if (fd == kFdUnused) {
- // Unused moving-to-space fd means we are using anonymous shared
- // mapping.
- DCHECK_EQ(shadow_to_space_map_.Size(), bump_pointer_space_->Capacity());
- mmap_flags |= MAP_ANONYMOUS;
- fd = -1;
- }
- // If the map is smaller than required, then we'll do mremap in the
- // compaction pause to increase the size.
- mmap_shadow_map(mmap_flags, fd);
- }
-
- for (auto& data : linear_alloc_spaces_data_) {
- DCHECK_EQ(mprotect(data.shadow_.Begin(), data.shadow_.Size(), PROT_READ | PROT_WRITE), 0)
- << "mprotect failed: " << strerror(errno);
- }
- } else if (!is_zygote && uffd_minor_fault_supported_) {
- // First GC after zygote-fork. We will still use uffd's copy mode but will
- // use it to move objects to MAP_SHARED (to prepare for subsequent GCs, which
- // will use uffd's minor-fault feature).
- if (shadow_to_space_map_.IsValid() &&
- shadow_to_space_map_.Size() == bump_pointer_space_->Capacity()) {
- mmap_shadow_map(MAP_SHARED | MAP_FIXED | MAP_ANONYMOUS, /*fd=*/-1);
- } else {
- size_t size = bump_pointer_space_->Capacity();
- DCHECK_EQ(moving_to_space_fd_, kFdUnused);
- DCHECK_EQ(moving_from_space_fd_, kFdUnused);
- const char* name = bump_pointer_space_->GetName();
- moving_to_space_fd_ = memfd_create(name, MFD_CLOEXEC);
- CHECK_NE(moving_to_space_fd_, -1)
- << "memfd_create: failed for " << name << ": " << strerror(errno);
- moving_from_space_fd_ = memfd_create(name, MFD_CLOEXEC);
- CHECK_NE(moving_from_space_fd_, -1)
- << "memfd_create: failed for " << name << ": " << strerror(errno);
-
- // memfds are considered as files from resource limits point of view.
- // And the moving space could be several hundred MBs. So increase the
- // limit, if it's lower than moving-space size.
- bool rlimit_changed = false;
- rlimit rlim_read;
- CHECK_EQ(getrlimit(RLIMIT_FSIZE, &rlim_read), 0) << "getrlimit failed: " << strerror(errno);
- if (rlim_read.rlim_cur < size) {
- rlimit_changed = true;
- rlimit rlim = rlim_read;
- rlim.rlim_cur = size;
- CHECK_EQ(setrlimit(RLIMIT_FSIZE, &rlim), 0) << "setrlimit failed: " << strerror(errno);
- }
-
- // moving-space will map this fd so that we compact objects into it.
- int ret = ftruncate(moving_to_space_fd_, size);
- CHECK_EQ(ret, 0) << "ftruncate failed for moving-space:" << strerror(errno);
- ret = ftruncate(moving_from_space_fd_, size);
- CHECK_EQ(ret, 0) << "ftruncate failed for moving-space:" << strerror(errno);
-
- if (rlimit_changed) {
- // reset the rlimit to the original limits.
- CHECK_EQ(setrlimit(RLIMIT_FSIZE, &rlim_read), 0)
- << "setrlimit failed: " << strerror(errno);
- }
- }
+ size_t num_threads = pool->GetThreadCount();
+ thread_pool_counter_ = num_threads;
+ for (size_t i = 0; i < num_threads; i++) {
+ pool->AddTask(thread_running_gc_, new ConcurrentCompactionGcTask(this, i + 1));
}
+ CHECK_EQ(pool->GetTaskCount(thread_running_gc_), num_threads);
}
}
@@ -1980,104 +1728,6 @@ void MarkCompact::SlideBlackPage(mirror::Object* first_obj,
}
}
-template <bool kFirstPageMapping>
-void MarkCompact::MapProcessedPages(uint8_t* to_space_start,
- Atomic<PageState>* state_arr,
- size_t arr_idx,
- size_t arr_len) {
- CHECK(minor_fault_initialized_);
- DCHECK_LT(arr_idx, arr_len);
- DCHECK_ALIGNED_PARAM(to_space_start, gPageSize);
- // Claim all the contiguous pages, which are ready to be mapped, and then do
- // so in a single ioctl. This helps avoid the overhead of invoking syscall
- // several times and also maps the already-processed pages, avoiding
- // unnecessary faults on them.
- size_t length = kFirstPageMapping ? gPageSize : 0;
- if (kFirstPageMapping) {
- arr_idx++;
- }
- // We need to guarantee that we don't end up sucsessfully marking a later
- // page 'mapping' and then fail to mark an earlier page. To guarantee that
- // we use acq_rel order.
- for (; arr_idx < arr_len; arr_idx++, length += gPageSize) {
- PageState expected_state = PageState::kProcessed;
- if (!state_arr[arr_idx].compare_exchange_strong(
- expected_state, PageState::kProcessedAndMapping, std::memory_order_acq_rel)) {
- break;
- }
- }
- if (length > 0) {
- // Note: We need the first page to be attempted (to be mapped) by the ioctl
- // as this function is called due to some mutator thread waiting on the
- // 'to_space_start' page. Therefore, the ioctl must always be called
- // with 'to_space_start' as the 'start' address because it can bail out in
- // the middle (not attempting to map the subsequent pages) if it finds any
- // page either already mapped in between, or missing on the shadow-map.
- struct uffdio_continue uffd_continue;
- uffd_continue.range.start = reinterpret_cast<uintptr_t>(to_space_start);
- uffd_continue.range.len = length;
- uffd_continue.mode = 0;
- int ret = ioctl(uffd_, UFFDIO_CONTINUE, &uffd_continue);
- if (UNLIKELY(ret == -1 && errno == EAGAIN)) {
- // This can happen only in linear-alloc.
- DCHECK(linear_alloc_spaces_data_.end() !=
- std::find_if(linear_alloc_spaces_data_.begin(),
- linear_alloc_spaces_data_.end(),
- [to_space_start](const LinearAllocSpaceData& data) {
- return data.begin_ <= to_space_start && to_space_start < data.end_;
- }));
-
- // This could happen if userfaultfd couldn't find any pages mapped in the
- // shadow map. For instance, if there are certain (contiguous) pages on
- // linear-alloc which are allocated and have first-object set-up but have
- // not been accessed yet.
- // Bail out by setting the remaining pages' state back to kProcessed and
- // then waking up any waiting threads.
- DCHECK_GE(uffd_continue.mapped, 0);
- DCHECK_ALIGNED_PARAM(uffd_continue.mapped, gPageSize);
- DCHECK_LT(uffd_continue.mapped, static_cast<ssize_t>(length));
- if (kFirstPageMapping) {
- // In this case the first page must be mapped.
- DCHECK_GE(uffd_continue.mapped, static_cast<ssize_t>(gPageSize));
- }
- // Nobody would modify these pages' state simultaneously so only atomic
- // store is sufficient. Use 'release' order to ensure that all states are
- // modified sequentially.
- for (size_t remaining_len = length - uffd_continue.mapped; remaining_len > 0;
- remaining_len -= gPageSize) {
- arr_idx--;
- DCHECK_EQ(state_arr[arr_idx].load(std::memory_order_relaxed),
- PageState::kProcessedAndMapping);
- state_arr[arr_idx].store(PageState::kProcessed, std::memory_order_release);
- }
- uffd_continue.range.start =
- reinterpret_cast<uintptr_t>(to_space_start) + uffd_continue.mapped;
- uffd_continue.range.len = length - uffd_continue.mapped;
- ret = ioctl(uffd_, UFFDIO_WAKE, &uffd_continue.range);
- CHECK_EQ(ret, 0) << "ioctl_userfaultfd: wake failed: " << strerror(errno);
- } else {
- // We may receive ENOENT if gc-thread unregisters the
- // range behind our back, which is fine because that
- // happens only when it knows compaction is done.
- CHECK(ret == 0 || !kFirstPageMapping || errno == ENOENT)
- << "ioctl_userfaultfd: continue failed: " << strerror(errno);
- if (ret == 0) {
- DCHECK_EQ(uffd_continue.mapped, static_cast<ssize_t>(length));
- }
- }
- if (use_uffd_sigbus_) {
- // Nobody else would modify these pages' state simultaneously so atomic
- // store is sufficient.
- for (; uffd_continue.mapped > 0; uffd_continue.mapped -= gPageSize) {
- arr_idx--;
- DCHECK_EQ(state_arr[arr_idx].load(std::memory_order_relaxed),
- PageState::kProcessedAndMapping);
- state_arr[arr_idx].store(PageState::kProcessedAndMapped, std::memory_order_release);
- }
- }
- }
-}
-
template <uint32_t kYieldMax = 5, uint64_t kSleepUs = 10>
static void BackOff(uint32_t i) {
// TODO: Consider adding x86 PAUSE and/or ARM YIELD here.
@@ -2238,18 +1888,6 @@ bool MarkCompact::DoPageCompactionWithStateChange(size_t page_idx,
// Store is sufficient as no other thread would modify the status at this point.
moving_pages_status_[page_idx].store(store_val, std::memory_order_release);
}
- } else if (kMode == kMinorFaultMode) {
- expected_state = static_cast<uint8_t>(PageState::kProcessing);
- desired_state = static_cast<uint8_t>(PageState::kProcessed);
- // the CAS needs to be with release order to ensure that stores to the
- // page makes it to memory *before* other threads observe that it's
- // ready to be mapped.
- if (!moving_pages_status_[page_idx].compare_exchange_strong(
- expected_state, desired_state, std::memory_order_release)) {
- // Some mutator has requested to map the page after processing it.
- DCHECK_EQ(expected_state, static_cast<uint8_t>(PageState::kProcessingAndMapping));
- }
- UNREACHABLE();
}
return true;
} else {
@@ -2392,8 +2030,7 @@ bool MarkCompact::FreeFromSpacePages(size_t cur_page_idx, int mode, size_t end_i
DCHECK_LT(gBufferPages, kMinFromSpaceMadviseSize);
size -= gBufferPages;
uint8_t* addr = last_reclaimed_page_ - size;
- int behavior = minor_fault_initialized_ ? MADV_REMOVE : MADV_DONTNEED;
- CHECK_EQ(madvise(addr + from_space_slide_diff_, size, behavior), 0)
+ CHECK_EQ(madvise(addr + from_space_slide_diff_, size, MADV_DONTNEED), 0)
<< "madvise of from-space failed: " << strerror(errno);
last_reclaimed_page_ = addr;
cur_reclaimable_page_ = addr;
@@ -2444,10 +2081,6 @@ void MarkCompact::CompactMovingSpace(uint8_t* page) {
size_t page_status_arr_len = moving_first_objs_count_ + black_page_count_;
size_t idx = page_status_arr_len;
uint8_t* to_space_end = bump_pointer_space_->Begin() + page_status_arr_len * gPageSize;
- uint8_t* shadow_space_end = nullptr;
- if (kMode == kMinorFaultMode) {
- shadow_space_end = shadow_to_space_map_.Begin() + page_status_arr_len * gPageSize;
- }
uint8_t* pre_compact_page = black_allocations_begin_ + (black_page_count_ * gPageSize);
DCHECK(IsAlignedParam(pre_compact_page, gPageSize));
@@ -2465,10 +2098,7 @@ void MarkCompact::CompactMovingSpace(uint8_t* page) {
idx--;
pre_compact_page -= gPageSize;
to_space_end -= gPageSize;
- if (kMode == kMinorFaultMode) {
- shadow_space_end -= gPageSize;
- page = shadow_space_end;
- } else if (kMode == kFallbackMode) {
+ if (kMode == kFallbackMode) {
page = to_space_end;
}
mirror::Object* first_obj = first_objs_moving_space_[idx].AsMirrorPtr();
@@ -2501,10 +2131,7 @@ void MarkCompact::CompactMovingSpace(uint8_t* page) {
while (idx > 0) {
idx--;
to_space_end -= gPageSize;
- if (kMode == kMinorFaultMode) {
- shadow_space_end -= gPageSize;
- page = shadow_space_end;
- } else if (kMode == kFallbackMode) {
+ if (kMode == kFallbackMode) {
page = to_space_end;
} else {
DCHECK_EQ(kMode, kCopyMode);
@@ -3284,11 +2911,7 @@ void MarkCompact::CompactionPause() {
stack_low_addr_ = nullptr;
}
-void MarkCompact::KernelPrepareRangeForUffd(uint8_t* to_addr,
- uint8_t* from_addr,
- size_t map_size,
- int fd,
- uint8_t* shadow_addr) {
+void MarkCompact::KernelPrepareRangeForUffd(uint8_t* to_addr, uint8_t* from_addr, size_t map_size) {
int mremap_flags = MREMAP_MAYMOVE | MREMAP_FIXED;
if (gHaveMremapDontunmap) {
mremap_flags |= MREMAP_DONTUNMAP;
@@ -3299,31 +2922,16 @@ void MarkCompact::KernelPrepareRangeForUffd(uint8_t* to_addr,
<< "mremap to move pages failed: " << strerror(errno)
<< ". space-addr=" << reinterpret_cast<void*>(to_addr) << " size=" << PrettySize(map_size);
- if (shadow_addr != nullptr) {
- DCHECK_EQ(fd, kFdUnused);
- DCHECK(gHaveMremapDontunmap);
- ret = mremap(shadow_addr, map_size, map_size, mremap_flags, to_addr);
- CHECK_EQ(ret, static_cast<void*>(to_addr))
- << "mremap from shadow to to-space map failed: " << strerror(errno);
- } else if (!gHaveMremapDontunmap || fd > kFdUnused) {
+ if (!gHaveMremapDontunmap) {
// Without MREMAP_DONTUNMAP the source mapping is unmapped by mremap. So mmap
// the moving space again.
int mmap_flags = MAP_FIXED;
- if (fd == kFdUnused) {
- // Use MAP_FIXED_NOREPLACE so that if someone else reserves 'to_addr'
- // mapping in meantime, which can happen when MREMAP_DONTUNMAP isn't
- // available, to avoid unmapping someone else' mapping and then causing
- // crashes elsewhere.
- mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE;
- // On some platforms MAP_ANONYMOUS expects fd to be -1.
- fd = -1;
- } else if (IsValidFd(fd)) {
- mmap_flags |= MAP_SHARED;
- } else {
- DCHECK_EQ(fd, kFdSharedAnon);
- mmap_flags |= MAP_SHARED | MAP_ANONYMOUS;
- }
- ret = mmap(to_addr, map_size, PROT_READ | PROT_WRITE, mmap_flags, fd, 0);
+ // Use MAP_FIXED_NOREPLACE so that if someone else reserves 'to_addr'
+ // mapping in meantime, which can happen when MREMAP_DONTUNMAP isn't
+ // available, to avoid unmapping someone else' mapping and then causing
+ // crashes elsewhere.
+ mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE;
+ ret = mmap(to_addr, map_size, PROT_READ | PROT_WRITE, mmap_flags, -1, 0);
CHECK_EQ(ret, static_cast<void*>(to_addr))
<< "mmap for moving space failed: " << strerror(errno);
}
@@ -3333,54 +2941,10 @@ void MarkCompact::KernelPreparation() {
TimingLogger::ScopedTiming t("(Paused)KernelPreparation", GetTimings());
uint8_t* moving_space_begin = bump_pointer_space_->Begin();
size_t moving_space_size = bump_pointer_space_->Capacity();
- int mode = kCopyMode;
size_t moving_space_register_sz = (moving_first_objs_count_ + black_page_count_) * gPageSize;
DCHECK_LE(moving_space_register_sz, moving_space_size);
- if (minor_fault_initialized_) {
- if (shadow_to_space_map_.IsValid()) {
- size_t shadow_size = shadow_to_space_map_.Size();
- void* addr = shadow_to_space_map_.Begin();
- if (shadow_size < moving_space_register_sz) {
- addr = mremap(addr,
- shadow_size,
- moving_space_register_sz,
- // Don't allow moving with obj-ptr poisoning as the
- // mapping needs to be in <4GB address space.
- kObjPtrPoisoning ? 0 : MREMAP_MAYMOVE,
- /*new_address=*/nullptr);
- if (addr != MAP_FAILED) {
- // Succeeded in expanding the mapping. Update the MemMap entry for shadow map.
- MemMap temp = MemMap::MapPlaceholder(
- "moving-space-shadow", static_cast<uint8_t*>(addr), moving_space_register_sz);
- std::swap(shadow_to_space_map_, temp);
- }
- }
- if (addr != MAP_FAILED) {
- mode = kMinorFaultMode;
- } else {
- // We are not going to use shadow map. So protect it to catch any
- // potential bugs.
- DCHECK_EQ(mprotect(shadow_to_space_map_.Begin(), shadow_to_space_map_.Size(), PROT_NONE), 0)
- << "mprotect failed: " << strerror(errno);
- }
- }
- }
- bool map_shared =
- minor_fault_initialized_ || (!Runtime::Current()->IsZygote() && uffd_minor_fault_supported_);
- uint8_t* shadow_addr = nullptr;
- if (moving_to_space_fd_ == kFdUnused && map_shared) {
- DCHECK(gHaveMremapDontunmap);
- DCHECK(shadow_to_space_map_.IsValid());
- DCHECK_EQ(shadow_to_space_map_.Size(), moving_space_size);
- shadow_addr = shadow_to_space_map_.Begin();
- }
-
- KernelPrepareRangeForUffd(moving_space_begin,
- from_space_begin_,
- moving_space_size,
- moving_to_space_fd_,
- shadow_addr);
+ KernelPrepareRangeForUffd(moving_space_begin, from_space_begin_, moving_space_size);
if (IsValidFd(uffd_)) {
if (moving_space_register_sz > 0) {
@@ -3411,47 +2975,22 @@ void MarkCompact::KernelPreparation() {
madvise(moving_space_begin, pmd_size, MADV_DONTNEED);
}
// Register the moving space with userfaultfd.
- RegisterUffd(moving_space_begin, moving_space_register_sz, mode);
+ RegisterUffd(moving_space_begin, moving_space_register_sz);
}
// Prepare linear-alloc for concurrent compaction.
for (auto& data : linear_alloc_spaces_data_) {
- bool mmap_again = map_shared && !data.already_shared_;
DCHECK_EQ(static_cast<ssize_t>(data.shadow_.Size()), data.end_ - data.begin_);
// There could be threads running in suspended mode when the compaction
// pause is being executed. In order to make the userfaultfd setup atomic,
// the registration has to be done *before* moving the pages to shadow map.
- if (!mmap_again) {
- // See the comment in the constructor as to why it's conditionally done.
- RegisterUffd(data.begin_,
- data.shadow_.Size(),
- minor_fault_initialized_ ? kMinorFaultMode : kCopyMode);
- }
- KernelPrepareRangeForUffd(data.begin_,
- data.shadow_.Begin(),
- data.shadow_.Size(),
- mmap_again ? kFdSharedAnon : kFdUnused);
- if (mmap_again) {
- data.already_shared_ = true;
- RegisterUffd(data.begin_,
- data.shadow_.Size(),
- minor_fault_initialized_ ? kMinorFaultMode : kCopyMode);
- }
+ RegisterUffd(data.begin_, data.shadow_.Size());
+ KernelPrepareRangeForUffd(data.begin_, data.shadow_.Begin(), data.shadow_.Size());
}
}
- if (map_shared) {
- // Start mapping linear-alloc MAP_SHARED only after the compaction pause of
- // the first GC in non-zygote processes. This is the GC which sets up
- // mappings for using minor-fault in future. Up to this point we run
- // userfaultfd in copy-mode, which requires the mappings (of linear-alloc)
- // to be MAP_PRIVATE.
- map_linear_alloc_shared_ = true;
- }
}
-template <int kMode>
void MarkCompact::ConcurrentCompaction(uint8_t* buf) {
- DCHECK_NE(kMode, kFallbackMode);
- DCHECK(kMode != kCopyMode || buf != nullptr);
+ DCHECK(buf != nullptr);
size_t nr_moving_space_used_pages = moving_first_objs_count_ + black_page_count_;
while (true) {
struct uffd_msg msg;
@@ -3485,13 +3024,9 @@ void MarkCompact::ConcurrentCompaction(uint8_t* buf) {
}
uint8_t* fault_page = AlignDown(fault_addr, gPageSize);
if (HasAddress(reinterpret_cast<mirror::Object*>(fault_addr))) {
- ConcurrentlyProcessMovingPage<kMode>(fault_page, buf, nr_moving_space_used_pages);
- } else if (minor_fault_initialized_) {
- ConcurrentlyProcessLinearAllocPage<kMinorFaultMode>(
- fault_page, (msg.arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_MINOR) != 0);
+ ConcurrentlyProcessMovingPage(fault_page, buf, nr_moving_space_used_pages);
} else {
- ConcurrentlyProcessLinearAllocPage<kCopyMode>(
- fault_page, (msg.arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_MINOR) != 0);
+ ConcurrentlyProcessLinearAllocPage(fault_page);
}
}
}
@@ -3543,23 +3078,14 @@ bool MarkCompact::SigbusHandler(siginfo_t* info) {
Thread* self = Thread::Current();
Locks::mutator_lock_->AssertSharedHeld(self);
size_t nr_moving_space_used_pages = moving_first_objs_count_ + black_page_count_;
- if (minor_fault_initialized_) {
- ConcurrentlyProcessMovingPage<kMinorFaultMode>(
- fault_page, nullptr, nr_moving_space_used_pages);
- } else {
- ConcurrentlyProcessMovingPage<kCopyMode>(
- fault_page, self->GetThreadLocalGcBuffer(), nr_moving_space_used_pages);
- }
+ ConcurrentlyProcessMovingPage(
+ fault_page, self->GetThreadLocalGcBuffer(), nr_moving_space_used_pages);
return true;
} else {
// Find the linear-alloc space containing fault-addr
for (auto& data : linear_alloc_spaces_data_) {
if (data.begin_ <= fault_page && data.end_ > fault_page) {
- if (minor_fault_initialized_) {
- ConcurrentlyProcessLinearAllocPage<kMinorFaultMode>(fault_page, false);
- } else {
- ConcurrentlyProcessLinearAllocPage<kCopyMode>(fault_page, false);
- }
+ ConcurrentlyProcessLinearAllocPage(fault_page);
return true;
}
}
@@ -3580,7 +3106,6 @@ bool MarkCompact::SigbusHandler(siginfo_t* info) {
}
}
-template <int kMode>
void MarkCompact::ConcurrentlyProcessMovingPage(uint8_t* fault_page,
uint8_t* buf,
size_t nr_moving_space_used_pages) {
@@ -3605,8 +3130,7 @@ void MarkCompact::ConcurrentlyProcessMovingPage(uint8_t* fault_page,
uint8_t* unused_space_begin =
bump_pointer_space_->Begin() + nr_moving_space_used_pages * gPageSize;
DCHECK(IsAlignedParam(unused_space_begin, gPageSize));
- DCHECK(kMode == kCopyMode || fault_page < unused_space_begin);
- if (kMode == kCopyMode && fault_page >= unused_space_begin) {
+ if (fault_page >= unused_space_begin) {
// There is a race which allows more than one thread to install a
// zero-page. But we can tolerate that. So absorb the EEXIST returned by
// the ioctl and move on.
@@ -3676,11 +3200,7 @@ void MarkCompact::ConcurrentlyProcessMovingPage(uint8_t* fault_page,
raw_state,
static_cast<uint8_t>(PageState::kMutatorProcessing),
std::memory_order_acq_rel)) {
- if (kMode == kMinorFaultMode) {
- DCHECK_EQ(buf, nullptr);
- buf = shadow_to_space_map_.Begin() + page_idx * gPageSize;
- } else if (UNLIKELY(buf == nullptr)) {
- DCHECK_EQ(kMode, kCopyMode);
+ if (UNLIKELY(buf == nullptr)) {
uint16_t idx = compaction_buffer_counter_.fetch_add(1, std::memory_order_relaxed);
// The buffer-map is one page bigger as the first buffer is used by GC-thread.
CHECK_LE(idx, kMutatorCompactionBufferCount);
@@ -3691,8 +3211,10 @@ void MarkCompact::ConcurrentlyProcessMovingPage(uint8_t* fault_page,
if (fault_page < post_compact_end_) {
// The page has to be compacted.
- CompactPage(
- first_obj, pre_compact_offset_moving_space_[page_idx], buf, kMode == kCopyMode);
+ CompactPage(first_obj,
+ pre_compact_offset_moving_space_[page_idx],
+ buf,
+ /*needs_memset_zero=*/true);
} else {
DCHECK_NE(first_obj, nullptr);
DCHECK_GT(pre_compact_offset_moving_space_[page_idx], 0u);
@@ -3708,7 +3230,7 @@ void MarkCompact::ConcurrentlyProcessMovingPage(uint8_t* fault_page,
first_chunk_size,
pre_compact_page,
buf,
- kMode == kCopyMode);
+ /*needs_memset_zero=*/true);
}
// Nobody else would simultaneously modify this page's state so an
// atomic store is sufficient. Use 'release' order to guarantee that
@@ -3718,16 +3240,11 @@ void MarkCompact::ConcurrentlyProcessMovingPage(uint8_t* fault_page,
// to immediately map the page, so that info is not needed.
moving_pages_status_[page_idx].store(static_cast<uint8_t>(PageState::kProcessedAndMapping),
std::memory_order_release);
- if (kMode == kCopyMode) {
- CopyIoctl(fault_page, buf, gPageSize, /*return_on_contention=*/false);
- // Store is sufficient as no other thread modifies the status at this stage.
- moving_pages_status_[page_idx].store(static_cast<uint8_t>(PageState::kProcessedAndMapped),
- std::memory_order_release);
- break;
- } else {
- // We don't support minor-fault feature anymore.
- UNREACHABLE();
- }
+ CopyIoctl(fault_page, buf, gPageSize, /*return_on_contention=*/false);
+ // Store is sufficient as no other thread modifies the status at this stage.
+ moving_pages_status_[page_idx].store(static_cast<uint8_t>(PageState::kProcessedAndMapped),
+ std::memory_order_release);
+ break;
}
state = GetPageStateFromWord(raw_state);
if (state == PageState::kProcessed) {
@@ -3753,7 +3270,6 @@ bool MarkCompact::MapUpdatedLinearAllocPages(uint8_t* start_page,
size_t length,
bool free_pages,
bool single_ioctl) {
- DCHECK(!minor_fault_initialized_);
DCHECK_ALIGNED_PARAM(length, gPageSize);
Atomic<PageState>* madv_state = state;
size_t madv_len = length;
@@ -3833,9 +3349,7 @@ bool MarkCompact::MapUpdatedLinearAllocPages(uint8_t* start_page,
return true;
}
-template <int kMode>
-void MarkCompact::ConcurrentlyProcessLinearAllocPage(uint8_t* fault_page, bool is_minor_fault) {
- DCHECK(!is_minor_fault || kMode == kMinorFaultMode);
+void MarkCompact::ConcurrentlyProcessLinearAllocPage(uint8_t* fault_page) {
auto arena_iter = linear_alloc_arenas_.end();
{
TrackedArena temp_arena(fault_page);
@@ -3882,47 +3396,31 @@ void MarkCompact::ConcurrentlyProcessLinearAllocPage(uint8_t* fault_page, bool i
// shared, before the CAS is successful.
if (state_arr[page_idx].compare_exchange_strong(
state, PageState::kProcessing, std::memory_order_acquire)) {
- if (kMode == kCopyMode || is_minor_fault) {
- LinearAllocPageUpdater updater(this);
- uint8_t* first_obj = arena_iter->first->GetFirstObject(fault_page);
- // null first_obj indicates that it's a page from arena for
- // intern-table/class-table. So first object isn't required.
- if (first_obj != nullptr) {
- updater.MultiObjectArena(fault_page + diff, first_obj + diff);
- } else {
- updater.SingleObjectArena(fault_page + diff, gPageSize);
- }
- if (kMode == kCopyMode) {
- if (updater.WasLastPageTouched()) {
- state_arr[page_idx].store(PageState::kProcessed, std::memory_order_release);
- state = PageState::kProcessed;
- continue;
- } else {
- // If the page wasn't touched, then it means it is empty and
- // is most likely not present on the shadow-side. Furthermore,
- // since the shadow is also userfaultfd registered doing copy
- // ioctl fails as the copy-from-user in the kernel will cause
- // userfault. Instead, just map a zeropage, which is not only
- // correct but also efficient as it avoids unnecessary memcpy
- // in the kernel.
- ZeropageIoctl(
- fault_page, gPageSize, /*tolerate_eexist=*/false, /*tolerate_enoent=*/false);
- state_arr[page_idx].store(PageState::kProcessedAndMapped,
- std::memory_order_release);
- return;
- }
- }
+ LinearAllocPageUpdater updater(this);
+ uint8_t* first_obj = arena_iter->first->GetFirstObject(fault_page);
+ // null first_obj indicates that it's a page from arena for
+ // intern-table/class-table. So first object isn't required.
+ if (first_obj != nullptr) {
+ updater.MultiObjectArena(fault_page + diff, first_obj + diff);
} else {
- // Don't touch the page in this case (there is no reason to do so
- // anyways) as it would mean reading from first_obj, which could be on
- // another missing page and hence may cause this thread to block, leading
- // to deadlocks.
- // Force read the page if it is missing so that a zeropage gets mapped on
- // the shadow map and then CONTINUE ioctl will map it on linear-alloc.
- ForceRead(fault_page + diff);
+ updater.SingleObjectArena(fault_page + diff, gPageSize);
+ }
+ if (updater.WasLastPageTouched()) {
+ state_arr[page_idx].store(PageState::kProcessed, std::memory_order_release);
+ state = PageState::kProcessed;
+ continue;
+ } else {
+ // If the page wasn't touched, then it means it is empty and
+ // is most likely not present on the shadow-side. Furthermore,
+ // since the shadow is also userfaultfd registered doing copy
+ // ioctl fails as the copy-from-user in the kernel will cause
+ // userfault. Instead, just map a zeropage, which is not only
+ // correct but also efficient as it avoids unnecessary memcpy
+ // in the kernel.
+ ZeropageIoctl(
+ fault_page, gPageSize, /*tolerate_eexist=*/false, /*tolerate_enoent=*/false);
+ state_arr[page_idx].store(PageState::kProcessedAndMapped, std::memory_order_release);
}
- MapProcessedPages</*kFirstPageMapping=*/true>(
- fault_page, state_arr, page_idx, space_data->page_status_map_.Size());
return;
}
}
@@ -3960,16 +3458,6 @@ void MarkCompact::ConcurrentlyProcessLinearAllocPage(uint8_t* fault_page, bool i
}
break;
}
-
- DCHECK_EQ(kMode, kMinorFaultMode);
- DCHECK_EQ(state, PageState::kProcessed);
- if (!is_minor_fault) {
- // Force read the page if it is missing so that a zeropage gets mapped on
- // the shadow map and then CONTINUE ioctl will map it on linear-alloc.
- ForceRead(fault_page + diff);
- }
- MapProcessedPages</*kFirstPageMapping=*/false>(
- fault_page, state_arr, page_idx, space_data->page_status_map_.Size());
}
}
@@ -4090,28 +3578,17 @@ void MarkCompact::ProcessLinearAlloc() {
updater.SingleObjectArena(page_begin + diff, page_size);
}
expected_state = PageState::kProcessing;
- if (!minor_fault_initialized_) {
- // Store is sufficient as no other thread could be modifying it. Use
- // release order to ensure that the writes to shadow page are
- // committed to memory before.
- if (updater.WasLastPageTouched()) {
- state_arr[page_idx].store(PageState::kProcessed, std::memory_order_release);
- } else {
- // See comment in ConcurrentlyProcessLinearAllocPage() with same situation.
- ZeropageIoctl(
- page_begin, gPageSize, /*tolerate_eexist=*/false, /*tolerate_enoent=*/false);
- // Ioctl will act as release fence.
- state_arr[page_idx].store(PageState::kProcessedAndMapped, std::memory_order_release);
- }
- } else if (!state_arr[page_idx].compare_exchange_strong(
- expected_state, PageState::kProcessed, std::memory_order_release)) {
- DCHECK_EQ(expected_state, PageState::kProcessingAndMapping);
- // Force read in case the page was missing and updater didn't touch it
- // as there was nothing to do. This will ensure that a zeropage is
- // faulted on the shadow map.
- ForceRead(page_begin + diff);
- MapProcessedPages</*kFirstPageMapping=*/true>(
- page_begin, state_arr, page_idx, space_data->page_status_map_.Size());
+ // Store is sufficient as no other thread could be modifying it. Use
+ // release order to ensure that the writes to shadow page are
+ // committed to memory before.
+ if (updater.WasLastPageTouched()) {
+ state_arr[page_idx].store(PageState::kProcessed, std::memory_order_release);
+ } else {
+ // See comment in ConcurrentlyProcessLinearAllocPage() with same situation.
+ ZeropageIoctl(
+ page_begin, gPageSize, /*tolerate_eexist=*/false, /*tolerate_enoent=*/false);
+ // Ioctl will act as release fence.
+ state_arr[page_idx].store(PageState::kProcessedAndMapped, std::memory_order_release);
}
}
};
@@ -4125,15 +3602,12 @@ void MarkCompact::ProcessLinearAlloc() {
}
}
-void MarkCompact::RegisterUffd(void* addr, size_t size, int mode) {
+void MarkCompact::RegisterUffd(void* addr, size_t size) {
DCHECK(IsValidFd(uffd_));
struct uffdio_register uffd_register;
uffd_register.range.start = reinterpret_cast<uintptr_t>(addr);
uffd_register.range.len = size;
uffd_register.mode = UFFDIO_REGISTER_MODE_MISSING;
- if (mode == kMinorFaultMode) {
- uffd_register.mode |= UFFDIO_REGISTER_MODE_MINOR;
- }
CHECK_EQ(ioctl(uffd_, UFFDIO_REGISTER, &uffd_register), 0)
<< "ioctl_userfaultfd: register failed: " << strerror(errno)
<< ". start:" << static_cast<void*>(addr) << " len:" << PrettySize(size);
@@ -4150,16 +3624,6 @@ void MarkCompact::UnregisterUffd(uint8_t* start, size_t len) {
CHECK_EQ(ioctl(uffd_, UFFDIO_UNREGISTER, &range), 0)
<< "ioctl_userfaultfd: unregister failed: " << strerror(errno)
<< ". addr:" << static_cast<void*>(start) << " len:" << PrettySize(len);
- // Due to an oversight in the kernel implementation of 'unregister', the
- // waiting threads are woken up only for copy uffds. Therefore, for now, we
- // have to explicitly wake up the threads in minor-fault case.
- // TODO: The fix in the kernel is being worked on. Once the kernel version
- // containing the fix is known, make it conditional on that as well.
- if (minor_fault_initialized_) {
- CHECK_EQ(ioctl(uffd_, UFFDIO_WAKE, &range), 0)
- << "ioctl_userfaultfd: wake failed: " << strerror(errno)
- << ". addr:" << static_cast<void*>(start) << " len:" << PrettySize(len);
- }
}
void MarkCompact::CompactionPhase() {
@@ -4170,11 +3634,7 @@ void MarkCompact::CompactionPhase() {
RecordFree(ObjectBytePair(freed_objects_, freed_bytes));
}
- if (CanCompactMovingSpaceWithMinorFault()) {
- CompactMovingSpace<kMinorFaultMode>(/*page=*/nullptr);
- } else {
- CompactMovingSpace<kCopyMode>(compaction_buffers_map_.Begin());
- }
+ CompactMovingSpace<kCopyMode>(compaction_buffers_map_.Begin());
ProcessLinearAlloc();
@@ -4209,10 +3669,6 @@ void MarkCompact::CompactionPhase() {
// madvise the shado-map as the pages from it were reclaimed in
// ProcessLinearAlloc() after arenas were mapped.
data.page_status_map_.MadviseDontNeedAndZero();
- if (minor_fault_initialized_) {
- DCHECK_EQ(mprotect(data.shadow_.Begin(), data.shadow_.Size(), PROT_NONE), 0)
- << "mprotect failed: " << strerror(errno);
- }
}
// Make sure no mutator is reading from the from-space before unregistering
@@ -4231,42 +3687,10 @@ void MarkCompact::CompactionPhase() {
UnregisterUffd(bump_pointer_space_->Begin(), used_size);
}
// Release all of the memory taken by moving-space's from-map
- if (minor_fault_initialized_) {
- if (IsValidFd(moving_from_space_fd_)) {
- // A strange behavior is observed wherein between GC cycles the from-space'
- // first page is accessed. But the memfd that is mapped on from-space, is
- // used on to-space in next GC cycle, causing issues with userfaultfd as the
- // page isn't missing. A possible reason for this could be prefetches. The
- // mprotect ensures that such accesses don't succeed.
- int ret = mprotect(from_space_begin_, moving_space_size, PROT_NONE);
- CHECK_EQ(ret, 0) << "mprotect(PROT_NONE) for from-space failed: " << strerror(errno);
- // madvise(MADV_REMOVE) needs PROT_WRITE. Use fallocate() instead, which
- // does the same thing.
- ret = fallocate(moving_from_space_fd_,
- FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
- /*offset=*/0,
- moving_space_size);
- CHECK_EQ(ret, 0) << "fallocate for from-space failed: " << strerror(errno);
- } else {
- // We don't have a valid fd, so use madvise(MADV_REMOVE) instead. mprotect
- // is not required in this case as we create fresh
- // MAP_SHARED+MAP_ANONYMOUS mapping in each GC cycle.
- int ret = madvise(from_space_begin_, moving_space_size, MADV_REMOVE);
- CHECK_EQ(ret, 0) << "madvise(MADV_REMOVE) failed for from-space map:" << strerror(errno);
- }
- } else {
- from_space_map_.MadviseDontNeedAndZero();
- }
+ from_space_map_.MadviseDontNeedAndZero();
// mprotect(PROT_NONE) all maps except to-space in debug-mode to catch any unexpected accesses.
- if (shadow_to_space_map_.IsValid()) {
- DCHECK_EQ(mprotect(shadow_to_space_map_.Begin(), shadow_to_space_map_.Size(), PROT_NONE), 0)
- << "mprotect(PROT_NONE) for shadow-map failed:" << strerror(errno);
- }
- if (!IsValidFd(moving_from_space_fd_)) {
- // The other case is already mprotected above.
- DCHECK_EQ(mprotect(from_space_begin_, moving_space_size, PROT_NONE), 0)
- << "mprotect(PROT_NONE) for from-space failed: " << strerror(errno);
- }
+ DCHECK_EQ(mprotect(from_space_begin_, moving_space_size, PROT_NONE), 0)
+ << "mprotect(PROT_NONE) for from-space failed: " << strerror(errno);
if (!use_uffd_sigbus_) {
heap_->GetThreadPool()->StopWorkers(thread_running_gc_);
@@ -4820,22 +4244,15 @@ void MarkCompact::FinishPhase() {
GetCurrentIteration()->SetScannedBytes(bytes_scanned_);
bool is_zygote = Runtime::Current()->IsZygote();
compacting_ = false;
- minor_fault_initialized_ = !is_zygote && uffd_minor_fault_supported_;
// Madvise compaction buffers. When using threaded implementation, skip the first page,
// which is used by the gc-thread for the next iteration. Otherwise, we get into a
// deadlock due to userfault on it in the next iteration. This page is not consuming any
// physical memory because we already madvised it above and then we triggered a read
// userfault, which maps a special zero-page.
- if (use_uffd_sigbus_ || !minor_fault_initialized_ || !shadow_to_space_map_.IsValid() ||
- shadow_to_space_map_.Size() < (moving_first_objs_count_ + black_page_count_) * gPageSize) {
- size_t adjustment = use_uffd_sigbus_ ? 0 : gPageSize;
- ZeroAndReleaseMemory(compaction_buffers_map_.Begin() + adjustment,
- compaction_buffers_map_.Size() - adjustment);
- } else if (shadow_to_space_map_.Size() == bump_pointer_space_->Capacity()) {
- // Now that we are going to use minor-faults from next GC cycle, we can
- // unmap the buffers used by worker threads.
- compaction_buffers_map_.SetSize(gPageSize);
- }
+ size_t adjustment = use_uffd_sigbus_ ? 0 : gPageSize;
+ ZeroAndReleaseMemory(compaction_buffers_map_.Begin() + adjustment,
+ compaction_buffers_map_.Size() - adjustment);
+
info_map_.MadviseDontNeedAndZero();
live_words_bitmap_->ClearBitmap();
// TODO: We can clear this bitmap right before compaction pause. But in that
@@ -4867,13 +4284,6 @@ void MarkCompact::FinishPhase() {
WriterMutexLock mu2(thread_running_gc_, *Locks::heap_bitmap_lock_);
heap_->ClearMarkedObjects();
}
- std::swap(moving_to_space_fd_, moving_from_space_fd_);
- if (IsValidFd(moving_to_space_fd_)) {
- // Confirm that the memfd to be used on to-space in next GC cycle is empty.
- struct stat buf;
- DCHECK_EQ(fstat(moving_to_space_fd_, &buf), 0) << "fstat failed: " << strerror(errno);
- DCHECK_EQ(buf.st_blocks, 0u);
- }
GcVisitedArenaPool* arena_pool =
static_cast<GcVisitedArenaPool*>(Runtime::Current()->GetLinearAllocArenaPool());
arena_pool->DeleteUnusedArenas();
diff --git a/runtime/gc/collector/mark_compact.h b/runtime/gc/collector/mark_compact.h
index dbadfd05d7..67857d2cf9 100644
--- a/runtime/gc/collector/mark_compact.h
+++ b/runtime/gc/collector/mark_compact.h
@@ -60,10 +60,8 @@ class MarkCompact final : public GarbageCollector {
static constexpr size_t kAlignment = kObjectAlignment;
static constexpr int kCopyMode = -1;
- static constexpr int kMinorFaultMode = -2;
// Fake file descriptor for fall back mode (when uffd isn't available)
static constexpr int kFallbackMode = -3;
- static constexpr int kFdSharedAnon = -1;
static constexpr int kFdUnused = -2;
// Bitmask for the compaction-done bit in the sigbus_in_progress_count_.
@@ -486,38 +484,27 @@ class MarkCompact final : public GarbageCollector {
void KernelPreparation();
// Called by KernelPreparation() for every memory range being prepared for
// userfaultfd registration.
- void KernelPrepareRangeForUffd(uint8_t* to_addr,
- uint8_t* from_addr,
- size_t map_size,
- int fd,
- uint8_t* shadow_addr = nullptr);
+ void KernelPrepareRangeForUffd(uint8_t* to_addr, uint8_t* from_addr, size_t map_size);
- void RegisterUffd(void* addr, size_t size, int mode);
+ void RegisterUffd(void* addr, size_t size);
void UnregisterUffd(uint8_t* start, size_t len);
// Called by thread-pool workers to read uffd_ and process fault events.
- template <int kMode>
void ConcurrentCompaction(uint8_t* buf) REQUIRES_SHARED(Locks::mutator_lock_);
// Called by thread-pool workers to compact and copy/map the fault page in
// moving space.
- template <int kMode>
void ConcurrentlyProcessMovingPage(uint8_t* fault_page,
uint8_t* buf,
size_t nr_moving_space_used_pages)
REQUIRES_SHARED(Locks::mutator_lock_);
// Called by thread-pool workers to process and copy/map the fault page in
// linear-alloc.
- template <int kMode>
- void ConcurrentlyProcessLinearAllocPage(uint8_t* fault_page, bool is_minor_fault)
+ void ConcurrentlyProcessLinearAllocPage(uint8_t* fault_page)
REQUIRES_SHARED(Locks::mutator_lock_);
// Process concurrently all the pages in linear-alloc. Called by gc-thread.
void ProcessLinearAlloc() REQUIRES_SHARED(Locks::mutator_lock_);
- // Returns true if the moving space can be compacted using uffd's minor-fault
- // feature.
- bool CanCompactMovingSpaceWithMinorFault();
-
// Does the following:
// 1. Checks the status of to-space pages in [cur_page_idx,
// last_checked_reclaim_page_idx_) range to see whether the corresponding
@@ -530,17 +517,6 @@ class MarkCompact final : public GarbageCollector {
bool FreeFromSpacePages(size_t cur_page_idx, int mode, size_t end_idx_for_mapping)
REQUIRES_SHARED(Locks::mutator_lock_);
- // Maps processed pages (from moving space and linear-alloc) for uffd's
- // minor-fault feature. We try to 'claim' all processed (and unmapped) pages
- // contiguous to 'to_space_start'.
- // kFirstPageMapping indicates if the first page is already claimed or not. It
- // also indicates that the ioctl must succeed in mapping the first page.
- template <bool kFirstPageMapping>
- void MapProcessedPages(uint8_t* to_space_start,
- Atomic<PageState>* state_arr,
- size_t arr_idx,
- size_t arr_len) REQUIRES_SHARED(Locks::mutator_lock_);
-
// Maps moving space pages in [start_idx, arr_len) range. It fetches the page
// address containing the compacted content from moving_pages_status_ array.
// 'from_fault' is true when called from userfault (sigbus handler).
@@ -639,7 +615,6 @@ class MarkCompact final : public GarbageCollector {
// that double updation doesn't happen in the first place.
std::unique_ptr<std::unordered_set<void*>> updated_roots_ GUARDED_BY(lock_);
MemMap from_space_map_;
- MemMap shadow_to_space_map_;
// Any array of live-bytes in logical chunks of kOffsetChunkSize size
// in the 'to-be-compacted' space.
MemMap info_map_;
@@ -664,23 +639,16 @@ class MarkCompact final : public GarbageCollector {
// have more than one, but this is to be ready for the worst case.
class LinearAllocSpaceData {
public:
- LinearAllocSpaceData(MemMap&& shadow,
- MemMap&& page_status_map,
- uint8_t* begin,
- uint8_t* end,
- bool already_shared)
+ LinearAllocSpaceData(MemMap&& shadow, MemMap&& page_status_map, uint8_t* begin, uint8_t* end)
: shadow_(std::move(shadow)),
page_status_map_(std::move(page_status_map)),
begin_(begin),
- end_(end),
- already_shared_(already_shared) {}
+ end_(end) {}
MemMap shadow_;
MemMap page_status_map_;
uint8_t* begin_;
uint8_t* end_;
- // Indicates if the linear-alloc is already MAP_SHARED.
- bool already_shared_;
};
std::vector<LinearAllocSpaceData> linear_alloc_spaces_data_;
@@ -830,11 +798,6 @@ class MarkCompact final : public GarbageCollector {
// in MarkingPause(). It reaches the correct count only once the marking phase
// is completed.
int32_t freed_objects_;
- // memfds for moving space for using userfaultfd's minor-fault feature.
- // Initialized to kFdUnused to indicate that mmap should be MAP_PRIVATE in
- // KernelPrepareRange().
- int moving_to_space_fd_;
- int moving_from_space_fd_;
// Userfault file descriptor, accessed only by the GC itself.
// kFallbackMode value indicates that we are in the fallback mode.
int uffd_;
@@ -860,20 +823,9 @@ class MarkCompact final : public GarbageCollector {
// Heap::PostForkChildAction() as it's invoked in app startup path. With
// this, we register the compaction-termination page on the first GC.
bool uffd_initialized_;
- // Flag indicating if userfaultfd supports minor-faults. Set appropriately in
- // CreateUserfaultfd(), where we get this information from the kernel.
- const bool uffd_minor_fault_supported_;
// Flag indicating if we should use sigbus signals instead of threads to
// handle userfaults.
const bool use_uffd_sigbus_;
- // For non-zygote processes this flag indicates if the spaces are ready to
- // start using userfaultfd's minor-fault feature. This initialization involves
- // starting to use shmem (memfd_create) for the userfaultfd protected spaces.
- bool minor_fault_initialized_;
- // Set to true when linear-alloc can start mapping with MAP_SHARED. Set on
- // non-zygote processes during first GC, which sets up everyting for using
- // minor-fault from next GC.
- bool map_linear_alloc_shared_;
// Clamping statue of `info_map_`. Initialized with 'NotDone'. Once heap is
// clamped but info_map_ is delayed, we set it to 'Pending'. Once 'info_map_'
// is also clamped, then we set it to 'Finished'.