Update native GC roots page-by-page
This CL enables updating native GC-roots during compaction page by
page. It does so by using a single-space linear allocator for allocating
ArtMethods/ArtFields/DexCache etc. and using a per-object header to
describe the kind of object/array and its size. Under the hood it still
uses arena allocator but the arenas are page-aligned regions taken from
a single-space.
This allows us in a future CL to use userfaultfd to protect this space
during the compaction pause and then concurrently update the pages
independently.
Bug: 160737021
Test: ART_USE_READ_BARRIER art/test/testrunner/testrunner.py --host
Change-Id: Ie52243741360f6008feccec76117d34c77ab1dcf
diff --git a/runtime/class_linker.cc b/runtime/class_linker.cc
index 67a20a8..c0d99b9 100644
--- a/runtime/class_linker.cc
+++ b/runtime/class_linker.cc
@@ -44,6 +44,7 @@
#include "base/hash_set.h"
#include "base/leb128.h"
#include "base/logging.h"
+#include "base/mem_map_arena_pool.h"
#include "base/metrics/metrics.h"
#include "base/mutex-inl.h"
#include "base/os.h"
@@ -97,7 +98,7 @@
#include "jit/jit_code_cache.h"
#include "jni/java_vm_ext.h"
#include "jni/jni_internal.h"
-#include "linear_alloc.h"
+#include "linear_alloc-inl.h"
#include "mirror/array-alloc-inl.h"
#include "mirror/array-inl.h"
#include "mirror/call_site.h"
@@ -3487,7 +3488,7 @@
// If the ArtField alignment changes, review all uses of LengthPrefixedArray<ArtField>.
static_assert(alignof(ArtField) == 4, "ArtField alignment is expected to be 4.");
size_t storage_size = LengthPrefixedArray<ArtField>::ComputeSize(length);
- void* array_storage = allocator->Alloc(self, storage_size);
+ void* array_storage = allocator->Alloc(self, storage_size, LinearAllocKind::kArtFieldArray);
auto* ret = new(array_storage) LengthPrefixedArray<ArtField>(length);
CHECK(ret != nullptr);
std::uninitialized_fill_n(&ret->At(0), length, ArtField());
@@ -3504,7 +3505,7 @@
const size_t method_size = ArtMethod::Size(image_pointer_size_);
const size_t storage_size =
LengthPrefixedArray<ArtMethod>::ComputeSize(length, method_size, method_alignment);
- void* array_storage = allocator->Alloc(self, storage_size);
+ void* array_storage = allocator->Alloc(self, storage_size, LinearAllocKind::kArtMethodArray);
auto* ret = new (array_storage) LengthPrefixedArray<ArtMethod>(length);
CHECK(ret != nullptr);
for (size_t i = 0; i < length; ++i) {
@@ -5918,7 +5919,9 @@
if (imt == nullptr) {
LinearAlloc* allocator = GetAllocatorForClassLoader(klass->GetClassLoader());
imt = reinterpret_cast<ImTable*>(
- allocator->Alloc(self, ImTable::SizeInBytes(image_pointer_size_)));
+ allocator->Alloc(self,
+ ImTable::SizeInBytes(image_pointer_size_),
+ LinearAllocKind::kNoGCRoots));
if (imt == nullptr) {
return false;
}
@@ -6201,8 +6204,9 @@
// Allocate a new table. Note that we will leak this table at the next conflict,
// but that's a tradeoff compared to making the table fixed size.
void* data = linear_alloc->Alloc(
- Thread::Current(), ImtConflictTable::ComputeSizeWithOneMoreEntry(current_table,
- image_pointer_size_));
+ Thread::Current(),
+ ImtConflictTable::ComputeSizeWithOneMoreEntry(current_table, image_pointer_size_),
+ LinearAllocKind::kNoGCRoots);
if (data == nullptr) {
LOG(ERROR) << "Failed to allocate conflict table";
return conflict_method;
@@ -6316,8 +6320,8 @@
LinearAlloc* linear_alloc,
PointerSize image_pointer_size) {
void* data = linear_alloc->Alloc(Thread::Current(),
- ImtConflictTable::ComputeSize(count,
- image_pointer_size));
+ ImtConflictTable::ComputeSize(count, image_pointer_size),
+ LinearAllocKind::kNoGCRoots);
return (data != nullptr) ? new (data) ImtConflictTable(count, image_pointer_size) : nullptr;
}
@@ -6933,7 +6937,7 @@
klass_(klass),
self_(self),
runtime_(runtime),
- stack_(runtime->GetLinearAlloc()->GetArenaPool()),
+ stack_(runtime->GetArenaPool()),
allocator_(&stack_),
copied_method_records_(copied_method_records_initial_buffer_,
kCopiedMethodRecordInitialBufferSize,
@@ -7013,6 +7017,10 @@
kMethodSize,
kMethodAlignment);
memset(old_methods, 0xFEu, old_size);
+ // Set size to 0 to avoid visiting declaring classes.
+ if (gUseUserfaultfd) {
+ old_methods->SetSize(0);
+ }
}
}
}
@@ -7615,16 +7623,25 @@
const size_t old_methods_ptr_size = (old_methods != nullptr) ? old_size : 0;
auto* methods = reinterpret_cast<LengthPrefixedArray<ArtMethod>*>(
class_linker_->GetAllocatorForClassLoader(klass->GetClassLoader())->Realloc(
- self_, old_methods, old_methods_ptr_size, new_size));
+ self_, old_methods, old_methods_ptr_size, new_size, LinearAllocKind::kArtMethodArray));
CHECK(methods != nullptr); // Native allocation failure aborts.
if (methods != old_methods) {
- StrideIterator<ArtMethod> out = methods->begin(kMethodSize, kMethodAlignment);
- // Copy over the old methods. The `ArtMethod::CopyFrom()` is only necessary to not miss
- // read barriers since `LinearAlloc::Realloc()` won't do read barriers when it copies.
- for (auto& m : klass->GetMethods(kPointerSize)) {
- out->CopyFrom(&m, kPointerSize);
- ++out;
+ if (gUseReadBarrier) {
+ StrideIterator<ArtMethod> out = methods->begin(kMethodSize, kMethodAlignment);
+ // Copy over the old methods. The `ArtMethod::CopyFrom()` is only necessary to not miss
+ // read barriers since `LinearAlloc::Realloc()` won't do read barriers when it copies.
+ for (auto& m : klass->GetMethods(kPointerSize)) {
+ out->CopyFrom(&m, kPointerSize);
+ ++out;
+ }
+ } else if (gUseUserfaultfd) {
+ // Clear the declaring class of the old dangling method array so that GC doesn't
+ // try to update them, which could cause crashes in userfaultfd GC due to
+ // checks in post-compact address computation.
+ for (auto& m : klass->GetMethods(kPointerSize)) {
+ m.SetDeclaringClass(nullptr);
+ }
}
}