diff options
55 files changed, 993 insertions, 353 deletions
diff --git a/build/Android.common_test.mk b/build/Android.common_test.mk index c750399387..f3e1cc3272 100644 --- a/build/Android.common_test.mk +++ b/build/Android.common_test.mk @@ -161,9 +161,9 @@ endef # $(5): a make variable used to collate target dependencies, e.g ART_TEST_TARGET_OAT_HelloWorld_DEX # $(6): a make variable used to collate host dependencies, e.g ART_TEST_HOST_OAT_HelloWorld_DEX # -# If the input test directory contains a file called main.list, then a -# multi-dex file is created passing main.list as the --main-dex-list argument -# to dx. +# If the input test directory contains a file called main.list and main.jpp, +# then a multi-dex file is created passing main.list as the --main-dex-list +# argument to dx and main.jpp for Jack. define build-art-test-dex ifeq ($(ART_BUILD_TARGET),true) include $(CLEAR_VARS) @@ -178,6 +178,7 @@ define build-art-test-dex LOCAL_DEX_PREOPT_IMAGE_LOCATION := $(TARGET_CORE_IMG_OUT) ifneq ($(wildcard $(LOCAL_PATH)/$(2)/main.list),) LOCAL_DX_FLAGS := --multi-dex --main-dex-list=$(LOCAL_PATH)/$(2)/main.list --minimal-main-dex + LOCAL_JACK_FLAGS := -D jack.dex.output.policy=minimal-multidex -D jack.preprocessor=true -D jack.preprocessor.file=$(LOCAL_PATH)/$(2)/main.jpp endif include $(BUILD_JAVA_LIBRARY) $(5) := $$(LOCAL_INSTALLED_MODULE) @@ -193,6 +194,7 @@ define build-art-test-dex LOCAL_DEX_PREOPT_IMAGE := $(HOST_CORE_IMG_LOCATION) ifneq ($(wildcard $(LOCAL_PATH)/$(2)/main.list),) LOCAL_DX_FLAGS := --multi-dex --main-dex-list=$(LOCAL_PATH)/$(2)/main.list --minimal-main-dex + LOCAL_JACK_FLAGS := -D jack.dex.output.policy=minimal-multidex -D jack.preprocessor=true -D jack.preprocessor.file=$(LOCAL_PATH)/$(2)/main.jpp endif include $(BUILD_HOST_DALVIK_JAVA_LIBRARY) $(6) := $$(LOCAL_INSTALLED_MODULE) diff --git a/compiler/dex/dex_to_dex_compiler.cc b/compiler/dex/dex_to_dex_compiler.cc index 7e916bee4a..fcefb6fbfc 100644 --- a/compiler/dex/dex_to_dex_compiler.cc +++ b/compiler/dex/dex_to_dex_compiler.cc @@ -252,10 +252,8 @@ void DexCompiler::CompileInstanceFieldAccess(Instruction* inst, } } -void DexCompiler::CompileInvokeVirtual(Instruction* inst, - uint32_t dex_pc, - Instruction::Code new_opcode, - bool is_range) { +void DexCompiler::CompileInvokeVirtual(Instruction* inst, uint32_t dex_pc, + Instruction::Code new_opcode, bool is_range) { if (!kEnableQuickening || !PerformOptimizations()) { return; } diff --git a/compiler/dex/mir_method_info.cc b/compiler/dex/mir_method_info.cc index 3d3d9790c3..34fb1bf0e0 100644 --- a/compiler/dex/mir_method_info.cc +++ b/compiler/dex/mir_method_info.cc @@ -58,8 +58,9 @@ void MirMethodLoweringInfo::Resolve(CompilerDriver* compiler_driver, auto current_dex_cache(hs.NewHandle<mirror::DexCache>(nullptr)); // Even if the referrer class is unresolved (i.e. we're compiling a method without class // definition) we still want to resolve methods and record all available info. + Runtime* const runtime = Runtime::Current(); const DexFile* const dex_file = mUnit->GetDexFile(); - const bool use_jit = Runtime::Current()->UseJit(); + const bool use_jit = runtime->UseJit(); const VerifiedMethod* const verified_method = mUnit->GetVerifiedMethod(); for (auto it = method_infos, end = method_infos + count; it != end; ++it) { @@ -80,7 +81,7 @@ void MirMethodLoweringInfo::Resolve(CompilerDriver* compiler_driver, it->target_method_idx_ = it->MethodIndex(); current_dex_cache.Assign(dex_cache.Get()); resolved_method = compiler_driver->ResolveMethod(soa, dex_cache, class_loader, mUnit, - it->MethodIndex(), invoke_type); + it->target_method_idx_, invoke_type, true); } else { // The method index is actually the dex PC in this case. // Calculate the proper dex file and target method idx. @@ -89,8 +90,7 @@ void MirMethodLoweringInfo::Resolve(CompilerDriver* compiler_driver, // Don't devirt if we are in a different dex file since we can't have direct invokes in // another dex file unless we always put a direct / patch pointer. devirt_target = nullptr; - current_dex_cache.Assign( - Runtime::Current()->GetClassLinker()->FindDexCache(*it->target_dex_file_)); + current_dex_cache.Assign(runtime->GetClassLinker()->FindDexCache(*it->target_dex_file_)); CHECK(current_dex_cache.Get() != nullptr); DexCompilationUnit cu( mUnit->GetCompilationUnit(), mUnit->GetClassLoader(), mUnit->GetClassLinker(), @@ -99,6 +99,14 @@ void MirMethodLoweringInfo::Resolve(CompilerDriver* compiler_driver, nullptr /* verified_method not used */); resolved_method = compiler_driver->ResolveMethod(soa, current_dex_cache, class_loader, &cu, it->target_method_idx_, invoke_type, false); + if (resolved_method == nullptr) { + // If the method is null then it should be a miranda method, in this case try + // re-loading it, this time as an interface method. The actual miranda method is in the + // vtable, but it will resolve to an interface method. + resolved_method = compiler_driver->ResolveMethod( + soa, current_dex_cache, class_loader, &cu, it->target_method_idx_, kInterface, false); + CHECK(resolved_method != nullptr); + } if (resolved_method != nullptr) { // Since this was a dequickened virtual, it is guaranteed to be resolved. However, it may be // resolved to an interface method. If this is the case then change the invoke type to @@ -123,10 +131,9 @@ void MirMethodLoweringInfo::Resolve(CompilerDriver* compiler_driver, it->vtable_idx_ = compiler_driver->GetResolvedMethodVTableIndex(resolved_method, invoke_type); } - MethodReference target_method(it->target_dex_file_, it->target_method_idx_); int fast_path_flags = compiler_driver->IsFastInvoke( - soa, dex_cache, class_loader, mUnit, referrer_class.Get(), resolved_method, + soa, current_dex_cache, class_loader, mUnit, referrer_class.Get(), resolved_method, &invoke_type, &target_method, devirt_target, &it->direct_code_, &it->direct_method_); const bool is_referrers_class = referrer_class.Get() == resolved_method->GetDeclaringClass(); const bool is_class_initialized = diff --git a/compiler/dex/verified_method.cc b/compiler/dex/verified_method.cc index 42d66be60d..5b90ba92e0 100644 --- a/compiler/dex/verified_method.cc +++ b/compiler/dex/verified_method.cc @@ -55,8 +55,8 @@ const VerifiedMethod* VerifiedMethod::Create(verifier::MethodVerifier* method_ve } // Only need dequicken info for JIT so far. - if (Runtime::Current()->UseJit()) { - verified_method->GenerateDequickenMap(method_verifier); + if (Runtime::Current()->UseJit() && !verified_method->GenerateDequickenMap(method_verifier)) { + return nullptr; } } @@ -194,9 +194,9 @@ void VerifiedMethod::ComputeGcMapSizes(verifier::MethodVerifier* method_verifier *log2_max_gc_pc = i; } -void VerifiedMethod::GenerateDequickenMap(verifier::MethodVerifier* method_verifier) { +bool VerifiedMethod::GenerateDequickenMap(verifier::MethodVerifier* method_verifier) { if (method_verifier->HasFailures()) { - return; + return false; } const DexFile::CodeItem* code_item = method_verifier->CodeItem(); const uint16_t* insns = code_item->insns_; @@ -209,8 +209,11 @@ void VerifiedMethod::GenerateDequickenMap(verifier::MethodVerifier* method_verif uint32_t dex_pc = inst->GetDexPc(insns); verifier::RegisterLine* line = method_verifier->GetRegLine(dex_pc); mirror::ArtMethod* method = - method_verifier->GetQuickInvokedMethod(inst, line, is_range_quick); - CHECK(method != nullptr); + method_verifier->GetQuickInvokedMethod(inst, line, is_range_quick, true); + if (method == nullptr) { + // It can be null if the line wasn't verified since it was unreachable. + return false; + } // The verifier must know what the type of the object was or else we would have gotten a // failure. Put the dex method index in the dequicken map since we need this to get number of // arguments in the compiler. @@ -220,7 +223,10 @@ void VerifiedMethod::GenerateDequickenMap(verifier::MethodVerifier* method_verif uint32_t dex_pc = inst->GetDexPc(insns); verifier::RegisterLine* line = method_verifier->GetRegLine(dex_pc); mirror::ArtField* field = method_verifier->GetQuickFieldAccess(inst, line); - CHECK(field != nullptr); + if (field == nullptr) { + // It can be null if the line wasn't verified since it was unreachable. + return false; + } // The verifier must know what the type of the field was or else we would have gotten a // failure. Put the dex field index in the dequicken map since we need this for lowering // in the compiler. @@ -228,6 +234,7 @@ void VerifiedMethod::GenerateDequickenMap(verifier::MethodVerifier* method_verif dequicken_map_.Put(dex_pc, DexFileReference(field->GetDexFile(), field->GetDexFieldIndex())); } } + return true; } void VerifiedMethod::GenerateDevirtMap(verifier::MethodVerifier* method_verifier) { diff --git a/compiler/dex/verified_method.h b/compiler/dex/verified_method.h index 748bdcb71c..954cbf4b26 100644 --- a/compiler/dex/verified_method.h +++ b/compiler/dex/verified_method.h @@ -93,8 +93,8 @@ class VerifiedMethod { void GenerateDevirtMap(verifier::MethodVerifier* method_verifier) SHARED_LOCKS_REQUIRED(Locks::mutator_lock_); - // Generate dequickening map into dequicken_map_. - void GenerateDequickenMap(verifier::MethodVerifier* method_verifier) + // Generate dequickening map into dequicken_map_. Returns false if there is an error. + bool GenerateDequickenMap(verifier::MethodVerifier* method_verifier) SHARED_LOCKS_REQUIRED(Locks::mutator_lock_); // Generate safe case set into safe_cast_set_. diff --git a/compiler/driver/compiler_driver.cc b/compiler/driver/compiler_driver.cc index 78dd6cc29e..be6c41a834 100644 --- a/compiler/driver/compiler_driver.cc +++ b/compiler/driver/compiler_driver.cc @@ -1860,6 +1860,12 @@ static void VerifyClass(const ParallelCompilationManager* manager, size_t class_ CHECK(klass->IsCompileTimeVerified() || klass->IsErroneous()) << PrettyDescriptor(klass.Get()) << ": state=" << klass->GetStatus(); + + // It is *very* problematic if there are verification errors in the boot classpath. For example, + // we rely on things working OK without verification when the decryption dialog is brought up. + // So abort in a debug build if we find this violated. + DCHECK(!manager->GetCompiler()->IsImage() || klass->IsVerified()) << "Boot classpath class " << + PrettyClass(klass.Get()) << " failed to fully verify."; } soa.Self()->AssertNoPendingException(); } diff --git a/compiler/optimizing/ssa_builder.cc b/compiler/optimizing/ssa_builder.cc index 1a8e784363..ba11e90d9c 100644 --- a/compiler/optimizing/ssa_builder.cc +++ b/compiler/optimizing/ssa_builder.cc @@ -199,9 +199,9 @@ void SsaBuilder::BuildSsa() { // our code generator will complain if the inputs of a phi do not have the same // type. The marking allows the type propagation to know which phis it needs // to handle. We mark but do not eliminate: the elimination will be done in - // step 8). - SsaDeadPhiElimination dead_phis(GetGraph()); - dead_phis.MarkDeadPhis(); + // step 9). + SsaDeadPhiElimination dead_phis_for_type_propagation(GetGraph()); + dead_phis_for_type_propagation.MarkDeadPhis(); // 4) Propagate types of phis. At this point, phis are typed void in the general // case, or float/double/reference when we created an equivalent phi. So we @@ -209,7 +209,11 @@ void SsaBuilder::BuildSsa() { PrimitiveTypePropagation type_propagation(GetGraph()); type_propagation.Run(); - // 5) Now that the graph is correclty typed, we can get rid of redundant phis. + // 5) Mark dead phis again. Steph 4) may have introduced new phis. + SsaDeadPhiElimination dead_phis(GetGraph()); + dead_phis.MarkDeadPhis(); + + // 6) Now that the graph is correclty typed, we can get rid of redundant phis. // Note that we cannot do this phase before type propagation, otherwise // we could get rid of phi equivalents, whose presence is a requirement for the // type propagation phase. Note that this is to satisfy statement (a) of the @@ -217,7 +221,7 @@ void SsaBuilder::BuildSsa() { SsaRedundantPhiElimination redundant_phi(GetGraph()); redundant_phi.Run(); - // 6) Make sure environments use the right phi "equivalent": a phi marked dead + // 7) Make sure environments use the right phi "equivalent": a phi marked dead // can have a phi equivalent that is not dead. We must therefore update // all environment uses of the dead phi to use its equivalent. Note that there // can be multiple phis for the same Dex register that are live (for example @@ -244,7 +248,7 @@ void SsaBuilder::BuildSsa() { } } - // 7) Deal with phis to guarantee liveness of phis in case of a debuggable + // 8) Deal with phis to guarantee liveness of phis in case of a debuggable // application. This is for satisfying statement (c) of the SsaBuilder // (see ssa_builder.h). if (GetGraph()->IsDebuggable()) { @@ -252,7 +256,7 @@ void SsaBuilder::BuildSsa() { dead_phi_handler.Run(); } - // 8) Now that the right phis are used for the environments, and we + // 9) Now that the right phis are used for the environments, and we // have potentially revive dead phis in case of a debuggable application, // we can eliminate phis we do not need. Regardless of the debuggable status, // this phase is necessary for statement (b) of the SsaBuilder (see ssa_builder.h), @@ -260,7 +264,7 @@ void SsaBuilder::BuildSsa() { // input types. dead_phis.EliminateDeadPhis(); - // 9) Clear locals. + // 10) Clear locals. for (HInstructionIterator it(GetGraph()->GetEntryBlock()->GetInstructions()); !it.Done(); it.Advance()) { diff --git a/runtime/common_runtime_test.cc b/runtime/common_runtime_test.cc index 84865973c6..e0d62d7012 100644 --- a/runtime/common_runtime_test.cc +++ b/runtime/common_runtime_test.cc @@ -263,6 +263,8 @@ void CommonRuntimeTest::SetUp() { // pool is created by the runtime. runtime_->GetHeap()->CreateThreadPool(); runtime_->GetHeap()->VerifyHeap(); // Check for heap corruption before the test + // Reduce timinig-dependent flakiness in OOME behavior (eg StubTest.AllocObject). + runtime_->GetHeap()->SetMinIntervalHomogeneousSpaceCompactionByOom(0U); // Get the boot class path from the runtime so it can be used in tests. boot_class_path_ = class_linker_->GetBootClassPath(); diff --git a/runtime/gc/accounting/mod_union_table_test.cc b/runtime/gc/accounting/mod_union_table_test.cc index 87ce166147..77809358e4 100644 --- a/runtime/gc/accounting/mod_union_table_test.cc +++ b/runtime/gc/accounting/mod_union_table_test.cc @@ -48,9 +48,9 @@ class ModUnionTableTest : public CommonRuntimeTest { SHARED_LOCKS_REQUIRED(Locks::mutator_lock_) { auto* klass = GetObjectArrayClass(self, space); const size_t size = ComputeArraySize(self, klass, component_count, 2); - size_t bytes_allocated = 0; + size_t bytes_allocated = 0, bytes_tl_bulk_allocated; auto* obj = down_cast<mirror::ObjectArray<mirror::Object>*>( - space->Alloc(self, size, &bytes_allocated, nullptr)); + space->Alloc(self, size, &bytes_allocated, nullptr, &bytes_tl_bulk_allocated)); if (obj != nullptr) { obj->SetClass(klass); obj->SetLength(static_cast<int32_t>(component_count)); @@ -77,9 +77,10 @@ class ModUnionTableTest : public CommonRuntimeTest { // copy of the class in the same space that we are allocating in. DCHECK(java_lang_object_array_ != nullptr); const size_t class_size = java_lang_object_array_->GetClassSize(); - size_t bytes_allocated = 0; + size_t bytes_allocated = 0, bytes_tl_bulk_allocated; auto* klass = down_cast<mirror::Class*>(space->Alloc(self, class_size, &bytes_allocated, - nullptr)); + nullptr, + &bytes_tl_bulk_allocated)); DCHECK(klass != nullptr); memcpy(klass, java_lang_object_array_, class_size); Runtime::Current()->GetHeap()->GetCardTable()->MarkCard(klass); diff --git a/runtime/gc/allocator/rosalloc-inl.h b/runtime/gc/allocator/rosalloc-inl.h index f6c9d3c144..bba92a1f40 100644 --- a/runtime/gc/allocator/rosalloc-inl.h +++ b/runtime/gc/allocator/rosalloc-inl.h @@ -28,15 +28,19 @@ inline ALWAYS_INLINE bool RosAlloc::ShouldCheckZeroMemory() { } template<bool kThreadSafe> -inline ALWAYS_INLINE void* RosAlloc::Alloc(Thread* self, size_t size, size_t* bytes_allocated) { +inline ALWAYS_INLINE void* RosAlloc::Alloc(Thread* self, size_t size, size_t* bytes_allocated, + size_t* usable_size, + size_t* bytes_tl_bulk_allocated) { if (UNLIKELY(size > kLargeSizeThreshold)) { - return AllocLargeObject(self, size, bytes_allocated); + return AllocLargeObject(self, size, bytes_allocated, usable_size, + bytes_tl_bulk_allocated); } void* m; if (kThreadSafe) { - m = AllocFromRun(self, size, bytes_allocated); + m = AllocFromRun(self, size, bytes_allocated, usable_size, bytes_tl_bulk_allocated); } else { - m = AllocFromRunThreadUnsafe(self, size, bytes_allocated); + m = AllocFromRunThreadUnsafe(self, size, bytes_allocated, usable_size, + bytes_tl_bulk_allocated); } // Check if the returned memory is really all zero. if (ShouldCheckZeroMemory() && m != nullptr) { @@ -48,6 +52,115 @@ inline ALWAYS_INLINE void* RosAlloc::Alloc(Thread* self, size_t size, size_t* by return m; } +inline bool RosAlloc::Run::IsFull() { + const size_t num_vec = NumberOfBitmapVectors(); + for (size_t v = 0; v < num_vec; ++v) { + if (~alloc_bit_map_[v] != 0) { + return false; + } + } + return true; +} + +inline bool RosAlloc::CanAllocFromThreadLocalRun(Thread* self, size_t size) { + if (UNLIKELY(!IsSizeForThreadLocal(size))) { + return false; + } + size_t bracket_size; + size_t idx = SizeToIndexAndBracketSize(size, &bracket_size); + DCHECK_EQ(idx, SizeToIndex(size)); + DCHECK_EQ(bracket_size, IndexToBracketSize(idx)); + DCHECK_EQ(bracket_size, bracketSizes[idx]); + DCHECK_LE(size, bracket_size); + DCHECK(size > 512 || bracket_size - size < 16); + DCHECK_LT(idx, kNumThreadLocalSizeBrackets); + Run* thread_local_run = reinterpret_cast<Run*>(self->GetRosAllocRun(idx)); + if (kIsDebugBuild) { + // Need the lock to prevent race conditions. + MutexLock mu(self, *size_bracket_locks_[idx]); + CHECK(non_full_runs_[idx].find(thread_local_run) == non_full_runs_[idx].end()); + CHECK(full_runs_[idx].find(thread_local_run) == full_runs_[idx].end()); + } + DCHECK(thread_local_run != nullptr); + DCHECK(thread_local_run->IsThreadLocal() || thread_local_run == dedicated_full_run_); + return !thread_local_run->IsFull(); +} + +inline void* RosAlloc::AllocFromThreadLocalRun(Thread* self, size_t size, + size_t* bytes_allocated) { + DCHECK(bytes_allocated != nullptr); + if (UNLIKELY(!IsSizeForThreadLocal(size))) { + return nullptr; + } + size_t bracket_size; + size_t idx = SizeToIndexAndBracketSize(size, &bracket_size); + Run* thread_local_run = reinterpret_cast<Run*>(self->GetRosAllocRun(idx)); + if (kIsDebugBuild) { + // Need the lock to prevent race conditions. + MutexLock mu(self, *size_bracket_locks_[idx]); + CHECK(non_full_runs_[idx].find(thread_local_run) == non_full_runs_[idx].end()); + CHECK(full_runs_[idx].find(thread_local_run) == full_runs_[idx].end()); + } + DCHECK(thread_local_run != nullptr); + DCHECK(thread_local_run->IsThreadLocal() || thread_local_run == dedicated_full_run_); + void* slot_addr = thread_local_run->AllocSlot(); + if (LIKELY(slot_addr != nullptr)) { + *bytes_allocated = bracket_size; + } + return slot_addr; +} + +inline size_t RosAlloc::MaxBytesBulkAllocatedFor(size_t size) { + if (UNLIKELY(!IsSizeForThreadLocal(size))) { + return size; + } + size_t bracket_size; + size_t idx = SizeToIndexAndBracketSize(size, &bracket_size); + return numOfSlots[idx] * bracket_size; +} + +inline void* RosAlloc::Run::AllocSlot() { + const size_t idx = size_bracket_idx_; + while (true) { + if (kIsDebugBuild) { + // Make sure that no slots leaked, the bitmap should be full for all previous vectors. + for (size_t i = 0; i < first_search_vec_idx_; ++i) { + CHECK_EQ(~alloc_bit_map_[i], 0U); + } + } + uint32_t* const alloc_bitmap_ptr = &alloc_bit_map_[first_search_vec_idx_]; + uint32_t ffz1 = __builtin_ffs(~*alloc_bitmap_ptr); + if (LIKELY(ffz1 != 0)) { + const uint32_t ffz = ffz1 - 1; + const uint32_t slot_idx = ffz + + first_search_vec_idx_ * sizeof(*alloc_bitmap_ptr) * kBitsPerByte; + const uint32_t mask = 1U << ffz; + DCHECK_LT(slot_idx, numOfSlots[idx]) << "out of range"; + // Found an empty slot. Set the bit. + DCHECK_EQ(*alloc_bitmap_ptr & mask, 0U); + *alloc_bitmap_ptr |= mask; + DCHECK_NE(*alloc_bitmap_ptr & mask, 0U); + uint8_t* slot_addr = reinterpret_cast<uint8_t*>(this) + + headerSizes[idx] + slot_idx * bracketSizes[idx]; + if (kTraceRosAlloc) { + LOG(INFO) << "RosAlloc::Run::AllocSlot() : 0x" << std::hex + << reinterpret_cast<intptr_t>(slot_addr) + << ", bracket_size=" << std::dec << bracketSizes[idx] + << ", slot_idx=" << slot_idx; + } + return slot_addr; + } + const size_t num_words = RoundUp(numOfSlots[idx], 32) / 32; + if (first_search_vec_idx_ + 1 >= num_words) { + DCHECK(IsFull()); + // Already at the last word, return null. + return nullptr; + } + // Increase the index to the next word and try again. + ++first_search_vec_idx_; + } +} + } // namespace allocator } // namespace gc } // namespace art diff --git a/runtime/gc/allocator/rosalloc.cc b/runtime/gc/allocator/rosalloc.cc index f51093aa57..f64a4ff8df 100644 --- a/runtime/gc/allocator/rosalloc.cc +++ b/runtime/gc/allocator/rosalloc.cc @@ -454,7 +454,10 @@ size_t RosAlloc::FreePages(Thread* self, void* ptr, bool already_zero) { return byte_size; } -void* RosAlloc::AllocLargeObject(Thread* self, size_t size, size_t* bytes_allocated) { +void* RosAlloc::AllocLargeObject(Thread* self, size_t size, size_t* bytes_allocated, + size_t* usable_size, size_t* bytes_tl_bulk_allocated) { + DCHECK(bytes_allocated != nullptr); + DCHECK(usable_size != nullptr); DCHECK_GT(size, kLargeSizeThreshold); size_t num_pages = RoundUp(size, kPageSize) / kPageSize; void* r; @@ -470,6 +473,8 @@ void* RosAlloc::AllocLargeObject(Thread* self, size_t size, size_t* bytes_alloca } const size_t total_bytes = num_pages * kPageSize; *bytes_allocated = total_bytes; + *usable_size = total_bytes; + *bytes_tl_bulk_allocated = total_bytes; if (kTraceRosAlloc) { LOG(INFO) << "RosAlloc::AllocLargeObject() : 0x" << std::hex << reinterpret_cast<intptr_t>(r) << "-0x" << (reinterpret_cast<intptr_t>(r) + num_pages * kPageSize) @@ -622,7 +627,12 @@ inline void* RosAlloc::AllocFromCurrentRunUnlocked(Thread* self, size_t idx) { return slot_addr; } -void* RosAlloc::AllocFromRunThreadUnsafe(Thread* self, size_t size, size_t* bytes_allocated) { +void* RosAlloc::AllocFromRunThreadUnsafe(Thread* self, size_t size, size_t* bytes_allocated, + size_t* usable_size, + size_t* bytes_tl_bulk_allocated) { + DCHECK(bytes_allocated != nullptr); + DCHECK(usable_size != nullptr); + DCHECK(bytes_tl_bulk_allocated != nullptr); DCHECK_LE(size, kLargeSizeThreshold); size_t bracket_size; size_t idx = SizeToIndexAndBracketSize(size, &bracket_size); @@ -634,14 +644,19 @@ void* RosAlloc::AllocFromRunThreadUnsafe(Thread* self, size_t size, size_t* byte Locks::mutator_lock_->AssertExclusiveHeld(self); void* slot_addr = AllocFromCurrentRunUnlocked(self, idx); if (LIKELY(slot_addr != nullptr)) { - DCHECK(bytes_allocated != nullptr); *bytes_allocated = bracket_size; - // Caller verifies that it is all 0. + *usable_size = bracket_size; + *bytes_tl_bulk_allocated = bracket_size; } + // Caller verifies that it is all 0. return slot_addr; } -void* RosAlloc::AllocFromRun(Thread* self, size_t size, size_t* bytes_allocated) { +void* RosAlloc::AllocFromRun(Thread* self, size_t size, size_t* bytes_allocated, + size_t* usable_size, size_t* bytes_tl_bulk_allocated) { + DCHECK(bytes_allocated != nullptr); + DCHECK(usable_size != nullptr); + DCHECK(bytes_tl_bulk_allocated != nullptr); DCHECK_LE(size, kLargeSizeThreshold); size_t bracket_size; size_t idx = SizeToIndexAndBracketSize(size, &bracket_size); @@ -712,31 +727,43 @@ void* RosAlloc::AllocFromRun(Thread* self, size_t size, size_t* bytes_allocated) self->SetRosAllocRun(idx, thread_local_run); DCHECK(!thread_local_run->IsFull()); } - DCHECK(thread_local_run != nullptr); DCHECK(!thread_local_run->IsFull()); DCHECK(thread_local_run->IsThreadLocal()); + // Account for all the free slots in the new or refreshed thread local run. + *bytes_tl_bulk_allocated = thread_local_run->NumberOfFreeSlots() * bracket_size; slot_addr = thread_local_run->AllocSlot(); // Must succeed now with a new run. DCHECK(slot_addr != nullptr); + } else { + // The slot is already counted. Leave it as is. + *bytes_tl_bulk_allocated = 0; } + DCHECK(slot_addr != nullptr); if (kTraceRosAlloc) { - LOG(INFO) << "RosAlloc::AllocFromRun() thread-local : 0x" << std::hex << reinterpret_cast<intptr_t>(slot_addr) + LOG(INFO) << "RosAlloc::AllocFromRun() thread-local : 0x" << std::hex + << reinterpret_cast<intptr_t>(slot_addr) << "-0x" << (reinterpret_cast<intptr_t>(slot_addr) + bracket_size) << "(" << std::dec << (bracket_size) << ")"; } + *bytes_allocated = bracket_size; + *usable_size = bracket_size; } else { // Use the (shared) current run. MutexLock mu(self, *size_bracket_locks_[idx]); slot_addr = AllocFromCurrentRunUnlocked(self, idx); if (kTraceRosAlloc) { - LOG(INFO) << "RosAlloc::AllocFromRun() : 0x" << std::hex << reinterpret_cast<intptr_t>(slot_addr) + LOG(INFO) << "RosAlloc::AllocFromRun() : 0x" << std::hex + << reinterpret_cast<intptr_t>(slot_addr) << "-0x" << (reinterpret_cast<intptr_t>(slot_addr) + bracket_size) << "(" << std::dec << (bracket_size) << ")"; } + if (LIKELY(slot_addr != nullptr)) { + *bytes_allocated = bracket_size; + *usable_size = bracket_size; + *bytes_tl_bulk_allocated = bracket_size; + } } - DCHECK(bytes_allocated != nullptr); - *bytes_allocated = bracket_size; // Caller verifies that it is all 0. return slot_addr; } @@ -852,44 +879,6 @@ std::string RosAlloc::Run::Dump() { return stream.str(); } -inline void* RosAlloc::Run::AllocSlot() { - const size_t idx = size_bracket_idx_; - while (true) { - if (kIsDebugBuild) { - // Make sure that no slots leaked, the bitmap should be full for all previous vectors. - for (size_t i = 0; i < first_search_vec_idx_; ++i) { - CHECK_EQ(~alloc_bit_map_[i], 0U); - } - } - uint32_t* const alloc_bitmap_ptr = &alloc_bit_map_[first_search_vec_idx_]; - uint32_t ffz1 = __builtin_ffs(~*alloc_bitmap_ptr); - if (LIKELY(ffz1 != 0)) { - const uint32_t ffz = ffz1 - 1; - const uint32_t slot_idx = ffz + first_search_vec_idx_ * sizeof(*alloc_bitmap_ptr) * kBitsPerByte; - const uint32_t mask = 1U << ffz; - DCHECK_LT(slot_idx, numOfSlots[idx]) << "out of range"; - // Found an empty slot. Set the bit. - DCHECK_EQ(*alloc_bitmap_ptr & mask, 0U); - *alloc_bitmap_ptr |= mask; - DCHECK_NE(*alloc_bitmap_ptr & mask, 0U); - uint8_t* slot_addr = reinterpret_cast<uint8_t*>(this) + headerSizes[idx] + slot_idx * bracketSizes[idx]; - if (kTraceRosAlloc) { - LOG(INFO) << "RosAlloc::Run::AllocSlot() : 0x" << std::hex << reinterpret_cast<intptr_t>(slot_addr) - << ", bracket_size=" << std::dec << bracketSizes[idx] << ", slot_idx=" << slot_idx; - } - return slot_addr; - } - const size_t num_words = RoundUp(numOfSlots[idx], 32) / 32; - if (first_search_vec_idx_ + 1 >= num_words) { - DCHECK(IsFull()); - // Already at the last word, return null. - return nullptr; - } - // Increase the index to the next word and try again. - ++first_search_vec_idx_; - } -} - void RosAlloc::Run::FreeSlot(void* ptr) { DCHECK(!IsThreadLocal()); const uint8_t idx = size_bracket_idx_; @@ -920,6 +909,25 @@ void RosAlloc::Run::FreeSlot(void* ptr) { } } +size_t RosAlloc::Run::NumberOfFreeSlots() { + size_t num_alloc_slots = 0; + const size_t idx = size_bracket_idx_; + const size_t num_slots = numOfSlots[idx]; + const size_t num_vec = RoundUp(num_slots, 32) / 32; + DCHECK_NE(num_vec, 0U); + for (size_t v = 0; v < num_vec - 1; v++) { + num_alloc_slots += POPCOUNT(alloc_bit_map_[v]); + } + // Don't count the invalid bits in the last vector. + uint32_t last_vec_masked = alloc_bit_map_[num_vec - 1] & + ~GetBitmapLastVectorMask(num_slots, num_vec); + num_alloc_slots += POPCOUNT(last_vec_masked); + size_t num_free_slots = num_slots - num_alloc_slots; + DCHECK_LE(num_alloc_slots, num_slots); + DCHECK_LE(num_free_slots, num_slots); + return num_free_slots; +} + inline bool RosAlloc::Run::MergeThreadLocalFreeBitMapToAllocBitMap(bool* is_all_free_after_out) { DCHECK(IsThreadLocal()); // Free slots in the alloc bit map based on the thread local free bit map. @@ -1055,16 +1063,6 @@ inline bool RosAlloc::Run::IsAllFree() { return alloc_bit_map_[num_vec - 1] == GetBitmapLastVectorMask(num_slots, num_vec); } -inline bool RosAlloc::Run::IsFull() { - const size_t num_vec = NumberOfBitmapVectors(); - for (size_t v = 0; v < num_vec; ++v) { - if (~alloc_bit_map_[v] != 0) { - return false; - } - } - return true; -} - inline bool RosAlloc::Run::IsBulkFreeBitmapClean() { const size_t num_vec = NumberOfBitmapVectors(); for (size_t v = 0; v < num_vec; v++) { @@ -1654,10 +1652,11 @@ void RosAlloc::SetFootprintLimit(size_t new_capacity) { } } -void RosAlloc::RevokeThreadLocalRuns(Thread* thread) { +size_t RosAlloc::RevokeThreadLocalRuns(Thread* thread) { Thread* self = Thread::Current(); // Avoid race conditions on the bulk free bit maps with BulkFree() (GC). ReaderMutexLock wmu(self, bulk_free_lock_); + size_t free_bytes = 0U; for (size_t idx = 0; idx < kNumThreadLocalSizeBrackets; idx++) { MutexLock mu(self, *size_bracket_locks_[idx]); Run* thread_local_run = reinterpret_cast<Run*>(thread->GetRosAllocRun(idx)); @@ -1665,9 +1664,12 @@ void RosAlloc::RevokeThreadLocalRuns(Thread* thread) { // Invalid means already revoked. DCHECK(thread_local_run->IsThreadLocal()); if (thread_local_run != dedicated_full_run_) { + // Note the thread local run may not be full here. thread->SetRosAllocRun(idx, dedicated_full_run_); DCHECK_EQ(thread_local_run->magic_num_, kMagicNum); - // Note the thread local run may not be full here. + // Count the number of free slots left. + size_t num_free_slots = thread_local_run->NumberOfFreeSlots(); + free_bytes += num_free_slots * bracketSizes[idx]; bool dont_care; thread_local_run->MergeThreadLocalFreeBitMapToAllocBitMap(&dont_care); thread_local_run->SetIsThreadLocal(false); @@ -1677,6 +1679,7 @@ void RosAlloc::RevokeThreadLocalRuns(Thread* thread) { RevokeRun(self, idx, thread_local_run); } } + return free_bytes; } void RosAlloc::RevokeRun(Thread* self, size_t idx, Run* run) { @@ -1719,16 +1722,18 @@ void RosAlloc::RevokeThreadUnsafeCurrentRuns() { } } -void RosAlloc::RevokeAllThreadLocalRuns() { +size_t RosAlloc::RevokeAllThreadLocalRuns() { // This is called when a mutator thread won't allocate such as at // the Zygote creation time or during the GC pause. MutexLock mu(Thread::Current(), *Locks::runtime_shutdown_lock_); MutexLock mu2(Thread::Current(), *Locks::thread_list_lock_); std::list<Thread*> thread_list = Runtime::Current()->GetThreadList()->GetList(); + size_t free_bytes = 0U; for (Thread* thread : thread_list) { - RevokeThreadLocalRuns(thread); + free_bytes += RevokeThreadLocalRuns(thread); } RevokeThreadUnsafeCurrentRuns(); + return free_bytes; } void RosAlloc::AssertThreadLocalRunsAreRevoked(Thread* thread) { diff --git a/runtime/gc/allocator/rosalloc.h b/runtime/gc/allocator/rosalloc.h index 3269e102bc..d1e7ad91a0 100644 --- a/runtime/gc/allocator/rosalloc.h +++ b/runtime/gc/allocator/rosalloc.h @@ -230,8 +230,10 @@ class RosAlloc { static uint32_t GetBitmapLastVectorMask(size_t num_slots, size_t num_vec); // Returns true if all the slots in the run are not in use. bool IsAllFree(); + // Returns the number of free slots. + size_t NumberOfFreeSlots(); // Returns true if all the slots in the run are in use. - bool IsFull(); + ALWAYS_INLINE bool IsFull(); // Returns true if the bulk free bit map is clean. bool IsBulkFreeBitmapClean(); // Returns true if the thread local free bit map is clean. @@ -309,6 +311,15 @@ class RosAlloc { DCHECK(bracketSizes[idx] == size); return idx; } + // Returns true if the given allocation size is for a thread local allocation. + static bool IsSizeForThreadLocal(size_t size) { + DCHECK_GT(kNumThreadLocalSizeBrackets, 0U); + size_t max_thread_local_bracket_idx = kNumThreadLocalSizeBrackets - 1; + bool is_size_for_thread_local = size <= bracketSizes[max_thread_local_bracket_idx]; + DCHECK(size > kLargeSizeThreshold || + (is_size_for_thread_local == (SizeToIndex(size) < kNumThreadLocalSizeBrackets))); + return is_size_for_thread_local; + } // Rounds up the size up the nearest bracket size. static size_t RoundToBracketSize(size_t size) { DCHECK(size <= kLargeSizeThreshold); @@ -504,11 +515,13 @@ class RosAlloc { size_t FreePages(Thread* self, void* ptr, bool already_zero) EXCLUSIVE_LOCKS_REQUIRED(lock_); // Allocate/free a run slot. - void* AllocFromRun(Thread* self, size_t size, size_t* bytes_allocated) + void* AllocFromRun(Thread* self, size_t size, size_t* bytes_allocated, size_t* usable_size, + size_t* bytes_tl_bulk_allocated) LOCKS_EXCLUDED(lock_); // Allocate/free a run slot without acquiring locks. // TODO: EXCLUSIVE_LOCKS_REQUIRED(Locks::mutator_lock_) - void* AllocFromRunThreadUnsafe(Thread* self, size_t size, size_t* bytes_allocated) + void* AllocFromRunThreadUnsafe(Thread* self, size_t size, size_t* bytes_allocated, + size_t* usable_size, size_t* bytes_tl_bulk_allocated) LOCKS_EXCLUDED(lock_); void* AllocFromCurrentRunUnlocked(Thread* self, size_t idx); @@ -527,7 +540,9 @@ class RosAlloc { size_t FreeInternal(Thread* self, void* ptr) LOCKS_EXCLUDED(lock_); // Allocates large objects. - void* AllocLargeObject(Thread* self, size_t size, size_t* bytes_allocated) LOCKS_EXCLUDED(lock_); + void* AllocLargeObject(Thread* self, size_t size, size_t* bytes_allocated, + size_t* usable_size, size_t* bytes_tl_bulk_allocated) + LOCKS_EXCLUDED(lock_); // Revoke a run by adding it to non_full_runs_ or freeing the pages. void RevokeRun(Thread* self, size_t idx, Run* run); @@ -551,13 +566,26 @@ class RosAlloc { // If kThreadUnsafe is true then the allocator may avoid acquiring some locks as an optimization. // If used, this may cause race conditions if multiple threads are allocating at the same time. template<bool kThreadSafe = true> - void* Alloc(Thread* self, size_t size, size_t* bytes_allocated) + void* Alloc(Thread* self, size_t size, size_t* bytes_allocated, size_t* usable_size, + size_t* bytes_tl_bulk_allocated) LOCKS_EXCLUDED(lock_); size_t Free(Thread* self, void* ptr) LOCKS_EXCLUDED(bulk_free_lock_); size_t BulkFree(Thread* self, void** ptrs, size_t num_ptrs) LOCKS_EXCLUDED(bulk_free_lock_); + // Returns true if the given allocation request can be allocated in + // an existing thread local run without allocating a new run. + ALWAYS_INLINE bool CanAllocFromThreadLocalRun(Thread* self, size_t size); + // Allocate the given allocation request in an existing thread local + // run without allocating a new run. + ALWAYS_INLINE void* AllocFromThreadLocalRun(Thread* self, size_t size, size_t* bytes_allocated); + + // Returns the maximum bytes that could be allocated for the given + // size in bulk, that is the maximum value for the + // bytes_allocated_bulk out param returned by RosAlloc::Alloc(). + ALWAYS_INLINE size_t MaxBytesBulkAllocatedFor(size_t size); + // Returns the size of the allocated slot for a given allocated memory chunk. size_t UsableSize(const void* ptr); // Returns the size of the allocated slot for a given size. @@ -586,9 +614,13 @@ class RosAlloc { void SetFootprintLimit(size_t bytes) LOCKS_EXCLUDED(lock_); // Releases the thread-local runs assigned to the given thread back to the common set of runs. - void RevokeThreadLocalRuns(Thread* thread); + // Returns the total bytes of free slots in the revoked thread local runs. This is to be + // subtracted from Heap::num_bytes_allocated_ to cancel out the ahead-of-time counting. + size_t RevokeThreadLocalRuns(Thread* thread); // Releases the thread-local runs assigned to all the threads back to the common set of runs. - void RevokeAllThreadLocalRuns() LOCKS_EXCLUDED(Locks::thread_list_lock_); + // Returns the total bytes of free slots in the revoked thread local runs. This is to be + // subtracted from Heap::num_bytes_allocated_ to cancel out the ahead-of-time counting. + size_t RevokeAllThreadLocalRuns() LOCKS_EXCLUDED(Locks::thread_list_lock_); // Assert the thread local runs of a thread are revoked. void AssertThreadLocalRunsAreRevoked(Thread* thread); // Assert all the thread local runs are revoked. diff --git a/runtime/gc/collector/concurrent_copying.cc b/runtime/gc/collector/concurrent_copying.cc index dd45ecab7f..db7a4ef7e7 100644 --- a/runtime/gc/collector/concurrent_copying.cc +++ b/runtime/gc/collector/concurrent_copying.cc @@ -1259,8 +1259,9 @@ mirror::Object* ConcurrentCopying::Copy(mirror::Object* from_ref) { size_t region_space_bytes_allocated = 0U; size_t non_moving_space_bytes_allocated = 0U; size_t bytes_allocated = 0U; + size_t dummy; mirror::Object* to_ref = region_space_->AllocNonvirtual<true>( - region_space_alloc_size, ®ion_space_bytes_allocated, nullptr); + region_space_alloc_size, ®ion_space_bytes_allocated, nullptr, &dummy); bytes_allocated = region_space_bytes_allocated; if (to_ref != nullptr) { DCHECK_EQ(region_space_alloc_size, region_space_bytes_allocated); @@ -1286,7 +1287,7 @@ mirror::Object* ConcurrentCopying::Copy(mirror::Object* from_ref) { } fall_back_to_non_moving = true; to_ref = heap_->non_moving_space_->Alloc(Thread::Current(), obj_size, - &non_moving_space_bytes_allocated, nullptr); + &non_moving_space_bytes_allocated, nullptr, &dummy); CHECK(to_ref != nullptr) << "Fall-back non-moving space allocation failed"; bytes_allocated = non_moving_space_bytes_allocated; // Mark it in the mark bitmap. diff --git a/runtime/gc/collector/garbage_collector.cc b/runtime/gc/collector/garbage_collector.cc index 8be18be676..eafcc45a13 100644 --- a/runtime/gc/collector/garbage_collector.cc +++ b/runtime/gc/collector/garbage_collector.cc @@ -48,6 +48,7 @@ void Iteration::Reset(GcCause gc_cause, bool clear_soft_references) { gc_cause_ = gc_cause; freed_ = ObjectBytePair(); freed_los_ = ObjectBytePair(); + freed_bytes_revoke_ = 0; } uint64_t Iteration::GetEstimatedThroughput() const { diff --git a/runtime/gc/collector/garbage_collector.h b/runtime/gc/collector/garbage_collector.h index b8094694b0..ed5207a356 100644 --- a/runtime/gc/collector/garbage_collector.h +++ b/runtime/gc/collector/garbage_collector.h @@ -75,6 +75,12 @@ class Iteration { uint64_t GetFreedLargeObjects() const { return freed_los_.objects; } + uint64_t GetFreedRevokeBytes() const { + return freed_bytes_revoke_; + } + void SetFreedRevoke(uint64_t freed) { + freed_bytes_revoke_ = freed; + } void Reset(GcCause gc_cause, bool clear_soft_references); // Returns the estimated throughput of the iteration. uint64_t GetEstimatedThroughput() const; @@ -99,6 +105,7 @@ class Iteration { TimingLogger timings_; ObjectBytePair freed_; ObjectBytePair freed_los_; + uint64_t freed_bytes_revoke_; // see Heap::num_bytes_freed_revoke_. std::vector<uint64_t> pause_times_; friend class GarbageCollector; diff --git a/runtime/gc/collector/mark_sweep.cc b/runtime/gc/collector/mark_sweep.cc index 8aac484f7f..ee4e752608 100644 --- a/runtime/gc/collector/mark_sweep.cc +++ b/runtime/gc/collector/mark_sweep.cc @@ -292,6 +292,7 @@ void MarkSweep::ReclaimPhase() { Runtime::Current()->AllowNewSystemWeaks(); { WriterMutexLock mu(self, *Locks::heap_bitmap_lock_); + GetHeap()->RecordFreeRevoke(); // Reclaim unmarked objects. Sweep(false); // Swap the live and mark bitmaps for each space which we modified space. This is an diff --git a/runtime/gc/collector/semi_space.cc b/runtime/gc/collector/semi_space.cc index c1ba5e3f72..b3d59f2a51 100644 --- a/runtime/gc/collector/semi_space.cc +++ b/runtime/gc/collector/semi_space.cc @@ -242,6 +242,7 @@ void SemiSpace::MarkingPhase() { // Revoke buffers before measuring how many objects were moved since the TLABs need to be revoked // before they are properly counted. RevokeAllThreadLocalBuffers(); + GetHeap()->RecordFreeRevoke(); // this is for the non-moving rosalloc space used by GSS. // Record freed memory. const int64_t from_bytes = from_space_->GetBytesAllocated(); const int64_t to_bytes = bytes_moved_; @@ -489,17 +490,18 @@ static inline size_t CopyAvoidingDirtyingPages(void* dest, const void* src, size mirror::Object* SemiSpace::MarkNonForwardedObject(mirror::Object* obj) { const size_t object_size = obj->SizeOf(); - size_t bytes_allocated; + size_t bytes_allocated, dummy; mirror::Object* forward_address = nullptr; if (generational_ && reinterpret_cast<uint8_t*>(obj) < last_gc_to_space_end_) { // If it's allocated before the last GC (older), move // (pseudo-promote) it to the main free list space (as sort // of an old generation.) forward_address = promo_dest_space_->AllocThreadUnsafe(self_, object_size, &bytes_allocated, - nullptr); + nullptr, &dummy); if (UNLIKELY(forward_address == nullptr)) { // If out of space, fall back to the to-space. - forward_address = to_space_->AllocThreadUnsafe(self_, object_size, &bytes_allocated, nullptr); + forward_address = to_space_->AllocThreadUnsafe(self_, object_size, &bytes_allocated, nullptr, + &dummy); // No logic for marking the bitmap, so it must be null. DCHECK(to_space_live_bitmap_ == nullptr); } else { @@ -544,7 +546,8 @@ mirror::Object* SemiSpace::MarkNonForwardedObject(mirror::Object* obj) { } } else { // If it's allocated after the last GC (younger), copy it to the to-space. - forward_address = to_space_->AllocThreadUnsafe(self_, object_size, &bytes_allocated, nullptr); + forward_address = to_space_->AllocThreadUnsafe(self_, object_size, &bytes_allocated, nullptr, + &dummy); if (forward_address != nullptr && to_space_live_bitmap_ != nullptr) { to_space_live_bitmap_->Set(forward_address); } @@ -552,7 +555,7 @@ mirror::Object* SemiSpace::MarkNonForwardedObject(mirror::Object* obj) { // If it's still null, attempt to use the fallback space. if (UNLIKELY(forward_address == nullptr)) { forward_address = fallback_space_->AllocThreadUnsafe(self_, object_size, &bytes_allocated, - nullptr); + nullptr, &dummy); CHECK(forward_address != nullptr) << "Out of memory in the to-space and fallback space."; accounting::ContinuousSpaceBitmap* bitmap = fallback_space_->GetLiveBitmap(); if (bitmap != nullptr) { diff --git a/runtime/gc/heap-inl.h b/runtime/gc/heap-inl.h index b8c24521a2..b770096671 100644 --- a/runtime/gc/heap-inl.h +++ b/runtime/gc/heap-inl.h @@ -64,6 +64,7 @@ inline mirror::Object* Heap::AllocObjectWithAllocator(Thread* self, mirror::Clas // fragmentation. } AllocationTimer alloc_timer(this, &obj); + // bytes allocated for the (individual) object. size_t bytes_allocated; size_t usable_size; size_t new_num_bytes_allocated = 0; @@ -86,13 +87,29 @@ inline mirror::Object* Heap::AllocObjectWithAllocator(Thread* self, mirror::Clas usable_size = bytes_allocated; pre_fence_visitor(obj, usable_size); QuasiAtomic::ThreadFenceForConstructor(); + } else if (!kInstrumented && allocator == kAllocatorTypeRosAlloc && + (obj = rosalloc_space_->AllocThreadLocal(self, byte_count, &bytes_allocated)) && + LIKELY(obj != nullptr)) { + DCHECK(!running_on_valgrind_); + obj->SetClass(klass); + if (kUseBakerOrBrooksReadBarrier) { + if (kUseBrooksReadBarrier) { + obj->SetReadBarrierPointer(obj); + } + obj->AssertReadBarrierPointer(); + } + usable_size = bytes_allocated; + pre_fence_visitor(obj, usable_size); + QuasiAtomic::ThreadFenceForConstructor(); } else { + // bytes allocated that takes bulk thread-local buffer allocations into account. + size_t bytes_tl_bulk_allocated = 0; obj = TryToAllocate<kInstrumented, false>(self, allocator, byte_count, &bytes_allocated, - &usable_size); + &usable_size, &bytes_tl_bulk_allocated); if (UNLIKELY(obj == nullptr)) { bool is_current_allocator = allocator == GetCurrentAllocator(); obj = AllocateInternalWithGc(self, allocator, byte_count, &bytes_allocated, &usable_size, - &klass); + &bytes_tl_bulk_allocated, &klass); if (obj == nullptr) { bool after_is_current_allocator = allocator == GetCurrentAllocator(); // If there is a pending exception, fail the allocation right away since the next one @@ -126,9 +143,9 @@ inline mirror::Object* Heap::AllocObjectWithAllocator(Thread* self, mirror::Clas WriteBarrierField(obj, mirror::Object::ClassOffset(), klass); } pre_fence_visitor(obj, usable_size); - new_num_bytes_allocated = - static_cast<size_t>(num_bytes_allocated_.FetchAndAddSequentiallyConsistent(bytes_allocated)) - + bytes_allocated; + new_num_bytes_allocated = static_cast<size_t>( + num_bytes_allocated_.FetchAndAddSequentiallyConsistent(bytes_tl_bulk_allocated)) + + bytes_tl_bulk_allocated; } if (kIsDebugBuild && Runtime::Current()->IsStarted()) { CHECK_LE(obj->SizeOf(), usable_size); @@ -196,8 +213,10 @@ inline mirror::Object* Heap::AllocLargeObject(Thread* self, mirror::Class** klas template <const bool kInstrumented, const bool kGrow> inline mirror::Object* Heap::TryToAllocate(Thread* self, AllocatorType allocator_type, size_t alloc_size, size_t* bytes_allocated, - size_t* usable_size) { + size_t* usable_size, + size_t* bytes_tl_bulk_allocated) { if (allocator_type != kAllocatorTypeTLAB && allocator_type != kAllocatorTypeRegionTLAB && + allocator_type != kAllocatorTypeRosAlloc && UNLIKELY(IsOutOfMemoryOnAllocation<kGrow>(allocator_type, alloc_size))) { return nullptr; } @@ -210,35 +229,56 @@ inline mirror::Object* Heap::TryToAllocate(Thread* self, AllocatorType allocator if (LIKELY(ret != nullptr)) { *bytes_allocated = alloc_size; *usable_size = alloc_size; + *bytes_tl_bulk_allocated = alloc_size; } break; } case kAllocatorTypeRosAlloc: { if (kInstrumented && UNLIKELY(running_on_valgrind_)) { // If running on valgrind, we should be using the instrumented path. - ret = rosalloc_space_->Alloc(self, alloc_size, bytes_allocated, usable_size); + size_t max_bytes_tl_bulk_allocated = rosalloc_space_->MaxBytesBulkAllocatedFor(alloc_size); + if (UNLIKELY(IsOutOfMemoryOnAllocation<kGrow>(allocator_type, + max_bytes_tl_bulk_allocated))) { + return nullptr; + } + ret = rosalloc_space_->Alloc(self, alloc_size, bytes_allocated, usable_size, + bytes_tl_bulk_allocated); } else { DCHECK(!running_on_valgrind_); - ret = rosalloc_space_->AllocNonvirtual(self, alloc_size, bytes_allocated, usable_size); + size_t max_bytes_tl_bulk_allocated = + rosalloc_space_->MaxBytesBulkAllocatedForNonvirtual(alloc_size); + if (UNLIKELY(IsOutOfMemoryOnAllocation<kGrow>(allocator_type, + max_bytes_tl_bulk_allocated))) { + return nullptr; + } + if (!kInstrumented) { + DCHECK(!rosalloc_space_->CanAllocThreadLocal(self, alloc_size)); + } + ret = rosalloc_space_->AllocNonvirtual(self, alloc_size, bytes_allocated, usable_size, + bytes_tl_bulk_allocated); } break; } case kAllocatorTypeDlMalloc: { if (kInstrumented && UNLIKELY(running_on_valgrind_)) { // If running on valgrind, we should be using the instrumented path. - ret = dlmalloc_space_->Alloc(self, alloc_size, bytes_allocated, usable_size); + ret = dlmalloc_space_->Alloc(self, alloc_size, bytes_allocated, usable_size, + bytes_tl_bulk_allocated); } else { DCHECK(!running_on_valgrind_); - ret = dlmalloc_space_->AllocNonvirtual(self, alloc_size, bytes_allocated, usable_size); + ret = dlmalloc_space_->AllocNonvirtual(self, alloc_size, bytes_allocated, usable_size, + bytes_tl_bulk_allocated); } break; } case kAllocatorTypeNonMoving: { - ret = non_moving_space_->Alloc(self, alloc_size, bytes_allocated, usable_size); + ret = non_moving_space_->Alloc(self, alloc_size, bytes_allocated, usable_size, + bytes_tl_bulk_allocated); break; } case kAllocatorTypeLOS: { - ret = large_object_space_->Alloc(self, alloc_size, bytes_allocated, usable_size); + ret = large_object_space_->Alloc(self, alloc_size, bytes_allocated, usable_size, + bytes_tl_bulk_allocated); // Note that the bump pointer spaces aren't necessarily next to // the other continuous spaces like the non-moving alloc space or // the zygote space. @@ -257,20 +297,22 @@ inline mirror::Object* Heap::TryToAllocate(Thread* self, AllocatorType allocator if (!bump_pointer_space_->AllocNewTlab(self, new_tlab_size)) { return nullptr; } - *bytes_allocated = new_tlab_size; + *bytes_tl_bulk_allocated = new_tlab_size; } else { - *bytes_allocated = 0; + *bytes_tl_bulk_allocated = 0; } // The allocation can't fail. ret = self->AllocTlab(alloc_size); DCHECK(ret != nullptr); + *bytes_allocated = alloc_size; *usable_size = alloc_size; break; } case kAllocatorTypeRegion: { DCHECK(region_space_ != nullptr); alloc_size = RoundUp(alloc_size, space::RegionSpace::kAlignment); - ret = region_space_->AllocNonvirtual<false>(alloc_size, bytes_allocated, usable_size); + ret = region_space_->AllocNonvirtual<false>(alloc_size, bytes_allocated, usable_size, + bytes_tl_bulk_allocated); break; } case kAllocatorTypeRegionTLAB: { @@ -283,15 +325,17 @@ inline mirror::Object* Heap::TryToAllocate(Thread* self, AllocatorType allocator // Try to allocate a tlab. if (!region_space_->AllocNewTlab(self)) { // Failed to allocate a tlab. Try non-tlab. - ret = region_space_->AllocNonvirtual<false>(alloc_size, bytes_allocated, usable_size); + ret = region_space_->AllocNonvirtual<false>(alloc_size, bytes_allocated, usable_size, + bytes_tl_bulk_allocated); return ret; } - *bytes_allocated = space::RegionSpace::kRegionSize; + *bytes_tl_bulk_allocated = space::RegionSpace::kRegionSize; // Fall-through. } else { // Check OOME for a non-tlab allocation. if (!IsOutOfMemoryOnAllocation<kGrow>(allocator_type, alloc_size)) { - ret = region_space_->AllocNonvirtual<false>(alloc_size, bytes_allocated, usable_size); + ret = region_space_->AllocNonvirtual<false>(alloc_size, bytes_allocated, usable_size, + bytes_tl_bulk_allocated); return ret; } else { // Neither tlab or non-tlab works. Give up. @@ -301,18 +345,20 @@ inline mirror::Object* Heap::TryToAllocate(Thread* self, AllocatorType allocator } else { // Large. Check OOME. if (LIKELY(!IsOutOfMemoryOnAllocation<kGrow>(allocator_type, alloc_size))) { - ret = region_space_->AllocNonvirtual<false>(alloc_size, bytes_allocated, usable_size); + ret = region_space_->AllocNonvirtual<false>(alloc_size, bytes_allocated, usable_size, + bytes_tl_bulk_allocated); return ret; } else { return nullptr; } } } else { - *bytes_allocated = 0; + *bytes_tl_bulk_allocated = 0; // Allocated in an existing buffer. } // The allocation can't fail. ret = self->AllocTlab(alloc_size); DCHECK(ret != nullptr); + *bytes_allocated = alloc_size; *usable_size = alloc_size; break; } diff --git a/runtime/gc/heap.cc b/runtime/gc/heap.cc index 7534515a8a..a41d65ce4a 100644 --- a/runtime/gc/heap.cc +++ b/runtime/gc/heap.cc @@ -156,6 +156,7 @@ Heap::Heap(size_t initial_size, size_t growth_limit, size_t min_free, size_t max total_objects_freed_ever_(0), num_bytes_allocated_(0), native_bytes_allocated_(0), + num_bytes_freed_revoke_(0), verify_missing_card_marks_(false), verify_system_weaks_(false), verify_pre_gc_heap_(verify_pre_gc_heap), @@ -1344,6 +1345,19 @@ void Heap::RecordFree(uint64_t freed_objects, int64_t freed_bytes) { } } +void Heap::RecordFreeRevoke() { + // Subtract num_bytes_freed_revoke_ from num_bytes_allocated_ to cancel out the + // the ahead-of-time, bulk counting of bytes allocated in rosalloc thread-local buffers. + // If there's a concurrent revoke, ok to not necessarily reset num_bytes_freed_revoke_ + // all the way to zero exactly as the remainder will be subtracted at the next GC. + size_t bytes_freed = num_bytes_freed_revoke_.LoadSequentiallyConsistent(); + CHECK_GE(num_bytes_freed_revoke_.FetchAndSubSequentiallyConsistent(bytes_freed), + bytes_freed) << "num_bytes_freed_revoke_ underflow"; + CHECK_GE(num_bytes_allocated_.FetchAndSubSequentiallyConsistent(bytes_freed), + bytes_freed) << "num_bytes_allocated_ underflow"; + GetCurrentGcIteration()->SetFreedRevoke(bytes_freed); +} + space::RosAllocSpace* Heap::GetRosAllocSpace(gc::allocator::RosAlloc* rosalloc) const { for (const auto& space : continuous_spaces_) { if (space->AsContinuousSpace()->IsRosAllocSpace()) { @@ -1358,6 +1372,7 @@ space::RosAllocSpace* Heap::GetRosAllocSpace(gc::allocator::RosAlloc* rosalloc) mirror::Object* Heap::AllocateInternalWithGc(Thread* self, AllocatorType allocator, size_t alloc_size, size_t* bytes_allocated, size_t* usable_size, + size_t* bytes_tl_bulk_allocated, mirror::Class** klass) { bool was_default_allocator = allocator == GetCurrentAllocator(); // Make sure there is no pending exception since we may need to throw an OOME. @@ -1377,7 +1392,7 @@ mirror::Object* Heap::AllocateInternalWithGc(Thread* self, AllocatorType allocat } // A GC was in progress and we blocked, retry allocation now that memory has been freed. mirror::Object* ptr = TryToAllocate<true, false>(self, allocator, alloc_size, bytes_allocated, - usable_size); + usable_size, bytes_tl_bulk_allocated); if (ptr != nullptr) { return ptr; } @@ -1391,7 +1406,7 @@ mirror::Object* Heap::AllocateInternalWithGc(Thread* self, AllocatorType allocat } if (gc_ran) { mirror::Object* ptr = TryToAllocate<true, false>(self, allocator, alloc_size, bytes_allocated, - usable_size); + usable_size, bytes_tl_bulk_allocated); if (ptr != nullptr) { return ptr; } @@ -1411,7 +1426,7 @@ mirror::Object* Heap::AllocateInternalWithGc(Thread* self, AllocatorType allocat if (plan_gc_ran) { // Did we free sufficient memory for the allocation to succeed? mirror::Object* ptr = TryToAllocate<true, false>(self, allocator, alloc_size, bytes_allocated, - usable_size); + usable_size, bytes_tl_bulk_allocated); if (ptr != nullptr) { return ptr; } @@ -1420,7 +1435,7 @@ mirror::Object* Heap::AllocateInternalWithGc(Thread* self, AllocatorType allocat // Allocations have failed after GCs; this is an exceptional state. // Try harder, growing the heap if necessary. mirror::Object* ptr = TryToAllocate<true, true>(self, allocator, alloc_size, bytes_allocated, - usable_size); + usable_size, bytes_tl_bulk_allocated); if (ptr != nullptr) { return ptr; } @@ -1437,7 +1452,8 @@ mirror::Object* Heap::AllocateInternalWithGc(Thread* self, AllocatorType allocat if (was_default_allocator && allocator != GetCurrentAllocator()) { return nullptr; } - ptr = TryToAllocate<true, true>(self, allocator, alloc_size, bytes_allocated, usable_size); + ptr = TryToAllocate<true, true>(self, allocator, alloc_size, bytes_allocated, usable_size, + bytes_tl_bulk_allocated); if (ptr == nullptr) { const uint64_t current_time = NanoTime(); switch (allocator) { @@ -1453,7 +1469,7 @@ mirror::Object* Heap::AllocateInternalWithGc(Thread* self, AllocatorType allocat case HomogeneousSpaceCompactResult::kSuccess: // If the allocation succeeded, we delayed an oom. ptr = TryToAllocate<true, true>(self, allocator, alloc_size, bytes_allocated, - usable_size); + usable_size, bytes_tl_bulk_allocated); if (ptr != nullptr) { count_delayed_oom_++; } @@ -1498,7 +1514,7 @@ mirror::Object* Heap::AllocateInternalWithGc(Thread* self, AllocatorType allocat } else { LOG(WARNING) << "Disabled moving GC due to the non moving space being full"; ptr = TryToAllocate<true, true>(self, allocator, alloc_size, bytes_allocated, - usable_size); + usable_size, bytes_tl_bulk_allocated); } } break; @@ -1984,8 +2000,8 @@ class ZygoteCompactingCollector FINAL : public collector::SemiSpace { if (it == bins_.end()) { // No available space in the bins, place it in the target space instead (grows the zygote // space). - size_t bytes_allocated; - forward_address = to_space_->Alloc(self_, object_size, &bytes_allocated, nullptr); + size_t bytes_allocated, dummy; + forward_address = to_space_->Alloc(self_, object_size, &bytes_allocated, nullptr, &dummy); if (to_space_live_bitmap_ != nullptr) { to_space_live_bitmap_->Set(forward_address); } else { @@ -3084,7 +3100,8 @@ void Heap::GrowForUtilization(collector::GarbageCollector* collector_ran, SetIdealFootprint(target_size); if (IsGcConcurrent()) { const uint64_t freed_bytes = current_gc_iteration_.GetFreedBytes() + - current_gc_iteration_.GetFreedLargeObjectBytes(); + current_gc_iteration_.GetFreedLargeObjectBytes() + + current_gc_iteration_.GetFreedRevokeBytes(); // Bytes allocated will shrink by freed_bytes after the GC runs, so if we want to figure out // how many bytes were allocated during the GC we need to add freed_bytes back on. CHECK_GE(bytes_allocated + freed_bytes, bytes_allocated_before_gc); @@ -3290,31 +3307,43 @@ void Heap::RequestTrim(Thread* self) { void Heap::RevokeThreadLocalBuffers(Thread* thread) { if (rosalloc_space_ != nullptr) { - rosalloc_space_->RevokeThreadLocalBuffers(thread); + size_t freed_bytes_revoke = rosalloc_space_->RevokeThreadLocalBuffers(thread); + if (freed_bytes_revoke > 0U) { + num_bytes_freed_revoke_.FetchAndAddSequentiallyConsistent(freed_bytes_revoke); + CHECK_GE(num_bytes_allocated_.LoadRelaxed(), num_bytes_freed_revoke_.LoadRelaxed()); + } } if (bump_pointer_space_ != nullptr) { - bump_pointer_space_->RevokeThreadLocalBuffers(thread); + CHECK_EQ(bump_pointer_space_->RevokeThreadLocalBuffers(thread), 0U); } if (region_space_ != nullptr) { - region_space_->RevokeThreadLocalBuffers(thread); + CHECK_EQ(region_space_->RevokeThreadLocalBuffers(thread), 0U); } } void Heap::RevokeRosAllocThreadLocalBuffers(Thread* thread) { if (rosalloc_space_ != nullptr) { - rosalloc_space_->RevokeThreadLocalBuffers(thread); + size_t freed_bytes_revoke = rosalloc_space_->RevokeThreadLocalBuffers(thread); + if (freed_bytes_revoke > 0U) { + num_bytes_freed_revoke_.FetchAndAddSequentiallyConsistent(freed_bytes_revoke); + CHECK_GE(num_bytes_allocated_.LoadRelaxed(), num_bytes_freed_revoke_.LoadRelaxed()); + } } } void Heap::RevokeAllThreadLocalBuffers() { if (rosalloc_space_ != nullptr) { - rosalloc_space_->RevokeAllThreadLocalBuffers(); + size_t freed_bytes_revoke = rosalloc_space_->RevokeAllThreadLocalBuffers(); + if (freed_bytes_revoke > 0U) { + num_bytes_freed_revoke_.FetchAndAddSequentiallyConsistent(freed_bytes_revoke); + CHECK_GE(num_bytes_allocated_.LoadRelaxed(), num_bytes_freed_revoke_.LoadRelaxed()); + } } if (bump_pointer_space_ != nullptr) { - bump_pointer_space_->RevokeAllThreadLocalBuffers(); + CHECK_EQ(bump_pointer_space_->RevokeAllThreadLocalBuffers(), 0U); } if (region_space_ != nullptr) { - region_space_->RevokeAllThreadLocalBuffers(); + CHECK_EQ(region_space_->RevokeAllThreadLocalBuffers(), 0U); } } diff --git a/runtime/gc/heap.h b/runtime/gc/heap.h index d41e17fb75..959ff18516 100644 --- a/runtime/gc/heap.h +++ b/runtime/gc/heap.h @@ -390,6 +390,9 @@ class Heap { // free-list backed space. void RecordFree(uint64_t freed_objects, int64_t freed_bytes); + // Record the bytes freed by thread-local buffer revoke. + void RecordFreeRevoke(); + // Must be called if a field of an Object in the heap changes, and before any GC safe-point. // The call is not needed if NULL is stored in the field. ALWAYS_INLINE void WriteBarrierField(const mirror::Object* dst, MemberOffset /*offset*/, @@ -664,6 +667,11 @@ class Heap { // Whether or not we may use a garbage collector, used so that we only create collectors we need. bool MayUseCollector(CollectorType type) const; + // Used by tests to reduce timinig-dependent flakiness in OOME behavior. + void SetMinIntervalHomogeneousSpaceCompactionByOom(uint64_t interval) { + min_interval_homogeneous_space_compaction_by_oom_ = interval; + } + private: class ConcurrentGCTask; class CollectorTransitionTask; @@ -724,6 +732,7 @@ class Heap { // an initial allocation attempt failed. mirror::Object* AllocateInternalWithGc(Thread* self, AllocatorType allocator, size_t num_bytes, size_t* bytes_allocated, size_t* usable_size, + size_t* bytes_tl_bulk_allocated, mirror::Class** klass) LOCKS_EXCLUDED(Locks::thread_suspend_count_lock_) SHARED_LOCKS_REQUIRED(Locks::mutator_lock_); @@ -742,7 +751,8 @@ class Heap { template <const bool kInstrumented, const bool kGrow> ALWAYS_INLINE mirror::Object* TryToAllocate(Thread* self, AllocatorType allocator_type, size_t alloc_size, size_t* bytes_allocated, - size_t* usable_size) + size_t* usable_size, + size_t* bytes_tl_bulk_allocated) SHARED_LOCKS_REQUIRED(Locks::mutator_lock_); void ThrowOutOfMemoryError(Thread* self, size_t byte_count, AllocatorType allocator_type) @@ -998,6 +1008,13 @@ class Heap { // Bytes which are allocated and managed by native code but still need to be accounted for. Atomic<size_t> native_bytes_allocated_; + // Number of bytes freed by thread local buffer revokes. This will + // cancel out the ahead-of-time bulk counting of bytes allocated in + // rosalloc thread-local buffers. It is temporarily accumulated + // here to be subtracted from num_bytes_allocated_ later at the next + // GC. + Atomic<size_t> num_bytes_freed_revoke_; + // Info related to the current or previous GC iteration. collector::Iteration current_gc_iteration_; diff --git a/runtime/gc/space/bump_pointer_space-inl.h b/runtime/gc/space/bump_pointer_space-inl.h index 9f1f9533d0..14a93d1611 100644 --- a/runtime/gc/space/bump_pointer_space-inl.h +++ b/runtime/gc/space/bump_pointer_space-inl.h @@ -24,7 +24,8 @@ namespace gc { namespace space { inline mirror::Object* BumpPointerSpace::Alloc(Thread*, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) { + size_t* usable_size, + size_t* bytes_tl_bulk_allocated) { num_bytes = RoundUp(num_bytes, kAlignment); mirror::Object* ret = AllocNonvirtual(num_bytes); if (LIKELY(ret != nullptr)) { @@ -32,13 +33,15 @@ inline mirror::Object* BumpPointerSpace::Alloc(Thread*, size_t num_bytes, size_t if (usable_size != nullptr) { *usable_size = num_bytes; } + *bytes_tl_bulk_allocated = num_bytes; } return ret; } inline mirror::Object* BumpPointerSpace::AllocThreadUnsafe(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) { + size_t* usable_size, + size_t* bytes_tl_bulk_allocated) { Locks::mutator_lock_->AssertExclusiveHeld(self); num_bytes = RoundUp(num_bytes, kAlignment); uint8_t* end = end_.LoadRelaxed(); @@ -54,6 +57,7 @@ inline mirror::Object* BumpPointerSpace::AllocThreadUnsafe(Thread* self, size_t if (UNLIKELY(usable_size != nullptr)) { *usable_size = num_bytes; } + *bytes_tl_bulk_allocated = num_bytes; return obj; } diff --git a/runtime/gc/space/bump_pointer_space.cc b/runtime/gc/space/bump_pointer_space.cc index fbfc4495e0..1303d7729e 100644 --- a/runtime/gc/space/bump_pointer_space.cc +++ b/runtime/gc/space/bump_pointer_space.cc @@ -93,12 +93,13 @@ mirror::Object* BumpPointerSpace::GetNextObject(mirror::Object* obj) { return reinterpret_cast<mirror::Object*>(RoundUp(position, kAlignment)); } -void BumpPointerSpace::RevokeThreadLocalBuffers(Thread* thread) { +size_t BumpPointerSpace::RevokeThreadLocalBuffers(Thread* thread) { MutexLock mu(Thread::Current(), block_lock_); RevokeThreadLocalBuffersLocked(thread); + return 0U; } -void BumpPointerSpace::RevokeAllThreadLocalBuffers() { +size_t BumpPointerSpace::RevokeAllThreadLocalBuffers() { Thread* self = Thread::Current(); MutexLock mu(self, *Locks::runtime_shutdown_lock_); MutexLock mu2(self, *Locks::thread_list_lock_); @@ -107,6 +108,7 @@ void BumpPointerSpace::RevokeAllThreadLocalBuffers() { for (Thread* thread : thread_list) { RevokeThreadLocalBuffers(thread); } + return 0U; } void BumpPointerSpace::AssertThreadLocalBuffersAreRevoked(Thread* thread) { diff --git a/runtime/gc/space/bump_pointer_space.h b/runtime/gc/space/bump_pointer_space.h index 089ede4453..c496a422e0 100644 --- a/runtime/gc/space/bump_pointer_space.h +++ b/runtime/gc/space/bump_pointer_space.h @@ -47,10 +47,10 @@ class BumpPointerSpace FINAL : public ContinuousMemMapAllocSpace { // Allocate num_bytes, returns nullptr if the space is full. mirror::Object* Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) OVERRIDE; + size_t* usable_size, size_t* bytes_tl_bulk_allocated) OVERRIDE; // Thread-unsafe allocation for when mutators are suspended, used by the semispace collector. mirror::Object* AllocThreadUnsafe(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) + size_t* usable_size, size_t* bytes_tl_bulk_allocated) OVERRIDE EXCLUSIVE_LOCKS_REQUIRED(Locks::mutator_lock_); mirror::Object* AllocNonvirtual(size_t num_bytes); @@ -103,9 +103,9 @@ class BumpPointerSpace FINAL : public ContinuousMemMapAllocSpace { void Dump(std::ostream& os) const; - void RevokeThreadLocalBuffers(Thread* thread) LOCKS_EXCLUDED(block_lock_); - void RevokeAllThreadLocalBuffers() LOCKS_EXCLUDED(Locks::runtime_shutdown_lock_, - Locks::thread_list_lock_); + size_t RevokeThreadLocalBuffers(Thread* thread) LOCKS_EXCLUDED(block_lock_); + size_t RevokeAllThreadLocalBuffers() LOCKS_EXCLUDED(Locks::runtime_shutdown_lock_, + Locks::thread_list_lock_); void AssertThreadLocalBuffersAreRevoked(Thread* thread) LOCKS_EXCLUDED(block_lock_); void AssertAllThreadLocalBuffersAreRevoked() LOCKS_EXCLUDED(Locks::runtime_shutdown_lock_, Locks::thread_list_lock_); diff --git a/runtime/gc/space/dlmalloc_space-inl.h b/runtime/gc/space/dlmalloc_space-inl.h index 4c8a35e0f7..9eace897e6 100644 --- a/runtime/gc/space/dlmalloc_space-inl.h +++ b/runtime/gc/space/dlmalloc_space-inl.h @@ -27,11 +27,13 @@ namespace space { inline mirror::Object* DlMallocSpace::AllocNonvirtual(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) { + size_t* usable_size, + size_t* bytes_tl_bulk_allocated) { mirror::Object* obj; { MutexLock mu(self, lock_); - obj = AllocWithoutGrowthLocked(self, num_bytes, bytes_allocated, usable_size); + obj = AllocWithoutGrowthLocked(self, num_bytes, bytes_allocated, usable_size, + bytes_tl_bulk_allocated); } if (LIKELY(obj != NULL)) { // Zero freshly allocated memory, done while not holding the space's lock. @@ -49,9 +51,11 @@ inline size_t DlMallocSpace::AllocationSizeNonvirtual(mirror::Object* obj, size_ return size + kChunkOverhead; } -inline mirror::Object* DlMallocSpace::AllocWithoutGrowthLocked(Thread* /*self*/, size_t num_bytes, - size_t* bytes_allocated, - size_t* usable_size) { +inline mirror::Object* DlMallocSpace::AllocWithoutGrowthLocked( + Thread* /*self*/, size_t num_bytes, + size_t* bytes_allocated, + size_t* usable_size, + size_t* bytes_tl_bulk_allocated) { mirror::Object* result = reinterpret_cast<mirror::Object*>(mspace_malloc(mspace_, num_bytes)); if (LIKELY(result != NULL)) { if (kDebugSpaces) { @@ -61,6 +65,7 @@ inline mirror::Object* DlMallocSpace::AllocWithoutGrowthLocked(Thread* /*self*/, size_t allocation_size = AllocationSizeNonvirtual(result, usable_size); DCHECK(bytes_allocated != NULL); *bytes_allocated = allocation_size; + *bytes_tl_bulk_allocated = allocation_size; } return result; } diff --git a/runtime/gc/space/dlmalloc_space.cc b/runtime/gc/space/dlmalloc_space.cc index b8a9dd6639..225861db60 100644 --- a/runtime/gc/space/dlmalloc_space.cc +++ b/runtime/gc/space/dlmalloc_space.cc @@ -123,7 +123,8 @@ void* DlMallocSpace::CreateMspace(void* begin, size_t morecore_start, size_t ini } mirror::Object* DlMallocSpace::AllocWithGrowth(Thread* self, size_t num_bytes, - size_t* bytes_allocated, size_t* usable_size) { + size_t* bytes_allocated, size_t* usable_size, + size_t* bytes_tl_bulk_allocated) { mirror::Object* result; { MutexLock mu(self, lock_); @@ -131,7 +132,8 @@ mirror::Object* DlMallocSpace::AllocWithGrowth(Thread* self, size_t num_bytes, size_t max_allowed = Capacity(); mspace_set_footprint_limit(mspace_, max_allowed); // Try the allocation. - result = AllocWithoutGrowthLocked(self, num_bytes, bytes_allocated, usable_size); + result = AllocWithoutGrowthLocked(self, num_bytes, bytes_allocated, usable_size, + bytes_tl_bulk_allocated); // Shrink back down as small as possible. size_t footprint = mspace_footprint(mspace_); mspace_set_footprint_limit(mspace_, footprint); diff --git a/runtime/gc/space/dlmalloc_space.h b/runtime/gc/space/dlmalloc_space.h index 6ce138c235..1f80f1fd6b 100644 --- a/runtime/gc/space/dlmalloc_space.h +++ b/runtime/gc/space/dlmalloc_space.h @@ -48,11 +48,15 @@ class DlMallocSpace : public MallocSpace { // Virtual to allow ValgrindMallocSpace to intercept. virtual mirror::Object* AllocWithGrowth(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) OVERRIDE LOCKS_EXCLUDED(lock_); + size_t* usable_size, + size_t* bytes_tl_bulk_allocated) + OVERRIDE LOCKS_EXCLUDED(lock_); // Virtual to allow ValgrindMallocSpace to intercept. virtual mirror::Object* Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) OVERRIDE LOCKS_EXCLUDED(lock_) { - return AllocNonvirtual(self, num_bytes, bytes_allocated, usable_size); + size_t* usable_size, size_t* bytes_tl_bulk_allocated) + OVERRIDE LOCKS_EXCLUDED(lock_) { + return AllocNonvirtual(self, num_bytes, bytes_allocated, usable_size, + bytes_tl_bulk_allocated); } // Virtual to allow ValgrindMallocSpace to intercept. virtual size_t AllocationSize(mirror::Object* obj, size_t* usable_size) OVERRIDE { @@ -67,15 +71,22 @@ class DlMallocSpace : public MallocSpace { LOCKS_EXCLUDED(lock_) SHARED_LOCKS_REQUIRED(Locks::mutator_lock_); + size_t MaxBytesBulkAllocatedFor(size_t num_bytes) OVERRIDE { + return num_bytes; + } + // DlMallocSpaces don't have thread local state. - void RevokeThreadLocalBuffers(art::Thread*) OVERRIDE { + size_t RevokeThreadLocalBuffers(art::Thread*) OVERRIDE { + return 0U; } - void RevokeAllThreadLocalBuffers() OVERRIDE { + size_t RevokeAllThreadLocalBuffers() OVERRIDE { + return 0U; } // Faster non-virtual allocation path. mirror::Object* AllocNonvirtual(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) LOCKS_EXCLUDED(lock_); + size_t* usable_size, size_t* bytes_tl_bulk_allocated) + LOCKS_EXCLUDED(lock_); // Faster non-virtual allocation size path. size_t AllocationSizeNonvirtual(mirror::Object* obj, size_t* usable_size); @@ -134,7 +145,8 @@ class DlMallocSpace : public MallocSpace { private: mirror::Object* AllocWithoutGrowthLocked(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) + size_t* usable_size, + size_t* bytes_tl_bulk_allocated) EXCLUSIVE_LOCKS_REQUIRED(lock_); void* CreateAllocator(void* base, size_t morecore_start, size_t initial_size, diff --git a/runtime/gc/space/large_object_space.cc b/runtime/gc/space/large_object_space.cc index 7523de58bf..5c8e4b9299 100644 --- a/runtime/gc/space/large_object_space.cc +++ b/runtime/gc/space/large_object_space.cc @@ -38,10 +38,11 @@ class ValgrindLargeObjectMapSpace FINAL : public LargeObjectMapSpace { } virtual mirror::Object* Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) OVERRIDE { + size_t* usable_size, size_t* bytes_tl_bulk_allocated) + OVERRIDE { mirror::Object* obj = LargeObjectMapSpace::Alloc(self, num_bytes + kValgrindRedZoneBytes * 2, bytes_allocated, - usable_size); + usable_size, bytes_tl_bulk_allocated); mirror::Object* object_without_rdz = reinterpret_cast<mirror::Object*>( reinterpret_cast<uintptr_t>(obj) + kValgrindRedZoneBytes); VALGRIND_MAKE_MEM_NOACCESS(reinterpret_cast<void*>(obj), kValgrindRedZoneBytes); @@ -108,7 +109,8 @@ LargeObjectMapSpace* LargeObjectMapSpace::Create(const std::string& name) { } mirror::Object* LargeObjectMapSpace::Alloc(Thread* self, size_t num_bytes, - size_t* bytes_allocated, size_t* usable_size) { + size_t* bytes_allocated, size_t* usable_size, + size_t* bytes_tl_bulk_allocated) { std::string error_msg; MemMap* mem_map = MemMap::MapAnonymous("large object space allocation", nullptr, num_bytes, PROT_READ | PROT_WRITE, true, false, &error_msg); @@ -131,6 +133,8 @@ mirror::Object* LargeObjectMapSpace::Alloc(Thread* self, size_t num_bytes, if (usable_size != nullptr) { *usable_size = allocation_size; } + DCHECK(bytes_tl_bulk_allocated != nullptr); + *bytes_tl_bulk_allocated = allocation_size; num_bytes_allocated_ += allocation_size; total_bytes_allocated_ += allocation_size; ++num_objects_allocated_; @@ -413,7 +417,7 @@ size_t FreeListSpace::AllocationSize(mirror::Object* obj, size_t* usable_size) { } mirror::Object* FreeListSpace::Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) { + size_t* usable_size, size_t* bytes_tl_bulk_allocated) { MutexLock mu(self, lock_); const size_t allocation_size = RoundUp(num_bytes, kAlignment); AllocationInfo temp_info; @@ -451,6 +455,8 @@ mirror::Object* FreeListSpace::Alloc(Thread* self, size_t num_bytes, size_t* byt if (usable_size != nullptr) { *usable_size = allocation_size; } + DCHECK(bytes_tl_bulk_allocated != nullptr); + *bytes_tl_bulk_allocated = allocation_size; // Need to do these inside of the lock. ++num_objects_allocated_; ++total_objects_allocated_; diff --git a/runtime/gc/space/large_object_space.h b/runtime/gc/space/large_object_space.h index 847f575815..d1f9386d09 100644 --- a/runtime/gc/space/large_object_space.h +++ b/runtime/gc/space/large_object_space.h @@ -62,9 +62,11 @@ class LargeObjectSpace : public DiscontinuousSpace, public AllocSpace { } size_t FreeList(Thread* self, size_t num_ptrs, mirror::Object** ptrs) OVERRIDE; // LargeObjectSpaces don't have thread local state. - void RevokeThreadLocalBuffers(art::Thread*) OVERRIDE { + size_t RevokeThreadLocalBuffers(art::Thread*) OVERRIDE { + return 0U; } - void RevokeAllThreadLocalBuffers() OVERRIDE { + size_t RevokeAllThreadLocalBuffers() OVERRIDE { + return 0U; } bool IsAllocSpace() const OVERRIDE { return true; @@ -124,7 +126,7 @@ class LargeObjectMapSpace : public LargeObjectSpace { // Return the storage space required by obj. size_t AllocationSize(mirror::Object* obj, size_t* usable_size); mirror::Object* Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size); + size_t* usable_size, size_t* bytes_tl_bulk_allocated); size_t Free(Thread* self, mirror::Object* ptr); void Walk(DlMallocSpace::WalkCallback, void* arg) OVERRIDE LOCKS_EXCLUDED(lock_); // TODO: disabling thread safety analysis as this may be called when we already hold lock_. @@ -153,7 +155,7 @@ class FreeListSpace FINAL : public LargeObjectSpace { size_t AllocationSize(mirror::Object* obj, size_t* usable_size) OVERRIDE EXCLUSIVE_LOCKS_REQUIRED(lock_); mirror::Object* Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) OVERRIDE; + size_t* usable_size, size_t* bytes_tl_bulk_allocated) OVERRIDE; size_t Free(Thread* self, mirror::Object* obj) OVERRIDE; void Walk(DlMallocSpace::WalkCallback callback, void* arg) OVERRIDE LOCKS_EXCLUDED(lock_); void Dump(std::ostream& os) const; diff --git a/runtime/gc/space/large_object_space_test.cc b/runtime/gc/space/large_object_space_test.cc index e17bad8a14..a261663ec7 100644 --- a/runtime/gc/space/large_object_space_test.cc +++ b/runtime/gc/space/large_object_space_test.cc @@ -49,11 +49,13 @@ void LargeObjectSpaceTest::LargeObjectTest() { while (requests.size() < num_allocations) { size_t request_size = test_rand(&rand_seed) % max_allocation_size; size_t allocation_size = 0; + size_t bytes_tl_bulk_allocated; mirror::Object* obj = los->Alloc(Thread::Current(), request_size, &allocation_size, - nullptr); + nullptr, &bytes_tl_bulk_allocated); ASSERT_TRUE(obj != nullptr); ASSERT_EQ(allocation_size, los->AllocationSize(obj, nullptr)); ASSERT_GE(allocation_size, request_size); + ASSERT_EQ(allocation_size, bytes_tl_bulk_allocated); // Fill in our magic value. uint8_t magic = (request_size & 0xFF) | 1; memset(obj, magic, request_size); @@ -83,9 +85,10 @@ void LargeObjectSpaceTest::LargeObjectTest() { // Test that dump doesn't crash. los->Dump(LOG(INFO)); - size_t bytes_allocated = 0; + size_t bytes_allocated = 0, bytes_tl_bulk_allocated; // Checks that the coalescing works. - mirror::Object* obj = los->Alloc(Thread::Current(), 100 * MB, &bytes_allocated, nullptr); + mirror::Object* obj = los->Alloc(Thread::Current(), 100 * MB, &bytes_allocated, nullptr, + &bytes_tl_bulk_allocated); EXPECT_TRUE(obj != nullptr); los->Free(Thread::Current(), obj); @@ -102,8 +105,9 @@ class AllocRaceTask : public Task { void Run(Thread* self) { for (size_t i = 0; i < iterations_ ; ++i) { - size_t alloc_size; - mirror::Object* ptr = los_->Alloc(self, size_, &alloc_size, nullptr); + size_t alloc_size, bytes_tl_bulk_allocated; + mirror::Object* ptr = los_->Alloc(self, size_, &alloc_size, nullptr, + &bytes_tl_bulk_allocated); NanoSleep((id_ + 3) * 1000); // (3+id) mu s diff --git a/runtime/gc/space/malloc_space.h b/runtime/gc/space/malloc_space.h index 06239e5e73..bbf1bbbdbd 100644 --- a/runtime/gc/space/malloc_space.h +++ b/runtime/gc/space/malloc_space.h @@ -55,10 +55,11 @@ class MallocSpace : public ContinuousMemMapAllocSpace { // Allocate num_bytes allowing the underlying space to grow. virtual mirror::Object* AllocWithGrowth(Thread* self, size_t num_bytes, - size_t* bytes_allocated, size_t* usable_size) = 0; + size_t* bytes_allocated, size_t* usable_size, + size_t* bytes_tl_bulk_allocated) = 0; // Allocate num_bytes without allowing the underlying space to grow. virtual mirror::Object* Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) = 0; + size_t* usable_size, size_t* bytes_tl_bulk_allocated) = 0; // Return the storage space required by obj. If usable_size isn't nullptr then it is set to the // amount of the storage space that may be used by obj. virtual size_t AllocationSize(mirror::Object* obj, size_t* usable_size) = 0; @@ -67,6 +68,11 @@ class MallocSpace : public ContinuousMemMapAllocSpace { virtual size_t FreeList(Thread* self, size_t num_ptrs, mirror::Object** ptrs) SHARED_LOCKS_REQUIRED(Locks::mutator_lock_) = 0; + // Returns the maximum bytes that could be allocated for the given + // size in bulk, that is the maximum value for the + // bytes_allocated_bulk out param returned by MallocSpace::Alloc(). + virtual size_t MaxBytesBulkAllocatedFor(size_t num_bytes) = 0; + #ifndef NDEBUG virtual void CheckMoreCoreForPrecondition() {} // to be overridden in the debug build. #else diff --git a/runtime/gc/space/region_space-inl.h b/runtime/gc/space/region_space-inl.h index a4ed7187c0..1cdf69dbe5 100644 --- a/runtime/gc/space/region_space-inl.h +++ b/runtime/gc/space/region_space-inl.h @@ -24,30 +24,36 @@ namespace gc { namespace space { inline mirror::Object* RegionSpace::Alloc(Thread*, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) { + size_t* usable_size, + size_t* bytes_tl_bulk_allocated) { num_bytes = RoundUp(num_bytes, kAlignment); - return AllocNonvirtual<false>(num_bytes, bytes_allocated, usable_size); + return AllocNonvirtual<false>(num_bytes, bytes_allocated, usable_size, + bytes_tl_bulk_allocated); } inline mirror::Object* RegionSpace::AllocThreadUnsafe(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) { + size_t* usable_size, + size_t* bytes_tl_bulk_allocated) { Locks::mutator_lock_->AssertExclusiveHeld(self); - return Alloc(self, num_bytes, bytes_allocated, usable_size); + return Alloc(self, num_bytes, bytes_allocated, usable_size, bytes_tl_bulk_allocated); } template<bool kForEvac> inline mirror::Object* RegionSpace::AllocNonvirtual(size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) { + size_t* usable_size, + size_t* bytes_tl_bulk_allocated) { DCHECK(IsAligned<kAlignment>(num_bytes)); mirror::Object* obj; if (LIKELY(num_bytes <= kRegionSize)) { // Non-large object. if (!kForEvac) { - obj = current_region_->Alloc(num_bytes, bytes_allocated, usable_size); + obj = current_region_->Alloc(num_bytes, bytes_allocated, usable_size, + bytes_tl_bulk_allocated); } else { DCHECK(evac_region_ != nullptr); - obj = evac_region_->Alloc(num_bytes, bytes_allocated, usable_size); + obj = evac_region_->Alloc(num_bytes, bytes_allocated, usable_size, + bytes_tl_bulk_allocated); } if (LIKELY(obj != nullptr)) { return obj; @@ -55,9 +61,11 @@ inline mirror::Object* RegionSpace::AllocNonvirtual(size_t num_bytes, size_t* by MutexLock mu(Thread::Current(), region_lock_); // Retry with current region since another thread may have updated it. if (!kForEvac) { - obj = current_region_->Alloc(num_bytes, bytes_allocated, usable_size); + obj = current_region_->Alloc(num_bytes, bytes_allocated, usable_size, + bytes_tl_bulk_allocated); } else { - obj = evac_region_->Alloc(num_bytes, bytes_allocated, usable_size); + obj = evac_region_->Alloc(num_bytes, bytes_allocated, usable_size, + bytes_tl_bulk_allocated); } if (LIKELY(obj != nullptr)) { return obj; @@ -73,7 +81,7 @@ inline mirror::Object* RegionSpace::AllocNonvirtual(size_t num_bytes, size_t* by r->Unfree(time_); r->SetNewlyAllocated(); ++num_non_free_regions_; - obj = r->Alloc(num_bytes, bytes_allocated, usable_size); + obj = r->Alloc(num_bytes, bytes_allocated, usable_size, bytes_tl_bulk_allocated); CHECK(obj != nullptr); current_region_ = r; return obj; @@ -85,7 +93,7 @@ inline mirror::Object* RegionSpace::AllocNonvirtual(size_t num_bytes, size_t* by if (r->IsFree()) { r->Unfree(time_); ++num_non_free_regions_; - obj = r->Alloc(num_bytes, bytes_allocated, usable_size); + obj = r->Alloc(num_bytes, bytes_allocated, usable_size, bytes_tl_bulk_allocated); CHECK(obj != nullptr); evac_region_ = r; return obj; @@ -94,7 +102,8 @@ inline mirror::Object* RegionSpace::AllocNonvirtual(size_t num_bytes, size_t* by } } else { // Large object. - obj = AllocLarge<kForEvac>(num_bytes, bytes_allocated, usable_size); + obj = AllocLarge<kForEvac>(num_bytes, bytes_allocated, usable_size, + bytes_tl_bulk_allocated); if (LIKELY(obj != nullptr)) { return obj; } @@ -103,7 +112,8 @@ inline mirror::Object* RegionSpace::AllocNonvirtual(size_t num_bytes, size_t* by } inline mirror::Object* RegionSpace::Region::Alloc(size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) { + size_t* usable_size, + size_t* bytes_tl_bulk_allocated) { DCHECK(IsAllocated() && IsInToSpace()); DCHECK(IsAligned<kAlignment>(num_bytes)); Atomic<uint8_t*>* atomic_top = reinterpret_cast<Atomic<uint8_t*>*>(&top_); @@ -124,6 +134,7 @@ inline mirror::Object* RegionSpace::Region::Alloc(size_t num_bytes, size_t* byte if (usable_size != nullptr) { *usable_size = num_bytes; } + *bytes_tl_bulk_allocated = num_bytes; return reinterpret_cast<mirror::Object*>(old_top); } @@ -253,7 +264,8 @@ inline mirror::Object* RegionSpace::GetNextObject(mirror::Object* obj) { template<bool kForEvac> mirror::Object* RegionSpace::AllocLarge(size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) { + size_t* usable_size, + size_t* bytes_tl_bulk_allocated) { DCHECK(IsAligned<kAlignment>(num_bytes)); DCHECK_GT(num_bytes, kRegionSize); size_t num_regs = RoundUp(num_bytes, kRegionSize) / kRegionSize; @@ -300,6 +312,7 @@ mirror::Object* RegionSpace::AllocLarge(size_t num_bytes, size_t* bytes_allocate if (usable_size != nullptr) { *usable_size = num_regs * kRegionSize; } + *bytes_tl_bulk_allocated = num_bytes; return reinterpret_cast<mirror::Object*>(first_reg->Begin()); } else { // right points to the non-free region. Start with the one after it. diff --git a/runtime/gc/space/region_space.cc b/runtime/gc/space/region_space.cc index 8bb73d614c..814ab6ce92 100644 --- a/runtime/gc/space/region_space.cc +++ b/runtime/gc/space/region_space.cc @@ -76,7 +76,7 @@ RegionSpace::RegionSpace(const std::string& name, MemMap* mem_map) current_region_ = &full_region_; evac_region_ = nullptr; size_t ignored; - DCHECK(full_region_.Alloc(kAlignment, &ignored, nullptr) == nullptr); + DCHECK(full_region_.Alloc(kAlignment, &ignored, nullptr, &ignored) == nullptr); } size_t RegionSpace::FromSpaceSize() { @@ -356,9 +356,10 @@ bool RegionSpace::AllocNewTlab(Thread* self) { return false; } -void RegionSpace::RevokeThreadLocalBuffers(Thread* thread) { +size_t RegionSpace::RevokeThreadLocalBuffers(Thread* thread) { MutexLock mu(Thread::Current(), region_lock_); RevokeThreadLocalBuffersLocked(thread); + return 0U; } void RegionSpace::RevokeThreadLocalBuffersLocked(Thread* thread) { @@ -377,7 +378,7 @@ void RegionSpace::RevokeThreadLocalBuffersLocked(Thread* thread) { thread->SetTlab(nullptr, nullptr); } -void RegionSpace::RevokeAllThreadLocalBuffers() { +size_t RegionSpace::RevokeAllThreadLocalBuffers() { Thread* self = Thread::Current(); MutexLock mu(self, *Locks::runtime_shutdown_lock_); MutexLock mu2(self, *Locks::thread_list_lock_); @@ -385,6 +386,7 @@ void RegionSpace::RevokeAllThreadLocalBuffers() { for (Thread* thread : thread_list) { RevokeThreadLocalBuffers(thread); } + return 0U; } void RegionSpace::AssertThreadLocalBuffersAreRevoked(Thread* thread) { diff --git a/runtime/gc/space/region_space.h b/runtime/gc/space/region_space.h index 416054716c..b88ce24114 100644 --- a/runtime/gc/space/region_space.h +++ b/runtime/gc/space/region_space.h @@ -42,18 +42,20 @@ class RegionSpace FINAL : public ContinuousMemMapAllocSpace { // Allocate num_bytes, returns nullptr if the space is full. mirror::Object* Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) OVERRIDE; + size_t* usable_size, size_t* bytes_tl_bulk_allocated) OVERRIDE; // Thread-unsafe allocation for when mutators are suspended, used by the semispace collector. mirror::Object* AllocThreadUnsafe(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) + size_t* usable_size, size_t* bytes_tl_bulk_allocated) OVERRIDE EXCLUSIVE_LOCKS_REQUIRED(Locks::mutator_lock_); // The main allocation routine. template<bool kForEvac> ALWAYS_INLINE mirror::Object* AllocNonvirtual(size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size); + size_t* usable_size, + size_t* bytes_tl_bulk_allocated); // Allocate/free large objects (objects that are larger than the region size.) template<bool kForEvac> - mirror::Object* AllocLarge(size_t num_bytes, size_t* bytes_allocated, size_t* usable_size); + mirror::Object* AllocLarge(size_t num_bytes, size_t* bytes_allocated, size_t* usable_size, + size_t* bytes_tl_bulk_allocated); void FreeLarge(mirror::Object* large_obj, size_t bytes_allocated); // Return the storage space required by obj. @@ -87,10 +89,10 @@ class RegionSpace FINAL : public ContinuousMemMapAllocSpace { void DumpRegions(std::ostream& os); void DumpNonFreeRegions(std::ostream& os); - void RevokeThreadLocalBuffers(Thread* thread) LOCKS_EXCLUDED(region_lock_); + size_t RevokeThreadLocalBuffers(Thread* thread) LOCKS_EXCLUDED(region_lock_); void RevokeThreadLocalBuffersLocked(Thread* thread) EXCLUSIVE_LOCKS_REQUIRED(region_lock_); - void RevokeAllThreadLocalBuffers() LOCKS_EXCLUDED(Locks::runtime_shutdown_lock_, - Locks::thread_list_lock_); + size_t RevokeAllThreadLocalBuffers() LOCKS_EXCLUDED(Locks::runtime_shutdown_lock_, + Locks::thread_list_lock_); void AssertThreadLocalBuffersAreRevoked(Thread* thread) LOCKS_EXCLUDED(region_lock_); void AssertAllThreadLocalBuffersAreRevoked() LOCKS_EXCLUDED(Locks::runtime_shutdown_lock_, Locks::thread_list_lock_); @@ -269,7 +271,8 @@ class RegionSpace FINAL : public ContinuousMemMapAllocSpace { } ALWAYS_INLINE mirror::Object* Alloc(size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size); + size_t* usable_size, + size_t* bytes_tl_bulk_allocated); bool IsFree() const { bool is_free = state_ == RegionState::kRegionStateFree; diff --git a/runtime/gc/space/rosalloc_space-inl.h b/runtime/gc/space/rosalloc_space-inl.h index 5d6642d349..9d582a3f86 100644 --- a/runtime/gc/space/rosalloc_space-inl.h +++ b/runtime/gc/space/rosalloc_space-inl.h @@ -26,13 +26,19 @@ namespace art { namespace gc { namespace space { +template<bool kMaybeRunningOnValgrind> inline size_t RosAllocSpace::AllocationSizeNonvirtual(mirror::Object* obj, size_t* usable_size) { // obj is a valid object. Use its class in the header to get the size. // Don't use verification since the object may be dead if we are sweeping. size_t size = obj->SizeOf<kVerifyNone>(); - bool running_on_valgrind = RUNNING_ON_VALGRIND != 0; - if (running_on_valgrind) { - size += 2 * kDefaultValgrindRedZoneBytes; + bool running_on_valgrind = false; + if (kMaybeRunningOnValgrind) { + running_on_valgrind = RUNNING_ON_VALGRIND != 0; + if (running_on_valgrind) { + size += 2 * kDefaultValgrindRedZoneBytes; + } + } else { + DCHECK_EQ(RUNNING_ON_VALGRIND, 0U); } size_t size_by_size = rosalloc_->UsableSize(size); if (kIsDebugBuild) { @@ -55,28 +61,50 @@ inline size_t RosAllocSpace::AllocationSizeNonvirtual(mirror::Object* obj, size_ template<bool kThreadSafe> inline mirror::Object* RosAllocSpace::AllocCommon(Thread* self, size_t num_bytes, - size_t* bytes_allocated, size_t* usable_size) { - size_t rosalloc_size = 0; + size_t* bytes_allocated, size_t* usable_size, + size_t* bytes_tl_bulk_allocated) { + size_t rosalloc_bytes_allocated = 0; + size_t rosalloc_usable_size = 0; + size_t rosalloc_bytes_tl_bulk_allocated = 0; if (!kThreadSafe) { Locks::mutator_lock_->AssertExclusiveHeld(self); } mirror::Object* result = reinterpret_cast<mirror::Object*>( - rosalloc_->Alloc<kThreadSafe>(self, num_bytes, &rosalloc_size)); + rosalloc_->Alloc<kThreadSafe>(self, num_bytes, &rosalloc_bytes_allocated, + &rosalloc_usable_size, + &rosalloc_bytes_tl_bulk_allocated)); if (LIKELY(result != NULL)) { if (kDebugSpaces) { CHECK(Contains(result)) << "Allocation (" << reinterpret_cast<void*>(result) << ") not in bounds of allocation space " << *this; } DCHECK(bytes_allocated != NULL); - *bytes_allocated = rosalloc_size; - DCHECK_EQ(rosalloc_size, rosalloc_->UsableSize(result)); + *bytes_allocated = rosalloc_bytes_allocated; + DCHECK_EQ(rosalloc_usable_size, rosalloc_->UsableSize(result)); if (usable_size != nullptr) { - *usable_size = rosalloc_size; + *usable_size = rosalloc_usable_size; } + DCHECK(bytes_tl_bulk_allocated != NULL); + *bytes_tl_bulk_allocated = rosalloc_bytes_tl_bulk_allocated; } return result; } +inline bool RosAllocSpace::CanAllocThreadLocal(Thread* self, size_t num_bytes) { + return rosalloc_->CanAllocFromThreadLocalRun(self, num_bytes); +} + +inline mirror::Object* RosAllocSpace::AllocThreadLocal(Thread* self, size_t num_bytes, + size_t* bytes_allocated) { + DCHECK(bytes_allocated != nullptr); + return reinterpret_cast<mirror::Object*>( + rosalloc_->AllocFromThreadLocalRun(self, num_bytes, bytes_allocated)); +} + +inline size_t RosAllocSpace::MaxBytesBulkAllocatedForNonvirtual(size_t num_bytes) { + return rosalloc_->MaxBytesBulkAllocatedFor(num_bytes); +} + } // namespace space } // namespace gc } // namespace art diff --git a/runtime/gc/space/rosalloc_space.cc b/runtime/gc/space/rosalloc_space.cc index ced25a40bb..f140021f76 100644 --- a/runtime/gc/space/rosalloc_space.cc +++ b/runtime/gc/space/rosalloc_space.cc @@ -154,7 +154,8 @@ allocator::RosAlloc* RosAllocSpace::CreateRosAlloc(void* begin, size_t morecore_ } mirror::Object* RosAllocSpace::AllocWithGrowth(Thread* self, size_t num_bytes, - size_t* bytes_allocated, size_t* usable_size) { + size_t* bytes_allocated, size_t* usable_size, + size_t* bytes_tl_bulk_allocated) { mirror::Object* result; { MutexLock mu(self, lock_); @@ -162,7 +163,8 @@ mirror::Object* RosAllocSpace::AllocWithGrowth(Thread* self, size_t num_bytes, size_t max_allowed = Capacity(); rosalloc_->SetFootprintLimit(max_allowed); // Try the allocation. - result = AllocCommon(self, num_bytes, bytes_allocated, usable_size); + result = AllocCommon(self, num_bytes, bytes_allocated, usable_size, + bytes_tl_bulk_allocated); // Shrink back down as small as possible. size_t footprint = rosalloc_->Footprint(); rosalloc_->SetFootprintLimit(footprint); @@ -209,7 +211,7 @@ size_t RosAllocSpace::FreeList(Thread* self, size_t num_ptrs, mirror::Object** p __builtin_prefetch(reinterpret_cast<char*>(ptrs[i + kPrefetchLookAhead])); } if (kVerifyFreedBytes) { - verify_bytes += AllocationSizeNonvirtual(ptrs[i], nullptr); + verify_bytes += AllocationSizeNonvirtual<true>(ptrs[i], nullptr); } } @@ -338,12 +340,12 @@ void RosAllocSpace::InspectAllRosAlloc(void (*callback)(void *start, void *end, } } -void RosAllocSpace::RevokeThreadLocalBuffers(Thread* thread) { - rosalloc_->RevokeThreadLocalRuns(thread); +size_t RosAllocSpace::RevokeThreadLocalBuffers(Thread* thread) { + return rosalloc_->RevokeThreadLocalRuns(thread); } -void RosAllocSpace::RevokeAllThreadLocalBuffers() { - rosalloc_->RevokeAllThreadLocalRuns(); +size_t RosAllocSpace::RevokeAllThreadLocalBuffers() { + return rosalloc_->RevokeAllThreadLocalRuns(); } void RosAllocSpace::AssertThreadLocalBuffersAreRevoked(Thread* thread) { diff --git a/runtime/gc/space/rosalloc_space.h b/runtime/gc/space/rosalloc_space.h index c856e9560a..36268f76f8 100644 --- a/runtime/gc/space/rosalloc_space.h +++ b/runtime/gc/space/rosalloc_space.h @@ -47,18 +47,21 @@ class RosAllocSpace : public MallocSpace { bool low_memory_mode, bool can_move_objects); mirror::Object* AllocWithGrowth(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) OVERRIDE LOCKS_EXCLUDED(lock_); + size_t* usable_size, size_t* bytes_tl_bulk_allocated) + OVERRIDE LOCKS_EXCLUDED(lock_); mirror::Object* Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) OVERRIDE { - return AllocNonvirtual(self, num_bytes, bytes_allocated, usable_size); + size_t* usable_size, size_t* bytes_tl_bulk_allocated) OVERRIDE { + return AllocNonvirtual(self, num_bytes, bytes_allocated, usable_size, + bytes_tl_bulk_allocated); } mirror::Object* AllocThreadUnsafe(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) + size_t* usable_size, size_t* bytes_tl_bulk_allocated) OVERRIDE EXCLUSIVE_LOCKS_REQUIRED(Locks::mutator_lock_) { - return AllocNonvirtualThreadUnsafe(self, num_bytes, bytes_allocated, usable_size); + return AllocNonvirtualThreadUnsafe(self, num_bytes, bytes_allocated, usable_size, + bytes_tl_bulk_allocated); } size_t AllocationSize(mirror::Object* obj, size_t* usable_size) OVERRIDE { - return AllocationSizeNonvirtual(obj, usable_size); + return AllocationSizeNonvirtual<true>(obj, usable_size); } size_t Free(Thread* self, mirror::Object* ptr) OVERRIDE SHARED_LOCKS_REQUIRED(Locks::mutator_lock_); @@ -66,17 +69,33 @@ class RosAllocSpace : public MallocSpace { SHARED_LOCKS_REQUIRED(Locks::mutator_lock_); mirror::Object* AllocNonvirtual(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) { + size_t* usable_size, size_t* bytes_tl_bulk_allocated) { // RosAlloc zeroes memory internally. - return AllocCommon(self, num_bytes, bytes_allocated, usable_size); + return AllocCommon(self, num_bytes, bytes_allocated, usable_size, + bytes_tl_bulk_allocated); } mirror::Object* AllocNonvirtualThreadUnsafe(Thread* self, size_t num_bytes, - size_t* bytes_allocated, size_t* usable_size) { + size_t* bytes_allocated, size_t* usable_size, + size_t* bytes_tl_bulk_allocated) { // RosAlloc zeroes memory internally. Pass in false for thread unsafe. - return AllocCommon<false>(self, num_bytes, bytes_allocated, usable_size); + return AllocCommon<false>(self, num_bytes, bytes_allocated, usable_size, + bytes_tl_bulk_allocated); } + // Returns true if the given allocation request can be allocated in + // an existing thread local run without allocating a new run. + ALWAYS_INLINE bool CanAllocThreadLocal(Thread* self, size_t num_bytes); + // Allocate the given allocation request in an existing thread local + // run without allocating a new run. + ALWAYS_INLINE mirror::Object* AllocThreadLocal(Thread* self, size_t num_bytes, + size_t* bytes_allocated); + size_t MaxBytesBulkAllocatedFor(size_t num_bytes) OVERRIDE { + return MaxBytesBulkAllocatedForNonvirtual(num_bytes); + } + ALWAYS_INLINE size_t MaxBytesBulkAllocatedForNonvirtual(size_t num_bytes); + // TODO: NO_THREAD_SAFETY_ANALYSIS because SizeOf() requires that mutator_lock is held. + template<bool kMaybeRunningOnValgrind> size_t AllocationSizeNonvirtual(mirror::Object* obj, size_t* usable_size) NO_THREAD_SAFETY_ANALYSIS; @@ -99,8 +118,8 @@ class RosAllocSpace : public MallocSpace { uint64_t GetBytesAllocated() OVERRIDE; uint64_t GetObjectsAllocated() OVERRIDE; - void RevokeThreadLocalBuffers(Thread* thread); - void RevokeAllThreadLocalBuffers(); + size_t RevokeThreadLocalBuffers(Thread* thread); + size_t RevokeAllThreadLocalBuffers(); void AssertThreadLocalBuffersAreRevoked(Thread* thread); void AssertAllThreadLocalBuffersAreRevoked(); @@ -134,7 +153,7 @@ class RosAllocSpace : public MallocSpace { private: template<bool kThreadSafe = true> mirror::Object* AllocCommon(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size); + size_t* usable_size, size_t* bytes_tl_bulk_allocated); void* CreateAllocator(void* base, size_t morecore_start, size_t initial_size, size_t maximum_size, bool low_memory_mode) OVERRIDE { diff --git a/runtime/gc/space/space.h b/runtime/gc/space/space.h index d24650b60d..f2378d9ff0 100644 --- a/runtime/gc/space/space.h +++ b/runtime/gc/space/space.h @@ -203,14 +203,24 @@ class AllocSpace { // succeeds, the output parameter bytes_allocated will be set to the // actually allocated bytes which is >= num_bytes. // Alloc can be called from multiple threads at the same time and must be thread-safe. + // + // bytes_tl_bulk_allocated - bytes allocated in bulk ahead of time for a thread local allocation, + // if applicable. It can be + // 1) equal to bytes_allocated if it's not a thread local allocation, + // 2) greater than bytes_allocated if it's a thread local + // allocation that required a new buffer, or + // 3) zero if it's a thread local allocation in an existing + // buffer. + // This is what is to be added to Heap::num_bytes_allocated_. virtual mirror::Object* Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) = 0; + size_t* usable_size, size_t* bytes_tl_bulk_allocated) = 0; // Thread-unsafe allocation for when mutators are suspended, used by the semispace collector. virtual mirror::Object* AllocThreadUnsafe(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) + size_t* usable_size, + size_t* bytes_tl_bulk_allocated) EXCLUSIVE_LOCKS_REQUIRED(Locks::mutator_lock_) { - return Alloc(self, num_bytes, bytes_allocated, usable_size); + return Alloc(self, num_bytes, bytes_allocated, usable_size, bytes_tl_bulk_allocated); } // Return the storage space required by obj. @@ -224,11 +234,15 @@ class AllocSpace { // Revoke any sort of thread-local buffers that are used to speed up allocations for the given // thread, if the alloc space implementation uses any. - virtual void RevokeThreadLocalBuffers(Thread* thread) = 0; + // Returns the total free bytes in the revoked thread local runs that's to be subtracted + // from Heap::num_bytes_allocated_ or zero if unnecessary. + virtual size_t RevokeThreadLocalBuffers(Thread* thread) = 0; // Revoke any sort of thread-local buffers that are used to speed up allocations for all the // threads, if the alloc space implementation uses any. - virtual void RevokeAllThreadLocalBuffers() = 0; + // Returns the total free bytes in the revoked thread local runs that's to be subtracted + // from Heap::num_bytes_allocated_ or zero if unnecessary. + virtual size_t RevokeAllThreadLocalBuffers() = 0; virtual void LogFragmentationAllocFailure(std::ostream& os, size_t failed_alloc_bytes) = 0; diff --git a/runtime/gc/space/space_test.h b/runtime/gc/space/space_test.h index 09d10dd94b..3e9e9f7a49 100644 --- a/runtime/gc/space/space_test.h +++ b/runtime/gc/space/space_test.h @@ -61,11 +61,13 @@ class SpaceTest : public CommonRuntimeTest { } mirror::Object* Alloc(space::MallocSpace* alloc_space, Thread* self, size_t bytes, - size_t* bytes_allocated, size_t* usable_size) + size_t* bytes_allocated, size_t* usable_size, + size_t* bytes_tl_bulk_allocated) SHARED_LOCKS_REQUIRED(Locks::mutator_lock_) { StackHandleScope<1> hs(self); Handle<mirror::Class> byte_array_class(hs.NewHandle(GetByteArrayClass(self))); - mirror::Object* obj = alloc_space->Alloc(self, bytes, bytes_allocated, usable_size); + mirror::Object* obj = alloc_space->Alloc(self, bytes, bytes_allocated, usable_size, + bytes_tl_bulk_allocated); if (obj != nullptr) { InstallClass(obj, byte_array_class.Get(), bytes); } @@ -73,11 +75,13 @@ class SpaceTest : public CommonRuntimeTest { } mirror::Object* AllocWithGrowth(space::MallocSpace* alloc_space, Thread* self, size_t bytes, - size_t* bytes_allocated, size_t* usable_size) + size_t* bytes_allocated, size_t* usable_size, + size_t* bytes_tl_bulk_allocated) SHARED_LOCKS_REQUIRED(Locks::mutator_lock_) { StackHandleScope<1> hs(self); Handle<mirror::Class> byte_array_class(hs.NewHandle(GetByteArrayClass(self))); - mirror::Object* obj = alloc_space->AllocWithGrowth(self, bytes, bytes_allocated, usable_size); + mirror::Object* obj = alloc_space->AllocWithGrowth(self, bytes, bytes_allocated, usable_size, + bytes_tl_bulk_allocated); if (obj != nullptr) { InstallClass(obj, byte_array_class.Get(), bytes); } @@ -182,34 +186,38 @@ void SpaceTest::ZygoteSpaceTestBody(CreateSpaceFn create_space) { ScopedObjectAccess soa(self); // Succeeds, fits without adjusting the footprint limit. - size_t ptr1_bytes_allocated, ptr1_usable_size; + size_t ptr1_bytes_allocated, ptr1_usable_size, ptr1_bytes_tl_bulk_allocated; StackHandleScope<3> hs(soa.Self()); MutableHandle<mirror::Object> ptr1( - hs.NewHandle(Alloc(space, self, 1 * MB, &ptr1_bytes_allocated, &ptr1_usable_size))); + hs.NewHandle(Alloc(space, self, 1 * MB, &ptr1_bytes_allocated, &ptr1_usable_size, + &ptr1_bytes_tl_bulk_allocated))); EXPECT_TRUE(ptr1.Get() != nullptr); EXPECT_LE(1U * MB, ptr1_bytes_allocated); EXPECT_LE(1U * MB, ptr1_usable_size); EXPECT_LE(ptr1_usable_size, ptr1_bytes_allocated); + EXPECT_EQ(ptr1_bytes_tl_bulk_allocated, ptr1_bytes_allocated); // Fails, requires a higher footprint limit. - mirror::Object* ptr2 = Alloc(space, self, 8 * MB, &dummy, nullptr); + mirror::Object* ptr2 = Alloc(space, self, 8 * MB, &dummy, nullptr, &dummy); EXPECT_TRUE(ptr2 == nullptr); // Succeeds, adjusts the footprint. - size_t ptr3_bytes_allocated, ptr3_usable_size; + size_t ptr3_bytes_allocated, ptr3_usable_size, ptr3_bytes_tl_bulk_allocated; MutableHandle<mirror::Object> ptr3( - hs.NewHandle(AllocWithGrowth(space, self, 8 * MB, &ptr3_bytes_allocated, &ptr3_usable_size))); + hs.NewHandle(AllocWithGrowth(space, self, 8 * MB, &ptr3_bytes_allocated, &ptr3_usable_size, + &ptr3_bytes_tl_bulk_allocated))); EXPECT_TRUE(ptr3.Get() != nullptr); EXPECT_LE(8U * MB, ptr3_bytes_allocated); EXPECT_LE(8U * MB, ptr3_usable_size); EXPECT_LE(ptr3_usable_size, ptr3_bytes_allocated); + EXPECT_EQ(ptr3_bytes_tl_bulk_allocated, ptr3_bytes_allocated); // Fails, requires a higher footprint limit. - mirror::Object* ptr4 = space->Alloc(self, 8 * MB, &dummy, nullptr); + mirror::Object* ptr4 = space->Alloc(self, 8 * MB, &dummy, nullptr, &dummy); EXPECT_TRUE(ptr4 == nullptr); // Also fails, requires a higher allowed footprint. - mirror::Object* ptr5 = space->AllocWithGrowth(self, 8 * MB, &dummy, nullptr); + mirror::Object* ptr5 = space->AllocWithGrowth(self, 8 * MB, &dummy, nullptr, &dummy); EXPECT_TRUE(ptr5 == nullptr); // Release some memory. @@ -219,13 +227,15 @@ void SpaceTest::ZygoteSpaceTestBody(CreateSpaceFn create_space) { EXPECT_LE(8U * MB, free3); // Succeeds, now that memory has been freed. - size_t ptr6_bytes_allocated, ptr6_usable_size; + size_t ptr6_bytes_allocated, ptr6_usable_size, ptr6_bytes_tl_bulk_allocated; Handle<mirror::Object> ptr6( - hs.NewHandle(AllocWithGrowth(space, self, 9 * MB, &ptr6_bytes_allocated, &ptr6_usable_size))); + hs.NewHandle(AllocWithGrowth(space, self, 9 * MB, &ptr6_bytes_allocated, &ptr6_usable_size, + &ptr6_bytes_tl_bulk_allocated))); EXPECT_TRUE(ptr6.Get() != nullptr); EXPECT_LE(9U * MB, ptr6_bytes_allocated); EXPECT_LE(9U * MB, ptr6_usable_size); EXPECT_LE(ptr6_usable_size, ptr6_bytes_allocated); + EXPECT_EQ(ptr6_bytes_tl_bulk_allocated, ptr6_bytes_allocated); // Final clean up. size_t free1 = space->AllocationSize(ptr1.Get(), nullptr); @@ -233,7 +243,7 @@ void SpaceTest::ZygoteSpaceTestBody(CreateSpaceFn create_space) { EXPECT_LE(1U * MB, free1); // Make sure that the zygote space isn't directly at the start of the space. - EXPECT_TRUE(space->Alloc(self, 1U * MB, &dummy, nullptr) != nullptr); + EXPECT_TRUE(space->Alloc(self, 1U * MB, &dummy, nullptr, &dummy) != nullptr); gc::Heap* heap = Runtime::Current()->GetHeap(); space::Space* old_space = space; @@ -250,22 +260,26 @@ void SpaceTest::ZygoteSpaceTestBody(CreateSpaceFn create_space) { AddSpace(space, false); // Succeeds, fits without adjusting the footprint limit. - ptr1.Assign(Alloc(space, self, 1 * MB, &ptr1_bytes_allocated, &ptr1_usable_size)); + ptr1.Assign(Alloc(space, self, 1 * MB, &ptr1_bytes_allocated, &ptr1_usable_size, + &ptr1_bytes_tl_bulk_allocated)); EXPECT_TRUE(ptr1.Get() != nullptr); EXPECT_LE(1U * MB, ptr1_bytes_allocated); EXPECT_LE(1U * MB, ptr1_usable_size); EXPECT_LE(ptr1_usable_size, ptr1_bytes_allocated); + EXPECT_EQ(ptr1_bytes_tl_bulk_allocated, ptr1_bytes_allocated); // Fails, requires a higher footprint limit. - ptr2 = Alloc(space, self, 8 * MB, &dummy, nullptr); + ptr2 = Alloc(space, self, 8 * MB, &dummy, nullptr, &dummy); EXPECT_TRUE(ptr2 == nullptr); // Succeeds, adjusts the footprint. - ptr3.Assign(AllocWithGrowth(space, self, 2 * MB, &ptr3_bytes_allocated, &ptr3_usable_size)); + ptr3.Assign(AllocWithGrowth(space, self, 2 * MB, &ptr3_bytes_allocated, &ptr3_usable_size, + &ptr3_bytes_tl_bulk_allocated)); EXPECT_TRUE(ptr3.Get() != nullptr); EXPECT_LE(2U * MB, ptr3_bytes_allocated); EXPECT_LE(2U * MB, ptr3_usable_size); EXPECT_LE(ptr3_usable_size, ptr3_bytes_allocated); + EXPECT_EQ(ptr3_bytes_tl_bulk_allocated, ptr3_bytes_allocated); space->Free(self, ptr3.Assign(nullptr)); // Final clean up. @@ -285,34 +299,38 @@ void SpaceTest::AllocAndFreeTestBody(CreateSpaceFn create_space) { AddSpace(space); // Succeeds, fits without adjusting the footprint limit. - size_t ptr1_bytes_allocated, ptr1_usable_size; + size_t ptr1_bytes_allocated, ptr1_usable_size, ptr1_bytes_tl_bulk_allocated; StackHandleScope<3> hs(soa.Self()); MutableHandle<mirror::Object> ptr1( - hs.NewHandle(Alloc(space, self, 1 * MB, &ptr1_bytes_allocated, &ptr1_usable_size))); + hs.NewHandle(Alloc(space, self, 1 * MB, &ptr1_bytes_allocated, &ptr1_usable_size, + &ptr1_bytes_tl_bulk_allocated))); EXPECT_TRUE(ptr1.Get() != nullptr); EXPECT_LE(1U * MB, ptr1_bytes_allocated); EXPECT_LE(1U * MB, ptr1_usable_size); EXPECT_LE(ptr1_usable_size, ptr1_bytes_allocated); + EXPECT_EQ(ptr1_bytes_tl_bulk_allocated, ptr1_bytes_allocated); // Fails, requires a higher footprint limit. - mirror::Object* ptr2 = Alloc(space, self, 8 * MB, &dummy, nullptr); + mirror::Object* ptr2 = Alloc(space, self, 8 * MB, &dummy, nullptr, &dummy); EXPECT_TRUE(ptr2 == nullptr); // Succeeds, adjusts the footprint. - size_t ptr3_bytes_allocated, ptr3_usable_size; + size_t ptr3_bytes_allocated, ptr3_usable_size, ptr3_bytes_tl_bulk_allocated; MutableHandle<mirror::Object> ptr3( - hs.NewHandle(AllocWithGrowth(space, self, 8 * MB, &ptr3_bytes_allocated, &ptr3_usable_size))); + hs.NewHandle(AllocWithGrowth(space, self, 8 * MB, &ptr3_bytes_allocated, &ptr3_usable_size, + &ptr3_bytes_tl_bulk_allocated))); EXPECT_TRUE(ptr3.Get() != nullptr); EXPECT_LE(8U * MB, ptr3_bytes_allocated); EXPECT_LE(8U * MB, ptr3_usable_size); EXPECT_LE(ptr3_usable_size, ptr3_bytes_allocated); + EXPECT_EQ(ptr3_bytes_tl_bulk_allocated, ptr3_bytes_allocated); // Fails, requires a higher footprint limit. - mirror::Object* ptr4 = Alloc(space, self, 8 * MB, &dummy, nullptr); + mirror::Object* ptr4 = Alloc(space, self, 8 * MB, &dummy, nullptr, &dummy); EXPECT_TRUE(ptr4 == nullptr); // Also fails, requires a higher allowed footprint. - mirror::Object* ptr5 = AllocWithGrowth(space, self, 8 * MB, &dummy, nullptr); + mirror::Object* ptr5 = AllocWithGrowth(space, self, 8 * MB, &dummy, nullptr, &dummy); EXPECT_TRUE(ptr5 == nullptr); // Release some memory. @@ -322,13 +340,15 @@ void SpaceTest::AllocAndFreeTestBody(CreateSpaceFn create_space) { EXPECT_LE(8U * MB, free3); // Succeeds, now that memory has been freed. - size_t ptr6_bytes_allocated, ptr6_usable_size; + size_t ptr6_bytes_allocated, ptr6_usable_size, ptr6_bytes_tl_bulk_allocated; Handle<mirror::Object> ptr6( - hs.NewHandle(AllocWithGrowth(space, self, 9 * MB, &ptr6_bytes_allocated, &ptr6_usable_size))); + hs.NewHandle(AllocWithGrowth(space, self, 9 * MB, &ptr6_bytes_allocated, &ptr6_usable_size, + &ptr6_bytes_tl_bulk_allocated))); EXPECT_TRUE(ptr6.Get() != nullptr); EXPECT_LE(9U * MB, ptr6_bytes_allocated); EXPECT_LE(9U * MB, ptr6_usable_size); EXPECT_LE(ptr6_usable_size, ptr6_bytes_allocated); + EXPECT_EQ(ptr6_bytes_tl_bulk_allocated, ptr6_bytes_allocated); // Final clean up. size_t free1 = space->AllocationSize(ptr1.Get(), nullptr); @@ -348,14 +368,16 @@ void SpaceTest::AllocAndFreeListTestBody(CreateSpaceFn create_space) { // Succeeds, fits without adjusting the max allowed footprint. mirror::Object* lots_of_objects[1024]; for (size_t i = 0; i < arraysize(lots_of_objects); i++) { - size_t allocation_size, usable_size; + size_t allocation_size, usable_size, bytes_tl_bulk_allocated; size_t size_of_zero_length_byte_array = SizeOfZeroLengthByteArray(); lots_of_objects[i] = Alloc(space, self, size_of_zero_length_byte_array, &allocation_size, - &usable_size); + &usable_size, &bytes_tl_bulk_allocated); EXPECT_TRUE(lots_of_objects[i] != nullptr); size_t computed_usable_size; EXPECT_EQ(allocation_size, space->AllocationSize(lots_of_objects[i], &computed_usable_size)); EXPECT_EQ(usable_size, computed_usable_size); + EXPECT_TRUE(bytes_tl_bulk_allocated == 0 || + bytes_tl_bulk_allocated >= allocation_size); } // Release memory. @@ -363,12 +385,15 @@ void SpaceTest::AllocAndFreeListTestBody(CreateSpaceFn create_space) { // Succeeds, fits by adjusting the max allowed footprint. for (size_t i = 0; i < arraysize(lots_of_objects); i++) { - size_t allocation_size, usable_size; - lots_of_objects[i] = AllocWithGrowth(space, self, 1024, &allocation_size, &usable_size); + size_t allocation_size, usable_size, bytes_tl_bulk_allocated; + lots_of_objects[i] = AllocWithGrowth(space, self, 1024, &allocation_size, &usable_size, + &bytes_tl_bulk_allocated); EXPECT_TRUE(lots_of_objects[i] != nullptr); size_t computed_usable_size; EXPECT_EQ(allocation_size, space->AllocationSize(lots_of_objects[i], &computed_usable_size)); EXPECT_EQ(usable_size, computed_usable_size); + EXPECT_TRUE(bytes_tl_bulk_allocated == 0 || + bytes_tl_bulk_allocated >= allocation_size); } // Release memory. @@ -425,10 +450,13 @@ void SpaceTest::SizeFootPrintGrowthLimitAndTrimBody(MallocSpace* space, intptr_t StackHandleScope<1> hs(soa.Self()); auto object(hs.NewHandle<mirror::Object>(nullptr)); size_t bytes_allocated = 0; + size_t bytes_tl_bulk_allocated; if (round <= 1) { - object.Assign(Alloc(space, self, alloc_size, &bytes_allocated, nullptr)); + object.Assign(Alloc(space, self, alloc_size, &bytes_allocated, nullptr, + &bytes_tl_bulk_allocated)); } else { - object.Assign(AllocWithGrowth(space, self, alloc_size, &bytes_allocated, nullptr)); + object.Assign(AllocWithGrowth(space, self, alloc_size, &bytes_allocated, nullptr, + &bytes_tl_bulk_allocated)); } footprint = space->GetFootprint(); EXPECT_GE(space->Size(), footprint); // invariant @@ -441,6 +469,8 @@ void SpaceTest::SizeFootPrintGrowthLimitAndTrimBody(MallocSpace* space, intptr_t } else { EXPECT_GE(allocation_size, 8u); } + EXPECT_TRUE(bytes_tl_bulk_allocated == 0 || + bytes_tl_bulk_allocated >= allocation_size); amount_allocated += allocation_size; break; } @@ -518,11 +548,13 @@ void SpaceTest::SizeFootPrintGrowthLimitAndTrimBody(MallocSpace* space, intptr_t auto large_object(hs.NewHandle<mirror::Object>(nullptr)); size_t three_quarters_space = (growth_limit / 2) + (growth_limit / 4); size_t bytes_allocated = 0; + size_t bytes_tl_bulk_allocated; if (round <= 1) { - large_object.Assign(Alloc(space, self, three_quarters_space, &bytes_allocated, nullptr)); + large_object.Assign(Alloc(space, self, three_quarters_space, &bytes_allocated, nullptr, + &bytes_tl_bulk_allocated)); } else { large_object.Assign(AllocWithGrowth(space, self, three_quarters_space, &bytes_allocated, - nullptr)); + nullptr, &bytes_tl_bulk_allocated)); } EXPECT_TRUE(large_object.Get() != nullptr); diff --git a/runtime/gc/space/valgrind_malloc_space-inl.h b/runtime/gc/space/valgrind_malloc_space-inl.h index ae8e892e29..bc329e129c 100644 --- a/runtime/gc/space/valgrind_malloc_space-inl.h +++ b/runtime/gc/space/valgrind_malloc_space-inl.h @@ -32,10 +32,15 @@ namespace valgrind_details { template <size_t kValgrindRedZoneBytes, bool kUseObjSizeForUsable> inline mirror::Object* AdjustForValgrind(void* obj_with_rdz, size_t num_bytes, size_t bytes_allocated, size_t usable_size, - size_t* bytes_allocated_out, size_t* usable_size_out) { + size_t bytes_tl_bulk_allocated, + size_t* bytes_allocated_out, size_t* usable_size_out, + size_t* bytes_tl_bulk_allocated_out) { if (bytes_allocated_out != nullptr) { *bytes_allocated_out = bytes_allocated; } + if (bytes_tl_bulk_allocated_out != nullptr) { + *bytes_tl_bulk_allocated_out = bytes_tl_bulk_allocated; + } // This cuts over-provision and is a trade-off between testing the over-provisioning code paths // vs checking overflows in the regular paths. @@ -82,20 +87,25 @@ ValgrindMallocSpace<S, kValgrindRedZoneBytes, kAdjustForRedzoneInAllocSize, kUseObjSizeForUsable>::AllocWithGrowth( - Thread* self, size_t num_bytes, size_t* bytes_allocated_out, size_t* usable_size_out) { + Thread* self, size_t num_bytes, size_t* bytes_allocated_out, size_t* usable_size_out, + size_t* bytes_tl_bulk_allocated_out) { size_t bytes_allocated; size_t usable_size; + size_t bytes_tl_bulk_allocated; void* obj_with_rdz = S::AllocWithGrowth(self, num_bytes + 2 * kValgrindRedZoneBytes, - &bytes_allocated, &usable_size); + &bytes_allocated, &usable_size, + &bytes_tl_bulk_allocated); if (obj_with_rdz == nullptr) { return nullptr; } - return valgrind_details::AdjustForValgrind<kValgrindRedZoneBytes, - kUseObjSizeForUsable>(obj_with_rdz, num_bytes, - bytes_allocated, usable_size, - bytes_allocated_out, - usable_size_out); + return valgrind_details::AdjustForValgrind<kValgrindRedZoneBytes, kUseObjSizeForUsable>( + obj_with_rdz, num_bytes, + bytes_allocated, usable_size, + bytes_tl_bulk_allocated, + bytes_allocated_out, + usable_size_out, + bytes_tl_bulk_allocated_out); } template <typename S, @@ -106,11 +116,13 @@ mirror::Object* ValgrindMallocSpace<S, kValgrindRedZoneBytes, kAdjustForRedzoneInAllocSize, kUseObjSizeForUsable>::Alloc( - Thread* self, size_t num_bytes, size_t* bytes_allocated_out, size_t* usable_size_out) { + Thread* self, size_t num_bytes, size_t* bytes_allocated_out, size_t* usable_size_out, + size_t* bytes_tl_bulk_allocated_out) { size_t bytes_allocated; size_t usable_size; + size_t bytes_tl_bulk_allocated; void* obj_with_rdz = S::Alloc(self, num_bytes + 2 * kValgrindRedZoneBytes, - &bytes_allocated, &usable_size); + &bytes_allocated, &usable_size, &bytes_tl_bulk_allocated); if (obj_with_rdz == nullptr) { return nullptr; } @@ -118,8 +130,10 @@ mirror::Object* ValgrindMallocSpace<S, return valgrind_details::AdjustForValgrind<kValgrindRedZoneBytes, kUseObjSizeForUsable>(obj_with_rdz, num_bytes, bytes_allocated, usable_size, + bytes_tl_bulk_allocated, bytes_allocated_out, - usable_size_out); + usable_size_out, + bytes_tl_bulk_allocated_out); } template <typename S, @@ -130,20 +144,25 @@ mirror::Object* ValgrindMallocSpace<S, kValgrindRedZoneBytes, kAdjustForRedzoneInAllocSize, kUseObjSizeForUsable>::AllocThreadUnsafe( - Thread* self, size_t num_bytes, size_t* bytes_allocated_out, size_t* usable_size_out) { + Thread* self, size_t num_bytes, size_t* bytes_allocated_out, size_t* usable_size_out, + size_t* bytes_tl_bulk_allocated_out) { size_t bytes_allocated; size_t usable_size; + size_t bytes_tl_bulk_allocated; void* obj_with_rdz = S::AllocThreadUnsafe(self, num_bytes + 2 * kValgrindRedZoneBytes, - &bytes_allocated, &usable_size); + &bytes_allocated, &usable_size, + &bytes_tl_bulk_allocated); if (obj_with_rdz == nullptr) { return nullptr; } - return valgrind_details::AdjustForValgrind<kValgrindRedZoneBytes, - kUseObjSizeForUsable>(obj_with_rdz, num_bytes, - bytes_allocated, usable_size, - bytes_allocated_out, - usable_size_out); + return valgrind_details::AdjustForValgrind<kValgrindRedZoneBytes, kUseObjSizeForUsable>( + obj_with_rdz, num_bytes, + bytes_allocated, usable_size, + bytes_tl_bulk_allocated, + bytes_allocated_out, + usable_size_out, + bytes_tl_bulk_allocated_out); } template <typename S, @@ -226,6 +245,17 @@ ValgrindMallocSpace<S, mem_map->Size() - initial_size); } +template <typename S, + size_t kValgrindRedZoneBytes, + bool kAdjustForRedzoneInAllocSize, + bool kUseObjSizeForUsable> +size_t ValgrindMallocSpace<S, + kValgrindRedZoneBytes, + kAdjustForRedzoneInAllocSize, + kUseObjSizeForUsable>::MaxBytesBulkAllocatedFor(size_t num_bytes) { + return S::MaxBytesBulkAllocatedFor(num_bytes + 2 * kValgrindRedZoneBytes); +} + } // namespace space } // namespace gc } // namespace art diff --git a/runtime/gc/space/valgrind_malloc_space.h b/runtime/gc/space/valgrind_malloc_space.h index 707ea69a20..a6b010a2a1 100644 --- a/runtime/gc/space/valgrind_malloc_space.h +++ b/runtime/gc/space/valgrind_malloc_space.h @@ -34,12 +34,13 @@ template <typename BaseMallocSpaceType, class ValgrindMallocSpace FINAL : public BaseMallocSpaceType { public: mirror::Object* AllocWithGrowth(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) OVERRIDE; + size_t* usable_size, size_t* bytes_tl_bulk_allocated) + OVERRIDE; mirror::Object* Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) OVERRIDE; + size_t* usable_size, size_t* bytes_tl_bulk_allocated) OVERRIDE; mirror::Object* AllocThreadUnsafe(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) OVERRIDE - EXCLUSIVE_LOCKS_REQUIRED(Locks::mutator_lock_); + size_t* usable_size, size_t* bytes_tl_bulk_allocated) + OVERRIDE EXCLUSIVE_LOCKS_REQUIRED(Locks::mutator_lock_); size_t AllocationSize(mirror::Object* obj, size_t* usable_size) OVERRIDE; @@ -53,6 +54,8 @@ class ValgrindMallocSpace FINAL : public BaseMallocSpaceType { UNUSED(ptr); } + size_t MaxBytesBulkAllocatedFor(size_t num_bytes) OVERRIDE; + template <typename... Params> explicit ValgrindMallocSpace(MemMap* mem_map, size_t initial_size, Params... params); virtual ~ValgrindMallocSpace() {} diff --git a/runtime/gc/space/zygote_space.cc b/runtime/gc/space/zygote_space.cc index a868e6831d..9e882a898e 100644 --- a/runtime/gc/space/zygote_space.cc +++ b/runtime/gc/space/zygote_space.cc @@ -77,7 +77,7 @@ void ZygoteSpace::Dump(std::ostream& os) const { << ",name=\"" << GetName() << "\"]"; } -mirror::Object* ZygoteSpace::Alloc(Thread*, size_t, size_t*, size_t*) { +mirror::Object* ZygoteSpace::Alloc(Thread*, size_t, size_t*, size_t*, size_t*) { UNIMPLEMENTED(FATAL); UNREACHABLE(); } diff --git a/runtime/gc/space/zygote_space.h b/runtime/gc/space/zygote_space.h index 0cf4bb139c..934a234345 100644 --- a/runtime/gc/space/zygote_space.h +++ b/runtime/gc/space/zygote_space.h @@ -46,7 +46,7 @@ class ZygoteSpace FINAL : public ContinuousMemMapAllocSpace { } mirror::Object* Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) OVERRIDE; + size_t* usable_size, size_t* bytes_tl_bulk_allocated) OVERRIDE; size_t AllocationSize(mirror::Object* obj, size_t* usable_size) OVERRIDE; @@ -55,9 +55,11 @@ class ZygoteSpace FINAL : public ContinuousMemMapAllocSpace { size_t FreeList(Thread* self, size_t num_ptrs, mirror::Object** ptrs) OVERRIDE; // ZygoteSpaces don't have thread local state. - void RevokeThreadLocalBuffers(art::Thread*) OVERRIDE { + size_t RevokeThreadLocalBuffers(art::Thread*) OVERRIDE { + return 0U; } - void RevokeAllThreadLocalBuffers() OVERRIDE { + size_t RevokeAllThreadLocalBuffers() OVERRIDE { + return 0U; } uint64_t GetBytesAllocated() { diff --git a/runtime/interpreter/unstarted_runtime.cc b/runtime/interpreter/unstarted_runtime.cc index 99ce1e2f4b..95f3357f7c 100644 --- a/runtime/interpreter/unstarted_runtime.cc +++ b/runtime/interpreter/unstarted_runtime.cc @@ -39,6 +39,19 @@ namespace art { namespace interpreter { +static void AbortTransactionOrFail(Thread* self, const char* fmt, ...) + SHARED_LOCKS_REQUIRED(Locks::mutator_lock_) { + va_list args; + va_start(args, fmt); + if (Runtime::Current()->IsActiveTransaction()) { + AbortTransaction(self, fmt, args); + va_end(args); + } else { + LOG(FATAL) << "Trying to abort, but not in transaction mode: " << StringPrintf(fmt, args); + UNREACHABLE(); + } +} + // Helper function to deal with class loading in an unstarted runtime. static void UnstartedRuntimeFindClass(Thread* self, Handle<mirror::String> className, Handle<mirror::ClassLoader> class_loader, JValue* result, @@ -52,8 +65,8 @@ static void UnstartedRuntimeFindClass(Thread* self, Handle<mirror::String> class mirror::Class* found = class_linker->FindClass(self, descriptor.c_str(), class_loader); if (found == nullptr && abort_if_not_found) { if (!self->IsExceptionPending()) { - AbortTransaction(self, "%s failed in un-started runtime for class: %s", - method_name.c_str(), PrettyDescriptor(descriptor.c_str()).c_str()); + AbortTransactionOrFail(self, "%s failed in un-started runtime for class: %s", + method_name.c_str(), PrettyDescriptor(descriptor.c_str()).c_str()); } return; } @@ -131,6 +144,22 @@ static void UnstartedClassNewInstance(Thread* self, ShadowFrame* shadow_frame, J StackHandleScope<3> hs(self); // Class, constructor, object. mirror::Class* klass = shadow_frame->GetVRegReference(arg_offset)->AsClass(); Handle<mirror::Class> h_klass(hs.NewHandle(klass)); + + // Check that it's not null. + if (h_klass.Get() == nullptr) { + AbortTransactionOrFail(self, "Class reference is null for newInstance"); + return; + } + + // If we're in a transaction, class must not be finalizable (it or a superclass has a finalizer). + if (Runtime::Current()->IsActiveTransaction()) { + if (h_klass.Get()->IsFinalizable()) { + AbortTransaction(self, "Class for newInstance is finalizable: '%s'", + PrettyClass(h_klass.Get()).c_str()); + return; + } + } + // There are two situations in which we'll abort this run. // 1) If the class isn't yet initialized and initialization fails. // 2) If we can't find the default constructor. We'll postpone the exception to runtime. @@ -154,10 +183,9 @@ static void UnstartedClassNewInstance(Thread* self, ShadowFrame* shadow_frame, J } } if (!ok) { - std::string error_msg = StringPrintf("Failed in Class.newInstance for '%s' with %s", - PrettyClass(h_klass.Get()).c_str(), - PrettyTypeOf(self->GetException()).c_str()); - Runtime::Current()->AbortTransactionAndThrowInternalError(self, error_msg); + AbortTransactionOrFail(self, "Failed in Class.newInstance for '%s' with %s", + PrettyClass(h_klass.Get()).c_str(), + PrettyTypeOf(self->GetException()).c_str()); } } @@ -185,9 +213,12 @@ static void UnstartedClassGetDeclaredField(Thread* self, ShadowFrame* shadow_fra } } } - CHECK(found != nullptr) - << "Failed to find field in Class.getDeclaredField in un-started runtime. name=" - << name2->ToModifiedUtf8() << " class=" << PrettyDescriptor(klass); + if (found == nullptr) { + AbortTransactionOrFail(self, "Failed to find field in Class.getDeclaredField in un-started " + " runtime. name=%s class=%s", name2->ToModifiedUtf8().c_str(), + PrettyDescriptor(klass).c_str()); + return; + } // TODO: getDeclaredField calls GetType once the field is found to ensure a // NoClassDefFoundError is thrown if the field's type cannot be resolved. mirror::Class* jlr_Field = self->DecodeJObject( @@ -261,9 +292,8 @@ static void UnstartedSystemArraycopy(Thread* self, ShadowFrame* shadow_frame, dst->Set(dstPos + i, src->Get(srcPos + i)); } } else { - std::string tmp = StringPrintf("Unimplemented System.arraycopy for type '%s'", - PrettyDescriptor(ctype).c_str()); - Runtime::Current()->AbortTransactionAndThrowInternalError(self, tmp); + AbortTransactionOrFail(self, "Unimplemented System.arraycopy for type '%s'", + PrettyDescriptor(ctype).c_str()); } } @@ -308,18 +338,12 @@ static void UnstartedThreadLocalGet(Thread* self, ShadowFrame* shadow_frame, JVa } } } - - if (!ok) { - // We'll abort, so clear exception. - self->ClearException(); - } } } } if (!ok) { - Runtime::Current()->AbortTransactionAndThrowInternalError(self, - "Could not create RealToString object"); + AbortTransactionOrFail(self, "Could not create RealToString object"); } } @@ -469,7 +493,7 @@ static void UnstartedJNIStringCompareTo(Thread* self, SHARED_LOCKS_REQUIRED(Locks::mutator_lock_) { mirror::String* rhs = reinterpret_cast<mirror::Object*>(args[0])->AsString(); if (rhs == nullptr) { - AbortTransaction(self, "String.compareTo with null object"); + AbortTransactionOrFail(self, "String.compareTo with null object"); } result->SetI(receiver->AsString()->CompareTo(rhs)); } diff --git a/runtime/verifier/method_verifier.cc b/runtime/verifier/method_verifier.cc index 87a29ed2f2..a48735b081 100644 --- a/runtime/verifier/method_verifier.cc +++ b/runtime/verifier/method_verifier.cc @@ -513,7 +513,7 @@ mirror::ArtMethod* MethodVerifier::FindInvokedMethodAtDexPc(uint32_t dex_pc) { } const Instruction* inst = Instruction::At(code_item_->insns_ + dex_pc); const bool is_range = (inst->Opcode() == Instruction::INVOKE_VIRTUAL_RANGE_QUICK); - return GetQuickInvokedMethod(inst, register_line, is_range); + return GetQuickInvokedMethod(inst, register_line, is_range, false); } bool MethodVerifier::Verify() { @@ -3431,10 +3431,14 @@ mirror::ArtMethod* MethodVerifier::VerifyInvocationArgs(const Instruction* inst, } mirror::ArtMethod* MethodVerifier::GetQuickInvokedMethod(const Instruction* inst, - RegisterLine* reg_line, bool is_range) { - DCHECK(inst->Opcode() == Instruction::INVOKE_VIRTUAL_QUICK || - inst->Opcode() == Instruction::INVOKE_VIRTUAL_RANGE_QUICK); - const RegType& actual_arg_type = reg_line->GetInvocationThis(this, inst, is_range); + RegisterLine* reg_line, bool is_range, + bool allow_failure) { + if (is_range) { + DCHECK_EQ(inst->Opcode(), Instruction::INVOKE_VIRTUAL_RANGE_QUICK); + } else { + DCHECK_EQ(inst->Opcode(), Instruction::INVOKE_VIRTUAL_QUICK); + } + const RegType& actual_arg_type = reg_line->GetInvocationThis(this, inst, is_range, allow_failure); if (!actual_arg_type.HasClass()) { VLOG(verifier) << "Failed to get mirror::Class* from '" << actual_arg_type << "'"; return nullptr; @@ -3445,29 +3449,29 @@ mirror::ArtMethod* MethodVerifier::GetQuickInvokedMethod(const Instruction* inst // Derive Object.class from Class.class.getSuperclass(). mirror::Class* object_klass = klass->GetClass()->GetSuperClass(); if (FailOrAbort(this, object_klass->IsObjectClass(), - "Failed to find Object class in quickened invoke receiver", - work_insn_idx_)) { + "Failed to find Object class in quickened invoke receiver", work_insn_idx_)) { return nullptr; } dispatch_class = object_klass; } else { dispatch_class = klass; } - if (FailOrAbort(this, dispatch_class->HasVTable(), - "Receiver class has no vtable for quickened invoke at ", - work_insn_idx_)) { + if (!dispatch_class->HasVTable()) { + FailOrAbort(this, allow_failure, "Receiver class has no vtable for quickened invoke at ", + work_insn_idx_); return nullptr; } uint16_t vtable_index = is_range ? inst->VRegB_3rc() : inst->VRegB_35c(); - if (FailOrAbort(this, static_cast<int32_t>(vtable_index) < dispatch_class->GetVTableLength(), - "Receiver class has not enough vtable slots for quickened invoke at ", - work_insn_idx_)) { + if (static_cast<int32_t>(vtable_index) >= dispatch_class->GetVTableLength()) { + FailOrAbort(this, allow_failure, + "Receiver class has not enough vtable slots for quickened invoke at ", + work_insn_idx_); return nullptr; } mirror::ArtMethod* res_method = dispatch_class->GetVTableEntry(vtable_index); - if (FailOrAbort(this, !self_->IsExceptionPending(), - "Unexpected exception pending for quickened invoke at ", - work_insn_idx_)) { + if (self_->IsExceptionPending()) { + FailOrAbort(this, allow_failure, "Unexpected exception pending for quickened invoke at ", + work_insn_idx_); return nullptr; } return res_method; @@ -3478,8 +3482,7 @@ mirror::ArtMethod* MethodVerifier::VerifyInvokeVirtualQuickArgs(const Instructio DCHECK(Runtime::Current()->IsStarted() || verify_to_dump_) << PrettyMethod(dex_method_idx_, *dex_file_, true) << "@" << work_insn_idx_; - mirror::ArtMethod* res_method = GetQuickInvokedMethod(inst, work_line_.get(), - is_range); + mirror::ArtMethod* res_method = GetQuickInvokedMethod(inst, work_line_.get(), is_range, false); if (res_method == nullptr) { Fail(VERIFY_ERROR_BAD_CLASS_HARD) << "Cannot infer method from " << inst->Name(); return nullptr; diff --git a/runtime/verifier/method_verifier.h b/runtime/verifier/method_verifier.h index bdd62596a6..d7c2071cbc 100644 --- a/runtime/verifier/method_verifier.h +++ b/runtime/verifier/method_verifier.h @@ -244,7 +244,7 @@ class MethodVerifier { SHARED_LOCKS_REQUIRED(Locks::mutator_lock_); // Returns the method of a quick invoke or nullptr if it cannot be found. mirror::ArtMethod* GetQuickInvokedMethod(const Instruction* inst, RegisterLine* reg_line, - bool is_range) + bool is_range, bool allow_failure) SHARED_LOCKS_REQUIRED(Locks::mutator_lock_); // Returns the access field of a quick field access (iget/iput-quick) or nullptr // if it cannot be found. diff --git a/runtime/verifier/register_line.cc b/runtime/verifier/register_line.cc index 3b098718db..ed588fcdac 100644 --- a/runtime/verifier/register_line.cc +++ b/runtime/verifier/register_line.cc @@ -81,18 +81,23 @@ bool RegisterLine::CheckConstructorReturn(MethodVerifier* verifier) const { } const RegType& RegisterLine::GetInvocationThis(MethodVerifier* verifier, const Instruction* inst, - bool is_range) { + bool is_range, bool allow_failure) { const size_t args_count = is_range ? inst->VRegA_3rc() : inst->VRegA_35c(); if (args_count < 1) { - verifier->Fail(VERIFY_ERROR_BAD_CLASS_HARD) << "invoke lacks 'this'"; + if (!allow_failure) { + verifier->Fail(VERIFY_ERROR_BAD_CLASS_HARD) << "invoke lacks 'this'"; + } return verifier->GetRegTypeCache()->Conflict(); } /* Get the element type of the array held in vsrc */ const uint32_t this_reg = (is_range) ? inst->VRegC_3rc() : inst->VRegC_35c(); const RegType& this_type = GetRegisterType(verifier, this_reg); if (!this_type.IsReferenceTypes()) { - verifier->Fail(VERIFY_ERROR_BAD_CLASS_HARD) << "tried to get class from non-reference register v" - << this_reg << " (type=" << this_type << ")"; + if (!allow_failure) { + verifier->Fail(VERIFY_ERROR_BAD_CLASS_HARD) + << "tried to get class from non-reference register v" << this_reg + << " (type=" << this_type << ")"; + } return verifier->GetRegTypeCache()->Conflict(); } return this_type; diff --git a/runtime/verifier/register_line.h b/runtime/verifier/register_line.h index ca61a0b8f0..376dbf1fad 100644 --- a/runtime/verifier/register_line.h +++ b/runtime/verifier/register_line.h @@ -188,9 +188,11 @@ class RegisterLine { * * The argument count is in vA, and the first argument is in vC, for both "simple" and "range" * versions. We just need to make sure vA is >= 1 and then return vC. + * allow_failure will return Conflict() instead of causing a verification failure if there is an + * error. */ const RegType& GetInvocationThis(MethodVerifier* verifier, const Instruction* inst, - bool is_range) + bool is_range, bool allow_failure = false) SHARED_LOCKS_REQUIRED(Locks::mutator_lock_); /* diff --git a/test/135-MirandaDispatch/expected.txt b/test/135-MirandaDispatch/expected.txt new file mode 100644 index 0000000000..134d8d0b47 --- /dev/null +++ b/test/135-MirandaDispatch/expected.txt @@ -0,0 +1 @@ +Finishing diff --git a/test/135-MirandaDispatch/info.txt b/test/135-MirandaDispatch/info.txt new file mode 100644 index 0000000000..22d2777b5c --- /dev/null +++ b/test/135-MirandaDispatch/info.txt @@ -0,0 +1,6 @@ +Regression test for JIT related incompatible class changes caused by miranda methods. +E.g. +java.lang.IncompatibleClassChangeError: The method 'void Main$TheInterface.m()' was expected to be of type virtual but instead was found to be of type interface (declaration of 'java.lang.reflect.ArtMethod' appears in out/host/linux-x86/framework/core-libart-hostdex.jar) + at Main.DoStuff(Main.java:37) + at Main.main(Main.java:44) + diff --git a/test/135-MirandaDispatch/src/Main.java b/test/135-MirandaDispatch/src/Main.java new file mode 100644 index 0000000000..bb005b0103 --- /dev/null +++ b/test/135-MirandaDispatch/src/Main.java @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public class Main { + // Enough to trigger JIT. + static final int loopIterations = 5000; + static int counter = 0; + + static interface TheInterface { + public void m(); + } + + static abstract class AbstractClass implements TheInterface { + } + + static class ConcreteClass extends AbstractClass { + public void m() { + ++counter; + } + } + + static void doStuff(AbstractClass c) { + for (int i = 0; i < loopIterations; ++i) { + c.m(); + } + } + + public static void main(String[] args) throws Exception { + ConcreteClass o = new ConcreteClass(); + for (int i = 0; i < loopIterations; ++i) { + doStuff(o); + } + if (counter != loopIterations * loopIterations) { + System.out.println("Expected " + loopIterations * loopIterations + " got " + counter); + } + System.out.println("Finishing"); + } +} diff --git a/test/459-dead-phi/expected.txt b/test/459-dead-phi/expected.txt new file mode 100644 index 0000000000..ba66466c2a --- /dev/null +++ b/test/459-dead-phi/expected.txt @@ -0,0 +1 @@ +0.0 diff --git a/test/459-dead-phi/info.txt b/test/459-dead-phi/info.txt new file mode 100644 index 0000000000..3f82ecb66e --- /dev/null +++ b/test/459-dead-phi/info.txt @@ -0,0 +1 @@ +Regression test for optimizing when it is building the SSA form. diff --git a/test/459-dead-phi/smali/EquivalentPhi.smali b/test/459-dead-phi/smali/EquivalentPhi.smali new file mode 100644 index 0000000000..4fa88a9e80 --- /dev/null +++ b/test/459-dead-phi/smali/EquivalentPhi.smali @@ -0,0 +1,41 @@ +# Copyright (C) 2015 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +.class public LEquivalentPhi; + +.super Ljava/lang/Object; + +.method public static equivalentPhi([F)F + .registers 5 + const/4 v0, 0x0 + # aget is initally expected to be an int, but will + # rightly become a float after type propagation. + aget v1, p0, v0 + move v2, v1 + if-eq v0, v0, :else + move v2, v0 + :else + # v2 will be a phi with (int, int) as input + move v3, v2 + if-eq v0, v0, :else2 + move v3, v0 + # v3 will be a phi with (int, int) as input. + : else2 + # This instruction will lead to creating a phi equivalent + # for v3 with float type, which in turn will lead to creating + # a phi equivalent for v2 of type float. We used to forget to + # delete the old phi, which ends up having incompatible input + # types. + return v3 +.end method diff --git a/test/459-dead-phi/src/Main.java b/test/459-dead-phi/src/Main.java new file mode 100644 index 0000000000..0ecc0bdcc4 --- /dev/null +++ b/test/459-dead-phi/src/Main.java @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.lang.reflect.Method; + +public class Main { + + // Workaround for b/18051191. + class InnerClass {} + + public static void main(String[] args) throws Exception { + Class<?> c = Class.forName("EquivalentPhi"); + Method m = c.getMethod("equivalentPhi", float[].class); + System.out.println(m.invoke(null, new float[] { 0.0f })); + } +} diff --git a/test/MultiDex/main.jpp b/test/MultiDex/main.jpp new file mode 100644 index 0000000000..a5d7a6c136 --- /dev/null +++ b/test/MultiDex/main.jpp @@ -0,0 +1,3 @@ +main: + @@com.android.jack.annotations.ForceInMainDex + class Main |