diff options
25 files changed, 225 insertions, 220 deletions
diff --git a/compiler/optimizing/code_generator_arm64.cc b/compiler/optimizing/code_generator_arm64.cc index f6d38fbaca..74efc9ea8d 100644 --- a/compiler/optimizing/code_generator_arm64.cc +++ b/compiler/optimizing/code_generator_arm64.cc @@ -1123,12 +1123,10 @@ void CodeGeneratorARM64::MaybeIncrementHotness(bool is_frame_entry) { __ Ldr(method, MemOperand(sp, 0)); } __ Ldrh(counter, MemOperand(method, ArtMethod::HotnessCountOffset().Int32Value())); - vixl::aarch64::Label done; - DCHECK_EQ(0u, interpreter::kNterpHotnessValue); - __ Cbz(counter, &done); - __ Add(counter, counter, -1); + __ Add(counter, counter, 1); + // Subtract one if the counter would overflow. + __ Sub(counter, counter, Operand(counter, LSR, 16)); __ Strh(counter, MemOperand(method, ArtMethod::HotnessCountOffset().Int32Value())); - __ Bind(&done); } if (GetGraph()->IsCompilingBaseline() && !Runtime::Current()->IsAotCompiler()) { diff --git a/compiler/optimizing/code_generator_arm_vixl.cc b/compiler/optimizing/code_generator_arm_vixl.cc index 29c72d8d07..700202ba20 100644 --- a/compiler/optimizing/code_generator_arm_vixl.cc +++ b/compiler/optimizing/code_generator_arm_vixl.cc @@ -2123,12 +2123,10 @@ void CodeGeneratorARMVIXL::MaybeIncrementHotness(bool is_frame_entry) { } // Load with zero extend to clear the high bits for integer overflow check. __ Ldrh(temp, MemOperand(kMethodRegister, ArtMethod::HotnessCountOffset().Int32Value())); - vixl::aarch32::Label done; - DCHECK_EQ(0u, interpreter::kNterpHotnessValue); - __ CompareAndBranchIfZero(temp, &done, /* is_far_target= */ false); - __ Add(temp, temp, -1); + __ Add(temp, temp, 1); + // Subtract one if the counter would overflow. + __ Sub(temp, temp, Operand(temp, ShiftType::LSR, 16)); __ Strh(temp, MemOperand(kMethodRegister, ArtMethod::HotnessCountOffset().Int32Value())); - __ Bind(&done); if (!is_frame_entry) { __ Pop(vixl32::Register(kMethodRegister)); GetAssembler()->cfi().AdjustCFAOffset(-static_cast<int>(kArmWordSize)); diff --git a/compiler/optimizing/code_generator_x86.cc b/compiler/optimizing/code_generator_x86.cc index e6e22d3c4a..c49b08ba69 100644 --- a/compiler/optimizing/code_generator_x86.cc +++ b/compiler/optimizing/code_generator_x86.cc @@ -1109,9 +1109,10 @@ void CodeGeneratorX86::MaybeIncrementHotness(bool is_frame_entry) { } NearLabel overflow; __ cmpw(Address(reg, ArtMethod::HotnessCountOffset().Int32Value()), - Immediate(interpreter::kNterpHotnessValue)); + Immediate(ArtMethod::MaxCounter())); __ j(kEqual, &overflow); - __ addw(Address(reg, ArtMethod::HotnessCountOffset().Int32Value()), Immediate(-1)); + __ addw(Address(reg, ArtMethod::HotnessCountOffset().Int32Value()), + Immediate(1)); __ Bind(&overflow); if (!is_frame_entry) { __ popl(EAX); diff --git a/compiler/optimizing/code_generator_x86_64.cc b/compiler/optimizing/code_generator_x86_64.cc index de78bf27ff..0584dc1365 100644 --- a/compiler/optimizing/code_generator_x86_64.cc +++ b/compiler/optimizing/code_generator_x86_64.cc @@ -1491,10 +1491,10 @@ void CodeGeneratorX86_64::MaybeIncrementHotness(bool is_frame_entry) { __ movq(CpuRegister(method), Address(CpuRegister(RSP), kCurrentMethodStackOffset)); } __ cmpw(Address(CpuRegister(method), ArtMethod::HotnessCountOffset().Int32Value()), - Immediate(interpreter::kNterpHotnessValue)); + Immediate(ArtMethod::MaxCounter())); __ j(kEqual, &overflow); __ addw(Address(CpuRegister(method), ArtMethod::HotnessCountOffset().Int32Value()), - Immediate(-1)); + Immediate(1)); __ Bind(&overflow); } diff --git a/runtime/art_method-inl.h b/runtime/art_method-inl.h index cb1e21d638..614a67f57c 100644 --- a/runtime/art_method-inl.h +++ b/runtime/art_method-inl.h @@ -421,42 +421,9 @@ inline CodeItemDebugInfoAccessor ArtMethod::DexInstructionDebugInfo() { return CodeItemDebugInfoAccessor(*GetDexFile(), GetCodeItem(), GetDexMethodIndex()); } -inline bool ArtMethod::CounterHasChanged() { +inline void ArtMethod::SetCounter(uint16_t hotness_count) { DCHECK(!IsAbstract()); - return hotness_count_ != interpreter::kNterpHotnessMask; -} - -inline void ArtMethod::ResetCounter() { - DCHECK(!IsAbstract()); - // Avoid dirtying the value if possible. - if (hotness_count_ != interpreter::kNterpHotnessMask) { - hotness_count_ = interpreter::kNterpHotnessMask; - } -} - -inline void ArtMethod::SetHotCounter() { - DCHECK(!IsAbstract()); - // Avoid dirtying the value if possible. - if (hotness_count_ != 0) { - hotness_count_ = 0; - } -} - -inline void ArtMethod::UpdateCounter(int32_t new_samples) { - DCHECK(!IsAbstract()); - DCHECK_GT(new_samples, 0); - DCHECK_LE(new_samples, std::numeric_limits<uint16_t>::max()); - uint16_t old_hotness_count = hotness_count_; - uint16_t new_count = (old_hotness_count <= new_samples) ? 0u : old_hotness_count - new_samples; - // Avoid dirtying the value if possible. - if (old_hotness_count != new_count) { - hotness_count_ = new_count; - } -} - -inline bool ArtMethod::CounterIsHot() { - DCHECK(!IsAbstract()); - return hotness_count_ == 0; + hotness_count_ = hotness_count; } inline uint16_t ArtMethod::GetCounter() { diff --git a/runtime/art_method.cc b/runtime/art_method.cc index 8b748d2ebd..2afd8dfce5 100644 --- a/runtime/art_method.cc +++ b/runtime/art_method.cc @@ -753,9 +753,7 @@ void ArtMethod::CopyFrom(ArtMethod* src, PointerSize image_pointer_size) { SetDataPtrSize(nullptr, image_pointer_size); } // Clear hotness to let the JIT properly decide when to compile this method. - if (!IsAbstract()) { - ResetCounter(); - } + hotness_count_ = 0; } bool ArtMethod::IsImagePointerSize(PointerSize pointer_size) { diff --git a/runtime/art_method.h b/runtime/art_method.h index 2ba9d6676a..3e8bccda5c 100644 --- a/runtime/art_method.h +++ b/runtime/art_method.h @@ -32,7 +32,6 @@ #include "dex/dex_file_structs.h" #include "dex/modifiers.h" #include "dex/primitive.h" -#include "interpreter/mterp/nterp.h" #include "gc_root.h" #include "obj_ptr.h" #include "offsets.h" @@ -76,7 +75,7 @@ class ArtMethod final { static constexpr uint32_t kRuntimeMethodDexMethodIndex = 0xFFFFFFFF; ArtMethod() : access_flags_(0), dex_method_index_(0), - method_index_(0), hotness_count_(interpreter::kNterpHotnessMask) { } + method_index_(0), hotness_count_(0) { } ArtMethod(ArtMethod* src, PointerSize image_pointer_size) { CopyFrom(src, image_pointer_size); @@ -684,12 +683,9 @@ class ArtMethod final { void CopyFrom(ArtMethod* src, PointerSize image_pointer_size) REQUIRES_SHARED(Locks::mutator_lock_); - ALWAYS_INLINE void ResetCounter(); - ALWAYS_INLINE void UpdateCounter(int32_t new_samples); - ALWAYS_INLINE void SetHotCounter(); - ALWAYS_INLINE bool CounterIsHot(); + ALWAYS_INLINE void SetCounter(uint16_t hotness_count); + ALWAYS_INLINE uint16_t GetCounter(); - ALWAYS_INLINE bool CounterHasChanged(); ALWAYS_INLINE static constexpr uint16_t MaxCounter() { return std::numeric_limits<decltype(hotness_count_)>::max(); diff --git a/runtime/class_linker.cc b/runtime/class_linker.cc index 4875298e68..69e87d3ba4 100644 --- a/runtime/class_linker.cc +++ b/runtime/class_linker.cc @@ -1956,9 +1956,6 @@ bool ClassLinker::AddImageSpace( const dex::CodeItem* code_item = method.GetDexFile()->GetCodeItem( reinterpret_cast32<uint32_t>(method.GetDataPtrSize(image_pointer_size_))); method.SetCodeItem(code_item); - // The hotness counter may have changed since we compiled the image, so - // reset it with the runtime value. - method.ResetCounter(); } // Set image methods' entry point that point to the interpreter bridge to the // nterp entry point. diff --git a/runtime/interpreter/interpreter_switch_impl-inl.h b/runtime/interpreter/interpreter_switch_impl-inl.h index 86b796244e..8e16e04bfa 100644 --- a/runtime/interpreter/interpreter_switch_impl-inl.h +++ b/runtime/interpreter/interpreter_switch_impl-inl.h @@ -255,7 +255,7 @@ class InstructionHandler { // Hotness update. jit::Jit* jit = Runtime::Current()->GetJit(); if (jit != nullptr) { - jit->AddSamples(Self(), shadow_frame_.GetMethod()); + jit->AddSamples(Self(), shadow_frame_.GetMethod(), 1, /*with_backedges=*/ true); } // Record new dex pc early to have consistent suspend point at loop header. shadow_frame_.SetDexPC(next_->GetDexPc(Insns())); diff --git a/runtime/interpreter/mterp/arm64ng/main.S b/runtime/interpreter/mterp/arm64ng/main.S index 08a7a804a3..79eca04c8a 100644 --- a/runtime/interpreter/mterp/arm64ng/main.S +++ b/runtime/interpreter/mterp/arm64ng/main.S @@ -301,13 +301,11 @@ END \name 2: ldr x0, [sp] ldrh w2, [x0, #ART_METHOD_HOTNESS_COUNT_OFFSET] -#if (NTERP_HOTNESS_VALUE != 0) -#error Expected 0 for hotness value -#endif - // If the counter is at zero, handle this in the runtime. - cbz w2, NterpHandleHotnessOverflow - add x2, x2, #-1 + add x2, x2, #1 + and w2, w2, #NTERP_HOTNESS_MASK strh w2, [x0, #ART_METHOD_HOTNESS_COUNT_OFFSET] + // If the counter overflows, handle this in the runtime. + cbz w2, NterpHandleHotnessOverflow // Otherwise, do a suspend check. ldr x0, [xSELF, #THREAD_FLAGS_OFFSET] ands x0, x0, #THREAD_SUSPEND_OR_CHECKPOINT_REQUEST @@ -424,13 +422,11 @@ END \name .macro START_EXECUTING_INSTRUCTIONS ldr x0, [sp] ldrh w2, [x0, #ART_METHOD_HOTNESS_COUNT_OFFSET] -#if (NTERP_HOTNESS_VALUE != 0) -#error Expected 0 for hotness value -#endif - // If the counter is at zero, handle this in the runtime. - cbz w2, 2f - add x2, x2, #-1 + add x2, x2, #1 + and w2, w2, #NTERP_HOTNESS_MASK strh w2, [x0, #ART_METHOD_HOTNESS_COUNT_OFFSET] + // If the counter overflows, handle this in the runtime. + cbz w2, 2f ldr x0, [xSELF, #THREAD_FLAGS_OFFSET] tst x0, #THREAD_SUSPEND_OR_CHECKPOINT_REQUEST b.ne 3f diff --git a/runtime/interpreter/mterp/armng/main.S b/runtime/interpreter/mterp/armng/main.S index 13d5dfe3fb..932f6acba0 100644 --- a/runtime/interpreter/mterp/armng/main.S +++ b/runtime/interpreter/mterp/armng/main.S @@ -312,10 +312,12 @@ END \name 2: ldr r0, [sp] ldrh r2, [r0, #ART_METHOD_HOTNESS_COUNT_OFFSET] - cmp r2, #NTERP_HOTNESS_VALUE - beq NterpHandleHotnessOverflow - add r2, r2, #-1 + add r2, r2, #1 + ubfx r2, r2, #0, #NTERP_HOTNESS_BITS strh r2, [r0, #ART_METHOD_HOTNESS_COUNT_OFFSET] + // If the counter overflows, handle this in the runtime. + cmp r2, #0 + beq NterpHandleHotnessOverflow // Otherwise, do a suspend check. ldr r0, [rSELF, #THREAD_FLAGS_OFFSET] ands r0, r0, #THREAD_SUSPEND_OR_CHECKPOINT_REQUEST @@ -449,10 +451,12 @@ END \name .macro START_EXECUTING_INSTRUCTIONS ldr r0, [sp] ldrh r2, [r0, #ART_METHOD_HOTNESS_COUNT_OFFSET] - cmp r2, #NTERP_HOTNESS_VALUE - beq 2f - add r2, r2, #-1 + add r2, r2, #1 + ubfx r2, r2, #0, #NTERP_HOTNESS_BITS strh r2, [r0, #ART_METHOD_HOTNESS_COUNT_OFFSET] + // If the counter overflows, handle this in the runtime. + cmp r2, #0 + beq 2f ldr r0, [rSELF, #THREAD_FLAGS_OFFSET] tst r0, #THREAD_SUSPEND_OR_CHECKPOINT_REQUEST bne 3f diff --git a/runtime/interpreter/mterp/nterp.cc b/runtime/interpreter/mterp/nterp.cc index 629f64e55c..ddef31d792 100644 --- a/runtime/interpreter/mterp/nterp.cc +++ b/runtime/interpreter/mterp/nterp.cc @@ -89,7 +89,18 @@ inline void UpdateHotness(ArtMethod* method) REQUIRES_SHARED(Locks::mutator_lock // The hotness we will add to a method when we perform a // field/method/class/string lookup. constexpr uint16_t kNterpHotnessLookup = 0xf; - method->UpdateCounter(kNterpHotnessLookup); + + // Convert to uint32_t to handle uint16_t overflow. + uint32_t counter = method->GetCounter(); + uint32_t new_counter = counter + kNterpHotnessLookup; + if (new_counter > kNterpHotnessMask) { + // Let the nterp code actually call the compilation: we want to make sure + // there's at least a second execution of the method or a back-edge to avoid + // compiling straightline initialization methods. + method->SetCounter(kNterpHotnessMask); + } else { + method->SetCounter(new_counter); + } } template<typename T> @@ -724,7 +735,6 @@ extern "C" mirror::Object* NterpFilledNewArrayRange(Thread* self, extern "C" jit::OsrData* NterpHotMethod(ArtMethod* method, uint16_t* dex_pc_ptr, uint32_t* vregs) REQUIRES_SHARED(Locks::mutator_lock_) { ScopedAssertNoThreadSuspension sants("In nterp"); - method->ResetCounter(); jit::Jit* jit = Runtime::Current()->GetJit(); if (jit != nullptr && jit->UseJitCompilation()) { // Nterp passes null on entry where we don't want to OSR. @@ -738,7 +748,7 @@ extern "C" jit::OsrData* NterpHotMethod(ArtMethod* method, uint16_t* dex_pc_ptr, return osr_data; } } - jit->EnqueueCompilation(method, Thread::Current()); + jit->EnqueueCompilationFromNterp(method, Thread::Current()); } return nullptr; } diff --git a/runtime/interpreter/mterp/nterp.h b/runtime/interpreter/mterp/nterp.h index 94e5b8a50e..97dee8c7ae 100644 --- a/runtime/interpreter/mterp/nterp.h +++ b/runtime/interpreter/mterp/nterp.h @@ -35,7 +35,6 @@ const void* GetNterpEntryPoint(); // The hotness threshold where we trigger JIT compilation or OSR. constexpr uint16_t kNterpHotnessMask = 0xffff; -constexpr uint16_t kNterpHotnessValue = 0; // The hotness threshold for the baseline compiler to trigger optimized // compilation. diff --git a/runtime/interpreter/mterp/x86_64ng/main.S b/runtime/interpreter/mterp/x86_64ng/main.S index 49800072f8..33912d76af 100644 --- a/runtime/interpreter/mterp/x86_64ng/main.S +++ b/runtime/interpreter/mterp/x86_64ng/main.S @@ -260,16 +260,10 @@ END_FUNCTION \name GOTO_NEXT 3: movq (%rsp), %rdi - movzwl ART_METHOD_HOTNESS_COUNT_OFFSET(%rdi), %esi -#if (NTERP_HOTNESS_VALUE != 0) -#error Expected 0 for hotness value -#endif - // If the counter is at zero, handle this in the runtime. - testw %si, %si - je NterpHandleHotnessOverflow - // Update counter. - addl $$-1, %esi - movw %si, ART_METHOD_HOTNESS_COUNT_OFFSET(%rdi) + addw $$1, ART_METHOD_HOTNESS_COUNT_OFFSET(%rdi) + andw $$(NTERP_HOTNESS_MASK), ART_METHOD_HOTNESS_COUNT_OFFSET(%rdi) + // If the counter overflows, handle this in the runtime. + jz NterpHandleHotnessOverflow // Otherwise, do a suspend check. testl $$(THREAD_SUSPEND_OR_CHECKPOINT_REQUEST), rSELF:THREAD_FLAGS_OFFSET jz 2b @@ -826,16 +820,9 @@ END_FUNCTION \name // Increase method hotness and do suspend check before starting executing the method. .macro START_EXECUTING_INSTRUCTIONS movq (%rsp), %rdi - movzwl ART_METHOD_HOTNESS_COUNT_OFFSET(%rdi), %esi -#if (NTERP_HOTNESS_VALUE != 0) -#error Expected 0 for hotness value -#endif - // If the counter is at zero, handle this in the runtime. - testw %si, %si - je 2f - // Update counter. - addl $$-1, %esi - movw %si, ART_METHOD_HOTNESS_COUNT_OFFSET(%rdi) + addw $$1, ART_METHOD_HOTNESS_COUNT_OFFSET(%rdi) + andw $$(NTERP_HOTNESS_MASK), ART_METHOD_HOTNESS_COUNT_OFFSET(%rdi) + jz 2f testl $$(THREAD_SUSPEND_OR_CHECKPOINT_REQUEST), rSELF:THREAD_FLAGS_OFFSET jz 1f EXPORT_PC diff --git a/runtime/interpreter/mterp/x86ng/main.S b/runtime/interpreter/mterp/x86ng/main.S index 7cc4126c6d..7f185b0c8c 100644 --- a/runtime/interpreter/mterp/x86ng/main.S +++ b/runtime/interpreter/mterp/x86ng/main.S @@ -302,16 +302,8 @@ END_FUNCTION \name GOTO_NEXT 3: movl (%esp), %eax - movzwl ART_METHOD_HOTNESS_COUNT_OFFSET(%eax), %ecx -#if (NTERP_HOTNESS_VALUE != 0) -#error Expected 0 for hotness value -#endif - // If the counter is at zero, handle this in the runtime. - testw %cx, %cx - je NterpHandleHotnessOverflow - // Update counter. - addl $$-1, %ecx - movw %cx, ART_METHOD_HOTNESS_COUNT_OFFSET(%eax) + addw $$1, ART_METHOD_HOTNESS_COUNT_OFFSET(%eax) + andw $$(NTERP_HOTNESS_MASK), ART_METHOD_HOTNESS_COUNT_OFFSET(%eax) // If the counter overflows, handle this in the runtime. jz NterpHandleHotnessOverflow // Otherwise, do a suspend check. @@ -973,16 +965,8 @@ END_FUNCTION \name // Increase method hotness and do suspend check before starting executing the method. .macro START_EXECUTING_INSTRUCTIONS movl (%esp), %eax - movzwl ART_METHOD_HOTNESS_COUNT_OFFSET(%eax), %ecx -#if (NTERP_HOTNESS_VALUE != 0) -#error Expected 0 for hotness value -#endif - // If the counter is at zero, handle this in the runtime. - testw %cx, %cx - je 2f - // Update counter. - addl $$-1, %ecx - movw %cx, ART_METHOD_HOTNESS_COUNT_OFFSET(%eax) + addw $$1, ART_METHOD_HOTNESS_COUNT_OFFSET(%eax) + andw $$(NTERP_HOTNESS_MASK), ART_METHOD_HOTNESS_COUNT_OFFSET(%eax) jz 2f testl $$(THREAD_SUSPEND_OR_CHECKPOINT_REQUEST), rSELF:THREAD_FLAGS_OFFSET jz 1f diff --git a/runtime/jit/jit-inl.h b/runtime/jit/jit-inl.h index 683fc38e1b..e6b4095e1d 100644 --- a/runtime/jit/jit-inl.h +++ b/runtime/jit/jit-inl.h @@ -27,16 +27,38 @@ namespace art { namespace jit { -inline void Jit::AddSamples(Thread* self, ArtMethod* method) { - if (IgnoreSamplesForMethod(method)) { - return; +inline bool Jit::ShouldUsePriorityThreadWeight(Thread* self) { + return self->IsJitSensitiveThread() && Runtime::Current()->InJankPerceptibleProcessState(); +} + +inline void Jit::AddSamples(Thread* self, + ArtMethod* method, + uint16_t samples, + bool with_backedges) { + if (Jit::ShouldUsePriorityThreadWeight(self)) { + samples *= PriorityThreadWeight(); } - if (method->CounterIsHot()) { - method->ResetCounter(); - EnqueueCompilation(method, self); - } else { - method->UpdateCounter(1); + uint32_t old_count = method->GetCounter(); + uint32_t new_count = old_count + samples; + + // The full check is fairly expensive so we just add to hotness most of the time, + // and we do the full check only when some of the higher bits of the count change. + // NB: The method needs to see the transitions of the counter past the thresholds. + uint32_t old_batch = RoundDown(old_count, kJitSamplesBatchSize); // Clear lower bits. + uint32_t new_batch = RoundDown(new_count, kJitSamplesBatchSize); // Clear lower bits. + if (UNLIKELY(kSlowMode)) { // Check every time in slow-debug mode. + if (!MaybeCompileMethod(self, method, old_count, new_count, with_backedges)) { + // Tests may check that the counter is 0 for methods that we never compile. + return; // Ignore the samples for now and retry later. + } + } else if (UNLIKELY(old_batch != new_batch)) { + if (!MaybeCompileMethod(self, method, old_batch, new_batch, with_backedges)) { + // OSR compilation will ignore the samples if they don't have backedges. + return; // Ignore the samples for now and retry later. + } } + + method->SetCounter(new_count); } } // namespace jit diff --git a/runtime/jit/jit.cc b/runtime/jit/jit.cc index 846ca4d795..bb421e6b20 100644 --- a/runtime/jit/jit.cc +++ b/runtime/jit/jit.cc @@ -1472,7 +1472,7 @@ uint32_t Jit::CompileMethodsFromProfile( } bool Jit::IgnoreSamplesForMethod(ArtMethod* method) REQUIRES_SHARED(Locks::mutator_lock_) { - if (method->IsClassInitializer() || !method->IsCompilable()) { + if (method->IsClassInitializer() || !method->IsCompilable() || method->IsPreCompiled()) { // We do not want to compile such methods. return true; } @@ -1492,6 +1492,64 @@ bool Jit::IgnoreSamplesForMethod(ArtMethod* method) REQUIRES_SHARED(Locks::mutat return false; } +bool Jit::MaybeCompileMethod(Thread* self, + ArtMethod* method, + uint32_t old_count, + uint32_t new_count, + bool with_backedges) { + if (thread_pool_ == nullptr) { + return false; + } + if (UNLIKELY(method->IsPreCompiled()) && !with_backedges /* don't check for OSR */) { + if (!NeedsClinitCheckBeforeCall(method) || + method->GetDeclaringClass()->IsVisiblyInitialized()) { + const void* entry_point = code_cache_->GetSavedEntryPointOfPreCompiledMethod(method); + if (entry_point != nullptr) { + Runtime::Current()->GetInstrumentation()->UpdateMethodsCode(method, entry_point); + return true; + } + } + } + + if (IgnoreSamplesForMethod(method)) { + return false; + } + if (HotMethodThreshold() == 0) { + // Tests might request JIT on first use (compiled synchronously in the interpreter). + return false; + } + DCHECK_GT(WarmMethodThreshold(), 0); + DCHECK_GT(HotMethodThreshold(), WarmMethodThreshold()); + DCHECK_GT(OSRMethodThreshold(), HotMethodThreshold()); + DCHECK_GE(PriorityThreadWeight(), 1); + DCHECK_LE(PriorityThreadWeight(), HotMethodThreshold()); + + if (UseJitCompilation()) { + if (old_count < HotMethodThreshold() && new_count >= HotMethodThreshold()) { + if (!code_cache_->ContainsPc(method->GetEntryPointFromQuickCompiledCode())) { + DCHECK(thread_pool_ != nullptr); + thread_pool_->AddTask( + self, + new JitCompileTask( + method, JitCompileTask::TaskKind::kCompile, CompilationKind::kBaseline)); + } + } + if (old_count < OSRMethodThreshold() && new_count >= OSRMethodThreshold()) { + if (!with_backedges) { + return false; + } + DCHECK(!method->IsNative()); // No back edges reported for native methods. + if (!code_cache_->IsOsrCompiled(method)) { + DCHECK(thread_pool_ != nullptr); + thread_pool_->AddTask( + self, + new JitCompileTask(method, JitCompileTask::TaskKind::kCompile, CompilationKind::kOsr)); + } + } + } + return true; +} + void Jit::EnqueueOptimizedCompilation(ArtMethod* method, Thread* self) { if (thread_pool_ == nullptr) { return; @@ -1540,7 +1598,7 @@ void Jit::MethodEntered(Thread* thread, ArtMethod* method) { return; } - AddSamples(thread, method); + AddSamples(thread, method, 1, /* with_backedges= */false); } void Jit::WaitForCompilationToFinish(Thread* self) { @@ -1734,48 +1792,19 @@ bool Jit::CanAssumeInitialized(ObjPtr<mirror::Class> cls, bool is_for_shared_reg } } -void Jit::EnqueueCompilation(ArtMethod* method, Thread* self) { +void Jit::EnqueueCompilationFromNterp(ArtMethod* method, Thread* self) { if (thread_pool_ == nullptr) { return; } - - if (JitAtFirstUse()) { - // Tests might request JIT on first use (compiled synchronously in the interpreter). - return; - } - - if (!UseJitCompilation()) { - return; - } - if (GetCodeCache()->ContainsPc(method->GetEntryPointFromQuickCompiledCode())) { - if (!method->IsNative() && !code_cache_->IsOsrCompiled(method)) { - // If we already have compiled code for it, nterp may be stuck in a loop. - // Compile OSR. - thread_pool_->AddTask( - self, - new JitCompileTask(method, JitCompileTask::TaskKind::kCompile, CompilationKind::kOsr)); - } - return; - } - - // Check if we have precompiled this method. - if (UNLIKELY(method->IsPreCompiled())) { - if (!NeedsClinitCheckBeforeCall(method) || - method->GetDeclaringClass()->IsVisiblyInitialized()) { - const void* entry_point = code_cache_->GetSavedEntryPointOfPreCompiledMethod(method); - if (entry_point != nullptr) { - Runtime::Current()->GetInstrumentation()->UpdateMethodsCode(method, entry_point); - } - } - return; - } - - if (IgnoreSamplesForMethod(method)) { + // If we already have compiled code for it, nterp may be stuck in a loop. + // Compile OSR. + thread_pool_->AddTask( + self, + new JitCompileTask(method, JitCompileTask::TaskKind::kCompile, CompilationKind::kOsr)); return; } - - if (!method->IsNative() && GetCodeCache()->CanAllocateProfilingInfo()) { + if (GetCodeCache()->CanAllocateProfilingInfo()) { thread_pool_->AddTask( self, new JitCompileTask(method, JitCompileTask::TaskKind::kCompile, CompilationKind::kBaseline)); diff --git a/runtime/jit/jit.h b/runtime/jit/jit.h index e06a6e04b7..a6e484f563 100644 --- a/runtime/jit/jit.h +++ b/runtime/jit/jit.h @@ -314,17 +314,24 @@ class Jit { void MethodEntered(Thread* thread, ArtMethod* method) REQUIRES_SHARED(Locks::mutator_lock_); - ALWAYS_INLINE void AddSamples(Thread* self, ArtMethod* method) + ALWAYS_INLINE void AddSamples(Thread* self, + ArtMethod* method, + uint16_t samples, + bool with_backedges) REQUIRES_SHARED(Locks::mutator_lock_); void NotifyInterpreterToCompiledCodeTransition(Thread* self, ArtMethod* caller) REQUIRES_SHARED(Locks::mutator_lock_) { - AddSamples(self, caller); + if (!IgnoreSamplesForMethod(caller)) { + AddSamples(self, caller, options_->GetInvokeTransitionWeight(), false); + } } void NotifyCompiledCodeToInterpreterTransition(Thread* self, ArtMethod* callee) REQUIRES_SHARED(Locks::mutator_lock_) { - AddSamples(self, callee); + if (!IgnoreSamplesForMethod(callee)) { + AddSamples(self, callee, options_->GetInvokeTransitionWeight(), false); + } } // Starts the profile saver if the config options allow profile recording. @@ -354,6 +361,9 @@ class Jit { // Return whether we can invoke JIT code for `method`. bool CanInvokeCompiledCode(ArtMethod* method); + // Return whether the runtime should use a priority thread weight when sampling. + static bool ShouldUsePriorityThreadWeight(Thread* self); + // Return the information required to do an OSR jump. Return null if the OSR // cannot be done. OsrData* PrepareForOsr(ArtMethod* method, uint32_t dex_pc, uint32_t* vregs) @@ -443,7 +453,7 @@ class Jit { void EnqueueOptimizedCompilation(ArtMethod* method, Thread* self); - void EnqueueCompilation(ArtMethod* method, Thread* self) + void EnqueueCompilationFromNterp(ArtMethod* method, Thread* self) REQUIRES_SHARED(Locks::mutator_lock_); private: @@ -464,6 +474,15 @@ class Jit { bool compile_after_boot) REQUIRES_SHARED(Locks::mutator_lock_); + // Compile the method if the number of samples passes a threshold. + // Returns false if we can not compile now - don't increment the counter and retry later. + bool MaybeCompileMethod(Thread* self, + ArtMethod* method, + uint32_t old_count, + uint32_t new_count, + bool with_backedges) + REQUIRES_SHARED(Locks::mutator_lock_); + static bool BindCompilerMethods(std::string* error_msg); // JIT compiler diff --git a/runtime/jit/jit_code_cache.cc b/runtime/jit/jit_code_cache.cc index 7545829700..9d9a7d3bd4 100644 --- a/runtime/jit/jit_code_cache.cc +++ b/runtime/jit/jit_code_cache.cc @@ -615,7 +615,12 @@ static void ClearMethodCounter(ArtMethod* method, bool was_warm) if (was_warm) { method->SetPreviouslyWarm(); } - method->ResetCounter(); + // We reset the counter to 1 so that the profile knows that the method was executed at least once. + // This is required for layout purposes. + // We also need to make sure we'll pass the warmup threshold again, so we set to 0 if + // the warmup threshold is 1. + uint16_t jit_warmup_threshold = Runtime::Current()->GetJITOptions()->GetWarmupThreshold(); + method->SetCounter(std::min(jit_warmup_threshold - 1, 1)); } void JitCodeCache::WaitForPotentialCollectionToCompleteRunnable(Thread* self) { @@ -789,7 +794,7 @@ bool JitCodeCache::RemoveMethod(ArtMethod* method, bool release_memory) { return false; } - ClearMethodCounter(method, /* was_warm= */ false); + method->SetCounter(0); Runtime::Current()->GetInstrumentation()->UpdateMethodsCode( method, GetQuickToInterpreterBridge()); VLOG(jit) @@ -1163,13 +1168,14 @@ void JitCodeCache::GarbageCollectCache(Thread* self) { if (!data.IsCompiled() || IsInZygoteExecSpace(data.GetCode())) { continue; } + // Make sure a single invocation of the GenericJNI trampoline tries to recompile. + uint16_t new_counter = Runtime::Current()->GetJit()->HotMethodThreshold() - 1u; const OatQuickMethodHeader* method_header = OatQuickMethodHeader::FromCodePointer(data.GetCode()); for (ArtMethod* method : data.GetMethods()) { if (method->GetEntryPointFromQuickCompiledCode() == method_header->GetEntryPoint()) { // Don't call Instrumentation::UpdateMethodsCode(), same as for normal methods above. - // Make sure a single invocation of the GenericJNI trampoline tries to recompile. - method->SetHotCounter(); + method->SetCounter(new_counter); method->SetEntryPointFromQuickCompiledCode(GetQuickGenericJniStub()); } } @@ -1294,7 +1300,6 @@ void JitCodeCache::DoCollection(Thread* self, bool collect_profiling_info) { OatQuickMethodHeader* method_header = OatQuickMethodHeader::FromEntryPoint(entry_point); if (CodeInfo::IsBaseline(method_header->GetOptimizedCodeInfoPtr())) { - info->GetMethod()->ResetCounter(); info->GetMethod()->SetEntryPointFromQuickCompiledCode(GetQuickToInterpreterBridge()); } } @@ -1484,6 +1489,7 @@ void JitCodeCache::GetProfiledMethods(const std::set<std::string>& dex_base_loca WaitUntilInlineCacheAccessible(self); MutexLock mu(self, *Locks::jit_lock_); ScopedTrace trace(__FUNCTION__); + uint16_t jit_compile_threshold = Runtime::Current()->GetJITOptions()->GetCompileThreshold(); for (auto it : profiling_infos_) { ProfilingInfo* info = it.second; ArtMethod* method = info->GetMethod(); @@ -1495,13 +1501,10 @@ void JitCodeCache::GetProfiledMethods(const std::set<std::string>& dex_base_loca } std::vector<ProfileMethodInfo::ProfileInlineCache> inline_caches; - // If the method is still baseline compiled, don't save the inline caches. + // If the method didn't reach the compilation threshold don't save the inline caches. // They might be incomplete and cause unnecessary deoptimizations. // If the inline cache is empty the compiler will generate a regular invoke virtual/interface. - const void* entry_point = method->GetEntryPointFromQuickCompiledCode(); - if (ContainsPc(entry_point) && - CodeInfo::IsBaseline( - OatQuickMethodHeader::FromEntryPoint(entry_point)->GetOptimizedCodeInfoPtr())) { + if (method->GetCounter() < jit_compile_threshold) { methods.emplace_back(/*ProfileMethodInfo*/ MethodReference(dex_file, method->GetDexMethodIndex()), inline_caches); continue; diff --git a/runtime/jit/profile_saver.cc b/runtime/jit/profile_saver.cc index 29747426a6..b86badcba5 100644 --- a/runtime/jit/profile_saver.cc +++ b/runtime/jit/profile_saver.cc @@ -626,6 +626,7 @@ void ProfileSaver::GetClassesAndMethodsHelper::UpdateProfile(const std::set<std: ProfileCompilationInfo* profile_info) { // Move members to local variables to allow the compiler to optimize this properly. const bool startup = startup_; + const uint32_t hot_method_sample_threshold = hot_method_sample_threshold_; const uint32_t base_flags = (startup ? Hotness::kFlagStartup : Hotness::kFlagPostStartup) | extra_flags_; @@ -636,10 +637,11 @@ void ProfileSaver::GetClassesAndMethodsHelper::UpdateProfile(const std::set<std: auto get_method_flags = [&](ArtMethod& method) { // Mark methods as hot if they have more than hot_method_sample_threshold // samples. This means they will get compiled by the compiler driver. - if (method.PreviouslyWarm() || method.CounterIsHot()) { + const uint16_t counter = method.GetCounter(); + if (method.PreviouslyWarm() || counter >= hot_method_sample_threshold) { ++number_of_hot_methods; return enum_cast<ProfileCompilationInfo::MethodHotness::Flag>(base_flags | Hotness::kFlagHot); - } else if (method.CounterHasChanged()) { + } else if (counter != 0u) { ++number_of_sampled_methods; return enum_cast<ProfileCompilationInfo::MethodHotness::Flag>(base_flags); } else { diff --git a/runtime/native/dalvik_system_VMRuntime.cc b/runtime/native/dalvik_system_VMRuntime.cc index 2e67b1aba6..842590408d 100644 --- a/runtime/native/dalvik_system_VMRuntime.cc +++ b/runtime/native/dalvik_system_VMRuntime.cc @@ -479,7 +479,9 @@ class ClearJitCountersVisitor : public ClassVisitor { } for (ArtMethod& m : klass->GetMethods(kRuntimePointerSize)) { if (!m.IsAbstract()) { - m.ResetCounter(); + if (m.GetCounter() != 0) { + m.SetCounter(0); + } } } return true; diff --git a/test/2230-profile-save-hotness/src-art/Main.java b/test/2230-profile-save-hotness/src-art/Main.java index c9132e6e7d..f71a891fa3 100644 --- a/test/2230-profile-save-hotness/src-art/Main.java +++ b/test/2230-profile-save-hotness/src-art/Main.java @@ -34,8 +34,6 @@ public class Main { return; } - String methodName = "$noinline$hotnessCount"; - int initialValue = getHotnessCounter(Main.class, methodName); File file = null; try { file = createTempFile(); @@ -48,22 +46,21 @@ public class Main { VMRuntime.CODE_PATH_TYPE_PRIMARY_APK); // Test that the profile saves an app method with a profiling info. - $noinline$hotnessCountWithLoop(100000); + $noinline$hotnessCountWithLoop(10000); ensureProfileProcessing(); + String methodName = "$noinline$hotnessCount"; Method appMethod = Main.class.getDeclaredMethod(methodName); if (!presentInProfile(file.getPath(), appMethod)) { System.out.println("App method not hot in profile " + getHotnessCounter(Main.class, methodName)); } - // Hardcoded assumption that the hotness value is zero. - if (getHotnessCounter(Main.class, methodName) != 0) { - System.out.println("Hotness should be zero " + + if (getHotnessCounter(Main.class, methodName) == 0) { + System.out.println("Hotness should be non zero " + getHotnessCounter(Main.class, methodName)); } VMRuntime.resetJitCounters(); - if (getHotnessCounter(Main.class, methodName) != initialValue) { - System.out.println( - "Expected " + initialValue +", got " + + getHotnessCounter(Main.class, methodName)); + if (getHotnessCounter(Main.class, methodName) != 0) { + System.out.println("Hotness should be zero " + getHotnessCounter(Main.class, methodName)); } } finally { if (file != null) { diff --git a/test/674-hotness-compiled/src/Main.java b/test/674-hotness-compiled/src/Main.java index a3330747eb..5f0d10ae03 100644 --- a/test/674-hotness-compiled/src/Main.java +++ b/test/674-hotness-compiled/src/Main.java @@ -29,25 +29,23 @@ public class Main { if (!isAotCompiled(Main.class, "main")) { return; } - int counter = getHotnessCounter(Main.class, "$noinline$hotnessCount"); $noinline$hotnessCount(); - int newCounter = getHotnessCounter(Main.class, "$noinline$hotnessCount"); - if (counter == newCounter) { + int counter = getHotnessCounter(Main.class, "$noinline$hotnessCount"); + if (counter == 0) { throw new Error("Expected hotness counter to be updated"); } - counter = getHotnessCounter(Main.class, "$noinline$hotnessCountWithLoop"); $noinline$hotnessCountWithLoop(1000); - newCounter = getHotnessCounter(Main.class, "$noinline$hotnessCountWithLoop"); - if (newCounter == counter) { - throw new Error("Expected counter " + newCounter + " to be different than " + counter); + int newCounter = getHotnessCounter(Main.class, "$noinline$hotnessCountWithLoop"); + if (newCounter <= counter) { + throw new Error("Expected counter " + newCounter + " to be larger than " + counter); } counter = newCounter; $noinline$hotnessCountWithLoop(65500); newCounter = getHotnessCounter(Main.class, "$noinline$hotnessCountWithLoop"); - if (newCounter == counter) { - throw new Error("Expected counter " + newCounter + " to be different than " + counter); + if (newCounter <= counter) { + throw new Error("Expected counter " + newCounter + " to be larger than " + counter); } } diff --git a/test/716-jli-jit-samples/src-art/Main.java b/test/716-jli-jit-samples/src-art/Main.java index ef63f167b1..def6b9f669 100644 --- a/test/716-jli-jit-samples/src-art/Main.java +++ b/test/716-jli-jit-samples/src-art/Main.java @@ -25,7 +25,6 @@ public class Main { private static final int ITERATIONS = 100; private static final VarHandle widgetIdVarHandle; - private static int initialHotnessCounter; public static native int getHotnessCounter(Class<?> cls, String methodName); @@ -85,8 +84,8 @@ public class Main { Widget.class, MethodType.methodType(void.class, int.class)); Widget w = (Widget) mh.invoke(3); w = (Widget) mh.invokeExact(3); - assertEquals(initialHotnessCounter, getHotnessCounter(MethodHandle.class, "invoke")); - assertEquals(initialHotnessCounter, getHotnessCounter(MethodHandle.class, "invokeExact")); + assertEquals(0, getHotnessCounter(MethodHandle.class, "invoke")); + assertEquals(0, getHotnessCounter(MethodHandle.class, "invokeExact")); // Reflective MethodHandle invocations String[] methodNames = {"invoke", "invokeExact"}; @@ -103,10 +102,8 @@ public class Main { assertEquals(ite.getCause().getClass(), UnsupportedOperationException.class); } } - assertEquals(initialHotnessCounter, - getHotnessCounter(MethodHandle.class, "invoke")); - assertEquals(initialHotnessCounter, - getHotnessCounter(MethodHandle.class, "invokeExact")); + assertEquals(0, getHotnessCounter(MethodHandle.class, "invoke")); + assertEquals(0, getHotnessCounter(MethodHandle.class, "invokeExact")); } System.out.println("MethodHandle OK"); @@ -118,8 +115,8 @@ public class Main { // Regular accessor invocations widgetIdVarHandle.set(w, i); assertEquals(i, widgetIdVarHandle.get(w)); - assertEquals(initialHotnessCounter, getHotnessCounter(VarHandle.class, "set")); - assertEquals(initialHotnessCounter, getHotnessCounter(VarHandle.class, "get")); + assertEquals(0, getHotnessCounter(VarHandle.class, "set")); + assertEquals(0, getHotnessCounter(VarHandle.class, "get")); // Reflective accessor invocations for (String accessorName : new String[] {"get", "set"}) { @@ -131,15 +128,14 @@ public class Main { assertEquals(ite.getCause().getClass(), UnsupportedOperationException.class); } } - assertEquals(initialHotnessCounter, getHotnessCounter(VarHandle.class, "set")); - assertEquals(initialHotnessCounter, getHotnessCounter(VarHandle.class, "get")); + assertEquals(0, getHotnessCounter(VarHandle.class, "set")); + assertEquals(0, getHotnessCounter(VarHandle.class, "get")); } System.out.println("VarHandle OK"); } public static void main(String[] args) throws Throwable { System.loadLibrary(args[0]); - initialHotnessCounter = getHotnessCounter(VarHandle.class, "set"); testMethodHandleCounters(); testVarHandleCounters(); } diff --git a/tools/cpp-define-generator/globals.def b/tools/cpp-define-generator/globals.def index 2572ea6f9b..ae8f5aabe5 100644 --- a/tools/cpp-define-generator/globals.def +++ b/tools/cpp-define-generator/globals.def @@ -64,8 +64,6 @@ ASM_DEFINE(NTERP_HANDLER_SIZE, art::interpreter::kNterpHandlerSize) ASM_DEFINE(NTERP_HANDLER_SIZE_LOG2, art::WhichPowerOf2(art::interpreter::kNterpHandlerSize)) -ASM_DEFINE(NTERP_HOTNESS_VALUE, - art::interpreter::kNterpHotnessValue) ASM_DEFINE(OBJECT_ALIGNMENT_MASK, art::kObjectAlignment - 1) ASM_DEFINE(OBJECT_ALIGNMENT_MASK_TOGGLED, @@ -82,3 +80,7 @@ ASM_DEFINE(STD_MEMORY_ORDER_RELAXED, std::memory_order_relaxed) ASM_DEFINE(STACK_OVERFLOW_RESERVED_BYTES, GetStackOverflowReservedBytes(art::kRuntimeISA)) +ASM_DEFINE(NTERP_HOTNESS_MASK, + art::interpreter::kNterpHotnessMask) +ASM_DEFINE(NTERP_HOTNESS_BITS, + art::POPCOUNT(art::interpreter::kNterpHotnessMask)) |