Reland "Change hotness counting."
This reverts commit ce131feb7c60ffdf28c315c5d67f9cac33a077ce.
Bug: 203810169
Test: test.py
Test: health/microbench/startup/hscapps/compile-speed-profile/open-clock
Reason for revert: Kept logic from before on what methods to save in the
profile.
Change-Id: Id67cd47a9fe31b4c6b154db20f632015238016d2
diff --git a/compiler/optimizing/code_generator_arm64.cc b/compiler/optimizing/code_generator_arm64.cc
index d455614..a323405 100644
--- a/compiler/optimizing/code_generator_arm64.cc
+++ b/compiler/optimizing/code_generator_arm64.cc
@@ -1189,10 +1189,12 @@
__ Ldr(method, MemOperand(sp, 0));
}
__ Ldrh(counter, MemOperand(method, ArtMethod::HotnessCountOffset().Int32Value()));
- __ Add(counter, counter, 1);
- // Subtract one if the counter would overflow.
- __ Sub(counter, counter, Operand(counter, LSR, 16));
+ vixl::aarch64::Label done;
+ DCHECK_EQ(0u, interpreter::kNterpHotnessValue);
+ __ Cbz(counter, &done);
+ __ Add(counter, counter, -1);
__ Strh(counter, MemOperand(method, ArtMethod::HotnessCountOffset().Int32Value()));
+ __ Bind(&done);
}
if (GetGraph()->IsCompilingBaseline() && !Runtime::Current()->IsAotCompiler()) {
diff --git a/compiler/optimizing/code_generator_arm_vixl.cc b/compiler/optimizing/code_generator_arm_vixl.cc
index bf0c77d..6d77f54 100644
--- a/compiler/optimizing/code_generator_arm_vixl.cc
+++ b/compiler/optimizing/code_generator_arm_vixl.cc
@@ -2186,10 +2186,12 @@
}
// Load with zero extend to clear the high bits for integer overflow check.
__ Ldrh(temp, MemOperand(kMethodRegister, ArtMethod::HotnessCountOffset().Int32Value()));
- __ Add(temp, temp, 1);
- // Subtract one if the counter would overflow.
- __ Sub(temp, temp, Operand(temp, ShiftType::LSR, 16));
+ vixl::aarch32::Label done;
+ DCHECK_EQ(0u, interpreter::kNterpHotnessValue);
+ __ CompareAndBranchIfZero(temp, &done, /* is_far_target= */ false);
+ __ Add(temp, temp, -1);
__ Strh(temp, MemOperand(kMethodRegister, ArtMethod::HotnessCountOffset().Int32Value()));
+ __ Bind(&done);
if (!is_frame_entry) {
__ Pop(vixl32::Register(kMethodRegister));
GetAssembler()->cfi().AdjustCFAOffset(-static_cast<int>(kArmWordSize));
diff --git a/compiler/optimizing/code_generator_x86.cc b/compiler/optimizing/code_generator_x86.cc
index a04b412..4008b7d 100644
--- a/compiler/optimizing/code_generator_x86.cc
+++ b/compiler/optimizing/code_generator_x86.cc
@@ -1197,10 +1197,9 @@
}
NearLabel overflow;
__ cmpw(Address(reg, ArtMethod::HotnessCountOffset().Int32Value()),
- Immediate(ArtMethod::MaxCounter()));
+ Immediate(interpreter::kNterpHotnessValue));
__ j(kEqual, &overflow);
- __ addw(Address(reg, ArtMethod::HotnessCountOffset().Int32Value()),
- Immediate(1));
+ __ addw(Address(reg, ArtMethod::HotnessCountOffset().Int32Value()), Immediate(-1));
__ Bind(&overflow);
if (!is_frame_entry) {
__ popl(EAX);
diff --git a/compiler/optimizing/code_generator_x86_64.cc b/compiler/optimizing/code_generator_x86_64.cc
index 4ec2dd7..ad81620 100644
--- a/compiler/optimizing/code_generator_x86_64.cc
+++ b/compiler/optimizing/code_generator_x86_64.cc
@@ -1591,10 +1591,10 @@
__ movq(CpuRegister(method), Address(CpuRegister(RSP), kCurrentMethodStackOffset));
}
__ cmpw(Address(CpuRegister(method), ArtMethod::HotnessCountOffset().Int32Value()),
- Immediate(ArtMethod::MaxCounter()));
+ Immediate(interpreter::kNterpHotnessValue));
__ j(kEqual, &overflow);
__ addw(Address(CpuRegister(method), ArtMethod::HotnessCountOffset().Int32Value()),
- Immediate(1));
+ Immediate(-1));
__ Bind(&overflow);
}
diff --git a/runtime/art_method-inl.h b/runtime/art_method-inl.h
index 7491fc7..69eaef9 100644
--- a/runtime/art_method-inl.h
+++ b/runtime/art_method-inl.h
@@ -423,9 +423,48 @@
return CodeItemDebugInfoAccessor(*GetDexFile(), GetCodeItem(), GetDexMethodIndex());
}
-inline void ArtMethod::SetCounter(uint16_t hotness_count) {
+inline bool ArtMethod::CounterHasChanged() {
DCHECK(!IsAbstract());
- hotness_count_ = hotness_count;
+ return hotness_count_ != interpreter::kNterpHotnessMask;
+}
+
+inline void ArtMethod::ResetCounter() {
+ DCHECK(!IsAbstract());
+ // Avoid dirtying the value if possible.
+ if (hotness_count_ != interpreter::kNterpHotnessMask) {
+ hotness_count_ = interpreter::kNterpHotnessMask;
+ }
+}
+
+inline void ArtMethod::SetHotCounter() {
+ DCHECK(!IsAbstract());
+ // Avoid dirtying the value if possible.
+ if (hotness_count_ != 0) {
+ hotness_count_ = 0;
+ }
+}
+
+inline void ArtMethod::UpdateCounter(int32_t new_samples) {
+ DCHECK(!IsAbstract());
+ DCHECK_GT(new_samples, 0);
+ DCHECK_LE(new_samples, std::numeric_limits<uint16_t>::max());
+ uint16_t old_hotness_count = hotness_count_;
+ uint16_t new_count = (old_hotness_count <= new_samples) ? 0u : old_hotness_count - new_samples;
+ // Avoid dirtying the value if possible.
+ if (old_hotness_count != new_count) {
+ hotness_count_ = new_count;
+ }
+}
+
+inline bool ArtMethod::CounterIsHot() {
+ DCHECK(!IsAbstract());
+ return hotness_count_ == 0;
+}
+
+inline bool ArtMethod::CounterHasReached(uint16_t samples) {
+ DCHECK(!IsAbstract());
+ DCHECK_LE(samples, interpreter::kNterpHotnessMask);
+ return hotness_count_ <= (interpreter::kNterpHotnessMask - samples);
}
inline uint16_t ArtMethod::GetCounter() {
diff --git a/runtime/art_method.cc b/runtime/art_method.cc
index 2afd8df..8b748d2 100644
--- a/runtime/art_method.cc
+++ b/runtime/art_method.cc
@@ -753,7 +753,9 @@
SetDataPtrSize(nullptr, image_pointer_size);
}
// Clear hotness to let the JIT properly decide when to compile this method.
- hotness_count_ = 0;
+ if (!IsAbstract()) {
+ ResetCounter();
+ }
}
bool ArtMethod::IsImagePointerSize(PointerSize pointer_size) {
diff --git a/runtime/art_method.h b/runtime/art_method.h
index 3e8bccd..a4bb78e 100644
--- a/runtime/art_method.h
+++ b/runtime/art_method.h
@@ -32,6 +32,7 @@
#include "dex/dex_file_structs.h"
#include "dex/modifiers.h"
#include "dex/primitive.h"
+#include "interpreter/mterp/nterp.h"
#include "gc_root.h"
#include "obj_ptr.h"
#include "offsets.h"
@@ -75,7 +76,7 @@
static constexpr uint32_t kRuntimeMethodDexMethodIndex = 0xFFFFFFFF;
ArtMethod() : access_flags_(0), dex_method_index_(0),
- method_index_(0), hotness_count_(0) { }
+ method_index_(0), hotness_count_(interpreter::kNterpHotnessMask) { }
ArtMethod(ArtMethod* src, PointerSize image_pointer_size) {
CopyFrom(src, image_pointer_size);
@@ -683,9 +684,13 @@
void CopyFrom(ArtMethod* src, PointerSize image_pointer_size)
REQUIRES_SHARED(Locks::mutator_lock_);
- ALWAYS_INLINE void SetCounter(uint16_t hotness_count);
-
+ ALWAYS_INLINE void ResetCounter();
+ ALWAYS_INLINE void UpdateCounter(int32_t new_samples);
+ ALWAYS_INLINE void SetHotCounter();
+ ALWAYS_INLINE bool CounterIsHot();
+ ALWAYS_INLINE bool CounterHasReached(uint16_t samples);
ALWAYS_INLINE uint16_t GetCounter();
+ ALWAYS_INLINE bool CounterHasChanged();
ALWAYS_INLINE static constexpr uint16_t MaxCounter() {
return std::numeric_limits<decltype(hotness_count_)>::max();
diff --git a/runtime/class_linker.cc b/runtime/class_linker.cc
index 7bdf70f..6dbb3ff 100644
--- a/runtime/class_linker.cc
+++ b/runtime/class_linker.cc
@@ -1956,6 +1956,9 @@
const dex::CodeItem* code_item = method.GetDexFile()->GetCodeItem(
reinterpret_cast32<uint32_t>(method.GetDataPtrSize(image_pointer_size_)));
method.SetCodeItem(code_item);
+ // The hotness counter may have changed since we compiled the image, so
+ // reset it with the runtime value.
+ method.ResetCounter();
}
// Set image methods' entry point that point to the interpreter bridge to the
// nterp entry point.
diff --git a/runtime/interpreter/interpreter_switch_impl-inl.h b/runtime/interpreter/interpreter_switch_impl-inl.h
index 42f7fc7..f58fb95 100644
--- a/runtime/interpreter/interpreter_switch_impl-inl.h
+++ b/runtime/interpreter/interpreter_switch_impl-inl.h
@@ -245,7 +245,7 @@
// Hotness update.
jit::Jit* jit = Runtime::Current()->GetJit();
if (jit != nullptr) {
- jit->AddSamples(Self(), shadow_frame_.GetMethod(), 1, /*with_backedges=*/ true);
+ jit->AddSamples(Self(), shadow_frame_.GetMethod());
}
// Record new dex pc early to have consistent suspend point at loop header.
shadow_frame_.SetDexPC(next_->GetDexPc(Insns()));
diff --git a/runtime/interpreter/mterp/arm64ng/main.S b/runtime/interpreter/mterp/arm64ng/main.S
index 79eca04..08a7a80 100644
--- a/runtime/interpreter/mterp/arm64ng/main.S
+++ b/runtime/interpreter/mterp/arm64ng/main.S
@@ -301,11 +301,13 @@
2:
ldr x0, [sp]
ldrh w2, [x0, #ART_METHOD_HOTNESS_COUNT_OFFSET]
- add x2, x2, #1
- and w2, w2, #NTERP_HOTNESS_MASK
- strh w2, [x0, #ART_METHOD_HOTNESS_COUNT_OFFSET]
- // If the counter overflows, handle this in the runtime.
+#if (NTERP_HOTNESS_VALUE != 0)
+#error Expected 0 for hotness value
+#endif
+ // If the counter is at zero, handle this in the runtime.
cbz w2, NterpHandleHotnessOverflow
+ add x2, x2, #-1
+ strh w2, [x0, #ART_METHOD_HOTNESS_COUNT_OFFSET]
// Otherwise, do a suspend check.
ldr x0, [xSELF, #THREAD_FLAGS_OFFSET]
ands x0, x0, #THREAD_SUSPEND_OR_CHECKPOINT_REQUEST
@@ -422,11 +424,13 @@
.macro START_EXECUTING_INSTRUCTIONS
ldr x0, [sp]
ldrh w2, [x0, #ART_METHOD_HOTNESS_COUNT_OFFSET]
- add x2, x2, #1
- and w2, w2, #NTERP_HOTNESS_MASK
- strh w2, [x0, #ART_METHOD_HOTNESS_COUNT_OFFSET]
- // If the counter overflows, handle this in the runtime.
+#if (NTERP_HOTNESS_VALUE != 0)
+#error Expected 0 for hotness value
+#endif
+ // If the counter is at zero, handle this in the runtime.
cbz w2, 2f
+ add x2, x2, #-1
+ strh w2, [x0, #ART_METHOD_HOTNESS_COUNT_OFFSET]
ldr x0, [xSELF, #THREAD_FLAGS_OFFSET]
tst x0, #THREAD_SUSPEND_OR_CHECKPOINT_REQUEST
b.ne 3f
diff --git a/runtime/interpreter/mterp/armng/main.S b/runtime/interpreter/mterp/armng/main.S
index 932f6ac..13d5dfe 100644
--- a/runtime/interpreter/mterp/armng/main.S
+++ b/runtime/interpreter/mterp/armng/main.S
@@ -312,12 +312,10 @@
2:
ldr r0, [sp]
ldrh r2, [r0, #ART_METHOD_HOTNESS_COUNT_OFFSET]
- add r2, r2, #1
- ubfx r2, r2, #0, #NTERP_HOTNESS_BITS
- strh r2, [r0, #ART_METHOD_HOTNESS_COUNT_OFFSET]
- // If the counter overflows, handle this in the runtime.
- cmp r2, #0
+ cmp r2, #NTERP_HOTNESS_VALUE
beq NterpHandleHotnessOverflow
+ add r2, r2, #-1
+ strh r2, [r0, #ART_METHOD_HOTNESS_COUNT_OFFSET]
// Otherwise, do a suspend check.
ldr r0, [rSELF, #THREAD_FLAGS_OFFSET]
ands r0, r0, #THREAD_SUSPEND_OR_CHECKPOINT_REQUEST
@@ -451,12 +449,10 @@
.macro START_EXECUTING_INSTRUCTIONS
ldr r0, [sp]
ldrh r2, [r0, #ART_METHOD_HOTNESS_COUNT_OFFSET]
- add r2, r2, #1
- ubfx r2, r2, #0, #NTERP_HOTNESS_BITS
- strh r2, [r0, #ART_METHOD_HOTNESS_COUNT_OFFSET]
- // If the counter overflows, handle this in the runtime.
- cmp r2, #0
+ cmp r2, #NTERP_HOTNESS_VALUE
beq 2f
+ add r2, r2, #-1
+ strh r2, [r0, #ART_METHOD_HOTNESS_COUNT_OFFSET]
ldr r0, [rSELF, #THREAD_FLAGS_OFFSET]
tst r0, #THREAD_SUSPEND_OR_CHECKPOINT_REQUEST
bne 3f
diff --git a/runtime/interpreter/mterp/nterp.cc b/runtime/interpreter/mterp/nterp.cc
index fd57ece..e339dcf 100644
--- a/runtime/interpreter/mterp/nterp.cc
+++ b/runtime/interpreter/mterp/nterp.cc
@@ -90,18 +90,7 @@
// The hotness we will add to a method when we perform a
// field/method/class/string lookup.
constexpr uint16_t kNterpHotnessLookup = 0xf;
-
- // Convert to uint32_t to handle uint16_t overflow.
- uint32_t counter = method->GetCounter();
- uint32_t new_counter = counter + kNterpHotnessLookup;
- if (new_counter > kNterpHotnessMask) {
- // Let the nterp code actually call the compilation: we want to make sure
- // there's at least a second execution of the method or a back-edge to avoid
- // compiling straightline initialization methods.
- method->SetCounter(kNterpHotnessMask);
- } else {
- method->SetCounter(new_counter);
- }
+ method->UpdateCounter(kNterpHotnessLookup);
}
template<typename T>
@@ -738,6 +727,7 @@
extern "C" jit::OsrData* NterpHotMethod(ArtMethod* method, uint16_t* dex_pc_ptr, uint32_t* vregs)
REQUIRES_SHARED(Locks::mutator_lock_) {
ScopedAssertNoThreadSuspension sants("In nterp");
+ method->ResetCounter();
jit::Jit* jit = Runtime::Current()->GetJit();
if (jit != nullptr && jit->UseJitCompilation()) {
// Nterp passes null on entry where we don't want to OSR.
@@ -751,7 +741,7 @@
return osr_data;
}
}
- jit->EnqueueCompilationFromNterp(method, Thread::Current());
+ jit->EnqueueCompilation(method, Thread::Current());
}
return nullptr;
}
diff --git a/runtime/interpreter/mterp/nterp.h b/runtime/interpreter/mterp/nterp.h
index 97dee8c..94e5b8a 100644
--- a/runtime/interpreter/mterp/nterp.h
+++ b/runtime/interpreter/mterp/nterp.h
@@ -35,6 +35,7 @@
// The hotness threshold where we trigger JIT compilation or OSR.
constexpr uint16_t kNterpHotnessMask = 0xffff;
+constexpr uint16_t kNterpHotnessValue = 0;
// The hotness threshold for the baseline compiler to trigger optimized
// compilation.
diff --git a/runtime/interpreter/mterp/x86_64ng/main.S b/runtime/interpreter/mterp/x86_64ng/main.S
index 33912d7..4980007 100644
--- a/runtime/interpreter/mterp/x86_64ng/main.S
+++ b/runtime/interpreter/mterp/x86_64ng/main.S
@@ -260,10 +260,16 @@
GOTO_NEXT
3:
movq (%rsp), %rdi
- addw $$1, ART_METHOD_HOTNESS_COUNT_OFFSET(%rdi)
- andw $$(NTERP_HOTNESS_MASK), ART_METHOD_HOTNESS_COUNT_OFFSET(%rdi)
- // If the counter overflows, handle this in the runtime.
- jz NterpHandleHotnessOverflow
+ movzwl ART_METHOD_HOTNESS_COUNT_OFFSET(%rdi), %esi
+#if (NTERP_HOTNESS_VALUE != 0)
+#error Expected 0 for hotness value
+#endif
+ // If the counter is at zero, handle this in the runtime.
+ testw %si, %si
+ je NterpHandleHotnessOverflow
+ // Update counter.
+ addl $$-1, %esi
+ movw %si, ART_METHOD_HOTNESS_COUNT_OFFSET(%rdi)
// Otherwise, do a suspend check.
testl $$(THREAD_SUSPEND_OR_CHECKPOINT_REQUEST), rSELF:THREAD_FLAGS_OFFSET
jz 2b
@@ -820,9 +826,16 @@
// Increase method hotness and do suspend check before starting executing the method.
.macro START_EXECUTING_INSTRUCTIONS
movq (%rsp), %rdi
- addw $$1, ART_METHOD_HOTNESS_COUNT_OFFSET(%rdi)
- andw $$(NTERP_HOTNESS_MASK), ART_METHOD_HOTNESS_COUNT_OFFSET(%rdi)
- jz 2f
+ movzwl ART_METHOD_HOTNESS_COUNT_OFFSET(%rdi), %esi
+#if (NTERP_HOTNESS_VALUE != 0)
+#error Expected 0 for hotness value
+#endif
+ // If the counter is at zero, handle this in the runtime.
+ testw %si, %si
+ je 2f
+ // Update counter.
+ addl $$-1, %esi
+ movw %si, ART_METHOD_HOTNESS_COUNT_OFFSET(%rdi)
testl $$(THREAD_SUSPEND_OR_CHECKPOINT_REQUEST), rSELF:THREAD_FLAGS_OFFSET
jz 1f
EXPORT_PC
diff --git a/runtime/interpreter/mterp/x86ng/main.S b/runtime/interpreter/mterp/x86ng/main.S
index 7f185b0..7cc4126 100644
--- a/runtime/interpreter/mterp/x86ng/main.S
+++ b/runtime/interpreter/mterp/x86ng/main.S
@@ -302,8 +302,16 @@
GOTO_NEXT
3:
movl (%esp), %eax
- addw $$1, ART_METHOD_HOTNESS_COUNT_OFFSET(%eax)
- andw $$(NTERP_HOTNESS_MASK), ART_METHOD_HOTNESS_COUNT_OFFSET(%eax)
+ movzwl ART_METHOD_HOTNESS_COUNT_OFFSET(%eax), %ecx
+#if (NTERP_HOTNESS_VALUE != 0)
+#error Expected 0 for hotness value
+#endif
+ // If the counter is at zero, handle this in the runtime.
+ testw %cx, %cx
+ je NterpHandleHotnessOverflow
+ // Update counter.
+ addl $$-1, %ecx
+ movw %cx, ART_METHOD_HOTNESS_COUNT_OFFSET(%eax)
// If the counter overflows, handle this in the runtime.
jz NterpHandleHotnessOverflow
// Otherwise, do a suspend check.
@@ -965,8 +973,16 @@
// Increase method hotness and do suspend check before starting executing the method.
.macro START_EXECUTING_INSTRUCTIONS
movl (%esp), %eax
- addw $$1, ART_METHOD_HOTNESS_COUNT_OFFSET(%eax)
- andw $$(NTERP_HOTNESS_MASK), ART_METHOD_HOTNESS_COUNT_OFFSET(%eax)
+ movzwl ART_METHOD_HOTNESS_COUNT_OFFSET(%eax), %ecx
+#if (NTERP_HOTNESS_VALUE != 0)
+#error Expected 0 for hotness value
+#endif
+ // If the counter is at zero, handle this in the runtime.
+ testw %cx, %cx
+ je 2f
+ // Update counter.
+ addl $$-1, %ecx
+ movw %cx, ART_METHOD_HOTNESS_COUNT_OFFSET(%eax)
jz 2f
testl $$(THREAD_SUSPEND_OR_CHECKPOINT_REQUEST), rSELF:THREAD_FLAGS_OFFSET
jz 1f
diff --git a/runtime/jit/jit-inl.h b/runtime/jit/jit-inl.h
index e6b4095..683fc38 100644
--- a/runtime/jit/jit-inl.h
+++ b/runtime/jit/jit-inl.h
@@ -27,38 +27,16 @@
namespace art {
namespace jit {
-inline bool Jit::ShouldUsePriorityThreadWeight(Thread* self) {
- return self->IsJitSensitiveThread() && Runtime::Current()->InJankPerceptibleProcessState();
-}
-
-inline void Jit::AddSamples(Thread* self,
- ArtMethod* method,
- uint16_t samples,
- bool with_backedges) {
- if (Jit::ShouldUsePriorityThreadWeight(self)) {
- samples *= PriorityThreadWeight();
+inline void Jit::AddSamples(Thread* self, ArtMethod* method) {
+ if (IgnoreSamplesForMethod(method)) {
+ return;
}
- uint32_t old_count = method->GetCounter();
- uint32_t new_count = old_count + samples;
-
- // The full check is fairly expensive so we just add to hotness most of the time,
- // and we do the full check only when some of the higher bits of the count change.
- // NB: The method needs to see the transitions of the counter past the thresholds.
- uint32_t old_batch = RoundDown(old_count, kJitSamplesBatchSize); // Clear lower bits.
- uint32_t new_batch = RoundDown(new_count, kJitSamplesBatchSize); // Clear lower bits.
- if (UNLIKELY(kSlowMode)) { // Check every time in slow-debug mode.
- if (!MaybeCompileMethod(self, method, old_count, new_count, with_backedges)) {
- // Tests may check that the counter is 0 for methods that we never compile.
- return; // Ignore the samples for now and retry later.
- }
- } else if (UNLIKELY(old_batch != new_batch)) {
- if (!MaybeCompileMethod(self, method, old_batch, new_batch, with_backedges)) {
- // OSR compilation will ignore the samples if they don't have backedges.
- return; // Ignore the samples for now and retry later.
- }
+ if (method->CounterIsHot()) {
+ method->ResetCounter();
+ EnqueueCompilation(method, self);
+ } else {
+ method->UpdateCounter(1);
}
-
- method->SetCounter(new_count);
}
} // namespace jit
diff --git a/runtime/jit/jit.cc b/runtime/jit/jit.cc
index bb421e6..846ca4d 100644
--- a/runtime/jit/jit.cc
+++ b/runtime/jit/jit.cc
@@ -1472,7 +1472,7 @@
}
bool Jit::IgnoreSamplesForMethod(ArtMethod* method) REQUIRES_SHARED(Locks::mutator_lock_) {
- if (method->IsClassInitializer() || !method->IsCompilable() || method->IsPreCompiled()) {
+ if (method->IsClassInitializer() || !method->IsCompilable()) {
// We do not want to compile such methods.
return true;
}
@@ -1492,64 +1492,6 @@
return false;
}
-bool Jit::MaybeCompileMethod(Thread* self,
- ArtMethod* method,
- uint32_t old_count,
- uint32_t new_count,
- bool with_backedges) {
- if (thread_pool_ == nullptr) {
- return false;
- }
- if (UNLIKELY(method->IsPreCompiled()) && !with_backedges /* don't check for OSR */) {
- if (!NeedsClinitCheckBeforeCall(method) ||
- method->GetDeclaringClass()->IsVisiblyInitialized()) {
- const void* entry_point = code_cache_->GetSavedEntryPointOfPreCompiledMethod(method);
- if (entry_point != nullptr) {
- Runtime::Current()->GetInstrumentation()->UpdateMethodsCode(method, entry_point);
- return true;
- }
- }
- }
-
- if (IgnoreSamplesForMethod(method)) {
- return false;
- }
- if (HotMethodThreshold() == 0) {
- // Tests might request JIT on first use (compiled synchronously in the interpreter).
- return false;
- }
- DCHECK_GT(WarmMethodThreshold(), 0);
- DCHECK_GT(HotMethodThreshold(), WarmMethodThreshold());
- DCHECK_GT(OSRMethodThreshold(), HotMethodThreshold());
- DCHECK_GE(PriorityThreadWeight(), 1);
- DCHECK_LE(PriorityThreadWeight(), HotMethodThreshold());
-
- if (UseJitCompilation()) {
- if (old_count < HotMethodThreshold() && new_count >= HotMethodThreshold()) {
- if (!code_cache_->ContainsPc(method->GetEntryPointFromQuickCompiledCode())) {
- DCHECK(thread_pool_ != nullptr);
- thread_pool_->AddTask(
- self,
- new JitCompileTask(
- method, JitCompileTask::TaskKind::kCompile, CompilationKind::kBaseline));
- }
- }
- if (old_count < OSRMethodThreshold() && new_count >= OSRMethodThreshold()) {
- if (!with_backedges) {
- return false;
- }
- DCHECK(!method->IsNative()); // No back edges reported for native methods.
- if (!code_cache_->IsOsrCompiled(method)) {
- DCHECK(thread_pool_ != nullptr);
- thread_pool_->AddTask(
- self,
- new JitCompileTask(method, JitCompileTask::TaskKind::kCompile, CompilationKind::kOsr));
- }
- }
- }
- return true;
-}
-
void Jit::EnqueueOptimizedCompilation(ArtMethod* method, Thread* self) {
if (thread_pool_ == nullptr) {
return;
@@ -1598,7 +1540,7 @@
return;
}
- AddSamples(thread, method, 1, /* with_backedges= */false);
+ AddSamples(thread, method);
}
void Jit::WaitForCompilationToFinish(Thread* self) {
@@ -1792,19 +1734,48 @@
}
}
-void Jit::EnqueueCompilationFromNterp(ArtMethod* method, Thread* self) {
+void Jit::EnqueueCompilation(ArtMethod* method, Thread* self) {
if (thread_pool_ == nullptr) {
return;
}
- if (GetCodeCache()->ContainsPc(method->GetEntryPointFromQuickCompiledCode())) {
- // If we already have compiled code for it, nterp may be stuck in a loop.
- // Compile OSR.
- thread_pool_->AddTask(
- self,
- new JitCompileTask(method, JitCompileTask::TaskKind::kCompile, CompilationKind::kOsr));
+
+ if (JitAtFirstUse()) {
+ // Tests might request JIT on first use (compiled synchronously in the interpreter).
return;
}
- if (GetCodeCache()->CanAllocateProfilingInfo()) {
+
+ if (!UseJitCompilation()) {
+ return;
+ }
+
+ if (GetCodeCache()->ContainsPc(method->GetEntryPointFromQuickCompiledCode())) {
+ if (!method->IsNative() && !code_cache_->IsOsrCompiled(method)) {
+ // If we already have compiled code for it, nterp may be stuck in a loop.
+ // Compile OSR.
+ thread_pool_->AddTask(
+ self,
+ new JitCompileTask(method, JitCompileTask::TaskKind::kCompile, CompilationKind::kOsr));
+ }
+ return;
+ }
+
+ // Check if we have precompiled this method.
+ if (UNLIKELY(method->IsPreCompiled())) {
+ if (!NeedsClinitCheckBeforeCall(method) ||
+ method->GetDeclaringClass()->IsVisiblyInitialized()) {
+ const void* entry_point = code_cache_->GetSavedEntryPointOfPreCompiledMethod(method);
+ if (entry_point != nullptr) {
+ Runtime::Current()->GetInstrumentation()->UpdateMethodsCode(method, entry_point);
+ }
+ }
+ return;
+ }
+
+ if (IgnoreSamplesForMethod(method)) {
+ return;
+ }
+
+ if (!method->IsNative() && GetCodeCache()->CanAllocateProfilingInfo()) {
thread_pool_->AddTask(
self,
new JitCompileTask(method, JitCompileTask::TaskKind::kCompile, CompilationKind::kBaseline));
diff --git a/runtime/jit/jit.h b/runtime/jit/jit.h
index a6e484f..e06a6e0 100644
--- a/runtime/jit/jit.h
+++ b/runtime/jit/jit.h
@@ -314,24 +314,17 @@
void MethodEntered(Thread* thread, ArtMethod* method)
REQUIRES_SHARED(Locks::mutator_lock_);
- ALWAYS_INLINE void AddSamples(Thread* self,
- ArtMethod* method,
- uint16_t samples,
- bool with_backedges)
+ ALWAYS_INLINE void AddSamples(Thread* self, ArtMethod* method)
REQUIRES_SHARED(Locks::mutator_lock_);
void NotifyInterpreterToCompiledCodeTransition(Thread* self, ArtMethod* caller)
REQUIRES_SHARED(Locks::mutator_lock_) {
- if (!IgnoreSamplesForMethod(caller)) {
- AddSamples(self, caller, options_->GetInvokeTransitionWeight(), false);
- }
+ AddSamples(self, caller);
}
void NotifyCompiledCodeToInterpreterTransition(Thread* self, ArtMethod* callee)
REQUIRES_SHARED(Locks::mutator_lock_) {
- if (!IgnoreSamplesForMethod(callee)) {
- AddSamples(self, callee, options_->GetInvokeTransitionWeight(), false);
- }
+ AddSamples(self, callee);
}
// Starts the profile saver if the config options allow profile recording.
@@ -361,9 +354,6 @@
// Return whether we can invoke JIT code for `method`.
bool CanInvokeCompiledCode(ArtMethod* method);
- // Return whether the runtime should use a priority thread weight when sampling.
- static bool ShouldUsePriorityThreadWeight(Thread* self);
-
// Return the information required to do an OSR jump. Return null if the OSR
// cannot be done.
OsrData* PrepareForOsr(ArtMethod* method, uint32_t dex_pc, uint32_t* vregs)
@@ -453,7 +443,7 @@
void EnqueueOptimizedCompilation(ArtMethod* method, Thread* self);
- void EnqueueCompilationFromNterp(ArtMethod* method, Thread* self)
+ void EnqueueCompilation(ArtMethod* method, Thread* self)
REQUIRES_SHARED(Locks::mutator_lock_);
private:
@@ -474,15 +464,6 @@
bool compile_after_boot)
REQUIRES_SHARED(Locks::mutator_lock_);
- // Compile the method if the number of samples passes a threshold.
- // Returns false if we can not compile now - don't increment the counter and retry later.
- bool MaybeCompileMethod(Thread* self,
- ArtMethod* method,
- uint32_t old_count,
- uint32_t new_count,
- bool with_backedges)
- REQUIRES_SHARED(Locks::mutator_lock_);
-
static bool BindCompilerMethods(std::string* error_msg);
// JIT compiler
diff --git a/runtime/jit/jit_code_cache.cc b/runtime/jit/jit_code_cache.cc
index 9d9a7d3..e05ca11 100644
--- a/runtime/jit/jit_code_cache.cc
+++ b/runtime/jit/jit_code_cache.cc
@@ -615,12 +615,10 @@
if (was_warm) {
method->SetPreviouslyWarm();
}
- // We reset the counter to 1 so that the profile knows that the method was executed at least once.
+ method->ResetCounter();
+ // We add one sample so that the profile knows that the method was executed at least once.
// This is required for layout purposes.
- // We also need to make sure we'll pass the warmup threshold again, so we set to 0 if
- // the warmup threshold is 1.
- uint16_t jit_warmup_threshold = Runtime::Current()->GetJITOptions()->GetWarmupThreshold();
- method->SetCounter(std::min(jit_warmup_threshold - 1, 1));
+ method->UpdateCounter(/* new_samples= */ 1);
}
void JitCodeCache::WaitForPotentialCollectionToCompleteRunnable(Thread* self) {
@@ -794,7 +792,7 @@
return false;
}
- method->SetCounter(0);
+ ClearMethodCounter(method, /* was_warm= */ false);
Runtime::Current()->GetInstrumentation()->UpdateMethodsCode(
method, GetQuickToInterpreterBridge());
VLOG(jit)
@@ -1168,14 +1166,13 @@
if (!data.IsCompiled() || IsInZygoteExecSpace(data.GetCode())) {
continue;
}
- // Make sure a single invocation of the GenericJNI trampoline tries to recompile.
- uint16_t new_counter = Runtime::Current()->GetJit()->HotMethodThreshold() - 1u;
const OatQuickMethodHeader* method_header =
OatQuickMethodHeader::FromCodePointer(data.GetCode());
for (ArtMethod* method : data.GetMethods()) {
if (method->GetEntryPointFromQuickCompiledCode() == method_header->GetEntryPoint()) {
// Don't call Instrumentation::UpdateMethodsCode(), same as for normal methods above.
- method->SetCounter(new_counter);
+ // Make sure a single invocation of the GenericJNI trampoline tries to recompile.
+ method->SetHotCounter();
method->SetEntryPointFromQuickCompiledCode(GetQuickGenericJniStub());
}
}
@@ -1300,6 +1297,7 @@
OatQuickMethodHeader* method_header =
OatQuickMethodHeader::FromEntryPoint(entry_point);
if (CodeInfo::IsBaseline(method_header->GetOptimizedCodeInfoPtr())) {
+ info->GetMethod()->ResetCounter();
info->GetMethod()->SetEntryPointFromQuickCompiledCode(GetQuickToInterpreterBridge());
}
}
@@ -1489,7 +1487,6 @@
WaitUntilInlineCacheAccessible(self);
MutexLock mu(self, *Locks::jit_lock_);
ScopedTrace trace(__FUNCTION__);
- uint16_t jit_compile_threshold = Runtime::Current()->GetJITOptions()->GetCompileThreshold();
for (auto it : profiling_infos_) {
ProfilingInfo* info = it.second;
ArtMethod* method = info->GetMethod();
@@ -1501,10 +1498,13 @@
}
std::vector<ProfileMethodInfo::ProfileInlineCache> inline_caches;
- // If the method didn't reach the compilation threshold don't save the inline caches.
+ // If the method is still baseline compiled, don't save the inline caches.
// They might be incomplete and cause unnecessary deoptimizations.
// If the inline cache is empty the compiler will generate a regular invoke virtual/interface.
- if (method->GetCounter() < jit_compile_threshold) {
+ const void* entry_point = method->GetEntryPointFromQuickCompiledCode();
+ if (ContainsPc(entry_point) &&
+ CodeInfo::IsBaseline(
+ OatQuickMethodHeader::FromEntryPoint(entry_point)->GetOptimizedCodeInfoPtr())) {
methods.emplace_back(/*ProfileMethodInfo*/
MethodReference(dex_file, method->GetDexMethodIndex()), inline_caches);
continue;
diff --git a/runtime/jit/profile_saver.cc b/runtime/jit/profile_saver.cc
index b86badc..fce779f 100644
--- a/runtime/jit/profile_saver.cc
+++ b/runtime/jit/profile_saver.cc
@@ -637,11 +637,10 @@
auto get_method_flags = [&](ArtMethod& method) {
// Mark methods as hot if they have more than hot_method_sample_threshold
// samples. This means they will get compiled by the compiler driver.
- const uint16_t counter = method.GetCounter();
- if (method.PreviouslyWarm() || counter >= hot_method_sample_threshold) {
+ if (method.PreviouslyWarm() || method.CounterHasReached(hot_method_sample_threshold)) {
++number_of_hot_methods;
return enum_cast<ProfileCompilationInfo::MethodHotness::Flag>(base_flags | Hotness::kFlagHot);
- } else if (counter != 0u) {
+ } else if (method.CounterHasChanged()) {
++number_of_sampled_methods;
return enum_cast<ProfileCompilationInfo::MethodHotness::Flag>(base_flags);
} else {
diff --git a/runtime/native/dalvik_system_VMRuntime.cc b/runtime/native/dalvik_system_VMRuntime.cc
index 8425904..2e67b1a 100644
--- a/runtime/native/dalvik_system_VMRuntime.cc
+++ b/runtime/native/dalvik_system_VMRuntime.cc
@@ -479,9 +479,7 @@
}
for (ArtMethod& m : klass->GetMethods(kRuntimePointerSize)) {
if (!m.IsAbstract()) {
- if (m.GetCounter() != 0) {
- m.SetCounter(0);
- }
+ m.ResetCounter();
}
}
return true;
diff --git a/test/2230-profile-save-hotness/src-art/Main.java b/test/2230-profile-save-hotness/src-art/Main.java
index f71a891..c9132e6 100644
--- a/test/2230-profile-save-hotness/src-art/Main.java
+++ b/test/2230-profile-save-hotness/src-art/Main.java
@@ -34,6 +34,8 @@
return;
}
+ String methodName = "$noinline$hotnessCount";
+ int initialValue = getHotnessCounter(Main.class, methodName);
File file = null;
try {
file = createTempFile();
@@ -46,21 +48,22 @@
VMRuntime.CODE_PATH_TYPE_PRIMARY_APK);
// Test that the profile saves an app method with a profiling info.
- $noinline$hotnessCountWithLoop(10000);
+ $noinline$hotnessCountWithLoop(100000);
ensureProfileProcessing();
- String methodName = "$noinline$hotnessCount";
Method appMethod = Main.class.getDeclaredMethod(methodName);
if (!presentInProfile(file.getPath(), appMethod)) {
System.out.println("App method not hot in profile " +
getHotnessCounter(Main.class, methodName));
}
- if (getHotnessCounter(Main.class, methodName) == 0) {
- System.out.println("Hotness should be non zero " +
+ // Hardcoded assumption that the hotness value is zero.
+ if (getHotnessCounter(Main.class, methodName) != 0) {
+ System.out.println("Hotness should be zero " +
getHotnessCounter(Main.class, methodName));
}
VMRuntime.resetJitCounters();
- if (getHotnessCounter(Main.class, methodName) != 0) {
- System.out.println("Hotness should be zero " + getHotnessCounter(Main.class, methodName));
+ if (getHotnessCounter(Main.class, methodName) != initialValue) {
+ System.out.println(
+ "Expected " + initialValue +", got " + + getHotnessCounter(Main.class, methodName));
}
} finally {
if (file != null) {
diff --git a/test/674-hotness-compiled/src/Main.java b/test/674-hotness-compiled/src/Main.java
index 5f0d10a..a333074 100644
--- a/test/674-hotness-compiled/src/Main.java
+++ b/test/674-hotness-compiled/src/Main.java
@@ -29,23 +29,25 @@
if (!isAotCompiled(Main.class, "main")) {
return;
}
- $noinline$hotnessCount();
int counter = getHotnessCounter(Main.class, "$noinline$hotnessCount");
- if (counter == 0) {
+ $noinline$hotnessCount();
+ int newCounter = getHotnessCounter(Main.class, "$noinline$hotnessCount");
+ if (counter == newCounter) {
throw new Error("Expected hotness counter to be updated");
}
+ counter = getHotnessCounter(Main.class, "$noinline$hotnessCountWithLoop");
$noinline$hotnessCountWithLoop(1000);
- int newCounter = getHotnessCounter(Main.class, "$noinline$hotnessCountWithLoop");
- if (newCounter <= counter) {
- throw new Error("Expected counter " + newCounter + " to be larger than " + counter);
+ newCounter = getHotnessCounter(Main.class, "$noinline$hotnessCountWithLoop");
+ if (newCounter == counter) {
+ throw new Error("Expected counter " + newCounter + " to be different than " + counter);
}
counter = newCounter;
$noinline$hotnessCountWithLoop(65500);
newCounter = getHotnessCounter(Main.class, "$noinline$hotnessCountWithLoop");
- if (newCounter <= counter) {
- throw new Error("Expected counter " + newCounter + " to be larger than " + counter);
+ if (newCounter == counter) {
+ throw new Error("Expected counter " + newCounter + " to be different than " + counter);
}
}
diff --git a/test/716-jli-jit-samples/src-art/Main.java b/test/716-jli-jit-samples/src-art/Main.java
index def6b9f..ef63f16 100644
--- a/test/716-jli-jit-samples/src-art/Main.java
+++ b/test/716-jli-jit-samples/src-art/Main.java
@@ -25,6 +25,7 @@
private static final int ITERATIONS = 100;
private static final VarHandle widgetIdVarHandle;
+ private static int initialHotnessCounter;
public static native int getHotnessCounter(Class<?> cls, String methodName);
@@ -84,8 +85,8 @@
Widget.class, MethodType.methodType(void.class, int.class));
Widget w = (Widget) mh.invoke(3);
w = (Widget) mh.invokeExact(3);
- assertEquals(0, getHotnessCounter(MethodHandle.class, "invoke"));
- assertEquals(0, getHotnessCounter(MethodHandle.class, "invokeExact"));
+ assertEquals(initialHotnessCounter, getHotnessCounter(MethodHandle.class, "invoke"));
+ assertEquals(initialHotnessCounter, getHotnessCounter(MethodHandle.class, "invokeExact"));
// Reflective MethodHandle invocations
String[] methodNames = {"invoke", "invokeExact"};
@@ -102,8 +103,10 @@
assertEquals(ite.getCause().getClass(), UnsupportedOperationException.class);
}
}
- assertEquals(0, getHotnessCounter(MethodHandle.class, "invoke"));
- assertEquals(0, getHotnessCounter(MethodHandle.class, "invokeExact"));
+ assertEquals(initialHotnessCounter,
+ getHotnessCounter(MethodHandle.class, "invoke"));
+ assertEquals(initialHotnessCounter,
+ getHotnessCounter(MethodHandle.class, "invokeExact"));
}
System.out.println("MethodHandle OK");
@@ -115,8 +118,8 @@
// Regular accessor invocations
widgetIdVarHandle.set(w, i);
assertEquals(i, widgetIdVarHandle.get(w));
- assertEquals(0, getHotnessCounter(VarHandle.class, "set"));
- assertEquals(0, getHotnessCounter(VarHandle.class, "get"));
+ assertEquals(initialHotnessCounter, getHotnessCounter(VarHandle.class, "set"));
+ assertEquals(initialHotnessCounter, getHotnessCounter(VarHandle.class, "get"));
// Reflective accessor invocations
for (String accessorName : new String[] {"get", "set"}) {
@@ -128,14 +131,15 @@
assertEquals(ite.getCause().getClass(), UnsupportedOperationException.class);
}
}
- assertEquals(0, getHotnessCounter(VarHandle.class, "set"));
- assertEquals(0, getHotnessCounter(VarHandle.class, "get"));
+ assertEquals(initialHotnessCounter, getHotnessCounter(VarHandle.class, "set"));
+ assertEquals(initialHotnessCounter, getHotnessCounter(VarHandle.class, "get"));
}
System.out.println("VarHandle OK");
}
public static void main(String[] args) throws Throwable {
System.loadLibrary(args[0]);
+ initialHotnessCounter = getHotnessCounter(VarHandle.class, "set");
testMethodHandleCounters();
testVarHandleCounters();
}
diff --git a/tools/cpp-define-generator/globals.def b/tools/cpp-define-generator/globals.def
index ae8f5aa..2572ea6 100644
--- a/tools/cpp-define-generator/globals.def
+++ b/tools/cpp-define-generator/globals.def
@@ -64,6 +64,8 @@
art::interpreter::kNterpHandlerSize)
ASM_DEFINE(NTERP_HANDLER_SIZE_LOG2,
art::WhichPowerOf2(art::interpreter::kNterpHandlerSize))
+ASM_DEFINE(NTERP_HOTNESS_VALUE,
+ art::interpreter::kNterpHotnessValue)
ASM_DEFINE(OBJECT_ALIGNMENT_MASK,
art::kObjectAlignment - 1)
ASM_DEFINE(OBJECT_ALIGNMENT_MASK_TOGGLED,
@@ -80,7 +82,3 @@
std::memory_order_relaxed)
ASM_DEFINE(STACK_OVERFLOW_RESERVED_BYTES,
GetStackOverflowReservedBytes(art::kRuntimeISA))
-ASM_DEFINE(NTERP_HOTNESS_MASK,
- art::interpreter::kNterpHotnessMask)
-ASM_DEFINE(NTERP_HOTNESS_BITS,
- art::POPCOUNT(art::interpreter::kNterpHotnessMask))