Revert "Change hotness counting."
This reverts commit c86869ab894c05e3181e7d15eb1893fa8a3fcd47.
Reason for revert: App startup performance regression.
Bug: 203810169
Change-Id: Ie816969fffd7740fcdfa330aeb645399f5351865
diff --git a/compiler/optimizing/code_generator_arm64.cc b/compiler/optimizing/code_generator_arm64.cc
index f6d38fb..74efc9e 100644
--- a/compiler/optimizing/code_generator_arm64.cc
+++ b/compiler/optimizing/code_generator_arm64.cc
@@ -1123,12 +1123,10 @@
__ Ldr(method, MemOperand(sp, 0));
}
__ Ldrh(counter, MemOperand(method, ArtMethod::HotnessCountOffset().Int32Value()));
- vixl::aarch64::Label done;
- DCHECK_EQ(0u, interpreter::kNterpHotnessValue);
- __ Cbz(counter, &done);
- __ Add(counter, counter, -1);
+ __ Add(counter, counter, 1);
+ // Subtract one if the counter would overflow.
+ __ Sub(counter, counter, Operand(counter, LSR, 16));
__ Strh(counter, MemOperand(method, ArtMethod::HotnessCountOffset().Int32Value()));
- __ Bind(&done);
}
if (GetGraph()->IsCompilingBaseline() && !Runtime::Current()->IsAotCompiler()) {
diff --git a/compiler/optimizing/code_generator_arm_vixl.cc b/compiler/optimizing/code_generator_arm_vixl.cc
index 29c72d8..700202b 100644
--- a/compiler/optimizing/code_generator_arm_vixl.cc
+++ b/compiler/optimizing/code_generator_arm_vixl.cc
@@ -2123,12 +2123,10 @@
}
// Load with zero extend to clear the high bits for integer overflow check.
__ Ldrh(temp, MemOperand(kMethodRegister, ArtMethod::HotnessCountOffset().Int32Value()));
- vixl::aarch32::Label done;
- DCHECK_EQ(0u, interpreter::kNterpHotnessValue);
- __ CompareAndBranchIfZero(temp, &done, /* is_far_target= */ false);
- __ Add(temp, temp, -1);
+ __ Add(temp, temp, 1);
+ // Subtract one if the counter would overflow.
+ __ Sub(temp, temp, Operand(temp, ShiftType::LSR, 16));
__ Strh(temp, MemOperand(kMethodRegister, ArtMethod::HotnessCountOffset().Int32Value()));
- __ Bind(&done);
if (!is_frame_entry) {
__ Pop(vixl32::Register(kMethodRegister));
GetAssembler()->cfi().AdjustCFAOffset(-static_cast<int>(kArmWordSize));
diff --git a/compiler/optimizing/code_generator_x86.cc b/compiler/optimizing/code_generator_x86.cc
index e6e22d3..c49b08b 100644
--- a/compiler/optimizing/code_generator_x86.cc
+++ b/compiler/optimizing/code_generator_x86.cc
@@ -1109,9 +1109,10 @@
}
NearLabel overflow;
__ cmpw(Address(reg, ArtMethod::HotnessCountOffset().Int32Value()),
- Immediate(interpreter::kNterpHotnessValue));
+ Immediate(ArtMethod::MaxCounter()));
__ j(kEqual, &overflow);
- __ addw(Address(reg, ArtMethod::HotnessCountOffset().Int32Value()), Immediate(-1));
+ __ addw(Address(reg, ArtMethod::HotnessCountOffset().Int32Value()),
+ Immediate(1));
__ Bind(&overflow);
if (!is_frame_entry) {
__ popl(EAX);
diff --git a/compiler/optimizing/code_generator_x86_64.cc b/compiler/optimizing/code_generator_x86_64.cc
index de78bf2..0584dc1 100644
--- a/compiler/optimizing/code_generator_x86_64.cc
+++ b/compiler/optimizing/code_generator_x86_64.cc
@@ -1491,10 +1491,10 @@
__ movq(CpuRegister(method), Address(CpuRegister(RSP), kCurrentMethodStackOffset));
}
__ cmpw(Address(CpuRegister(method), ArtMethod::HotnessCountOffset().Int32Value()),
- Immediate(interpreter::kNterpHotnessValue));
+ Immediate(ArtMethod::MaxCounter()));
__ j(kEqual, &overflow);
__ addw(Address(CpuRegister(method), ArtMethod::HotnessCountOffset().Int32Value()),
- Immediate(-1));
+ Immediate(1));
__ Bind(&overflow);
}
diff --git a/runtime/art_method-inl.h b/runtime/art_method-inl.h
index cb1e21d..614a67f 100644
--- a/runtime/art_method-inl.h
+++ b/runtime/art_method-inl.h
@@ -421,42 +421,9 @@
return CodeItemDebugInfoAccessor(*GetDexFile(), GetCodeItem(), GetDexMethodIndex());
}
-inline bool ArtMethod::CounterHasChanged() {
+inline void ArtMethod::SetCounter(uint16_t hotness_count) {
DCHECK(!IsAbstract());
- return hotness_count_ != interpreter::kNterpHotnessMask;
-}
-
-inline void ArtMethod::ResetCounter() {
- DCHECK(!IsAbstract());
- // Avoid dirtying the value if possible.
- if (hotness_count_ != interpreter::kNterpHotnessMask) {
- hotness_count_ = interpreter::kNterpHotnessMask;
- }
-}
-
-inline void ArtMethod::SetHotCounter() {
- DCHECK(!IsAbstract());
- // Avoid dirtying the value if possible.
- if (hotness_count_ != 0) {
- hotness_count_ = 0;
- }
-}
-
-inline void ArtMethod::UpdateCounter(int32_t new_samples) {
- DCHECK(!IsAbstract());
- DCHECK_GT(new_samples, 0);
- DCHECK_LE(new_samples, std::numeric_limits<uint16_t>::max());
- uint16_t old_hotness_count = hotness_count_;
- uint16_t new_count = (old_hotness_count <= new_samples) ? 0u : old_hotness_count - new_samples;
- // Avoid dirtying the value if possible.
- if (old_hotness_count != new_count) {
- hotness_count_ = new_count;
- }
-}
-
-inline bool ArtMethod::CounterIsHot() {
- DCHECK(!IsAbstract());
- return hotness_count_ == 0;
+ hotness_count_ = hotness_count;
}
inline uint16_t ArtMethod::GetCounter() {
diff --git a/runtime/art_method.cc b/runtime/art_method.cc
index 8b748d2..2afd8df 100644
--- a/runtime/art_method.cc
+++ b/runtime/art_method.cc
@@ -753,9 +753,7 @@
SetDataPtrSize(nullptr, image_pointer_size);
}
// Clear hotness to let the JIT properly decide when to compile this method.
- if (!IsAbstract()) {
- ResetCounter();
- }
+ hotness_count_ = 0;
}
bool ArtMethod::IsImagePointerSize(PointerSize pointer_size) {
diff --git a/runtime/art_method.h b/runtime/art_method.h
index 2ba9d66..3e8bccd 100644
--- a/runtime/art_method.h
+++ b/runtime/art_method.h
@@ -32,7 +32,6 @@
#include "dex/dex_file_structs.h"
#include "dex/modifiers.h"
#include "dex/primitive.h"
-#include "interpreter/mterp/nterp.h"
#include "gc_root.h"
#include "obj_ptr.h"
#include "offsets.h"
@@ -76,7 +75,7 @@
static constexpr uint32_t kRuntimeMethodDexMethodIndex = 0xFFFFFFFF;
ArtMethod() : access_flags_(0), dex_method_index_(0),
- method_index_(0), hotness_count_(interpreter::kNterpHotnessMask) { }
+ method_index_(0), hotness_count_(0) { }
ArtMethod(ArtMethod* src, PointerSize image_pointer_size) {
CopyFrom(src, image_pointer_size);
@@ -684,12 +683,9 @@
void CopyFrom(ArtMethod* src, PointerSize image_pointer_size)
REQUIRES_SHARED(Locks::mutator_lock_);
- ALWAYS_INLINE void ResetCounter();
- ALWAYS_INLINE void UpdateCounter(int32_t new_samples);
- ALWAYS_INLINE void SetHotCounter();
- ALWAYS_INLINE bool CounterIsHot();
+ ALWAYS_INLINE void SetCounter(uint16_t hotness_count);
+
ALWAYS_INLINE uint16_t GetCounter();
- ALWAYS_INLINE bool CounterHasChanged();
ALWAYS_INLINE static constexpr uint16_t MaxCounter() {
return std::numeric_limits<decltype(hotness_count_)>::max();
diff --git a/runtime/class_linker.cc b/runtime/class_linker.cc
index 4875298..69e87d3 100644
--- a/runtime/class_linker.cc
+++ b/runtime/class_linker.cc
@@ -1956,9 +1956,6 @@
const dex::CodeItem* code_item = method.GetDexFile()->GetCodeItem(
reinterpret_cast32<uint32_t>(method.GetDataPtrSize(image_pointer_size_)));
method.SetCodeItem(code_item);
- // The hotness counter may have changed since we compiled the image, so
- // reset it with the runtime value.
- method.ResetCounter();
}
// Set image methods' entry point that point to the interpreter bridge to the
// nterp entry point.
diff --git a/runtime/interpreter/interpreter_switch_impl-inl.h b/runtime/interpreter/interpreter_switch_impl-inl.h
index 86b7962..8e16e04 100644
--- a/runtime/interpreter/interpreter_switch_impl-inl.h
+++ b/runtime/interpreter/interpreter_switch_impl-inl.h
@@ -255,7 +255,7 @@
// Hotness update.
jit::Jit* jit = Runtime::Current()->GetJit();
if (jit != nullptr) {
- jit->AddSamples(Self(), shadow_frame_.GetMethod());
+ jit->AddSamples(Self(), shadow_frame_.GetMethod(), 1, /*with_backedges=*/ true);
}
// Record new dex pc early to have consistent suspend point at loop header.
shadow_frame_.SetDexPC(next_->GetDexPc(Insns()));
diff --git a/runtime/interpreter/mterp/arm64ng/main.S b/runtime/interpreter/mterp/arm64ng/main.S
index 08a7a80..79eca04 100644
--- a/runtime/interpreter/mterp/arm64ng/main.S
+++ b/runtime/interpreter/mterp/arm64ng/main.S
@@ -301,13 +301,11 @@
2:
ldr x0, [sp]
ldrh w2, [x0, #ART_METHOD_HOTNESS_COUNT_OFFSET]
-#if (NTERP_HOTNESS_VALUE != 0)
-#error Expected 0 for hotness value
-#endif
- // If the counter is at zero, handle this in the runtime.
- cbz w2, NterpHandleHotnessOverflow
- add x2, x2, #-1
+ add x2, x2, #1
+ and w2, w2, #NTERP_HOTNESS_MASK
strh w2, [x0, #ART_METHOD_HOTNESS_COUNT_OFFSET]
+ // If the counter overflows, handle this in the runtime.
+ cbz w2, NterpHandleHotnessOverflow
// Otherwise, do a suspend check.
ldr x0, [xSELF, #THREAD_FLAGS_OFFSET]
ands x0, x0, #THREAD_SUSPEND_OR_CHECKPOINT_REQUEST
@@ -424,13 +422,11 @@
.macro START_EXECUTING_INSTRUCTIONS
ldr x0, [sp]
ldrh w2, [x0, #ART_METHOD_HOTNESS_COUNT_OFFSET]
-#if (NTERP_HOTNESS_VALUE != 0)
-#error Expected 0 for hotness value
-#endif
- // If the counter is at zero, handle this in the runtime.
- cbz w2, 2f
- add x2, x2, #-1
+ add x2, x2, #1
+ and w2, w2, #NTERP_HOTNESS_MASK
strh w2, [x0, #ART_METHOD_HOTNESS_COUNT_OFFSET]
+ // If the counter overflows, handle this in the runtime.
+ cbz w2, 2f
ldr x0, [xSELF, #THREAD_FLAGS_OFFSET]
tst x0, #THREAD_SUSPEND_OR_CHECKPOINT_REQUEST
b.ne 3f
diff --git a/runtime/interpreter/mterp/armng/main.S b/runtime/interpreter/mterp/armng/main.S
index 13d5dfe..932f6ac 100644
--- a/runtime/interpreter/mterp/armng/main.S
+++ b/runtime/interpreter/mterp/armng/main.S
@@ -312,10 +312,12 @@
2:
ldr r0, [sp]
ldrh r2, [r0, #ART_METHOD_HOTNESS_COUNT_OFFSET]
- cmp r2, #NTERP_HOTNESS_VALUE
- beq NterpHandleHotnessOverflow
- add r2, r2, #-1
+ add r2, r2, #1
+ ubfx r2, r2, #0, #NTERP_HOTNESS_BITS
strh r2, [r0, #ART_METHOD_HOTNESS_COUNT_OFFSET]
+ // If the counter overflows, handle this in the runtime.
+ cmp r2, #0
+ beq NterpHandleHotnessOverflow
// Otherwise, do a suspend check.
ldr r0, [rSELF, #THREAD_FLAGS_OFFSET]
ands r0, r0, #THREAD_SUSPEND_OR_CHECKPOINT_REQUEST
@@ -449,10 +451,12 @@
.macro START_EXECUTING_INSTRUCTIONS
ldr r0, [sp]
ldrh r2, [r0, #ART_METHOD_HOTNESS_COUNT_OFFSET]
- cmp r2, #NTERP_HOTNESS_VALUE
- beq 2f
- add r2, r2, #-1
+ add r2, r2, #1
+ ubfx r2, r2, #0, #NTERP_HOTNESS_BITS
strh r2, [r0, #ART_METHOD_HOTNESS_COUNT_OFFSET]
+ // If the counter overflows, handle this in the runtime.
+ cmp r2, #0
+ beq 2f
ldr r0, [rSELF, #THREAD_FLAGS_OFFSET]
tst r0, #THREAD_SUSPEND_OR_CHECKPOINT_REQUEST
bne 3f
diff --git a/runtime/interpreter/mterp/nterp.cc b/runtime/interpreter/mterp/nterp.cc
index 629f64e..ddef31d 100644
--- a/runtime/interpreter/mterp/nterp.cc
+++ b/runtime/interpreter/mterp/nterp.cc
@@ -89,7 +89,18 @@
// The hotness we will add to a method when we perform a
// field/method/class/string lookup.
constexpr uint16_t kNterpHotnessLookup = 0xf;
- method->UpdateCounter(kNterpHotnessLookup);
+
+ // Convert to uint32_t to handle uint16_t overflow.
+ uint32_t counter = method->GetCounter();
+ uint32_t new_counter = counter + kNterpHotnessLookup;
+ if (new_counter > kNterpHotnessMask) {
+ // Let the nterp code actually call the compilation: we want to make sure
+ // there's at least a second execution of the method or a back-edge to avoid
+ // compiling straightline initialization methods.
+ method->SetCounter(kNterpHotnessMask);
+ } else {
+ method->SetCounter(new_counter);
+ }
}
template<typename T>
@@ -724,7 +735,6 @@
extern "C" jit::OsrData* NterpHotMethod(ArtMethod* method, uint16_t* dex_pc_ptr, uint32_t* vregs)
REQUIRES_SHARED(Locks::mutator_lock_) {
ScopedAssertNoThreadSuspension sants("In nterp");
- method->ResetCounter();
jit::Jit* jit = Runtime::Current()->GetJit();
if (jit != nullptr && jit->UseJitCompilation()) {
// Nterp passes null on entry where we don't want to OSR.
@@ -738,7 +748,7 @@
return osr_data;
}
}
- jit->EnqueueCompilation(method, Thread::Current());
+ jit->EnqueueCompilationFromNterp(method, Thread::Current());
}
return nullptr;
}
diff --git a/runtime/interpreter/mterp/nterp.h b/runtime/interpreter/mterp/nterp.h
index 94e5b8a..97dee8c 100644
--- a/runtime/interpreter/mterp/nterp.h
+++ b/runtime/interpreter/mterp/nterp.h
@@ -35,7 +35,6 @@
// The hotness threshold where we trigger JIT compilation or OSR.
constexpr uint16_t kNterpHotnessMask = 0xffff;
-constexpr uint16_t kNterpHotnessValue = 0;
// The hotness threshold for the baseline compiler to trigger optimized
// compilation.
diff --git a/runtime/interpreter/mterp/x86_64ng/main.S b/runtime/interpreter/mterp/x86_64ng/main.S
index 4980007..33912d7 100644
--- a/runtime/interpreter/mterp/x86_64ng/main.S
+++ b/runtime/interpreter/mterp/x86_64ng/main.S
@@ -260,16 +260,10 @@
GOTO_NEXT
3:
movq (%rsp), %rdi
- movzwl ART_METHOD_HOTNESS_COUNT_OFFSET(%rdi), %esi
-#if (NTERP_HOTNESS_VALUE != 0)
-#error Expected 0 for hotness value
-#endif
- // If the counter is at zero, handle this in the runtime.
- testw %si, %si
- je NterpHandleHotnessOverflow
- // Update counter.
- addl $$-1, %esi
- movw %si, ART_METHOD_HOTNESS_COUNT_OFFSET(%rdi)
+ addw $$1, ART_METHOD_HOTNESS_COUNT_OFFSET(%rdi)
+ andw $$(NTERP_HOTNESS_MASK), ART_METHOD_HOTNESS_COUNT_OFFSET(%rdi)
+ // If the counter overflows, handle this in the runtime.
+ jz NterpHandleHotnessOverflow
// Otherwise, do a suspend check.
testl $$(THREAD_SUSPEND_OR_CHECKPOINT_REQUEST), rSELF:THREAD_FLAGS_OFFSET
jz 2b
@@ -826,16 +820,9 @@
// Increase method hotness and do suspend check before starting executing the method.
.macro START_EXECUTING_INSTRUCTIONS
movq (%rsp), %rdi
- movzwl ART_METHOD_HOTNESS_COUNT_OFFSET(%rdi), %esi
-#if (NTERP_HOTNESS_VALUE != 0)
-#error Expected 0 for hotness value
-#endif
- // If the counter is at zero, handle this in the runtime.
- testw %si, %si
- je 2f
- // Update counter.
- addl $$-1, %esi
- movw %si, ART_METHOD_HOTNESS_COUNT_OFFSET(%rdi)
+ addw $$1, ART_METHOD_HOTNESS_COUNT_OFFSET(%rdi)
+ andw $$(NTERP_HOTNESS_MASK), ART_METHOD_HOTNESS_COUNT_OFFSET(%rdi)
+ jz 2f
testl $$(THREAD_SUSPEND_OR_CHECKPOINT_REQUEST), rSELF:THREAD_FLAGS_OFFSET
jz 1f
EXPORT_PC
diff --git a/runtime/interpreter/mterp/x86ng/main.S b/runtime/interpreter/mterp/x86ng/main.S
index 7cc4126..7f185b0 100644
--- a/runtime/interpreter/mterp/x86ng/main.S
+++ b/runtime/interpreter/mterp/x86ng/main.S
@@ -302,16 +302,8 @@
GOTO_NEXT
3:
movl (%esp), %eax
- movzwl ART_METHOD_HOTNESS_COUNT_OFFSET(%eax), %ecx
-#if (NTERP_HOTNESS_VALUE != 0)
-#error Expected 0 for hotness value
-#endif
- // If the counter is at zero, handle this in the runtime.
- testw %cx, %cx
- je NterpHandleHotnessOverflow
- // Update counter.
- addl $$-1, %ecx
- movw %cx, ART_METHOD_HOTNESS_COUNT_OFFSET(%eax)
+ addw $$1, ART_METHOD_HOTNESS_COUNT_OFFSET(%eax)
+ andw $$(NTERP_HOTNESS_MASK), ART_METHOD_HOTNESS_COUNT_OFFSET(%eax)
// If the counter overflows, handle this in the runtime.
jz NterpHandleHotnessOverflow
// Otherwise, do a suspend check.
@@ -973,16 +965,8 @@
// Increase method hotness and do suspend check before starting executing the method.
.macro START_EXECUTING_INSTRUCTIONS
movl (%esp), %eax
- movzwl ART_METHOD_HOTNESS_COUNT_OFFSET(%eax), %ecx
-#if (NTERP_HOTNESS_VALUE != 0)
-#error Expected 0 for hotness value
-#endif
- // If the counter is at zero, handle this in the runtime.
- testw %cx, %cx
- je 2f
- // Update counter.
- addl $$-1, %ecx
- movw %cx, ART_METHOD_HOTNESS_COUNT_OFFSET(%eax)
+ addw $$1, ART_METHOD_HOTNESS_COUNT_OFFSET(%eax)
+ andw $$(NTERP_HOTNESS_MASK), ART_METHOD_HOTNESS_COUNT_OFFSET(%eax)
jz 2f
testl $$(THREAD_SUSPEND_OR_CHECKPOINT_REQUEST), rSELF:THREAD_FLAGS_OFFSET
jz 1f
diff --git a/runtime/jit/jit-inl.h b/runtime/jit/jit-inl.h
index 683fc38..e6b4095 100644
--- a/runtime/jit/jit-inl.h
+++ b/runtime/jit/jit-inl.h
@@ -27,16 +27,38 @@
namespace art {
namespace jit {
-inline void Jit::AddSamples(Thread* self, ArtMethod* method) {
- if (IgnoreSamplesForMethod(method)) {
- return;
+inline bool Jit::ShouldUsePriorityThreadWeight(Thread* self) {
+ return self->IsJitSensitiveThread() && Runtime::Current()->InJankPerceptibleProcessState();
+}
+
+inline void Jit::AddSamples(Thread* self,
+ ArtMethod* method,
+ uint16_t samples,
+ bool with_backedges) {
+ if (Jit::ShouldUsePriorityThreadWeight(self)) {
+ samples *= PriorityThreadWeight();
}
- if (method->CounterIsHot()) {
- method->ResetCounter();
- EnqueueCompilation(method, self);
- } else {
- method->UpdateCounter(1);
+ uint32_t old_count = method->GetCounter();
+ uint32_t new_count = old_count + samples;
+
+ // The full check is fairly expensive so we just add to hotness most of the time,
+ // and we do the full check only when some of the higher bits of the count change.
+ // NB: The method needs to see the transitions of the counter past the thresholds.
+ uint32_t old_batch = RoundDown(old_count, kJitSamplesBatchSize); // Clear lower bits.
+ uint32_t new_batch = RoundDown(new_count, kJitSamplesBatchSize); // Clear lower bits.
+ if (UNLIKELY(kSlowMode)) { // Check every time in slow-debug mode.
+ if (!MaybeCompileMethod(self, method, old_count, new_count, with_backedges)) {
+ // Tests may check that the counter is 0 for methods that we never compile.
+ return; // Ignore the samples for now and retry later.
+ }
+ } else if (UNLIKELY(old_batch != new_batch)) {
+ if (!MaybeCompileMethod(self, method, old_batch, new_batch, with_backedges)) {
+ // OSR compilation will ignore the samples if they don't have backedges.
+ return; // Ignore the samples for now and retry later.
+ }
}
+
+ method->SetCounter(new_count);
}
} // namespace jit
diff --git a/runtime/jit/jit.cc b/runtime/jit/jit.cc
index 846ca4d..bb421e6 100644
--- a/runtime/jit/jit.cc
+++ b/runtime/jit/jit.cc
@@ -1472,7 +1472,7 @@
}
bool Jit::IgnoreSamplesForMethod(ArtMethod* method) REQUIRES_SHARED(Locks::mutator_lock_) {
- if (method->IsClassInitializer() || !method->IsCompilable()) {
+ if (method->IsClassInitializer() || !method->IsCompilable() || method->IsPreCompiled()) {
// We do not want to compile such methods.
return true;
}
@@ -1492,6 +1492,64 @@
return false;
}
+bool Jit::MaybeCompileMethod(Thread* self,
+ ArtMethod* method,
+ uint32_t old_count,
+ uint32_t new_count,
+ bool with_backedges) {
+ if (thread_pool_ == nullptr) {
+ return false;
+ }
+ if (UNLIKELY(method->IsPreCompiled()) && !with_backedges /* don't check for OSR */) {
+ if (!NeedsClinitCheckBeforeCall(method) ||
+ method->GetDeclaringClass()->IsVisiblyInitialized()) {
+ const void* entry_point = code_cache_->GetSavedEntryPointOfPreCompiledMethod(method);
+ if (entry_point != nullptr) {
+ Runtime::Current()->GetInstrumentation()->UpdateMethodsCode(method, entry_point);
+ return true;
+ }
+ }
+ }
+
+ if (IgnoreSamplesForMethod(method)) {
+ return false;
+ }
+ if (HotMethodThreshold() == 0) {
+ // Tests might request JIT on first use (compiled synchronously in the interpreter).
+ return false;
+ }
+ DCHECK_GT(WarmMethodThreshold(), 0);
+ DCHECK_GT(HotMethodThreshold(), WarmMethodThreshold());
+ DCHECK_GT(OSRMethodThreshold(), HotMethodThreshold());
+ DCHECK_GE(PriorityThreadWeight(), 1);
+ DCHECK_LE(PriorityThreadWeight(), HotMethodThreshold());
+
+ if (UseJitCompilation()) {
+ if (old_count < HotMethodThreshold() && new_count >= HotMethodThreshold()) {
+ if (!code_cache_->ContainsPc(method->GetEntryPointFromQuickCompiledCode())) {
+ DCHECK(thread_pool_ != nullptr);
+ thread_pool_->AddTask(
+ self,
+ new JitCompileTask(
+ method, JitCompileTask::TaskKind::kCompile, CompilationKind::kBaseline));
+ }
+ }
+ if (old_count < OSRMethodThreshold() && new_count >= OSRMethodThreshold()) {
+ if (!with_backedges) {
+ return false;
+ }
+ DCHECK(!method->IsNative()); // No back edges reported for native methods.
+ if (!code_cache_->IsOsrCompiled(method)) {
+ DCHECK(thread_pool_ != nullptr);
+ thread_pool_->AddTask(
+ self,
+ new JitCompileTask(method, JitCompileTask::TaskKind::kCompile, CompilationKind::kOsr));
+ }
+ }
+ }
+ return true;
+}
+
void Jit::EnqueueOptimizedCompilation(ArtMethod* method, Thread* self) {
if (thread_pool_ == nullptr) {
return;
@@ -1540,7 +1598,7 @@
return;
}
- AddSamples(thread, method);
+ AddSamples(thread, method, 1, /* with_backedges= */false);
}
void Jit::WaitForCompilationToFinish(Thread* self) {
@@ -1734,48 +1792,19 @@
}
}
-void Jit::EnqueueCompilation(ArtMethod* method, Thread* self) {
+void Jit::EnqueueCompilationFromNterp(ArtMethod* method, Thread* self) {
if (thread_pool_ == nullptr) {
return;
}
-
- if (JitAtFirstUse()) {
- // Tests might request JIT on first use (compiled synchronously in the interpreter).
- return;
- }
-
- if (!UseJitCompilation()) {
- return;
- }
-
if (GetCodeCache()->ContainsPc(method->GetEntryPointFromQuickCompiledCode())) {
- if (!method->IsNative() && !code_cache_->IsOsrCompiled(method)) {
- // If we already have compiled code for it, nterp may be stuck in a loop.
- // Compile OSR.
- thread_pool_->AddTask(
- self,
- new JitCompileTask(method, JitCompileTask::TaskKind::kCompile, CompilationKind::kOsr));
- }
+ // If we already have compiled code for it, nterp may be stuck in a loop.
+ // Compile OSR.
+ thread_pool_->AddTask(
+ self,
+ new JitCompileTask(method, JitCompileTask::TaskKind::kCompile, CompilationKind::kOsr));
return;
}
-
- // Check if we have precompiled this method.
- if (UNLIKELY(method->IsPreCompiled())) {
- if (!NeedsClinitCheckBeforeCall(method) ||
- method->GetDeclaringClass()->IsVisiblyInitialized()) {
- const void* entry_point = code_cache_->GetSavedEntryPointOfPreCompiledMethod(method);
- if (entry_point != nullptr) {
- Runtime::Current()->GetInstrumentation()->UpdateMethodsCode(method, entry_point);
- }
- }
- return;
- }
-
- if (IgnoreSamplesForMethod(method)) {
- return;
- }
-
- if (!method->IsNative() && GetCodeCache()->CanAllocateProfilingInfo()) {
+ if (GetCodeCache()->CanAllocateProfilingInfo()) {
thread_pool_->AddTask(
self,
new JitCompileTask(method, JitCompileTask::TaskKind::kCompile, CompilationKind::kBaseline));
diff --git a/runtime/jit/jit.h b/runtime/jit/jit.h
index e06a6e0..a6e484f 100644
--- a/runtime/jit/jit.h
+++ b/runtime/jit/jit.h
@@ -314,17 +314,24 @@
void MethodEntered(Thread* thread, ArtMethod* method)
REQUIRES_SHARED(Locks::mutator_lock_);
- ALWAYS_INLINE void AddSamples(Thread* self, ArtMethod* method)
+ ALWAYS_INLINE void AddSamples(Thread* self,
+ ArtMethod* method,
+ uint16_t samples,
+ bool with_backedges)
REQUIRES_SHARED(Locks::mutator_lock_);
void NotifyInterpreterToCompiledCodeTransition(Thread* self, ArtMethod* caller)
REQUIRES_SHARED(Locks::mutator_lock_) {
- AddSamples(self, caller);
+ if (!IgnoreSamplesForMethod(caller)) {
+ AddSamples(self, caller, options_->GetInvokeTransitionWeight(), false);
+ }
}
void NotifyCompiledCodeToInterpreterTransition(Thread* self, ArtMethod* callee)
REQUIRES_SHARED(Locks::mutator_lock_) {
- AddSamples(self, callee);
+ if (!IgnoreSamplesForMethod(callee)) {
+ AddSamples(self, callee, options_->GetInvokeTransitionWeight(), false);
+ }
}
// Starts the profile saver if the config options allow profile recording.
@@ -354,6 +361,9 @@
// Return whether we can invoke JIT code for `method`.
bool CanInvokeCompiledCode(ArtMethod* method);
+ // Return whether the runtime should use a priority thread weight when sampling.
+ static bool ShouldUsePriorityThreadWeight(Thread* self);
+
// Return the information required to do an OSR jump. Return null if the OSR
// cannot be done.
OsrData* PrepareForOsr(ArtMethod* method, uint32_t dex_pc, uint32_t* vregs)
@@ -443,7 +453,7 @@
void EnqueueOptimizedCompilation(ArtMethod* method, Thread* self);
- void EnqueueCompilation(ArtMethod* method, Thread* self)
+ void EnqueueCompilationFromNterp(ArtMethod* method, Thread* self)
REQUIRES_SHARED(Locks::mutator_lock_);
private:
@@ -464,6 +474,15 @@
bool compile_after_boot)
REQUIRES_SHARED(Locks::mutator_lock_);
+ // Compile the method if the number of samples passes a threshold.
+ // Returns false if we can not compile now - don't increment the counter and retry later.
+ bool MaybeCompileMethod(Thread* self,
+ ArtMethod* method,
+ uint32_t old_count,
+ uint32_t new_count,
+ bool with_backedges)
+ REQUIRES_SHARED(Locks::mutator_lock_);
+
static bool BindCompilerMethods(std::string* error_msg);
// JIT compiler
diff --git a/runtime/jit/jit_code_cache.cc b/runtime/jit/jit_code_cache.cc
index 7545829..9d9a7d3 100644
--- a/runtime/jit/jit_code_cache.cc
+++ b/runtime/jit/jit_code_cache.cc
@@ -615,7 +615,12 @@
if (was_warm) {
method->SetPreviouslyWarm();
}
- method->ResetCounter();
+ // We reset the counter to 1 so that the profile knows that the method was executed at least once.
+ // This is required for layout purposes.
+ // We also need to make sure we'll pass the warmup threshold again, so we set to 0 if
+ // the warmup threshold is 1.
+ uint16_t jit_warmup_threshold = Runtime::Current()->GetJITOptions()->GetWarmupThreshold();
+ method->SetCounter(std::min(jit_warmup_threshold - 1, 1));
}
void JitCodeCache::WaitForPotentialCollectionToCompleteRunnable(Thread* self) {
@@ -789,7 +794,7 @@
return false;
}
- ClearMethodCounter(method, /* was_warm= */ false);
+ method->SetCounter(0);
Runtime::Current()->GetInstrumentation()->UpdateMethodsCode(
method, GetQuickToInterpreterBridge());
VLOG(jit)
@@ -1163,13 +1168,14 @@
if (!data.IsCompiled() || IsInZygoteExecSpace(data.GetCode())) {
continue;
}
+ // Make sure a single invocation of the GenericJNI trampoline tries to recompile.
+ uint16_t new_counter = Runtime::Current()->GetJit()->HotMethodThreshold() - 1u;
const OatQuickMethodHeader* method_header =
OatQuickMethodHeader::FromCodePointer(data.GetCode());
for (ArtMethod* method : data.GetMethods()) {
if (method->GetEntryPointFromQuickCompiledCode() == method_header->GetEntryPoint()) {
// Don't call Instrumentation::UpdateMethodsCode(), same as for normal methods above.
- // Make sure a single invocation of the GenericJNI trampoline tries to recompile.
- method->SetHotCounter();
+ method->SetCounter(new_counter);
method->SetEntryPointFromQuickCompiledCode(GetQuickGenericJniStub());
}
}
@@ -1294,7 +1300,6 @@
OatQuickMethodHeader* method_header =
OatQuickMethodHeader::FromEntryPoint(entry_point);
if (CodeInfo::IsBaseline(method_header->GetOptimizedCodeInfoPtr())) {
- info->GetMethod()->ResetCounter();
info->GetMethod()->SetEntryPointFromQuickCompiledCode(GetQuickToInterpreterBridge());
}
}
@@ -1484,6 +1489,7 @@
WaitUntilInlineCacheAccessible(self);
MutexLock mu(self, *Locks::jit_lock_);
ScopedTrace trace(__FUNCTION__);
+ uint16_t jit_compile_threshold = Runtime::Current()->GetJITOptions()->GetCompileThreshold();
for (auto it : profiling_infos_) {
ProfilingInfo* info = it.second;
ArtMethod* method = info->GetMethod();
@@ -1495,13 +1501,10 @@
}
std::vector<ProfileMethodInfo::ProfileInlineCache> inline_caches;
- // If the method is still baseline compiled, don't save the inline caches.
+ // If the method didn't reach the compilation threshold don't save the inline caches.
// They might be incomplete and cause unnecessary deoptimizations.
// If the inline cache is empty the compiler will generate a regular invoke virtual/interface.
- const void* entry_point = method->GetEntryPointFromQuickCompiledCode();
- if (ContainsPc(entry_point) &&
- CodeInfo::IsBaseline(
- OatQuickMethodHeader::FromEntryPoint(entry_point)->GetOptimizedCodeInfoPtr())) {
+ if (method->GetCounter() < jit_compile_threshold) {
methods.emplace_back(/*ProfileMethodInfo*/
MethodReference(dex_file, method->GetDexMethodIndex()), inline_caches);
continue;
diff --git a/runtime/jit/profile_saver.cc b/runtime/jit/profile_saver.cc
index 2974742..b86badc 100644
--- a/runtime/jit/profile_saver.cc
+++ b/runtime/jit/profile_saver.cc
@@ -626,6 +626,7 @@
ProfileCompilationInfo* profile_info) {
// Move members to local variables to allow the compiler to optimize this properly.
const bool startup = startup_;
+ const uint32_t hot_method_sample_threshold = hot_method_sample_threshold_;
const uint32_t base_flags =
(startup ? Hotness::kFlagStartup : Hotness::kFlagPostStartup) | extra_flags_;
@@ -636,10 +637,11 @@
auto get_method_flags = [&](ArtMethod& method) {
// Mark methods as hot if they have more than hot_method_sample_threshold
// samples. This means they will get compiled by the compiler driver.
- if (method.PreviouslyWarm() || method.CounterIsHot()) {
+ const uint16_t counter = method.GetCounter();
+ if (method.PreviouslyWarm() || counter >= hot_method_sample_threshold) {
++number_of_hot_methods;
return enum_cast<ProfileCompilationInfo::MethodHotness::Flag>(base_flags | Hotness::kFlagHot);
- } else if (method.CounterHasChanged()) {
+ } else if (counter != 0u) {
++number_of_sampled_methods;
return enum_cast<ProfileCompilationInfo::MethodHotness::Flag>(base_flags);
} else {
diff --git a/runtime/native/dalvik_system_VMRuntime.cc b/runtime/native/dalvik_system_VMRuntime.cc
index 2e67b1a..8425904 100644
--- a/runtime/native/dalvik_system_VMRuntime.cc
+++ b/runtime/native/dalvik_system_VMRuntime.cc
@@ -479,7 +479,9 @@
}
for (ArtMethod& m : klass->GetMethods(kRuntimePointerSize)) {
if (!m.IsAbstract()) {
- m.ResetCounter();
+ if (m.GetCounter() != 0) {
+ m.SetCounter(0);
+ }
}
}
return true;
diff --git a/test/2230-profile-save-hotness/src-art/Main.java b/test/2230-profile-save-hotness/src-art/Main.java
index c9132e6..f71a891 100644
--- a/test/2230-profile-save-hotness/src-art/Main.java
+++ b/test/2230-profile-save-hotness/src-art/Main.java
@@ -34,8 +34,6 @@
return;
}
- String methodName = "$noinline$hotnessCount";
- int initialValue = getHotnessCounter(Main.class, methodName);
File file = null;
try {
file = createTempFile();
@@ -48,22 +46,21 @@
VMRuntime.CODE_PATH_TYPE_PRIMARY_APK);
// Test that the profile saves an app method with a profiling info.
- $noinline$hotnessCountWithLoop(100000);
+ $noinline$hotnessCountWithLoop(10000);
ensureProfileProcessing();
+ String methodName = "$noinline$hotnessCount";
Method appMethod = Main.class.getDeclaredMethod(methodName);
if (!presentInProfile(file.getPath(), appMethod)) {
System.out.println("App method not hot in profile " +
getHotnessCounter(Main.class, methodName));
}
- // Hardcoded assumption that the hotness value is zero.
- if (getHotnessCounter(Main.class, methodName) != 0) {
- System.out.println("Hotness should be zero " +
+ if (getHotnessCounter(Main.class, methodName) == 0) {
+ System.out.println("Hotness should be non zero " +
getHotnessCounter(Main.class, methodName));
}
VMRuntime.resetJitCounters();
- if (getHotnessCounter(Main.class, methodName) != initialValue) {
- System.out.println(
- "Expected " + initialValue +", got " + + getHotnessCounter(Main.class, methodName));
+ if (getHotnessCounter(Main.class, methodName) != 0) {
+ System.out.println("Hotness should be zero " + getHotnessCounter(Main.class, methodName));
}
} finally {
if (file != null) {
diff --git a/test/674-hotness-compiled/src/Main.java b/test/674-hotness-compiled/src/Main.java
index a333074..5f0d10a 100644
--- a/test/674-hotness-compiled/src/Main.java
+++ b/test/674-hotness-compiled/src/Main.java
@@ -29,25 +29,23 @@
if (!isAotCompiled(Main.class, "main")) {
return;
}
- int counter = getHotnessCounter(Main.class, "$noinline$hotnessCount");
$noinline$hotnessCount();
- int newCounter = getHotnessCounter(Main.class, "$noinline$hotnessCount");
- if (counter == newCounter) {
+ int counter = getHotnessCounter(Main.class, "$noinline$hotnessCount");
+ if (counter == 0) {
throw new Error("Expected hotness counter to be updated");
}
- counter = getHotnessCounter(Main.class, "$noinline$hotnessCountWithLoop");
$noinline$hotnessCountWithLoop(1000);
- newCounter = getHotnessCounter(Main.class, "$noinline$hotnessCountWithLoop");
- if (newCounter == counter) {
- throw new Error("Expected counter " + newCounter + " to be different than " + counter);
+ int newCounter = getHotnessCounter(Main.class, "$noinline$hotnessCountWithLoop");
+ if (newCounter <= counter) {
+ throw new Error("Expected counter " + newCounter + " to be larger than " + counter);
}
counter = newCounter;
$noinline$hotnessCountWithLoop(65500);
newCounter = getHotnessCounter(Main.class, "$noinline$hotnessCountWithLoop");
- if (newCounter == counter) {
- throw new Error("Expected counter " + newCounter + " to be different than " + counter);
+ if (newCounter <= counter) {
+ throw new Error("Expected counter " + newCounter + " to be larger than " + counter);
}
}
diff --git a/test/716-jli-jit-samples/src-art/Main.java b/test/716-jli-jit-samples/src-art/Main.java
index ef63f16..def6b9f 100644
--- a/test/716-jli-jit-samples/src-art/Main.java
+++ b/test/716-jli-jit-samples/src-art/Main.java
@@ -25,7 +25,6 @@
private static final int ITERATIONS = 100;
private static final VarHandle widgetIdVarHandle;
- private static int initialHotnessCounter;
public static native int getHotnessCounter(Class<?> cls, String methodName);
@@ -85,8 +84,8 @@
Widget.class, MethodType.methodType(void.class, int.class));
Widget w = (Widget) mh.invoke(3);
w = (Widget) mh.invokeExact(3);
- assertEquals(initialHotnessCounter, getHotnessCounter(MethodHandle.class, "invoke"));
- assertEquals(initialHotnessCounter, getHotnessCounter(MethodHandle.class, "invokeExact"));
+ assertEquals(0, getHotnessCounter(MethodHandle.class, "invoke"));
+ assertEquals(0, getHotnessCounter(MethodHandle.class, "invokeExact"));
// Reflective MethodHandle invocations
String[] methodNames = {"invoke", "invokeExact"};
@@ -103,10 +102,8 @@
assertEquals(ite.getCause().getClass(), UnsupportedOperationException.class);
}
}
- assertEquals(initialHotnessCounter,
- getHotnessCounter(MethodHandle.class, "invoke"));
- assertEquals(initialHotnessCounter,
- getHotnessCounter(MethodHandle.class, "invokeExact"));
+ assertEquals(0, getHotnessCounter(MethodHandle.class, "invoke"));
+ assertEquals(0, getHotnessCounter(MethodHandle.class, "invokeExact"));
}
System.out.println("MethodHandle OK");
@@ -118,8 +115,8 @@
// Regular accessor invocations
widgetIdVarHandle.set(w, i);
assertEquals(i, widgetIdVarHandle.get(w));
- assertEquals(initialHotnessCounter, getHotnessCounter(VarHandle.class, "set"));
- assertEquals(initialHotnessCounter, getHotnessCounter(VarHandle.class, "get"));
+ assertEquals(0, getHotnessCounter(VarHandle.class, "set"));
+ assertEquals(0, getHotnessCounter(VarHandle.class, "get"));
// Reflective accessor invocations
for (String accessorName : new String[] {"get", "set"}) {
@@ -131,15 +128,14 @@
assertEquals(ite.getCause().getClass(), UnsupportedOperationException.class);
}
}
- assertEquals(initialHotnessCounter, getHotnessCounter(VarHandle.class, "set"));
- assertEquals(initialHotnessCounter, getHotnessCounter(VarHandle.class, "get"));
+ assertEquals(0, getHotnessCounter(VarHandle.class, "set"));
+ assertEquals(0, getHotnessCounter(VarHandle.class, "get"));
}
System.out.println("VarHandle OK");
}
public static void main(String[] args) throws Throwable {
System.loadLibrary(args[0]);
- initialHotnessCounter = getHotnessCounter(VarHandle.class, "set");
testMethodHandleCounters();
testVarHandleCounters();
}
diff --git a/tools/cpp-define-generator/globals.def b/tools/cpp-define-generator/globals.def
index 2572ea6..ae8f5aa 100644
--- a/tools/cpp-define-generator/globals.def
+++ b/tools/cpp-define-generator/globals.def
@@ -64,8 +64,6 @@
art::interpreter::kNterpHandlerSize)
ASM_DEFINE(NTERP_HANDLER_SIZE_LOG2,
art::WhichPowerOf2(art::interpreter::kNterpHandlerSize))
-ASM_DEFINE(NTERP_HOTNESS_VALUE,
- art::interpreter::kNterpHotnessValue)
ASM_DEFINE(OBJECT_ALIGNMENT_MASK,
art::kObjectAlignment - 1)
ASM_DEFINE(OBJECT_ALIGNMENT_MASK_TOGGLED,
@@ -82,3 +80,7 @@
std::memory_order_relaxed)
ASM_DEFINE(STACK_OVERFLOW_RESERVED_BYTES,
GetStackOverflowReservedBytes(art::kRuntimeISA))
+ASM_DEFINE(NTERP_HOTNESS_MASK,
+ art::interpreter::kNterpHotnessMask)
+ASM_DEFINE(NTERP_HOTNESS_BITS,
+ art::POPCOUNT(art::interpreter::kNterpHotnessMask))