Introduce a nterp with clinit check entrypoint.
It will be used post-zygote fork, so that methods whose class is not
initialized yet keep the same entrypoint even after initialization. This
avoids dirtying shared pages.
Test: imgdiag
Bug: 162110941
Change-Id: Idb861f23b368604e79f6d3efa7ad0dff287e0209
diff --git a/runtime/art_method.cc b/runtime/art_method.cc
index 2b7c238..cef9d7b 100644
--- a/runtime/art_method.cc
+++ b/runtime/art_method.cc
@@ -734,16 +734,16 @@
// the entry point to the JIT code, but this would require taking the JIT code cache
// lock to notify it, which we do not want at this level.
Runtime* runtime = Runtime::Current();
+ const void* entry_point = GetEntryPointFromQuickCompiledCodePtrSize(image_pointer_size);
if (runtime->UseJitCompilation()) {
- if (runtime->GetJit()->GetCodeCache()->ContainsPc(GetEntryPointFromQuickCompiledCode())) {
+ if (runtime->GetJit()->GetCodeCache()->ContainsPc(entry_point)) {
SetEntryPointFromQuickCompiledCodePtrSize(
src->IsNative() ? GetQuickGenericJniStub() : GetQuickToInterpreterBridge(),
image_pointer_size);
}
}
- if (interpreter::IsNterpSupported() &&
- (GetEntryPointFromQuickCompiledCodePtrSize(image_pointer_size) ==
- interpreter::GetNterpEntryPoint())) {
+ ClassLinker* class_linker = Runtime::Current()->GetClassLinker();
+ if (interpreter::IsNterpSupported() && class_linker->IsNterpEntryPoint(entry_point)) {
// If the entrypoint is nterp, it's too early to check if the new method
// will support it. So for simplicity, use the interpreter bridge.
SetEntryPointFromQuickCompiledCodePtrSize(GetQuickToInterpreterBridge(), image_pointer_size);
diff --git a/runtime/class_linker.cc b/runtime/class_linker.cc
index c8dbc75..d21aecc 100644
--- a/runtime/class_linker.cc
+++ b/runtime/class_linker.cc
@@ -3389,11 +3389,9 @@
}
instrumentation::Instrumentation* instrumentation = runtime->GetInstrumentation();
- // Link the code of methods skipped by LinkCode.
for (size_t method_index = 0; method_index < num_direct_methods; ++method_index) {
ArtMethod* method = klass->GetDirectMethod(method_index, pointer_size);
- if (!method->IsStatic()) {
- // Only update static methods.
+ if (!NeedsClinitCheckBeforeCall(method)) {
continue;
}
instrumentation->UpdateMethodsCode(method, instrumentation->GetCodeForInvoke(method));
diff --git a/runtime/class_linker.h b/runtime/class_linker.h
index 3b20c75b..895d820 100644
--- a/runtime/class_linker.h
+++ b/runtime/class_linker.h
@@ -36,6 +36,7 @@
#include "dex/dex_file_types.h"
#include "gc_root.h"
#include "handle.h"
+#include "interpreter/mterp/nterp.h"
#include "jni.h"
#include "mirror/class.h"
#include "mirror/object.h"
@@ -608,6 +609,11 @@
return nterp_trampoline_ == entry_point;
}
+ bool IsNterpEntryPoint(const void* entry_point) const {
+ return entry_point == interpreter::GetNterpEntryPoint() ||
+ entry_point == interpreter::GetNterpWithClinitEntryPoint();
+ }
+
const void* GetQuickToInterpreterBridgeTrampoline() const {
return quick_to_interpreter_bridge_trampoline_;
}
diff --git a/runtime/instrumentation.cc b/runtime/instrumentation.cc
index 39e6aff..61b0e52 100644
--- a/runtime/instrumentation.cc
+++ b/runtime/instrumentation.cc
@@ -189,6 +189,7 @@
return class_linker->IsQuickResolutionStub(code) ||
class_linker->IsQuickToInterpreterBridge(code) ||
class_linker->IsQuickGenericJniStub(code) ||
+ (code == interpreter::GetNterpWithClinitEntryPoint()) ||
(code == GetQuickInstrumentationEntryPoint());
}
@@ -382,7 +383,12 @@
// stub only if we have compiled code or we can execute nterp, and the method needs a class
// initialization check.
if (aot_code != nullptr || method->IsNative() || CanUseNterp(method)) {
- UpdateEntryPoints(method, GetQuickResolutionStub());
+ if (kIsDebugBuild && CanUseNterp(method)) {
+ // Adds some test coverage for the nterp clinit entrypoint.
+ UpdateEntryPoints(method, interpreter::GetNterpWithClinitEntryPoint());
+ } else {
+ UpdateEntryPoints(method, GetQuickResolutionStub());
+ }
} else {
UpdateEntryPoints(method, GetQuickToInterpreterBridge());
}
@@ -1089,6 +1095,8 @@
return "obsolete";
} else if (code == interpreter::GetNterpEntryPoint()) {
return "nterp";
+ } else if (code == interpreter::GetNterpWithClinitEntryPoint()) {
+ return "nterp with clinit";
} else if (class_linker->IsQuickGenericJniStub(code)) {
return "generic jni";
} else if (Runtime::Current()->GetOatFileManager().ContainsPc(code)) {
@@ -1301,10 +1309,14 @@
DCHECK(!method->IsProxyMethod()) << method->PrettyMethod();
ClassLinker* class_linker = Runtime::Current()->GetClassLinker();
const void* code = method->GetEntryPointFromQuickCompiledCodePtrSize(kRuntimePointerSize);
- // If we don't have the instrumentation, the resolution stub, or the
- // interpreter as entrypoint, just return the current entrypoint, assuming
- // it's the most optimized.
+ // If we don't have the instrumentation, the resolution stub, the
+ // interpreter, or the nterp with clinit as entrypoint, just return the current entrypoint,
+ // assuming it's the most optimized.
+ // We don't want to return the nterp with clinit entrypoint as it calls the
+ // resolution stub, and the resolution stub will call `GetCodeForInvoke` to know the actual
+ // code to invoke.
if (code != GetQuickInstrumentationEntryPoint() &&
+ code != interpreter::GetNterpWithClinitEntryPoint() &&
!class_linker->IsQuickResolutionStub(code) &&
!class_linker->IsQuickToInterpreterBridge(code)) {
return code;
diff --git a/runtime/interpreter/mterp/arm64ng/main.S b/runtime/interpreter/mterp/arm64ng/main.S
index 89de81f..81d6b7b 100644
--- a/runtime/interpreter/mterp/arm64ng/main.S
+++ b/runtime/interpreter/mterp/arm64ng/main.S
@@ -1590,6 +1590,14 @@
* rest method parameters
*/
+OAT_ENTRY ExecuteNterpWithClinitImpl, EndExecuteNterpWithClinitImpl
+ ldr wip, [x0, ART_METHOD_DECLARING_CLASS_OFFSET]
+ ldrb wip, [ip, MIRROR_CLASS_IS_VISIBLY_INITIALIZED_OFFSET]
+ cmp ip, #MIRROR_CLASS_IS_VISIBLY_INITIALIZED_VALUE
+ bcs ExecuteNterpImpl
+ b art_quick_resolution_trampoline
+EndExecuteNterpWithClinitImpl:
+
OAT_ENTRY ExecuteNterpImpl, EndExecuteNterpImpl
.cfi_startproc
sub x16, sp, #STACK_OVERFLOW_RESERVED_BYTES
diff --git a/runtime/interpreter/mterp/armng/main.S b/runtime/interpreter/mterp/armng/main.S
index 310a3fd..f89db40 100644
--- a/runtime/interpreter/mterp/armng/main.S
+++ b/runtime/interpreter/mterp/armng/main.S
@@ -1608,6 +1608,14 @@
* rest method parameters
*/
+OAT_ENTRY ExecuteNterpWithClinitImpl, EndExecuteNterpWithClinitImpl
+ ldr ip, [r0, ART_METHOD_DECLARING_CLASS_OFFSET]
+ ldrb ip, [ip, MIRROR_CLASS_IS_VISIBLY_INITIALIZED_OFFSET]
+ cmp ip, #MIRROR_CLASS_IS_VISIBLY_INITIALIZED_VALUE
+ bcs ExecuteNterpImpl
+ b art_quick_resolution_trampoline
+EndExecuteNterpWithClinitImpl:
+
OAT_ENTRY ExecuteNterpImpl, EndExecuteNterpImpl
.cfi_startproc
sub ip, sp, #STACK_OVERFLOW_RESERVED_BYTES
diff --git a/runtime/interpreter/mterp/nterp.cc b/runtime/interpreter/mterp/nterp.cc
index d70a846..17f17af 100644
--- a/runtime/interpreter/mterp/nterp.cc
+++ b/runtime/interpreter/mterp/nterp.cc
@@ -58,10 +58,17 @@
// The entrypoint for nterp, which ArtMethods can directly point to.
extern "C" void ExecuteNterpImpl() REQUIRES_SHARED(Locks::mutator_lock_);
+// Another entrypoint, which does a clinit check at entry.
+extern "C" void ExecuteNterpWithClinitImpl() REQUIRES_SHARED(Locks::mutator_lock_);
+
const void* GetNterpEntryPoint() {
return reinterpret_cast<const void*>(interpreter::ExecuteNterpImpl);
}
+const void* GetNterpWithClinitEntryPoint() {
+ return reinterpret_cast<const void*>(interpreter::ExecuteNterpWithClinitImpl);
+}
+
/*
* Verify some constants used by the nterp interpreter.
*/
diff --git a/runtime/interpreter/mterp/nterp.h b/runtime/interpreter/mterp/nterp.h
index 1590b28..4d5af39 100644
--- a/runtime/interpreter/mterp/nterp.h
+++ b/runtime/interpreter/mterp/nterp.h
@@ -32,6 +32,7 @@
bool IsNterpSupported();
bool CanRuntimeUseNterp();
const void* GetNterpEntryPoint();
+const void* GetNterpWithClinitEntryPoint();
constexpr uint16_t kNterpHotnessValue = 0;
diff --git a/runtime/interpreter/mterp/x86_64ng/main.S b/runtime/interpreter/mterp/x86_64ng/main.S
index bd191c0..3e476db 100644
--- a/runtime/interpreter/mterp/x86_64ng/main.S
+++ b/runtime/interpreter/mterp/x86_64ng/main.S
@@ -1694,6 +1694,13 @@
* rest method parameters
*/
+OAT_ENTRY ExecuteNterpWithClinitImpl, EndExecuteNterpWithClinitImpl
+ movl ART_METHOD_DECLARING_CLASS_OFFSET(%rdi), %r10d
+ cmpb $$(MIRROR_CLASS_IS_VISIBLY_INITIALIZED_VALUE), MIRROR_CLASS_IS_VISIBLY_INITIALIZED_OFFSET(%r10d)
+ jae ExecuteNterpImpl
+ jmp art_quick_resolution_trampoline
+EndExecuteNterpWithClinitImpl:
+
OAT_ENTRY ExecuteNterpImpl, EndExecuteNterpImpl
.cfi_startproc
.cfi_def_cfa rsp, 8
diff --git a/runtime/interpreter/mterp/x86ng/main.S b/runtime/interpreter/mterp/x86ng/main.S
index db8519b..7872520 100644
--- a/runtime/interpreter/mterp/x86ng/main.S
+++ b/runtime/interpreter/mterp/x86ng/main.S
@@ -1757,6 +1757,15 @@
* rest method parameters
*/
+OAT_ENTRY ExecuteNterpWithClinitImpl, EndExecuteNterpWithClinitImpl
+ push %esi
+ movl ART_METHOD_DECLARING_CLASS_OFFSET(%eax), %esi
+ cmpb $$(MIRROR_CLASS_IS_VISIBLY_INITIALIZED_VALUE), MIRROR_CLASS_IS_VISIBLY_INITIALIZED_OFFSET(%esi)
+ pop %esi
+ jae ExecuteNterpImpl
+ jmp art_quick_resolution_trampoline
+EndExecuteNterpWithClinitImpl:
+
OAT_ENTRY ExecuteNterpImpl, EndExecuteNterpImpl
.cfi_startproc
.cfi_def_cfa esp, 4
diff --git a/runtime/jit/jit.cc b/runtime/jit/jit.cc
index 2deed43..526cba9 100644
--- a/runtime/jit/jit.cc
+++ b/runtime/jit/jit.cc
@@ -1366,9 +1366,10 @@
const void* entry_point = method->GetEntryPointFromQuickCompiledCode();
if (class_linker->IsQuickToInterpreterBridge(entry_point) ||
class_linker->IsQuickGenericJniStub(entry_point) ||
- (entry_point == interpreter::GetNterpEntryPoint()) ||
- // We explicitly check for the stub. The trampoline is for methods backed by
- // a .oat file that has a compiled version of the method.
+ class_linker->IsNterpEntryPoint(entry_point) ||
+ // We explicitly check for the resolution stub, and not the resolution trampoline.
+ // The trampoline is for methods backed by a .oat file that has a compiled version of
+ // the method.
(entry_point == GetQuickResolutionStub())) {
VLOG(jit) << "JIT Zygote processing method " << ArtMethod::PrettyMethod(method)
<< " from profile";
diff --git a/runtime/runtime.cc b/runtime/runtime.cc
index d3ca77d..72fa118 100644
--- a/runtime/runtime.cc
+++ b/runtime/runtime.cc
@@ -152,6 +152,7 @@
#include "native_bridge_art_interface.h"
#include "native_stack_dump.h"
#include "nativehelper/scoped_local_ref.h"
+#include "nterp_helpers.h"
#include "oat.h"
#include "oat_file_manager.h"
#include "oat_quick_method_header.h"
@@ -698,38 +699,52 @@
// notreached
}
-class FindNativeMethodsVisitor : public ClassVisitor {
+/**
+ * Update entrypoints (native and Java) of methods before the first fork. This
+ * helps sharing pages where ArtMethods are allocated between the zygote and
+ * forked apps.
+ */
+class UpdateMethodsPreFirstForkVisitor : public ClassVisitor {
public:
- FindNativeMethodsVisitor(Thread* self, ClassLinker* class_linker)
+ UpdateMethodsPreFirstForkVisitor(Thread* self, ClassLinker* class_linker)
: vm_(down_cast<JNIEnvExt*>(self->GetJniEnv())->GetVm()),
self_(self),
- class_linker_(class_linker) {}
+ class_linker_(class_linker),
+ can_use_nterp_(interpreter::CanRuntimeUseNterp()) {}
bool operator()(ObjPtr<mirror::Class> klass) override REQUIRES_SHARED(Locks::mutator_lock_) {
bool is_initialized = klass->IsVisiblyInitialized();
for (ArtMethod& method : klass->GetDeclaredMethods(kRuntimePointerSize)) {
- if (method.IsNative() && (is_initialized || !NeedsClinitCheckBeforeCall(&method))) {
- const void* existing = method.GetEntryPointFromJni();
- if (method.IsCriticalNative()
- ? class_linker_->IsJniDlsymLookupCriticalStub(existing)
- : class_linker_->IsJniDlsymLookupStub(existing)) {
- const void* native_code =
- vm_->FindCodeForNativeMethod(&method, /*error_msg=*/ nullptr, /*can_suspend=*/ false);
- if (native_code != nullptr) {
- class_linker_->RegisterNative(self_, &method, native_code);
+ if (is_initialized || !NeedsClinitCheckBeforeCall(&method)) {
+ if (method.IsNative()) {
+ const void* existing = method.GetEntryPointFromJni();
+ if (method.IsCriticalNative()
+ ? class_linker_->IsJniDlsymLookupCriticalStub(existing)
+ : class_linker_->IsJniDlsymLookupStub(existing)) {
+ const void* native_code =
+ vm_->FindCodeForNativeMethod(&method, /*error_msg=*/ nullptr, /*can_suspend=*/ false);
+ if (native_code != nullptr) {
+ class_linker_->RegisterNative(self_, &method, native_code);
+ }
}
}
+ } else if (can_use_nterp_) {
+ const void* existing = method.GetEntryPointFromQuickCompiledCode();
+ if (class_linker_->IsQuickResolutionStub(existing) && CanMethodUseNterp(&method)) {
+ method.SetEntryPointFromQuickCompiledCode(interpreter::GetNterpWithClinitEntryPoint());
+ }
}
}
return true;
}
private:
- JavaVMExt* vm_;
- Thread* self_;
- ClassLinker* class_linker_;
+ JavaVMExt* const vm_;
+ Thread* const self_;
+ ClassLinker* const class_linker_;
+ const bool can_use_nterp_;
- DISALLOW_COPY_AND_ASSIGN(FindNativeMethodsVisitor);
+ DISALLOW_COPY_AND_ASSIGN(UpdateMethodsPreFirstForkVisitor);
};
void Runtime::PreZygoteFork() {
@@ -743,8 +758,7 @@
// Ensure we call FixupStaticTrampolines on all methods that are
// initialized.
class_linker_->MakeInitializedClassesVisiblyInitialized(soa.Self(), /*wait=*/ true);
- // Update native method JNI entrypoints.
- FindNativeMethodsVisitor visitor(soa.Self(), class_linker_);
+ UpdateMethodsPreFirstForkVisitor visitor(soa.Self(), class_linker_);
class_linker_->VisitClasses(&visitor);
}
heap_->PreZygoteFork();
@@ -3428,6 +3442,8 @@
}
}
+// Return whether a boot image has a profile. This means we'll need to pre-JIT
+// methods in that profile for performance.
bool Runtime::HasImageWithProfile() const {
for (gc::space::ImageSpace* space : GetHeap()->GetBootImageSpaces()) {
if (!space->GetProfileFiles().empty()) {
diff --git a/tools/cpp-define-generator/mirror_class.def b/tools/cpp-define-generator/mirror_class.def
index 8cfd54e..c01aab3 100644
--- a/tools/cpp-define-generator/mirror_class.def
+++ b/tools/cpp-define-generator/mirror_class.def
@@ -16,6 +16,7 @@
#if ASM_DEFINE_INCLUDE_DEPENDENCIES
#include "mirror/class.h"
+#include "subtype_check.h"
#endif
ASM_DEFINE(MIRROR_CLASS_ACCESS_FLAGS_OFFSET,
@@ -49,3 +50,9 @@
ASM_DEFINE(MIRROR_CLASS_IS_INTERFACE_FLAG, art::kAccInterface)
ASM_DEFINE(MIRROR_CLASS_IS_INTERFACE_FLAG_BIT,
art::WhichPowerOf2(art::kAccInterface))
+ASM_DEFINE(MIRROR_CLASS_IS_VISIBLY_INITIALIZED_OFFSET,
+ art::mirror::Class::StatusOffset().SizeValue() +
+ (art::SubtypeCheckBits::BitStructSizeOf() / art::kBitsPerByte))
+ASM_DEFINE(MIRROR_CLASS_IS_VISIBLY_INITIALIZED_VALUE,
+ art::enum_cast<uint32_t>(art::ClassStatus::kVisiblyInitialized) <<
+ (art::SubtypeCheckBits::BitStructSizeOf() % art::kBitsPerByte))