JNI: Rewrite read barrier slow path.
Preserve all argument registers in the slow path to prepare
for moving arguments in registers for @FastNative. Move the
read barrier check earlier as it logically belongs to the
transition frame creation. For Baker read barriers, add a
mark bit check with fast return to the main path.
Test: m test-art-host-gtest
Test: testrunner.py --host --optimizing
Test: run-gtests.sh
Test: testrunner.py --target --optimizing
Bug: 172332525
Change-Id: I50bbc0bc9d54577281e7667aafebb4a53a539af1
diff --git a/compiler/utils/arm/jni_macro_assembler_arm_vixl.cc b/compiler/utils/arm/jni_macro_assembler_arm_vixl.cc
index e81e378..2857ff4 100644
--- a/compiler/utils/arm/jni_macro_assembler_arm_vixl.cc
+++ b/compiler/utils/arm/jni_macro_assembler_arm_vixl.cc
@@ -20,6 +20,7 @@
#include <type_traits>
#include "entrypoints/quick/quick_entrypoints.h"
+#include "lock_word.h"
#include "thread.h"
using namespace vixl::aarch32; // NOLINT(build/namespaces)
@@ -1065,6 +1066,29 @@
}
}
+void ArmVIXLJNIMacroAssembler::TestMarkBit(ManagedRegister mref,
+ JNIMacroLabel* label,
+ JNIMacroUnaryCondition cond) {
+ DCHECK(kUseBakerReadBarrier);
+ vixl32::Register ref = AsVIXLRegister(mref.AsArm());
+ UseScratchRegisterScope temps(asm_.GetVIXLAssembler());
+ vixl32::Register scratch = temps.Acquire();
+ ___ Ldr(scratch, MemOperand(ref, mirror::Object::MonitorOffset().SizeValue()));
+ static_assert(LockWord::kMarkBitStateSize == 1u);
+ ___ Tst(scratch, LockWord::kMarkBitStateMaskShifted);
+ switch (cond) {
+ case JNIMacroUnaryCondition::kZero:
+ ___ B(eq, ArmVIXLJNIMacroLabel::Cast(label)->AsArm());
+ break;
+ case JNIMacroUnaryCondition::kNotZero:
+ ___ B(ne, ArmVIXLJNIMacroLabel::Cast(label)->AsArm());
+ break;
+ default:
+ LOG(FATAL) << "Not implemented unary condition: " << static_cast<int>(cond);
+ UNREACHABLE();
+ }
+}
+
void ArmVIXLJNIMacroAssembler::Bind(JNIMacroLabel* label) {
CHECK(label != nullptr);
___ Bind(ArmVIXLJNIMacroLabel::Cast(label)->AsArm());
diff --git a/compiler/utils/arm/jni_macro_assembler_arm_vixl.h b/compiler/utils/arm/jni_macro_assembler_arm_vixl.h
index 07ace97..f4dc1d1 100644
--- a/compiler/utils/arm/jni_macro_assembler_arm_vixl.h
+++ b/compiler/utils/arm/jni_macro_assembler_arm_vixl.h
@@ -197,6 +197,8 @@
void Jump(JNIMacroLabel* label) override;
// Emit a conditional jump to the label by applying a unary condition test to the GC marking flag.
void TestGcMarking(JNIMacroLabel* label, JNIMacroUnaryCondition cond) override;
+ // Emit a conditional jump to the label by applying a unary condition test to object's mark bit.
+ void TestMarkBit(ManagedRegister ref, JNIMacroLabel* label, JNIMacroUnaryCondition cond) override;
// Code at this offset will serve as the target for the Jump call.
void Bind(JNIMacroLabel* label) override;
diff --git a/compiler/utils/arm64/jni_macro_assembler_arm64.cc b/compiler/utils/arm64/jni_macro_assembler_arm64.cc
index f7144d0..57e0823 100644
--- a/compiler/utils/arm64/jni_macro_assembler_arm64.cc
+++ b/compiler/utils/arm64/jni_macro_assembler_arm64.cc
@@ -17,6 +17,7 @@
#include "jni_macro_assembler_arm64.h"
#include "entrypoints/quick/quick_entrypoints.h"
+#include "lock_word.h"
#include "managed_register_arm64.h"
#include "offsets.h"
#include "thread.h"
@@ -804,6 +805,28 @@
}
}
+void Arm64JNIMacroAssembler::TestMarkBit(ManagedRegister m_ref,
+ JNIMacroLabel* label,
+ JNIMacroUnaryCondition cond) {
+ DCHECK(kUseBakerReadBarrier);
+ Register ref = reg_x(m_ref.AsArm64().AsOverlappingXRegister());
+ UseScratchRegisterScope temps(asm_.GetVIXLAssembler());
+ Register scratch = temps.AcquireW();
+ ___ Ldr(scratch, MEM_OP(ref, mirror::Object::MonitorOffset().SizeValue()));
+ static_assert(LockWord::kMarkBitStateSize == 1u);
+ switch (cond) {
+ case JNIMacroUnaryCondition::kZero:
+ ___ Tbz(scratch, LockWord::kMarkBitStateShift, Arm64JNIMacroLabel::Cast(label)->AsArm64());
+ break;
+ case JNIMacroUnaryCondition::kNotZero:
+ ___ Tbnz(scratch, LockWord::kMarkBitStateShift, Arm64JNIMacroLabel::Cast(label)->AsArm64());
+ break;
+ default:
+ LOG(FATAL) << "Not implemented unary condition: " << static_cast<int>(cond);
+ UNREACHABLE();
+ }
+}
+
void Arm64JNIMacroAssembler::Bind(JNIMacroLabel* label) {
CHECK(label != nullptr);
___ Bind(Arm64JNIMacroLabel::Cast(label)->AsArm64());
diff --git a/compiler/utils/arm64/jni_macro_assembler_arm64.h b/compiler/utils/arm64/jni_macro_assembler_arm64.h
index 5d6a0e4..32f1ea9 100644
--- a/compiler/utils/arm64/jni_macro_assembler_arm64.h
+++ b/compiler/utils/arm64/jni_macro_assembler_arm64.h
@@ -181,6 +181,8 @@
void Jump(JNIMacroLabel* label) override;
// Emit a conditional jump to the label by applying a unary condition test to the GC marking flag.
void TestGcMarking(JNIMacroLabel* label, JNIMacroUnaryCondition cond) override;
+ // Emit a conditional jump to the label by applying a unary condition test to object's mark bit.
+ void TestMarkBit(ManagedRegister ref, JNIMacroLabel* label, JNIMacroUnaryCondition cond) override;
// Code at this offset will serve as the target for the Jump call.
void Bind(JNIMacroLabel* label) override;
diff --git a/compiler/utils/jni_macro_assembler.h b/compiler/utils/jni_macro_assembler.h
index 0ccf4cd..7f5dc2f 100644
--- a/compiler/utils/jni_macro_assembler.h
+++ b/compiler/utils/jni_macro_assembler.h
@@ -257,6 +257,10 @@
virtual void Jump(JNIMacroLabel* label) = 0;
// Emit a conditional jump to the label by applying a unary condition test to the GC marking flag.
virtual void TestGcMarking(JNIMacroLabel* label, JNIMacroUnaryCondition cond) = 0;
+ // Emit a conditional jump to the label by applying a unary condition test to object's mark bit.
+ virtual void TestMarkBit(ManagedRegister ref,
+ JNIMacroLabel* label,
+ JNIMacroUnaryCondition cond) = 0;
// Code at this offset will serve as the target for the Jump call.
virtual void Bind(JNIMacroLabel* label) = 0;
diff --git a/compiler/utils/x86/jni_macro_assembler_x86.cc b/compiler/utils/x86/jni_macro_assembler_x86.cc
index f805556..2e7f23d 100644
--- a/compiler/utils/x86/jni_macro_assembler_x86.cc
+++ b/compiler/utils/x86/jni_macro_assembler_x86.cc
@@ -18,6 +18,7 @@
#include "base/casts.h"
#include "entrypoints/quick/quick_entrypoints.h"
+#include "lock_word.h"
#include "thread.h"
#include "utils/assembler.h"
@@ -590,27 +591,37 @@
__ jmp(X86JNIMacroLabel::Cast(label)->AsX86());
}
-void X86JNIMacroAssembler::TestGcMarking(JNIMacroLabel* label, JNIMacroUnaryCondition cond) {
- CHECK(label != nullptr);
-
- art::x86::Condition x86_cond;
+static Condition UnaryConditionToX86Condition(JNIMacroUnaryCondition cond) {
switch (cond) {
case JNIMacroUnaryCondition::kZero:
- x86_cond = art::x86::kZero;
- break;
+ return kZero;
case JNIMacroUnaryCondition::kNotZero:
- x86_cond = art::x86::kNotZero;
- break;
+ return kNotZero;
default:
LOG(FATAL) << "Not implemented condition: " << static_cast<int>(cond);
UNREACHABLE();
}
+}
+
+void X86JNIMacroAssembler::TestGcMarking(JNIMacroLabel* label, JNIMacroUnaryCondition cond) {
+ CHECK(label != nullptr);
// CMP self->tls32_.is_gc_marking, 0
// Jcc <Offset>
DCHECK_EQ(Thread::IsGcMarkingSize(), 4u);
__ fs()->cmpl(Address::Absolute(Thread::IsGcMarkingOffset<kX86PointerSize>()), Immediate(0));
- __ j(x86_cond, X86JNIMacroLabel::Cast(label)->AsX86());
+ __ j(UnaryConditionToX86Condition(cond), X86JNIMacroLabel::Cast(label)->AsX86());
+}
+
+void X86JNIMacroAssembler::TestMarkBit(ManagedRegister mref,
+ JNIMacroLabel* label,
+ JNIMacroUnaryCondition cond) {
+ DCHECK(kUseBakerReadBarrier);
+ Register ref = mref.AsX86().AsCpuRegister();
+ static_assert(LockWord::kMarkBitStateSize == 1u);
+ __ testl(Address(ref, mirror::Object::MonitorOffset().SizeValue()),
+ Immediate(LockWord::kMarkBitStateMaskShifted));
+ __ j(UnaryConditionToX86Condition(cond), X86JNIMacroLabel::Cast(label)->AsX86());
}
void X86JNIMacroAssembler::Bind(JNIMacroLabel* label) {
diff --git a/compiler/utils/x86/jni_macro_assembler_x86.h b/compiler/utils/x86/jni_macro_assembler_x86.h
index 486cd7e..68822f8 100644
--- a/compiler/utils/x86/jni_macro_assembler_x86.h
+++ b/compiler/utils/x86/jni_macro_assembler_x86.h
@@ -173,6 +173,8 @@
void Jump(JNIMacroLabel* label) override;
// Emit a conditional jump to the label by applying a unary condition test to the GC marking flag.
void TestGcMarking(JNIMacroLabel* label, JNIMacroUnaryCondition cond) override;
+ // Emit a conditional jump to the label by applying a unary condition test to object's mark bit.
+ void TestMarkBit(ManagedRegister ref, JNIMacroLabel* label, JNIMacroUnaryCondition cond) override;
// Code at this offset will serve as the target for the Jump call.
void Bind(JNIMacroLabel* label) override;
diff --git a/compiler/utils/x86_64/jni_macro_assembler_x86_64.cc b/compiler/utils/x86_64/jni_macro_assembler_x86_64.cc
index fcc517e..afed413 100644
--- a/compiler/utils/x86_64/jni_macro_assembler_x86_64.cc
+++ b/compiler/utils/x86_64/jni_macro_assembler_x86_64.cc
@@ -19,6 +19,7 @@
#include "base/casts.h"
#include "base/memory_region.h"
#include "entrypoints/quick/quick_entrypoints.h"
+#include "lock_word.h"
#include "thread.h"
namespace art {
@@ -673,28 +674,38 @@
__ jmp(X86_64JNIMacroLabel::Cast(label)->AsX86_64());
}
-void X86_64JNIMacroAssembler::TestGcMarking(JNIMacroLabel* label, JNIMacroUnaryCondition cond) {
- CHECK(label != nullptr);
-
- art::x86_64::Condition x86_64_cond;
+static Condition UnaryConditionToX86_64Condition(JNIMacroUnaryCondition cond) {
switch (cond) {
case JNIMacroUnaryCondition::kZero:
- x86_64_cond = art::x86_64::kZero;
- break;
+ return kZero;
case JNIMacroUnaryCondition::kNotZero:
- x86_64_cond = art::x86_64::kNotZero;
- break;
+ return kNotZero;
default:
LOG(FATAL) << "Not implemented condition: " << static_cast<int>(cond);
UNREACHABLE();
}
+}
+
+void X86_64JNIMacroAssembler::TestGcMarking(JNIMacroLabel* label, JNIMacroUnaryCondition cond) {
+ CHECK(label != nullptr);
// CMP self->tls32_.is_gc_marking, 0
// Jcc <Offset>
DCHECK_EQ(Thread::IsGcMarkingSize(), 4u);
__ gs()->cmpl(Address::Absolute(Thread::IsGcMarkingOffset<kX86_64PointerSize>(), true),
Immediate(0));
- __ j(x86_64_cond, X86_64JNIMacroLabel::Cast(label)->AsX86_64());
+ __ j(UnaryConditionToX86_64Condition(cond), X86_64JNIMacroLabel::Cast(label)->AsX86_64());
+}
+
+void X86_64JNIMacroAssembler::TestMarkBit(ManagedRegister mref,
+ JNIMacroLabel* label,
+ JNIMacroUnaryCondition cond) {
+ DCHECK(kUseBakerReadBarrier);
+ CpuRegister ref = mref.AsX86_64().AsCpuRegister();
+ static_assert(LockWord::kMarkBitStateSize == 1u);
+ __ testl(Address(ref, mirror::Object::MonitorOffset().SizeValue()),
+ Immediate(LockWord::kMarkBitStateMaskShifted));
+ __ j(UnaryConditionToX86_64Condition(cond), X86_64JNIMacroLabel::Cast(label)->AsX86_64());
}
void X86_64JNIMacroAssembler::Bind(JNIMacroLabel* label) {
diff --git a/compiler/utils/x86_64/jni_macro_assembler_x86_64.h b/compiler/utils/x86_64/jni_macro_assembler_x86_64.h
index baebf48..71c3035 100644
--- a/compiler/utils/x86_64/jni_macro_assembler_x86_64.h
+++ b/compiler/utils/x86_64/jni_macro_assembler_x86_64.h
@@ -193,6 +193,8 @@
void Jump(JNIMacroLabel* label) override;
// Emit a conditional jump to the label by applying a unary condition test to the GC marking flag.
void TestGcMarking(JNIMacroLabel* label, JNIMacroUnaryCondition cond) override;
+ // Emit a conditional jump to the label by applying a unary condition test to object's mark bit.
+ void TestMarkBit(ManagedRegister ref, JNIMacroLabel* label, JNIMacroUnaryCondition cond) override;
// Code at this offset will serve as the target for the Jump call.
void Bind(JNIMacroLabel* label) override;