summaryrefslogtreecommitdiff
path: root/compiler
diff options
context:
space:
mode:
Diffstat (limited to 'compiler')
-rw-r--r--compiler/dex/quick/arm/call_arm.cc103
-rw-r--r--compiler/dex/quick/arm64/call_arm64.cc32
-rw-r--r--compiler/image_writer.cc6
3 files changed, 110 insertions, 31 deletions
diff --git a/compiler/dex/quick/arm/call_arm.cc b/compiler/dex/quick/arm/call_arm.cc
index 9cf005bc48..1a9dbeae0f 100644
--- a/compiler/dex/quick/arm/call_arm.cc
+++ b/compiler/dex/quick/arm/call_arm.cc
@@ -161,7 +161,11 @@ void ArmMir2Lir::GenMonitorEnter(int opt_flags, RegLocation rl_src) {
NewLIR3(kThumb2Ldrex, rs_r1.GetReg(), rs_r0.GetReg(),
mirror::Object::MonitorOffset().Int32Value() >> 2);
MarkPossibleNullPointerException(opt_flags);
- LIR* not_unlocked_branch = OpCmpImmBranch(kCondNe, rs_r1, 0, NULL);
+ // Zero out the read barrier bits.
+ OpRegRegImm(kOpAnd, rs_r3, rs_r1, LockWord::kReadBarrierStateMaskShiftedToggled);
+ LIR* not_unlocked_branch = OpCmpImmBranch(kCondNe, rs_r3, 0, NULL);
+ // r1 is zero except for the rb bits here. Copy the read barrier bits into r2.
+ OpRegRegReg(kOpOr, rs_r2, rs_r2, rs_r1);
NewLIR4(kThumb2Strex, rs_r1.GetReg(), rs_r2.GetReg(), rs_r0.GetReg(),
mirror::Object::MonitorOffset().Int32Value() >> 2);
LIR* lock_success_branch = OpCmpImmBranch(kCondEq, rs_r1, 0, NULL);
@@ -189,7 +193,14 @@ void ArmMir2Lir::GenMonitorEnter(int opt_flags, RegLocation rl_src) {
NewLIR3(kThumb2Ldrex, rs_r1.GetReg(), rs_r0.GetReg(),
mirror::Object::MonitorOffset().Int32Value() >> 2);
MarkPossibleNullPointerException(opt_flags);
- OpRegImm(kOpCmp, rs_r1, 0);
+ // Zero out the read barrier bits.
+ OpRegRegImm(kOpAnd, rs_r3, rs_r1, LockWord::kReadBarrierStateMaskShiftedToggled);
+ // r1 will be zero except for the rb bits if the following
+ // cmp-and-branch branches to eq where r2 will be used. Copy the
+ // read barrier bits into r2.
+ OpRegRegReg(kOpOr, rs_r2, rs_r2, rs_r1);
+ OpRegImm(kOpCmp, rs_r3, 0);
+
LIR* it = OpIT(kCondEq, "");
NewLIR4(kThumb2Strex/*eq*/, rs_r1.GetReg(), rs_r2.GetReg(), rs_r0.GetReg(),
mirror::Object::MonitorOffset().Int32Value() >> 2);
@@ -228,14 +239,28 @@ void ArmMir2Lir::GenMonitorExit(int opt_flags, RegLocation rl_src) {
null_check_branch = OpCmpImmBranch(kCondEq, rs_r0, 0, NULL);
}
}
- Load32Disp(rs_r0, mirror::Object::MonitorOffset().Int32Value(), rs_r1);
+ if (!kUseReadBarrier) {
+ Load32Disp(rs_r0, mirror::Object::MonitorOffset().Int32Value(), rs_r1); // Get lock
+ } else {
+ NewLIR3(kThumb2Ldrex, rs_r1.GetReg(), rs_r0.GetReg(),
+ mirror::Object::MonitorOffset().Int32Value() >> 2);
+ }
MarkPossibleNullPointerException(opt_flags);
- LoadConstantNoClobber(rs_r3, 0);
- LIR* slow_unlock_branch = OpCmpBranch(kCondNe, rs_r1, rs_r2, NULL);
+ // Zero out the read barrier bits.
+ OpRegRegImm(kOpAnd, rs_r3, rs_r1, LockWord::kReadBarrierStateMaskShiftedToggled);
+ // Zero out except the read barrier bits.
+ OpRegRegImm(kOpAnd, rs_r1, rs_r1, LockWord::kReadBarrierStateMaskShifted);
+ LIR* slow_unlock_branch = OpCmpBranch(kCondNe, rs_r3, rs_r2, NULL);
GenMemBarrier(kAnyStore);
- Store32Disp(rs_r0, mirror::Object::MonitorOffset().Int32Value(), rs_r3);
- LIR* unlock_success_branch = OpUnconditionalBranch(NULL);
-
+ LIR* unlock_success_branch;
+ if (!kUseReadBarrier) {
+ Store32Disp(rs_r0, mirror::Object::MonitorOffset().Int32Value(), rs_r1);
+ unlock_success_branch = OpUnconditionalBranch(NULL);
+ } else {
+ NewLIR4(kThumb2Strex, rs_r2.GetReg(), rs_r1.GetReg(), rs_r0.GetReg(),
+ mirror::Object::MonitorOffset().Int32Value() >> 2);
+ unlock_success_branch = OpCmpImmBranch(kCondEq, rs_r2, 0, NULL);
+ }
LIR* slow_path_target = NewLIR0(kPseudoTargetLabel);
slow_unlock_branch->target = slow_path_target;
if (null_check_branch != nullptr) {
@@ -253,25 +278,57 @@ void ArmMir2Lir::GenMonitorExit(int opt_flags, RegLocation rl_src) {
} else {
// Explicit null-check as slow-path is entered using an IT.
GenNullCheck(rs_r0, opt_flags);
- Load32Disp(rs_r0, mirror::Object::MonitorOffset().Int32Value(), rs_r1); // Get lock
+ if (!kUseReadBarrier) {
+ Load32Disp(rs_r0, mirror::Object::MonitorOffset().Int32Value(), rs_r1); // Get lock
+ } else {
+ // If we use read barriers, we need to use atomic instructions.
+ NewLIR3(kThumb2Ldrex, rs_r1.GetReg(), rs_r0.GetReg(),
+ mirror::Object::MonitorOffset().Int32Value() >> 2);
+ }
MarkPossibleNullPointerException(opt_flags);
Load32Disp(rs_rARM_SELF, Thread::ThinLockIdOffset<4>().Int32Value(), rs_r2);
- LoadConstantNoClobber(rs_r3, 0);
+ // Zero out the read barrier bits.
+ OpRegRegImm(kOpAnd, rs_r3, rs_r1, LockWord::kReadBarrierStateMaskShiftedToggled);
+ // Zero out except the read barrier bits.
+ OpRegRegImm(kOpAnd, rs_r1, rs_r1, LockWord::kReadBarrierStateMaskShifted);
// Is lock unheld on lock or held by us (==thread_id) on unlock?
- OpRegReg(kOpCmp, rs_r1, rs_r2);
-
- LIR* it = OpIT(kCondEq, "EE");
- if (GenMemBarrier(kAnyStore)) {
- UpdateIT(it, "TEE");
+ OpRegReg(kOpCmp, rs_r3, rs_r2);
+ if (!kUseReadBarrier) {
+ LIR* it = OpIT(kCondEq, "EE");
+ if (GenMemBarrier(kAnyStore)) {
+ UpdateIT(it, "TEE");
+ }
+ Store32Disp/*eq*/(rs_r0, mirror::Object::MonitorOffset().Int32Value(), rs_r1);
+ // Go expensive route - UnlockObjectFromCode(obj);
+ LoadWordDisp/*ne*/(rs_rARM_SELF, QUICK_ENTRYPOINT_OFFSET(4, pUnlockObject).Int32Value(),
+ rs_rARM_LR);
+ ClobberCallerSave();
+ LIR* call_inst = OpReg(kOpBlx/*ne*/, rs_rARM_LR);
+ OpEndIT(it);
+ MarkSafepointPC(call_inst);
+ } else {
+ // If we use read barriers, we need to use atomic instructions.
+ LIR* it = OpIT(kCondEq, "");
+ if (GenMemBarrier(kAnyStore)) {
+ UpdateIT(it, "T");
+ }
+ NewLIR4/*eq*/(kThumb2Strex, rs_r2.GetReg(), rs_r1.GetReg(), rs_r0.GetReg(),
+ mirror::Object::MonitorOffset().Int32Value() >> 2);
+ OpEndIT(it);
+ // Since we know r2 wasn't zero before the above it instruction,
+ // if r2 is zero here, we know r3 was equal to r2 and the strex
+ // suceeded (we're done). Otherwise (either r3 wasn't equal to r2
+ // or the strex failed), call the entrypoint.
+ OpRegImm(kOpCmp, rs_r2, 0);
+ LIR* it2 = OpIT(kCondNe, "T");
+ // Go expensive route - UnlockObjectFromCode(obj);
+ LoadWordDisp/*ne*/(rs_rARM_SELF, QUICK_ENTRYPOINT_OFFSET(4, pUnlockObject).Int32Value(),
+ rs_rARM_LR);
+ ClobberCallerSave();
+ LIR* call_inst = OpReg(kOpBlx/*ne*/, rs_rARM_LR);
+ OpEndIT(it2);
+ MarkSafepointPC(call_inst);
}
- Store32Disp/*eq*/(rs_r0, mirror::Object::MonitorOffset().Int32Value(), rs_r3);
- // Go expensive route - UnlockObjectFromCode(obj);
- LoadWordDisp/*ne*/(rs_rARM_SELF, QUICK_ENTRYPOINT_OFFSET(4, pUnlockObject).Int32Value(),
- rs_rARM_LR);
- ClobberCallerSave();
- LIR* call_inst = OpReg(kOpBlx/*ne*/, rs_rARM_LR);
- OpEndIT(it);
- MarkSafepointPC(call_inst);
}
}
diff --git a/compiler/dex/quick/arm64/call_arm64.cc b/compiler/dex/quick/arm64/call_arm64.cc
index 24e8fdff80..15edcc5142 100644
--- a/compiler/dex/quick/arm64/call_arm64.cc
+++ b/compiler/dex/quick/arm64/call_arm64.cc
@@ -172,7 +172,12 @@ void Arm64Mir2Lir::GenMonitorEnter(int opt_flags, RegLocation rl_src) {
OpRegRegImm(kOpAdd, rs_x2, rs_x0, mirror::Object::MonitorOffset().Int32Value());
NewLIR2(kA64Ldxr2rX, rw3, rx2);
MarkPossibleNullPointerException(opt_flags);
- LIR* not_unlocked_branch = OpCmpImmBranch(kCondNe, rs_w3, 0, NULL);
+ // Zero out the read barrier bits.
+ OpRegRegImm(kOpAnd, rs_w2, rs_w3, LockWord::kReadBarrierStateMaskShiftedToggled);
+ LIR* not_unlocked_branch = OpCmpImmBranch(kCondNe, rs_w2, 0, NULL);
+ // w3 is zero except for the rb bits here. Copy the read barrier bits into w1.
+ OpRegRegReg(kOpOr, rs_w1, rs_w1, rs_w3);
+ OpRegRegImm(kOpAdd, rs_x2, rs_x0, mirror::Object::MonitorOffset().Int32Value());
NewLIR3(kA64Stxr3wrX, rw3, rw1, rx2);
LIR* lock_success_branch = OpCmpImmBranch(kCondEq, rs_w3, 0, NULL);
@@ -217,13 +222,28 @@ void Arm64Mir2Lir::GenMonitorExit(int opt_flags, RegLocation rl_src) {
}
}
Load32Disp(rs_xSELF, Thread::ThinLockIdOffset<8>().Int32Value(), rs_w1);
- Load32Disp(rs_x0, mirror::Object::MonitorOffset().Int32Value(), rs_w2);
+ if (!kUseReadBarrier) {
+ Load32Disp(rs_x0, mirror::Object::MonitorOffset().Int32Value(), rs_w2);
+ } else {
+ OpRegRegImm(kOpAdd, rs_x3, rs_x0, mirror::Object::MonitorOffset().Int32Value());
+ NewLIR2(kA64Ldxr2rX, rw2, rx3);
+ }
MarkPossibleNullPointerException(opt_flags);
- LIR* slow_unlock_branch = OpCmpBranch(kCondNe, rs_w1, rs_w2, NULL);
+ // Zero out the read barrier bits.
+ OpRegRegImm(kOpAnd, rs_w3, rs_w2, LockWord::kReadBarrierStateMaskShiftedToggled);
+ // Zero out except the read barrier bits.
+ OpRegRegImm(kOpAnd, rs_w2, rs_w2, LockWord::kReadBarrierStateMaskShifted);
+ LIR* slow_unlock_branch = OpCmpBranch(kCondNe, rs_w3, rs_w1, NULL);
GenMemBarrier(kAnyStore);
- Store32Disp(rs_x0, mirror::Object::MonitorOffset().Int32Value(), rs_wzr);
- LIR* unlock_success_branch = OpUnconditionalBranch(NULL);
-
+ LIR* unlock_success_branch;
+ if (!kUseReadBarrier) {
+ Store32Disp(rs_x0, mirror::Object::MonitorOffset().Int32Value(), rs_w2);
+ unlock_success_branch = OpUnconditionalBranch(NULL);
+ } else {
+ OpRegRegImm(kOpAdd, rs_x3, rs_x0, mirror::Object::MonitorOffset().Int32Value());
+ NewLIR3(kA64Stxr3wrX, rw1, rw2, rx3);
+ unlock_success_branch = OpCmpImmBranch(kCondEq, rs_w1, 0, NULL);
+ }
LIR* slow_path_target = NewLIR0(kPseudoTargetLabel);
slow_unlock_branch->target = slow_path_target;
if (null_check_branch != nullptr) {
diff --git a/compiler/image_writer.cc b/compiler/image_writer.cc
index f5f9320532..b4732c87c8 100644
--- a/compiler/image_writer.cc
+++ b/compiler/image_writer.cc
@@ -909,7 +909,9 @@ void ImageWriter::CopyAndFixupObjects() {
heap->VisitObjects(CopyAndFixupObjectsCallback, this);
// Fix up the object previously had hash codes.
for (const std::pair<mirror::Object*, uint32_t>& hash_pair : saved_hashes_) {
- hash_pair.first->SetLockWord(LockWord::FromHashCode(hash_pair.second), false);
+ Object* obj = hash_pair.first;
+ DCHECK_EQ(obj->GetLockWord(false).ReadBarrierState(), 0U);
+ obj->SetLockWord(LockWord::FromHashCode(hash_pair.second, 0U), false);
}
saved_hashes_.clear();
}
@@ -935,7 +937,7 @@ void ImageWriter::CopyAndFixupObjectsCallback(Object* obj, void* arg) {
Object* copy = reinterpret_cast<Object*>(dst);
// Write in a hash code of objects which have inflated monitors or a hash code in their monitor
// word.
- copy->SetLockWord(LockWord(), false);
+ copy->SetLockWord(LockWord::Default(), false);
image_writer->FixupObject(obj, copy);
}
generated by cgit v1.2.3-59-g8ed1b (git 2.39.0) at 2026-01-01 11:49:10 +0000