summaryrefslogtreecommitdiff
path: root/compiler/optimizing
diff options
context:
space:
mode:
Diffstat (limited to 'compiler/optimizing')
-rw-r--r--compiler/optimizing/code_generator_arm64.cc2
-rw-r--r--compiler/optimizing/code_generator_arm64.h19
-rw-r--r--compiler/optimizing/intrinsics_arm64.cc138
-rw-r--r--compiler/optimizing/intrinsics_utils.h10
4 files changed, 150 insertions, 19 deletions
diff --git a/compiler/optimizing/code_generator_arm64.cc b/compiler/optimizing/code_generator_arm64.cc
index f5a6f8dd73..a184dad900 100644
--- a/compiler/optimizing/code_generator_arm64.cc
+++ b/compiler/optimizing/code_generator_arm64.cc
@@ -6738,7 +6738,7 @@ void CodeGeneratorARM64::GenerateGcRootFieldLoad(
MaybeGenerateMarkingRegisterCheck(/* code= */ __LINE__);
}
-void CodeGeneratorARM64::GenerateIntrinsicCasMoveWithBakerReadBarrier(
+void CodeGeneratorARM64::GenerateIntrinsicMoveWithBakerReadBarrier(
vixl::aarch64::Register marked_old_value,
vixl::aarch64::Register old_value) {
DCHECK(EmitBakerReadBarrier());
diff --git a/compiler/optimizing/code_generator_arm64.h b/compiler/optimizing/code_generator_arm64.h
index 52321810bc..7ff08f55cb 100644
--- a/compiler/optimizing/code_generator_arm64.h
+++ b/compiler/optimizing/code_generator_arm64.h
@@ -144,19 +144,8 @@ Location ARM64ReturnLocation(DataType::Type return_type);
V(SystemArrayCopyByte) \
V(SystemArrayCopyInt) \
/* 1.8 */ \
- V(UnsafeGetAndAddInt) \
- V(UnsafeGetAndAddLong) \
- V(UnsafeGetAndSetInt) \
- V(UnsafeGetAndSetLong) \
- V(UnsafeGetAndSetObject) \
V(MethodHandleInvokeExact) \
- V(MethodHandleInvoke) \
- /* OpenJDK 11 */ \
- V(JdkUnsafeGetAndAddInt) \
- V(JdkUnsafeGetAndAddLong) \
- V(JdkUnsafeGetAndSetInt) \
- V(JdkUnsafeGetAndSetLong) \
- V(JdkUnsafeGetAndSetReference)
+ V(MethodHandleInvoke)
class SlowPathCodeARM64 : public SlowPathCode {
public:
@@ -904,9 +893,9 @@ class CodeGeneratorARM64 : public CodeGenerator {
uint32_t offset,
vixl::aarch64::Label* fixup_label,
ReadBarrierOption read_barrier_option);
- // Generate MOV for the `old_value` in intrinsic CAS and mark it with Baker read barrier.
- void GenerateIntrinsicCasMoveWithBakerReadBarrier(vixl::aarch64::Register marked_old_value,
- vixl::aarch64::Register old_value);
+ // Generate MOV for the `old_value` in intrinsic and mark it with Baker read barrier.
+ void GenerateIntrinsicMoveWithBakerReadBarrier(vixl::aarch64::Register marked_old_value,
+ vixl::aarch64::Register old_value);
// Fast path implementation of ReadBarrier::Barrier for a heap
// reference field load when Baker's read barriers are used.
// Overload suitable for Unsafe.getObject/-Volatile() intrinsic.
diff --git a/compiler/optimizing/intrinsics_arm64.cc b/compiler/optimizing/intrinsics_arm64.cc
index 9add804d24..be3fb9ac77 100644
--- a/compiler/optimizing/intrinsics_arm64.cc
+++ b/compiler/optimizing/intrinsics_arm64.cc
@@ -1360,7 +1360,7 @@ class ReadBarrierCasSlowPathARM64 : public SlowPathCodeARM64 {
// Mark the `old_value_` from the main path and compare with `expected_`.
if (kUseBakerReadBarrier) {
DCHECK(mark_old_value_slow_path_ == nullptr);
- arm64_codegen->GenerateIntrinsicCasMoveWithBakerReadBarrier(old_value_temp_, old_value_);
+ arm64_codegen->GenerateIntrinsicMoveWithBakerReadBarrier(old_value_temp_, old_value_);
} else {
DCHECK(mark_old_value_slow_path_ != nullptr);
__ B(mark_old_value_slow_path_->GetEntryLabel());
@@ -1414,7 +1414,7 @@ class ReadBarrierCasSlowPathARM64 : public SlowPathCodeARM64 {
__ Bind(&mark_old_value);
if (kUseBakerReadBarrier) {
DCHECK(update_old_value_slow_path_ == nullptr);
- arm64_codegen->GenerateIntrinsicCasMoveWithBakerReadBarrier(old_value_, old_value_temp_);
+ arm64_codegen->GenerateIntrinsicMoveWithBakerReadBarrier(old_value_, old_value_temp_);
} else {
// Note: We could redirect the `failure` above directly to the entry label and bind
// the exit label in the main path, but the main path would need to access the
@@ -1692,6 +1692,138 @@ static void GenerateGetAndUpdate(CodeGeneratorARM64* codegen,
__ Cbnz(store_result, &loop_label);
}
+static void CreateUnsafeGetAndUpdateLocations(ArenaAllocator* allocator,
+ HInvoke* invoke,
+ CodeGeneratorARM64* codegen) {
+ const bool can_call = codegen->EmitReadBarrier() && IsUnsafeGetAndSetReference(invoke);
+ LocationSummary* locations =
+ new (allocator) LocationSummary(invoke,
+ can_call
+ ? LocationSummary::kCallOnSlowPath
+ : LocationSummary::kNoCall,
+ kIntrinsified);
+ if (can_call && kUseBakerReadBarrier) {
+ locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers.
+ }
+ locations->SetInAt(0, Location::NoLocation()); // Unused receiver.
+ locations->SetInAt(1, Location::RequiresRegister());
+ locations->SetInAt(2, Location::RequiresRegister());
+ locations->SetInAt(3, Location::RequiresRegister());
+ locations->AddTemp(Location::RequiresRegister());
+
+ locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
+}
+
+static void GenUnsafeGetAndUpdate(HInvoke* invoke,
+ DataType::Type type,
+ CodeGeneratorARM64* codegen,
+ GetAndUpdateOp get_and_update_op) {
+ MacroAssembler* masm = codegen->GetVIXLAssembler();
+ LocationSummary* locations = invoke->GetLocations();
+
+ Register out = RegisterFrom(locations->Out(), type); // Result.
+ Register base = WRegisterFrom(locations->InAt(1)); // Object pointer.
+ Register offset = XRegisterFrom(locations->InAt(2)); // Long offset.
+ Register arg = RegisterFrom(locations->InAt(3), type); // Expected.
+ Register tmp_ptr = XRegisterFrom(locations->GetTemp(0)); // Pointer to actual memory.
+
+ // This needs to be before the temp registers, as MarkGCCard also uses VIXL temps.
+ if (type == DataType::Type::kReference) {
+ DCHECK(get_and_update_op == GetAndUpdateOp::kSet);
+ // Mark card for object as a new value shall be stored.
+ bool new_value_can_be_null = true; // TODO: Worth finding out this information?
+ codegen->MarkGCCard(base, /*value=*/ arg, new_value_can_be_null);
+ }
+
+ __ Add(tmp_ptr, base.X(), Operand(offset));
+ GenerateGetAndUpdate(codegen,
+ get_and_update_op,
+ type,
+ std::memory_order_seq_cst,
+ tmp_ptr,
+ arg,
+ /*old_value=*/ out);
+
+ if (type == DataType::Type::kReference && codegen->EmitReadBarrier()) {
+ DCHECK(get_and_update_op == GetAndUpdateOp::kSet);
+ if (kUseBakerReadBarrier) {
+ codegen->GenerateIntrinsicMoveWithBakerReadBarrier(out.W(), out.W());
+ } else {
+ codegen->GenerateReadBarrierSlow(
+ invoke,
+ Location::RegisterLocation(out.GetCode()),
+ Location::RegisterLocation(out.GetCode()),
+ Location::RegisterLocation(base.GetCode()),
+ /*offset=*/ 0u,
+ /*index=*/ Location::RegisterLocation(offset.GetCode()));
+ }
+ }
+}
+
+void IntrinsicLocationsBuilderARM64::VisitUnsafeGetAndAddInt(HInvoke* invoke) {
+ VisitJdkUnsafeGetAndAddInt(invoke);
+}
+void IntrinsicLocationsBuilderARM64::VisitUnsafeGetAndAddLong(HInvoke* invoke) {
+ VisitJdkUnsafeGetAndAddLong(invoke);
+}
+void IntrinsicLocationsBuilderARM64::VisitUnsafeGetAndSetInt(HInvoke* invoke) {
+ VisitJdkUnsafeGetAndSetInt(invoke);
+}
+void IntrinsicLocationsBuilderARM64::VisitUnsafeGetAndSetLong(HInvoke* invoke) {
+ VisitJdkUnsafeGetAndSetLong(invoke);
+}
+void IntrinsicLocationsBuilderARM64::VisitUnsafeGetAndSetObject(HInvoke* invoke) {
+ VisitJdkUnsafeGetAndSetReference(invoke);
+}
+
+void IntrinsicLocationsBuilderARM64::VisitJdkUnsafeGetAndAddInt(HInvoke* invoke) {
+ CreateUnsafeGetAndUpdateLocations(allocator_, invoke, codegen_);
+}
+void IntrinsicLocationsBuilderARM64::VisitJdkUnsafeGetAndAddLong(HInvoke* invoke) {
+ CreateUnsafeGetAndUpdateLocations(allocator_, invoke, codegen_);
+}
+void IntrinsicLocationsBuilderARM64::VisitJdkUnsafeGetAndSetInt(HInvoke* invoke) {
+ CreateUnsafeGetAndUpdateLocations(allocator_, invoke, codegen_);
+}
+void IntrinsicLocationsBuilderARM64::VisitJdkUnsafeGetAndSetLong(HInvoke* invoke) {
+ CreateUnsafeGetAndUpdateLocations(allocator_, invoke, codegen_);
+}
+void IntrinsicLocationsBuilderARM64::VisitJdkUnsafeGetAndSetReference(HInvoke* invoke) {
+ CreateUnsafeGetAndUpdateLocations(allocator_, invoke, codegen_);
+}
+
+void IntrinsicCodeGeneratorARM64::VisitUnsafeGetAndAddInt(HInvoke* invoke) {
+ VisitJdkUnsafeGetAndAddInt(invoke);
+}
+void IntrinsicCodeGeneratorARM64::VisitUnsafeGetAndAddLong(HInvoke* invoke) {
+ VisitJdkUnsafeGetAndAddLong(invoke);
+}
+void IntrinsicCodeGeneratorARM64::VisitUnsafeGetAndSetInt(HInvoke* invoke) {
+ VisitJdkUnsafeGetAndSetInt(invoke);
+}
+void IntrinsicCodeGeneratorARM64::VisitUnsafeGetAndSetLong(HInvoke* invoke) {
+ VisitJdkUnsafeGetAndSetLong(invoke);
+}
+void IntrinsicCodeGeneratorARM64::VisitUnsafeGetAndSetObject(HInvoke* invoke) {
+ VisitJdkUnsafeGetAndSetReference(invoke);
+}
+
+void IntrinsicCodeGeneratorARM64::VisitJdkUnsafeGetAndAddInt(HInvoke* invoke) {
+ GenUnsafeGetAndUpdate(invoke, DataType::Type::kInt32, codegen_, GetAndUpdateOp::kAdd);
+}
+void IntrinsicCodeGeneratorARM64::VisitJdkUnsafeGetAndAddLong(HInvoke* invoke) {
+ GenUnsafeGetAndUpdate(invoke, DataType::Type::kInt64, codegen_, GetAndUpdateOp::kAdd);
+}
+void IntrinsicCodeGeneratorARM64::VisitJdkUnsafeGetAndSetInt(HInvoke* invoke) {
+ GenUnsafeGetAndUpdate(invoke, DataType::Type::kInt32, codegen_, GetAndUpdateOp::kSet);
+}
+void IntrinsicCodeGeneratorARM64::VisitJdkUnsafeGetAndSetLong(HInvoke* invoke) {
+ GenUnsafeGetAndUpdate(invoke, DataType::Type::kInt64, codegen_, GetAndUpdateOp::kSet);
+}
+void IntrinsicCodeGeneratorARM64::VisitJdkUnsafeGetAndSetReference(HInvoke* invoke) {
+ GenUnsafeGetAndUpdate(invoke, DataType::Type::kReference, codegen_, GetAndUpdateOp::kSet);
+}
+
void IntrinsicLocationsBuilderARM64::VisitStringCompareTo(HInvoke* invoke) {
LocationSummary* locations =
new (allocator_) LocationSummary(invoke,
@@ -5571,7 +5703,7 @@ static void GenerateVarHandleGetAndUpdate(HInvoke* invoke,
__ Sxth(out.W(), old_value.W());
} else if (value_type == DataType::Type::kReference && codegen->EmitReadBarrier()) {
if (kUseBakerReadBarrier) {
- codegen->GenerateIntrinsicCasMoveWithBakerReadBarrier(out.W(), old_value.W());
+ codegen->GenerateIntrinsicMoveWithBakerReadBarrier(out.W(), old_value.W());
} else {
codegen->GenerateReadBarrierSlow(
invoke,
diff --git a/compiler/optimizing/intrinsics_utils.h b/compiler/optimizing/intrinsics_utils.h
index 380011e608..590bc34ee9 100644
--- a/compiler/optimizing/intrinsics_utils.h
+++ b/compiler/optimizing/intrinsics_utils.h
@@ -177,6 +177,16 @@ static inline bool IsUnsafeCASReference(HInvoke* invoke) {
}
}
+static inline bool IsUnsafeGetAndSetReference(HInvoke* invoke) {
+ switch (invoke->GetIntrinsic()) {
+ case Intrinsics::kUnsafeGetAndSetObject:
+ case Intrinsics::kJdkUnsafeGetAndSetReference:
+ return true;
+ default:
+ return false;
+ }
+}
+
static inline bool IsVarHandleCASFamily(HInvoke* invoke) {
mirror::VarHandle::AccessModeTemplate access_mode =
mirror::VarHandle::GetAccessModeTemplateByIntrinsic(invoke->GetIntrinsic());
generated by cgit v1.2.3-59-g8ed1b (git 2.39.0) at 2025-10-20 12:48:47 +0000