Implement Reference.refersTo() intrinsic.

Test: Added tests to 122-npe and 160-read-barrier-stress
Test: m test-art-host-gtest
Test: testrunner.py --host --optimizing
Test: run-gtests.sh
Test: testrunner.py --target --optimizing
Bug: 172573708
Change-Id: I8342510565289058df218d3249ffac1eb993ca4f

8 files changed, 260 insertions, 4 deletions

diff --git a/compiler/optimizing/intrinsics.cc b/compiler/optimizing/intrinsics.cc
index 10d0b8992c..16c3029770 100644
--- a/compiler/optimizing/intrinsics.cc
+++ b/compiler/optimizing/intrinsics.cc
@@ -392,6 +392,20 @@ void IntrinsicVisitor::CreateReferenceGetReferentLocations(HInvoke* invoke,
   locations->SetOut(Location::RequiresRegister());
 }
 
+void IntrinsicVisitor::CreateReferenceRefersToLocations(HInvoke* invoke) {
+  if (kEmitCompilerReadBarrier && !kUseBakerReadBarrier) {
+    // Unimplemented for non-Baker read barrier.
+    return;
+  }
+
+  ArenaAllocator* allocator = invoke->GetBlock()->GetGraph()->GetAllocator();
+  LocationSummary* locations =
+      new (allocator) LocationSummary(invoke, LocationSummary::kCallOnSlowPath, kIntrinsified);
+  locations->SetInAt(0, Location::RequiresRegister());
+  locations->SetInAt(1, Location::RequiresRegister());
+  locations->SetOut(Location::RequiresRegister());
+}
+
 void IntrinsicVisitor::AssertNonMovableStringClass() {
   if (kIsDebugBuild) {
     ScopedObjectAccess soa(Thread::Current());
diff --git a/compiler/optimizing/intrinsics.h b/compiler/optimizing/intrinsics.h
index 48a103530e..62b5faa7d0 100644
--- a/compiler/optimizing/intrinsics.h
+++ b/compiler/optimizing/intrinsics.h
@@ -139,6 +139,7 @@ class IntrinsicVisitor : public ValueObject {
   static MemberOffset GetReferenceDisableIntrinsicOffset();
   static MemberOffset GetReferenceSlowPathEnabledOffset();
   static void CreateReferenceGetReferentLocations(HInvoke* invoke, CodeGenerator* codegen);
+  static void CreateReferenceRefersToLocations(HInvoke* invoke);
 
  protected:
   IntrinsicVisitor() {}
diff --git a/compiler/optimizing/intrinsics_arm64.cc b/compiler/optimizing/intrinsics_arm64.cc
index 252865f230..6774a2600f 100644
--- a/compiler/optimizing/intrinsics_arm64.cc
+++ b/compiler/optimizing/intrinsics_arm64.cc
@@ -3292,6 +3292,61 @@ void IntrinsicCodeGeneratorARM64::VisitReferenceGetReferent(HInvoke* invoke) {
   __ Bind(slow_path->GetExitLabel());
 }
 
+void IntrinsicLocationsBuilderARM64::VisitReferenceRefersTo(HInvoke* invoke) {
+  IntrinsicVisitor::CreateReferenceRefersToLocations(invoke);
+}
+
+void IntrinsicCodeGeneratorARM64::VisitReferenceRefersTo(HInvoke* invoke) {
+  LocationSummary* locations = invoke->GetLocations();
+  MacroAssembler* masm = codegen_->GetVIXLAssembler();
+  UseScratchRegisterScope temps(masm);
+
+  Register obj = WRegisterFrom(locations->InAt(0));
+  Register other = WRegisterFrom(locations->InAt(1));
+  Register out = WRegisterFrom(locations->Out());
+  Register tmp = temps.AcquireW();
+
+  uint32_t referent_offset = mirror::Reference::ReferentOffset().Uint32Value();
+  uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value();
+
+  MemOperand field = HeapOperand(obj, referent_offset);
+  codegen_->LoadAcquire(invoke, DataType::Type::kReference, tmp, field, /*needs_null_check=*/ true);
+
+  __ Cmp(tmp, other);
+
+  if (kEmitCompilerReadBarrier) {
+    DCHECK(kUseBakerReadBarrier);
+
+    vixl::aarch64::Label calculate_result;
+
+    // If the GC is not marking, the comparison result is final.
+    __ Cbz(mr, &calculate_result);
+
+    __ B(&calculate_result, eq);  // ZF set if taken.
+
+    // Check if the loaded reference is null.
+    __ Cbz(tmp, &calculate_result);  // ZF clear if taken.
+
+    // For correct memory visibility, we need a barrier before loading the lock word.
+    codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny);
+
+    // Load the lockword and check if it is a forwarding address.
+    static_assert(LockWord::kStateShift == 30u);
+    static_assert(LockWord::kStateForwardingAddress == 3u);
+    __ Ldr(tmp, HeapOperand(tmp, monitor_offset));
+    __ Cmp(tmp, Operand(0xc0000000));
+    __ B(&calculate_result, lo);   // ZF clear if taken.
+
+    // Extract the forwarding address and compare with `other`.
+    __ Cmp(other, Operand(tmp, LSL, LockWord::kForwardingAddressShift));
+
+    __ Bind(&calculate_result);
+  }
+
+  // Convert ZF into the Boolean result.
+  __ Cset(out, eq);
+}
+
 void IntrinsicLocationsBuilderARM64::VisitThreadInterrupted(HInvoke* invoke) {
   LocationSummary* locations =
       new (allocator_) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified);
diff --git a/compiler/optimizing/intrinsics_arm_vixl.cc b/compiler/optimizing/intrinsics_arm_vixl.cc
index 3766f35498..b652234289 100644
--- a/compiler/optimizing/intrinsics_arm_vixl.cc
+++ b/compiler/optimizing/intrinsics_arm_vixl.cc
@@ -2559,6 +2559,68 @@ void IntrinsicCodeGeneratorARMVIXL::VisitReferenceGetReferent(HInvoke* invoke) {
   __ Bind(slow_path->GetExitLabel());
 }
 
+void IntrinsicLocationsBuilderARMVIXL::VisitReferenceRefersTo(HInvoke* invoke) {
+  IntrinsicVisitor::CreateReferenceRefersToLocations(invoke);
+}
+
+void IntrinsicCodeGeneratorARMVIXL::VisitReferenceRefersTo(HInvoke* invoke) {
+  LocationSummary* locations = invoke->GetLocations();
+  ArmVIXLAssembler* assembler = GetAssembler();
+  UseScratchRegisterScope temps(assembler->GetVIXLAssembler());
+
+  vixl32::Register obj = RegisterFrom(locations->InAt(0));
+  vixl32::Register other = RegisterFrom(locations->InAt(1));
+  vixl32::Register out = RegisterFrom(locations->Out());
+  vixl32::Register tmp = temps.Acquire();
+
+  uint32_t referent_offset = mirror::Reference::ReferentOffset().Uint32Value();
+  uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value();
+
+  {
+    // Ensure that between load and MaybeRecordImplicitNullCheck there are no pools emitted.
+    // Loading scratch register always uses 32-bit encoding.
+    vixl::ExactAssemblyScope eas(assembler->GetVIXLAssembler(),
+                                 vixl32::k32BitT32InstructionSizeInBytes);
+    __ ldr(tmp, MemOperand(obj, referent_offset));
+    codegen_->MaybeRecordImplicitNullCheck(invoke);
+  }
+  codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny);  // `referent` is volatile.
+
+  if (kEmitCompilerReadBarrier) {
+    DCHECK(kUseBakerReadBarrier);
+
+    vixl32::Label calculate_result;
+    __ Subs(out, tmp, other);
+    __ B(eq, &calculate_result);  // `out` is 0 if taken.
+
+    // Check if the loaded reference is null.
+    __ Cmp(tmp, 0);
+    __ B(eq, &calculate_result);  // `out` is not 0 if taken.
+
+    // For correct memory visibility, we need a barrier before loading the lock word
+    // but we already have the barrier emitted for volatile load above which is sufficient.
+
+    // Load the lockword and check if it is a forwarding address.
+    static_assert(LockWord::kStateShift == 30u);
+    static_assert(LockWord::kStateForwardingAddress == 3u);
+    __ Ldr(tmp, MemOperand(tmp, monitor_offset));
+    __ Cmp(tmp, Operand(0xc0000000));
+    __ B(lo, &calculate_result);   // `out` is not 0 if taken.
+
+    // Extract the forwarding address and subtract from `other`.
+    __ Sub(out, other, Operand(tmp, LSL, LockWord::kForwardingAddressShift));
+
+    __ Bind(&calculate_result);
+  } else {
+    DCHECK(!kEmitCompilerReadBarrier);
+    __ Sub(out, tmp, other);
+  }
+
+  // Convert 0 to 1 and non-zero to 0 for the Boolean result (`out = (out == 0)`).
+  __ Clz(out, out);
+  __ Lsr(out, out, WhichPowerOf2(out.GetSizeInBits()));
+}
+
 void IntrinsicLocationsBuilderARMVIXL::VisitThreadInterrupted(HInvoke* invoke) {
   LocationSummary* locations =
       new (allocator_) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified);
diff --git a/compiler/optimizing/intrinsics_x86.cc b/compiler/optimizing/intrinsics_x86.cc
index 83a2e39e5f..cda3500d7f 100644
--- a/compiler/optimizing/intrinsics_x86.cc
+++ b/compiler/optimizing/intrinsics_x86.cc
@@ -3167,6 +3167,66 @@ void IntrinsicCodeGeneratorX86::VisitReferenceGetReferent(HInvoke* invoke) {
   __ Bind(slow_path->GetExitLabel());
 }
 
+void IntrinsicLocationsBuilderX86::VisitReferenceRefersTo(HInvoke* invoke) {
+  IntrinsicVisitor::CreateReferenceRefersToLocations(invoke);
+}
+
+void IntrinsicCodeGeneratorX86::VisitReferenceRefersTo(HInvoke* invoke) {
+  X86Assembler* assembler = GetAssembler();
+  LocationSummary* locations = invoke->GetLocations();
+
+  Register obj = locations->InAt(0).AsRegister<Register>();
+  Register other = locations->InAt(1).AsRegister<Register>();
+  Register out = locations->Out().AsRegister<Register>();
+
+  uint32_t referent_offset = mirror::Reference::ReferentOffset().Uint32Value();
+  uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value();
+
+  __ movl(out, Address(obj, referent_offset));
+  codegen_->MaybeRecordImplicitNullCheck(invoke);
+  // Note that the fence is a no-op, thanks to the x86 memory model.
+  codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny);  // `referent` is volatile.
+
+  NearLabel end, return_true, return_false;
+  __ cmpl(out, other);
+
+  if (kEmitCompilerReadBarrier) {
+    DCHECK(kUseBakerReadBarrier);
+
+    __ j(kEqual, &return_true);
+
+    // Check if the loaded reference is null.
+    __ testl(out, out);
+    __ j(kZero, &return_false);
+
+    // For correct memory visibility, we need a barrier before loading the lock word
+    // but we already have the barrier emitted for volatile load above which is sufficient.
+
+    // Load the lockword and check if it is a forwarding address.
+    static_assert(LockWord::kStateShift == 30u);
+    static_assert(LockWord::kStateForwardingAddress == 3u);
+    __ movl(out, Address(out, monitor_offset));
+    __ cmpl(out, Immediate(static_cast<int32_t>(0xc0000000)));
+    __ j(kBelow, &return_false);
+
+    // Extract the forwarding address and compare with `other`.
+    __ shll(out, Immediate(LockWord::kForwardingAddressShift));
+    __ cmpl(out, other);
+  }
+
+  __ j(kNotEqual, &return_false);
+
+  // Return true and exit the function.
+  __ Bind(&return_true);
+  __ movl(out, Immediate(1));
+  __ jmp(&end);
+
+  // Return false and exit the function.
+  __ Bind(&return_false);
+  __ xorl(out, out);
+  __ Bind(&end);
+}
+
 void IntrinsicLocationsBuilderX86::VisitThreadInterrupted(HInvoke* invoke) {
   LocationSummary* locations =
       new (allocator_) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified);
diff --git a/compiler/optimizing/intrinsics_x86_64.cc b/compiler/optimizing/intrinsics_x86_64.cc
index 73010afc69..5f05c4333d 100644
--- a/compiler/optimizing/intrinsics_x86_64.cc
+++ b/compiler/optimizing/intrinsics_x86_64.cc
@@ -2712,6 +2712,60 @@ void IntrinsicCodeGeneratorX86_64::VisitReferenceGetReferent(HInvoke* invoke) {
   __ Bind(slow_path->GetExitLabel());
 }
 
+void IntrinsicLocationsBuilderX86_64::VisitReferenceRefersTo(HInvoke* invoke) {
+  IntrinsicVisitor::CreateReferenceRefersToLocations(invoke);
+}
+
+void IntrinsicCodeGeneratorX86_64::VisitReferenceRefersTo(HInvoke* invoke) {
+  X86_64Assembler* assembler = GetAssembler();
+  LocationSummary* locations = invoke->GetLocations();
+
+  CpuRegister obj = locations->InAt(0).AsRegister<CpuRegister>();
+  CpuRegister other = locations->InAt(1).AsRegister<CpuRegister>();
+  CpuRegister out = locations->Out().AsRegister<CpuRegister>();
+
+  uint32_t referent_offset = mirror::Reference::ReferentOffset().Uint32Value();
+  uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value();
+
+  __ movl(out, Address(obj, referent_offset));
+  codegen_->MaybeRecordImplicitNullCheck(invoke);
+  // Note that the fence is a no-op, thanks to the x86-64 memory model.
+  codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny);  // `referent` is volatile.
+
+  __ cmpl(out, other);
+
+  if (kEmitCompilerReadBarrier) {
+    DCHECK(kUseBakerReadBarrier);
+
+    NearLabel calculate_result;
+    __ j(kEqual, &calculate_result);  // ZF set if taken.
+
+    // Check if the loaded reference is null in a way that leaves ZF clear for null.
+    __ cmpl(out, Immediate(1));
+    __ j(kBelow, &calculate_result);  // ZF clear if taken.
+
+    // For correct memory visibility, we need a barrier before loading the lock word
+    // but we already have the barrier emitted for volatile load above which is sufficient.
+
+    // Load the lockword and check if it is a forwarding address.
+    static_assert(LockWord::kStateShift == 30u);
+    static_assert(LockWord::kStateForwardingAddress == 3u);
+    __ movl(out, Address(out, monitor_offset));
+    __ cmpl(out, Immediate(static_cast<int32_t>(0xc0000000)));
+    __ j(kBelow, &calculate_result);   // ZF clear if taken.
+
+    // Extract the forwarding address and compare with `other`.
+    __ shll(out, Immediate(LockWord::kForwardingAddressShift));
+    __ cmpl(out, other);
+
+    __ Bind(&calculate_result);
+  }
+
+  // Convert ZF into the Boolean result.
+  __ setcc(kEqual, out);
+  __ movzxb(out, out);
+}
+
 void IntrinsicLocationsBuilderX86_64::VisitThreadInterrupted(HInvoke* invoke) {
   LocationSummary* locations =
       new (allocator_) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified);
diff --git a/compiler/optimizing/nodes.cc b/compiler/optimizing/nodes.cc
index e815474b6e..4bb7ffc792 100644
--- a/compiler/optimizing/nodes.cc
+++ b/compiler/optimizing/nodes.cc
@@ -3199,6 +3199,19 @@ std::ostream& operator<<(std::ostream& os, HInvokeStaticOrDirect::ClinitCheckReq
   }
 }
 
+bool HInvokeVirtual::CanDoImplicitNullCheckOn(HInstruction* obj) const {
+  if (obj != InputAt(0)) {
+    return false;
+  }
+  switch (GetIntrinsic()) {
+    case Intrinsics::kReferenceRefersTo:
+      return true;
+    default:
+      // TODO: Add implicit null checks in more intrinsics.
+      return false;
+  }
+}
+
 bool HLoadClass::InstructionDataEquals(const HInstruction* other) const {
   const HLoadClass* other_load_class = other->AsLoadClass();
   // TODO: To allow GVN for HLoadClass from different dex files, we should compare the type
diff --git a/compiler/optimizing/nodes.h b/compiler/optimizing/nodes.h
index 6381b2c356..c47ca3bbb5 100644
--- a/compiler/optimizing/nodes.h
+++ b/compiler/optimizing/nodes.h
@@ -5112,10 +5112,7 @@ class HInvokeVirtual final : public HInvoke {
     }
   }
 
-  bool CanDoImplicitNullCheckOn(HInstruction* obj) const override {
-    // TODO: Add implicit null checks in intrinsics.
-    return (obj == InputAt(0)) && !IsIntrinsic();
-  }
+  bool CanDoImplicitNullCheckOn(HInstruction* obj) const override;
 
   uint32_t GetVTableIndex() const { return vtable_index_; }