arm64: VarHandle.{get,set}{Opaque,Acquire,Volatile}.

Extend the VarHandle.{get,set} for fields to similar
variants with additional memory ordering requirements.

Test: testrunner.py --target --64 -t 712-varhandle-invocations
Test: Repeat with ART_USE_READ_BARRIER=false ART_HEAP_POISONING=true.
Test: Repeat with ART_READ_BARRIER_TYPE=TABLELOOKUP.
Bug: 65872996
Change-Id: I400802a1e089a5a81149316f88bb90979f8988e6
diff --git a/compiler/optimizing/code_generator_arm64.cc b/compiler/optimizing/code_generator_arm64.cc
index a5d2060..36040ca 100644
--- a/compiler/optimizing/code_generator_arm64.cc
+++ b/compiler/optimizing/code_generator_arm64.cc
@@ -37,6 +37,7 @@
 #include "lock_word.h"
 #include "mirror/array-inl.h"
 #include "mirror/class-inl.h"
+#include "mirror/var_handle.h"
 #include "offsets.h"
 #include "thread.h"
 #include "utils/arm64/assembler_arm64.h"
@@ -719,9 +720,11 @@
         // to an object field within an object.
         DCHECK(instruction_->IsInvoke()) << instruction_->DebugName();
         DCHECK(instruction_->GetLocations()->Intrinsified());
-        DCHECK((instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObject) ||
-               (instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile ||
-               (instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kVarHandleGet)))
+        Intrinsics intrinsic = instruction_->AsInvoke()->GetIntrinsic();
+        DCHECK(intrinsic == Intrinsics::kUnsafeGetObject ||
+               intrinsic == Intrinsics::kUnsafeGetObjectVolatile ||
+               mirror::VarHandle::GetAccessModeTemplateByIntrinsic(intrinsic) ==
+                   mirror::VarHandle::AccessModeTemplate::kGet)
             << instruction_->AsInvoke()->GetIntrinsic();
         DCHECK_EQ(offset_, 0u);
         DCHECK(index_.IsRegister());
diff --git a/compiler/optimizing/intrinsics_arm64.cc b/compiler/optimizing/intrinsics_arm64.cc
index a547551..c38f5d6 100644
--- a/compiler/optimizing/intrinsics_arm64.cc
+++ b/compiler/optimizing/intrinsics_arm64.cc
@@ -3540,7 +3540,7 @@
   }
 }
 
-void IntrinsicLocationsBuilderARM64::VisitVarHandleGet(HInvoke* invoke) {
+static void CreateVarHandleGetLocations(HInvoke* invoke) {
   DataType::Type type = invoke->GetType();
   if (type == DataType::Type::kVoid) {
     // Return type should not be void for get.
@@ -3584,7 +3584,9 @@
   }
 }
 
-void IntrinsicCodeGeneratorARM64::VisitVarHandleGet(HInvoke* invoke) {
+static void GenerateVarHandleGet(HInvoke* invoke,
+                                 CodeGeneratorARM64* codegen,
+                                 bool use_load_acquire) {
   // Implemented only for fields.
   size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke);
   DCHECK_LE(expected_coordinates_count, 1u);
@@ -3592,15 +3594,15 @@
   DCHECK_NE(type, DataType::Type::kVoid);
 
   LocationSummary* locations = invoke->GetLocations();
-  MacroAssembler* masm = GetVIXLAssembler();
+  MacroAssembler* masm = codegen->GetVIXLAssembler();
   CPURegister out = helpers::OutputCPURegister(invoke);
 
   SlowPathCodeARM64* slow_path =
-      new (codegen_->GetScopedAllocator()) IntrinsicSlowPathARM64(invoke);
-  codegen_->AddSlowPath(slow_path);
+      new (codegen->GetScopedAllocator()) IntrinsicSlowPathARM64(invoke);
+  codegen->AddSlowPath(slow_path);
 
-  GenerateVarHandleFieldCheck(invoke, codegen_, slow_path);
-  GenerateVarHandleAccessModeAndVarTypeChecks(invoke, codegen_, slow_path, type);
+  GenerateVarHandleFieldCheck(invoke, codegen, slow_path);
+  GenerateVarHandleAccessModeAndVarTypeChecks(invoke, codegen, slow_path, type);
 
   // Use `out` for offset if it is a core register, except for non-Baker read barrier.
   Register offset =
@@ -3612,34 +3614,72 @@
       ? WRegisterFrom(locations->GetTemp(0u))
       : InputRegisterAt(invoke, 1);
 
-  GenerateVarHandleFieldReference(invoke, codegen_, object, offset);
+  GenerateVarHandleFieldReference(invoke, codegen, object, offset);
 
   // Load the value from the field.
   if (type == DataType::Type::kReference && kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
     // Piggy-back on the field load path using introspection for the Baker read barrier.
     // The `offset` is either the `out` or a temporary, use it for field address.
     __ Add(offset.X(), object.X(), offset.X());
-    codegen_->GenerateFieldLoadWithBakerReadBarrier(invoke,
-                                                    locations->Out(),
-                                                    object,
-                                                    MemOperand(offset.X()),
-                                                    /*needs_null_check=*/ false,
-                                                    /*use_load_acquire=*/ false);
+    codegen->GenerateFieldLoadWithBakerReadBarrier(invoke,
+                                                   locations->Out(),
+                                                   object,
+                                                   MemOperand(offset.X()),
+                                                   /*needs_null_check=*/ false,
+                                                   use_load_acquire);
   } else {
-    codegen_->Load(type, out, MemOperand(object.X(), offset.X()));
+    MemOperand address(object.X(), offset.X());
+    if (use_load_acquire) {
+      codegen->LoadAcquire(invoke, out, address, /*needs_null_check=*/ false);
+    } else {
+      codegen->Load(type, out, address);
+    }
     if (type == DataType::Type::kReference) {
       DCHECK(out.IsW());
       Location out_loc = locations->Out();
       Location object_loc = LocationFrom(object);
       Location offset_loc = LocationFrom(offset);
-      codegen_->MaybeGenerateReadBarrierSlow(invoke, out_loc, out_loc, object_loc, 0u, offset_loc);
+      codegen->MaybeGenerateReadBarrierSlow(invoke, out_loc, out_loc, object_loc, 0u, offset_loc);
     }
   }
 
   __ Bind(slow_path->GetExitLabel());
 }
 
-void IntrinsicLocationsBuilderARM64::VisitVarHandleSet(HInvoke* invoke) {
+void IntrinsicLocationsBuilderARM64::VisitVarHandleGet(HInvoke* invoke) {
+  CreateVarHandleGetLocations(invoke);
+}
+
+void IntrinsicCodeGeneratorARM64::VisitVarHandleGet(HInvoke* invoke) {
+  GenerateVarHandleGet(invoke, codegen_, /*use_load_acquire=*/ false);
+}
+
+void IntrinsicLocationsBuilderARM64::VisitVarHandleGetOpaque(HInvoke* invoke) {
+  CreateVarHandleGetLocations(invoke);
+}
+
+void IntrinsicCodeGeneratorARM64::VisitVarHandleGetOpaque(HInvoke* invoke) {
+  GenerateVarHandleGet(invoke, codegen_, /*use_load_acquire=*/ false);
+}
+
+void IntrinsicLocationsBuilderARM64::VisitVarHandleGetAcquire(HInvoke* invoke) {
+  CreateVarHandleGetLocations(invoke);
+}
+
+void IntrinsicCodeGeneratorARM64::VisitVarHandleGetAcquire(HInvoke* invoke) {
+  GenerateVarHandleGet(invoke, codegen_, /*use_load_acquire=*/ true);
+}
+
+void IntrinsicLocationsBuilderARM64::VisitVarHandleGetVolatile(HInvoke* invoke) {
+  CreateVarHandleGetLocations(invoke);
+}
+
+void IntrinsicCodeGeneratorARM64::VisitVarHandleGetVolatile(HInvoke* invoke) {
+  // ARM64 load-acquire instructions are implicitly sequentially consistent.
+  GenerateVarHandleGet(invoke, codegen_, /*use_load_acquire=*/ true);
+}
+
+static void CreateVarHandleSetLocations(HInvoke* invoke) {
   if (invoke->GetType() != DataType::Type::kVoid) {
     // Return type should be void for set.
     return;
@@ -3688,31 +3728,33 @@
   }
 }
 
-void IntrinsicCodeGeneratorARM64::VisitVarHandleSet(HInvoke* invoke) {
+static void GenerateVarHandleSet(HInvoke* invoke,
+                                 CodeGeneratorARM64* codegen,
+                                 bool use_store_release) {
   // Implemented only for fields.
   size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke);
   DCHECK_LE(expected_coordinates_count, 1u);
   uint32_t value_index = invoke->GetNumberOfArguments() - 1;
   DataType::Type value_type = GetDataTypeFromShorty(invoke, value_index);
 
-  MacroAssembler* masm = GetVIXLAssembler();
+  MacroAssembler* masm = codegen->GetVIXLAssembler();
   CPURegister value = InputCPURegisterOrZeroRegAt(invoke, value_index);
 
   SlowPathCodeARM64* slow_path =
-      new (codegen_->GetScopedAllocator()) IntrinsicSlowPathARM64(invoke);
-  codegen_->AddSlowPath(slow_path);
+      new (codegen->GetScopedAllocator()) IntrinsicSlowPathARM64(invoke);
+  codegen->AddSlowPath(slow_path);
 
-  GenerateVarHandleFieldCheck(invoke, codegen_, slow_path);
+  GenerateVarHandleFieldCheck(invoke, codegen, slow_path);
   {
     UseScratchRegisterScope temps(masm);
     Register var_type_no_rb = temps.AcquireW();
     GenerateVarHandleAccessModeAndVarTypeChecks(
-        invoke, codegen_, slow_path, value_type, var_type_no_rb);
+        invoke, codegen, slow_path, value_type, var_type_no_rb);
     if (value_type == DataType::Type::kReference && !value.IsZero()) {
       // If the value type is a reference, check it against the varType.
       // False negatives due to varType being an interface or array type
       // or due to the missing read barrier are handled by the slow path.
-      GenerateSubTypeObjectCheckNoReadBarrier(codegen_, slow_path, value.W(), var_type_no_rb);
+      GenerateSubTypeObjectCheckNoReadBarrier(codegen, slow_path, value.W(), var_type_no_rb);
     }
   }
 
@@ -3723,7 +3765,7 @@
       ? WRegisterFrom(invoke->GetLocations()->GetTemp(1u))
       : InputRegisterAt(invoke, 1);
 
-  GenerateVarHandleFieldReference(invoke, codegen_, object, offset);
+  GenerateVarHandleFieldReference(invoke, codegen, object, offset);
 
   // Store the value to the field.
   {
@@ -3733,19 +3775,57 @@
       DCHECK(value.IsW());
       Register temp = temps.AcquireW();
       __ Mov(temp, value.W());
-      codegen_->GetAssembler()->PoisonHeapReference(temp);
+      codegen->GetAssembler()->PoisonHeapReference(temp);
       source = temp;
     }
-    codegen_->Store(value_type, source, MemOperand(object.X(), offset.X()));
+    MemOperand address(object.X(), offset.X());
+    if (use_store_release) {
+      codegen->StoreRelease(invoke, value_type, source, address, /*needs_null_check=*/ false);
+    } else {
+      codegen->Store(value_type, source, address);
+    }
   }
 
   if (CodeGenerator::StoreNeedsWriteBarrier(value_type, invoke->InputAt(value_index))) {
-    codegen_->MarkGCCard(object, Register(value), /*value_can_be_null=*/ true);
+    codegen->MarkGCCard(object, Register(value), /*value_can_be_null=*/ true);
   }
 
   __ Bind(slow_path->GetExitLabel());
 }
 
+void IntrinsicLocationsBuilderARM64::VisitVarHandleSet(HInvoke* invoke) {
+  CreateVarHandleSetLocations(invoke);
+}
+
+void IntrinsicCodeGeneratorARM64::VisitVarHandleSet(HInvoke* invoke) {
+  GenerateVarHandleSet(invoke, codegen_, /*use_store_release=*/ false);
+}
+
+void IntrinsicLocationsBuilderARM64::VisitVarHandleSetOpaque(HInvoke* invoke) {
+  CreateVarHandleSetLocations(invoke);
+}
+
+void IntrinsicCodeGeneratorARM64::VisitVarHandleSetOpaque(HInvoke* invoke) {
+  GenerateVarHandleSet(invoke, codegen_, /*use_store_release=*/ false);
+}
+
+void IntrinsicLocationsBuilderARM64::VisitVarHandleSetRelease(HInvoke* invoke) {
+  CreateVarHandleSetLocations(invoke);
+}
+
+void IntrinsicCodeGeneratorARM64::VisitVarHandleSetRelease(HInvoke* invoke) {
+  GenerateVarHandleSet(invoke, codegen_, /*use_store_release=*/ true);
+}
+
+void IntrinsicLocationsBuilderARM64::VisitVarHandleSetVolatile(HInvoke* invoke) {
+  CreateVarHandleSetLocations(invoke);
+}
+
+void IntrinsicCodeGeneratorARM64::VisitVarHandleSetVolatile(HInvoke* invoke) {
+  // ARM64 store-release instructions are implicitly sequentially consistent.
+  GenerateVarHandleSet(invoke, codegen_, /*use_store_release=*/ true);
+}
+
 UNIMPLEMENTED_INTRINSIC(ARM64, ReferenceGetReferent)
 
 UNIMPLEMENTED_INTRINSIC(ARM64, StringStringIndexOf);
@@ -3784,7 +3864,6 @@
 UNIMPLEMENTED_INTRINSIC(ARM64, VarHandleCompareAndExchangeAcquire)
 UNIMPLEMENTED_INTRINSIC(ARM64, VarHandleCompareAndExchangeRelease)
 UNIMPLEMENTED_INTRINSIC(ARM64, VarHandleCompareAndSet)
-UNIMPLEMENTED_INTRINSIC(ARM64, VarHandleGetAcquire)
 UNIMPLEMENTED_INTRINSIC(ARM64, VarHandleGetAndAdd)
 UNIMPLEMENTED_INTRINSIC(ARM64, VarHandleGetAndAddAcquire)
 UNIMPLEMENTED_INTRINSIC(ARM64, VarHandleGetAndAddRelease)
@@ -3800,11 +3879,6 @@
 UNIMPLEMENTED_INTRINSIC(ARM64, VarHandleGetAndSet)
 UNIMPLEMENTED_INTRINSIC(ARM64, VarHandleGetAndSetAcquire)
 UNIMPLEMENTED_INTRINSIC(ARM64, VarHandleGetAndSetRelease)
-UNIMPLEMENTED_INTRINSIC(ARM64, VarHandleGetOpaque)
-UNIMPLEMENTED_INTRINSIC(ARM64, VarHandleGetVolatile)
-UNIMPLEMENTED_INTRINSIC(ARM64, VarHandleSetOpaque)
-UNIMPLEMENTED_INTRINSIC(ARM64, VarHandleSetRelease)
-UNIMPLEMENTED_INTRINSIC(ARM64, VarHandleSetVolatile)
 UNIMPLEMENTED_INTRINSIC(ARM64, VarHandleWeakCompareAndSet)
 UNIMPLEMENTED_INTRINSIC(ARM64, VarHandleWeakCompareAndSetAcquire)
 UNIMPLEMENTED_INTRINSIC(ARM64, VarHandleWeakCompareAndSetPlain)