X86: Implement VarHandle.getAndAdd intrinsic

This commit implements VarHandle getAndAdd intrinsic. This also implied
adding xadd instruction and tests for it.

Test: art/test.py --host -r -t 712-varhandle-invocation --32
Test: m test-art-host-gtest
Bug: 65872996
Change-Id: I84dd95ba6464c8a73ace03a13817147c7099677a

1 files changed, 168 insertions, 13 deletions

diff --git a/compiler/optimizing/intrinsics_x86.cc b/compiler/optimizing/intrinsics_x86.cc
index 823b011a4f..4e24aab1ad 100644
--- a/compiler/optimizing/intrinsics_x86.cc
+++ b/compiler/optimizing/intrinsics_x86.cc
@@ -1962,18 +1962,16 @@ void IntrinsicLocationsBuilderX86::VisitUnsafeCASObject(HInvoke* invoke) {
   CreateIntIntIntIntIntToInt(allocator_, DataType::Type::kReference, invoke);
 }
 
-static void GenPrimitiveCAS(DataType::Type type,
-                            CodeGeneratorX86* codegen,
-                            Location expected_value,
-                            Location new_value,
-                            Register base,
-                            Register offset,
-                            Location out,
-                            // Only necessary for floating point
-                            Register temp = Register::kNoRegister) {
+static void GenPrimitiveLockedCmpxchg(DataType::Type type,
+                                      CodeGeneratorX86* codegen,
+                                      Location expected_value,
+                                      Location new_value,
+                                      Register base,
+                                      Register offset,
+                                      // Only necessary for floating point
+                                      Register temp = Register::kNoRegister) {
   X86Assembler* assembler = down_cast<X86Assembler*>(codegen->GetAssembler());
 
-  DCHECK_EQ(out.AsRegister<Register>(), EAX);
   if (DataType::Kind(type) == DataType::Type::kInt32) {
     DCHECK_EQ(expected_value.AsRegister<Register>(), EAX);
   }
@@ -2016,6 +2014,21 @@ static void GenPrimitiveCAS(DataType::Type type,
   }
   // LOCK CMPXCHG/LOCK CMPXCHG8B have full barrier semantics, and we
   // don't need scheduling barriers at this time.
+}
+
+static void GenPrimitiveCAS(DataType::Type type,
+                            CodeGeneratorX86* codegen,
+                            Location expected_value,
+                            Location new_value,
+                            Register base,
+                            Register offset,
+                            Location out,
+                            // Only necessary for floating point
+                            Register temp = Register::kNoRegister) {
+  X86Assembler* assembler = down_cast<X86Assembler*>(codegen->GetAssembler());
+  DCHECK_EQ(out.AsRegister<Register>(), EAX);
+
+  GenPrimitiveLockedCmpxchg(type, codegen, expected_value, new_value, base, offset, temp);
 
   // Convert ZF into the Boolean result.
   __ setb(kZero, out.AsRegister<Register>());
@@ -3133,6 +3146,7 @@ static bool IsValidFieldVarHandleExpected(HInvoke* invoke) {
     return false;
   }
 
+  uint32_t number_of_arguments = invoke->GetNumberOfArguments();
   DataType::Type type = invoke->GetType();
   mirror::VarHandle::AccessModeTemplate access_mode_template =
       mirror::VarHandle::GetAccessModeTemplateByIntrinsic(invoke->GetIntrinsic());
@@ -3146,7 +3160,6 @@ static bool IsValidFieldVarHandleExpected(HInvoke* invoke) {
       if (type != DataType::Type::kBool) {
         return false;
       }
-      uint32_t number_of_arguments = invoke->GetNumberOfArguments();
       uint32_t expected_value_index = number_of_arguments - 2;
       uint32_t new_value_index = number_of_arguments - 1;
       DataType::Type expected_value_type = GetDataTypeFromShorty(invoke, expected_value_index);
@@ -3157,13 +3170,25 @@ static bool IsValidFieldVarHandleExpected(HInvoke* invoke) {
       }
       break;
     }
+    case mirror::VarHandle::AccessModeTemplate::kGetAndUpdate: {
+      DataType::Type value_type = GetDataTypeFromShorty(invoke, number_of_arguments - 1);
+      if (invoke->GetIntrinsic() == Intrinsics::kVarHandleGetAndAdd) {
+        if (value_type == DataType::Type::kReference || value_type == DataType::Type::kVoid) {
+          // We should not add references
+          return false;
+        }
+      }
+      if (value_type != type) {
+        return false;
+      }
+      break;
+    }
     case mirror::VarHandle::AccessModeTemplate::kGet:
       // The return type should be the same as varType, so it shouldn't be void
       if (type == DataType::Type::kVoid) {
         return false;
       }
       break;
-    case mirror::VarHandle::AccessModeTemplate::kGetAndUpdate:
     case mirror::VarHandle::AccessModeTemplate::kCompareAndExchange:
       // Unimplemented intrinsics
       UNREACHABLE();
@@ -3776,6 +3801,137 @@ void IntrinsicCodeGeneratorX86::VisitVarHandleWeakCompareAndSetRelease(HInvoke*
   GenerateVarHandleCompareAndSet(invoke, codegen_);
 }
 
+void IntrinsicLocationsBuilderX86::VisitVarHandleGetAndAdd(HInvoke* invoke) {
+  // The only read barrier implementation supporting the
+  // VarHandleGet intrinsic is the Baker-style read barriers.
+  if (kEmitCompilerReadBarrier && !kUseBakerReadBarrier) {
+    return;
+  }
+
+  if (!IsValidFieldVarHandleExpected(invoke)) {
+    return;
+  }
+
+  // The last argument should be the value we intend to set.
+  uint32_t value_index = invoke->GetNumberOfArguments() - 1;
+  DataType::Type value_type = GetDataTypeFromShorty(invoke, value_index);
+  if (DataType::Is64BitType(value_type)) {
+    // We avoid the case of an Int64/Float64 value because we would need to place it in a register
+    // pair. If the slow path is taken, the ParallelMove might fail to move the pair according to
+    // the X86DexCallingConvention in case of an overlap (e.g., move the 64 bit value from
+    // <EAX, EBX> to <EBX, ECX>). (Bug: b/168687887)
+    return;
+  }
+
+  ArenaAllocator* allocator = invoke->GetBlock()->GetGraph()->GetAllocator();
+  LocationSummary* locations = new (allocator) LocationSummary(
+      invoke, LocationSummary::kCallOnSlowPath, kIntrinsified);
+  locations->AddTemp(Location::RequiresRegister());
+  locations->AddTemp(Location::RequiresRegister());
+  locations->SetInAt(0, Location::RequiresRegister());
+  size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke);
+  if (expected_coordinates_count == 1u) {
+    // For instance fields, this is the source object
+    locations->SetInAt(1, Location::RequiresRegister());
+  } else {
+    // For static fields, we need another temp because one will be busy with the declaring class.
+    locations->AddTemp(Location::RequiresRegister());
+  }
+
+  if (DataType::IsFloatingPointType(value_type)) {
+    locations->AddTemp(Location::RequiresFpuRegister());
+    locations->AddTemp(Location::RegisterLocation(EAX));
+    locations->SetInAt(value_index, Location::RequiresFpuRegister());
+    locations->SetOut(Location::RequiresFpuRegister());
+  } else {
+    // xadd updates the register argument with the old value. ByteRegister required for xaddb.
+    locations->SetInAt(value_index, Location::RegisterLocation(EAX));
+    locations->SetOut(Location::RegisterLocation(EAX));
+  }
+}
+
+void IntrinsicCodeGeneratorX86::VisitVarHandleGetAndAdd(HInvoke* invoke) {
+  // The only read barrier implementation supporting the
+  // VarHandleGet intrinsic is the Baker-style read barriers.
+  DCHECK(!kEmitCompilerReadBarrier || kUseBakerReadBarrier);
+
+  CodeGeneratorX86* codegen = down_cast<CodeGeneratorX86*>(codegen_);
+  X86Assembler* assembler = codegen->GetAssembler();
+  LocationSummary* locations = invoke->GetLocations();
+  uint32_t number_of_arguments = invoke->GetNumberOfArguments();
+  uint32_t value_index = number_of_arguments - 1;
+  DataType::Type type = GetDataTypeFromShorty(invoke, value_index);
+  DCHECK_EQ(type, invoke->GetType());
+  Location value_loc = locations->InAt(value_index);
+  Register vh_object = locations->InAt(0).AsRegister<Register>();
+  Register temp = locations->GetTemp(0).AsRegister<Register>();
+  SlowPathCode* slow_path = new (codegen->GetScopedAllocator()) IntrinsicSlowPathX86(invoke);
+  codegen->AddSlowPath(slow_path);
+
+  GenerateVarHandleCommonChecks(invoke, temp, slow_path, assembler);
+
+  GenerateVarTypePrimitiveTypeCheck(vh_object, temp, type, slow_path, assembler);
+
+  Register offset = locations->GetTemp(1).AsRegister<Register>();
+  // Get the field referred by the VarHandle. The returned register contains the object reference
+  // or the declaring class. The field offset will be placed in 'offset'. For static fields, the
+  // declaring class will be placed in 'temp' register.
+  Register reference = GenerateVarHandleFieldReference(invoke, codegen, temp, offset);
+
+  size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke);
+  temp = (expected_coordinates_count == 1u) ? temp : locations->GetTemp(2).AsRegister<Register>();
+  DCHECK_NE(temp, reference);
+  Address field_addr(reference, offset, TIMES_1, 0);
+
+  switch (type) {
+    case DataType::Type::kInt8:
+      __ LockXaddb(field_addr, value_loc.AsRegister<ByteRegister>());
+      __ movsxb(locations->Out().AsRegister<Register>(),
+                locations->Out().AsRegister<ByteRegister>());
+      break;
+    case DataType::Type::kInt16:
+      __ LockXaddw(field_addr, value_loc.AsRegister<Register>());
+      __ movsxw(locations->Out().AsRegister<Register>(), locations->Out().AsRegister<Register>());
+      break;
+    case DataType::Type::kUint16:
+      __ LockXaddw(field_addr, value_loc.AsRegister<Register>());
+      __ movzxw(locations->Out().AsRegister<Register>(), locations->Out().AsRegister<Register>());
+      break;
+    case DataType::Type::kInt32:
+      __ LockXaddl(field_addr, value_loc.AsRegister<Register>());
+      break;
+    case DataType::Type::kFloat32: {
+      Location temp_float =
+          (expected_coordinates_count == 1u) ? locations->GetTemp(2) : locations->GetTemp(3);
+      DCHECK(temp_float.IsFpuRegister());
+      Location eax = Location::RegisterLocation(EAX);
+      NearLabel try_again;
+      __ Bind(&try_again);
+      codegen->MoveFromMemory(type, temp_float, reference, offset);
+      __ movd(EAX, temp_float.AsFpuRegister<XmmRegister>());
+      __ addss(temp_float.AsFpuRegister<XmmRegister>(),
+              value_loc.AsFpuRegister<XmmRegister>());
+      GenPrimitiveLockedCmpxchg(type,
+                                codegen,
+                                /* expected= */ eax,
+                                /* new_value= */ temp_float,
+                                reference,
+                                offset,
+                                temp);
+      __ j(kNotZero, &try_again);
+
+      // The old value is present in EAX.
+      codegen->Move32(locations->Out(), eax);
+      break;
+    }
+    default:
+      UNREACHABLE();
+  }
+
+  __ Bind(slow_path->GetExitLabel());
+}
+
+
 UNIMPLEMENTED_INTRINSIC(X86, MathRoundDouble)
 UNIMPLEMENTED_INTRINSIC(X86, ReferenceGetReferent)
 UNIMPLEMENTED_INTRINSIC(X86, FloatIsInfinite)
@@ -3831,7 +3987,6 @@ UNIMPLEMENTED_INTRINSIC(X86, VarHandleCompareAndExchange)
 UNIMPLEMENTED_INTRINSIC(X86, VarHandleCompareAndExchangeAcquire)
 UNIMPLEMENTED_INTRINSIC(X86, VarHandleCompareAndExchangeRelease)
 UNIMPLEMENTED_INTRINSIC(X86, VarHandleGetAcquire)
-UNIMPLEMENTED_INTRINSIC(X86, VarHandleGetAndAdd)
 UNIMPLEMENTED_INTRINSIC(X86, VarHandleGetAndAddAcquire)
 UNIMPLEMENTED_INTRINSIC(X86, VarHandleGetAndAddRelease)
 UNIMPLEMENTED_INTRINSIC(X86, VarHandleGetAndBitwiseAnd)