riscv64: [codegen] Implement Baker read barriers.

Implement codegen changes and entrypoints for Baker read
barriers. Also implement resolution and initialization
entrypoints and enable codegen for certain instructions to
allow stress-testing the Baker read barrier implementation.

Fix `CodeGeneratorRISCV64::Finalize()` to avoid finalizing
the code twice. This double finaization bug was exposed by
enabling compilation of some larger methods.

Test: # Edit `run-test` to disable checker, then
      testrunner.py --target --64 --ndebug --optimizing
      # Ignore 7 pre-existing failures.
Bug: 283082089
Change-Id: I55a128921b388fae1bf818bfbda0bcb18f6dbfb3

5 files changed, 70 insertions, 47 deletions

diff --git a/compiler/optimizing/code_generator_riscv64.cc b/compiler/optimizing/code_generator_riscv64.cc
index e617b65907..7f23730143 100644
--- a/compiler/optimizing/code_generator_riscv64.cc
+++ b/compiler/optimizing/code_generator_riscv64.cc
@@ -136,6 +136,17 @@ static constexpr int64_t ShiftedSignExtendedClassStatusValue() {
   return static_cast<int64_t>(kShiftedStatusValue) - (INT64_C(1) << 32);
 }
 
+int32_t ReadBarrierMarkEntrypointOffset(Location ref) {
+  DCHECK(ref.IsRegister());
+  int reg = ref.reg();
+  DCHECK(T0 <= reg && reg <= T6 && reg != TR) << reg;
+  // Note: Entrypoints for registers X30 (T5) and X31 (T6) are stored in entries
+  // for X0 (Zero) and X1 (RA) because these are not valid registers for marking
+  // and we currently have slots only up to register 29.
+  int entry_point_number = (reg >= 30) ? reg - 30 : reg;
+  return Thread::ReadBarrierMarkEntryPointsOffset<kRiscv64PointerSize>(entry_point_number);
+}
+
 Location InvokeRuntimeCallingConvention::GetReturnLocation(DataType::Type return_type) {
   return Riscv64ReturnLocation(return_type);
 }
@@ -658,7 +669,7 @@ class ReadBarrierMarkSlowPathRISCV64 : public SlowPathCodeRISCV64 {
     //
     riscv64_codegen->ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction_, this);
     DCHECK_NE(entrypoint_.AsRegister<XRegister>(), TMP);  // A taken branch can clobber `TMP`.
-    __ Jalr(entrypoint_.AsRegister<XRegister>());
+    __ Jalr(entrypoint_.AsRegister<XRegister>());  // Clobbers `RA` (used as the `entrypoint_`).
     __ J(GetExitLabel());
   }
 
@@ -1178,16 +1189,13 @@ void InstructionCodeGeneratorRISCV64::GenerateGcRootFieldLoad(HInstruction* inst
                     "have different sizes.");
 
       // Slow path marking the GC root `root`.
-      ScratchRegisterScope srs(GetAssembler());
-      srs.ExcludeXRegister(TMP);  // A taken branch can clobber `TMP`.
-      XRegister tmp = srs.AllocateXRegister();
+      XRegister tmp = RA;  // Use RA as temp. It is clobbered in the slow path anyway.
       SlowPathCodeRISCV64* slow_path =
           new (codegen_->GetScopedAllocator()) ReadBarrierMarkSlowPathRISCV64(
               instruction, root, Location::RegisterLocation(tmp));
       codegen_->AddSlowPath(slow_path);
 
-      const int32_t entry_point_offset =
-          Thread::ReadBarrierMarkEntryPointsOffset<kRiscv64PointerSize>(root.reg());
+      const int32_t entry_point_offset = ReadBarrierMarkEntrypointOffset(root);
       // Loading the entrypoint does not require a load acquire since it is only changed when
       // threads are suspended or running a checkpoint.
       __ Loadd(tmp, TR, entry_point_offset);
@@ -1710,13 +1718,8 @@ void CodeGeneratorRISCV64::GenerateFieldLoadWithBakerReadBarrier(HInstruction* i
                                                                  uint32_t offset,
                                                                  Location temp,
                                                                  bool needs_null_check) {
-  UNUSED(instruction);
-  UNUSED(ref);
-  UNUSED(obj);
-  UNUSED(offset);
-  UNUSED(temp);
-  UNUSED(needs_null_check);
-  LOG(FATAL) << "Unimplemented";
+  GenerateReferenceLoadWithBakerReadBarrier(
+      instruction, ref, obj, offset, /*index=*/ Location::NoLocation(), temp, needs_null_check);
 }
 
 void CodeGeneratorRISCV64::GenerateArrayLoadWithBakerReadBarrier(HInstruction* instruction,
@@ -1726,14 +1729,8 @@ void CodeGeneratorRISCV64::GenerateArrayLoadWithBakerReadBarrier(HInstruction* i
                                                                  Location index,
                                                                  Location temp,
                                                                  bool needs_null_check) {
-  UNUSED(instruction);
-  UNUSED(ref);
-  UNUSED(obj);
-  UNUSED(data_offset);
-  UNUSED(index);
-  UNUSED(temp);
-  UNUSED(needs_null_check);
-  LOG(FATAL) << "Unimplemented";
+  GenerateReferenceLoadWithBakerReadBarrier(
+      instruction, ref, obj, data_offset, index, temp, needs_null_check);
 }
 
 void CodeGeneratorRISCV64::GenerateReferenceLoadWithBakerReadBarrier(HInstruction* instruction,
@@ -1741,20 +1738,43 @@ void CodeGeneratorRISCV64::GenerateReferenceLoadWithBakerReadBarrier(HInstructio
                                                                      XRegister obj,
                                                                      uint32_t offset,
                                                                      Location index,
-                                                                     ScaleFactor scale_factor,
                                                                      Location temp,
-                                                                     bool needs_null_check,
-                                                                     bool always_update_field) {
-  UNUSED(instruction);
-  UNUSED(ref);
-  UNUSED(obj);
-  UNUSED(offset);
-  UNUSED(index);
-  UNUSED(scale_factor);
+                                                                     bool needs_null_check) {
+  // For now, use the same approach as for GC roots plus unpoison the reference if needed.
+  // TODO(riscv64): Implement checking if the holder is black.
   UNUSED(temp);
-  UNUSED(needs_null_check);
-  UNUSED(always_update_field);
-  LOG(FATAL) << "Unimplemented";
+
+  XRegister reg = ref.AsRegister<XRegister>();
+  if (index.IsValid()) {
+    DCHECK(instruction->IsArrayGet());
+    DCHECK(!needs_null_check);
+    DCHECK(index.IsRegister());
+    // /* HeapReference<Object> */ ref = *(obj + index * element_size + offset)
+    DataType::Type type = DataType::Type::kReference;
+    DCHECK_EQ(type, instruction->GetType());
+    instruction_visitor_.ShNAdd(reg, index.AsRegister<XRegister>(), obj, type);
+    __ Loadwu(reg, reg, offset);
+  } else {
+    // /* HeapReference<Object> */ ref = *(obj + offset)
+    __ Loadwu(reg, obj, offset);
+    if (needs_null_check) {
+      MaybeRecordImplicitNullCheck(instruction);
+    }
+  }
+  MaybeUnpoisonHeapReference(reg);
+
+  // Slow path marking the reference.
+  XRegister tmp = RA;  // Use RA as temp. It is clobbered in the slow path anyway.
+  SlowPathCodeRISCV64* slow_path = new (GetScopedAllocator()) ReadBarrierMarkSlowPathRISCV64(
+      instruction, ref, Location::RegisterLocation(tmp));
+  AddSlowPath(slow_path);
+
+  const int32_t entry_point_offset = ReadBarrierMarkEntrypointOffset(ref);
+  // Loading the entrypoint does not require a load acquire since it is only changed when
+  // threads are suspended or running a checkpoint.
+  __ Loadd(tmp, TR, entry_point_offset);
+  __ Bnez(tmp, slow_path->GetEntryLabel());
+  __ Bind(slow_path->GetExitLabel());
 }
 
 void CodeGeneratorRISCV64::GenerateReadBarrierSlow(HInstruction* instruction,
@@ -5837,8 +5857,6 @@ void CodeGeneratorRISCV64::Finalize() {
       entry.code_interval.end = __ GetAdjustedPosition(entry.code_interval.end);
     }
   }
-
-  CodeGenerator::Finalize();
 }
 
 // Generate code to invoke a runtime entry point.
diff --git a/compiler/optimizing/code_generator_riscv64.h b/compiler/optimizing/code_generator_riscv64.h
index 08df77e6aa..375cec957f 100644
--- a/compiler/optimizing/code_generator_riscv64.h
+++ b/compiler/optimizing/code_generator_riscv64.h
@@ -406,6 +406,8 @@ class InstructionCodeGeneratorRISCV64 : public InstructionCodeGenerator {
 
   void GenerateMemoryBarrier(MemBarrierKind kind);
 
+  void ShNAdd(XRegister rd, XRegister rs1, XRegister rs2, DataType::Type type);
+
  protected:
   void GenerateClassInitializationCheck(SlowPathCodeRISCV64* slow_path, XRegister class_reg);
   void GenerateBitstringTypeCheckCompare(HTypeCheckInstruction* check, XRegister temp);
@@ -519,8 +521,6 @@ class InstructionCodeGeneratorRISCV64 : public InstructionCodeGenerator {
   void Load(Location out, XRegister rs1, int32_t offset, DataType::Type type);
   void Store(Location value, XRegister rs1, int32_t offset, DataType::Type type);
 
-  void ShNAdd(XRegister rd, XRegister rs1, XRegister rs2, DataType::Type type);
-
   Riscv64Assembler* const assembler_;
   CodeGeneratorRISCV64* const codegen_;
 
@@ -759,22 +759,13 @@ class CodeGeneratorRISCV64 : public CodeGenerator {
                                              bool needs_null_check);
   // Factored implementation, used by GenerateFieldLoadWithBakerReadBarrier
   // and GenerateArrayLoadWithBakerReadBarrier.
-  //
-  // Load the object reference located at the address
-  // `obj + offset + (index << scale_factor)`, held by object `obj`, into
-  // `ref`, and mark it if needed.
-  //
-  // If `always_update_field` is true, the value of the reference is
-  // atomically updated in the holder (`obj`).
   void GenerateReferenceLoadWithBakerReadBarrier(HInstruction* instruction,
                                                  Location ref,
                                                  XRegister obj,
                                                  uint32_t offset,
                                                  Location index,
-                                                 ScaleFactor scale_factor,
                                                  Location temp,
-                                                 bool needs_null_check,
-                                                 bool always_update_field = false);
+                                                 bool needs_null_check);
 
   // Generate a read barrier for a heap reference within `instruction`
   // using a slow path.
diff --git a/compiler/optimizing/optimizing_compiler.cc b/compiler/optimizing/optimizing_compiler.cc
index cca11e0c7e..040c2449a7 100644
--- a/compiler/optimizing/optimizing_compiler.cc
+++ b/compiler/optimizing/optimizing_compiler.cc
@@ -751,6 +751,14 @@ static bool CanAssembleGraphForRiscv64(HGraph* graph) {
         case HInstruction::kFloatConstant:
         case HInstruction::kIntConstant:
         case HInstruction::kLongConstant:
+        case HInstruction::kNullConstant:
+        case HInstruction::kLoadClass:
+        case HInstruction::kLoadString:
+        case HInstruction::kLoadMethodHandle:
+        case HInstruction::kLoadMethodType:
+        case HInstruction::kInstanceFieldGet:
+        case HInstruction::kStaticFieldGet:
+        case HInstruction::kArrayGet:
         case HInstruction::kAbove:
         case HInstruction::kAboveOrEqual:
         case HInstruction::kBelow:
diff --git a/compiler/utils/riscv64/assembler_riscv64.cc b/compiler/utils/riscv64/assembler_riscv64.cc
index fd714c7594..089bc5dfe6 100644
--- a/compiler/utils/riscv64/assembler_riscv64.cc
+++ b/compiler/utils/riscv64/assembler_riscv64.cc
@@ -50,6 +50,7 @@ ALWAYS_INLINE static inline int32_t ToInt12(uint32_t uint12) {
 }
 
 void Riscv64Assembler::FinalizeCode() {
+  CHECK(!finalized_);
   Assembler::FinalizeCode();
   ReserveJumpTableSpace();
   EmitLiterals();
@@ -57,6 +58,7 @@ void Riscv64Assembler::FinalizeCode() {
   EmitBranches();
   EmitJumpTables();
   PatchCFI();
+  finalized_ = true;
 }
 
 void Riscv64Assembler::Emit(uint32_t value) {
diff --git a/compiler/utils/riscv64/assembler_riscv64.h b/compiler/utils/riscv64/assembler_riscv64.h
index d7583459b4..15f2518c87 100644
--- a/compiler/utils/riscv64/assembler_riscv64.h
+++ b/compiler/utils/riscv64/assembler_riscv64.h
@@ -163,6 +163,7 @@ class Riscv64Assembler final : public Assembler {
                             const Riscv64InstructionSetFeatures* instruction_set_features = nullptr)
       : Assembler(allocator),
         branches_(allocator->Adapter(kArenaAllocAssembler)),
+        finalized_(false),
         overwriting_(false),
         overwrite_location_(0),
         literals_(allocator->Adapter(kArenaAllocAssembler)),
@@ -1042,6 +1043,9 @@ class Riscv64Assembler final : public Assembler {
 
   ArenaVector<Branch> branches_;
 
+  // For checking that we finalize the code only once.
+  bool finalized_;
+
   // Whether appending instructions at the end of the buffer or overwriting the existing ones.
   bool overwriting_;
   // The current overwrite location.