riscv64: Implement JNI compiler for @FastNative.

Implement all JNI macro assembler functions needed by the
JNI compiler to compile stubs for @FastNative methods.
Enable JNI compiler tests for @FastNative methods.

Test: m test-art-host-gtest
Test: run-gtests.sh
Bug: 283082089
Change-Id: I34ccc3a89de4833a18e2140b656e251a354fec01

5 files changed, 209 insertions, 29 deletions

diff --git a/compiler/jni/jni_compiler_test.cc b/compiler/jni/jni_compiler_test.cc
index 93dd715286..ec68ff2f4d 100644
--- a/compiler/jni/jni_compiler_test.cc
+++ b/compiler/jni/jni_compiler_test.cc
@@ -465,7 +465,6 @@ LockWord JniCompilerTest::GetLockWord(jobject obj) {
 #define JNI_TEST(TestName) \
   JNI_TEST_NORMAL_ONLY(TestName)                 \
   TEST_F(JniCompilerTest, TestName ## FastCompiler) {    \
-    TEST_DISABLED_FOR_RISCV64(); \
     ScopedCheckHandleScope top_handle_scope_check;  \
     SCOPED_TRACE("@FastNative JNI with compiler");  \
     gCurrentJni = static_cast<uint32_t>(JniKind::kFast); \
@@ -473,7 +472,6 @@ LockWord JniCompilerTest::GetLockWord(jobject obj) {
   }                                              \
                                                  \
   TEST_F(JniCompilerTest, TestName ## FastGeneric) { \
-    TEST_DISABLED_FOR_RISCV64(); \
     ScopedCheckHandleScope top_handle_scope_check;  \
     SCOPED_TRACE("@FastNative JNI with generic");  \
     gCurrentJni = static_cast<uint32_t>(JniKind::kFast); \
diff --git a/compiler/utils/riscv64/jni_macro_assembler_riscv64.cc b/compiler/utils/riscv64/jni_macro_assembler_riscv64.cc
index b698d56c90..59c6f11a10 100644
--- a/compiler/utils/riscv64/jni_macro_assembler_riscv64.cc
+++ b/compiler/utils/riscv64/jni_macro_assembler_riscv64.cc
@@ -19,6 +19,8 @@
 #include "base/bit_utils_iterator.h"
 #include "dwarf/register.h"
 #include "entrypoints/quick/quick_entrypoints.h"
+#include "indirect_reference_table.h"
+#include "lock_word.h"
 #include "managed_register_riscv64.h"
 #include "offsets.h"
 #include "thread.h"
@@ -374,8 +376,17 @@ void Riscv64JNIMacroAssembler::GetCurrentThread(FrameOffset offset) {
 void Riscv64JNIMacroAssembler::DecodeJNITransitionOrLocalJObject(ManagedRegister m_reg,
                                                                  JNIMacroLabel* slow_path,
                                                                  JNIMacroLabel* resume) {
-  // TODO(riscv64): Implement this.
-  UNUSED(m_reg, slow_path, resume);
+  // This implements the fast-path of `Thread::DecodeJObject()`.
+  constexpr int64_t kGlobalOrWeakGlobalMask = IndirectReferenceTable::GetGlobalOrWeakGlobalMask();
+  DCHECK(IsInt<12>(kGlobalOrWeakGlobalMask));
+  constexpr int64_t kIndirectRefKindMask = IndirectReferenceTable::GetIndirectRefKindMask();
+  DCHECK(IsInt<12>(kIndirectRefKindMask));
+  XRegister reg = m_reg.AsRiscv64().AsXRegister();
+  __ Beqz(reg, Riscv64JNIMacroLabel::Cast(resume)->AsRiscv64());  // Skip test and load for null.
+  __ Andi(TMP, reg, kGlobalOrWeakGlobalMask);
+  __ Bnez(TMP, Riscv64JNIMacroLabel::Cast(slow_path)->AsRiscv64());
+  __ Andi(reg, reg, ~kIndirectRefKindMask);
+  __ Loadw(reg, reg, 0);
 }
 
 void Riscv64JNIMacroAssembler::VerifyObject([[maybe_unused]] ManagedRegister m_src,
@@ -423,12 +434,14 @@ void Riscv64JNIMacroAssembler::TryToTransitionFromNativeToRunnable(
 }
 
 void Riscv64JNIMacroAssembler::SuspendCheck(JNIMacroLabel* label) {
-  // TODO(riscv64): Implement this.
-  UNUSED(label);
+  __ Loadw(TMP, TR, Thread::ThreadFlagsOffset<kRiscv64PointerSize>().Int32Value());
+  DCHECK(IsInt<12>(dchecked_integral_cast<int32_t>(Thread::SuspendOrCheckpointRequestFlags())));
+  __ Andi(TMP, TMP, dchecked_integral_cast<int32_t>(Thread::SuspendOrCheckpointRequestFlags()));
+  __ Bnez(TMP, Riscv64JNIMacroLabel::Cast(label)->AsRiscv64());
 }
 
 void Riscv64JNIMacroAssembler::ExceptionPoll(JNIMacroLabel* label) {
-  __ Loadd(TMP, TR, Thread::ExceptionOffset<kArm64PointerSize>().Int32Value());
+  __ Loadd(TMP, TR, Thread::ExceptionOffset<kRiscv64PointerSize>().Int32Value());
   __ Bnez(TMP, Riscv64JNIMacroLabel::Cast(label)->AsRiscv64());
 }
 
@@ -436,8 +449,8 @@ void Riscv64JNIMacroAssembler::DeliverPendingException() {
   // Pass exception object as argument.
   // Don't care about preserving A0 as this won't return.
   // Note: The scratch register from `ExceptionPoll()` may have been clobbered.
-  __ Loadd(A0, TR, Thread::ExceptionOffset<kArm64PointerSize>().Int32Value());
-  __ Loadd(RA, TR, QUICK_ENTRYPOINT_OFFSET(kArm64PointerSize, pDeliverException).Int32Value());
+  __ Loadd(A0, TR, Thread::ExceptionOffset<kRiscv64PointerSize>().Int32Value());
+  __ Loadd(RA, TR, QUICK_ENTRYPOINT_OFFSET(kRiscv64PointerSize, pDeliverException).Int32Value());
   __ Jalr(RA);
   // Call should never return.
   __ Ebreak();
@@ -459,7 +472,7 @@ void Riscv64JNIMacroAssembler::TestGcMarking(JNIMacroLabel* label, JNIMacroUnary
   DCHECK(gUseReadBarrier);
 
   XRegister test_reg = TMP;
-  int32_t is_gc_marking_offset = Thread::IsGcMarkingOffset<kArm64PointerSize>().Int32Value();
+  int32_t is_gc_marking_offset = Thread::IsGcMarkingOffset<kRiscv64PointerSize>().Int32Value();
   __ Loadw(test_reg, TR, is_gc_marking_offset);
   switch (cond) {
     case JNIMacroUnaryCondition::kZero:
@@ -474,21 +487,37 @@ void Riscv64JNIMacroAssembler::TestGcMarking(JNIMacroLabel* label, JNIMacroUnary
   }
 }
 
-void Riscv64JNIMacroAssembler::TestMarkBit(ManagedRegister ref,
+void Riscv64JNIMacroAssembler::TestMarkBit(ManagedRegister m_ref,
                                            JNIMacroLabel* label,
                                            JNIMacroUnaryCondition cond) {
-  // TODO(riscv64): Implement this.
-  UNUSED(ref, label, cond);
+  DCHECK(kUseBakerReadBarrier);
+  XRegister ref = m_ref.AsRiscv64().AsXRegister();
+  __ Loadw(TMP, ref, mirror::Object::MonitorOffset().Int32Value());
+  // Move the bit we want to check to the sign bit, so that we can use BGEZ/BLTZ
+  // to check it. Extracting the bit for BEQZ/BNEZ would require one more instruction.
+  static_assert(LockWord::kMarkBitStateSize == 1u);
+  __ Slliw(TMP, TMP, 31 - LockWord::kMarkBitStateShift);
+  switch (cond) {
+    case JNIMacroUnaryCondition::kZero:
+      __ Bgez(TMP, Riscv64JNIMacroLabel::Cast(label)->AsRiscv64());
+      break;
+    case JNIMacroUnaryCondition::kNotZero:
+      __ Bltz(TMP, Riscv64JNIMacroLabel::Cast(label)->AsRiscv64());
+      break;
+    default:
+      LOG(FATAL) << "Not implemented unary condition: " << static_cast<int>(cond);
+      UNREACHABLE();
+  }
 }
 
 void Riscv64JNIMacroAssembler::TestByteAndJumpIfNotZero(uintptr_t address, JNIMacroLabel* label) {
-  XRegister test_reg = TMP;
   int32_t small_offset = dchecked_integral_cast<int32_t>(address & 0xfff) -
                          dchecked_integral_cast<int32_t>((address & 0x800) << 1);
-  int32_t remainder = static_cast<int64_t>(address) - small_offset;
-  __ Li(test_reg, remainder);
-  __ Lb(test_reg, test_reg, small_offset);
-  __ Bnez(test_reg, down_cast<Riscv64Label*>(Riscv64JNIMacroLabel::Cast(label)->AsRiscv64()));
+  int64_t remainder = static_cast<int64_t>(address) - small_offset;
+  // Note: We use `TMP2` here because `TMP` can be used by `LoadConst64()`.
+  __ LoadConst64(TMP2, remainder);
+  __ Lb(TMP2, TMP2, small_offset);
+  __ Bnez(TMP2, down_cast<Riscv64Label*>(Riscv64JNIMacroLabel::Cast(label)->AsRiscv64()));
 }
 
 void Riscv64JNIMacroAssembler::Bind(JNIMacroLabel* label) {
@@ -510,7 +539,7 @@ void Riscv64JNIMacroAssembler::CreateJObject(ManagedRegister m_dest,
     if (!dest.Equals(ref)) {
       __ Li(dest.AsXRegister(), 0);
     }
-    __ Bnez(ref.AsXRegister(), &null_label);
+    __ Beqz(ref.AsXRegister(), &null_label);
   }
   __ AddConst64(dest.AsXRegister(), SP, spilled_reference_offset.Int32Value());
   if (null_allowed) {
@@ -518,7 +547,7 @@ void Riscv64JNIMacroAssembler::CreateJObject(ManagedRegister m_dest,
   }
 }
 
-#undef ___
+#undef __
 
 }  // namespace riscv64
 }  // namespace art
diff --git a/compiler/utils/riscv64/jni_macro_assembler_riscv64.h b/compiler/utils/riscv64/jni_macro_assembler_riscv64.h
index 903c702f2d..daa2a58978 100644
--- a/compiler/utils/riscv64/jni_macro_assembler_riscv64.h
+++ b/compiler/utils/riscv64/jni_macro_assembler_riscv64.h
@@ -143,6 +143,8 @@ class Riscv64JNIMacroAssembler  : public JNIMacroAssemblerFwd<Riscv64Assembler,
                      FrameOffset spilled_reference_offset,
                      ManagedRegister m_ref,
                      bool null_allowed);
+
+  ART_FRIEND_TEST(JniMacroAssemblerRiscv64Test, CreateJObject);
 };
 
 class Riscv64JNIMacroLabel final
diff --git a/compiler/utils/riscv64/jni_macro_assembler_riscv64_test.cc b/compiler/utils/riscv64/jni_macro_assembler_riscv64_test.cc
index 6a27885e04..f4e7d397a4 100644
--- a/compiler/utils/riscv64/jni_macro_assembler_riscv64_test.cc
+++ b/compiler/utils/riscv64/jni_macro_assembler_riscv64_test.cc
@@ -25,6 +25,8 @@
 
 #include "gtest/gtest.h"
 
+#include "indirect_reference_table.h"
+#include "lock_word.h"
 #include "jni/quick/calling_convention.h"
 #include "utils/riscv64/jni_macro_assembler_riscv64.h"
 #include "utils/assembler_test_base.h"
@@ -35,6 +37,8 @@
 namespace art HIDDEN {
 namespace riscv64 {
 
+#define __ assembler_.
+
 class JniMacroAssemblerRiscv64Test : public AssemblerTestBase {
  public:
   JniMacroAssemblerRiscv64Test() : pool_(), allocator_(&pool_), assembler_(&allocator_) { }
@@ -59,6 +63,13 @@ class JniMacroAssemblerRiscv64Test : public AssemblerTestBase {
     return Riscv64ManagedRegister::FromFRegister(reg);
   }
 
+  std::string EmitRet() {
+    __ RemoveFrame(/*frame_size=*/ 0u,
+                   /*callee_save_regs=*/ ArrayRef<const ManagedRegister>(),
+                   /*may_suspend=*/ false);
+    return "ret\n";
+  }
+
   static const size_t kWordSize = 4u;
   static const size_t kDoubleWordSize = 8u;
 
@@ -67,8 +78,6 @@ class JniMacroAssemblerRiscv64Test : public AssemblerTestBase {
   Riscv64JNIMacroAssembler assembler_;
 };
 
-#define __ assembler_.
-
 TEST_F(JniMacroAssemblerRiscv64Test, StackFrame) {
   std::string expected;
 
@@ -260,6 +269,27 @@ TEST_F(JniMacroAssemblerRiscv64Test, Load) {
   DriverStr(expected, "Load");
 }
 
+TEST_F(JniMacroAssemblerRiscv64Test, CreateJObject) {
+  std::string expected;
+
+  __ CreateJObject(AsManaged(A0), FrameOffset(8), AsManaged(A0), /*null_allowed=*/ true);
+  expected += "beqz a0, 1f\n"
+              "addi a0, sp, 8\n"
+              "1:\n";
+  __ CreateJObject(AsManaged(A1), FrameOffset(12), AsManaged(A1), /*null_allowed=*/ false);
+  expected += "addi a1, sp, 12\n";
+  __ CreateJObject(AsManaged(A2), FrameOffset(16), AsManaged(A3), /*null_allowed=*/ true);
+  expected += "li a2, 0\n"
+              "beqz a3, 2f\n"
+              "addi a2, sp, 16\n"
+              "2:\n";
+  __ CreateJObject(AsManaged(A4), FrameOffset(2048), AsManaged(A5), /*null_allowed=*/ false);
+  expected += "addi t6, sp, 2047\n"
+              "addi a4, t6, 1\n";
+
+  DriverStr(expected, "CreateJObject");
+}
+
 TEST_F(JniMacroAssemblerRiscv64Test, MoveArguments) {
   // TODO(riscv64): Test `MoveArguments()`.
   // We do not add the test yet while there is an outstanding FIXME in `MoveArguments()`.
@@ -294,6 +324,36 @@ TEST_F(JniMacroAssemblerRiscv64Test, GetCurrentThread) {
   DriverStr(expected, "GetCurrentThread");
 }
 
+TEST_F(JniMacroAssemblerRiscv64Test, DecodeJNITransitionOrLocalJObject) {
+  std::string expected;
+
+  constexpr int64_t kGlobalOrWeakGlobalMask = IndirectReferenceTable::GetGlobalOrWeakGlobalMask();
+  constexpr int64_t kIndirectRefKindMask = IndirectReferenceTable::GetIndirectRefKindMask();
+
+  std::unique_ptr<JNIMacroLabel> slow_path = __ CreateLabel();
+  std::unique_ptr<JNIMacroLabel> resume = __ CreateLabel();
+
+  __ DecodeJNITransitionOrLocalJObject(AsManaged(A0), slow_path.get(), resume.get());
+  expected += "beqz a0, 1f\n"
+              "andi t6, a0, " + std::to_string(kGlobalOrWeakGlobalMask) + "\n"
+              "bnez t6, 2f\n"
+              "andi a0, a0, ~" + std::to_string(kIndirectRefKindMask) + "\n"
+              "lw a0, (a0)\n";
+
+  __ Bind(resume.get());
+  expected += "1:\n";
+
+  expected += EmitRet();
+
+  __ Bind(slow_path.get());
+  expected += "2:\n";
+
+  __ Jump(resume.get());
+  expected += "j 1b\n";
+
+  DriverStr(expected, "DecodeJNITransitionOrLocalJObject");
+}
+
 TEST_F(JniMacroAssemblerRiscv64Test, JumpCodePointer) {
   std::string expected;
 
@@ -333,6 +393,33 @@ TEST_F(JniMacroAssemblerRiscv64Test, Call) {
   DriverStr(expected, "Call");
 }
 
+TEST_F(JniMacroAssemblerRiscv64Test, SuspendCheck) {
+  std::string expected;
+
+  ThreadOffset64 thread_flags_offet = Thread::ThreadFlagsOffset<kRiscv64PointerSize>();
+
+  std::unique_ptr<JNIMacroLabel> slow_path = __ CreateLabel();
+  std::unique_ptr<JNIMacroLabel> resume = __ CreateLabel();
+
+  __ SuspendCheck(slow_path.get());
+  expected += "lw t6, " + std::to_string(thread_flags_offet.Int32Value()) + "(s1)\n"
+              "andi t6, t6, " + std::to_string(Thread::SuspendOrCheckpointRequestFlags()) + "\n"
+              "bnez t6, 2f\n";
+
+  __ Bind(resume.get());
+  expected += "1:\n";
+
+  expected += EmitRet();
+
+  __ Bind(slow_path.get());
+  expected += "2:\n";
+
+  __ Jump(resume.get());
+  expected += "j 1b";
+
+  DriverStr(expected, "SuspendCheck");
+}
+
 TEST_F(JniMacroAssemblerRiscv64Test, Exception) {
   std::string expected;
 
@@ -345,10 +432,7 @@ TEST_F(JniMacroAssemblerRiscv64Test, Exception) {
   expected += "ld t6, " + std::to_string(exception_offset.Int32Value()) + "(s1)\n"
               "bnez t6, 1f\n";
 
-  __ RemoveFrame(/*frame_size=*/ 0u,
-                 /*callee_save_regs=*/ ArrayRef<const ManagedRegister>(),
-                 /*may_suspend=*/ false);
-  expected += "ret\n";
+  expected += EmitRet();
 
   __ Bind(slow_path.get());
   expected += "1:\n";
@@ -387,11 +471,77 @@ TEST_F(JniMacroAssemblerRiscv64Test, JumpLabel) {
 }
 
 TEST_F(JniMacroAssemblerRiscv64Test, ReadBarrier) {
-  // TODO(riscv64): Test `TestGcMarking()` and `TestMarkBit()`.
+  std::string expected;
+
+  ThreadOffset64 is_gc_marking_offset = Thread::IsGcMarkingOffset<kRiscv64PointerSize>();
+  MemberOffset monitor_offset = mirror::Object::MonitorOffset();
+
+  std::unique_ptr<JNIMacroLabel> slow_path = __ CreateLabel();
+  std::unique_ptr<JNIMacroLabel> resume = __ CreateLabel();
+
+  __ TestGcMarking(slow_path.get(), JNIMacroUnaryCondition::kNotZero);
+  expected += "lw t6, " + std::to_string(is_gc_marking_offset.Int32Value()) + "(s1)\n"
+              "bnez t6, 2f\n";
+
+  __ TestGcMarking(slow_path.get(), JNIMacroUnaryCondition::kZero);
+  expected += "lw t6, " + std::to_string(is_gc_marking_offset.Int32Value()) + "(s1)\n"
+              "beqz t6, 2f\n";
+
+  __ Bind(resume.get());
+  expected += "1:\n";
+
+  expected += EmitRet();
+
+  __ Bind(slow_path.get());
+  expected += "2:\n";
+
+  __ TestMarkBit(AsManaged(A0), resume.get(), JNIMacroUnaryCondition::kNotZero);
+  expected += "lw t6, " + std::to_string(monitor_offset.Int32Value()) + "(a0)\n"
+              "slliw t6, t6, " + std::to_string(31 - LockWord::kMarkBitStateShift) + "\n"
+              "bltz t6, 1b\n";
+
+  __ TestMarkBit(AsManaged(T0), resume.get(), JNIMacroUnaryCondition::kZero);
+  expected += "lw t6, " + std::to_string(monitor_offset.Int32Value()) + "(t0)\n"
+              "slliw t6, t6, " + std::to_string(31 - LockWord::kMarkBitStateShift) + "\n"
+              "bgez t6, 1b\n";
+
+  DriverStr(expected, "ReadBarrier");
 }
 
 TEST_F(JniMacroAssemblerRiscv64Test, TestByteAndJumpIfNotZero) {
-  // TODO(riscv64): Test `TestByteAndJumpIfNotZero()`.
+  // Note: The `TestByteAndJumpIfNotZero()` takes the address as a `uintptr_t`.
+  // Use 32-bit addresses, so that we can include this test in 32-bit host tests.
+
+  std::string expected;
+
+  std::unique_ptr<JNIMacroLabel> slow_path = __ CreateLabel();
+  std::unique_ptr<JNIMacroLabel> resume = __ CreateLabel();
+
+  __ TestByteAndJumpIfNotZero(0x12345678u, slow_path.get());
+  expected += "lui t5, 0x12345\n"
+              "lb t5, 0x678(t5)\n"
+              "bnez t5, 2f\n";
+
+  __ TestByteAndJumpIfNotZero(0x87654321u, slow_path.get());
+  expected += "lui t5, 0x87654/4\n"
+              "slli t5, t5, 2\n"
+              "lb t5, 0x321(t5)\n"
+              "bnez t5, 2f\n";
+
+  __ Bind(resume.get());
+  expected += "1:\n";
+
+  expected += EmitRet();
+
+  __ Bind(slow_path.get());
+  expected += "2:\n";
+
+  __ TestByteAndJumpIfNotZero(0x456789abu, resume.get());
+  expected += "lui t5, 0x45678+1\n"
+              "lb t5, 0x9ab-0x1000(t5)\n"
+              "bnez t5, 1b\n";
+
+  DriverStr(expected, "TestByteAndJumpIfNotZero");
 }
 
 #undef __
diff --git a/runtime/arch/riscv64/quick_entrypoints_riscv64.S b/runtime/arch/riscv64/quick_entrypoints_riscv64.S
index 19f5a10d0e..e9eb3130bc 100644
--- a/runtime/arch/riscv64/quick_entrypoints_riscv64.S
+++ b/runtime/arch/riscv64/quick_entrypoints_riscv64.S
@@ -554,6 +554,8 @@ ENTRY \c_name
 END \c_name
 .endm
 
+// Called by managed code to deliver an exception.
+ONE_ARG_RUNTIME_EXCEPTION art_quick_deliver_exception, artDeliverExceptionFromCode
 
 // Called to attempt to execute an obsolete method.
 ONE_ARG_RUNTIME_EXCEPTION art_invoke_obsolete_method_stub, artInvokeObsoleteMethod
@@ -787,7 +789,6 @@ UNDEFINED art_quick_invoke_super_trampoline_with_access_check
 UNDEFINED art_quick_invoke_virtual_trampoline_with_access_check
 UNDEFINED art_quick_invoke_polymorphic
 UNDEFINED art_quick_invoke_custom
-UNDEFINED art_quick_deliver_exception
 UNDEFINED art_quick_throw_array_bounds
 UNDEFINED art_quick_throw_div_zero
 UNDEFINED art_quick_throw_null_pointer_exception