Add a fast path for nterp entrypoint to avoid fetching the shorty.

When all parameters are references, we don't need to look at the shorty.
Use the 0x00100000 flag in the modifiers which is free for non-native
methods.

Test: test.py
Bug: 112676029
Change-Id: Ied9a253f7f7230045dd13188a5b806fb1d6d019d

8 files changed, 132 insertions, 2 deletions

diff --git a/libdexfile/dex/modifiers.h b/libdexfile/dex/modifiers.h
index 7b15cca8a4..5fad46cb07 100644
--- a/libdexfile/dex/modifiers.h
+++ b/libdexfile/dex/modifiers.h
@@ -97,6 +97,9 @@ static constexpr uint32_t kAccMustCountLocks =        0x04000000;  // method (ru
 // virtual call.
 static constexpr uint32_t kAccSingleImplementation =  0x08000000;  // method (runtime)
 
+// Whether nterp can take a fast path when entering this method (runtime; non-native)
+static constexpr uint32_t kAccNterpEntryPointFastPathFlag = 0x00100000;
+
 static constexpr uint32_t kAccPublicApi =             0x10000000;  // field, method
 static constexpr uint32_t kAccCorePlatformApi =       0x20000000;  // field, method
 
diff --git a/runtime/art_method.h b/runtime/art_method.h
index b2c211f7cf..412da07fff 100644
--- a/runtime/art_method.h
+++ b/runtime/art_method.h
@@ -394,6 +394,16 @@ class ArtMethod final {
     ClearAccessFlags(kAccSkipAccessChecks);
   }
 
+  bool HasNterpEntryPointFastPathFlag() const {
+    constexpr uint32_t mask = kAccNative | kAccNterpEntryPointFastPathFlag;
+    return (GetAccessFlags() & mask) == kAccNterpEntryPointFastPathFlag;
+  }
+
+  void SetNterpEntryPointFastPathFlag() REQUIRES_SHARED(Locks::mutator_lock_) {
+    DCHECK(!IsNative());
+    AddAccessFlags(kAccNterpEntryPointFastPathFlag);
+  }
+
   // Returns true if this method could be overridden by a default method.
   bool IsOverridableByDefaultMethod() REQUIRES_SHARED(Locks::mutator_lock_);
 
diff --git a/runtime/class_linker.cc b/runtime/class_linker.cc
index e894037ae3..2040263cc6 100644
--- a/runtime/class_linker.cc
+++ b/runtime/class_linker.cc
@@ -3838,6 +3838,17 @@ void ClassLinker::LoadMethod(const DexFile& dex_file,
     } else {
       dst->SetCodeItem(dst->GetDexFile()->GetCodeItem(method.GetCodeItemOffset()));
     }
+    bool has_all_references = true;
+    const char* shorty = dst->GetShorty();
+    for (size_t i = 1, e = strlen(shorty); i < e; ++i) {
+      if (shorty[i] != 'L') {
+        has_all_references = false;
+        break;
+      }
+    }
+    if (has_all_references) {
+      dst->SetNterpEntryPointFastPathFlag();
+    }
   } else {
     dst->SetDataPtrSize(nullptr, image_pointer_size_);
     DCHECK_EQ(method.GetCodeItemOffset(), 0u);
diff --git a/runtime/image.cc b/runtime/image.cc
index 788c981e29..7c4cc39804 100644
--- a/runtime/image.cc
+++ b/runtime/image.cc
@@ -29,8 +29,8 @@
 namespace art {
 
 const uint8_t ImageHeader::kImageMagic[] = { 'a', 'r', 't', '\n' };
-// Last change: Remove kAccMiranda, kAccDefaultConflict.
-const uint8_t ImageHeader::kImageVersion[] = { '0', '9', '7', '\0' };
+// Last change: Add kAccNterpEntryPointFastPathFlag
+const uint8_t ImageHeader::kImageVersion[] = { '0', '9', '8', '\0' };
 
 ImageHeader::ImageHeader(uint32_t image_reservation_size,
                          uint32_t component_count,
diff --git a/runtime/interpreter/mterp/arm64ng/main.S b/runtime/interpreter/mterp/arm64ng/main.S
index 3ba175d73f..27c4b7c85d 100644
--- a/runtime/interpreter/mterp/arm64ng/main.S
+++ b/runtime/interpreter/mterp/arm64ng/main.S
@@ -1438,6 +1438,25 @@ END \name
     b 1b
 .endm
 
+.macro SETUP_REFERENCE_PARAMETER_IN_GPR gpr32, regs, refs, ins, arg_offset, finished
+    str \gpr32, [\regs, \arg_offset]
+    sub \ins, \ins, #1
+    str \gpr32, [\refs, \arg_offset]
+    add \arg_offset, \arg_offset, #4
+    cbz \ins, \finished
+.endm
+
+// Uses ip2 as temporary.
+.macro SETUP_REFERENCE_PARAMETERS_IN_STACK regs, refs, ins, stack_ptr, arg_offset
+1:
+    ldr wip2, [\stack_ptr, \arg_offset]
+    sub \ins, \ins, #1
+    str wip2, [\regs, \arg_offset]
+    str wip2, [\refs, \arg_offset]
+    add \arg_offset, \arg_offset, #4
+    cbnz \ins, 1b
+.endm
+
 %def entry():
 /*
  * ArtMethod entry point.
@@ -1465,6 +1484,23 @@ OAT_ENTRY ExecuteNterpImpl, EndExecuteNterpImpl
     ldr w26, [x0, #ART_METHOD_ACCESS_FLAGS_OFFSET]
     lsl x21, ip2, #2 // x21 is now the offset for inputs into the registers array.
 
+    tbz w26, #ART_METHOD_NTERP_ENTRY_POINT_FAST_PATH_FLAG_BIT, .Lsetup_slow_path
+    // Setup pointer to inputs in FP and pointer to inputs in REFS
+    add x10, xFP, x21
+    add x11, xREFS, x21
+    mov x12, #0
+    SETUP_REFERENCE_PARAMETER_IN_GPR w1, x10, x11, w15, x12, .Lxmm_setup_finished
+    SETUP_REFERENCE_PARAMETER_IN_GPR w2, x10, x11, w15, x12, .Lxmm_setup_finished
+    SETUP_REFERENCE_PARAMETER_IN_GPR w3, x10, x11, w15, x12, .Lxmm_setup_finished
+    SETUP_REFERENCE_PARAMETER_IN_GPR w4, x10, x11, w15, x12, .Lxmm_setup_finished
+    SETUP_REFERENCE_PARAMETER_IN_GPR w5, x10, x11, w15, x12, .Lxmm_setup_finished
+    SETUP_REFERENCE_PARAMETER_IN_GPR w6, x10, x11, w15, x12, .Lxmm_setup_finished
+    SETUP_REFERENCE_PARAMETER_IN_GPR w7, x10, x11, w15, x12, .Lxmm_setup_finished
+    add x28, x28, #OFFSET_TO_FIRST_ARGUMENT_IN_STACK
+    SETUP_REFERENCE_PARAMETERS_IN_STACK x10, x11, w15, x28, x12
+    b .Lxmm_setup_finished
+
+.Lsetup_slow_path:
     // If the method is not static and there is one argument ('this'), we don't need to fetch the
     // shorty.
     tbnz w26, #ART_METHOD_IS_STATIC_FLAG_BIT, .Lsetup_with_shorty
diff --git a/runtime/interpreter/mterp/armng/main.S b/runtime/interpreter/mterp/armng/main.S
index 4b0a70f3d9..d7f1bf893f 100644
--- a/runtime/interpreter/mterp/armng/main.S
+++ b/runtime/interpreter/mterp/armng/main.S
@@ -1472,6 +1472,24 @@ END \name
     b 1b
 .endm
 
+.macro SETUP_REFERENCE_PARAMETER_IN_GPR gpr32, regs, refs, ins, arg_offset, finished
+    str \gpr32, [\regs, \arg_offset]
+    subs \ins, \ins, #1
+    str \gpr32, [\refs, \arg_offset]
+    add \arg_offset, \arg_offset, #4
+    beq \finished
+.endm
+
+.macro SETUP_REFERENCE_PARAMETERS_IN_STACK regs, refs, ins, stack_ptr, arg_offset
+1:
+    ldr ip, [\stack_ptr, \arg_offset]
+    subs \ins, \ins, #1
+    str ip, [\regs, \arg_offset]
+    str ip, [\refs, \arg_offset]
+    add \arg_offset, \arg_offset, #4
+    bne 1b
+.endm
+
 %def entry():
 /*
  * ArtMethod entry point.
@@ -1502,6 +1520,20 @@ OAT_ENTRY ExecuteNterpImpl, EndExecuteNterpImpl
     lsl rINST, rINST, #2 // rINST is now the offset for inputs into the registers array.
     mov rIBASE, ip // rIBASE contains the old stack pointer
 
+    tst r8, #ART_METHOD_NTERP_ENTRY_POINT_FAST_PATH_FLAG
+    beq .Lsetup_slow_path
+    // Setup pointer to inputs in FP and pointer to inputs in REFS
+    add lr, rFP, rINST
+    add r8, rREFS, rINST
+    mov r0, #0
+    SETUP_REFERENCE_PARAMETER_IN_GPR r1, lr, r8, r4, r0, .Lxmm_setup_finished
+    SETUP_REFERENCE_PARAMETER_IN_GPR r2, lr, r8, r4, r0, .Lxmm_setup_finished
+    SETUP_REFERENCE_PARAMETER_IN_GPR r3, lr, r8, r4, r0, .Lxmm_setup_finished
+    add rIBASE, rIBASE, #OFFSET_TO_FIRST_ARGUMENT_IN_STACK
+    SETUP_REFERENCE_PARAMETERS_IN_STACK lr, r8, r4, rIBASE, r0
+    b .Lxmm_setup_finished
+
+.Lsetup_slow_path:
     // If the method is not static and there is one argument ('this'), we don't need to fetch the
     // shorty.
     tst r8, #ART_METHOD_IS_STATIC_FLAG
diff --git a/runtime/interpreter/mterp/x86_64ng/main.S b/runtime/interpreter/mterp/x86_64ng/main.S
index 21214bb023..0089429696 100644
--- a/runtime/interpreter/mterp/x86_64ng/main.S
+++ b/runtime/interpreter/mterp/x86_64ng/main.S
@@ -1553,6 +1553,25 @@ END_FUNCTION \name
    jmp 1b
 .endm
 
+.macro SETUP_REFERENCE_PARAMETER_IN_GPR gpr32, regs, refs, ins, arg_offset, finished
+    movl REG_VAR(gpr32), (REG_VAR(regs), REG_VAR(arg_offset))
+    movl REG_VAR(gpr32), (REG_VAR(refs), REG_VAR(arg_offset))
+    addq MACRO_LITERAL(4), REG_VAR(arg_offset)
+    subl MACRO_LITERAL(1), REG_VAR(ins)
+    je \finished
+.endm
+
+// Uses eax as temporary
+.macro SETUP_REFERENCE_PARAMETERS_IN_STACK regs, refs, ins, stack_ptr, arg_offset
+1:
+    movl OFFSET_TO_FIRST_ARGUMENT_IN_STACK(REG_VAR(stack_ptr), REG_VAR(arg_offset)), %eax
+    movl %eax, (REG_VAR(regs), REG_VAR(arg_offset))
+    movl %eax, (REG_VAR(refs), REG_VAR(arg_offset))
+    addq MACRO_LITERAL(4), REG_VAR(arg_offset)
+    subl MACRO_LITERAL(1), REG_VAR(ins)
+    jne 1b
+.endm
+
 %def entry():
 /*
  * ArtMethod entry point.
@@ -1581,6 +1600,21 @@ OAT_ENTRY ExecuteNterpImpl, EndExecuteNterpImpl
     subq %r14, %rbx
     salq $$2, %rbx // rbx is now the offset for inputs into the registers array.
 
+    testl $$ART_METHOD_NTERP_ENTRY_POINT_FAST_PATH_FLAG, ART_METHOD_ACCESS_FLAGS_OFFSET(%rdi)
+    je .Lsetup_slow_path
+    leaq (rFP, %rbx, 1), %rdi
+    leaq (rREFS, %rbx, 1), %rbx
+    movq $$0, %r10
+
+    SETUP_REFERENCE_PARAMETER_IN_GPR esi, rdi, rbx, r14d, r10, .Lxmm_setup_finished
+    SETUP_REFERENCE_PARAMETER_IN_GPR edx, rdi, rbx, r14d, r10, .Lxmm_setup_finished
+    SETUP_REFERENCE_PARAMETER_IN_GPR ecx, rdi, rbx, r14d, r10, .Lxmm_setup_finished
+    SETUP_REFERENCE_PARAMETER_IN_GPR r8d, rdi, rbx, r14d, r10, .Lxmm_setup_finished
+    SETUP_REFERENCE_PARAMETER_IN_GPR r9d, rdi, rbx, r14d, r10, .Lxmm_setup_finished
+    SETUP_REFERENCE_PARAMETERS_IN_STACK rdi, rbx, r14d, r11, r10
+    jmp .Lxmm_setup_finished
+
+.Lsetup_slow_path:
     // If the method is not static and there is one argument ('this'), we don't need to fetch the
     // shorty.
     testl $$ART_METHOD_IS_STATIC_FLAG, ART_METHOD_ACCESS_FLAGS_OFFSET(%rdi)
diff --git a/tools/cpp-define-generator/art_method.def b/tools/cpp-define-generator/art_method.def
index c2e18b1913..5e09565df1 100644
--- a/tools/cpp-define-generator/art_method.def
+++ b/tools/cpp-define-generator/art_method.def
@@ -23,10 +23,14 @@ ASM_DEFINE(ART_METHOD_ACCESS_FLAGS_OFFSET,
            art::ArtMethod::AccessFlagsOffset().Int32Value())
 ASM_DEFINE(ART_METHOD_IS_STATIC_FLAG,
            art::kAccStatic)
+ASM_DEFINE(ART_METHOD_NTERP_ENTRY_POINT_FAST_PATH_FLAG,
+           art::kAccNterpEntryPointFastPathFlag)
 ASM_DEFINE(ART_METHOD_IMT_MASK,
            art::ImTable::kSizeTruncToPowerOfTwo - 1)
 ASM_DEFINE(ART_METHOD_IS_STATIC_FLAG_BIT,
            art::MostSignificantBit(art::kAccStatic))
+ASM_DEFINE(ART_METHOD_NTERP_ENTRY_POINT_FAST_PATH_FLAG_BIT,
+           art::MostSignificantBit(art::kAccNterpEntryPointFastPathFlag))
 ASM_DEFINE(ART_METHOD_DECLARING_CLASS_OFFSET,
            art::ArtMethod::DeclaringClassOffset().Int32Value())
 ASM_DEFINE(ART_METHOD_JNI_OFFSET_32,