jni: Fast path for @FastNative annotated java methods

Adds a faster path for java methods annotated with
dalvik.annotation.optimization.FastNative .

Intended to replace usage of fast JNI (registering with "!(FOO)BAR" descriptors).

Performance Microbenchmark Results (Angler):
* Regular JNI cost in nanoseconds: 115
* Fast JNI cost in nanoseconds: 60
* @FastNative cost in nanoseconds: 36

Summary: Up to 67% faster (vs fast jni) JNI transition cost

Change-Id: Ic23823ae0f232270c068ec999fd89aa993894b0e

25 files changed, 284 insertions, 24 deletions

diff --git a/compiler/compiler.h b/compiler/compiler.h
index 487a27fec0..a955f3c481 100644
--- a/compiler/compiler.h
+++ b/compiler/compiler.h
@@ -38,6 +38,11 @@ class Compiler {
     kOptimizing
   };
 
+  enum JniOptimizationFlags {
+    kNone,
+    kFastNative,
+  };
+
   static Compiler* Create(CompilerDriver* driver, Kind kind);
 
   virtual void Init() = 0;
@@ -57,7 +62,8 @@ class Compiler {
 
   virtual CompiledMethod* JniCompile(uint32_t access_flags,
                                      uint32_t method_idx,
-                                     const DexFile& dex_file) const = 0;
+                                     const DexFile& dex_file,
+                                     JniOptimizationFlags optimization_flags) const = 0;
 
   virtual bool JitCompile(Thread* self ATTRIBUTE_UNUSED,
                           jit::JitCodeCache* code_cache ATTRIBUTE_UNUSED,
diff --git a/compiler/driver/compiler_driver.cc b/compiler/driver/compiler_driver.cc
index d0a8335a99..758cd936a2 100644
--- a/compiler/driver/compiler_driver.cc
+++ b/compiler/driver/compiler_driver.cc
@@ -599,7 +599,38 @@ static void CompileMethod(Thread* self,
         InstructionSetHasGenericJniStub(driver->GetInstructionSet())) {
       // Leaving this empty will trigger the generic JNI version
     } else {
-      compiled_method = driver->GetCompiler()->JniCompile(access_flags, method_idx, dex_file);
+      // Look-up the ArtMethod associated with this code_item (if any)
+      // -- It is later used to lookup any [optimization] annotations for this method.
+      ScopedObjectAccess soa(self);
+      StackHandleScope<1> hs(soa.Self());
+      Handle<mirror::ClassLoader> class_loader_handle(hs.NewHandle(
+          soa.Decode<mirror::ClassLoader*>(class_loader)));
+
+      // TODO: Lookup annotation from DexFile directly without resolving method.
+      ArtMethod* method =
+          Runtime::Current()->GetClassLinker()->ResolveMethod<ClassLinker::kNoICCECheckForCache>(
+              dex_file,
+              method_idx,
+              dex_cache,
+              class_loader_handle,
+              /* referrer */ nullptr,
+              invoke_type);
+
+      bool fast_native = false;
+      if (LIKELY(method != nullptr)) {
+        fast_native = method->IsAnnotatedWithFastNative();
+      } else {
+        // Failed method resolutions happen very rarely, e.g. ancestor class cannot be resolved.
+        DCHECK(self->IsExceptionPending());
+        self->ClearException();
+      }
+
+      Compiler::JniOptimizationFlags optimization_flags =
+          fast_native ? Compiler::kFastNative : Compiler::kNone;
+      compiled_method = driver->GetCompiler()->JniCompile(access_flags,
+                                                          method_idx,
+                                                          dex_file,
+                                                          optimization_flags);
       CHECK(compiled_method != nullptr);
     }
   } else if ((access_flags & kAccAbstract) != 0) {
@@ -2874,7 +2905,7 @@ bool CompilerDriver::IsStringTypeIndex(uint16_t type_index, const DexFile* dex_f
 
 bool CompilerDriver::IsStringInit(uint32_t method_index, const DexFile* dex_file, int32_t* offset) {
   DexFileMethodInliner* inliner = GetMethodInlinerMap()->GetMethodInliner(dex_file);
-  PointerSize pointer_size = InstructionSetPointerSize(GetInstructionSet());
+  const PointerSize pointer_size = InstructionSetPointerSize(GetInstructionSet());
   *offset = inliner->GetOffsetForStringInit(method_index, pointer_size);
   return inliner->IsStringInitMethodIndex(method_index);
 }
diff --git a/compiler/jni/jni_compiler_test.cc b/compiler/jni/jni_compiler_test.cc
index c4c2399ccd..b83985a771 100644
--- a/compiler/jni/jni_compiler_test.cc
+++ b/compiler/jni/jni_compiler_test.cc
@@ -175,6 +175,9 @@ class JniCompilerTest : public CommonCompilerTest {
   void StackArgsMixedImpl();
   void StackArgsSignExtendedMips64Impl();
 
+  void NormalNativeImpl();
+  void FastNativeImpl();
+
   JNIEnv* env_;
   jstring library_search_path_;
   jmethodID jmethod_;
@@ -1772,4 +1775,44 @@ void JniCompilerTest::StackArgsSignExtendedMips64Impl() {
 
 JNI_TEST(StackArgsSignExtendedMips64)
 
+void Java_MyClassNatives_normalNative(JNIEnv*, jclass) {
+  // Intentionally left empty.
+}
+
+// Methods not annotated with anything are not considered "fast native"
+// -- Check that the annotation lookup does not find it.
+void JniCompilerTest::NormalNativeImpl() {
+  SetUpForTest(/* direct */ true,
+               "normalNative",
+               "()V",
+               reinterpret_cast<void*>(&Java_MyClassNatives_normalNative));
+
+  ScopedObjectAccess soa(Thread::Current());
+  ArtMethod* method = soa.DecodeMethod(jmethod_);
+  ASSERT_TRUE(method != nullptr);
+
+  EXPECT_FALSE(method->IsAnnotatedWithFastNative());
+}
+JNI_TEST(NormalNative)
+
+// Methods annotated with @FastNative are considered "fast native"
+// -- Check that the annotation lookup succeeds.
+void Java_MyClassNatives_fastNative(JNIEnv*, jclass) {
+  // Intentionally left empty.
+}
+
+void JniCompilerTest::FastNativeImpl() {
+  SetUpForTest(/* direct */ true,
+               "fastNative",
+               "()V",
+               reinterpret_cast<void*>(&Java_MyClassNatives_fastNative));
+
+  ScopedObjectAccess soa(Thread::Current());
+  ArtMethod* method = soa.DecodeMethod(jmethod_);
+  ASSERT_TRUE(method != nullptr);
+
+  EXPECT_TRUE(method->IsAnnotatedWithFastNative());
+}
+JNI_TEST(FastNative)
+
 }  // namespace art
diff --git a/compiler/jni/quick/jni_compiler.cc b/compiler/jni/quick/jni_compiler.cc
index f99f6a8792..d092c3f1f6 100644
--- a/compiler/jni/quick/jni_compiler.cc
+++ b/compiler/jni/quick/jni_compiler.cc
@@ -17,6 +17,7 @@
 #include "jni_compiler.h"
 
 #include <algorithm>
+#include <ios>
 #include <memory>
 #include <vector>
 #include <fstream>
@@ -44,12 +45,15 @@
 #include "utils/mips/managed_register_mips.h"
 #include "utils/mips64/managed_register_mips64.h"
 #include "utils/x86/managed_register_x86.h"
+#include "utils.h"
 #include "thread.h"
 
 #define __ jni_asm->
 
 namespace art {
 
+using JniOptimizationFlags = Compiler::JniOptimizationFlags;
+
 template <PointerSize kPointerSize>
 static void CopyParameter(JNIMacroAssembler<kPointerSize>* jni_asm,
                           ManagedRuntimeCallingConvention* mr_conv,
@@ -75,7 +79,8 @@ template <PointerSize kPointerSize>
 static CompiledMethod* ArtJniCompileMethodInternal(CompilerDriver* driver,
                                                    uint32_t access_flags,
                                                    uint32_t method_idx,
-                                                   const DexFile& dex_file) {
+                                                   const DexFile& dex_file,
+                                                   JniOptimizationFlags optimization_flags) {
   const bool is_native = (access_flags & kAccNative) != 0;
   CHECK(is_native);
   const bool is_static = (access_flags & kAccStatic) != 0;
@@ -84,6 +89,19 @@ static CompiledMethod* ArtJniCompileMethodInternal(CompilerDriver* driver,
   InstructionSet instruction_set = driver->GetInstructionSet();
   const InstructionSetFeatures* instruction_set_features = driver->GetInstructionSetFeatures();
 
+  // i.e. if the method was annotated with @FastNative
+  const bool is_fast_native =
+      (static_cast<uint32_t>(optimization_flags) & Compiler::kFastNative) != 0;
+
+  VLOG(jni) << "JniCompile: Method :: "
+              << art::PrettyMethod(method_idx, dex_file, /* with signature */ true)
+              << " :: access_flags = " << std::hex << access_flags << std::dec;
+
+  if (UNLIKELY(is_fast_native)) {
+    VLOG(jni) << "JniCompile: Fast native method detected :: "
+              << art::PrettyMethod(method_idx, dex_file, /* with signature */ true);
+  }
+
   ArenaPool pool;
   ArenaAllocator arena(&pool);
 
@@ -240,7 +258,10 @@ static CompiledMethod* ArtJniCompileMethodInternal(CompilerDriver* driver,
   ThreadOffset<kPointerSize> jni_start =
       is_synchronized
           ? QUICK_ENTRYPOINT_OFFSET(kPointerSize, pJniMethodStartSynchronized)
-          : QUICK_ENTRYPOINT_OFFSET(kPointerSize, pJniMethodStart);
+          : (is_fast_native
+                 ? QUICK_ENTRYPOINT_OFFSET(kPointerSize, pJniMethodFastStart)
+                 : QUICK_ENTRYPOINT_OFFSET(kPointerSize, pJniMethodStart));
+
   main_jni_conv->ResetIterator(FrameOffset(main_out_arg_size));
   FrameOffset locked_object_handle_scope_offset(0);
   if (is_synchronized) {
@@ -385,6 +406,7 @@ static CompiledMethod* ArtJniCompileMethodInternal(CompilerDriver* driver,
   }
   //     thread.
   end_jni_conv->ResetIterator(FrameOffset(end_out_arg_size));
+
   ThreadOffset<kPointerSize> jni_end(-1);
   if (reference_return) {
     // Pass result.
@@ -396,7 +418,9 @@ static CompiledMethod* ArtJniCompileMethodInternal(CompilerDriver* driver,
   } else {
     jni_end = is_synchronized
                   ? QUICK_ENTRYPOINT_OFFSET(kPointerSize, pJniMethodEndSynchronized)
-                  : QUICK_ENTRYPOINT_OFFSET(kPointerSize, pJniMethodEnd);
+                  : (is_fast_native
+                         ? QUICK_ENTRYPOINT_OFFSET(kPointerSize, pJniMethodFastEnd)
+                         : QUICK_ENTRYPOINT_OFFSET(kPointerSize, pJniMethodEnd));
   }
   // Pass saved local reference state.
   if (end_jni_conv->IsCurrentParamOnStack()) {
@@ -573,14 +597,17 @@ static void SetNativeParameter(JNIMacroAssembler<kPointerSize>* jni_asm,
   }
 }
 
-CompiledMethod* ArtQuickJniCompileMethod(CompilerDriver* compiler, uint32_t access_flags,
-                                         uint32_t method_idx, const DexFile& dex_file) {
+CompiledMethod* ArtQuickJniCompileMethod(CompilerDriver* compiler,
+                                         uint32_t access_flags,
+                                         uint32_t method_idx,
+                                         const DexFile& dex_file,
+                                         Compiler::JniOptimizationFlags optimization_flags) {
   if (Is64BitInstructionSet(compiler->GetInstructionSet())) {
     return ArtJniCompileMethodInternal<PointerSize::k64>(
-        compiler, access_flags, method_idx, dex_file);
+        compiler, access_flags, method_idx, dex_file, optimization_flags);
   } else {
     return ArtJniCompileMethodInternal<PointerSize::k32>(
-        compiler, access_flags, method_idx, dex_file);
+        compiler, access_flags, method_idx, dex_file, optimization_flags);
   }
 }
 
diff --git a/compiler/jni/quick/jni_compiler.h b/compiler/jni/quick/jni_compiler.h
index 46277f105c..26c32a31b8 100644
--- a/compiler/jni/quick/jni_compiler.h
+++ b/compiler/jni/quick/jni_compiler.h
@@ -17,6 +17,7 @@
 #ifndef ART_COMPILER_JNI_QUICK_JNI_COMPILER_H_
 #define ART_COMPILER_JNI_QUICK_JNI_COMPILER_H_
 
+#include "compiler.h"
 #include "dex_file.h"
 
 namespace art {
@@ -24,8 +25,11 @@ namespace art {
 class CompilerDriver;
 class CompiledMethod;
 
-CompiledMethod* ArtQuickJniCompileMethod(CompilerDriver* compiler, uint32_t access_flags,
-                                         uint32_t method_idx, const DexFile& dex_file);
+CompiledMethod* ArtQuickJniCompileMethod(CompilerDriver* compiler,
+                                         uint32_t access_flags,
+                                         uint32_t method_idx,
+                                         const DexFile& dex_file,
+                                         Compiler::JniOptimizationFlags optimization_flags);
 
 }  // namespace art
 
diff --git a/compiler/oat_test.cc b/compiler/oat_test.cc
index ce044e8e54..bf53bb2d0b 100644
--- a/compiler/oat_test.cc
+++ b/compiler/oat_test.cc
@@ -445,7 +445,7 @@ TEST_F(OatTest, OatHeaderSizeCheck) {
   EXPECT_EQ(72U, sizeof(OatHeader));
   EXPECT_EQ(4U, sizeof(OatMethodOffsets));
   EXPECT_EQ(20U, sizeof(OatQuickMethodHeader));
-  EXPECT_EQ(162 * static_cast<size_t>(GetInstructionSetPointerSize(kRuntimeISA)),
+  EXPECT_EQ(164 * static_cast<size_t>(GetInstructionSetPointerSize(kRuntimeISA)),
             sizeof(QuickEntryPoints));
 }
 
diff --git a/compiler/optimizing/optimizing_compiler.cc b/compiler/optimizing/optimizing_compiler.cc
index f7c82d1987..6aaa15fa02 100644
--- a/compiler/optimizing/optimizing_compiler.cc
+++ b/compiler/optimizing/optimizing_compiler.cc
@@ -283,8 +283,13 @@ class OptimizingCompiler FINAL : public Compiler {
 
   CompiledMethod* JniCompile(uint32_t access_flags,
                              uint32_t method_idx,
-                             const DexFile& dex_file) const OVERRIDE {
-    return ArtQuickJniCompileMethod(GetCompilerDriver(), access_flags, method_idx, dex_file);
+                             const DexFile& dex_file,
+                             JniOptimizationFlags optimization_flags) const OVERRIDE {
+    return ArtQuickJniCompileMethod(GetCompilerDriver(),
+                                    access_flags,
+                                    method_idx,
+                                    dex_file,
+                                    optimization_flags);
   }
 
   uintptr_t GetEntryPointOf(ArtMethod* method) const OVERRIDE
diff --git a/runtime/art_method.cc b/runtime/art_method.cc
index 60975d427a..d812590cc7 100644
--- a/runtime/art_method.cc
+++ b/runtime/art_method.cc
@@ -334,6 +334,23 @@ bool ArtMethod::IsOverridableByDefaultMethod() {
   return GetDeclaringClass()->IsInterface();
 }
 
+bool ArtMethod::IsAnnotatedWithFastNative() {
+  Thread* self = Thread::Current();
+  ScopedObjectAccess soa(self);
+  StackHandleScope<1> shs(self);
+
+  const DexFile& dex_file = GetDeclaringClass()->GetDexFile();
+
+  mirror::Class* fast_native_annotation =
+      soa.Decode<mirror::Class*>(WellKnownClasses::dalvik_annotation_optimization_FastNative);
+  Handle<mirror::Class> fast_native_handle(shs.NewHandle(fast_native_annotation));
+
+  // Note: Resolves any method annotations' classes as a side-effect.
+  // -- This seems allowed by the spec since it says we can preload any classes
+  //    referenced by another classes's constant pool table.
+  return dex_file.IsMethodAnnotationPresent(this, fast_native_handle, DexFile::kDexVisibilityBuild);
+}
+
 bool ArtMethod::EqualParameters(Handle<mirror::ObjectArray<mirror::Class>> params) {
   auto* dex_cache = GetDexCache();
   auto* dex_file = dex_cache->GetDexFile();
diff --git a/runtime/art_method.h b/runtime/art_method.h
index acf06fdc58..a90ef2359e 100644
--- a/runtime/art_method.h
+++ b/runtime/art_method.h
@@ -375,6 +375,10 @@ class ArtMethod FINAL {
     return (GetAccessFlags() & kAccMustCountLocks) != 0;
   }
 
+  // Checks to see if the method was annotated with @dalvik.annotation.optimization.FastNative
+  // -- Independent of kAccFastNative access flags.
+  bool IsAnnotatedWithFastNative();
+
   // Returns true if this method could be overridden by a default method.
   bool IsOverridableByDefaultMethod() SHARED_REQUIRES(Locks::mutator_lock_);
 
diff --git a/runtime/asm_support.h b/runtime/asm_support.h
index d4cee44458..e318f562c4 100644
--- a/runtime/asm_support.h
+++ b/runtime/asm_support.h
@@ -87,7 +87,7 @@ ADD_TEST_EQ(THREAD_SELF_OFFSET,
             art::Thread::SelfOffset<POINTER_SIZE>().Int32Value())
 
 // Offset of field Thread::tlsPtr_.thread_local_objects.
-#define THREAD_LOCAL_OBJECTS_OFFSET (THREAD_CARD_TABLE_OFFSET + 197 * __SIZEOF_POINTER__)
+#define THREAD_LOCAL_OBJECTS_OFFSET (THREAD_CARD_TABLE_OFFSET + 199 * __SIZEOF_POINTER__)
 ADD_TEST_EQ(THREAD_LOCAL_OBJECTS_OFFSET,
             art::Thread::ThreadLocalObjectsOffset<POINTER_SIZE>().Int32Value())
 // Offset of field Thread::tlsPtr_.thread_local_pos.
diff --git a/runtime/dex_file.cc b/runtime/dex_file.cc
index a6eb5f6261..90c678c1a8 100644
--- a/runtime/dex_file.cc
+++ b/runtime/dex_file.cc
@@ -1403,7 +1403,9 @@ mirror::ObjectArray<mirror::String>* DexFile::GetSignatureAnnotationForMethod(Ar
   return GetSignatureValue(method_class, annotation_set);
 }
 
-bool DexFile::IsMethodAnnotationPresent(ArtMethod* method, Handle<mirror::Class> annotation_class)
+bool DexFile::IsMethodAnnotationPresent(ArtMethod* method,
+                                        Handle<mirror::Class> annotation_class,
+                                        uint32_t visibility /* = kDexVisibilityRuntime */)
     const {
   const AnnotationSetItem* annotation_set = FindAnnotationSetForMethod(method);
   if (annotation_set == nullptr) {
@@ -1411,8 +1413,10 @@ bool DexFile::IsMethodAnnotationPresent(ArtMethod* method, Handle<mirror::Class>
   }
   StackHandleScope<1> hs(Thread::Current());
   Handle<mirror::Class> method_class(hs.NewHandle(method->GetDeclaringClass()));
-  const AnnotationItem* annotation_item = GetAnnotationItemFromAnnotationSet(
-      method_class, annotation_set, kDexVisibilityRuntime, annotation_class);
+  const AnnotationItem* annotation_item = GetAnnotationItemFromAnnotationSet(method_class,
+                                                                             annotation_set,
+                                                                             visibility,
+                                                                             annotation_class);
   return annotation_item != nullptr;
 }
 
diff --git a/runtime/dex_file.h b/runtime/dex_file.h
index 2eca495a13..59339eff9b 100644
--- a/runtime/dex_file.h
+++ b/runtime/dex_file.h
@@ -960,7 +960,9 @@ class DexFile {
       SHARED_REQUIRES(Locks::mutator_lock_);
   mirror::ObjectArray<mirror::String>* GetSignatureAnnotationForMethod(ArtMethod* method) const
       SHARED_REQUIRES(Locks::mutator_lock_);
-  bool IsMethodAnnotationPresent(ArtMethod* method, Handle<mirror::Class> annotation_class) const
+  bool IsMethodAnnotationPresent(ArtMethod* method,
+                                 Handle<mirror::Class> annotation_class,
+                                 uint32_t visibility = kDexVisibilityRuntime) const
       SHARED_REQUIRES(Locks::mutator_lock_);
 
   const AnnotationSetItem* FindAnnotationSetForClass(Handle<mirror::Class> klass) const
diff --git a/runtime/entrypoints/quick/quick_default_init_entrypoints.h b/runtime/entrypoints/quick/quick_default_init_entrypoints.h
index f98de95fcb..2a206c286a 100644
--- a/runtime/entrypoints/quick/quick_default_init_entrypoints.h
+++ b/runtime/entrypoints/quick/quick_default_init_entrypoints.h
@@ -73,11 +73,13 @@ void DefaultInitEntryPoints(JniEntryPoints* jpoints, QuickEntryPoints* qpoints)
 
   // JNI
   qpoints->pJniMethodStart = JniMethodStart;
+  qpoints->pJniMethodFastStart = JniMethodFastStart;
   qpoints->pJniMethodStartSynchronized = JniMethodStartSynchronized;
   qpoints->pJniMethodEnd = JniMethodEnd;
   qpoints->pJniMethodEndSynchronized = JniMethodEndSynchronized;
   qpoints->pJniMethodEndWithReference = JniMethodEndWithReference;
   qpoints->pJniMethodEndWithReferenceSynchronized = JniMethodEndWithReferenceSynchronized;
+  qpoints->pJniMethodFastEnd = JniMethodFastEnd;
   qpoints->pQuickGenericJniTrampoline = art_quick_generic_jni_trampoline;
 
   // Locks
diff --git a/runtime/entrypoints/quick/quick_entrypoints.h b/runtime/entrypoints/quick/quick_entrypoints.h
index f5b68fa922..08e0d6e41e 100644
--- a/runtime/entrypoints/quick/quick_entrypoints.h
+++ b/runtime/entrypoints/quick/quick_entrypoints.h
@@ -52,10 +52,13 @@ struct PACKED(4) QuickEntryPoints {
 // JNI entrypoints.
 // TODO: NO_THREAD_SAFETY_ANALYSIS due to different control paths depending on fast JNI.
 extern uint32_t JniMethodStart(Thread* self) NO_THREAD_SAFETY_ANALYSIS HOT_ATTR;
+extern uint32_t JniMethodFastStart(Thread* self) NO_THREAD_SAFETY_ANALYSIS HOT_ATTR;
 extern uint32_t JniMethodStartSynchronized(jobject to_lock, Thread* self)
     NO_THREAD_SAFETY_ANALYSIS HOT_ATTR;
 extern void JniMethodEnd(uint32_t saved_local_ref_cookie, Thread* self)
     NO_THREAD_SAFETY_ANALYSIS HOT_ATTR;
+extern void JniMethodFastEnd(uint32_t saved_local_ref_cookie, Thread* self)
+    NO_THREAD_SAFETY_ANALYSIS HOT_ATTR;
 extern void JniMethodEndSynchronized(uint32_t saved_local_ref_cookie, jobject locked,
                                      Thread* self)
     NO_THREAD_SAFETY_ANALYSIS HOT_ATTR;
diff --git a/runtime/entrypoints/quick/quick_entrypoints_list.h b/runtime/entrypoints/quick/quick_entrypoints_list.h
index 07f0394773..74c928ad8d 100644
--- a/runtime/entrypoints/quick/quick_entrypoints_list.h
+++ b/runtime/entrypoints/quick/quick_entrypoints_list.h
@@ -72,8 +72,10 @@
   V(HandleFillArrayData, void, void*, void*) \
 \
   V(JniMethodStart, uint32_t, Thread*) \
+  V(JniMethodFastStart, uint32_t, Thread*) \
   V(JniMethodStartSynchronized, uint32_t, jobject, Thread*) \
   V(JniMethodEnd, void, uint32_t, Thread*) \
+  V(JniMethodFastEnd, void, uint32_t, Thread*) \
   V(JniMethodEndSynchronized, void, uint32_t, jobject, Thread*) \
   V(JniMethodEndWithReference, mirror::Object*, jobject, uint32_t, Thread*) \
   V(JniMethodEndWithReferenceSynchronized, mirror::Object*, jobject, uint32_t, jobject, Thread*) \
@@ -195,7 +197,8 @@
   V(ReadBarrierMarkReg28, mirror::Object*, mirror::Object*) \
   V(ReadBarrierMarkReg29, mirror::Object*, mirror::Object*) \
   V(ReadBarrierSlow, mirror::Object*, mirror::Object*, mirror::Object*, uint32_t) \
-  V(ReadBarrierForRootSlow, mirror::Object*, GcRoot<mirror::Object>*)
+  V(ReadBarrierForRootSlow, mirror::Object*, GcRoot<mirror::Object>*) \
+\
 
 #endif  // ART_RUNTIME_ENTRYPOINTS_QUICK_QUICK_ENTRYPOINTS_LIST_H_
 #undef ART_RUNTIME_ENTRYPOINTS_QUICK_QUICK_ENTRYPOINTS_LIST_H_   // #define is only for lint.
diff --git a/runtime/entrypoints/quick/quick_jni_entrypoints.cc b/runtime/entrypoints/quick/quick_jni_entrypoints.cc
index 58f256a191..c06824c777 100644
--- a/runtime/entrypoints/quick/quick_jni_entrypoints.cc
+++ b/runtime/entrypoints/quick/quick_jni_entrypoints.cc
@@ -29,6 +29,21 @@ extern void ReadBarrierJni(mirror::CompressedReference<mirror::Object>* handle_o
   handle_on_stack->Assign(to_ref);
 }
 
+// Called on entry to fast JNI, push a new local reference table only.
+extern uint32_t JniMethodFastStart(Thread* self) {
+  JNIEnvExt* env = self->GetJniEnv();
+  DCHECK(env != nullptr);
+  uint32_t saved_local_ref_cookie = env->local_ref_cookie;
+  env->local_ref_cookie = env->locals.GetSegmentState();
+
+  if (kIsDebugBuild) {
+    ArtMethod* native_method = *self->GetManagedStack()->GetTopQuickFrame();
+    CHECK(native_method->IsAnnotatedWithFastNative()) << PrettyMethod(native_method);
+  }
+
+  return saved_local_ref_cookie;
+}
+
 // Called on entry to JNI, transition out of Runnable and release share of mutator_lock_.
 extern uint32_t JniMethodStart(Thread* self) {
   JNIEnvExt* env = self->GetJniEnv();
@@ -73,11 +88,32 @@ static void PopLocalReferences(uint32_t saved_local_ref_cookie, Thread* self)
   self->PopHandleScope();
 }
 
+// TODO: These should probably be templatized or macro-ized.
+// Otherwise there's just too much repetitive boilerplate.
+
 extern void JniMethodEnd(uint32_t saved_local_ref_cookie, Thread* self) {
   GoToRunnable(self);
   PopLocalReferences(saved_local_ref_cookie, self);
 }
 
+extern void JniMethodFastEnd(uint32_t saved_local_ref_cookie, Thread* self) {
+  // inlined fast version of GoToRunnable(self);
+
+  if (kIsDebugBuild) {
+    ArtMethod* native_method = *self->GetManagedStack()->GetTopQuickFrame();
+    CHECK(native_method->IsAnnotatedWithFastNative()) << PrettyMethod(native_method);
+  }
+
+  if (UNLIKELY(self->TestAllFlags())) {
+    // In fast JNI mode we never transitioned out of runnable. Perform a suspend check if there
+    // is a flag raised.
+    DCHECK(Locks::mutator_lock_->IsSharedHeld(self));
+    self->CheckSuspend();
+  }
+
+  PopLocalReferences(saved_local_ref_cookie, self);
+}
+
 extern void JniMethodEndSynchronized(uint32_t saved_local_ref_cookie, jobject locked,
                                      Thread* self) {
   GoToRunnable(self);
@@ -85,6 +121,10 @@ extern void JniMethodEndSynchronized(uint32_t saved_local_ref_cookie, jobject lo
   PopLocalReferences(saved_local_ref_cookie, self);
 }
 
+// TODO: JniMethodFastEndWithReference
+// (Probably don't need to have a synchronized variant since
+// it already has to do atomic operations)
+
 // Common result handling for EndWithReference.
 static mirror::Object* JniMethodEndWithReferenceHandleResult(jobject result,
                                                              uint32_t saved_local_ref_cookie,
diff --git a/runtime/entrypoints_order_test.cc b/runtime/entrypoints_order_test.cc
index e3203dc14f..004cdc47b9 100644
--- a/runtime/entrypoints_order_test.cc
+++ b/runtime/entrypoints_order_test.cc
@@ -211,11 +211,14 @@ class EntrypointsOrderTest : public CommonRuntimeTest {
     EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pAputObjectWithBoundCheck, pAputObject, sizeof(void*));
     EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pAputObject, pHandleFillArrayData, sizeof(void*));
     EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pHandleFillArrayData, pJniMethodStart, sizeof(void*));
-    EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pJniMethodStart, pJniMethodStartSynchronized,
+    EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pJniMethodStart, pJniMethodFastStart,
+                         sizeof(void*));
+    EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pJniMethodFastStart, pJniMethodStartSynchronized,
                          sizeof(void*));
     EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pJniMethodStartSynchronized, pJniMethodEnd,
                          sizeof(void*));
-    EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pJniMethodEnd, pJniMethodEndSynchronized, sizeof(void*));
+    EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pJniMethodEnd, pJniMethodFastEnd, sizeof(void*));
+    EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pJniMethodFastEnd, pJniMethodEndSynchronized, sizeof(void*));
     EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pJniMethodEndSynchronized, pJniMethodEndWithReference,
                          sizeof(void*));
     EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pJniMethodEndWithReference,
diff --git a/runtime/jni_internal.cc b/runtime/jni_internal.cc
index e1a4e2a7b6..c3224757d8 100644
--- a/runtime/jni_internal.cc
+++ b/runtime/jni_internal.cc
@@ -2205,6 +2205,7 @@ class JNI {
 
       VLOG(jni) << "[Registering JNI native method " << PrettyMethod(m) << "]";
 
+      is_fast = is_fast || m->IsFastNative();  // Merge with @FastNative state.
       m->RegisterNative(fnPtr, is_fast);
     }
     return JNI_OK;
diff --git a/runtime/parsed_options.cc b/runtime/parsed_options.cc
index c7e4f8b343..2a040a3d5e 100644
--- a/runtime/parsed_options.cc
+++ b/runtime/parsed_options.cc
@@ -487,7 +487,7 @@ bool ParsedOptions::DoParse(const RuntimeOptions& options,
   args.SetIfMissing(M::ParallelGCThreads, gc::Heap::kDefaultEnableParallelGC ?
       static_cast<unsigned int>(sysconf(_SC_NPROCESSORS_CONF) - 1u) : 0u);
 
-  // -Xverbose:
+  // -verbose:
   {
     LogVerbosity *log_verbosity = args.Get(M::Verbose);
     if (log_verbosity != nullptr) {
diff --git a/runtime/thread.cc b/runtime/thread.cc
index 33267364e5..b35a614e99 100644
--- a/runtime/thread.cc
+++ b/runtime/thread.cc
@@ -2599,6 +2599,9 @@ void Thread::DumpThreadOffset(std::ostream& os, uint32_t offset) {
   QUICK_ENTRY_POINT_INFO(pReadBarrierMarkReg29)
   QUICK_ENTRY_POINT_INFO(pReadBarrierSlow)
   QUICK_ENTRY_POINT_INFO(pReadBarrierForRootSlow)
+
+  QUICK_ENTRY_POINT_INFO(pJniMethodFastStart)
+  QUICK_ENTRY_POINT_INFO(pJniMethodFastEnd)
 #undef QUICK_ENTRY_POINT_INFO
 
   os << offset;
diff --git a/runtime/well_known_classes.cc b/runtime/well_known_classes.cc
index 48deb35c1e..ddce344af3 100644
--- a/runtime/well_known_classes.cc
+++ b/runtime/well_known_classes.cc
@@ -30,6 +30,7 @@
 namespace art {
 
 jclass WellKnownClasses::com_android_dex_Dex;
+jclass WellKnownClasses::dalvik_annotation_optimization_FastNative;
 jclass WellKnownClasses::dalvik_system_DexFile;
 jclass WellKnownClasses::dalvik_system_DexPathList;
 jclass WellKnownClasses::dalvik_system_DexPathList__Element;
@@ -215,6 +216,7 @@ static jmethodID CachePrimitiveBoxingMethod(JNIEnv* env, char prim_name, const c
 
 void WellKnownClasses::Init(JNIEnv* env) {
   com_android_dex_Dex = CacheClass(env, "com/android/dex/Dex");
+  dalvik_annotation_optimization_FastNative = CacheClass(env, "dalvik/annotation/optimization/FastNative");
   dalvik_system_DexFile = CacheClass(env, "dalvik/system/DexFile");
   dalvik_system_DexPathList = CacheClass(env, "dalvik/system/DexPathList");
   dalvik_system_DexPathList__Element = CacheClass(env, "dalvik/system/DexPathList$Element");
diff --git a/runtime/well_known_classes.h b/runtime/well_known_classes.h
index c9faf69680..b8e05b8e9d 100644
--- a/runtime/well_known_classes.h
+++ b/runtime/well_known_classes.h
@@ -41,6 +41,7 @@ struct WellKnownClasses {
       SHARED_REQUIRES(Locks::mutator_lock_);
 
   static jclass com_android_dex_Dex;
+  static jclass dalvik_annotation_optimization_FastNative;
   static jclass dalvik_system_DexFile;
   static jclass dalvik_system_DexPathList;
   static jclass dalvik_system_DexPathList__Element;
diff --git a/test/004-JniTest/jni_test.cc b/test/004-JniTest/jni_test.cc
index 8619ff7f3e..bb18a707f4 100644
--- a/test/004-JniTest/jni_test.cc
+++ b/test/004-JniTest/jni_test.cc
@@ -27,11 +27,18 @@ namespace art {
 
 static JavaVM* jvm = nullptr;
 
+static jint Java_Main_intFastNativeMethod(JNIEnv*, jclass, jint a, jint b, jint c);
+
+static JNINativeMethod sMainMethods[] = {
+  {"intFastNativeMethod", "(III)I", reinterpret_cast<void*>(Java_Main_intFastNativeMethod) }
+};
+
 extern "C" JNIEXPORT jint JNI_OnLoad(JavaVM *vm, void*) {
   CHECK(vm != nullptr);
   CHECK(jvm == nullptr);
   jvm = vm;
   std::cout << "JNI_OnLoad called" << std::endl;
+
   return JNI_VERSION_1_6;
 }
 
@@ -740,5 +747,24 @@ extern "C" JNIEXPORT void JNICALL Java_Main_testInvokeLambdaMethod(JNIEnv* e, jc
   InvokeSpecificMethod(e, l, "sayHi");
 }
 
+// Register on-demand because many tests share this JNI library and
+// we can't unconditionally register them.
+extern "C" JNIEXPORT jboolean JNICALL Java_Main_registerNativesJniTest(JNIEnv* e, jclass kls) {
+  const size_t numMethods = sizeof(sMainMethods)/sizeof(JNINativeMethod);
+
+  if (e->RegisterNatives(kls, sMainMethods, numMethods) < 0) {
+      std::cerr << "RegisterNatives failed for 'Main'" << std::endl;
+      return JNI_FALSE;
+  }
+
+  return JNI_TRUE;
+}
+
+// Annotated with @FastNative in Java code. Doesn't need to be explicitly registered with "!".
+// NOTE: Has to be registered explicitly to avoid mutator lock check failures.
+static jint Java_Main_intFastNativeMethod(JNIEnv*, jclass, jint a, jint b, jint c) {
+  return a + b + c;
+}
+
 }  // namespace art
 
diff --git a/test/004-JniTest/src/Main.java b/test/004-JniTest/src/Main.java
index 0221900113..573afdbd13 100644
--- a/test/004-JniTest/src/Main.java
+++ b/test/004-JniTest/src/Main.java
@@ -18,6 +18,8 @@ import java.lang.reflect.InvocationHandler;
 import java.lang.reflect.Method;
 import java.lang.reflect.Proxy;
 
+import dalvik.annotation.optimization.FastNative;
+
 public class Main {
     public static void main(String[] args) {
         System.loadLibrary(args[0]);
@@ -44,8 +46,13 @@ public class Main {
         testInvokeLambdaMethod(() -> { System.out.println("hi-lambda: " + lambda); });
         String def = "δ";
         testInvokeLambdaDefaultMethod(() -> { System.out.println("hi-default " + def + lambda); });
+
+        registerNativesJniTest();
+        testFastNativeMethods();
     }
 
+    private static native boolean registerNativesJniTest();
+
     private static native void testCallDefaultMethods();
 
     private static native void testFindClassOnAttachedNativeThread();
@@ -263,6 +270,25 @@ public class Main {
     private static native void testInvokeLambdaMethod(LambdaInterface iface);
 
     private static native void testInvokeLambdaDefaultMethod(LambdaInterface iface);
+
+    // Test invoking @FastNative methods works correctly.
+
+    // Return sum of a+b+c.
+    @FastNative
+    static native int intFastNativeMethod(int a, int b, int c);
+
+    private static void testFastNativeMethods() {
+      int returns[] = { 0, 3, 6, 9, 12 };
+      for (int i = 0; i < returns.length; i++) {
+        int result = intFastNativeMethod(i, i, i);
+        if (returns[i] != result) {
+          System.out.println("FastNative Int Run " + i + " with " + returns[i] + " vs " + result);
+          throw new AssertionError();
+        }
+      }
+    }
+
+
 }
 
 @FunctionalInterface
diff --git a/test/MyClassNatives/MyClassNatives.java b/test/MyClassNatives/MyClassNatives.java
index 4a8b0e0241..45cfd0f59e 100644
--- a/test/MyClassNatives/MyClassNatives.java
+++ b/test/MyClassNatives/MyClassNatives.java
@@ -14,6 +14,8 @@
  * limitations under the License.
  */
 
+import dalvik.annotation.optimization.FastNative;
+
 class MyClassNatives {
     native void throwException();
     native void foo();
@@ -102,4 +104,9 @@ class MyClassNatives {
     static native boolean returnTrue();
     static native boolean returnFalse();
     static native int returnInt();
+
+    // Check for @FastNative annotation presence [or lack of presence].
+    public static native void normalNative();
+    @FastNative
+    public static native void fastNative();
 }