jni: Add read barrier fast path to jni compiler

Static method dispatch via JNI requires a read barrier
for the ArtMethod::GetDeclaringClass() load before adding it to the
JNI StackHandleScope.

We used to call ReadBarrierJni unconditionally but add a branch
to skip calling it if the GC is not currently in the marking phase.

Test: ART_USE_READ_BARRIER=true make test-art-host test-art-target
Bug: 30437917
Change-Id: I4f505ebde17c0a67209c7bb51b3f39e37a06373a
diff --git a/compiler/jni/quick/jni_compiler.cc b/compiler/jni/quick/jni_compiler.cc
index bfb342f..13d8c16 100644
--- a/compiler/jni/quick/jni_compiler.cc
+++ b/compiler/jni/quick/jni_compiler.cc
@@ -276,10 +276,32 @@
   __ IncreaseFrameSize(main_out_arg_size);
 
   // Call the read barrier for the declaring class loaded from the method for a static call.
+  // Skip this for @CriticalNative because we didn't build a HandleScope to begin with.
   // Note that we always have outgoing param space available for at least two params.
   if (kUseReadBarrier && is_static && !is_critical_native) {
-    // XX: Why is this necessary only for the jclass? Why not for every single object ref?
-    // Skip this for @CriticalNative because we didn't build a HandleScope to begin with.
+    const bool kReadBarrierFastPath =
+        (instruction_set != kMips) && (instruction_set != kMips64);
+    std::unique_ptr<JNIMacroLabel> skip_cold_path_label;
+    if (kReadBarrierFastPath) {
+      skip_cold_path_label = __ CreateLabel();
+      // Fast path for supported targets.
+      //
+      // Check if gc_is_marking is set -- if it's not, we don't need
+      // a read barrier so skip it.
+      __ LoadFromThread(main_jni_conv->InterproceduralScratchRegister(),
+                        Thread::IsGcMarkingOffset<kPointerSize>(),
+                        Thread::IsGcMarkingSize());
+      // Jump over the slow path if gc is marking is false.
+      __ Jump(skip_cold_path_label.get(),
+              JNIMacroUnaryCondition::kZero,
+              main_jni_conv->InterproceduralScratchRegister());
+    }
+
+    // Construct slow path for read barrier:
+    //
+    // Call into the runtime's ReadBarrierJni and have it fix up
+    // the object address if it was moved.
+
     ThreadOffset<kPointerSize> read_barrier = QUICK_ENTRYPOINT_OFFSET(kPointerSize,
                                                                       pReadBarrierJni);
     main_jni_conv->ResetIterator(FrameOffset(main_out_arg_size));
@@ -310,6 +332,10 @@
       __ CallFromThread(read_barrier, main_jni_conv->InterproceduralScratchRegister());
     }
     main_jni_conv->ResetIterator(FrameOffset(main_out_arg_size));  // Reset.
+
+    if (kReadBarrierFastPath) {
+      __ Bind(skip_cold_path_label.get());
+    }
   }
 
   // 6. Call into appropriate JniMethodStart passing Thread* so that transition out of Runnable
diff --git a/compiler/utils/arm/jni_macro_assembler_arm_vixl.cc b/compiler/utils/arm/jni_macro_assembler_arm_vixl.cc
index 14d29c4..8a9fd90 100644
--- a/compiler/utils/arm/jni_macro_assembler_arm_vixl.cc
+++ b/compiler/utils/arm/jni_macro_assembler_arm_vixl.cc
@@ -254,10 +254,10 @@
   return Load(m_dst.AsArm(), sp, src.Int32Value(), size);
 }
 
-void ArmVIXLJNIMacroAssembler::LoadFromThread(ManagedRegister m_dst ATTRIBUTE_UNUSED,
-                                              ThreadOffset32 src ATTRIBUTE_UNUSED,
-                                              size_t size ATTRIBUTE_UNUSED) {
-  UNIMPLEMENTED(FATAL);
+void ArmVIXLJNIMacroAssembler::LoadFromThread(ManagedRegister m_dst,
+                                              ThreadOffset32 src,
+                                              size_t size) {
+  return Load(m_dst.AsArm(), tr, src.Int32Value(), size);
 }
 
 void ArmVIXLJNIMacroAssembler::LoadRawPtrFromThread(ManagedRegister m_dst, ThreadOffset32 offs) {
@@ -558,6 +558,38 @@
   // TODO: think about using CBNZ here.
 }
 
+std::unique_ptr<JNIMacroLabel> ArmVIXLJNIMacroAssembler::CreateLabel() {
+  return std::unique_ptr<JNIMacroLabel>(new ArmVIXLJNIMacroLabel());
+}
+
+void ArmVIXLJNIMacroAssembler::Jump(JNIMacroLabel* label) {
+  CHECK(label != nullptr);
+  ___ B(ArmVIXLJNIMacroLabel::Cast(label)->AsArm());
+}
+
+void ArmVIXLJNIMacroAssembler::Jump(JNIMacroLabel* label,
+                                    JNIMacroUnaryCondition condition,
+                                    ManagedRegister test) {
+  CHECK(label != nullptr);
+
+  switch (condition) {
+    case JNIMacroUnaryCondition::kZero:
+      ___ Cbz(test.AsArm().AsVIXLRegister(), ArmVIXLJNIMacroLabel::Cast(label)->AsArm());
+      break;
+    case JNIMacroUnaryCondition::kNotZero:
+      ___ Cbnz(test.AsArm().AsVIXLRegister(), ArmVIXLJNIMacroLabel::Cast(label)->AsArm());
+      break;
+    default:
+      LOG(FATAL) << "Not implemented unary condition: " << static_cast<int>(condition);
+      UNREACHABLE();
+  }
+}
+
+void ArmVIXLJNIMacroAssembler::Bind(JNIMacroLabel* label) {
+  CHECK(label != nullptr);
+  ___ Bind(ArmVIXLJNIMacroLabel::Cast(label)->AsArm());
+}
+
 void ArmVIXLJNIMacroAssembler::EmitExceptionPoll(
     ArmVIXLJNIMacroAssembler::ArmException* exception) {
   ___ Bind(exception->Entry());
@@ -588,9 +620,14 @@
   if (dest.IsNoRegister()) {
     CHECK_EQ(0u, size) << dest;
   } else if (dest.IsCoreRegister()) {
-    CHECK_EQ(4u, size) << dest;
     CHECK(!dest.AsVIXLRegister().Is(sp)) << dest;
-    ___ Ldr(dest.AsVIXLRegister(), MemOperand(base, offset));
+
+    if (size == 1u) {
+      ___ Ldrb(dest.AsVIXLRegister(), MemOperand(base, offset));
+    } else {
+      CHECK_EQ(4u, size) << dest;
+      ___ Ldr(dest.AsVIXLRegister(), MemOperand(base, offset));
+    }
   } else if (dest.IsRegisterPair()) {
     CHECK_EQ(8u, size) << dest;
     ___ Ldr(dest.AsVIXLRegisterPairLow(),  MemOperand(base, offset));
diff --git a/compiler/utils/arm/jni_macro_assembler_arm_vixl.h b/compiler/utils/arm/jni_macro_assembler_arm_vixl.h
index 9fc683d..f3baf1f 100644
--- a/compiler/utils/arm/jni_macro_assembler_arm_vixl.h
+++ b/compiler/utils/arm/jni_macro_assembler_arm_vixl.h
@@ -187,6 +187,15 @@
   // and branch to a ExceptionSlowPath if it is.
   void ExceptionPoll(ManagedRegister scratch, size_t stack_adjust);
 
+  // Create a new label that can be used with Jump/Bind calls.
+  std::unique_ptr<JNIMacroLabel> CreateLabel() OVERRIDE;
+  // Emit an unconditional jump to the label.
+  void Jump(JNIMacroLabel* label) OVERRIDE;
+  // Emit a conditional jump to the label by applying a unary condition test to the register.
+  void Jump(JNIMacroLabel* label, JNIMacroUnaryCondition cond, ManagedRegister test) OVERRIDE;
+  // Code at this offset will serve as the target for the Jump call.
+  void Bind(JNIMacroLabel* label) OVERRIDE;
+
   void MemoryBarrier(ManagedRegister scratch) OVERRIDE;
 
   void EmitExceptionPoll(ArmVIXLJNIMacroAssembler::ArmException *exception);
@@ -219,6 +228,16 @@
   friend class ArmVIXLAssemblerTest_VixlStoreToOffset_Test;
 };
 
+class ArmVIXLJNIMacroLabel FINAL
+    : public JNIMacroLabelCommon<ArmVIXLJNIMacroLabel,
+                                 vixl32::Label,
+                                 kArm> {
+ public:
+  vixl32::Label* AsArm() {
+    return AsPlatformLabel();
+  }
+};
+
 }  // namespace arm
 }  // namespace art
 
diff --git a/compiler/utils/arm64/jni_macro_assembler_arm64.cc b/compiler/utils/arm64/jni_macro_assembler_arm64.cc
index dfdcd11..9cd6884 100644
--- a/compiler/utils/arm64/jni_macro_assembler_arm64.cc
+++ b/compiler/utils/arm64/jni_macro_assembler_arm64.cc
@@ -262,9 +262,12 @@
     ___ Ldr(reg_w(dest.AsWRegister()), MEM_OP(reg_x(base), offset));
   } else if (dest.IsXRegister()) {
     CHECK_NE(dest.AsXRegister(), SP) << dest;
-    if (size == 4u) {
+
+    if (size == 1u) {
+      ___ Ldrb(reg_w(dest.AsOverlappingWRegister()), MEM_OP(reg_x(base), offset));
+    } else if (size == 4u) {
       ___ Ldr(reg_w(dest.AsOverlappingWRegister()), MEM_OP(reg_x(base), offset));
-    } else {
+    }  else {
       CHECK_EQ(8u, size) << dest;
       ___ Ldr(reg_x(dest.AsXRegister()), MEM_OP(reg_x(base), offset));
     }
@@ -627,6 +630,38 @@
   ___ Cbnz(reg_x(scratch.AsXRegister()), exception_blocks_.back()->Entry());
 }
 
+std::unique_ptr<JNIMacroLabel> Arm64JNIMacroAssembler::CreateLabel() {
+  return std::unique_ptr<JNIMacroLabel>(new Arm64JNIMacroLabel());
+}
+
+void Arm64JNIMacroAssembler::Jump(JNIMacroLabel* label) {
+  CHECK(label != nullptr);
+  ___ B(Arm64JNIMacroLabel::Cast(label)->AsArm64());
+}
+
+void Arm64JNIMacroAssembler::Jump(JNIMacroLabel* label,
+                                  JNIMacroUnaryCondition condition,
+                                  ManagedRegister test) {
+  CHECK(label != nullptr);
+
+  switch (condition) {
+    case JNIMacroUnaryCondition::kZero:
+      ___ Cbz(reg_x(test.AsArm64().AsXRegister()), Arm64JNIMacroLabel::Cast(label)->AsArm64());
+      break;
+    case JNIMacroUnaryCondition::kNotZero:
+      ___ Cbnz(reg_x(test.AsArm64().AsXRegister()), Arm64JNIMacroLabel::Cast(label)->AsArm64());
+      break;
+    default:
+      LOG(FATAL) << "Not implemented unary condition: " << static_cast<int>(condition);
+      UNREACHABLE();
+  }
+}
+
+void Arm64JNIMacroAssembler::Bind(JNIMacroLabel* label) {
+  CHECK(label != nullptr);
+  ___ Bind(Arm64JNIMacroLabel::Cast(label)->AsArm64());
+}
+
 void Arm64JNIMacroAssembler::EmitExceptionPoll(Arm64Exception *exception) {
   UseScratchRegisterScope temps(asm_.GetVIXLAssembler());
   temps.Exclude(reg_x(exception->scratch_.AsXRegister()));
diff --git a/compiler/utils/arm64/jni_macro_assembler_arm64.h b/compiler/utils/arm64/jni_macro_assembler_arm64.h
index b9f6854..264e99a 100644
--- a/compiler/utils/arm64/jni_macro_assembler_arm64.h
+++ b/compiler/utils/arm64/jni_macro_assembler_arm64.h
@@ -168,6 +168,15 @@
   // and branch to a ExceptionSlowPath if it is.
   void ExceptionPoll(ManagedRegister scratch, size_t stack_adjust) OVERRIDE;
 
+  // Create a new label that can be used with Jump/Bind calls.
+  std::unique_ptr<JNIMacroLabel> CreateLabel() OVERRIDE;
+  // Emit an unconditional jump to the label.
+  void Jump(JNIMacroLabel* label) OVERRIDE;
+  // Emit a conditional jump to the label by applying a unary condition test to the register.
+  void Jump(JNIMacroLabel* label, JNIMacroUnaryCondition cond, ManagedRegister test) OVERRIDE;
+  // Code at this offset will serve as the target for the Jump call.
+  void Bind(JNIMacroLabel* label) OVERRIDE;
+
  private:
   class Arm64Exception {
    public:
@@ -222,6 +231,16 @@
   ArenaVector<std::unique_ptr<Arm64Exception>> exception_blocks_;
 };
 
+class Arm64JNIMacroLabel FINAL
+    : public JNIMacroLabelCommon<Arm64JNIMacroLabel,
+                                 vixl::aarch64::Label,
+                                 kArm64> {
+ public:
+  vixl::aarch64::Label* AsArm64() {
+    return AsPlatformLabel();
+  }
+};
+
 }  // namespace arm64
 }  // namespace art
 
diff --git a/compiler/utils/jni_macro_assembler.h b/compiler/utils/jni_macro_assembler.h
index 0119ae9..59a1a48 100644
--- a/compiler/utils/jni_macro_assembler.h
+++ b/compiler/utils/jni_macro_assembler.h
@@ -35,6 +35,12 @@
 class DebugFrameOpCodeWriterForAssembler;
 class InstructionSetFeatures;
 class MemoryRegion;
+class JNIMacroLabel;
+
+enum class JNIMacroUnaryCondition {
+  kZero,
+  kNotZero
+};
 
 template <PointerSize kPointerSize>
 class JNIMacroAssembler : public DeletableArenaObject<kArenaAllocAssembler> {
@@ -193,6 +199,15 @@
   // and branch to a ExceptionSlowPath if it is.
   virtual void ExceptionPoll(ManagedRegister scratch, size_t stack_adjust) = 0;
 
+  // Create a new label that can be used with Jump/Bind calls.
+  virtual std::unique_ptr<JNIMacroLabel> CreateLabel() = 0;
+  // Emit an unconditional jump to the label.
+  virtual void Jump(JNIMacroLabel* label) = 0;
+  // Emit a conditional jump to the label by applying a unary condition test to the register.
+  virtual void Jump(JNIMacroLabel* label, JNIMacroUnaryCondition cond, ManagedRegister test) = 0;
+  // Code at this offset will serve as the target for the Jump call.
+  virtual void Bind(JNIMacroLabel* label) = 0;
+
   virtual ~JNIMacroAssembler() {}
 
   /**
@@ -205,6 +220,28 @@
   explicit JNIMacroAssembler() {}
 };
 
+// A "Label" class used with the JNIMacroAssembler
+// allowing one to use branches (jumping from one place to another).
+//
+// This is just an interface, so every platform must provide
+// its own implementation of it.
+//
+// It is only safe to use a label created
+// via JNIMacroAssembler::CreateLabel with that same macro assembler.
+class JNIMacroLabel {
+ public:
+  virtual ~JNIMacroLabel() = 0;
+
+  const InstructionSet isa_;
+ protected:
+  explicit JNIMacroLabel(InstructionSet isa) : isa_(isa) {}
+};
+
+inline JNIMacroLabel::~JNIMacroLabel() {
+  // Compulsory definition for a pure virtual destructor
+  // to avoid linking errors.
+}
+
 template <typename T, PointerSize kPointerSize>
 class JNIMacroAssemblerFwd : public JNIMacroAssembler<kPointerSize> {
  public:
@@ -230,6 +267,30 @@
   T asm_;
 };
 
+template <typename Self, typename PlatformLabel, InstructionSet kIsa>
+class JNIMacroLabelCommon : public JNIMacroLabel {
+ public:
+  static Self* Cast(JNIMacroLabel* label) {
+    CHECK(label != nullptr);
+    CHECK_EQ(kIsa, label->isa_);
+
+    return reinterpret_cast<Self*>(label);
+  }
+
+ protected:
+  PlatformLabel* AsPlatformLabel() {
+    return &label_;
+  }
+
+  JNIMacroLabelCommon() : JNIMacroLabel(kIsa) {
+  }
+
+  virtual ~JNIMacroLabelCommon() OVERRIDE {}
+
+ private:
+  PlatformLabel label_;
+};
+
 }  // namespace art
 
 #endif  // ART_COMPILER_UTILS_JNI_MACRO_ASSEMBLER_H_
diff --git a/compiler/utils/mips/assembler_mips.h b/compiler/utils/mips/assembler_mips.h
index e1255f7..b932fb8 100644
--- a/compiler/utils/mips/assembler_mips.h
+++ b/compiler/utils/mips/assembler_mips.h
@@ -179,6 +179,8 @@
 
 class MipsAssembler FINAL : public Assembler, public JNIMacroAssembler<PointerSize::k32> {
  public:
+  using JNIBase = JNIMacroAssembler<PointerSize::k32>;
+
   explicit MipsAssembler(ArenaAllocator* arena,
                          const MipsInstructionSetFeatures* instruction_set_features = nullptr)
       : Assembler(arena),
@@ -723,6 +725,34 @@
     UNIMPLEMENTED(FATAL) << "Do not use Jump for MIPS";
   }
 
+  // Don't warn about a different virtual Bind/Jump in the base class.
+  using JNIBase::Bind;
+  using JNIBase::Jump;
+
+  // Create a new label that can be used with Jump/Bind calls.
+  std::unique_ptr<JNIMacroLabel> CreateLabel() OVERRIDE {
+    LOG(FATAL) << "Not implemented on MIPS32";
+    UNREACHABLE();
+  }
+  // Emit an unconditional jump to the label.
+  void Jump(JNIMacroLabel* label ATTRIBUTE_UNUSED) OVERRIDE {
+    LOG(FATAL) << "Not implemented on MIPS32";
+    UNREACHABLE();
+  }
+  // Emit a conditional jump to the label by applying a unary condition test to the register.
+  void Jump(JNIMacroLabel* label ATTRIBUTE_UNUSED,
+            JNIMacroUnaryCondition cond ATTRIBUTE_UNUSED,
+            ManagedRegister test ATTRIBUTE_UNUSED) OVERRIDE {
+    LOG(FATAL) << "Not implemented on MIPS32";
+    UNREACHABLE();
+  }
+
+  // Code at this offset will serve as the target for the Jump call.
+  void Bind(JNIMacroLabel* label ATTRIBUTE_UNUSED) OVERRIDE {
+    LOG(FATAL) << "Not implemented on MIPS32";
+    UNREACHABLE();
+  }
+
   // Create a new literal with a given value.
   // NOTE: Force the template parameter to be explicitly specified.
   template <typename T>
diff --git a/compiler/utils/mips64/assembler_mips64.h b/compiler/utils/mips64/assembler_mips64.h
index 6277b5d..238cb9d 100644
--- a/compiler/utils/mips64/assembler_mips64.h
+++ b/compiler/utils/mips64/assembler_mips64.h
@@ -335,6 +335,8 @@
 
 class Mips64Assembler FINAL : public Assembler, public JNIMacroAssembler<PointerSize::k64> {
  public:
+  using JNIBase = JNIMacroAssembler<PointerSize::k64>;
+
   explicit Mips64Assembler(ArenaAllocator* arena)
       : Assembler(arena),
         overwriting_(false),
@@ -574,6 +576,35 @@
   }
 
   void Bind(Mips64Label* label);
+
+  // Don't warn about a different virtual Bind/Jump in the base class.
+  using JNIBase::Bind;
+  using JNIBase::Jump;
+
+  // Create a new label that can be used with Jump/Bind calls.
+  std::unique_ptr<JNIMacroLabel> CreateLabel() OVERRIDE {
+    LOG(FATAL) << "Not implemented on MIPS64";
+    UNREACHABLE();
+  }
+  // Emit an unconditional jump to the label.
+  void Jump(JNIMacroLabel* label ATTRIBUTE_UNUSED) OVERRIDE {
+    LOG(FATAL) << "Not implemented on MIPS64";
+    UNREACHABLE();
+  }
+  // Emit a conditional jump to the label by applying a unary condition test to the register.
+  void Jump(JNIMacroLabel* label ATTRIBUTE_UNUSED,
+            JNIMacroUnaryCondition cond ATTRIBUTE_UNUSED,
+            ManagedRegister test ATTRIBUTE_UNUSED) OVERRIDE {
+    LOG(FATAL) << "Not implemented on MIPS64";
+    UNREACHABLE();
+  }
+
+  // Code at this offset will serve as the target for the Jump call.
+  void Bind(JNIMacroLabel* label ATTRIBUTE_UNUSED) OVERRIDE {
+    LOG(FATAL) << "Not implemented on MIPS64";
+    UNREACHABLE();
+  }
+
   void Bc(Mips64Label* label);
   void Jialc(Mips64Label* label, GpuRegister indirect_reg);
   void Bltc(GpuRegister rs, GpuRegister rt, Mips64Label* label);
diff --git a/compiler/utils/x86/jni_macro_assembler_x86.cc b/compiler/utils/x86/jni_macro_assembler_x86.cc
index 77af885..cfdf80b 100644
--- a/compiler/utils/x86/jni_macro_assembler_x86.cc
+++ b/compiler/utils/x86/jni_macro_assembler_x86.cc
@@ -215,8 +215,12 @@
   if (dest.IsNoRegister()) {
     CHECK_EQ(0u, size);
   } else if (dest.IsCpuRegister()) {
-    CHECK_EQ(4u, size);
-    __ fs()->movl(dest.AsCpuRegister(), Address::Absolute(src));
+    if (size == 1u) {
+      __ fs()->movzxb(dest.AsCpuRegister(), Address::Absolute(src));
+    } else {
+      CHECK_EQ(4u, size);
+      __ fs()->movl(dest.AsCpuRegister(), Address::Absolute(src));
+    }
   } else if (dest.IsRegisterPair()) {
     CHECK_EQ(8u, size);
     __ fs()->movl(dest.AsRegisterPairLow(), Address::Absolute(src));
@@ -519,6 +523,48 @@
   __ j(kNotEqual, slow->Entry());
 }
 
+std::unique_ptr<JNIMacroLabel> X86JNIMacroAssembler::CreateLabel() {
+  return std::unique_ptr<JNIMacroLabel>(new X86JNIMacroLabel());
+}
+
+void X86JNIMacroAssembler::Jump(JNIMacroLabel* label) {
+  CHECK(label != nullptr);
+  __ jmp(X86JNIMacroLabel::Cast(label)->AsX86());
+}
+
+void X86JNIMacroAssembler::Jump(JNIMacroLabel* label,
+                                JNIMacroUnaryCondition condition,
+                                ManagedRegister test) {
+  CHECK(label != nullptr);
+
+  art::x86::Condition x86_cond;
+  switch (condition) {
+    case JNIMacroUnaryCondition::kZero:
+      x86_cond = art::x86::kZero;
+      break;
+    case JNIMacroUnaryCondition::kNotZero:
+      x86_cond = art::x86::kNotZero;
+      break;
+    default:
+      LOG(FATAL) << "Not implemented condition: " << static_cast<int>(condition);
+      UNREACHABLE();
+  }
+
+  // TEST reg, reg
+  // Jcc <Offset>
+  __ testl(test.AsX86().AsCpuRegister(), test.AsX86().AsCpuRegister());
+  __ j(x86_cond, X86JNIMacroLabel::Cast(label)->AsX86());
+
+
+  // X86 also has JCZX, JECZX, however it's not worth it to implement
+  // because we aren't likely to codegen with ECX+kZero check.
+}
+
+void X86JNIMacroAssembler::Bind(JNIMacroLabel* label) {
+  CHECK(label != nullptr);
+  __ Bind(X86JNIMacroLabel::Cast(label)->AsX86());
+}
+
 #undef __
 
 void X86ExceptionSlowPath::Emit(Assembler *sasm) {
diff --git a/compiler/utils/x86/jni_macro_assembler_x86.h b/compiler/utils/x86/jni_macro_assembler_x86.h
index 015584c..8ffda64 100644
--- a/compiler/utils/x86/jni_macro_assembler_x86.h
+++ b/compiler/utils/x86/jni_macro_assembler_x86.h
@@ -30,6 +30,8 @@
 namespace art {
 namespace x86 {
 
+class X86JNIMacroLabel;
+
 class X86JNIMacroAssembler FINAL : public JNIMacroAssemblerFwd<X86Assembler, PointerSize::k32> {
  public:
   explicit X86JNIMacroAssembler(ArenaAllocator* arena) : JNIMacroAssemblerFwd(arena) {}
@@ -152,10 +154,29 @@
   // and branch to a ExceptionSlowPath if it is.
   void ExceptionPoll(ManagedRegister scratch, size_t stack_adjust) OVERRIDE;
 
+  // Create a new label that can be used with Jump/Bind calls.
+  std::unique_ptr<JNIMacroLabel> CreateLabel() OVERRIDE;
+  // Emit an unconditional jump to the label.
+  void Jump(JNIMacroLabel* label) OVERRIDE;
+  // Emit a conditional jump to the label by applying a unary condition test to the register.
+  void Jump(JNIMacroLabel* label, JNIMacroUnaryCondition cond, ManagedRegister test) OVERRIDE;
+  // Code at this offset will serve as the target for the Jump call.
+  void Bind(JNIMacroLabel* label) OVERRIDE;
+
  private:
   DISALLOW_COPY_AND_ASSIGN(X86JNIMacroAssembler);
 };
 
+class X86JNIMacroLabel FINAL
+    : public JNIMacroLabelCommon<X86JNIMacroLabel,
+                                 art::Label,
+                                 kX86> {
+ public:
+  art::Label* AsX86() {
+    return AsPlatformLabel();
+  }
+};
+
 }  // namespace x86
 }  // namespace art
 
diff --git a/compiler/utils/x86_64/jni_macro_assembler_x86_64.cc b/compiler/utils/x86_64/jni_macro_assembler_x86_64.cc
index 3e687a7..ec86254 100644
--- a/compiler/utils/x86_64/jni_macro_assembler_x86_64.cc
+++ b/compiler/utils/x86_64/jni_macro_assembler_x86_64.cc
@@ -260,8 +260,12 @@
   if (dest.IsNoRegister()) {
     CHECK_EQ(0u, size);
   } else if (dest.IsCpuRegister()) {
-    CHECK_EQ(4u, size);
-    __ gs()->movl(dest.AsCpuRegister(), Address::Absolute(src, true));
+    if (size == 1u) {
+      __ gs()->movzxb(dest.AsCpuRegister(), Address::Absolute(src, true));
+    } else {
+      CHECK_EQ(4u, size);
+      __ gs()->movl(dest.AsCpuRegister(), Address::Absolute(src, true));
+    }
   } else if (dest.IsRegisterPair()) {
     CHECK_EQ(8u, size);
     __ gs()->movq(dest.AsRegisterPairLow(), Address::Absolute(src, true));
@@ -585,6 +589,44 @@
   __ j(kNotEqual, slow->Entry());
 }
 
+std::unique_ptr<JNIMacroLabel> X86_64JNIMacroAssembler::CreateLabel() {
+  return std::unique_ptr<JNIMacroLabel>(new X86_64JNIMacroLabel());
+}
+
+void X86_64JNIMacroAssembler::Jump(JNIMacroLabel* label) {
+  CHECK(label != nullptr);
+  __ jmp(X86_64JNIMacroLabel::Cast(label)->AsX86_64());
+}
+
+void X86_64JNIMacroAssembler::Jump(JNIMacroLabel* label,
+                                   JNIMacroUnaryCondition condition,
+                                   ManagedRegister test) {
+  CHECK(label != nullptr);
+
+  art::x86_64::Condition x86_64_cond;
+  switch (condition) {
+    case JNIMacroUnaryCondition::kZero:
+      x86_64_cond = art::x86_64::kZero;
+      break;
+    case JNIMacroUnaryCondition::kNotZero:
+      x86_64_cond = art::x86_64::kNotZero;
+      break;
+    default:
+      LOG(FATAL) << "Not implemented condition: " << static_cast<int>(condition);
+      UNREACHABLE();
+  }
+
+  // TEST reg, reg
+  // Jcc <Offset>
+  __ testq(test.AsX86_64().AsCpuRegister(), test.AsX86_64().AsCpuRegister());
+  __ j(x86_64_cond, X86_64JNIMacroLabel::Cast(label)->AsX86_64());
+}
+
+void X86_64JNIMacroAssembler::Bind(JNIMacroLabel* label) {
+  CHECK(label != nullptr);
+  __ Bind(X86_64JNIMacroLabel::Cast(label)->AsX86_64());
+}
+
 #undef __
 
 void X86_64ExceptionSlowPath::Emit(Assembler *sasm) {
diff --git a/compiler/utils/x86_64/jni_macro_assembler_x86_64.h b/compiler/utils/x86_64/jni_macro_assembler_x86_64.h
index 9107f3c..aa058f7 100644
--- a/compiler/utils/x86_64/jni_macro_assembler_x86_64.h
+++ b/compiler/utils/x86_64/jni_macro_assembler_x86_64.h
@@ -180,10 +180,29 @@
   // and branch to a ExceptionSlowPath if it is.
   void ExceptionPoll(ManagedRegister scratch, size_t stack_adjust) OVERRIDE;
 
+  // Create a new label that can be used with Jump/Bind calls.
+  std::unique_ptr<JNIMacroLabel> CreateLabel() OVERRIDE;
+  // Emit an unconditional jump to the label.
+  void Jump(JNIMacroLabel* label) OVERRIDE;
+  // Emit a conditional jump to the label by applying a unary condition test to the register.
+  void Jump(JNIMacroLabel* label, JNIMacroUnaryCondition cond, ManagedRegister test) OVERRIDE;
+  // Code at this offset will serve as the target for the Jump call.
+  void Bind(JNIMacroLabel* label) OVERRIDE;
+
  private:
   DISALLOW_COPY_AND_ASSIGN(X86_64JNIMacroAssembler);
 };
 
+class X86_64JNIMacroLabel FINAL
+    : public JNIMacroLabelCommon<X86_64JNIMacroLabel,
+                                 art::Label,
+                                 kX86_64> {
+ public:
+  art::Label* AsX86_64() {
+    return AsPlatformLabel();
+  }
+};
+
 }  // namespace x86_64
 }  // namespace art
 
diff --git a/runtime/thread.h b/runtime/thread.h
index 376a69c..6f5913e 100644
--- a/runtime/thread.h
+++ b/runtime/thread.h
@@ -570,6 +570,10 @@
         OFFSETOF_MEMBER(tls_32bit_sized_values, is_gc_marking));
   }
 
+  static constexpr size_t IsGcMarkingSize() {
+    return sizeof(tls32_.is_gc_marking);
+  }
+
   // Deoptimize the Java stack.
   void DeoptimizeWithDeoptimizationException(JValue* result) REQUIRES_SHARED(Locks::mutator_lock_);