x86/x86-64 nterp: Add support for heap poisoning.

Test: ART_HEAP_POISONING=true testrunner.py -b --host --jit
Change-Id: Ifacafc068c10f202218ec816c1396342d93730e4
diff --git a/runtime/interpreter/mterp/nterp.cc b/runtime/interpreter/mterp/nterp.cc
index 0541228..52cdeae 100644
--- a/runtime/interpreter/mterp/nterp.cc
+++ b/runtime/interpreter/mterp/nterp.cc
@@ -19,6 +19,7 @@
  */
 #include "nterp.h"
 
+#include "arch/instruction_set.h"
 #include "base/quasi_atomic.h"
 #include "class_linker-inl.h"
 #include "dex/dex_instruction_utils.h"
@@ -35,8 +36,20 @@
 namespace interpreter {
 
 bool IsNterpSupported() {
-  return !kPoisonHeapReferences && kReserveMarkingRegister &&
-         kRuntimeISA != InstructionSet::kRiscv64;
+  switch (kRuntimeISA) {
+    case InstructionSet::kArm:
+    case InstructionSet::kThumb2:
+    case InstructionSet::kArm64:
+      return !kPoisonHeapReferences && kReserveMarkingRegister;
+    case InstructionSet::kRiscv64:
+      // TODO(riscv64): Support nterp.
+      return false;
+    case InstructionSet::kX86:
+    case InstructionSet::kX86_64:
+      return true;
+    default:
+      return false;
+  }
 }
 
 bool CanRuntimeUseNterp() REQUIRES_SHARED(Locks::mutator_lock_) {
diff --git a/runtime/interpreter/mterp/x86_64ng/array.S b/runtime/interpreter/mterp/x86_64ng/array.S
index c8dd8b0..7aa34c7 100644
--- a/runtime/interpreter/mterp/x86_64ng/array.S
+++ b/runtime/interpreter/mterp/x86_64ng/array.S
@@ -22,10 +22,12 @@
     testb $$READ_BARRIER_TEST_VALUE, GRAY_BYTE_OFFSET(%edi)
     $load   $data_offset(%rdi,%rsi,$shift), %eax
     jnz 2f
+    UNPOISON_HEAP_REF eax  // Affects flags, so we cannot unpoison before the jnz.
 1:
     SET_VREG_OBJECT %eax, rINSTq
     ADVANCE_PC_FETCH_AND_GOTO_NEXT 2
 2:
+    UNPOISON_HEAP_REF eax
     // reg00 is eax
     call art_quick_read_barrier_mark_reg00
     jmp 1b
diff --git a/runtime/interpreter/mterp/x86_64ng/invoke.S b/runtime/interpreter/mterp/x86_64ng/invoke.S
index 731a2bc..f8f09af 100644
--- a/runtime/interpreter/mterp/x86_64ng/invoke.S
+++ b/runtime/interpreter/mterp/x86_64ng/invoke.S
@@ -84,6 +84,7 @@
    .endif
    movl (rFP, %r11, 4), %esi
    movl MIRROR_OBJECT_CLASS_OFFSET(%esi), %edx
+   UNPOISON_HEAP_REF edx
    // Test the first two bits of the fetched ArtMethod:
    // - If the first bit is set, this is a method on j.l.Object
    // - If the second bit is set, this is a default method.
@@ -158,6 +159,7 @@
    movl (rFP, %r11, 4), %esi
    // Note: if esi is null, this will be handled by our SIGSEGV handler.
    movl MIRROR_OBJECT_CLASS_OFFSET(%esi), %edx
+   UNPOISON_HEAP_REF edx
    movq MIRROR_CLASS_VTABLE_OFFSET_64(%edx, %edi, 8), %rdi
    jmp $helper
 2:
diff --git a/runtime/interpreter/mterp/x86_64ng/main.S b/runtime/interpreter/mterp/x86_64ng/main.S
index e00980c..7dc8178 100644
--- a/runtime/interpreter/mterp/x86_64ng/main.S
+++ b/runtime/interpreter/mterp/x86_64ng/main.S
@@ -1987,6 +1987,7 @@
    GET_VREG rINST, rINSTq                  # vB (object we're operating on)
    testl   rINST, rINST                    # is object null?
    je      common_errNullObject
+   POISON_HEAP_REF ecx
    movl %ecx, (rINSTq,%rax,1)
    testl %ecx, %ecx
    je 4f
@@ -2010,6 +2011,7 @@
    testl   rINST, rINST                    # is object null?
    je      common_errNullObject
    negl %eax
+   POISON_HEAP_REF ecx
    movl %ecx, (rINSTq,%rax,1)
    testl %ecx, %ecx
    je 5f
@@ -2032,6 +2034,7 @@
    testb $$READ_BARRIER_TEST_VALUE, GRAY_BYTE_OFFSET(%ecx)
    movl (%rcx,%rax,1), %eax
    jnz 3f
+   UNPOISON_HEAP_REF eax  // Affects flags, so we cannot unpoison before the jnz.
 4:
    andb    $$0xf,rINSTbl                   # rINST <- A
    SET_VREG_OBJECT %eax, rINSTq            # fp[A] <- value
@@ -2050,6 +2053,7 @@
    negl %eax
    jmp 1b
 3:
+   UNPOISON_HEAP_REF eax
    // reg00 is eax
    call art_quick_read_barrier_mark_reg00
    jmp 4b
@@ -2064,6 +2068,7 @@
    cmpq $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET
    jne 3f
 5:
+   POISON_HEAP_REF ebp
    movl %ebp, (%eax, %edx, 1)
    testl %ebp, %ebp
    je 4f
@@ -2089,6 +2094,7 @@
    cmpq $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET
    jne 7f
 6:
+   POISON_HEAP_REF ebp
    movl %ebp, (%eax, %edx, 1)
    testl %ebp, %ebp
    je 8f
@@ -2117,6 +2123,7 @@
    testb $$READ_BARRIER_TEST_VALUE, GRAY_BYTE_OFFSET(%eax)
    movl (%eax, %edx, 1), %eax
    jnz 3f
+   UNPOISON_HEAP_REF eax  // Affects flags, so we cannot unpoison before the jnz.
 4:
    SET_VREG_OBJECT %eax, rINSTq            # fp[A] <- value
    ADVANCE_PC_FETCH_AND_GOTO_NEXT 2
@@ -2130,6 +2137,7 @@
    andq $$-2, %rax
    jmp 1b
 3:
+   UNPOISON_HEAP_REF eax
    call art_quick_read_barrier_mark_reg00
    jmp 4b
 5:
diff --git a/runtime/interpreter/mterp/x86_64ng/object.S b/runtime/interpreter/mterp/x86_64ng/object.S
index 21a6e67..9edf4ec 100644
--- a/runtime/interpreter/mterp/x86_64ng/object.S
+++ b/runtime/interpreter/mterp/x86_64ng/object.S
@@ -7,6 +7,7 @@
    testl %edi, %edi
    je .L${opcode}_resume
    // Fast path without read barriers.
+   POISON_HEAP_REF esi  // Poison class reference for in-memory comparison.
    cmpl MIRROR_OBJECT_CLASS_OFFSET(%edi), %esi
    jne ${slow_path}
 .L${opcode}_resume:
@@ -21,13 +22,16 @@
    jmp 1b
 
 %def op_check_cast_slow_path():
+   UNPOISON_HEAP_REF esi  // Unpoison class reference poisoned in main path.
    testl $$MIRROR_CLASS_IS_INTERFACE_FLAG, MIRROR_CLASS_ACCESS_FLAGS_OFFSET(%rsi)
    jne 2f
    movl MIRROR_OBJECT_CLASS_OFFSET(%edi), %eax
+   UNPOISON_HEAP_REF eax
    cmpl $$0, MIRROR_CLASS_COMPONENT_TYPE_OFFSET(%rsi)
    jne 2f
 1:
    movl MIRROR_CLASS_SUPER_CLASS_OFFSET(%eax), %eax
+   UNPOISON_HEAP_REF eax
    cmpl %eax, %esi
    je .L${opcode}_resume
    testl %eax, %eax
@@ -45,10 +49,12 @@
    jmp 3b
 5:
    movl MIRROR_CLASS_COMPONENT_TYPE_OFFSET(%eax), %eax
+   UNPOISON_HEAP_REF eax
    // Check if object is an array.
    testl %eax, %eax
    je 2b
    movl MIRROR_CLASS_COMPONENT_TYPE_OFFSET(%esi), %ecx
+   UNPOISON_HEAP_REF ecx
    cmpl $$0, MIRROR_CLASS_SUPER_CLASS_OFFSET(%ecx)
    jne 2b
    cmpw $$0, MIRROR_CLASS_OBJECT_PRIMITIVE_TYPE_OFFSET(%ecx)
@@ -90,6 +96,7 @@
    testl %edi, %edi
    je .L${opcode}_set_vreg
    // Fast path without read barriers.
+   POISON_HEAP_REF esi  // Poison class reference for in-memory comparison.
    cmpl MIRROR_OBJECT_CLASS_OFFSET(%edi), %esi
    jne ${slow_path}
 .L${opcode}_set_one:
@@ -100,6 +107,7 @@
    ADVANCE_PC_FETCH_AND_GOTO_NEXT 2
 
 %def op_instance_of_slow_path():
+   UNPOISON_HEAP_REF esi  // Unpoison class reference poisoned in main path.
    // Go slow path if we are marking. Checking now allows
    // not going to slow path if the super class hierarchy check fails.
    cmpq $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET
@@ -107,10 +115,12 @@
    testl $$MIRROR_CLASS_IS_INTERFACE_FLAG, MIRROR_CLASS_ACCESS_FLAGS_OFFSET(%rsi)
    jne 5f
    movl MIRROR_OBJECT_CLASS_OFFSET(%edi), %eax
+   UNPOISON_HEAP_REF eax
    cmpl $$0, MIRROR_CLASS_COMPONENT_TYPE_OFFSET(%rsi)
    jne 3f
 1:
    movl MIRROR_CLASS_SUPER_CLASS_OFFSET(%eax), %eax
+   UNPOISON_HEAP_REF eax
    cmpl %eax, %esi
    je .L${opcode}_set_one
    testl %eax, %eax
@@ -120,10 +130,12 @@
    jmp       .L${opcode}_resume
 3:
    movl MIRROR_CLASS_COMPONENT_TYPE_OFFSET(%eax), %eax
+   UNPOISON_HEAP_REF eax
    // Check if object is an array.
    testl %eax, %eax
    je 2b
    movl MIRROR_CLASS_COMPONENT_TYPE_OFFSET(%esi), %ecx
+   UNPOISON_HEAP_REF ecx
    cmpl $$0, MIRROR_CLASS_SUPER_CLASS_OFFSET(%ecx)
    jne 5f
    cmpw $$0, MIRROR_CLASS_OBJECT_PRIMITIVE_TYPE_OFFSET(%ecx)
diff --git a/runtime/interpreter/mterp/x86ng/array.S b/runtime/interpreter/mterp/x86ng/array.S
index fced2e8..8a00769 100644
--- a/runtime/interpreter/mterp/x86ng/array.S
+++ b/runtime/interpreter/mterp/x86ng/array.S
@@ -22,10 +22,12 @@
     testb $$READ_BARRIER_TEST_VALUE, GRAY_BYTE_OFFSET(%eax)
     $load   $data_offset(%eax,%ecx,$multiplier), %eax
     jnz 2f
+    UNPOISON_HEAP_REF eax  // Affects flags, so we cannot unpoison before the jnz.
 1:
     SET_VREG_OBJECT %eax, rINST
     ADVANCE_PC_FETCH_AND_GOTO_NEXT 2
 2:
+    UNPOISON_HEAP_REF eax
     // reg00 is eax
     call art_quick_read_barrier_mark_reg00
     jmp 1b
diff --git a/runtime/interpreter/mterp/x86ng/invoke.S b/runtime/interpreter/mterp/x86ng/invoke.S
index f45ff80..91a9077 100644
--- a/runtime/interpreter/mterp/x86ng/invoke.S
+++ b/runtime/interpreter/mterp/x86ng/invoke.S
@@ -86,6 +86,7 @@
    .endif
    movl (rFP, %ecx, 4), %ecx
    movl MIRROR_OBJECT_CLASS_OFFSET(%ecx), %edx
+   UNPOISON_HEAP_REF edx
    // Test the first two bits of the fetched ArtMethod:
    // - If the first bit is set, this is a method on j.l.Object
    // - If the second bit is set, this is a default method.
@@ -163,6 +164,7 @@
    movl (rFP, %ecx, 4), %ecx
    // Note: if ecx is null, this will be handled by our SIGSEGV handler.
    movl MIRROR_OBJECT_CLASS_OFFSET(%ecx), %edx
+   UNPOISON_HEAP_REF edx
    movl MIRROR_CLASS_VTABLE_OFFSET_32(%edx, %eax, 4), %eax
    jmp $helper
 2:
diff --git a/runtime/interpreter/mterp/x86ng/main.S b/runtime/interpreter/mterp/x86ng/main.S
index 4bc57ec..ceb7bf7 100644
--- a/runtime/interpreter/mterp/x86ng/main.S
+++ b/runtime/interpreter/mterp/x86ng/main.S
@@ -2061,6 +2061,7 @@
    GET_VREG rINST, rINST                   # vB (object we're operating on)
    testl   rINST, rINST                    # is object null?
    je      common_errNullObject
+   POISON_HEAP_REF ecx
    movl %ecx, (rINST, %eax, 1)
    testl %ecx, %ecx
    je 4f
@@ -2090,6 +2091,7 @@
    GET_VREG rINST, rINST                   # vB (object we're operating on)
    testl   rINST, rINST                    # is object null?
    je      common_errNullObject
+   POISON_HEAP_REF ecx
    movl %ecx, (rINST, %eax, 1)
    testl %ecx, %ecx
    je 5f
@@ -2112,6 +2114,7 @@
    testb $$READ_BARRIER_TEST_VALUE, GRAY_BYTE_OFFSET(%ecx)
    movl (%ecx,%eax,1), %eax
    jnz 3f
+   UNPOISON_HEAP_REF eax  // Affects flags, so we cannot unpoison before the jnz.
 4:
    andb    $$0xf,rINSTbl                   # rINST <- A
    SET_VREG_OBJECT %eax, rINST             # fp[A] <- value
@@ -2130,6 +2133,7 @@
    negl %eax
    jmp 1b
 3:
+   UNPOISON_HEAP_REF eax
    // reg00 is eax
    call art_quick_read_barrier_mark_reg00
    jmp 4b
@@ -2144,6 +2148,7 @@
    cmpl $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET
    jne 3f
 5:
+   POISON_HEAP_REF ebx  // `rINST` is `%ebx` but we need to pass `ebx`.
    movl rINST, (%eax, %ecx, 1)
    testl rINST, rINST
    je 4f
@@ -2169,6 +2174,7 @@
    cmpl $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET
    jne 7f
 6:
+   POISON_HEAP_REF ebx  // `rINST` is `%ebx` but we need to pass `ebx`.
    movl rINST, (%eax, %ecx, 1)
    testl rINST, rINST
    je 8f
@@ -2197,6 +2203,7 @@
    testb $$READ_BARRIER_TEST_VALUE, GRAY_BYTE_OFFSET(%eax)
    movl (%eax, %ecx, 1), %eax
    jnz 3f
+   UNPOISON_HEAP_REF eax  // Affects flags, so we cannot unpoison before the jnz.
 4:
    SET_VREG_OBJECT %eax, rINST             # fp[A] <- value
    ADVANCE_PC_FETCH_AND_GOTO_NEXT 2
@@ -2210,6 +2217,7 @@
    CLEAR_VOLATILE_MARKER %eax
    jmp 1b
 3:
+   UNPOISON_HEAP_REF eax
    call art_quick_read_barrier_mark_reg00
    jmp 4b
 5:
diff --git a/runtime/interpreter/mterp/x86ng/object.S b/runtime/interpreter/mterp/x86ng/object.S
index 39091ce..b56084e 100644
--- a/runtime/interpreter/mterp/x86ng/object.S
+++ b/runtime/interpreter/mterp/x86ng/object.S
@@ -7,6 +7,7 @@
    testl %eax, %eax
    je .L${opcode}_resume
    // Fast path without read barriers.
+   POISON_HEAP_REF ecx  // Poison class reference for in-memory comparison.
    cmpl MIRROR_OBJECT_CLASS_OFFSET(%eax), %ecx
    jne ${slow_path}
 .L${opcode}_resume:
@@ -21,6 +22,7 @@
    jmp 1b
 
 %def op_check_cast_slow_path():
+   UNPOISON_HEAP_REF ecx  // Unpoison class reference poisoned in main path.
    cmpl $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET
    jne 2f
 1:
@@ -45,6 +47,7 @@
    testl %eax, %eax
    je .L${opcode}_resume
    // Fast path without read barriers.
+   POISON_HEAP_REF ecx  // Poison class reference for in-memory comparison.
    cmpl MIRROR_OBJECT_CLASS_OFFSET(%eax), %ecx
    jne ${slow_path}
 .L${opcode}_set_one:
@@ -63,6 +66,7 @@
    jmp 1b
 
 %def op_instance_of_slow_path():
+   UNPOISON_HEAP_REF ecx  // Unpoison class reference poisoned in main path.
    cmpl $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET
    jne 2f
    testl $$MIRROR_CLASS_IS_INTERFACE_FLAG, MIRROR_CLASS_ACCESS_FLAGS_OFFSET(%ecx)
@@ -70,8 +74,10 @@
    cmpl $$0, MIRROR_CLASS_COMPONENT_TYPE_OFFSET(%ecx)
    jne 3f
    movl MIRROR_OBJECT_CLASS_OFFSET(%eax), %eax
+   UNPOISON_HEAP_REF eax
 1:
    movl MIRROR_CLASS_SUPER_CLASS_OFFSET(%eax), %eax
+   UNPOISON_HEAP_REF eax
    cmpl %eax, %ecx
    je .L${opcode}_set_one
    testl %eax, %eax