x86/x86-64 nterp: Add support for heap poisoning.
Test: ART_HEAP_POISONING=true testrunner.py -b --host --jit
Change-Id: Ifacafc068c10f202218ec816c1396342d93730e4
diff --git a/runtime/interpreter/mterp/nterp.cc b/runtime/interpreter/mterp/nterp.cc
index 0541228..52cdeae 100644
--- a/runtime/interpreter/mterp/nterp.cc
+++ b/runtime/interpreter/mterp/nterp.cc
@@ -19,6 +19,7 @@
*/
#include "nterp.h"
+#include "arch/instruction_set.h"
#include "base/quasi_atomic.h"
#include "class_linker-inl.h"
#include "dex/dex_instruction_utils.h"
@@ -35,8 +36,20 @@
namespace interpreter {
bool IsNterpSupported() {
- return !kPoisonHeapReferences && kReserveMarkingRegister &&
- kRuntimeISA != InstructionSet::kRiscv64;
+ switch (kRuntimeISA) {
+ case InstructionSet::kArm:
+ case InstructionSet::kThumb2:
+ case InstructionSet::kArm64:
+ return !kPoisonHeapReferences && kReserveMarkingRegister;
+ case InstructionSet::kRiscv64:
+ // TODO(riscv64): Support nterp.
+ return false;
+ case InstructionSet::kX86:
+ case InstructionSet::kX86_64:
+ return true;
+ default:
+ return false;
+ }
}
bool CanRuntimeUseNterp() REQUIRES_SHARED(Locks::mutator_lock_) {
diff --git a/runtime/interpreter/mterp/x86_64ng/array.S b/runtime/interpreter/mterp/x86_64ng/array.S
index c8dd8b0..7aa34c7 100644
--- a/runtime/interpreter/mterp/x86_64ng/array.S
+++ b/runtime/interpreter/mterp/x86_64ng/array.S
@@ -22,10 +22,12 @@
testb $$READ_BARRIER_TEST_VALUE, GRAY_BYTE_OFFSET(%edi)
$load $data_offset(%rdi,%rsi,$shift), %eax
jnz 2f
+ UNPOISON_HEAP_REF eax // Affects flags, so we cannot unpoison before the jnz.
1:
SET_VREG_OBJECT %eax, rINSTq
ADVANCE_PC_FETCH_AND_GOTO_NEXT 2
2:
+ UNPOISON_HEAP_REF eax
// reg00 is eax
call art_quick_read_barrier_mark_reg00
jmp 1b
diff --git a/runtime/interpreter/mterp/x86_64ng/invoke.S b/runtime/interpreter/mterp/x86_64ng/invoke.S
index 731a2bc..f8f09af 100644
--- a/runtime/interpreter/mterp/x86_64ng/invoke.S
+++ b/runtime/interpreter/mterp/x86_64ng/invoke.S
@@ -84,6 +84,7 @@
.endif
movl (rFP, %r11, 4), %esi
movl MIRROR_OBJECT_CLASS_OFFSET(%esi), %edx
+ UNPOISON_HEAP_REF edx
// Test the first two bits of the fetched ArtMethod:
// - If the first bit is set, this is a method on j.l.Object
// - If the second bit is set, this is a default method.
@@ -158,6 +159,7 @@
movl (rFP, %r11, 4), %esi
// Note: if esi is null, this will be handled by our SIGSEGV handler.
movl MIRROR_OBJECT_CLASS_OFFSET(%esi), %edx
+ UNPOISON_HEAP_REF edx
movq MIRROR_CLASS_VTABLE_OFFSET_64(%edx, %edi, 8), %rdi
jmp $helper
2:
diff --git a/runtime/interpreter/mterp/x86_64ng/main.S b/runtime/interpreter/mterp/x86_64ng/main.S
index e00980c..7dc8178 100644
--- a/runtime/interpreter/mterp/x86_64ng/main.S
+++ b/runtime/interpreter/mterp/x86_64ng/main.S
@@ -1987,6 +1987,7 @@
GET_VREG rINST, rINSTq # vB (object we're operating on)
testl rINST, rINST # is object null?
je common_errNullObject
+ POISON_HEAP_REF ecx
movl %ecx, (rINSTq,%rax,1)
testl %ecx, %ecx
je 4f
@@ -2010,6 +2011,7 @@
testl rINST, rINST # is object null?
je common_errNullObject
negl %eax
+ POISON_HEAP_REF ecx
movl %ecx, (rINSTq,%rax,1)
testl %ecx, %ecx
je 5f
@@ -2032,6 +2034,7 @@
testb $$READ_BARRIER_TEST_VALUE, GRAY_BYTE_OFFSET(%ecx)
movl (%rcx,%rax,1), %eax
jnz 3f
+ UNPOISON_HEAP_REF eax // Affects flags, so we cannot unpoison before the jnz.
4:
andb $$0xf,rINSTbl # rINST <- A
SET_VREG_OBJECT %eax, rINSTq # fp[A] <- value
@@ -2050,6 +2053,7 @@
negl %eax
jmp 1b
3:
+ UNPOISON_HEAP_REF eax
// reg00 is eax
call art_quick_read_barrier_mark_reg00
jmp 4b
@@ -2064,6 +2068,7 @@
cmpq $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET
jne 3f
5:
+ POISON_HEAP_REF ebp
movl %ebp, (%eax, %edx, 1)
testl %ebp, %ebp
je 4f
@@ -2089,6 +2094,7 @@
cmpq $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET
jne 7f
6:
+ POISON_HEAP_REF ebp
movl %ebp, (%eax, %edx, 1)
testl %ebp, %ebp
je 8f
@@ -2117,6 +2123,7 @@
testb $$READ_BARRIER_TEST_VALUE, GRAY_BYTE_OFFSET(%eax)
movl (%eax, %edx, 1), %eax
jnz 3f
+ UNPOISON_HEAP_REF eax // Affects flags, so we cannot unpoison before the jnz.
4:
SET_VREG_OBJECT %eax, rINSTq # fp[A] <- value
ADVANCE_PC_FETCH_AND_GOTO_NEXT 2
@@ -2130,6 +2137,7 @@
andq $$-2, %rax
jmp 1b
3:
+ UNPOISON_HEAP_REF eax
call art_quick_read_barrier_mark_reg00
jmp 4b
5:
diff --git a/runtime/interpreter/mterp/x86_64ng/object.S b/runtime/interpreter/mterp/x86_64ng/object.S
index 21a6e67..9edf4ec 100644
--- a/runtime/interpreter/mterp/x86_64ng/object.S
+++ b/runtime/interpreter/mterp/x86_64ng/object.S
@@ -7,6 +7,7 @@
testl %edi, %edi
je .L${opcode}_resume
// Fast path without read barriers.
+ POISON_HEAP_REF esi // Poison class reference for in-memory comparison.
cmpl MIRROR_OBJECT_CLASS_OFFSET(%edi), %esi
jne ${slow_path}
.L${opcode}_resume:
@@ -21,13 +22,16 @@
jmp 1b
%def op_check_cast_slow_path():
+ UNPOISON_HEAP_REF esi // Unpoison class reference poisoned in main path.
testl $$MIRROR_CLASS_IS_INTERFACE_FLAG, MIRROR_CLASS_ACCESS_FLAGS_OFFSET(%rsi)
jne 2f
movl MIRROR_OBJECT_CLASS_OFFSET(%edi), %eax
+ UNPOISON_HEAP_REF eax
cmpl $$0, MIRROR_CLASS_COMPONENT_TYPE_OFFSET(%rsi)
jne 2f
1:
movl MIRROR_CLASS_SUPER_CLASS_OFFSET(%eax), %eax
+ UNPOISON_HEAP_REF eax
cmpl %eax, %esi
je .L${opcode}_resume
testl %eax, %eax
@@ -45,10 +49,12 @@
jmp 3b
5:
movl MIRROR_CLASS_COMPONENT_TYPE_OFFSET(%eax), %eax
+ UNPOISON_HEAP_REF eax
// Check if object is an array.
testl %eax, %eax
je 2b
movl MIRROR_CLASS_COMPONENT_TYPE_OFFSET(%esi), %ecx
+ UNPOISON_HEAP_REF ecx
cmpl $$0, MIRROR_CLASS_SUPER_CLASS_OFFSET(%ecx)
jne 2b
cmpw $$0, MIRROR_CLASS_OBJECT_PRIMITIVE_TYPE_OFFSET(%ecx)
@@ -90,6 +96,7 @@
testl %edi, %edi
je .L${opcode}_set_vreg
// Fast path without read barriers.
+ POISON_HEAP_REF esi // Poison class reference for in-memory comparison.
cmpl MIRROR_OBJECT_CLASS_OFFSET(%edi), %esi
jne ${slow_path}
.L${opcode}_set_one:
@@ -100,6 +107,7 @@
ADVANCE_PC_FETCH_AND_GOTO_NEXT 2
%def op_instance_of_slow_path():
+ UNPOISON_HEAP_REF esi // Unpoison class reference poisoned in main path.
// Go slow path if we are marking. Checking now allows
// not going to slow path if the super class hierarchy check fails.
cmpq $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET
@@ -107,10 +115,12 @@
testl $$MIRROR_CLASS_IS_INTERFACE_FLAG, MIRROR_CLASS_ACCESS_FLAGS_OFFSET(%rsi)
jne 5f
movl MIRROR_OBJECT_CLASS_OFFSET(%edi), %eax
+ UNPOISON_HEAP_REF eax
cmpl $$0, MIRROR_CLASS_COMPONENT_TYPE_OFFSET(%rsi)
jne 3f
1:
movl MIRROR_CLASS_SUPER_CLASS_OFFSET(%eax), %eax
+ UNPOISON_HEAP_REF eax
cmpl %eax, %esi
je .L${opcode}_set_one
testl %eax, %eax
@@ -120,10 +130,12 @@
jmp .L${opcode}_resume
3:
movl MIRROR_CLASS_COMPONENT_TYPE_OFFSET(%eax), %eax
+ UNPOISON_HEAP_REF eax
// Check if object is an array.
testl %eax, %eax
je 2b
movl MIRROR_CLASS_COMPONENT_TYPE_OFFSET(%esi), %ecx
+ UNPOISON_HEAP_REF ecx
cmpl $$0, MIRROR_CLASS_SUPER_CLASS_OFFSET(%ecx)
jne 5f
cmpw $$0, MIRROR_CLASS_OBJECT_PRIMITIVE_TYPE_OFFSET(%ecx)
diff --git a/runtime/interpreter/mterp/x86ng/array.S b/runtime/interpreter/mterp/x86ng/array.S
index fced2e8..8a00769 100644
--- a/runtime/interpreter/mterp/x86ng/array.S
+++ b/runtime/interpreter/mterp/x86ng/array.S
@@ -22,10 +22,12 @@
testb $$READ_BARRIER_TEST_VALUE, GRAY_BYTE_OFFSET(%eax)
$load $data_offset(%eax,%ecx,$multiplier), %eax
jnz 2f
+ UNPOISON_HEAP_REF eax // Affects flags, so we cannot unpoison before the jnz.
1:
SET_VREG_OBJECT %eax, rINST
ADVANCE_PC_FETCH_AND_GOTO_NEXT 2
2:
+ UNPOISON_HEAP_REF eax
// reg00 is eax
call art_quick_read_barrier_mark_reg00
jmp 1b
diff --git a/runtime/interpreter/mterp/x86ng/invoke.S b/runtime/interpreter/mterp/x86ng/invoke.S
index f45ff80..91a9077 100644
--- a/runtime/interpreter/mterp/x86ng/invoke.S
+++ b/runtime/interpreter/mterp/x86ng/invoke.S
@@ -86,6 +86,7 @@
.endif
movl (rFP, %ecx, 4), %ecx
movl MIRROR_OBJECT_CLASS_OFFSET(%ecx), %edx
+ UNPOISON_HEAP_REF edx
// Test the first two bits of the fetched ArtMethod:
// - If the first bit is set, this is a method on j.l.Object
// - If the second bit is set, this is a default method.
@@ -163,6 +164,7 @@
movl (rFP, %ecx, 4), %ecx
// Note: if ecx is null, this will be handled by our SIGSEGV handler.
movl MIRROR_OBJECT_CLASS_OFFSET(%ecx), %edx
+ UNPOISON_HEAP_REF edx
movl MIRROR_CLASS_VTABLE_OFFSET_32(%edx, %eax, 4), %eax
jmp $helper
2:
diff --git a/runtime/interpreter/mterp/x86ng/main.S b/runtime/interpreter/mterp/x86ng/main.S
index 4bc57ec..ceb7bf7 100644
--- a/runtime/interpreter/mterp/x86ng/main.S
+++ b/runtime/interpreter/mterp/x86ng/main.S
@@ -2061,6 +2061,7 @@
GET_VREG rINST, rINST # vB (object we're operating on)
testl rINST, rINST # is object null?
je common_errNullObject
+ POISON_HEAP_REF ecx
movl %ecx, (rINST, %eax, 1)
testl %ecx, %ecx
je 4f
@@ -2090,6 +2091,7 @@
GET_VREG rINST, rINST # vB (object we're operating on)
testl rINST, rINST # is object null?
je common_errNullObject
+ POISON_HEAP_REF ecx
movl %ecx, (rINST, %eax, 1)
testl %ecx, %ecx
je 5f
@@ -2112,6 +2114,7 @@
testb $$READ_BARRIER_TEST_VALUE, GRAY_BYTE_OFFSET(%ecx)
movl (%ecx,%eax,1), %eax
jnz 3f
+ UNPOISON_HEAP_REF eax // Affects flags, so we cannot unpoison before the jnz.
4:
andb $$0xf,rINSTbl # rINST <- A
SET_VREG_OBJECT %eax, rINST # fp[A] <- value
@@ -2130,6 +2133,7 @@
negl %eax
jmp 1b
3:
+ UNPOISON_HEAP_REF eax
// reg00 is eax
call art_quick_read_barrier_mark_reg00
jmp 4b
@@ -2144,6 +2148,7 @@
cmpl $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET
jne 3f
5:
+ POISON_HEAP_REF ebx // `rINST` is `%ebx` but we need to pass `ebx`.
movl rINST, (%eax, %ecx, 1)
testl rINST, rINST
je 4f
@@ -2169,6 +2174,7 @@
cmpl $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET
jne 7f
6:
+ POISON_HEAP_REF ebx // `rINST` is `%ebx` but we need to pass `ebx`.
movl rINST, (%eax, %ecx, 1)
testl rINST, rINST
je 8f
@@ -2197,6 +2203,7 @@
testb $$READ_BARRIER_TEST_VALUE, GRAY_BYTE_OFFSET(%eax)
movl (%eax, %ecx, 1), %eax
jnz 3f
+ UNPOISON_HEAP_REF eax // Affects flags, so we cannot unpoison before the jnz.
4:
SET_VREG_OBJECT %eax, rINST # fp[A] <- value
ADVANCE_PC_FETCH_AND_GOTO_NEXT 2
@@ -2210,6 +2217,7 @@
CLEAR_VOLATILE_MARKER %eax
jmp 1b
3:
+ UNPOISON_HEAP_REF eax
call art_quick_read_barrier_mark_reg00
jmp 4b
5:
diff --git a/runtime/interpreter/mterp/x86ng/object.S b/runtime/interpreter/mterp/x86ng/object.S
index 39091ce..b56084e 100644
--- a/runtime/interpreter/mterp/x86ng/object.S
+++ b/runtime/interpreter/mterp/x86ng/object.S
@@ -7,6 +7,7 @@
testl %eax, %eax
je .L${opcode}_resume
// Fast path without read barriers.
+ POISON_HEAP_REF ecx // Poison class reference for in-memory comparison.
cmpl MIRROR_OBJECT_CLASS_OFFSET(%eax), %ecx
jne ${slow_path}
.L${opcode}_resume:
@@ -21,6 +22,7 @@
jmp 1b
%def op_check_cast_slow_path():
+ UNPOISON_HEAP_REF ecx // Unpoison class reference poisoned in main path.
cmpl $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET
jne 2f
1:
@@ -45,6 +47,7 @@
testl %eax, %eax
je .L${opcode}_resume
// Fast path without read barriers.
+ POISON_HEAP_REF ecx // Poison class reference for in-memory comparison.
cmpl MIRROR_OBJECT_CLASS_OFFSET(%eax), %ecx
jne ${slow_path}
.L${opcode}_set_one:
@@ -63,6 +66,7 @@
jmp 1b
%def op_instance_of_slow_path():
+ UNPOISON_HEAP_REF ecx // Unpoison class reference poisoned in main path.
cmpl $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET
jne 2f
testl $$MIRROR_CLASS_IS_INTERFACE_FLAG, MIRROR_CLASS_ACCESS_FLAGS_OFFSET(%ecx)
@@ -70,8 +74,10 @@
cmpl $$0, MIRROR_CLASS_COMPONENT_TYPE_OFFSET(%ecx)
jne 3f
movl MIRROR_OBJECT_CLASS_OFFSET(%eax), %eax
+ UNPOISON_HEAP_REF eax
1:
movl MIRROR_CLASS_SUPER_CLASS_OFFSET(%eax), %eax
+ UNPOISON_HEAP_REF eax
cmpl %eax, %ecx
je .L${opcode}_set_one
testl %eax, %eax