arm64 nterp: Add support for heap poisoning.
This pushed the invoke-interface handler beyond the handler
instruction limit, so one part had to be split out to a
slow-path. (Note that if we moved the two loads before the
call to the helper and moved the default method and Object
method cases to the slow path as well, the handler would be
just 64 bytes even with the reference unpoisoning.)
Test: testrunner.py --target --jit --64
Change-Id: I14061c725d67f94c6ccd9aa6c735ecb73e8b2f89
diff --git a/runtime/interpreter/mterp/arm64ng/array.S b/runtime/interpreter/mterp/arm64ng/array.S
index 1689b01..a08fbe1 100644
--- a/runtime/interpreter/mterp/arm64ng/array.S
+++ b/runtime/interpreter/mterp/arm64ng/array.S
@@ -23,6 +23,7 @@
GOTO_OPCODE x10 // jump to next instruction
.elseif $is_object
$load w2, [x0, #$data_offset] // w2<- vBB[vCC]
+ UNPOISON_HEAP_REF w2
TEST_IF_MARKING 2f
1:
SET_VREG_OBJECT w2, w9 // vAA<- w2
diff --git a/runtime/interpreter/mterp/arm64ng/invoke.S b/runtime/interpreter/mterp/arm64ng/invoke.S
index 9dd7b64..cfc012b 100644
--- a/runtime/interpreter/mterp/arm64ng/invoke.S
+++ b/runtime/interpreter/mterp/arm64ng/invoke.S
@@ -70,10 +70,11 @@
b NterpCommonInvokePolymorphicRange
%def invoke_interface(range=""):
+% slow_path = add_slow_path(invoke_interface_slow_path, range)
EXPORT_PC
// Fast-path which gets the method from thread-local cache.
-% fetch_from_thread_cache("x26", miss_label="5f")
-1:
+% fetch_from_thread_cache("x26", miss_label=slow_path)
+.L${opcode}_resume:
// First argument is the 'this' pointer.
FETCH w1, 2
.if !$range
@@ -82,13 +83,14 @@
GET_VREG w1, w1
// Note: if w1 is null, this will be handled by our SIGSEGV handler.
ldr w2, [x1, #MIRROR_OBJECT_CLASS_OFFSET]
+ UNPOISON_HEAP_REF w2
// Test the first two bits of the fetched ArtMethod:
// - If the first bit is set, this is a method on j.l.Object
// - If the second bit is set, this is a default method.
tst w26, #0x3
- b.ne 3f
+ b.ne 2f
ldrh w3, [x26, #ART_METHOD_IMT_INDEX_OFFSET]
-2:
+1:
ldr x2, [x2, #MIRROR_CLASS_IMT_PTR_OFFSET_64]
ldr x0, [x2, w3, uxtw #3]
.if $range
@@ -96,13 +98,13 @@
.else
b NterpCommonInvokeInterface
.endif
-3:
- tbnz w26, #0, 4f
+2:
+ tbnz w26, #0, 3f
and x26, x26, #-4
ldrh w3, [x26, #ART_METHOD_METHOD_INDEX_OFFSET]
and w3, w3, #ART_METHOD_IMT_MASK
- b 2b
-4:
+ b 1b
+3:
lsr w26, w26, #16
add w2, w2, #MIRROR_CLASS_VTABLE_OFFSET_64
ldr x0, [x2, w26, uxtw #3]
@@ -111,13 +113,14 @@
.else
b NterpCommonInvokeInstance
.endif
-5:
+
+%def invoke_interface_slow_path(range):
mov x0, xSELF
ldr x1, [sp]
mov x2, xPC
bl nterp_get_method
mov x26, x0
- b 1b
+ b .L${opcode}_resume
%def op_invoke_interface():
% invoke_interface(range="0")
@@ -155,6 +158,7 @@
GET_VREG w1, w1
// Note: if w1 is null, this will be handled by our SIGSEGV handler.
ldr w0, [x1, #MIRROR_OBJECT_CLASS_OFFSET]
+ UNPOISON_HEAP_REF w0
add w0, w0, #MIRROR_CLASS_VTABLE_OFFSET_64
ldr x0, [x0, w2, uxtw #3]
b $helper
diff --git a/runtime/interpreter/mterp/arm64ng/main.S b/runtime/interpreter/mterp/arm64ng/main.S
index 2492aec..0ed237c 100644
--- a/runtime/interpreter/mterp/arm64ng/main.S
+++ b/runtime/interpreter/mterp/arm64ng/main.S
@@ -1404,6 +1404,12 @@
GOTO_OPCODE ip
.endm
+.macro POISON_HEAP_REF_IF_OBJECT is_object, rRef
+ .if \is_object
+ POISON_HEAP_REF \rRef
+ .endif
+.endm
+
.macro WRITE_BARRIER_IF_OBJECT is_object, value, holder, label
.if \is_object
cbz \value, \label
diff --git a/runtime/interpreter/mterp/arm64ng/object.S b/runtime/interpreter/mterp/arm64ng/object.S
index b6a6e24..5202ebc 100644
--- a/runtime/interpreter/mterp/arm64ng/object.S
+++ b/runtime/interpreter/mterp/arm64ng/object.S
@@ -7,6 +7,7 @@
GET_VREG w0, w2 // w0<- vA (object)
cbz w0, .L${opcode}_resume
ldr w2, [x0, #MIRROR_OBJECT_CLASS_OFFSET]
+ UNPOISON_HEAP_REF w2
// Fast path: do a comparison without read barrier.
cmp w1, w2
bne ${slow_path}
@@ -32,9 +33,11 @@
ldr w3, [x1, #MIRROR_CLASS_ACCESS_FLAGS_OFFSET]
tbnz w3, #MIRROR_CLASS_IS_INTERFACE_FLAG_BIT, 2f
ldr w3, [x1, #MIRROR_CLASS_COMPONENT_TYPE_OFFSET]
+ UNPOISON_HEAP_REF w3
cbnz w3, 5f
1:
ldr w2, [x2, #MIRROR_CLASS_SUPER_CLASS_OFFSET]
+ UNPOISON_HEAP_REF w2
cmp w1, w2
beq .L${opcode}_resume
cbnz w2, 1b
@@ -50,9 +53,11 @@
5:
// Class in w1 is an array, w3 is the component type.
ldr w2, [x2, #MIRROR_CLASS_COMPONENT_TYPE_OFFSET]
+ UNPOISON_HEAP_REF w2
// Check if object is an array.
cbz w2, 2b
ldr w4, [x3, #MIRROR_CLASS_SUPER_CLASS_OFFSET]
+ UNPOISON_HEAP_REF w4
// If the super class of the component type is not null, go slow path.
cbnz w4, 2b
ldrh w3, [x3, #MIRROR_CLASS_OBJECT_PRIMITIVE_TYPE_OFFSET]
@@ -74,6 +79,7 @@
GET_VREG w0, w2 // w0<- vB (object)
cbz w0, .L${opcode}_resume
ldr w2, [x0, #MIRROR_OBJECT_CLASS_OFFSET]
+ UNPOISON_HEAP_REF w2
// Fast path: do a comparison without read barrier.
cmp w1, w2
bne ${slow_path}
@@ -101,9 +107,11 @@
ldr w3, [x1, #MIRROR_CLASS_ACCESS_FLAGS_OFFSET]
tbnz w3, #MIRROR_CLASS_IS_INTERFACE_FLAG_BIT, 5f
ldr w3, [x1, #MIRROR_CLASS_COMPONENT_TYPE_OFFSET]
+ UNPOISON_HEAP_REF w3
cbnz w3, 3f
1:
ldr w2, [x2, #MIRROR_CLASS_SUPER_CLASS_OFFSET]
+ UNPOISON_HEAP_REF w2
cmp w1, w2
beq .L${opcode}_set_one
cbnz w2, 1b
@@ -113,10 +121,12 @@
3:
// Class in x1 is an array, x3 is the component type of x1, and x2 is the class of the object.
ldr w2, [x2, #MIRROR_CLASS_COMPONENT_TYPE_OFFSET]
+ UNPOISON_HEAP_REF w2
// Check if object is an array.
cbz w2, 2b
// Check of x1 is Object[]
ldr w4, [x3, #MIRROR_CLASS_SUPER_CLASS_OFFSET]
+ UNPOISON_HEAP_REF w4
// If the super class is not Object, go to slow path.
cbnz w4, 5f
// Super class is null, this could either be a primitive array or Object[].
@@ -161,6 +171,7 @@
SET_VREG_WIDE x0, w2 // fp[A] <- value
.elseif $is_object
$load w0, [x3, x0]
+ UNPOISON_HEAP_REF w0
TEST_IF_MARKING .L${opcode}_read_barrier
.L${opcode}_resume_after_read_barrier:
SET_VREG_OBJECT w0, w2 // fp[A] <- value
@@ -196,6 +207,7 @@
SET_VREG_WIDE x0, w2 // fp[A] <- value
.elseif $is_object
$volatile_load w0, [x3]
+ UNPOISON_HEAP_REF w0
TEST_IF_MARKING .L${opcode}_read_barrier
SET_VREG_OBJECT w0, w2 // fp[A] <- value
.else
@@ -242,6 +254,7 @@
.if $wide
$store x26, [x2, x0]
.else
+ POISON_HEAP_REF_IF_OBJECT $is_object, w26
$store w26, [x2, x0]
WRITE_BARRIER_IF_OBJECT $is_object, w26, w2, .L${opcode}_skip_write_barrier
.endif
@@ -274,6 +287,7 @@
.if $wide
$volatile_store x26, [x3]
.else
+ POISON_HEAP_REF_IF_OBJECT $is_object, w26
$volatile_store w26, [x3]
WRITE_BARRIER_IF_OBJECT $is_object, w26, w2, .L${opcode}_slow_path_skip_write_barrier
.endif
@@ -314,6 +328,7 @@
SET_VREG_WIDE x0, w2 // fp[A] <- value
.elseif $is_object
$load w0, [x0, x1]
+ UNPOISON_HEAP_REF w0
// No need to check the marking register, we know it's not set here.
.L${opcode}_after_reference_load:
SET_VREG_OBJECT w0, w2 // fp[A] <- value
@@ -328,6 +343,7 @@
bl art_quick_read_barrier_mark_reg00
.if $is_object
$load w0, [x0, x1]
+ UNPOISON_HEAP_REF w0
.L${opcode}_mark_after_load:
// Here, we know the marking register is set.
bl art_quick_read_barrier_mark_reg00
@@ -356,6 +372,7 @@
SET_VREG_WIDE x0, w2 // fp[A] <- value
.elseif $is_object
$volatile_load w0, [x0]
+ UNPOISON_HEAP_REF w0
TEST_IF_MARKING .L${opcode}_mark_after_load
SET_VREG_OBJECT w0, w2 // fp[A] <- value
.else
@@ -406,6 +423,7 @@
.if $wide
$store x26, [x0, x1]
.else
+ POISON_HEAP_REF_IF_OBJECT $is_object, w26
$store w26, [x0, x1]
WRITE_BARRIER_IF_OBJECT $is_object, w26, w0, .L${opcode}_skip_write_barrier
.endif
@@ -442,6 +460,7 @@
.if $wide
$volatile_store x26, [x1]
.else
+ POISON_HEAP_REF_IF_OBJECT $is_object, w26
$volatile_store w26, [x1]
WRITE_BARRIER_IF_OBJECT $is_object, w26, w0, .L${opcode}_slow_path_skip_write_barrier
.endif
diff --git a/runtime/interpreter/mterp/nterp.cc b/runtime/interpreter/mterp/nterp.cc
index 52cdeae..6ccc5e9 100644
--- a/runtime/interpreter/mterp/nterp.cc
+++ b/runtime/interpreter/mterp/nterp.cc
@@ -39,8 +39,9 @@
switch (kRuntimeISA) {
case InstructionSet::kArm:
case InstructionSet::kThumb2:
- case InstructionSet::kArm64:
return !kPoisonHeapReferences && kReserveMarkingRegister;
+ case InstructionSet::kArm64:
+ return kReserveMarkingRegister;
case InstructionSet::kRiscv64:
// TODO(riscv64): Support nterp.
return false;