diff options
24 files changed, 513 insertions, 97 deletions
diff --git a/compiler/oat_test.cc b/compiler/oat_test.cc index c98a5f8ba8..88dc29e6ab 100644 --- a/compiler/oat_test.cc +++ b/compiler/oat_test.cc @@ -183,7 +183,7 @@ TEST_F(OatTest, OatHeaderSizeCheck) { EXPECT_EQ(72U, sizeof(OatHeader)); EXPECT_EQ(4U, sizeof(OatMethodOffsets)); EXPECT_EQ(28U, sizeof(OatQuickMethodHeader)); - EXPECT_EQ(112 * GetInstructionSetPointerSize(kRuntimeISA), sizeof(QuickEntryPoints)); + EXPECT_EQ(113 * GetInstructionSetPointerSize(kRuntimeISA), sizeof(QuickEntryPoints)); } TEST_F(OatTest, OatHeaderIsValid) { diff --git a/runtime/arch/arm/entrypoints_init_arm.cc b/runtime/arch/arm/entrypoints_init_arm.cc index 2f2654d4f6..be9af9871d 100644 --- a/runtime/arch/arm/entrypoints_init_arm.cc +++ b/runtime/arch/arm/entrypoints_init_arm.cc @@ -171,6 +171,7 @@ void InitEntryPoints(InterpreterEntryPoints* ipoints, JniEntryPoints* jpoints, // Read barrier qpoints->pReadBarrierJni = ReadBarrierJni; + qpoints->pReadBarrierSlow = artReadBarrierSlow; } } // namespace art diff --git a/runtime/arch/arm/quick_entrypoints_arm.S b/runtime/arch/arm/quick_entrypoints_arm.S index 20001109a6..f6d954f4f1 100644 --- a/runtime/arch/arm/quick_entrypoints_arm.S +++ b/runtime/arch/arm/quick_entrypoints_arm.S @@ -51,7 +51,6 @@ sub sp, #12 @ 3 words of space, bottom word will hold Method* .cfi_adjust_cfa_offset 12 RUNTIME_CURRENT1 \rTemp1, \rTemp2 @ Load Runtime::Current into rTemp1. - THIS_LOAD_REQUIRES_READ_BARRIER ldr \rTemp1, [\rTemp1, #RUNTIME_SAVE_ALL_CALLEE_SAVE_FRAME_OFFSET] @ rTemp1 is kSaveAll Method*. str \rTemp1, [sp, #0] @ Place Method* at bottom of stack. str sp, [r9, #THREAD_TOP_QUICK_FRAME_OFFSET] @ Place sp in Thread::Current()->top_quick_frame. @@ -79,7 +78,6 @@ sub sp, #4 @ bottom word will hold Method* .cfi_adjust_cfa_offset 4 RUNTIME_CURRENT2 \rTemp1, \rTemp2 @ Load Runtime::Current into rTemp1. - THIS_LOAD_REQUIRES_READ_BARRIER ldr \rTemp1, [\rTemp1, #RUNTIME_REFS_ONLY_CALLEE_SAVE_FRAME_OFFSET] @ rTemp1 is kRefsOnly Method*. str \rTemp1, [sp, #0] @ Place Method* at bottom of stack. str sp, [r9, #THREAD_TOP_QUICK_FRAME_OFFSET] @ Place sp in Thread::Current()->top_quick_frame. @@ -139,7 +137,6 @@ .macro SETUP_REFS_AND_ARGS_CALLEE_SAVE_FRAME rTemp1, rTemp2 SETUP_REFS_AND_ARGS_CALLEE_SAVE_FRAME_REGISTERS_ONLY RUNTIME_CURRENT3 \rTemp1, \rTemp2 @ Load Runtime::Current into rTemp1. - THIS_LOAD_REQUIRES_READ_BARRIER @ rTemp1 is kRefsAndArgs Method*. ldr \rTemp1, [\rTemp1, #RUNTIME_REFS_AND_ARGS_CALLEE_SAVE_FRAME_OFFSET] str \rTemp1, [sp, #0] @ Place Method* at bottom of stack. @@ -171,7 +168,6 @@ .cfi_adjust_cfa_offset -40 .endm - .macro RETURN_IF_RESULT_IS_ZERO cbnz r0, 1f @ result non-zero branch over bx lr @ return @@ -588,6 +584,59 @@ ENTRY art_quick_check_cast bkpt END art_quick_check_cast +// Restore rReg's value from [sp, #offset] if rReg is not the same as rExclude. +.macro POP_REG_NE rReg, offset, rExclude + .ifnc \rReg, \rExclude + ldr \rReg, [sp, #\offset] @ restore rReg + .cfi_restore \rReg + .endif +.endm + + /* + * Macro to insert read barrier, only used in art_quick_aput_obj. + * rObj and rDest are registers, offset is a defined literal such as MIRROR_OBJECT_CLASS_OFFSET. + * TODO: When read barrier has a fast path, add heap unpoisoning support for the fast path. + */ +.macro READ_BARRIER rDest, rObj, offset +#ifdef USE_READ_BARRIER + push {r0-r3, ip, lr} @ 6 words for saved registers (used in art_quick_aput_obj) + .cfi_adjust_cfa_offset 24 + .cfi_rel_offset r0, 0 + .cfi_rel_offset r1, 4 + .cfi_rel_offset r2, 8 + .cfi_rel_offset r3, 12 + .cfi_rel_offset ip, 16 + .cfi_rel_offset lr, 20 + sub sp, #8 @ push padding + .cfi_adjust_cfa_offset 8 + @ mov r0, r0 @ pass ref in r0 (no-op for now since parameter ref is unused) + .ifnc \rObj, r1 + mov r1, \rObj @ pass rObj + .endif + mov r2, #\offset @ pass offset + bl artReadBarrierSlow @ artReadBarrierSlow(ref, rObj, offset) + @ No need to unpoison return value in r0, artReadBarrierSlow() would do the unpoisoning. + .ifnc \rDest, r0 + mov \rDest, r0 @ save return value in rDest + .endif + add sp, #8 @ pop padding + .cfi_adjust_cfa_offset -8 + POP_REG_NE r0, 0, \rDest @ conditionally restore saved registers + POP_REG_NE r1, 4, \rDest + POP_REG_NE r2, 8, \rDest + POP_REG_NE r3, 12, \rDest + POP_REG_NE ip, 16, \rDest + add sp, #20 + .cfi_adjust_cfa_offset -20 + pop {lr} @ restore lr + .cfi_adjust_cfa_offset -4 + .cfi_restore lr +#else + ldr \rDest, [\rObj, #\offset] + UNPOISON_HEAP_REF \rDest +#endif // USE_READ_BARRIER +.endm + /* * Entry from managed code for array put operations of objects where the value being stored * needs to be checked for compatibility. @@ -609,15 +658,21 @@ ENTRY art_quick_aput_obj_with_bound_check b art_quick_throw_array_bounds END art_quick_aput_obj_with_bound_check +#ifdef USE_READ_BARRIER + .extern artReadBarrierSlow +#endif .hidden art_quick_aput_obj ENTRY art_quick_aput_obj +#ifdef USE_READ_BARRIER + @ The offset to .Ldo_aput_null is too large to use cbz due to expansion from READ_BARRIER macro. + tst r2, r2 + beq .Ldo_aput_null +#else cbz r2, .Ldo_aput_null - ldr r3, [r0, #MIRROR_OBJECT_CLASS_OFFSET] - UNPOISON_HEAP_REF r3 - ldr ip, [r2, #MIRROR_OBJECT_CLASS_OFFSET] - UNPOISON_HEAP_REF ip - ldr r3, [r3, #MIRROR_CLASS_COMPONENT_TYPE_OFFSET] - UNPOISON_HEAP_REF r3 +#endif // USE_READ_BARRIER + READ_BARRIER r3, r0, MIRROR_OBJECT_CLASS_OFFSET + READ_BARRIER ip, r2, MIRROR_OBJECT_CLASS_OFFSET + READ_BARRIER r3, r3, MIRROR_CLASS_COMPONENT_TYPE_OFFSET cmp r3, ip @ value's type == array's component type - trivial assignability bne .Lcheck_assignability .Ldo_aput: diff --git a/runtime/arch/arm64/entrypoints_init_arm64.cc b/runtime/arch/arm64/entrypoints_init_arm64.cc index 2ce2a29bbf..0f06727d0d 100644 --- a/runtime/arch/arm64/entrypoints_init_arm64.cc +++ b/runtime/arch/arm64/entrypoints_init_arm64.cc @@ -155,6 +155,7 @@ void InitEntryPoints(InterpreterEntryPoints* ipoints, JniEntryPoints* jpoints, // Read barrier qpoints->pReadBarrierJni = ReadBarrierJni; + qpoints->pReadBarrierSlow = artReadBarrierSlow; }; } // namespace art diff --git a/runtime/arch/arm64/quick_entrypoints_arm64.S b/runtime/arch/arm64/quick_entrypoints_arm64.S index 6d9b44a1d2..548ab47f82 100644 --- a/runtime/arch/arm64/quick_entrypoints_arm64.S +++ b/runtime/arch/arm64/quick_entrypoints_arm64.S @@ -31,8 +31,6 @@ ldr xIP0, [xIP0] // xIP0 = & (art::Runtime * art::Runtime.instance_) . // xIP0 = (ArtMethod*) Runtime.instance_.callee_save_methods[kRefAndArgs] . - THIS_LOAD_REQUIRES_READ_BARRIER - // Loads appropriate callee-save-method. ldr xIP0, [xIP0, RUNTIME_SAVE_ALL_CALLEE_SAVE_FRAME_OFFSET ] @@ -95,8 +93,6 @@ ldr xIP0, [xIP0] // xIP0 = & (art::Runtime * art::Runtime.instance_) . // xIP0 = (ArtMethod*) Runtime.instance_.callee_save_methods[kRefOnly] . - THIS_LOAD_REQUIRES_READ_BARRIER - // Loads appropriate callee-save-method. ldr xIP0, [xIP0, RUNTIME_REFS_ONLY_CALLEE_SAVE_FRAME_OFFSET ] @@ -251,7 +247,6 @@ ldr xIP0, [xIP0] // xIP0 = & (art::Runtime * art::Runtime.instance_) . // xIP0 = (ArtMethod*) Runtime.instance_.callee_save_methods[kRefAndArgs] . - THIS_LOAD_REQUIRES_READ_BARRIER ldr xIP0, [xIP0, RUNTIME_REFS_AND_ARGS_CALLEE_SAVE_FRAME_OFFSET ] SETUP_REFS_AND_ARGS_CALLEE_SAVE_FRAME_INTERNAL @@ -1119,6 +1114,62 @@ ENTRY art_quick_check_cast brk 0 // We should not return here... END art_quick_check_cast +// Restore xReg's value from [sp, #offset] if xReg is not the same as xExclude. +.macro POP_REG_NE xReg, offset, xExclude + .ifnc \xReg, \xExclude + ldr \xReg, [sp, #\offset] // restore xReg + .cfi_restore \xReg + .endif +.endm + + /* + * Macro to insert read barrier, only used in art_quick_aput_obj. + * xDest, wDest and xObj are registers, offset is a defined literal such as + * MIRROR_OBJECT_CLASS_OFFSET. Dest needs both x and w versions of the same register to handle + * name mismatch between instructions. This macro uses the lower 32b of register when possible. + * TODO: When read barrier has a fast path, add heap unpoisoning support for the fast path. + */ +.macro READ_BARRIER xDest, wDest, xObj, offset +#ifdef USE_READ_BARRIER + // Store registers used in art_quick_aput_obj (x0-x4, LR), stack is 16B aligned. + stp x0, x1, [sp, #-48]! + .cfi_adjust_cfa_offset 48 + .cfi_rel_offset x0, 0 + .cfi_rel_offset x1, 8 + stp x2, x3, [sp, #16] + .cfi_rel_offset x2, 16 + .cfi_rel_offset x3, 24 + stp x4, xLR, [sp, #32] + .cfi_rel_offset x4, 32 + .cfi_rel_offset x30, 40 + + // mov x0, x0 // pass ref in x0 (no-op for now since parameter ref is unused) + .ifnc \xObj, x1 + mov x1, \xObj // pass xObj + .endif + mov w2, #\offset // pass offset + bl artReadBarrierSlow // artReadBarrierSlow(ref, xObj, offset) + // No need to unpoison return value in w0, artReadBarrierSlow() would do the unpoisoning. + .ifnc \wDest, w0 + mov \wDest, w0 // save return value in wDest + .endif + + // Conditionally restore saved registers + POP_REG_NE x0, 0, \xDest + POP_REG_NE x1, 8, \xDest + POP_REG_NE x2, 16, \xDest + POP_REG_NE x3, 24, \xDest + POP_REG_NE x4, 32, \xDest + ldr xLR, [sp, #40] + .cfi_restore x30 + add sp, sp, #48 + .cfi_adjust_cfa_offset -48 +#else + ldr \wDest, [\xObj, #\offset] // Heap reference = 32b. This also zero-extends to \xDest. + UNPOISON_HEAP_REF \wDest +#endif // USE_READ_BARRIER +.endm + /* * Entry from managed code for array put operations of objects where the value being stored * needs to be checked for compatibility. @@ -1146,17 +1197,17 @@ ENTRY art_quick_aput_obj_with_bound_check b art_quick_throw_array_bounds END art_quick_aput_obj_with_bound_check +#ifdef USE_READ_BARRIER + .extern artReadBarrierSlow +#endif ENTRY art_quick_aput_obj cbz x2, .Ldo_aput_null - ldr w3, [x0, #MIRROR_OBJECT_CLASS_OFFSET] // Heap reference = 32b + READ_BARRIER x3, w3, x0, MIRROR_OBJECT_CLASS_OFFSET // Heap reference = 32b // This also zero-extends to x3 - UNPOISON_HEAP_REF w3 - ldr w4, [x2, #MIRROR_OBJECT_CLASS_OFFSET] // Heap reference = 32b + READ_BARRIER x4, w4, x2, MIRROR_OBJECT_CLASS_OFFSET // Heap reference = 32b // This also zero-extends to x4 - UNPOISON_HEAP_REF w4 - ldr w3, [x3, #MIRROR_CLASS_COMPONENT_TYPE_OFFSET] // Heap reference = 32b + READ_BARRIER x3, w3, x3, MIRROR_CLASS_COMPONENT_TYPE_OFFSET // Heap reference = 32b // This also zero-extends to x3 - UNPOISON_HEAP_REF w3 cmp w3, w4 // value's type == array's component type - trivial assignability bne .Lcheck_assignability .Ldo_aput: diff --git a/runtime/arch/mips/entrypoints_direct_mips.h b/runtime/arch/mips/entrypoints_direct_mips.h index b1aa3ee63f..f9c53152f6 100644 --- a/runtime/arch/mips/entrypoints_direct_mips.h +++ b/runtime/arch/mips/entrypoints_direct_mips.h @@ -44,7 +44,8 @@ static constexpr bool IsDirectEntrypoint(QuickEntrypointEnum entrypoint) { entrypoint == kQuickCmpgDouble || entrypoint == kQuickCmpgFloat || entrypoint == kQuickCmplDouble || - entrypoint == kQuickCmplFloat; + entrypoint == kQuickCmplFloat || + entrypoint == kQuickReadBarrierSlow; } } // namespace art diff --git a/runtime/arch/mips/entrypoints_init_mips.cc b/runtime/arch/mips/entrypoints_init_mips.cc index 09a018ebc6..4e4b91fdcd 100644 --- a/runtime/arch/mips/entrypoints_init_mips.cc +++ b/runtime/arch/mips/entrypoints_init_mips.cc @@ -279,6 +279,8 @@ void InitEntryPoints(InterpreterEntryPoints* ipoints, JniEntryPoints* jpoints, qpoints->pReadBarrierJni = ReadBarrierJni; static_assert(!IsDirectEntrypoint(kQuickReadBarrierJni), "Non-direct C stub marked direct."); + qpoints->pReadBarrierSlow = artReadBarrierSlow; + static_assert(IsDirectEntrypoint(kQuickReadBarrierSlow), "Direct C stub not marked direct."); }; } // namespace art diff --git a/runtime/arch/mips/quick_entrypoints_mips.S b/runtime/arch/mips/quick_entrypoints_mips.S index 2819f92a0d..4d5004f444 100644 --- a/runtime/arch/mips/quick_entrypoints_mips.S +++ b/runtime/arch/mips/quick_entrypoints_mips.S @@ -79,7 +79,6 @@ lw $t0, %got(_ZN3art7Runtime9instance_E)($gp) lw $t0, 0($t0) - THIS_LOAD_REQUIRES_READ_BARRIER lw $t0, RUNTIME_SAVE_ALL_CALLEE_SAVE_FRAME_OFFSET($t0) sw $t0, 0($sp) # Place Method* at bottom of stack. sw $sp, THREAD_TOP_QUICK_FRAME_OFFSET(rSELF) # Place sp in Thread::Current()->top_quick_frame. @@ -127,7 +126,6 @@ lw $t0, %got(_ZN3art7Runtime9instance_E)($gp) lw $t0, 0($t0) - THIS_LOAD_REQUIRES_READ_BARRIER lw $t0, RUNTIME_REFS_ONLY_CALLEE_SAVE_FRAME_OFFSET($t0) sw $t0, 0($sp) # Place Method* at bottom of stack. sw $sp, THREAD_TOP_QUICK_FRAME_OFFSET(rSELF) # Place sp in Thread::Current()->top_quick_frame. @@ -219,7 +217,6 @@ SETUP_REFS_AND_ARGS_CALLEE_SAVE_FRAME_REGISTERS_ONLY lw $t0, %got(_ZN3art7Runtime9instance_E)($gp) lw $t0, 0($t0) - THIS_LOAD_REQUIRES_READ_BARRIER lw $t0, RUNTIME_REFS_AND_ARGS_CALLEE_SAVE_FRAME_OFFSET($t0) sw $t0, 0($sp) # Place Method* at bottom of stack. sw $sp, THREAD_TOP_QUICK_FRAME_OFFSET(rSELF) # Place sp in Thread::Current()->top_quick_frame. @@ -627,6 +624,76 @@ ENTRY art_quick_check_cast END art_quick_check_cast /* + * Restore rReg's value from offset($sp) if rReg is not the same as rExclude. + * nReg is the register number for rReg. + */ +.macro POP_REG_NE rReg, nReg, offset, rExclude + .ifnc \rReg, \rExclude + lw \rReg, \offset($sp) # restore rReg + .cfi_restore \nReg + .endif +.endm + + /* + * Macro to insert read barrier, only used in art_quick_aput_obj. + * rObj and rDest are registers, offset is a defined literal such as MIRROR_OBJECT_CLASS_OFFSET. + * TODO: When read barrier has a fast path, add heap unpoisoning support for the fast path. + */ +.macro READ_BARRIER rDest, rObj, offset +#ifdef USE_READ_BARRIER + # saved registers used in art_quick_aput_obj: a0-a2, t0-t1, t9, ra. 8 words for 16B alignment. + addiu $sp, $sp, -32 + .cfi_adjust_cfa_offset 32 + sw $ra, 28($sp) + .cfi_rel_offset 31, 28 + sw $t9, 24($sp) + .cfi_rel_offset 25, 24 + sw $t1, 20($sp) + .cfi_rel_offset 9, 20 + sw $t0, 16($sp) + .cfi_rel_offset 8, 16 + sw $a2, 8($sp) # padding slot at offset 12 (padding can be any slot in the 32B) + .cfi_rel_offset 6, 8 + sw $a1, 4($sp) + .cfi_rel_offset 5, 4 + sw $a0, 0($sp) + .cfi_rel_offset 4, 0 + + # move $a0, $a0 # pass ref in a0 (no-op for now since parameter ref is unused) + .ifnc \rObj, $a1 + move $a1, \rObj # pass rObj + .endif + addiu $a2, $zero, \offset # pass offset + jal artReadBarrierSlow # artReadBarrierSlow(ref, rObj, offset) + addiu $sp, $sp, -16 # Use branch delay slot to reserve argument slots on the stack + # before the call to artReadBarrierSlow. + addiu $sp, $sp, 16 # restore stack after call to artReadBarrierSlow + # No need to unpoison return value in v0, artReadBarrierSlow() would do the unpoisoning. + move \rDest, $v0 # save return value in rDest + # (rDest cannot be v0 in art_quick_aput_obj) + + lw $a0, 0($sp) # restore registers except rDest + # (rDest can only be t0 or t1 in art_quick_aput_obj) + .cfi_restore 4 + lw $a1, 4($sp) + .cfi_restore 5 + lw $a2, 8($sp) + .cfi_restore 6 + POP_REG_NE $t0, 8, 16, \rDest + POP_REG_NE $t1, 9, 20, \rDest + lw $t9, 24($sp) + .cfi_restore 25 + lw $ra, 28($sp) # restore $ra + .cfi_restore 31 + addiu $sp, $sp, 32 + .cfi_adjust_cfa_offset -32 +#else + lw \rDest, \offset(\rObj) + UNPOISON_HEAP_REF \rDest +#endif // USE_READ_BARRIER +.endm + + /* * Entry from managed code for array put operations of objects where the value being stored * needs to be checked for compatibility. * a0 = array, a1 = index, a2 = value @@ -648,15 +715,15 @@ ENTRY art_quick_aput_obj_with_bound_check move $a1, $t0 END art_quick_aput_obj_with_bound_check +#ifdef USE_READ_BARRIER + .extern artReadBarrierSlow +#endif ENTRY art_quick_aput_obj beqz $a2, .Ldo_aput_null nop - lw $t0, MIRROR_OBJECT_CLASS_OFFSET($a0) - UNPOISON_HEAP_REF $t0 - lw $t1, MIRROR_OBJECT_CLASS_OFFSET($a2) - UNPOISON_HEAP_REF $t1 - lw $t0, MIRROR_CLASS_COMPONENT_TYPE_OFFSET($t0) - UNPOISON_HEAP_REF $t0 + READ_BARRIER $t0, $a0, MIRROR_OBJECT_CLASS_OFFSET + READ_BARRIER $t1, $a2, MIRROR_OBJECT_CLASS_OFFSET + READ_BARRIER $t0, $t0, MIRROR_CLASS_COMPONENT_TYPE_OFFSET bne $t1, $t0, .Lcheck_assignability # value's type == array's component type - trivial assignability nop .Ldo_aput: diff --git a/runtime/arch/mips64/entrypoints_init_mips64.cc b/runtime/arch/mips64/entrypoints_init_mips64.cc index 4904af9cfc..ec02d5ab69 100644 --- a/runtime/arch/mips64/entrypoints_init_mips64.cc +++ b/runtime/arch/mips64/entrypoints_init_mips64.cc @@ -186,6 +186,7 @@ void InitEntryPoints(InterpreterEntryPoints* ipoints, JniEntryPoints* jpoints, // Read barrier qpoints->pReadBarrierJni = ReadBarrierJni; + qpoints->pReadBarrierSlow = artReadBarrierSlow; }; } // namespace art diff --git a/runtime/arch/mips64/quick_entrypoints_mips64.S b/runtime/arch/mips64/quick_entrypoints_mips64.S index abca70b363..c30e6ca93f 100644 --- a/runtime/arch/mips64/quick_entrypoints_mips64.S +++ b/runtime/arch/mips64/quick_entrypoints_mips64.S @@ -89,7 +89,6 @@ # load appropriate callee-save-method ld $t1, %got(_ZN3art7Runtime9instance_E)($gp) ld $t1, 0($t1) - THIS_LOAD_REQUIRES_READ_BARRIER ld $t1, RUNTIME_SAVE_ALL_CALLEE_SAVE_FRAME_OFFSET($t1) sd $t1, 0($sp) # Place ArtMethod* at bottom of stack. sd $sp, THREAD_TOP_QUICK_FRAME_OFFSET(rSELF) # Place sp in Thread::Current()->top_quick_frame. @@ -132,7 +131,6 @@ # load appropriate callee-save-method ld $t1, %got(_ZN3art7Runtime9instance_E)($gp) ld $t1, 0($t1) - THIS_LOAD_REQUIRES_READ_BARRIER ld $t1, RUNTIME_REFS_ONLY_CALLEE_SAVE_FRAME_OFFSET($t1) sd $t1, 0($sp) # Place Method* at bottom of stack. sd $sp, THREAD_TOP_QUICK_FRAME_OFFSET(rSELF) # Place sp in Thread::Current()->top_quick_frame. @@ -255,7 +253,6 @@ # load appropriate callee-save-method ld $t1, %got(_ZN3art7Runtime9instance_E)($gp) ld $t1, 0($t1) - THIS_LOAD_REQUIRES_READ_BARRIER ld $t1, RUNTIME_REFS_AND_ARGS_CALLEE_SAVE_FRAME_OFFSET($t1) sd $t1, 0($sp) # Place Method* at bottom of stack. sd $sp, THREAD_TOP_QUICK_FRAME_OFFSET(rSELF) # Place sp in Thread::Current()->top_quick_frame. @@ -888,6 +885,77 @@ ENTRY art_quick_check_cast move $a2, rSELF # pass Thread::Current END art_quick_check_cast + + /* + * Restore rReg's value from offset($sp) if rReg is not the same as rExclude. + * nReg is the register number for rReg. + */ +.macro POP_REG_NE rReg, nReg, offset, rExclude + .ifnc \rReg, \rExclude + ld \rReg, \offset($sp) # restore rReg + .cfi_restore \nReg + .endif +.endm + + /* + * Macro to insert read barrier, only used in art_quick_aput_obj. + * rObj and rDest are registers, offset is a defined literal such as MIRROR_OBJECT_CLASS_OFFSET. + * TODO: When read barrier has a fast path, add heap unpoisoning support for the fast path. + */ +.macro READ_BARRIER rDest, rObj, offset +#ifdef USE_READ_BARRIER + # saved registers used in art_quick_aput_obj: a0-a2, t0-t1, t9, ra. 16B-aligned. + daddiu $sp, $sp, -64 + .cfi_adjust_cfa_offset 64 + sd $ra, 56($sp) + .cfi_rel_offset 31, 56 + sd $t9, 48($sp) + .cfi_rel_offset 25, 48 + sd $t1, 40($sp) + .cfi_rel_offset 13, 40 + sd $t0, 32($sp) + .cfi_rel_offset 12, 32 + sd $a2, 16($sp) # padding slot at offset 24 (padding can be any slot in the 64B) + .cfi_rel_offset 6, 16 + sd $a1, 8($sp) + .cfi_rel_offset 5, 8 + sd $a0, 0($sp) + .cfi_rel_offset 4, 0 + + # move $a0, $a0 # pass ref in a0 (no-op for now since parameter ref is unused) + .ifnc \rObj, $a1 + move $a1, \rObj # pass rObj + .endif + daddiu $a2, $zero, \offset # pass offset + jal artReadBarrierSlow # artReadBarrierSlow(ref, rObj, offset) + .cpreturn # Restore gp from t8 in branch delay slot. + # t8 may be clobbered in artReadBarrierSlow. + # No need to unpoison return value in v0, artReadBarrierSlow() would do the unpoisoning. + move \rDest, $v0 # save return value in rDest + # (rDest cannot be v0 in art_quick_aput_obj) + + ld $a0, 0($sp) # restore registers except rDest + # (rDest can only be t0 or t1 in art_quick_aput_obj) + .cfi_restore 4 + ld $a1, 8($sp) + .cfi_restore 5 + ld $a2, 16($sp) + .cfi_restore 6 + POP_REG_NE $t0, 12, 32, \rDest + POP_REG_NE $t1, 13, 40, \rDest + ld $t9, 48($sp) + .cfi_restore 25 + ld $ra, 56($sp) # restore $ra + .cfi_restore 31 + daddiu $sp, $sp, 64 + .cfi_adjust_cfa_offset -64 + SETUP_GP # set up gp because we are not returning +#else + lwu \rDest, \offset(\rObj) + UNPOISON_HEAP_REF \rDest +#endif // USE_READ_BARRIER +.endm + /* * Entry from managed code for array put operations of objects where the value being stored * needs to be checked for compatibility. @@ -913,12 +981,9 @@ END art_quick_aput_obj_with_bound_check ENTRY art_quick_aput_obj beq $a2, $zero, .Ldo_aput_null nop - lwu $t0, MIRROR_OBJECT_CLASS_OFFSET($a0) - UNPOISON_HEAP_REF $t0 - lwu $t1, MIRROR_OBJECT_CLASS_OFFSET($a2) - UNPOISON_HEAP_REF $t1 - lwu $t0, MIRROR_CLASS_COMPONENT_TYPE_OFFSET($t0) - UNPOISON_HEAP_REF $t0 + READ_BARRIER $t0, $a0, MIRROR_OBJECT_CLASS_OFFSET + READ_BARRIER $t1, $a2, MIRROR_OBJECT_CLASS_OFFSET + READ_BARRIER $t0, $t0, MIRROR_CLASS_COMPONENT_TYPE_OFFSET bne $t1, $t0, .Lcheck_assignability # value's type == array's component type - trivial assignability nop .Ldo_aput: diff --git a/runtime/arch/stub_test.cc b/runtime/arch/stub_test.cc index 0831c26e9a..cf7db34ca1 100644 --- a/runtime/arch/stub_test.cc +++ b/runtime/arch/stub_test.cc @@ -1124,8 +1124,6 @@ TEST_F(StubTest, CheckCast) { TEST_F(StubTest, APutObj) { - TEST_DISABLED_FOR_READ_BARRIER(); - #if defined(__i386__) || defined(__arm__) || defined(__aarch64__) || defined(__mips__) || \ (defined(__x86_64__) && !defined(__APPLE__)) Thread* self = Thread::Current(); @@ -1258,8 +1256,6 @@ TEST_F(StubTest, APutObj) { } TEST_F(StubTest, AllocObject) { - TEST_DISABLED_FOR_READ_BARRIER(); - #if defined(__i386__) || defined(__arm__) || defined(__aarch64__) || defined(__mips__) || \ (defined(__x86_64__) && !defined(__APPLE__)) // This will lead to OOM error messages in the log. @@ -1385,8 +1381,6 @@ TEST_F(StubTest, AllocObject) { } TEST_F(StubTest, AllocObjectArray) { - TEST_DISABLED_FOR_READ_BARRIER(); - #if defined(__i386__) || defined(__arm__) || defined(__aarch64__) || defined(__mips__) || \ (defined(__x86_64__) && !defined(__APPLE__)) // TODO: Check the "Unresolved" allocation stubs @@ -1474,8 +1468,6 @@ TEST_F(StubTest, AllocObjectArray) { TEST_F(StubTest, StringCompareTo) { - TEST_DISABLED_FOR_READ_BARRIER(); - #if defined(__i386__) || defined(__arm__) || defined(__aarch64__) || (defined(__x86_64__) && !defined(__APPLE__)) // TODO: Check the "Unresolved" allocation stubs @@ -2152,8 +2144,6 @@ static void TestFields(Thread* self, StubTest* test, Primitive::Type test_type) } TEST_F(StubTest, Fields8) { - TEST_DISABLED_FOR_READ_BARRIER(); - Thread* self = Thread::Current(); self->TransitionFromSuspendedToRunnable(); @@ -2166,8 +2156,6 @@ TEST_F(StubTest, Fields8) { } TEST_F(StubTest, Fields16) { - TEST_DISABLED_FOR_READ_BARRIER(); - Thread* self = Thread::Current(); self->TransitionFromSuspendedToRunnable(); @@ -2180,8 +2168,6 @@ TEST_F(StubTest, Fields16) { } TEST_F(StubTest, Fields32) { - TEST_DISABLED_FOR_READ_BARRIER(); - Thread* self = Thread::Current(); self->TransitionFromSuspendedToRunnable(); @@ -2193,8 +2179,6 @@ TEST_F(StubTest, Fields32) { } TEST_F(StubTest, FieldsObj) { - TEST_DISABLED_FOR_READ_BARRIER(); - Thread* self = Thread::Current(); self->TransitionFromSuspendedToRunnable(); @@ -2206,8 +2190,6 @@ TEST_F(StubTest, FieldsObj) { } TEST_F(StubTest, Fields64) { - TEST_DISABLED_FOR_READ_BARRIER(); - Thread* self = Thread::Current(); self->TransitionFromSuspendedToRunnable(); @@ -2221,8 +2203,6 @@ TEST_F(StubTest, Fields64) { TEST_F(StubTest, IMT) { #if defined(__i386__) || defined(__arm__) || defined(__aarch64__) || defined(__mips__) || \ (defined(__x86_64__) && !defined(__APPLE__)) - TEST_DISABLED_FOR_READ_BARRIER(); - Thread* self = Thread::Current(); ScopedObjectAccess soa(self); @@ -2342,8 +2322,6 @@ TEST_F(StubTest, IMT) { TEST_F(StubTest, StringIndexOf) { #if defined(__arm__) || defined(__aarch64__) - TEST_DISABLED_FOR_READ_BARRIER(); - Thread* self = Thread::Current(); ScopedObjectAccess soa(self); // garbage is created during ClassLinker::Init @@ -2416,4 +2394,40 @@ TEST_F(StubTest, StringIndexOf) { #endif } +TEST_F(StubTest, ReadBarrier) { +#if defined(ART_USE_READ_BARRIER) && (defined(__i386__) || defined(__arm__) || \ + defined(__aarch64__) || defined(__mips__) || (defined(__x86_64__) && !defined(__APPLE__))) + Thread* self = Thread::Current(); + + const uintptr_t readBarrierSlow = StubTest::GetEntrypoint(self, kQuickReadBarrierSlow); + + // Create an object + ScopedObjectAccess soa(self); + // garbage is created during ClassLinker::Init + + StackHandleScope<2> hs(soa.Self()); + Handle<mirror::Class> c( + hs.NewHandle(class_linker_->FindSystemClass(soa.Self(), "Ljava/lang/Object;"))); + + // Build an object instance + Handle<mirror::Object> obj(hs.NewHandle(c->AllocObject(soa.Self()))); + + EXPECT_FALSE(self->IsExceptionPending()); + + size_t result = Invoke3(0U, reinterpret_cast<size_t>(obj.Get()), + mirror::Object::ClassOffset().SizeValue(), readBarrierSlow, self); + + EXPECT_FALSE(self->IsExceptionPending()); + EXPECT_NE(reinterpret_cast<size_t>(nullptr), result); + mirror::Class* klass = reinterpret_cast<mirror::Class*>(result); + EXPECT_EQ(klass, obj->GetClass()); + + // Tests done. +#else + LOG(INFO) << "Skipping read_barrier_slow"; + // Force-print to std::cout so it's also outside the logcat. + std::cout << "Skipping read_barrier_slow" << std::endl; +#endif +} + } // namespace art diff --git a/runtime/arch/x86/entrypoints_init_x86.cc b/runtime/arch/x86/entrypoints_init_x86.cc index 737f4d1c5b..e2632c103b 100644 --- a/runtime/arch/x86/entrypoints_init_x86.cc +++ b/runtime/arch/x86/entrypoints_init_x86.cc @@ -28,6 +28,9 @@ namespace art { extern "C" uint32_t art_quick_is_assignable(const mirror::Class* klass, const mirror::Class* ref_class); +// Read barrier entrypoints. +extern "C" mirror::Object* art_quick_read_barrier_slow(mirror::Object*, mirror::Object*, uint32_t); + void InitEntryPoints(InterpreterEntryPoints* ipoints, JniEntryPoints* jpoints, QuickEntryPoints* qpoints) { // Interpreter @@ -141,6 +144,7 @@ void InitEntryPoints(InterpreterEntryPoints* ipoints, JniEntryPoints* jpoints, // Read barrier qpoints->pReadBarrierJni = ReadBarrierJni; + qpoints->pReadBarrierSlow = art_quick_read_barrier_slow; }; } // namespace art diff --git a/runtime/arch/x86/quick_entrypoints_x86.S b/runtime/arch/x86/quick_entrypoints_x86.S index ebfb3faf4b..1da5a2ff17 100644 --- a/runtime/arch/x86/quick_entrypoints_x86.S +++ b/runtime/arch/x86/quick_entrypoints_x86.S @@ -33,7 +33,6 @@ MACRO2(SETUP_SAVE_ALL_CALLEE_SAVE_FRAME, got_reg, temp_reg) movl SYMBOL(_ZN3art7Runtime9instance_E)@GOT(REG_VAR(got_reg)), REG_VAR(temp_reg) movl (REG_VAR(temp_reg)), REG_VAR(temp_reg) // Push save all callee-save method. - THIS_LOAD_REQUIRES_READ_BARRIER pushl RUNTIME_SAVE_ALL_CALLEE_SAVE_FRAME_OFFSET(REG_VAR(temp_reg)) CFI_ADJUST_CFA_OFFSET(4) // Store esp as the top quick frame. @@ -60,7 +59,6 @@ MACRO2(SETUP_REFS_ONLY_CALLEE_SAVE_FRAME, got_reg, temp_reg) movl SYMBOL(_ZN3art7Runtime9instance_E)@GOT(REG_VAR(got_reg)), REG_VAR(temp_reg) movl (REG_VAR(temp_reg)), REG_VAR(temp_reg) // Push save all callee-save method. - THIS_LOAD_REQUIRES_READ_BARRIER pushl RUNTIME_REFS_ONLY_CALLEE_SAVE_FRAME_OFFSET(REG_VAR(temp_reg)) CFI_ADJUST_CFA_OFFSET(4) // Store esp as the top quick frame. @@ -106,7 +104,6 @@ MACRO2(SETUP_REFS_AND_ARGS_CALLEE_SAVE_FRAME, got_reg, temp_reg) movl SYMBOL(_ZN3art7Runtime9instance_E)@GOT(REG_VAR(got_reg)), REG_VAR(temp_reg) movl (REG_VAR(temp_reg)), REG_VAR(temp_reg) // Push save all callee-save method. - THIS_LOAD_REQUIRES_READ_BARRIER pushl RUNTIME_REFS_AND_ARGS_CALLEE_SAVE_FRAME_OFFSET(REG_VAR(temp_reg)) CFI_ADJUST_CFA_OFFSET(4) // Store esp as the stop quick frame. @@ -1126,6 +1123,53 @@ DEFINE_FUNCTION art_quick_check_cast UNREACHABLE END_FUNCTION art_quick_check_cast +// Restore reg's value if reg is not the same as exclude_reg, otherwise just adjust stack. +MACRO2(POP_REG_NE, reg, exclude_reg) + .ifc RAW_VAR(reg), RAW_VAR(exclude_reg) + addl MACRO_LITERAL(4), %esp + CFI_ADJUST_CFA_OFFSET(-4) + .else + POP RAW_VAR(reg) + .endif +END_MACRO + + /* + * Macro to insert read barrier, only used in art_quick_aput_obj. + * obj_reg and dest_reg are registers, offset is a defined literal such as + * MIRROR_OBJECT_CLASS_OFFSET. + * pop_eax is a boolean flag, indicating if eax is popped after the call. + * TODO: When read barrier has a fast path, add heap unpoisoning support for the fast path. + */ +MACRO4(READ_BARRIER, obj_reg, offset, dest_reg, pop_eax) +#ifdef USE_READ_BARRIER + PUSH eax // save registers used in art_quick_aput_obj + PUSH ebx + PUSH edx + PUSH ecx + // Outgoing argument set up + pushl MACRO_LITERAL((RAW_VAR(offset))) // pass offset, double parentheses are necessary + CFI_ADJUST_CFA_OFFSET(4) + PUSH RAW_VAR(obj_reg) // pass obj_reg + PUSH eax // pass ref, just pass eax for now since parameter ref is unused + call SYMBOL(artReadBarrierSlow) // artReadBarrierSlow(ref, obj_reg, offset) + // No need to unpoison return value in eax, artReadBarrierSlow() would do the unpoisoning. + .ifnc RAW_VAR(dest_reg), eax + movl %eax, REG_VAR(dest_reg) // save loaded ref in dest_reg + .endif + addl MACRO_LITERAL(12), %esp // pop arguments + CFI_ADJUST_CFA_OFFSET(-12) + POP_REG_NE ecx, RAW_VAR(dest_reg) // Restore args except dest_reg + POP_REG_NE edx, RAW_VAR(dest_reg) + POP_REG_NE ebx, RAW_VAR(dest_reg) + .ifc RAW_VAR(pop_eax), true + POP_REG_NE eax, RAW_VAR(dest_reg) + .endif +#else + movl RAW_VAR(offset)(REG_VAR(obj_reg)), REG_VAR(dest_reg) + UNPOISON_HEAP_REF RAW_VAR(dest_reg) +#endif // USE_READ_BARRIER +END_MACRO + /* * Entry from managed code for array put operations of objects where the value being stored * needs to be checked for compatibility. @@ -1149,17 +1193,20 @@ END_FUNCTION art_quick_aput_obj_with_bound_check DEFINE_FUNCTION art_quick_aput_obj test %edx, %edx // store of null jz .Ldo_aput_null - movl MIRROR_OBJECT_CLASS_OFFSET(%eax), %ebx - UNPOISON_HEAP_REF ebx - movl MIRROR_CLASS_COMPONENT_TYPE_OFFSET(%ebx), %ebx - UNPOISON_HEAP_REF ebx + READ_BARRIER eax, MIRROR_OBJECT_CLASS_OFFSET, ebx, true + READ_BARRIER ebx, MIRROR_CLASS_COMPONENT_TYPE_OFFSET, ebx, true // value's type == array's component type - trivial assignability -#ifdef USE_HEAP_POISONING - PUSH eax // save eax +#if defined(USE_READ_BARRIER) + READ_BARRIER edx, MIRROR_OBJECT_CLASS_OFFSET, eax, false + cmpl %eax, %ebx + POP eax // restore eax from the push in the beginning of READ_BARRIER macro +#elif defined(USE_HEAP_POISONING) + PUSH eax // save eax + // Cannot call READ_BARRIER macro here, because the above push messes up stack alignment. movl MIRROR_OBJECT_CLASS_OFFSET(%edx), %eax UNPOISON_HEAP_REF eax cmpl %eax, %ebx - POP eax // restore eax + POP eax // restore eax #else cmpl MIRROR_OBJECT_CLASS_OFFSET(%edx), %ebx #endif @@ -1181,6 +1228,8 @@ DEFINE_FUNCTION art_quick_aput_obj subl LITERAL(8), %esp // alignment padding CFI_ADJUST_CFA_OFFSET(8) #ifdef USE_HEAP_POISONING + // This load does not need read barrier, since edx is unchanged and there's no GC safe point + // from last read of MIRROR_OBJECT_CLASS_OFFSET(%edx). movl MIRROR_OBJECT_CLASS_OFFSET(%edx), %eax // pass arg2 - type of the value to be stored UNPOISON_HEAP_REF eax PUSH eax @@ -1696,5 +1745,15 @@ DEFINE_FUNCTION art_nested_signal_return UNREACHABLE END_FUNCTION art_nested_signal_return +DEFINE_FUNCTION art_quick_read_barrier_slow + PUSH edx // pass arg3 - offset + PUSH ecx // pass arg2 - obj + PUSH eax // pass arg1 - ref + call SYMBOL(artReadBarrierSlow) // artReadBarrierSlow(ref, obj, offset) + addl LITERAL(12), %esp // pop arguments + CFI_ADJUST_CFA_OFFSET(-12) + ret +END_FUNCTION art_quick_read_barrier_slow + // TODO: implement these! UNIMPLEMENTED art_quick_memcmp16 diff --git a/runtime/arch/x86_64/asm_support_x86_64.S b/runtime/arch/x86_64/asm_support_x86_64.S index 706ae58d91..cf0039c84e 100644 --- a/runtime/arch/x86_64/asm_support_x86_64.S +++ b/runtime/arch/x86_64/asm_support_x86_64.S @@ -24,6 +24,7 @@ #define MACRO1(macro_name, macro_arg1) .macro macro_name macro_arg1 #define MACRO2(macro_name, macro_arg1, macro_arg2) .macro macro_name macro_arg1, macro_arg2 #define MACRO3(macro_name, macro_arg1, macro_arg2, macro_arg3) .macro macro_name macro_arg1, macro_arg2, macro_arg3 +#define MACRO4(macro_name, macro_arg1, macro_arg2, macro_arg3, macro_arg4) .macro macro_name macro_arg1, macro_arg2, macro_arg3, macro_arg4 #define END_MACRO .endm #if defined(__clang__) diff --git a/runtime/arch/x86_64/entrypoints_init_x86_64.cc b/runtime/arch/x86_64/entrypoints_init_x86_64.cc index d0ab9d5d49..ef1bb5f9a7 100644 --- a/runtime/arch/x86_64/entrypoints_init_x86_64.cc +++ b/runtime/arch/x86_64/entrypoints_init_x86_64.cc @@ -29,6 +29,9 @@ namespace art { extern "C" uint32_t art_quick_assignable_from_code(const mirror::Class* klass, const mirror::Class* ref_class); +// Read barrier entrypoints. +extern "C" mirror::Object* art_quick_read_barrier_slow(mirror::Object*, mirror::Object*, uint32_t); + void InitEntryPoints(InterpreterEntryPoints* ipoints, JniEntryPoints* jpoints, QuickEntryPoints* qpoints) { #if defined(__APPLE__) @@ -145,6 +148,7 @@ void InitEntryPoints(InterpreterEntryPoints* ipoints, JniEntryPoints* jpoints, // Read barrier qpoints->pReadBarrierJni = ReadBarrierJni; + qpoints->pReadBarrierSlow = art_quick_read_barrier_slow; #endif // __APPLE__ }; diff --git a/runtime/arch/x86_64/quick_entrypoints_x86_64.S b/runtime/arch/x86_64/quick_entrypoints_x86_64.S index 0eeb03a526..f4c9488260 100644 --- a/runtime/arch/x86_64/quick_entrypoints_x86_64.S +++ b/runtime/arch/x86_64/quick_entrypoints_x86_64.S @@ -66,7 +66,6 @@ MACRO0(SETUP_SAVE_ALL_CALLEE_SAVE_FRAME) movq %xmm14, 24(%rsp) movq %xmm15, 32(%rsp) // R10 := ArtMethod* for save all callee save frame method. - THIS_LOAD_REQUIRES_READ_BARRIER movq RUNTIME_SAVE_ALL_CALLEE_SAVE_FRAME_OFFSET(%r10), %r10 // Store ArtMethod* to bottom of stack. movq %r10, 0(%rsp) @@ -109,7 +108,6 @@ MACRO0(SETUP_REFS_ONLY_CALLEE_SAVE_FRAME) movq %xmm14, 24(%rsp) movq %xmm15, 32(%rsp) // R10 := ArtMethod* for refs only callee save frame method. - THIS_LOAD_REQUIRES_READ_BARRIER movq RUNTIME_REFS_ONLY_CALLEE_SAVE_FRAME_OFFSET(%r10), %r10 // Store ArtMethod* to bottom of stack. movq %r10, 0(%rsp) @@ -168,7 +166,6 @@ MACRO0(SETUP_REFS_AND_ARGS_CALLEE_SAVE_FRAME) subq MACRO_LITERAL(80 + 4 * 8), %rsp CFI_ADJUST_CFA_OFFSET(80 + 4 * 8) // R10 := ArtMethod* for ref and args callee save frame method. - THIS_LOAD_REQUIRES_READ_BARRIER movq RUNTIME_REFS_AND_ARGS_CALLEE_SAVE_FRAME_OFFSET(%r10), %r10 // Save FPRs. movq %xmm0, 16(%rsp) @@ -920,8 +917,12 @@ DEFINE_FUNCTION art_quick_alloc_object_tlab // Fast path tlab allocation. // RDI: uint32_t type_idx, RSI: ArtMethod* // RDX, RCX, R8, R9: free. RAX: return val. + // TODO: Add read barrier when this function is used. + // Might need a special macro since rsi and edx is 32b/64b mismatched. movl ART_METHOD_DEX_CACHE_TYPES_OFFSET(%rsi), %edx // Load dex cache resolved types array UNPOISON_HEAP_REF edx + // TODO: Add read barrier when this function is used. + // Might need to break down into multiple instructions to get the base address in a register. // Load the class movl MIRROR_OBJECT_ARRAY_DATA_OFFSET(%rdx, %rdi, MIRROR_OBJECT_ARRAY_COMPONENT_SIZE), %edx UNPOISON_HEAP_REF edx @@ -1153,6 +1154,60 @@ DEFINE_FUNCTION art_quick_check_cast END_FUNCTION art_quick_check_cast +// Restore reg's value if reg is not the same as exclude_reg, otherwise just adjust stack. +MACRO2(POP_REG_NE, reg, exclude_reg) + .ifc RAW_VAR(reg), RAW_VAR(exclude_reg) + addq MACRO_LITERAL(8), %rsp + CFI_ADJUST_CFA_OFFSET(-8) + .else + POP RAW_VAR(reg) + .endif +END_MACRO + + /* + * Macro to insert read barrier, used in art_quick_aput_obj and art_quick_alloc_object_tlab. + * obj_reg and dest_reg{32|64} are registers, offset is a defined literal such as + * MIRROR_OBJECT_CLASS_OFFSET. dest_reg needs two versions to handle the mismatch between + * 64b PUSH/POP and 32b argument. + * TODO: When read barrier has a fast path, add heap unpoisoning support for the fast path. + * + * As with art_quick_aput_obj* functions, the 64b versions are in comments. + */ +MACRO4(READ_BARRIER, obj_reg, offset, dest_reg32, dest_reg64) +#ifdef USE_READ_BARRIER + PUSH rax // save registers that might be used + PUSH rdi + PUSH rsi + PUSH rdx + PUSH rcx + SETUP_FP_CALLEE_SAVE_FRAME + // Outgoing argument set up + // movl %edi, %edi // pass ref, no-op for now since parameter ref is unused + // // movq %rdi, %rdi + movl REG_VAR(obj_reg), %esi // pass obj_reg + // movq REG_VAR(obj_reg), %rsi + movl MACRO_LITERAL((RAW_VAR(offset))), %edx // pass offset, double parentheses are necessary + // movq MACRO_LITERAL((RAW_VAR(offset))), %rdx + call SYMBOL(artReadBarrierSlow) // artReadBarrierSlow(ref, obj_reg, offset) + // No need to unpoison return value in rax, artReadBarrierSlow() would do the unpoisoning. + .ifnc RAW_VAR(dest_reg32), eax + // .ifnc RAW_VAR(dest_reg64), rax + movl %eax, REG_VAR(dest_reg32) // save loaded ref in dest_reg + // movq %rax, REG_VAR(dest_reg64) + .endif + RESTORE_FP_CALLEE_SAVE_FRAME + POP_REG_NE rcx, RAW_VAR(dest_reg64) // Restore registers except dest_reg + POP_REG_NE rdx, RAW_VAR(dest_reg64) + POP_REG_NE rsi, RAW_VAR(dest_reg64) + POP_REG_NE rdi, RAW_VAR(dest_reg64) + POP_REG_NE rax, RAW_VAR(dest_reg64) +#else + movl RAW_VAR(offset)(REG_VAR(obj_reg)), REG_VAR(dest_reg32) + // movq RAW_VAR(offset)(REG_VAR(obj_reg)), REG_VAR(dest_reg64) + UNPOISON_HEAP_REF RAW_VAR(dest_reg32) // UNPOISON_HEAP_REF only takes a 32b register +#endif // USE_READ_BARRIER +END_MACRO + /* * Entry from managed code for array put operations of objects where the value being stored * needs to be checked for compatibility. @@ -1197,15 +1252,13 @@ DEFINE_FUNCTION art_quick_aput_obj testl %edx, %edx // store of null // test %rdx, %rdx jz .Ldo_aput_null - movl MIRROR_OBJECT_CLASS_OFFSET(%edi), %ecx -// movq MIRROR_OBJECT_CLASS_OFFSET(%rdi), %rcx - UNPOISON_HEAP_REF ecx - movl MIRROR_CLASS_COMPONENT_TYPE_OFFSET(%ecx), %ecx -// movq MIRROR_CLASS_COMPONENT_TYPE_OFFSET(%rcx), %rcx - UNPOISON_HEAP_REF ecx -#ifdef USE_HEAP_POISONING - movl MIRROR_OBJECT_CLASS_OFFSET(%edx), %eax // rax is free. - UNPOISON_HEAP_REF eax + READ_BARRIER edi, MIRROR_OBJECT_CLASS_OFFSET, ecx, rcx + // READ_BARRIER rdi, MIRROR_OBJECT_CLASS_OFFSET, ecx, rcx + READ_BARRIER ecx, MIRROR_CLASS_COMPONENT_TYPE_OFFSET, ecx, rcx + // READ_BARRIER rcx, MIRROR_CLASS_COMPONENT_TYPE_OFFSET, ecx, rcx +#if defined(USE_HEAP_POISONING) || defined(USE_READ_BARRIER) + READ_BARRIER edx, MIRROR_OBJECT_CLASS_OFFSET, eax, rax // rax is free. + // READ_BARRIER rdx, MIRROR_OBJECT_CLASS_OFFSET, eax, rax cmpl %eax, %ecx // value's type == array's component type - trivial assignability #else cmpl MIRROR_OBJECT_CLASS_OFFSET(%edx), %ecx // value's type == array's component type - trivial assignability @@ -1232,9 +1285,14 @@ DEFINE_FUNCTION art_quick_aput_obj PUSH rdx SETUP_FP_CALLEE_SAVE_FRAME - // "Uncompress" = do nothing, as already zero-extended on load. - movl MIRROR_OBJECT_CLASS_OFFSET(%edx), %esi // Pass arg2 = value's class. - UNPOISON_HEAP_REF esi +#if defined(USE_HEAP_POISONING) || defined(USE_READ_BARRIER) + // The load of MIRROR_OBJECT_CLASS_OFFSET(%edx) is redundant, eax still holds the value. + movl %eax, %esi // Pass arg2 = value's class. + // movq %rax, %rsi +#else + // "Uncompress" = do nothing, as already zero-extended on load. + movl MIRROR_OBJECT_CLASS_OFFSET(%edx), %esi // Pass arg2 = value's class. +#endif movq %rcx, %rdi // Pass arg1 = array's component type. call SYMBOL(artIsAssignableFromCode) // (Class* a, Class* b) @@ -1735,3 +1793,14 @@ DEFINE_FUNCTION art_nested_signal_return call PLT_SYMBOL(longjmp) UNREACHABLE END_FUNCTION art_nested_signal_return + +DEFINE_FUNCTION art_quick_read_barrier_slow + SETUP_FP_CALLEE_SAVE_FRAME + subq LITERAL(8), %rsp // Alignment padding. + CFI_ADJUST_CFA_OFFSET(8) + call SYMBOL(artReadBarrierSlow) // artReadBarrierSlow(ref, obj, offset) + addq LITERAL(8), %rsp + CFI_ADJUST_CFA_OFFSET(-8) + RESTORE_FP_CALLEE_SAVE_FRAME + ret +END_FUNCTION art_quick_read_barrier_slow diff --git a/runtime/asm_support.h b/runtime/asm_support.h index f4f8eaf759..350a0d4c15 100644 --- a/runtime/asm_support.h +++ b/runtime/asm_support.h @@ -109,7 +109,7 @@ ADD_TEST_EQ(THREAD_SELF_OFFSET, art::Thread::SelfOffset<__SIZEOF_POINTER__>().Int32Value()) // Offset of field Thread::tlsPtr_.thread_local_pos. -#define THREAD_LOCAL_POS_OFFSET (THREAD_CARD_TABLE_OFFSET + 150 * __SIZEOF_POINTER__) +#define THREAD_LOCAL_POS_OFFSET (THREAD_CARD_TABLE_OFFSET + 151 * __SIZEOF_POINTER__) ADD_TEST_EQ(THREAD_LOCAL_POS_OFFSET, art::Thread::ThreadLocalPosOffset<__SIZEOF_POINTER__>().Int32Value()) // Offset of field Thread::tlsPtr_.thread_local_end. diff --git a/runtime/entrypoints/quick/quick_entrypoints.h b/runtime/entrypoints/quick/quick_entrypoints.h index cef2510451..3d3f7a1bdb 100644 --- a/runtime/entrypoints/quick/quick_entrypoints.h +++ b/runtime/entrypoints/quick/quick_entrypoints.h @@ -20,6 +20,7 @@ #include <jni.h> #include "base/macros.h" +#include "base/mutex.h" #include "offsets.h" #define QUICK_ENTRYPOINT_OFFSET(ptr_size, x) \ @@ -71,6 +72,16 @@ extern void ReadBarrierJni(mirror::CompressedReference<mirror::Object>* handle_o Thread* self) NO_THREAD_SAFETY_ANALYSIS HOT_ATTR; +// Read barrier entrypoints. +// Compilers for ARM, ARM64, MIPS, MIPS64 can insert a call to this function directly. +// For x86 and x86_64, compilers need a wrapper assembly function, to handle mismatch in ABI. +// This is the read barrier slow path for instance and static fields and reference-type arrays. +// TODO: Currently the read barrier does not have a fast path for compilers to directly generate. +// Ideally the slow path should only take one parameter "ref". +extern "C" mirror::Object* artReadBarrierSlow(mirror::Object* ref, mirror::Object* obj, + uint32_t offset) + SHARED_REQUIRES(Locks::mutator_lock_) HOT_ATTR; + } // namespace art #endif // ART_RUNTIME_ENTRYPOINTS_QUICK_QUICK_ENTRYPOINTS_H_ diff --git a/runtime/entrypoints/quick/quick_entrypoints_list.h b/runtime/entrypoints/quick/quick_entrypoints_list.h index 60bbf4ac82..73d8ae76ae 100644 --- a/runtime/entrypoints/quick/quick_entrypoints_list.h +++ b/runtime/entrypoints/quick/quick_entrypoints_list.h @@ -145,7 +145,8 @@ V(NewStringFromStringBuffer, void) \ V(NewStringFromStringBuilder, void) \ \ - V(ReadBarrierJni, void, mirror::CompressedReference<mirror::Object>*, Thread*) + V(ReadBarrierJni, void, mirror::CompressedReference<mirror::Object>*, Thread*) \ + V(ReadBarrierSlow, mirror::Object*, mirror::Object*, mirror::Object*, uint32_t) #endif // ART_RUNTIME_ENTRYPOINTS_QUICK_QUICK_ENTRYPOINTS_LIST_H_ #undef ART_RUNTIME_ENTRYPOINTS_QUICK_QUICK_ENTRYPOINTS_LIST_H_ // #define is only for lint. diff --git a/runtime/entrypoints/quick/quick_field_entrypoints.cc b/runtime/entrypoints/quick/quick_field_entrypoints.cc index 25a943a82a..0a1d80648d 100644 --- a/runtime/entrypoints/quick/quick_field_entrypoints.cc +++ b/runtime/entrypoints/quick/quick_field_entrypoints.cc @@ -557,4 +557,16 @@ extern "C" int artSetObjInstanceFromCode(uint32_t field_idx, mirror::Object* obj return -1; // failure } +// TODO: Currently the read barrier does not have a fast path. Ideally the slow path should only +// take one parameter "ref", which is generated by the fast path. +extern "C" mirror::Object* artReadBarrierSlow(mirror::Object* ref ATTRIBUTE_UNUSED, + mirror::Object* obj, uint32_t offset) { + DCHECK(kUseReadBarrier); + uint8_t* raw_addr = reinterpret_cast<uint8_t*>(obj) + offset; + mirror::HeapReference<mirror::Object>* ref_addr = + reinterpret_cast<mirror::HeapReference<mirror::Object>*>(raw_addr); + return ReadBarrier::Barrier<mirror::Object, kWithReadBarrier, true>(obj, MemberOffset(offset), + ref_addr); +} + } // namespace art diff --git a/runtime/entrypoints_order_test.cc b/runtime/entrypoints_order_test.cc index c05c93555c..f7a3cd53cd 100644 --- a/runtime/entrypoints_order_test.cc +++ b/runtime/entrypoints_order_test.cc @@ -311,8 +311,9 @@ class EntrypointsOrderTest : public CommonRuntimeTest { sizeof(void*)); EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pNewStringFromStringBuilder, pReadBarrierJni, sizeof(void*)); + EXPECT_OFFSET_DIFFNP(QuickEntryPoints, pReadBarrierJni, pReadBarrierSlow, sizeof(void*)); - CHECKED(OFFSETOF_MEMBER(QuickEntryPoints, pReadBarrierJni) + CHECKED(OFFSETOF_MEMBER(QuickEntryPoints, pReadBarrierSlow) + sizeof(void*) == sizeof(QuickEntryPoints), QuickEntryPoints_all); } }; diff --git a/runtime/oat.h b/runtime/oat.h index ee2f3f60f3..29dd76ce5e 100644 --- a/runtime/oat.h +++ b/runtime/oat.h @@ -32,7 +32,7 @@ class InstructionSetFeatures; class PACKED(4) OatHeader { public: static constexpr uint8_t kOatMagic[] = { 'o', 'a', 't', '\n' }; - static constexpr uint8_t kOatVersion[] = { '0', '6', '7', '\0' }; + static constexpr uint8_t kOatVersion[] = { '0', '6', '8', '\0' }; static constexpr const char* kImageLocationKey = "image-location"; static constexpr const char* kDex2OatCmdLineKey = "dex2oat-cmdline"; diff --git a/runtime/read_barrier_c.h b/runtime/read_barrier_c.h index 4f408dd5c1..710c21f03e 100644 --- a/runtime/read_barrier_c.h +++ b/runtime/read_barrier_c.h @@ -47,9 +47,4 @@ #error "Only one of Baker or Brooks can be enabled at a time." #endif -// A placeholder marker to indicate places to add read barriers in the -// assembly code. This is a development time aid and to be removed -// after read barriers are added. -#define THIS_LOAD_REQUIRES_READ_BARRIER - #endif // ART_RUNTIME_READ_BARRIER_C_H_ diff --git a/runtime/thread.cc b/runtime/thread.cc index 6949b0bd34..2b977af51c 100644 --- a/runtime/thread.cc +++ b/runtime/thread.cc @@ -2304,6 +2304,7 @@ void Thread::DumpThreadOffset(std::ostream& os, uint32_t offset) { QUICK_ENTRY_POINT_INFO(pNewStringFromStringBuffer) QUICK_ENTRY_POINT_INFO(pNewStringFromStringBuilder) QUICK_ENTRY_POINT_INFO(pReadBarrierJni) + QUICK_ENTRY_POINT_INFO(pReadBarrierSlow) #undef QUICK_ENTRY_POINT_INFO os << offset; |