diff options
| author | 2021-12-08 11:27:02 +0000 | |
|---|---|---|
| committer | 2022-04-22 11:38:39 +0000 | |
| commit | 640267052653a8c177b6f325675e7217b83d7d3c (patch) | |
| tree | ad9a47af4317bc11ede0455a709a29ae468e8e4b | |
| parent | 9d31daa0b3f8e748a19555870932bace11f2b199 (diff) | |
Update how deoptimization from runtime methods is supported
We need to support deoptimization when returning from runtime methods
- To support setting break points or other debug related features for
long running loops we should be able to deopt at suspend points.
- To deoptimize the full stack when we have runtime functions like
art_quick_to_interpreter_bridge on the stack.
- To deoptimize the optimized code after redefining the class. Once
we redefine the class the offsets of the fields could change
because the field offsets are grouped together by type when
allocating. This would invalidate all optimized code and
deoptimization cannot be pushed to a later point.
Earlier this was achieved by instrumenting the stack and updating
the return pcs of the runtime functions to instrumentation exit
stub, so we could deoptimize the method. However runtime methods
can be called in different scenarios which complicated the logic
and was inaccurate in some scenarios.
This CL changes the approach to check if a deoptimization is
required before returning from the slow path of runtime functions
that can suspend. Checking for deopts only at suspend points is not
sufficient because of the class redefinition support as mentioned
earlier.
Bug: 204766614,208587492
Test: art/test.py
Change-Id: Ia655d957bae4ff891f8f32a0f3110c80e7a80bfd
25 files changed, 643 insertions, 344 deletions
diff --git a/compiler/optimizing/code_generator.h b/compiler/optimizing/code_generator.h index d81a7b5382..b09219a2ed 100644 --- a/compiler/optimizing/code_generator.h +++ b/compiler/optimizing/code_generator.h @@ -33,6 +33,7 @@ #include "graph_visualizer.h" #include "locations.h" #include "nodes.h" +#include "oat_quick_method_header.h" #include "optimizing_compiler_stats.h" #include "read_barrier_option.h" #include "stack.h" diff --git a/compiler/utils/assembler_thumb_test_expected.cc.inc b/compiler/utils/assembler_thumb_test_expected.cc.inc index b6c6025e41..1775014e9e 100644 --- a/compiler/utils/assembler_thumb_test_expected.cc.inc +++ b/compiler/utils/assembler_thumb_test_expected.cc.inc @@ -76,7 +76,7 @@ const char* const VixlJniHelpersResults = { " f2: bb f1 00 0f cmp.w r11, #0\n" " f6: 18 bf it ne\n" " f8: e3 46 movne r11, r12\n" - " fa: d9 f8 94 c0 ldr.w r12, [r9, #148]\n" + " fa: d9 f8 9c c0 ldr.w r12, [r9, #156]\n" " fe: bc f1 00 0f cmp.w r12, #0\n" " 102: 71 d1 bne 0x1e8 @ imm = #226\n" " 104: cd f8 ff c7 str.w r12, [sp, #2047]\n" @@ -152,8 +152,8 @@ const char* const VixlJniHelpersResults = { " 218: bd e8 e0 4d pop.w {r5, r6, r7, r8, r10, r11, lr}\n" " 21c: d9 f8 24 80 ldr.w r8, [r9, #36]\n" " 220: 70 47 bx lr\n" - " 222: d9 f8 94 00 ldr.w r0, [r9, #148]\n" - " 226: d9 f8 c8 e2 ldr.w lr, [r9, #712]\n" + " 222: d9 f8 9c 00 ldr.w r0, [r9, #156]\n" + " 226: d9 f8 d0 e2 ldr.w lr, [r9, #720]\n" " 22a: f0 47 blx lr\n" }; diff --git a/runtime/arch/arm/asm_support_arm.S b/runtime/arch/arm/asm_support_arm.S index 23d82bac38..68afc24091 100644 --- a/runtime/arch/arm/asm_support_arm.S +++ b/runtime/arch/arm/asm_support_arm.S @@ -315,10 +315,6 @@ DELIVER_PENDING_EXCEPTION .endm -.macro RETURN_OR_DELIVER_PENDING_EXCEPTION_R1 - RETURN_OR_DELIVER_PENDING_EXCEPTION_REG r1 -.endm - .macro RETURN_OR_DELIVER_PENDING_EXCEPTION ldr ip, [rSELF, #THREAD_EXCEPTION_OFFSET] @ Get exception field. cmp ip, #0 diff --git a/runtime/arch/arm/asm_support_arm.h b/runtime/arch/arm/asm_support_arm.h index aff055e611..5e3f85484a 100644 --- a/runtime/arch/arm/asm_support_arm.h +++ b/runtime/arch/arm/asm_support_arm.h @@ -25,6 +25,8 @@ #define FRAME_SIZE_SAVE_EVERYTHING 192 #define FRAME_SIZE_SAVE_EVERYTHING_FOR_CLINIT FRAME_SIZE_SAVE_EVERYTHING #define FRAME_SIZE_SAVE_EVERYTHING_FOR_SUSPEND_CHECK FRAME_SIZE_SAVE_EVERYTHING +#define SAVE_EVERYTHING_FRAME_R0_OFFSET \ + (FRAME_SIZE_SAVE_EVERYTHING - CALLEE_SAVE_EVERYTHING_NUM_CORE_SPILLS * POINTER_SIZE) // The offset from the art_quick_read_barrier_mark_introspection (used for field // loads with 32-bit LDR) to the entrypoint for field loads with 16-bit LDR, diff --git a/runtime/arch/arm/quick_entrypoints_arm.S b/runtime/arch/arm/quick_entrypoints_arm.S index d6f129be50..5a9757232f 100644 --- a/runtime/arch/arm/quick_entrypoints_arm.S +++ b/runtime/arch/arm/quick_entrypoints_arm.S @@ -134,16 +134,50 @@ .cfi_adjust_cfa_offset -52 .endm -.macro RETURN_IF_RESULT_IS_ZERO - cbnz r0, 1f @ result non-zero branch over - bx lr @ return +.macro RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION + ldr r1, [rSELF, # THREAD_EXCEPTION_OFFSET] // Get exception field. + cbnz r1, 1f + DEOPT_OR_RETURN r1 // Check if deopt is required 1: + DELIVER_PENDING_EXCEPTION .endm -.macro RETURN_IF_RESULT_IS_NON_ZERO - cbz r0, 1f @ result zero branch over - bx lr @ return -1: +.macro DEOPT_OR_RETURN temp, is_ref = 0 + ldr \temp, [rSELF, #THREAD_DEOPT_CHECK_REQUIRED_OFFSET] + cbnz \temp, 2f + bx lr +2: + SETUP_SAVE_EVERYTHING_FRAME \temp + mov r2, \is_ref // pass if result is a reference + mov r1, r0 // pass the result + mov r0, rSELF // Thread::Current + bl artDeoptimizeIfNeeded + .cfi_remember_state + RESTORE_SAVE_EVERYTHING_FRAME + REFRESH_MARKING_REGISTER + bx lr + CFI_RESTORE_STATE_AND_DEF_CFA sp, FRAME_SIZE_SAVE_EVERYTHING +.endm + +.macro DEOPT_OR_RESTORE_SAVE_EVERYTHING_FRAME_AND_RETURN_R0 temp, is_ref + ldr \temp, [rSELF, #THREAD_DEOPT_CHECK_REQUIRED_OFFSET] + cbnz \temp, 2f + .cfi_remember_state + RESTORE_SAVE_EVERYTHING_FRAME_KEEP_R0 + REFRESH_MARKING_REGISTER + bx lr + CFI_RESTORE_STATE_AND_DEF_CFA sp, FRAME_SIZE_SAVE_EVERYTHING +2: + str r0, [sp, SAVE_EVERYTHING_FRAME_R0_OFFSET] // update result in the frame + mov r2, \is_ref // pass if result is a reference + mov r1, r0 // pass the result + mov r0, rSELF // Thread::Current + bl artDeoptimizeIfNeeded + .cfi_remember_state + RESTORE_SAVE_EVERYTHING_FRAME + REFRESH_MARKING_REGISTER + bx lr + CFI_RESTORE_STATE_AND_DEF_CFA sp, FRAME_SIZE_SAVE_EVERYTHING .endm .macro NO_ARG_RUNTIME_EXCEPTION c_name, cxx_name @@ -183,12 +217,16 @@ END \c_name .endm .macro RETURN_IF_RESULT_IS_ZERO_OR_DELIVER - RETURN_IF_RESULT_IS_ZERO + cbnz r0, 1f @ result non-zero branch over + DEOPT_OR_RETURN r1 +1: DELIVER_PENDING_EXCEPTION .endm -.macro RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER - RETURN_IF_RESULT_IS_NON_ZERO +.macro RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER + cbz r0, 1f @ result zero branch over + DEOPT_OR_RETURN r1, /*is_ref=*/1 +1: DELIVER_PENDING_EXCEPTION .endm @@ -517,8 +555,7 @@ ENTRY art_quick_lock_object_no_inline bl artLockObjectFromCode @ (Object* obj, Thread*) RESTORE_SAVE_REFS_ONLY_FRAME REFRESH_MARKING_REGISTER - RETURN_IF_RESULT_IS_ZERO - DELIVER_PENDING_EXCEPTION + RETURN_IF_RESULT_IS_ZERO_OR_DELIVER END art_quick_lock_object_no_inline /* @@ -548,8 +585,7 @@ ENTRY art_quick_unlock_object_no_inline bl artUnlockObjectFromCode @ (Object* obj, Thread*) RESTORE_SAVE_REFS_ONLY_FRAME REFRESH_MARKING_REGISTER - RETURN_IF_RESULT_IS_ZERO - DELIVER_PENDING_EXCEPTION + RETURN_IF_RESULT_IS_ZERO_OR_DELIVER END art_quick_unlock_object_no_inline /* @@ -782,11 +818,8 @@ ENTRY \name mov r1, rSELF @ pass Thread::Current bl \entrypoint @ (uint32_t index, Thread*) cbz r0, 1f @ If result is null, deliver the OOME. - .cfi_remember_state - RESTORE_SAVE_EVERYTHING_FRAME_KEEP_R0 - REFRESH_MARKING_REGISTER - bx lr - CFI_RESTORE_STATE_AND_DEF_CFA sp, FRAME_SIZE_SAVE_EVERYTHING + str r0, [sp, #136] @ store result in the frame + DEOPT_OR_RESTORE_SAVE_EVERYTHING_FRAME_AND_RETURN_R0 r1, /* is_ref= */ 1 1: DELIVER_PENDING_EXCEPTION_FRAME_READY END \name @@ -809,12 +842,12 @@ ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_resolve_string, artResolveStringFromC /* * Called by managed code to resolve a static field and load a non-wide value. */ -ONE_ARG_REF_DOWNCALL art_quick_get_byte_static, artGetByteStaticFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_R1 -ONE_ARG_REF_DOWNCALL art_quick_get_boolean_static, artGetBooleanStaticFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_R1 -ONE_ARG_REF_DOWNCALL art_quick_get_short_static, artGetShortStaticFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_R1 -ONE_ARG_REF_DOWNCALL art_quick_get_char_static, artGetCharStaticFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_R1 -ONE_ARG_REF_DOWNCALL art_quick_get32_static, artGet32StaticFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_R1 -ONE_ARG_REF_DOWNCALL art_quick_get_obj_static, artGetObjStaticFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_R1 +ONE_ARG_REF_DOWNCALL art_quick_get_byte_static, artGetByteStaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION +ONE_ARG_REF_DOWNCALL art_quick_get_boolean_static, artGetBooleanStaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION +ONE_ARG_REF_DOWNCALL art_quick_get_short_static, artGetShortStaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION +ONE_ARG_REF_DOWNCALL art_quick_get_char_static, artGetCharStaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION +ONE_ARG_REF_DOWNCALL art_quick_get32_static, artGet32StaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION +ONE_ARG_REF_DOWNCALL art_quick_get_obj_static, artGetObjStaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION /* * Called by managed code to resolve a static field and load a 64-bit primitive value. */ @@ -827,7 +860,7 @@ ENTRY art_quick_get64_static RESTORE_SAVE_REFS_ONLY_FRAME REFRESH_MARKING_REGISTER cbnz r2, 1f @ success if no exception pending - bx lr @ return on success + DEOPT_OR_RETURN r2 @ check if deopt is required or return 1: DELIVER_PENDING_EXCEPTION END art_quick_get64_static @@ -835,12 +868,12 @@ END art_quick_get64_static /* * Called by managed code to resolve an instance field and load a non-wide value. */ -TWO_ARG_REF_DOWNCALL art_quick_get_byte_instance, artGetByteInstanceFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_R1 -TWO_ARG_REF_DOWNCALL art_quick_get_boolean_instance, artGetBooleanInstanceFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_R1 -TWO_ARG_REF_DOWNCALL art_quick_get_short_instance, artGetShortInstanceFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_R1 -TWO_ARG_REF_DOWNCALL art_quick_get_char_instance, artGetCharInstanceFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_R1 -TWO_ARG_REF_DOWNCALL art_quick_get32_instance, artGet32InstanceFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_R1 -TWO_ARG_REF_DOWNCALL art_quick_get_obj_instance, artGetObjInstanceFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_R1 +TWO_ARG_REF_DOWNCALL art_quick_get_byte_instance, artGetByteInstanceFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION +TWO_ARG_REF_DOWNCALL art_quick_get_boolean_instance, artGetBooleanInstanceFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION +TWO_ARG_REF_DOWNCALL art_quick_get_short_instance, artGetShortInstanceFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION +TWO_ARG_REF_DOWNCALL art_quick_get_char_instance, artGetCharInstanceFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION +TWO_ARG_REF_DOWNCALL art_quick_get32_instance, artGet32InstanceFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION +TWO_ARG_REF_DOWNCALL art_quick_get_obj_instance, artGetObjInstanceFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION /* * Called by managed code to resolve an instance field and load a 64-bit primitive value. */ @@ -853,7 +886,7 @@ ENTRY art_quick_get64_instance RESTORE_SAVE_REFS_ONLY_FRAME REFRESH_MARKING_REGISTER cbnz r2, 1f @ success if no exception pending - bx lr @ return on success + DEOPT_OR_RETURN r2 @ check if deopt is required or return 1: DELIVER_PENDING_EXCEPTION END art_quick_get64_instance @@ -888,8 +921,7 @@ ENTRY art_quick_set64_instance .cfi_adjust_cfa_offset -16 RESTORE_SAVE_REFS_ONLY_FRAME @ TODO: we can clearly save an add here REFRESH_MARKING_REGISTER - RETURN_IF_RESULT_IS_ZERO - DELIVER_PENDING_EXCEPTION + RETURN_IF_RESULT_IS_ZERO_OR_DELIVER END art_quick_set64_instance .extern artSet64StaticFromCompiledCode @@ -903,8 +935,7 @@ ENTRY art_quick_set64_static .cfi_adjust_cfa_offset -16 RESTORE_SAVE_REFS_ONLY_FRAME @ TODO: we can clearly save an add here REFRESH_MARKING_REGISTER - RETURN_IF_RESULT_IS_ZERO - DELIVER_PENDING_EXCEPTION + RETURN_IF_RESULT_IS_ZERO_OR_DELIVER END art_quick_set64_static // Generate the allocation entrypoints for each allocator. @@ -1037,7 +1068,7 @@ ENTRY \c_name bl \cxx_name @ (mirror::Class* cls, Thread*) RESTORE_SAVE_REFS_ONLY_FRAME REFRESH_MARKING_REGISTER - RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER + RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER END \c_name .endm @@ -1122,7 +1153,7 @@ ENTRY \name bl \entrypoint // (mirror::Class* klass, Thread*) RESTORE_SAVE_REFS_ONLY_FRAME REFRESH_MARKING_REGISTER - RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER + RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER END \name .endm @@ -1195,7 +1226,7 @@ ENTRY \name bl \entrypoint RESTORE_SAVE_REFS_ONLY_FRAME REFRESH_MARKING_REGISTER - RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER + RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER END \name .endm @@ -1883,7 +1914,7 @@ ENTRY art_quick_string_builder_append bl artStringBuilderAppend @ (uint32_t, const unit32_t*, Thread*) RESTORE_SAVE_REFS_ONLY_FRAME REFRESH_MARKING_REGISTER - RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER + RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER END art_quick_string_builder_append /* diff --git a/runtime/arch/arm64/asm_support_arm64.h b/runtime/arch/arm64/asm_support_arm64.h index 887ee0259c..5a285a4d9d 100644 --- a/runtime/arch/arm64/asm_support_arm64.h +++ b/runtime/arch/arm64/asm_support_arm64.h @@ -19,6 +19,7 @@ #include "asm_support.h" +// TODO(mythria): Change these to use constants from callee_save_frame_arm64.h #define CALLEE_SAVES_SIZE (12 * 8 + 8 * 8) // +8 for the ArtMethod, +8 for alignment. #define FRAME_SIZE_SAVE_ALL_CALLEE_SAVES (CALLEE_SAVES_SIZE + 16) @@ -27,6 +28,8 @@ #define FRAME_SIZE_SAVE_EVERYTHING 512 #define FRAME_SIZE_SAVE_EVERYTHING_FOR_CLINIT FRAME_SIZE_SAVE_EVERYTHING #define FRAME_SIZE_SAVE_EVERYTHING_FOR_SUSPEND_CHECK FRAME_SIZE_SAVE_EVERYTHING +#define SAVE_EVERYTHING_FRAME_X0_OFFSET \ + (FRAME_SIZE_SAVE_EVERYTHING - CALLEE_SAVE_EVERYTHING_NUM_CORE_SPILLS * POINTER_SIZE) // The offset from art_quick_read_barrier_mark_introspection to the array switch cases, // i.e. art_quick_read_barrier_mark_introspection_arrays. diff --git a/runtime/arch/arm64/quick_entrypoints_arm64.S b/runtime/arch/arm64/quick_entrypoints_arm64.S index d8c91e11b9..5f74f7ddf7 100644 --- a/runtime/arch/arm64/quick_entrypoints_arm64.S +++ b/runtime/arch/arm64/quick_entrypoints_arm64.S @@ -194,26 +194,56 @@ RESTORE_SAVE_EVERYTHING_FRAME_KEEP_X0 .endm -.macro RETURN_IF_RESULT_IS_ZERO - cbnz x0, 1f // result non-zero branch over - ret // return -1: +.macro RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION + ldr x1, [xSELF, # THREAD_EXCEPTION_OFFSET] // Get exception field. + cbnz x1, 1f + DEOPT_OR_RETURN x1 // Check if deopt is required +1: // deliver exception on current thread + DELIVER_PENDING_EXCEPTION .endm -.macro RETURN_IF_RESULT_IS_NON_ZERO - cbz x0, 1f // result zero branch over - ret // return -1: +.macro DEOPT_OR_RETURN temp, is_ref = 0 + ldr \temp, [xSELF, #THREAD_DEOPT_CHECK_REQUIRED_OFFSET] + cbnz \temp, 2f + ret +2: + SETUP_SAVE_EVERYTHING_FRAME + mov x2, \is_ref // pass if result is a reference + mov x1, x0 // pass the result + mov x0, xSELF // Thread::Current + bl artDeoptimizeIfNeeded + .cfi_remember_state + RESTORE_SAVE_EVERYTHING_FRAME + REFRESH_MARKING_REGISTER + ret + CFI_RESTORE_STATE_AND_DEF_CFA sp, FRAME_SIZE_SAVE_EVERYTHING .endm -// Same as above with x1. This is helpful in stubs that want to avoid clobbering another register. -.macro RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 - RETURN_OR_DELIVER_PENDING_EXCEPTION_REG x1 +.macro DEOPT_OR_RESTORE_SAVE_EVERYTHING_FRAME_AND_RETURN_X0 temp, is_ref + ldr \temp, [xSELF, #THREAD_DEOPT_CHECK_REQUIRED_OFFSET] + cbnz \temp, 2f + .cfi_remember_state + RESTORE_SAVE_EVERYTHING_FRAME_KEEP_X0 + REFRESH_MARKING_REGISTER + ret + CFI_RESTORE_STATE_AND_DEF_CFA sp, FRAME_SIZE_SAVE_EVERYTHING +2: + str x0, [sp, #SAVE_EVERYTHING_FRAME_X0_OFFSET] // update result in the frame + mov x2, \is_ref // pass if result is a reference + mov x1, x0 // pass the result + mov x0, xSELF // Thread::Current + bl artDeoptimizeIfNeeded + .cfi_remember_state + RESTORE_SAVE_EVERYTHING_FRAME + REFRESH_MARKING_REGISTER + ret + CFI_RESTORE_STATE_AND_DEF_CFA sp, FRAME_SIZE_SAVE_EVERYTHING .endm + .macro RETURN_IF_W0_IS_ZERO_OR_DELIVER cbnz w0, 1f // result non-zero branch over - ret // return + DEOPT_OR_RETURN x1 1: DELIVER_PENDING_EXCEPTION .endm @@ -1214,11 +1244,7 @@ ENTRY \name mov x1, xSELF // pass Thread::Current bl \entrypoint // (int32_t index, Thread* self) cbz w0, 1f // If result is null, deliver the OOME. - .cfi_remember_state - RESTORE_SAVE_EVERYTHING_FRAME_KEEP_X0 - REFRESH_MARKING_REGISTER - ret // return - CFI_RESTORE_STATE_AND_DEF_CFA sp, FRAME_SIZE_SAVE_EVERYTHING + DEOPT_OR_RESTORE_SAVE_EVERYTHING_FRAME_AND_RETURN_X0 x1, /* is_ref= */ 1 1: DELIVER_PENDING_EXCEPTION_FRAME_READY END \name @@ -1228,13 +1254,14 @@ END \name ONE_ARG_SAVE_EVERYTHING_DOWNCALL \name, \entrypoint, RUNTIME_SAVE_EVERYTHING_FOR_CLINIT_METHOD_OFFSET .endm -.macro RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER - cbz w0, 1f // result zero branch over - ret // return +.macro RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER + cbz w0, 1f // result zero branch over + DEOPT_OR_RETURN x1, /*is_ref=*/1 // check for deopt or return 1: DELIVER_PENDING_EXCEPTION .endm + /* * Entry from managed code that calls artHandleFillArrayDataFromCode and delivers exception on * failure. @@ -1256,21 +1283,21 @@ ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_resolve_string, artResolveStringFromC // Note: Functions `art{Get,Set}<Kind>{Static,Instance}FromCompiledCode` are // defined with a macro in runtime/entrypoints/quick/quick_field_entrypoints.cc. -ONE_ARG_REF_DOWNCALL art_quick_get_boolean_static, artGetBooleanStaticFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 -ONE_ARG_REF_DOWNCALL art_quick_get_byte_static, artGetByteStaticFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 -ONE_ARG_REF_DOWNCALL art_quick_get_char_static, artGetCharStaticFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 -ONE_ARG_REF_DOWNCALL art_quick_get_short_static, artGetShortStaticFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 -ONE_ARG_REF_DOWNCALL art_quick_get32_static, artGet32StaticFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 -ONE_ARG_REF_DOWNCALL art_quick_get64_static, artGet64StaticFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 -ONE_ARG_REF_DOWNCALL art_quick_get_obj_static, artGetObjStaticFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 - -TWO_ARG_REF_DOWNCALL art_quick_get_boolean_instance, artGetBooleanInstanceFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 -TWO_ARG_REF_DOWNCALL art_quick_get_byte_instance, artGetByteInstanceFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 -TWO_ARG_REF_DOWNCALL art_quick_get_char_instance, artGetCharInstanceFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 -TWO_ARG_REF_DOWNCALL art_quick_get_short_instance, artGetShortInstanceFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 -TWO_ARG_REF_DOWNCALL art_quick_get32_instance, artGet32InstanceFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 -TWO_ARG_REF_DOWNCALL art_quick_get64_instance, artGet64InstanceFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 -TWO_ARG_REF_DOWNCALL art_quick_get_obj_instance, artGetObjInstanceFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_X1 +ONE_ARG_REF_DOWNCALL art_quick_get_boolean_static, artGetBooleanStaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION +ONE_ARG_REF_DOWNCALL art_quick_get_byte_static, artGetByteStaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION +ONE_ARG_REF_DOWNCALL art_quick_get_char_static, artGetCharStaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION +ONE_ARG_REF_DOWNCALL art_quick_get_short_static, artGetShortStaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION +ONE_ARG_REF_DOWNCALL art_quick_get32_static, artGet32StaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION +ONE_ARG_REF_DOWNCALL art_quick_get64_static, artGet64StaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION +ONE_ARG_REF_DOWNCALL art_quick_get_obj_static, artGetObjStaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION + +TWO_ARG_REF_DOWNCALL art_quick_get_boolean_instance, artGetBooleanInstanceFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION +TWO_ARG_REF_DOWNCALL art_quick_get_byte_instance, artGetByteInstanceFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION +TWO_ARG_REF_DOWNCALL art_quick_get_char_instance, artGetCharInstanceFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION +TWO_ARG_REF_DOWNCALL art_quick_get_short_instance, artGetShortInstanceFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION +TWO_ARG_REF_DOWNCALL art_quick_get32_instance, artGet32InstanceFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION +TWO_ARG_REF_DOWNCALL art_quick_get64_instance, artGet64InstanceFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION +TWO_ARG_REF_DOWNCALL art_quick_get_obj_instance, artGetObjInstanceFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION TWO_ARG_REF_DOWNCALL art_quick_set8_static, artSet8StaticFromCompiledCode, RETURN_IF_W0_IS_ZERO_OR_DELIVER TWO_ARG_REF_DOWNCALL art_quick_set16_static, artSet16StaticFromCompiledCode, RETURN_IF_W0_IS_ZERO_OR_DELIVER @@ -1410,7 +1437,7 @@ ENTRY \c_name bl \cxx_name RESTORE_SAVE_REFS_ONLY_FRAME REFRESH_MARKING_REGISTER - RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER + RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER END \c_name .endm @@ -1465,7 +1492,7 @@ ENTRY \name bl \entrypoint // (mirror::Class*, Thread*) RESTORE_SAVE_REFS_ONLY_FRAME REFRESH_MARKING_REGISTER - RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER + RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER END \name .endm @@ -1539,7 +1566,7 @@ ENTRY \name bl \entrypoint RESTORE_SAVE_REFS_ONLY_FRAME REFRESH_MARKING_REGISTER - RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER + RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER END \name .endm @@ -2102,7 +2129,7 @@ ENTRY art_quick_string_builder_append bl artStringBuilderAppend // (uint32_t, const unit32_t*, Thread*) RESTORE_SAVE_REFS_ONLY_FRAME REFRESH_MARKING_REGISTER - RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER + RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER END art_quick_string_builder_append /* diff --git a/runtime/arch/quick_alloc_entrypoints.S b/runtime/arch/quick_alloc_entrypoints.S index 32888edf7b..5d4b24bc6b 100644 --- a/runtime/arch/quick_alloc_entrypoints.S +++ b/runtime/arch/quick_alloc_entrypoints.S @@ -16,27 +16,27 @@ .macro GENERATE_ALLOC_ENTRYPOINTS c_suffix, cxx_suffix // Called by managed code to allocate an object of a resolved class. -ONE_ARG_DOWNCALL art_quick_alloc_object_resolved\c_suffix, artAllocObjectFromCodeResolved\cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER +ONE_ARG_DOWNCALL art_quick_alloc_object_resolved\c_suffix, artAllocObjectFromCodeResolved\cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER // Called by managed code to allocate an object of an initialized class. -ONE_ARG_DOWNCALL art_quick_alloc_object_initialized\c_suffix, artAllocObjectFromCodeInitialized\cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER +ONE_ARG_DOWNCALL art_quick_alloc_object_initialized\c_suffix, artAllocObjectFromCodeInitialized\cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER // Called by managed code to allocate an object when the caller doesn't know whether it has access // to the created type. -ONE_ARG_DOWNCALL art_quick_alloc_object_with_checks\c_suffix, artAllocObjectFromCodeWithChecks\cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER +ONE_ARG_DOWNCALL art_quick_alloc_object_with_checks\c_suffix, artAllocObjectFromCodeWithChecks\cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER // Called by managed code to allocate a string if it could not be removed by any optimizations -ONE_ARG_DOWNCALL art_quick_alloc_string_object\c_suffix, artAllocStringObject\cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER +ONE_ARG_DOWNCALL art_quick_alloc_string_object\c_suffix, artAllocStringObject\cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER // Called by managed code to allocate an array of a resolve class. -TWO_ARG_DOWNCALL art_quick_alloc_array_resolved\c_suffix, artAllocArrayFromCodeResolved\cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER +TWO_ARG_DOWNCALL art_quick_alloc_array_resolved\c_suffix, artAllocArrayFromCodeResolved\cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER // Called by managed code to allocate a string from bytes -FOUR_ARG_DOWNCALL art_quick_alloc_string_from_bytes\c_suffix, artAllocStringFromBytesFromCode\cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER +FOUR_ARG_DOWNCALL art_quick_alloc_string_from_bytes\c_suffix, artAllocStringFromBytesFromCode\cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER // Called by managed code to allocate a string from chars -THREE_ARG_DOWNCALL art_quick_alloc_string_from_chars\c_suffix, artAllocStringFromCharsFromCode\cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER +THREE_ARG_DOWNCALL art_quick_alloc_string_from_chars\c_suffix, artAllocStringFromCharsFromCode\cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER // Called by managed code to allocate a string from string -ONE_ARG_DOWNCALL art_quick_alloc_string_from_string\c_suffix, artAllocStringFromStringFromCode\cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER +ONE_ARG_DOWNCALL art_quick_alloc_string_from_string\c_suffix, artAllocStringFromStringFromCode\cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER -TWO_ARG_DOWNCALL art_quick_alloc_array_resolved8\c_suffix, artAllocArrayFromCodeResolved\cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER -TWO_ARG_DOWNCALL art_quick_alloc_array_resolved16\c_suffix, artAllocArrayFromCodeResolved\cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER -TWO_ARG_DOWNCALL art_quick_alloc_array_resolved32\c_suffix, artAllocArrayFromCodeResolved\cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER -TWO_ARG_DOWNCALL art_quick_alloc_array_resolved64\c_suffix, artAllocArrayFromCodeResolved\cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER +TWO_ARG_DOWNCALL art_quick_alloc_array_resolved8\c_suffix, artAllocArrayFromCodeResolved\cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER +TWO_ARG_DOWNCALL art_quick_alloc_array_resolved16\c_suffix, artAllocArrayFromCodeResolved\cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER +TWO_ARG_DOWNCALL art_quick_alloc_array_resolved32\c_suffix, artAllocArrayFromCodeResolved\cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER +TWO_ARG_DOWNCALL art_quick_alloc_array_resolved64\c_suffix, artAllocArrayFromCodeResolved\cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER .endm .macro GENERATE_ALL_ALLOC_ENTRYPOINTS @@ -58,29 +58,29 @@ GENERATE_ALLOC_ENTRYPOINTS _region_tlab_instrumented, RegionTLABInstrumented // GENERATE_ALL_ALLOC_ENTRYPOINTS for selectively implementing allocation fast paths in // hand-written assembly. #define GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_RESOLVED(c_suffix, cxx_suffix) \ - ONE_ARG_DOWNCALL art_quick_alloc_object_resolved ## c_suffix, artAllocObjectFromCodeResolved ## cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER + ONE_ARG_DOWNCALL art_quick_alloc_object_resolved ## c_suffix, artAllocObjectFromCodeResolved ## cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER #define GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_INITIALIZED(c_suffix, cxx_suffix) \ - ONE_ARG_DOWNCALL art_quick_alloc_object_initialized ## c_suffix, artAllocObjectFromCodeInitialized ## cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER + ONE_ARG_DOWNCALL art_quick_alloc_object_initialized ## c_suffix, artAllocObjectFromCodeInitialized ## cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER #define GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_WITH_ACCESS_CHECK(c_suffix, cxx_suffix) \ - ONE_ARG_DOWNCALL art_quick_alloc_object_with_checks ## c_suffix, artAllocObjectFromCodeWithChecks ## cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER + ONE_ARG_DOWNCALL art_quick_alloc_object_with_checks ## c_suffix, artAllocObjectFromCodeWithChecks ## cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER #define GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_OBJECT(c_suffix, cxx_suffix) \ - ONE_ARG_DOWNCALL art_quick_alloc_string_object ## c_suffix, artAllocStringObject ## cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER + ONE_ARG_DOWNCALL art_quick_alloc_string_object ## c_suffix, artAllocStringObject ## cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER #define GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_BYTES(c_suffix, cxx_suffix) \ - FOUR_ARG_DOWNCALL art_quick_alloc_string_from_bytes ## c_suffix, artAllocStringFromBytesFromCode ## cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER + FOUR_ARG_DOWNCALL art_quick_alloc_string_from_bytes ## c_suffix, artAllocStringFromBytesFromCode ## cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER #define GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_CHARS(c_suffix, cxx_suffix) \ - THREE_ARG_DOWNCALL art_quick_alloc_string_from_chars ## c_suffix, artAllocStringFromCharsFromCode ## cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER + THREE_ARG_DOWNCALL art_quick_alloc_string_from_chars ## c_suffix, artAllocStringFromCharsFromCode ## cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER #define GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_STRING(c_suffix, cxx_suffix) \ - ONE_ARG_DOWNCALL art_quick_alloc_string_from_string ## c_suffix, artAllocStringFromStringFromCode ## cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER + ONE_ARG_DOWNCALL art_quick_alloc_string_from_string ## c_suffix, artAllocStringFromStringFromCode ## cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER #define GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED(c_suffix, cxx_suffix) \ - TWO_ARG_DOWNCALL art_quick_alloc_array_resolved ## c_suffix, artAllocArrayFromCodeResolved ## cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER + TWO_ARG_DOWNCALL art_quick_alloc_array_resolved ## c_suffix, artAllocArrayFromCodeResolved ## cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER #define GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED8(c_suffix, cxx_suffix) \ - TWO_ARG_DOWNCALL art_quick_alloc_array_resolved8 ## c_suffix, artAllocArrayFromCodeResolved ## cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER + TWO_ARG_DOWNCALL art_quick_alloc_array_resolved8 ## c_suffix, artAllocArrayFromCodeResolved ## cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER #define GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED16(c_suffix, cxx_suffix) \ - TWO_ARG_DOWNCALL art_quick_alloc_array_resolved16 ## c_suffix, artAllocArrayFromCodeResolved ## cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER + TWO_ARG_DOWNCALL art_quick_alloc_array_resolved16 ## c_suffix, artAllocArrayFromCodeResolved ## cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER #define GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED32(c_suffix, cxx_suffix) \ - TWO_ARG_DOWNCALL art_quick_alloc_array_resolved32 ## c_suffix, artAllocArrayFromCodeResolved ## cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER + TWO_ARG_DOWNCALL art_quick_alloc_array_resolved32 ## c_suffix, artAllocArrayFromCodeResolved ## cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER #define GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED64(c_suffix, cxx_suffix) \ - TWO_ARG_DOWNCALL art_quick_alloc_array_resolved64 ## c_suffix, artAllocArrayFromCodeResolved ## cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER + TWO_ARG_DOWNCALL art_quick_alloc_array_resolved64 ## c_suffix, artAllocArrayFromCodeResolved ## cxx_suffix, RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER .macro GENERATE_ALLOC_ENTRYPOINTS_FOR_REGION_TLAB_ALLOCATOR // This is to be separately defined for each architecture to allow a hand-written assembly fast path. diff --git a/runtime/arch/x86/asm_support_x86.h b/runtime/arch/x86/asm_support_x86.h index 737d736f01..f6889334df 100644 --- a/runtime/arch/x86/asm_support_x86.h +++ b/runtime/arch/x86/asm_support_x86.h @@ -25,5 +25,7 @@ #define FRAME_SIZE_SAVE_EVERYTHING (48 + 64) #define FRAME_SIZE_SAVE_EVERYTHING_FOR_CLINIT FRAME_SIZE_SAVE_EVERYTHING #define FRAME_SIZE_SAVE_EVERYTHING_FOR_SUSPEND_CHECK FRAME_SIZE_SAVE_EVERYTHING +#define SAVE_EVERYTHING_FRAME_EAX_OFFSET \ + (FRAME_SIZE_SAVE_EVERYTHING - CALLEE_SAVE_EVERYTHING_NUM_CORE_SPILLS * POINTER_SIZE) #endif // ART_RUNTIME_ARCH_X86_ASM_SUPPORT_X86_H_ diff --git a/runtime/arch/x86/quick_entrypoints_x86.S b/runtime/arch/x86/quick_entrypoints_x86.S index 7f1311c01e..c66d893d7d 100644 --- a/runtime/arch/x86/quick_entrypoints_x86.S +++ b/runtime/arch/x86/quick_entrypoints_x86.S @@ -794,12 +794,9 @@ MACRO3(ONE_ARG_SAVE_EVERYTHING_DOWNCALL, c_name, cxx_name, runtime_method_offset call CALLVAR(cxx_name) // cxx_name(arg1, Thread*) addl MACRO_LITERAL(16), %esp // pop arguments CFI_ADJUST_CFA_OFFSET(-16) - testl %eax, %eax // If result is null, deliver the OOME. + testl %eax, %eax // If result is null deliver pending exception jz 1f - CFI_REMEMBER_STATE - RESTORE_SAVE_EVERYTHING_FRAME_KEEP_EAX // restore frame up to return address - ret // return - CFI_RESTORE_STATE_AND_DEF_CFA esp, FRAME_SIZE_SAVE_EVERYTHING + DEOPT_OR_RESTORE_SAVE_EVERYTHING_FRAME_AND_RETURN_EAX ebx, /* is_ref= */1 // Check for deopt 1: DELIVER_PENDING_EXCEPTION_FRAME_READY END_FUNCTION VAR(c_name) @@ -809,18 +806,72 @@ MACRO2(ONE_ARG_SAVE_EVERYTHING_DOWNCALL_FOR_CLINIT, c_name, cxx_name) ONE_ARG_SAVE_EVERYTHING_DOWNCALL \c_name, \cxx_name, RUNTIME_SAVE_EVERYTHING_FOR_CLINIT_METHOD_OFFSET END_MACRO -MACRO0(RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER) - testl %eax, %eax // eax == 0 ? - jz 1f // if eax == 0 goto 1 - ret // return -1: // deliver exception on current thread +MACRO0(RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER) + testl %eax, %eax // eax == 0 ? + jz 1f // if eax == 0 goto 1 + DEOPT_OR_RETURN ebx, /*is_ref=*/1 // check if deopt is required +1: // deliver exception on current thread + DELIVER_PENDING_EXCEPTION +END_MACRO + +MACRO0(RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION) + cmpl MACRO_LITERAL(0),%fs:THREAD_EXCEPTION_OFFSET // exception field == 0 ? + jne 1f // if exception field != 0 goto 1 + DEOPT_OR_RETURN ebx // check if deopt is required +1: // deliver exception on current thread DELIVER_PENDING_EXCEPTION END_MACRO +MACRO2(DEOPT_OR_RETURN, temp, is_ref = 0) + cmpl LITERAL(0), %fs:THREAD_DEOPT_CHECK_REQUIRED_OFFSET + jne 2f + ret +2: + SETUP_SAVE_EVERYTHING_FRAME \temp + subl MACRO_LITERAL(4), %esp // alignment padding + CFI_ADJUST_CFA_OFFSET(4) + pushl MACRO_LITERAL(\is_ref) // is_ref + CFI_ADJUST_CFA_OFFSET(4) + PUSH_ARG eax // result + pushl %fs:THREAD_SELF_OFFSET // Pass Thread::Current + CFI_ADJUST_CFA_OFFSET(4) + call SYMBOL(artDeoptimizeIfNeeded) + addl LITERAL(16), %esp // pop arguments + CFI_REMEMBER_STATE + RESTORE_SAVE_EVERYTHING_FRAME + ret + CFI_RESTORE_STATE_AND_DEF_CFA esp, FRAME_SIZE_SAVE_EVERYTHING +END_MACRO + +MACRO2(DEOPT_OR_RESTORE_SAVE_EVERYTHING_FRAME_AND_RETURN_EAX, temp, is_ref = 0) + cmpl LITERAL(0), %fs:THREAD_DEOPT_CHECK_REQUIRED_OFFSET + jne 2f + CFI_REMEMBER_STATE + RESTORE_SAVE_EVERYTHING_FRAME_KEEP_EAX + ret + CFI_RESTORE_STATE_AND_DEF_CFA esp, FRAME_SIZE_SAVE_EVERYTHING +2: + movl %eax, SAVE_EVERYTHING_FRAME_EAX_OFFSET(%esp) // update eax in the frame + subl MACRO_LITERAL(4), %esp // alignment padding + CFI_ADJUST_CFA_OFFSET(4) + pushl MACRO_LITERAL(\is_ref) // is_ref + CFI_ADJUST_CFA_OFFSET(4) + PUSH_ARG eax // result + pushl %fs:THREAD_SELF_OFFSET // Pass Thread::Current + CFI_ADJUST_CFA_OFFSET(4) + call SYMBOL(artDeoptimizeIfNeeded) + addl LITERAL(16), %esp // pop arguments + CFI_REMEMBER_STATE + RESTORE_SAVE_EVERYTHING_FRAME + ret + CFI_RESTORE_STATE_AND_DEF_CFA esp, FRAME_SIZE_SAVE_EVERYTHING +END_MACRO + + MACRO0(RETURN_IF_EAX_ZERO) testl %eax, %eax // eax == 0 ? jnz 1f // if eax != 0 goto 1 - ret // return + DEOPT_OR_RETURN ebx // check if deopt is needed 1: // deliver exception on current thread DELIVER_PENDING_EXCEPTION END_MACRO @@ -927,7 +978,7 @@ MACRO2(ART_QUICK_ALLOC_OBJECT_ROSALLOC, c_name, cxx_name) addl LITERAL(16), %esp // pop arguments CFI_ADJUST_CFA_OFFSET(-16) RESTORE_SAVE_REFS_ONLY_FRAME // restore frame up to return address - RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER // return or deliver exception + RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER // return or deliver exception END_FUNCTION VAR(c_name) END_MACRO @@ -974,7 +1025,7 @@ MACRO1(ALLOC_OBJECT_RESOLVED_TLAB_SLOW_PATH, cxx_name) addl LITERAL(16), %esp CFI_ADJUST_CFA_OFFSET(-16) RESTORE_SAVE_REFS_ONLY_FRAME // restore frame up to return address - RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER // return or deliver exception + RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER // return or deliver exception END_MACRO MACRO2(ART_QUICK_ALLOC_OBJECT_TLAB, c_name, cxx_name) @@ -1107,7 +1158,7 @@ MACRO3(GENERATE_ALLOC_ARRAY_TLAB, c_entrypoint, cxx_name, size_setup) addl LITERAL(16), %esp // pop arguments CFI_ADJUST_CFA_OFFSET(-16) RESTORE_SAVE_REFS_ONLY_FRAME // restore frame up to return address - RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER // return or deliver exception + RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER // return or deliver exception END_FUNCTION VAR(c_entrypoint) END_MACRO @@ -1501,21 +1552,21 @@ END_FUNCTION art_quick_lushr // Note: Functions `art{Get,Set}<Kind>{Static,Instance}FromCompiledCode` are // defined with a macro in runtime/entrypoints/quick/quick_field_entrypoints.cc. -ONE_ARG_REF_DOWNCALL art_quick_get_boolean_static, artGetBooleanStaticFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION -ONE_ARG_REF_DOWNCALL art_quick_get_byte_static, artGetByteStaticFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION -ONE_ARG_REF_DOWNCALL art_quick_get_char_static, artGetCharStaticFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION -ONE_ARG_REF_DOWNCALL art_quick_get_short_static, artGetShortStaticFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION -ONE_ARG_REF_DOWNCALL art_quick_get32_static, artGet32StaticFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION -ONE_ARG_REF_DOWNCALL art_quick_get64_static, artGet64StaticFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION -ONE_ARG_REF_DOWNCALL art_quick_get_obj_static, artGetObjStaticFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION - -TWO_ARG_REF_DOWNCALL art_quick_get_boolean_instance, artGetBooleanInstanceFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION -TWO_ARG_REF_DOWNCALL art_quick_get_byte_instance, artGetByteInstanceFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION -TWO_ARG_REF_DOWNCALL art_quick_get_char_instance, artGetCharInstanceFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION -TWO_ARG_REF_DOWNCALL art_quick_get_short_instance, artGetShortInstanceFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION -TWO_ARG_REF_DOWNCALL art_quick_get32_instance, artGet32InstanceFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION -TWO_ARG_REF_DOWNCALL art_quick_get64_instance, artGet64InstanceFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION -TWO_ARG_REF_DOWNCALL art_quick_get_obj_instance, artGetObjInstanceFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION +ONE_ARG_REF_DOWNCALL art_quick_get_boolean_static, artGetBooleanStaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION +ONE_ARG_REF_DOWNCALL art_quick_get_byte_static, artGetByteStaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION +ONE_ARG_REF_DOWNCALL art_quick_get_char_static, artGetCharStaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION +ONE_ARG_REF_DOWNCALL art_quick_get_short_static, artGetShortStaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION +ONE_ARG_REF_DOWNCALL art_quick_get32_static, artGet32StaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION +ONE_ARG_REF_DOWNCALL art_quick_get64_static, artGet64StaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION +ONE_ARG_REF_DOWNCALL art_quick_get_obj_static, artGetObjStaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION + +TWO_ARG_REF_DOWNCALL art_quick_get_boolean_instance, artGetBooleanInstanceFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION +TWO_ARG_REF_DOWNCALL art_quick_get_byte_instance, artGetByteInstanceFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION +TWO_ARG_REF_DOWNCALL art_quick_get_char_instance, artGetCharInstanceFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION +TWO_ARG_REF_DOWNCALL art_quick_get_short_instance, artGetShortInstanceFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION +TWO_ARG_REF_DOWNCALL art_quick_get32_instance, artGet32InstanceFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION +TWO_ARG_REF_DOWNCALL art_quick_get64_instance, artGet64InstanceFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION +TWO_ARG_REF_DOWNCALL art_quick_get_obj_instance, artGetObjInstanceFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION TWO_ARG_REF_DOWNCALL art_quick_set8_static, artSet8StaticFromCompiledCode, RETURN_IF_EAX_ZERO TWO_ARG_REF_DOWNCALL art_quick_set16_static, artSet16StaticFromCompiledCode, RETURN_IF_EAX_ZERO @@ -1616,7 +1667,7 @@ DEFINE_FUNCTION art_quick_resolution_trampoline movl %eax, %edi // remember code pointer in EDI addl LITERAL(16), %esp // pop arguments CFI_ADJUST_CFA_OFFSET(-16) - test %eax, %eax // if code pointer is null goto deliver pending exception + test %eax, %eax // if code pointer is null goto deliver the OOME. jz 1f RESTORE_SAVE_REFS_AND_ARGS_FRAME_AND_JUMP 1: @@ -1974,19 +2025,19 @@ DEFINE_FUNCTION art_quick_string_builder_append SETUP_SAVE_REFS_ONLY_FRAME ebx // save ref containing registers for GC // Outgoing argument set up leal FRAME_SIZE_SAVE_REFS_ONLY + __SIZEOF_POINTER__(%esp), %edi // prepare args - push %eax // push padding + push %eax // push padding CFI_ADJUST_CFA_OFFSET(4) - pushl %fs:THREAD_SELF_OFFSET // pass Thread::Current() + pushl %fs:THREAD_SELF_OFFSET // pass Thread::Current() CFI_ADJUST_CFA_OFFSET(4) - push %edi // pass args + push %edi // pass args CFI_ADJUST_CFA_OFFSET(4) - push %eax // pass format + push %eax // pass format CFI_ADJUST_CFA_OFFSET(4) - call SYMBOL(artStringBuilderAppend) // (uint32_t, const unit32_t*, Thread*) - addl MACRO_LITERAL(16), %esp // pop arguments + call SYMBOL(artStringBuilderAppend) // (uint32_t, const unit32_t*, Thread*) + addl MACRO_LITERAL(16), %esp // pop arguments CFI_ADJUST_CFA_OFFSET(-16) - RESTORE_SAVE_REFS_ONLY_FRAME // restore frame up to return address - RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER // return or deliver exception + RESTORE_SAVE_REFS_ONLY_FRAME // restore frame up to return address + RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER // return or deliver exception END_FUNCTION art_quick_string_builder_append // Create a function `name` calling the ReadBarrier::Mark routine, diff --git a/runtime/arch/x86_64/asm_support_x86_64.h b/runtime/arch/x86_64/asm_support_x86_64.h index 51befbe7b8..e389c781e5 100644 --- a/runtime/arch/x86_64/asm_support_x86_64.h +++ b/runtime/arch/x86_64/asm_support_x86_64.h @@ -25,5 +25,7 @@ #define FRAME_SIZE_SAVE_EVERYTHING (144 + 16*8) #define FRAME_SIZE_SAVE_EVERYTHING_FOR_CLINIT FRAME_SIZE_SAVE_EVERYTHING #define FRAME_SIZE_SAVE_EVERYTHING_FOR_SUSPEND_CHECK FRAME_SIZE_SAVE_EVERYTHING +#define SAVE_EVERYTHING_FRAME_RAX_OFFSET \ + (FRAME_SIZE_SAVE_EVERYTHING - CALLEE_SAVE_EVERYTHING_NUM_CORE_SPILLS * POINTER_SIZE) #endif // ART_RUNTIME_ARCH_X86_64_ASM_SUPPORT_X86_64_H_ diff --git a/runtime/arch/x86_64/quick_entrypoints_x86_64.S b/runtime/arch/x86_64/quick_entrypoints_x86_64.S index 673696c714..2888d531cf 100644 --- a/runtime/arch/x86_64/quick_entrypoints_x86_64.S +++ b/runtime/arch/x86_64/quick_entrypoints_x86_64.S @@ -731,12 +731,9 @@ MACRO3(ONE_ARG_SAVE_EVERYTHING_DOWNCALL, c_name, cxx_name, runtime_method_offset movl %eax, %edi // pass the index of the constant as arg0 movq %gs:THREAD_SELF_OFFSET, %rsi // pass Thread::Current() call CALLVAR(cxx_name) // cxx_name(arg0, Thread*) - testl %eax, %eax // If result is null, deliver the OOME. + testl %eax, %eax // If result is null, deliver pending exception. jz 1f - CFI_REMEMBER_STATE - RESTORE_SAVE_EVERYTHING_FRAME_KEEP_RAX // restore frame up to return address - ret - CFI_RESTORE_STATE_AND_DEF_CFA rsp, FRAME_SIZE_SAVE_EVERYTHING + DEOPT_OR_RESTORE_SAVE_EVERYTHING_FRAME_AND_RETURN_RAX /*is_ref=*/1 1: DELIVER_PENDING_EXCEPTION_FRAME_READY END_FUNCTION VAR(c_name) @@ -746,18 +743,65 @@ MACRO2(ONE_ARG_SAVE_EVERYTHING_DOWNCALL_FOR_CLINIT, c_name, cxx_name) ONE_ARG_SAVE_EVERYTHING_DOWNCALL \c_name, \cxx_name, RUNTIME_SAVE_EVERYTHING_FOR_CLINIT_METHOD_OFFSET END_MACRO -MACRO0(RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER) +MACRO0(RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER) testq %rax, %rax // rax == 0 ? jz 1f // if rax == 0 goto 1 - ret // return + DEOPT_OR_RETURN /*is_ref=*/1 // Check if deopt is required +1: // deliver exception on current thread + DELIVER_PENDING_EXCEPTION +END_MACRO + + +MACRO0(RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION) + movq %gs:THREAD_EXCEPTION_OFFSET, %rcx // get exception field + testq %rcx, %rcx // rcx == 0 ? + jnz 1f // if rcx != 0 goto 1 + DEOPT_OR_RETURN // Check if deopt is required 1: // deliver exception on current thread DELIVER_PENDING_EXCEPTION END_MACRO +MACRO1(DEOPT_OR_RETURN, is_ref = 0) + cmpl LITERAL(0), %gs:THREAD_DEOPT_CHECK_REQUIRED_OFFSET + jne 2f + ret +2: + SETUP_SAVE_EVERYTHING_FRAME + movq LITERAL(\is_ref), %rdx // pass if result is a reference + movq %rax, %rsi // pass the result + movq %gs:THREAD_SELF_OFFSET, %rdi // pass Thread::Current + call SYMBOL(artDeoptimizeIfNeeded) + CFI_REMEMBER_STATE + RESTORE_SAVE_EVERYTHING_FRAME + ret + CFI_RESTORE_STATE_AND_DEF_CFA rsp, FRAME_SIZE_SAVE_EVERYTHING +END_MACRO + +MACRO1(DEOPT_OR_RESTORE_SAVE_EVERYTHING_FRAME_AND_RETURN_RAX, is_ref = 0) + cmpl LITERAL(0), %gs:THREAD_DEOPT_CHECK_REQUIRED_OFFSET + jne 2f + CFI_REMEMBER_STATE + RESTORE_SAVE_EVERYTHING_FRAME_KEEP_RAX + ret + CFI_RESTORE_STATE_AND_DEF_CFA rsp, FRAME_SIZE_SAVE_EVERYTHING +2: + movq %rax, SAVE_EVERYTHING_FRAME_RAX_OFFSET(%rsp) // update result in the frame + movq LITERAL(\is_ref), %rdx // pass if result is a reference + movq %rax, %rsi // pass the result + movq %gs:THREAD_SELF_OFFSET, %rdi // pass Thread::Current + call SYMBOL(artDeoptimizeIfNeeded) + CFI_REMEMBER_STATE + RESTORE_SAVE_EVERYTHING_FRAME + ret + CFI_RESTORE_STATE_AND_DEF_CFA rsp, FRAME_SIZE_SAVE_EVERYTHING +END_MACRO + + + MACRO0(RETURN_IF_EAX_ZERO) testl %eax, %eax // eax == 0 ? jnz 1f // if eax != 0 goto 1 - ret // return + DEOPT_OR_RETURN // Check if we need a deopt 1: // deliver exception on current thread DELIVER_PENDING_EXCEPTION END_MACRO @@ -859,7 +903,7 @@ MACRO2(ART_QUICK_ALLOC_OBJECT_ROSALLOC, c_name, cxx_name) movq %gs:THREAD_SELF_OFFSET, %rsi // pass Thread::Current() call CALLVAR(cxx_name) // cxx_name(arg0, Thread*) RESTORE_SAVE_REFS_ONLY_FRAME // restore frame up to return address - RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER // return or deliver exception + RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER // return or deliver exception END_FUNCTION VAR(c_name) END_MACRO @@ -931,7 +975,7 @@ MACRO1(ALLOC_OBJECT_TLAB_SLOW_PATH, cxx_name) movq %gs:THREAD_SELF_OFFSET, %rsi // pass Thread::Current() call CALLVAR(cxx_name) // cxx_name(arg0, Thread*) RESTORE_SAVE_REFS_ONLY_FRAME // restore frame up to return address - RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER // return or deliver exception + RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER // return or deliver exception END_MACRO // A hand-written override for GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_RESOLVED(_tlab, TLAB). May be @@ -1019,7 +1063,7 @@ MACRO3(GENERATE_ALLOC_ARRAY_TLAB, c_entrypoint, cxx_name, size_setup) movq %gs:THREAD_SELF_OFFSET, %rdx // pass Thread::Current() call CALLVAR(cxx_name) // cxx_name(arg0, arg1, Thread*) RESTORE_SAVE_REFS_ONLY_FRAME // restore frame up to return address - RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER // return or deliver exception + RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER // return or deliver exception END_FUNCTION VAR(c_entrypoint) END_MACRO @@ -1324,27 +1368,27 @@ THREE_ARG_REF_DOWNCALL art_quick_set32_instance, artSet32InstanceFromCompiledCod THREE_ARG_REF_DOWNCALL art_quick_set64_instance, artSet64InstanceFromCompiledCode, RETURN_IF_EAX_ZERO THREE_ARG_REF_DOWNCALL art_quick_set_obj_instance, artSetObjInstanceFromCompiledCode, RETURN_IF_EAX_ZERO -TWO_ARG_REF_DOWNCALL art_quick_get_byte_instance, artGetByteInstanceFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION -TWO_ARG_REF_DOWNCALL art_quick_get_boolean_instance, artGetBooleanInstanceFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION -TWO_ARG_REF_DOWNCALL art_quick_get_short_instance, artGetShortInstanceFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION -TWO_ARG_REF_DOWNCALL art_quick_get_char_instance, artGetCharInstanceFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION -TWO_ARG_REF_DOWNCALL art_quick_get32_instance, artGet32InstanceFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION -TWO_ARG_REF_DOWNCALL art_quick_get64_instance, artGet64InstanceFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION -TWO_ARG_REF_DOWNCALL art_quick_get_obj_instance, artGetObjInstanceFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION +TWO_ARG_REF_DOWNCALL art_quick_get_byte_instance, artGetByteInstanceFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION +TWO_ARG_REF_DOWNCALL art_quick_get_boolean_instance, artGetBooleanInstanceFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION +TWO_ARG_REF_DOWNCALL art_quick_get_short_instance, artGetShortInstanceFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION +TWO_ARG_REF_DOWNCALL art_quick_get_char_instance, artGetCharInstanceFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION +TWO_ARG_REF_DOWNCALL art_quick_get32_instance, artGet32InstanceFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION +TWO_ARG_REF_DOWNCALL art_quick_get64_instance, artGet64InstanceFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION +TWO_ARG_REF_DOWNCALL art_quick_get_obj_instance, artGetObjInstanceFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION TWO_ARG_REF_DOWNCALL art_quick_set8_static, artSet8StaticFromCompiledCode, RETURN_IF_EAX_ZERO TWO_ARG_REF_DOWNCALL art_quick_set16_static, artSet16StaticFromCompiledCode, RETURN_IF_EAX_ZERO TWO_ARG_REF_DOWNCALL art_quick_set32_static, artSet32StaticFromCompiledCode, RETURN_IF_EAX_ZERO -TWO_ARG_REF_DOWNCALL art_quick_set64_static, artSet64StaticFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION +TWO_ARG_REF_DOWNCALL art_quick_set64_static, artSet64StaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION TWO_ARG_REF_DOWNCALL art_quick_set_obj_static, artSetObjStaticFromCompiledCode, RETURN_IF_EAX_ZERO -ONE_ARG_REF_DOWNCALL art_quick_get_byte_static, artGetByteStaticFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION -ONE_ARG_REF_DOWNCALL art_quick_get_boolean_static, artGetBooleanStaticFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION -ONE_ARG_REF_DOWNCALL art_quick_get_short_static, artGetShortStaticFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION -ONE_ARG_REF_DOWNCALL art_quick_get_char_static, artGetCharStaticFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION -ONE_ARG_REF_DOWNCALL art_quick_get32_static, artGet32StaticFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION -ONE_ARG_REF_DOWNCALL art_quick_get64_static, artGet64StaticFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION -ONE_ARG_REF_DOWNCALL art_quick_get_obj_static, artGetObjStaticFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION +ONE_ARG_REF_DOWNCALL art_quick_get_byte_static, artGetByteStaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION +ONE_ARG_REF_DOWNCALL art_quick_get_boolean_static, artGetBooleanStaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION +ONE_ARG_REF_DOWNCALL art_quick_get_short_static, artGetShortStaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION +ONE_ARG_REF_DOWNCALL art_quick_get_char_static, artGetCharStaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION +ONE_ARG_REF_DOWNCALL art_quick_get32_static, artGet32StaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION +ONE_ARG_REF_DOWNCALL art_quick_get64_static, artGet64StaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION +ONE_ARG_REF_DOWNCALL art_quick_get_obj_static, artGetObjStaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION DEFINE_FUNCTION art_quick_proxy_invoke_handler SETUP_SAVE_REFS_AND_ARGS_FRAME_WITH_METHOD_IN_RDI @@ -1846,7 +1890,7 @@ DEFINE_FUNCTION art_quick_string_builder_append movq %gs:THREAD_SELF_OFFSET, %rdx // pass Thread::Current() call artStringBuilderAppend // (uint32_t, const unit32_t*, Thread*) RESTORE_SAVE_REFS_ONLY_FRAME // restore frame up to return address - RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER // return or deliver exception + RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER // return or deopt or deliver exception END_FUNCTION art_quick_string_builder_append // Create a function `name` calling the ReadBarrier::Mark routine, diff --git a/runtime/entrypoints/quick/quick_dexcache_entrypoints.cc b/runtime/entrypoints/quick/quick_dexcache_entrypoints.cc index 60a5875c5e..76bee2152a 100644 --- a/runtime/entrypoints/quick/quick_dexcache_entrypoints.cc +++ b/runtime/entrypoints/quick/quick_dexcache_entrypoints.cc @@ -23,6 +23,7 @@ #include "dex/dex_file_types.h" #include "entrypoints/entrypoint_utils-inl.h" #include "gc/heap.h" +#include "jvalue-inl.h" #include "mirror/class-inl.h" #include "mirror/class_loader.h" #include "mirror/object-inl.h" diff --git a/runtime/entrypoints/quick/quick_thread_entrypoints.cc b/runtime/entrypoints/quick/quick_thread_entrypoints.cc index 93422cf056..5dca58ab04 100644 --- a/runtime/entrypoints/quick/quick_thread_entrypoints.cc +++ b/runtime/entrypoints/quick/quick_thread_entrypoints.cc @@ -21,16 +21,46 @@ namespace art { +extern "C" void artDeoptimizeIfNeeded(Thread* self, uintptr_t result, bool is_ref) + REQUIRES_SHARED(Locks::mutator_lock_) { + instrumentation::Instrumentation* instr = Runtime::Current()->GetInstrumentation(); + DCHECK(!self->IsExceptionPending()); + + ArtMethod** sp = self->GetManagedStack()->GetTopQuickFrame(); + DCHECK(sp != nullptr && (*sp)->IsRuntimeMethod()); + + DeoptimizationMethodType type = instr->GetDeoptimizationMethodType(*sp); + JValue jvalue; + jvalue.SetJ(result); + instr->DeoptimizeIfNeeded(self, sp, type, jvalue, is_ref); +} + extern "C" void artTestSuspendFromCode(Thread* self) REQUIRES_SHARED(Locks::mutator_lock_) { // Called when there is a pending checkpoint or suspend request. ScopedQuickEntrypointChecks sqec(self); self->CheckSuspend(); + + // We could have other dex instructions at the same dex pc as suspend and we need to execute + // those instructions. So we should start executing from the current dex pc. + ArtMethod** sp = self->GetManagedStack()->GetTopQuickFrame(); + JValue result; + result.SetJ(0); + Runtime::Current()->GetInstrumentation()->DeoptimizeIfNeeded( + self, sp, DeoptimizationMethodType::kKeepDexPc, result, /* is_ref= */ false); } extern "C" void artImplicitSuspendFromCode(Thread* self) REQUIRES_SHARED(Locks::mutator_lock_) { // Called when there is a pending checkpoint or suspend request. ScopedQuickEntrypointChecks sqec(self); self->CheckSuspend(/*implicit=*/ true); + + // We could have other dex instructions at the same dex pc as suspend and we need to execute + // those instructions. So we should start executing from the current dex pc. + ArtMethod** sp = self->GetManagedStack()->GetTopQuickFrame(); + JValue result; + result.SetJ(0); + Runtime::Current()->GetInstrumentation()->DeoptimizeIfNeeded( + self, sp, DeoptimizationMethodType::kKeepDexPc, result, /* is_ref= */ false); } extern "C" void artCompileOptimized(ArtMethod* method, Thread* self) diff --git a/runtime/entrypoints/quick/quick_trampoline_entrypoints.cc b/runtime/entrypoints/quick/quick_trampoline_entrypoints.cc index 2086fa234c..0707477df3 100644 --- a/runtime/entrypoints/quick/quick_trampoline_entrypoints.cc +++ b/runtime/entrypoints/quick/quick_trampoline_entrypoints.cc @@ -713,41 +713,34 @@ extern "C" uint64_t artQuickToInterpreterBridge(ArtMethod* method, Thread* self, // Pop transition. self->PopManagedStackFragment(fragment); - // Request a stack deoptimization if needed - ArtMethod* caller = QuickArgumentVisitor::GetCallingMethod(sp); - uintptr_t caller_pc = QuickArgumentVisitor::GetCallingPc(sp); + // Check if caller needs to be deoptimized for instrumentation reasons. + instrumentation::Instrumentation* instr = Runtime::Current()->GetInstrumentation(); // If caller_pc is the instrumentation exit stub, the stub will check to see if deoptimization // should be done and it knows the real return pc. NB If the upcall is null we don't need to do // anything. This can happen during shutdown or early startup. - if (UNLIKELY( - caller != nullptr && - caller_pc != reinterpret_cast<uintptr_t>(GetQuickInstrumentationExitPc()) && - (self->IsForceInterpreter() || Dbg::IsForcedInterpreterNeededForUpcall(self, caller)))) { - if (!Runtime::Current()->IsAsyncDeoptimizeable(caller_pc)) { - LOG(WARNING) << "Got a deoptimization request on un-deoptimizable method " - << caller->PrettyMethod(); - } else { - VLOG(deopt) << "Forcing deoptimization on return from method " << method->PrettyMethod() - << " to " << caller->PrettyMethod() - << (force_frame_pop ? " for frame-pop" : ""); - DCHECK_IMPLIES(force_frame_pop, result.GetJ() == 0) - << "Force frame pop should have no result."; - if (force_frame_pop && self->GetException() != nullptr) { - LOG(WARNING) << "Suppressing exception for instruction-retry: " - << self->GetException()->Dump(); - } - // Push the context of the deoptimization stack so we can restore the return value and the - // exception before executing the deoptimized frames. - self->PushDeoptimizationContext( - result, - shorty[0] == 'L' || shorty[0] == '[', /* class or array */ - force_frame_pop ? nullptr : self->GetException(), - /* from_code= */ false, - DeoptimizationMethodType::kDefault); - - // Set special exception to cause deoptimization. - self->SetException(Thread::GetDeoptimizationException()); + if (UNLIKELY(instr->ShouldDeoptimizeCaller(self, sp))) { + ArtMethod* caller = QuickArgumentVisitor::GetOuterMethod(sp); + uintptr_t caller_pc = QuickArgumentVisitor::GetCallingPc(sp); + DCHECK(Runtime::Current()->IsAsyncDeoptimizeable(caller_pc)); + DCHECK(caller != nullptr); + VLOG(deopt) << "Forcing deoptimization on return from method " << method->PrettyMethod() + << " to " << caller->PrettyMethod() << (force_frame_pop ? " for frame-pop" : ""); + DCHECK(!force_frame_pop || result.GetJ() == 0) << "Force frame pop should have no result."; + if (force_frame_pop && self->GetException() != nullptr) { + LOG(WARNING) << "Suppressing exception for instruction-retry: " + << self->GetException()->Dump(); } + DCHECK(self->GetException() != Thread::GetDeoptimizationException()); + // Push the context of the deoptimization stack so we can restore the return value and the + // exception before executing the deoptimized frames. + self->PushDeoptimizationContext(result, + shorty[0] == 'L' || shorty[0] == '[', /* class or array */ + force_frame_pop ? nullptr : self->GetException(), + /* from_code= */ false, + DeoptimizationMethodType::kDefault); + + // Set special exception to cause deoptimization. + self->SetException(Thread::GetDeoptimizationException()); } // No need to restore the args since the method has already been run by the interpreter. @@ -1076,6 +1069,7 @@ extern "C" const void* artInstrumentationMethodEntryFromCode(ArtMethod* method, } } + DCHECK(!method->IsRuntimeMethod()); instrumentation->PushInstrumentationStackFrame(self, is_static ? nullptr : h_object.Get(), method, @@ -2600,6 +2594,10 @@ extern "C" uint64_t artInvokePolymorphic(mirror::Object* raw_receiver, Thread* s // Pop transition record. self->PopManagedStackFragment(fragment); + bool is_ref = (shorty[0] == 'L'); + Runtime::Current()->GetInstrumentation()->PushDeoptContextIfNeeded( + self, DeoptimizationMethodType::kDefault, is_ref, result); + return result.GetJ(); } @@ -2658,6 +2656,10 @@ extern "C" uint64_t artInvokeCustom(uint32_t call_site_idx, Thread* self, ArtMet // Pop transition record. self->PopManagedStackFragment(fragment); + bool is_ref = (shorty[0] == 'L'); + Runtime::Current()->GetInstrumentation()->PushDeoptContextIfNeeded( + self, DeoptimizationMethodType::kDefault, is_ref, result); + return result.GetJ(); } @@ -2688,7 +2690,7 @@ extern "C" int artMethodExitHook(Thread* self, instrumentation::Instrumentation* instr = Runtime::Current()->GetInstrumentation(); DCHECK(instr->AreExitStubsInstalled()); bool is_ref; - JValue return_value = instr->GetReturnValue(self, method, &is_ref, gpr_result, fpr_result); + JValue return_value = instr->GetReturnValue(method, &is_ref, gpr_result, fpr_result); bool deoptimize = false; { StackHandleScope<1> hs(self); @@ -2703,7 +2705,7 @@ extern "C" int artMethodExitHook(Thread* self, // back to an upcall. NthCallerVisitor visitor(self, 1, /*include_runtime_and_upcalls=*/false); visitor.WalkStack(true); - deoptimize = instr->ShouldDeoptimizeMethod(self, visitor); + deoptimize = instr->ShouldDeoptimizeCaller(self, visitor); // If we need a deoptimization MethodExitEvent will be called by the interpreter when it // re-executes the return instruction. diff --git a/runtime/instrumentation.cc b/runtime/instrumentation.cc index 1e328a31d2..9c2f7f637d 100644 --- a/runtime/instrumentation.cc +++ b/runtime/instrumentation.cc @@ -55,6 +55,9 @@ #include "thread_list.h" namespace art { +extern "C" NO_RETURN void artDeoptimize(Thread* self); +extern "C" NO_RETURN void artDeliverPendingExceptionFromCode(Thread* self); + namespace instrumentation { constexpr bool kVerboseInstrumentation = false; @@ -482,15 +485,16 @@ void InstrumentationInstallStack(Thread* thread, void* arg, bool deopt_all_frame instrumentation_exit_pc_(instrumentation_exit_pc), reached_existing_instrumentation_frames_(false), force_deopt_id_(force_deopt_id), - deopt_all_frames_(deopt_all_frames) {} + deopt_all_frames_(deopt_all_frames), + runtime_methods_need_deopt_check_(false) {} bool VisitFrame() override REQUIRES_SHARED(Locks::mutator_lock_) { ArtMethod* m = GetMethod(); - if (m == nullptr) { + if (m == nullptr || m->IsRuntimeMethod()) { if (kVerboseInstrumentation) { - LOG(INFO) << " Skipping upcall. Frame " << GetFrameId(); + LOG(INFO) << " Skipping upcall / runtime method. Frame " << GetFrameId(); } - return true; // Ignore upcalls. + return true; // Ignore upcalls and runtime methods. } if (GetCurrentQuickFrame() == nullptr) { if (kVerboseInstrumentation) { @@ -507,11 +511,6 @@ void InstrumentationInstallStack(Thread* thread, void* arg, bool deopt_all_frame auto it = instrumentation_stack_->find(GetReturnPcAddr()); CHECK(it != instrumentation_stack_->end()); const InstrumentationStackFrame& frame = it->second; - if (m->IsRuntimeMethod()) { - if (frame.interpreter_entry_) { - return true; - } - } // We've reached a frame which has already been installed with instrumentation exit stub. // We should have already installed instrumentation or be interpreter on previous frames. @@ -534,12 +533,14 @@ void InstrumentationInstallStack(Thread* thread, void* arg, bool deopt_all_frame const OatQuickMethodHeader* method_header = GetCurrentOatQuickMethodHeader(); if (method_header != nullptr && method_header->HasShouldDeoptimizeFlag()) { if (deopt_all_frames_) { + runtime_methods_need_deopt_check_ = true; SetShouldDeoptimizeFlag(DeoptimizeFlagValue::kDebug); } return true; } CHECK_NE(return_pc, 0U); - if (UNLIKELY(reached_existing_instrumentation_frames_ && !m->IsRuntimeMethod())) { + DCHECK(!m->IsRuntimeMethod()); + if (UNLIKELY(reached_existing_instrumentation_frames_)) { // We already saw an existing instrumentation frame so this should be a runtime-method // inserted by the interpreter or runtime. std::string thread_name; @@ -550,21 +551,9 @@ void InstrumentationInstallStack(Thread* thread, void* arg, bool deopt_all_frame << " return_pc is " << std::hex << return_pc; UNREACHABLE(); } - if (m->IsRuntimeMethod()) { - size_t frame_size = GetCurrentQuickFrameInfo().FrameSizeInBytes(); - ArtMethod** caller_frame = reinterpret_cast<ArtMethod**>( - reinterpret_cast<uint8_t*>(GetCurrentQuickFrame()) + frame_size); - if (*caller_frame != nullptr && (*caller_frame)->IsNative()) { - // Do not install instrumentation exit on return to JNI stubs. - return true; - } - } + InstrumentationStackFrame instrumentation_frame( - m->IsRuntimeMethod() ? nullptr : GetThisObject().Ptr(), - m, - return_pc, - false, - force_deopt_id_); + GetThisObject().Ptr(), m, return_pc, false, force_deopt_id_); if (kVerboseInstrumentation) { LOG(INFO) << "Pushing frame " << instrumentation_frame.Dump(); } @@ -584,6 +573,7 @@ void InstrumentationInstallStack(Thread* thread, void* arg, bool deopt_all_frame bool reached_existing_instrumentation_frames_; uint64_t force_deopt_id_; bool deopt_all_frames_; + bool runtime_methods_need_deopt_check_; }; if (kVerboseInstrumentation) { std::string thread_name; @@ -601,6 +591,10 @@ void InstrumentationInstallStack(Thread* thread, void* arg, bool deopt_all_frame deopt_all_frames); visitor.WalkStack(true); + if (visitor.runtime_methods_need_deopt_check_) { + thread->SetDeoptCheckRequired(true); + } + if (instrumentation->ShouldNotifyMethodEnterExitEvents()) { // Create method enter events for all methods currently on the thread's stack. We only do this // if we haven't already processed the method enter events. @@ -622,14 +616,16 @@ static void InstrumentationRestoreStack(Thread* thread, void* arg) Locks::mutator_lock_->AssertExclusiveHeld(Thread::Current()); struct RestoreStackVisitor final : public StackVisitor { - RestoreStackVisitor(Thread* thread_in, uintptr_t instrumentation_exit_pc, + RestoreStackVisitor(Thread* thread_in, + uintptr_t instrumentation_exit_pc, Instrumentation* instrumentation) : StackVisitor(thread_in, nullptr, kInstrumentationStackWalk), thread_(thread_in), instrumentation_exit_pc_(instrumentation_exit_pc), instrumentation_(instrumentation), instrumentation_stack_(thread_in->GetInstrumentationStack()), - frames_removed_(0) {} + frames_removed_(0), + runtime_methods_need_deopt_check_(false) {} bool VisitFrame() override REQUIRES_SHARED(Locks::mutator_lock_) { if (instrumentation_stack_->size() == 0) { @@ -643,11 +639,19 @@ static void InstrumentationRestoreStack(Thread* thread, void* arg) } return true; // Ignore shadow frames. } - if (m == nullptr) { + if (m == nullptr || m->IsRuntimeMethod()) { if (kVerboseInstrumentation) { - LOG(INFO) << " Skipping upcall. Frame " << GetFrameId(); + LOG(INFO) << " Skipping upcall / runtime method. Frame " << GetFrameId(); } - return true; // Ignore upcalls. + return true; // Ignore upcalls and runtime methods. + } + const OatQuickMethodHeader* method_header = GetCurrentOatQuickMethodHeader(); + // For JITed frames, we don't install instrumentation stubs. + if (method_header != nullptr && method_header->HasShouldDeoptimizeFlag()) { + if (IsShouldDeoptimizeFlagForDebugSet()) { + runtime_methods_need_deopt_check_ = true; + } + return true; } auto it = instrumentation_stack_->find(GetReturnPcAddr()); if (it != instrumentation_stack_->end()) { @@ -684,6 +688,7 @@ static void InstrumentationRestoreStack(Thread* thread, void* arg) Instrumentation* const instrumentation_; std::map<uintptr_t, instrumentation::InstrumentationStackFrame>* const instrumentation_stack_; size_t frames_removed_; + bool runtime_methods_need_deopt_check_; }; if (kVerboseInstrumentation) { std::string thread_name; @@ -698,6 +703,10 @@ static void InstrumentationRestoreStack(Thread* thread, void* arg) reinterpret_cast<uintptr_t>(GetQuickInstrumentationExitPc()); RestoreStackVisitor visitor(thread, instrumentation_exit_pc, instrumentation); visitor.WalkStack(true); + DCHECK_IMPLIES(visitor.runtime_methods_need_deopt_check_, thread->IsDeoptCheckRequired()); + if (!visitor.runtime_methods_need_deopt_check_) { + thread->SetDeoptCheckRequired(false); + } CHECK_EQ(visitor.frames_removed_, stack->size()); stack->clear(); } @@ -1521,83 +1530,18 @@ DeoptimizationMethodType Instrumentation::GetDeoptimizationMethodType(ArtMethod* return DeoptimizationMethodType::kDefault; } -// Try to get the shorty of a runtime method if it's an invocation stub. -static char GetRuntimeMethodShorty(Thread* thread) REQUIRES_SHARED(Locks::mutator_lock_) { - char shorty = 'V'; - StackVisitor::WalkStack( - [&shorty](const art::StackVisitor* stack_visitor) REQUIRES_SHARED(Locks::mutator_lock_) { - ArtMethod* m = stack_visitor->GetMethod(); - if (m == nullptr || m->IsRuntimeMethod()) { - return true; - } - // The first Java method. - if (m->IsNative()) { - // Use JNI method's shorty for the jni stub. - shorty = m->GetShorty()[0]; - } else if (m->IsProxyMethod()) { - // Proxy method just invokes its proxied method via - // art_quick_proxy_invoke_handler. - shorty = m->GetInterfaceMethodIfProxy(kRuntimePointerSize)->GetShorty()[0]; - } else { - const Instruction& instr = m->DexInstructions().InstructionAt(stack_visitor->GetDexPc()); - if (instr.IsInvoke()) { - uint16_t method_index = static_cast<uint16_t>(instr.VRegB()); - const DexFile* dex_file = m->GetDexFile(); - if (interpreter::IsStringInit(dex_file, method_index)) { - // Invoking string init constructor is turned into invoking - // StringFactory.newStringFromChars() which returns a string. - shorty = 'L'; - } else { - shorty = dex_file->GetMethodShorty(method_index)[0]; - } - - } else { - // It could be that a non-invoke opcode invokes a stub, which in turn - // invokes Java code. In such cases, we should never expect a return - // value from the stub. - } - } - // Stop stack walking since we've seen a Java frame. - return false; - }, - thread, - /* context= */ nullptr, - art::StackVisitor::StackWalkKind::kIncludeInlinedFrames); - return shorty; -} - -JValue Instrumentation::GetReturnValue( - Thread* self, ArtMethod* method, bool* is_ref, uint64_t* gpr_result, uint64_t* fpr_result) { +JValue Instrumentation::GetReturnValue(ArtMethod* method, + bool* is_ref, + uint64_t* gpr_result, + uint64_t* fpr_result) { uint32_t length; const PointerSize pointer_size = Runtime::Current()->GetClassLinker()->GetImagePointerSize(); - char return_shorty; // Runtime method does not call into MethodExitEvent() so there should not be // suspension point below. ScopedAssertNoThreadSuspension ants(__FUNCTION__, method->IsRuntimeMethod()); - if (method->IsRuntimeMethod()) { - Runtime* runtime = Runtime::Current(); - if (method != runtime->GetCalleeSaveMethod(CalleeSaveType::kSaveEverythingForClinit) && - method != runtime->GetCalleeSaveMethod(CalleeSaveType::kSaveEverythingForSuspendCheck)) { - // If the caller is at an invocation point and the runtime method is not - // for clinit, we need to pass return results to the caller. - // We need the correct shorty to decide whether we need to pass the return - // result for deoptimization below. - return_shorty = GetRuntimeMethodShorty(self); - } else { - // Some runtime methods such as allocations, unresolved field getters, etc. - // have return value. We don't need to set return_value since MethodExitEvent() - // below isn't called for runtime methods. Deoptimization doesn't need the - // value either since the dex instruction will be re-executed by the - // interpreter, except these two cases: - // (1) For an invoke, which is handled above to get the correct shorty. - // (2) For MONITOR_ENTER/EXIT, which cannot be re-executed since it's not - // idempotent. However there is no return value for it anyway. - return_shorty = 'V'; - } - } else { - return_shorty = method->GetInterfaceMethodIfProxy(pointer_size)->GetShorty(&length)[0]; - } + DCHECK(!method->IsRuntimeMethod()); + char return_shorty = method->GetInterfaceMethodIfProxy(pointer_size)->GetShorty(&length)[0]; *is_ref = return_shorty == '[' || return_shorty == 'L'; JValue return_value; @@ -1611,27 +1555,127 @@ JValue Instrumentation::GetReturnValue( return return_value; } -bool Instrumentation::ShouldDeoptimizeMethod(Thread* self, const NthCallerVisitor& visitor) { - bool should_deoptimize_frame = false; - const OatQuickMethodHeader* header = visitor.GetCurrentOatQuickMethodHeader(); - if (header != nullptr && header->HasShouldDeoptimizeFlag()) { - uint8_t should_deopt_flag = visitor.GetShouldDeoptimizeFlag(); - // DeoptimizeFlag could be set for debugging or for CHA invalidations. - // Deoptimize here only if it was requested for debugging. CHA - // invalidations are handled in the JITed code. - if ((should_deopt_flag & static_cast<uint8_t>(DeoptimizeFlagValue::kDebug)) != 0) { - should_deoptimize_frame = true; - } +bool Instrumentation::PushDeoptContextIfNeeded(Thread* self, + DeoptimizationMethodType deopt_type, + bool is_ref, + const JValue& return_value) + REQUIRES_SHARED(Locks::mutator_lock_) { + if (self->IsExceptionPending()) { + return false; + } + + ArtMethod** sp = self->GetManagedStack()->GetTopQuickFrame(); + DCHECK(sp != nullptr && (*sp)->IsRuntimeMethod()); + if (!ShouldDeoptimizeCaller(self, sp)) { + return false; } - return (visitor.caller != nullptr) && - (InterpreterStubsInstalled() || IsDeoptimized(visitor.caller) || + + // TODO(mythria): The current deopt behaviour is we just re-execute the + // alloc instruction so we don't need the return value. For instrumentation + // related deopts, we actually don't need to and can use the result we got + // here. Since this is a debug only feature it is not very important but + // consider reusing the result in future. + self->PushDeoptimizationContext( + return_value, is_ref, nullptr, /* from_code= */ false, deopt_type); + self->SetException(Thread::GetDeoptimizationException()); + return true; +} + +void Instrumentation::DeoptimizeIfNeeded(Thread* self, + ArtMethod** sp, + DeoptimizationMethodType type, + JValue return_value, + bool is_reference) { + if (self->IsAsyncExceptionPending() || ShouldDeoptimizeCaller(self, sp)) { + self->PushDeoptimizationContext(return_value, + is_reference, + nullptr, + /* from_code= */ false, + type); + artDeoptimize(self); + } +} + +bool Instrumentation::NeedsSlowInterpreterForMethod(Thread* self, ArtMethod* method) { + return (method != nullptr) && + (InterpreterStubsInstalled() || + IsDeoptimized(method) || self->IsForceInterpreter() || // NB Since structurally obsolete compiled methods might have the offsets of // methods/fields compiled in we need to go back to interpreter whenever we hit // them. - visitor.caller->GetDeclaringClass()->IsObsoleteObject() || - Dbg::IsForcedInterpreterNeededForUpcall(self, visitor.caller) || - should_deoptimize_frame); + method->GetDeclaringClass()->IsObsoleteObject() || + Dbg::IsForcedInterpreterNeededForUpcall(self, method)); +} + +bool Instrumentation::ShouldDeoptimizeCaller(Thread* self, ArtMethod** sp) { + ArtMethod* runtime_method = *sp; + DCHECK(runtime_method->IsRuntimeMethod()); + QuickMethodFrameInfo frame_info = Runtime::Current()->GetRuntimeMethodFrameInfo(runtime_method); + + uintptr_t caller_sp = reinterpret_cast<uintptr_t>(sp) + frame_info.FrameSizeInBytes(); + ArtMethod* caller = *(reinterpret_cast<ArtMethod**>(caller_sp)); + uintptr_t caller_pc_addr = reinterpret_cast<uintptr_t>(sp) + frame_info.GetReturnPcOffset(); + uintptr_t caller_pc = *reinterpret_cast<uintptr_t*>(caller_pc_addr); + + if (caller == nullptr || + caller->IsNative() || + caller_pc == reinterpret_cast<uintptr_t>(GetQuickInstrumentationExitPc())) { + // If caller_pc is QuickInstrumentationExit then deoptimization will be handled by the + // instrumentation exit trampoline so we don't need to handle deoptimizations here. + // We need to check for a deoptimization here because when a redefinition happens it is + // not safe to use any compiled code because the field offsets might change. For native + // methods, we don't embed any field offsets so no need to check for a deoptimization. + // If the caller is null we don't need to do anything. This can happen when the caller + // is being interpreted by the switch interpreter (when called from + // artQuickToInterpreterBridge) / during shutdown / early startup. + return false; + } + + if (!Runtime::Current()->IsAsyncDeoptimizeable(caller_pc)) { + LOG(WARNING) << "Got a deoptimization request on un-deoptimizable method " + << caller->PrettyMethod(); + return false; + } + + if (NeedsSlowInterpreterForMethod(self, caller)) { + return true; + } + + // Non java debuggable apps don't support redefinition and hence it isn't required to check if + // frame needs to be deoptimized. + if (!Runtime::Current()->IsJavaDebuggable()) { + return false; + } + + bool should_deoptimize_frame = false; + const OatQuickMethodHeader* header = caller->GetOatQuickMethodHeader(caller_pc); + if (header != nullptr && header->HasShouldDeoptimizeFlag()) { + DCHECK(header->IsOptimized()); + uint8_t* should_deopt_flag_addr = + reinterpret_cast<uint8_t*>(caller_sp) + header->GetShouldDeoptimizeFlagOffset(); + if ((*should_deopt_flag_addr & static_cast<uint8_t>(DeoptimizeFlagValue::kDebug)) != 0) { + should_deoptimize_frame = true; + } + } + return should_deoptimize_frame; +} + +bool Instrumentation::ShouldDeoptimizeCaller(Thread* self, const NthCallerVisitor& visitor) { + bool should_deoptimize_frame = false; + if (visitor.caller != nullptr) { + const OatQuickMethodHeader* header = visitor.GetCurrentOatQuickMethodHeader(); + if (header != nullptr && header->HasShouldDeoptimizeFlag()) { + uint8_t should_deopt_flag = visitor.GetShouldDeoptimizeFlag(); + // DeoptimizeFlag could be set for debugging or for CHA invalidations. + // Deoptimize here only if it was requested for debugging. CHA + // invalidations are handled in the JITed code. + if ((should_deopt_flag & static_cast<uint8_t>(DeoptimizeFlagValue::kDebug)) != 0) { + should_deoptimize_frame = true; + } + } + } + return NeedsSlowInterpreterForMethod(self, visitor.caller) || should_deoptimize_frame; } TwoWordReturn Instrumentation::PopInstrumentationStackFrame(Thread* self, @@ -1656,19 +1700,19 @@ TwoWordReturn Instrumentation::PopInstrumentationStackFrame(Thread* self, self->VerifyStack(); ArtMethod* method = instrumentation_frame.method_; + DCHECK(!method->IsRuntimeMethod()); bool is_ref; - JValue return_value = GetReturnValue(self, method, &is_ref, gpr_result, fpr_result); + JValue return_value = GetReturnValue(method, &is_ref, gpr_result, fpr_result); StackHandleScope<1> hs(self); MutableHandle<mirror::Object> res(hs.NewHandle<mirror::Object>(nullptr)); if (is_ref) { // Take a handle to the return value so we won't lose it if we suspend. - // FIXME: The `is_ref` is often guessed wrong, so even object aligment - // assertion would fail for some tests. See b/204766614 . - // DCHECK_ALIGNED(return_value.GetL(), kObjectAlignment); + DCHECK_ALIGNED(return_value.GetL(), kObjectAlignment); res.Assign(return_value.GetL()); } - if (!method->IsRuntimeMethod() && !instrumentation_frame.interpreter_entry_) { + if (!instrumentation_frame.interpreter_entry_) { + DCHECK(!method->IsRuntimeMethod()); // Note that sending the event may change the contents of *return_pc_addr. MethodExitEvent(self, instrumentation_frame.method_, OptionalFrame{}, return_value); } @@ -1680,7 +1724,7 @@ TwoWordReturn Instrumentation::PopInstrumentationStackFrame(Thread* self, // Check if we forced all threads to deoptimize in the time between this frame being created and // now. bool should_deoptimize_frame = instrumentation_frame.force_deopt_id_ != current_force_deopt_id_; - bool deoptimize = ShouldDeoptimizeMethod(self, visitor) || should_deoptimize_frame; + bool deoptimize = ShouldDeoptimizeCaller(self, visitor) || should_deoptimize_frame; if (is_ref) { // Restore the return value if it's a reference since it might have moved. diff --git a/runtime/instrumentation.h b/runtime/instrumentation.h index b1631091ae..c63e73eeea 100644 --- a/runtime/instrumentation.h +++ b/runtime/instrumentation.h @@ -31,6 +31,7 @@ #include "base/macros.h" #include "base/safe_map.h" #include "gc_root.h" +#include "jvalue.h" #include "offsets.h" namespace art { @@ -479,12 +480,29 @@ class Instrumentation { void ExceptionHandledEvent(Thread* thread, ObjPtr<mirror::Throwable> exception_object) const REQUIRES_SHARED(Locks::mutator_lock_); - JValue GetReturnValue(Thread* self, - ArtMethod* method, - bool* is_ref, - uint64_t* gpr_result, - uint64_t* fpr_result) REQUIRES_SHARED(Locks::mutator_lock_); - bool ShouldDeoptimizeMethod(Thread* self, const NthCallerVisitor& visitor) + JValue GetReturnValue(ArtMethod* method, bool* is_ref, uint64_t* gpr_result, uint64_t* fpr_result) + REQUIRES_SHARED(Locks::mutator_lock_); + bool PushDeoptContextIfNeeded(Thread* self, + DeoptimizationMethodType deopt_type, + bool is_ref, + const JValue& result) REQUIRES_SHARED(Locks::mutator_lock_); + void DeoptimizeIfNeeded(Thread* self, + ArtMethod** sp, + DeoptimizationMethodType type, + JValue result, + bool is_ref) REQUIRES_SHARED(Locks::mutator_lock_); + // TODO(mythria): Update uses of ShouldDeoptimizeCaller that takes a visitor by a method that + // doesn't need to walk the stack. This is used on method exits to check if the caller needs a + // deoptimization. + bool ShouldDeoptimizeCaller(Thread* self, const NthCallerVisitor& visitor) + REQUIRES_SHARED(Locks::mutator_lock_); + // This returns if the caller of runtime method requires a deoptimization. This checks both if the + // method requires a deopt or if this particular frame needs a deopt because of a class + // redefinition. + bool ShouldDeoptimizeCaller(Thread* self, ArtMethod** sp) REQUIRES_SHARED(Locks::mutator_lock_); + // This returns if the specified method requires a deoptimization. This doesn't account if a stack + // frame involving this method requires a deoptimization. + bool NeedsSlowInterpreterForMethod(Thread* self, ArtMethod* method) REQUIRES_SHARED(Locks::mutator_lock_); // Called when an instrumented method is entered. The intended link register (lr) is saved so diff --git a/runtime/interpreter/interpreter.cc b/runtime/interpreter/interpreter.cc index 38c94abf06..dfb9f3e4f2 100644 --- a/runtime/interpreter/interpreter.cc +++ b/runtime/interpreter/interpreter.cc @@ -476,6 +476,7 @@ void EnterInterpreterFromDeoptimize(Thread* self, const uint32_t dex_pc = shadow_frame->GetDexPC(); uint32_t new_dex_pc = dex_pc; if (UNLIKELY(self->IsExceptionPending())) { + DCHECK(self->GetException() != Thread::GetDeoptimizationException()); // If we deoptimize from the QuickExceptionHandler, we already reported the exception throw // event to the instrumentation. Skip throw listeners for the first frame. The deopt check // should happen after the throw listener is called as throw listener can trigger a diff --git a/runtime/oat.h b/runtime/oat.h index 462d41cdf0..14b389d670 100644 --- a/runtime/oat.h +++ b/runtime/oat.h @@ -32,8 +32,8 @@ class InstructionSetFeatures; class PACKED(4) OatHeader { public: static constexpr std::array<uint8_t, 4> kOatMagic { { 'o', 'a', 't', '\n' } }; - // Last oat version changed reason: Revert^4 "bss support for inlining BCP into non-BCP". - static constexpr std::array<uint8_t, 4> kOatVersion { { '2', '2', '5', '\0' } }; + // Last oat version changed reason: Update deoptimization from runtime methods. + static constexpr std::array<uint8_t, 4> kOatVersion { { '2', '2', '6', '\0' } }; static constexpr const char* kDex2OatCmdLineKey = "dex2oat-cmdline"; static constexpr const char* kDebuggableKey = "debuggable"; diff --git a/runtime/oat_quick_method_header.h b/runtime/oat_quick_method_header.h index e347588f3e..0bbf23fc27 100644 --- a/runtime/oat_quick_method_header.h +++ b/runtime/oat_quick_method_header.h @@ -28,6 +28,11 @@ namespace art { class ArtMethod; +// Size in bytes of the should_deoptimize flag on stack. +// We just need 4 bytes for our purpose regardless of the architecture. Frame size +// calculation will automatically do alignment for the final frame size. +static constexpr size_t kShouldDeoptimizeFlagSize = 4; + // OatQuickMethodHeader precedes the raw code chunk generated by the compiler. class PACKED(4) OatQuickMethodHeader { public: @@ -145,6 +150,17 @@ class PACKED(4) OatQuickMethodHeader { return CodeInfo::DecodeFrameInfo(GetOptimizedCodeInfoPtr()); } + size_t GetShouldDeoptimizeFlagOffset() const { + DCHECK(IsOptimized()); + QuickMethodFrameInfo frame_info = GetFrameInfo(); + size_t frame_size = frame_info.FrameSizeInBytes(); + size_t core_spill_size = + POPCOUNT(frame_info.CoreSpillMask()) * GetBytesPerGprSpillLocation(kRuntimeISA); + size_t fpu_spill_size = + POPCOUNT(frame_info.FpSpillMask()) * GetBytesPerFprSpillLocation(kRuntimeISA); + return frame_size - core_spill_size - fpu_spill_size - kShouldDeoptimizeFlagSize; + } + uintptr_t ToNativeQuickPc(ArtMethod* method, const uint32_t dex_pc, bool is_for_catch_handler, diff --git a/runtime/stack.h b/runtime/stack.h index 1b00b54acb..bfda57b136 100644 --- a/runtime/stack.h +++ b/runtime/stack.h @@ -58,11 +58,6 @@ enum VRegKind { }; std::ostream& operator<<(std::ostream& os, VRegKind rhs); -// Size in bytes of the should_deoptimize flag on stack. -// We just need 4 bytes for our purpose regardless of the architecture. Frame size -// calculation will automatically do alignment for the final frame size. -static constexpr size_t kShouldDeoptimizeFlagSize = 4; - /* * Our current stack layout. * The Dalvik registers come first, followed by the @@ -306,6 +301,11 @@ class StackVisitor { return *GetShouldDeoptimizeFlagAddr(); } + bool IsShouldDeoptimizeFlagForDebugSet() const REQUIRES_SHARED(Locks::mutator_lock_) { + uint8_t should_deopt_flag = GetShouldDeoptimizeFlag(); + return (should_deopt_flag & static_cast<uint8_t>(DeoptimizeFlagValue::kDebug)) != 0; + } + private: // Private constructor known in the case that num_frames_ has already been computed. StackVisitor(Thread* thread, diff --git a/runtime/thread.cc b/runtime/thread.cc index 78ba26dec0..97cfb7a30c 100644 --- a/runtime/thread.cc +++ b/runtime/thread.cc @@ -272,6 +272,7 @@ void Thread::PushDeoptimizationContext(const JValue& return_value, ObjPtr<mirror::Throwable> exception, bool from_code, DeoptimizationMethodType method_type) { + DCHECK(exception != Thread::GetDeoptimizationException()); DeoptimizationContextRecord* record = new DeoptimizationContextRecord( return_value, is_reference, @@ -3696,6 +3697,9 @@ void Thread::QuickDeliverException() { ObjPtr<mirror::Throwable> exception = GetException(); CHECK(exception != nullptr); if (exception == GetDeoptimizationException()) { + // This wasn't a real exception, so just clear it here. If there was an actual exception it + // will be recorded in the DeoptimizationContext and it will be restored later. + ClearException(); artDeoptimize(this); UNREACHABLE(); } diff --git a/runtime/thread.h b/runtime/thread.h index dd8b061b95..b32e3c2637 100644 --- a/runtime/thread.h +++ b/runtime/thread.h @@ -766,6 +766,13 @@ class Thread { OFFSETOF_MEMBER(tls_32bit_sized_values, is_gc_marking)); } + template <PointerSize pointer_size> + static constexpr ThreadOffset<pointer_size> DeoptCheckRequiredOffset() { + return ThreadOffset<pointer_size>( + OFFSETOF_MEMBER(Thread, tls32_) + + OFFSETOF_MEMBER(tls_32bit_sized_values, is_deopt_check_required)); + } + static constexpr size_t IsGcMarkingSize() { return sizeof(tls32_.is_gc_marking); } @@ -1017,6 +1024,10 @@ class Thread { void SetIsGcMarkingAndUpdateEntrypoints(bool is_marking); + bool IsDeoptCheckRequired() const { return tls32_.is_deopt_check_required; } + + void SetDeoptCheckRequired(bool flag) { tls32_.is_deopt_check_required = flag; } + bool GetWeakRefAccessEnabled() const; // Only safe for current thread. void SetWeakRefAccessEnabled(bool enabled) { @@ -1712,6 +1723,7 @@ class Thread { thread_exit_check_count(0), is_transitioning_to_runnable(false), is_gc_marking(false), + is_deopt_check_required(false), weak_ref_access_enabled(WeakRefAccessState::kVisiblyEnabled), disable_thread_flip_count(0), user_code_suspend_count(0), @@ -1766,6 +1778,12 @@ class Thread { // GC roots. bool32_t is_gc_marking; + // True if we need to check for deoptimization when returning from the runtime functions. This + // is required only when a class is redefined to prevent executing code that has field offsets + // embedded. For non-debuggable apps redefinition is not allowed and this flag should always be + // set to false. + bool32_t is_deopt_check_required; + // Thread "interrupted" status; stays raised until queried or thrown. Atomic<bool32_t> interrupted; diff --git a/tools/cpp-define-generator/globals.def b/tools/cpp-define-generator/globals.def index 2572ea6f9b..8dcc023cee 100644 --- a/tools/cpp-define-generator/globals.def +++ b/tools/cpp-define-generator/globals.def @@ -28,6 +28,7 @@ #include "mirror/object_reference.h" #include "runtime_globals.h" #include "stack.h" +#include "entrypoints/quick/callee_save_frame.h" #endif ASM_DEFINE(ACCESS_FLAGS_METHOD_IS_NATIVE, @@ -82,3 +83,6 @@ ASM_DEFINE(STD_MEMORY_ORDER_RELAXED, std::memory_order_relaxed) ASM_DEFINE(STACK_OVERFLOW_RESERVED_BYTES, GetStackOverflowReservedBytes(art::kRuntimeISA)) +ASM_DEFINE(CALLEE_SAVE_EVERYTHING_NUM_CORE_SPILLS, + art::POPCOUNT(art::RuntimeCalleeSaveFrame::GetCoreSpills( + art::CalleeSaveType::kSaveEverything))) diff --git a/tools/cpp-define-generator/thread.def b/tools/cpp-define-generator/thread.def index bae92009b2..d796542a32 100644 --- a/tools/cpp-define-generator/thread.def +++ b/tools/cpp-define-generator/thread.def @@ -37,6 +37,8 @@ ASM_DEFINE(THREAD_INTERPRETER_CACHE_SIZE_SHIFT, (art::WhichPowerOf2(sizeof(art::InterpreterCache::Entry)) - 2)) ASM_DEFINE(THREAD_IS_GC_MARKING_OFFSET, art::Thread::IsGcMarkingOffset<art::kRuntimePointerSize>().Int32Value()) +ASM_DEFINE(THREAD_DEOPT_CHECK_REQUIRED_OFFSET, + art::Thread::DeoptCheckRequiredOffset<art::kRuntimePointerSize>().Int32Value()) ASM_DEFINE(THREAD_LOCAL_ALLOC_STACK_END_OFFSET, art::Thread::ThreadLocalAllocStackEndOffset<art::kRuntimePointerSize>().Int32Value()) ASM_DEFINE(THREAD_LOCAL_ALLOC_STACK_TOP_OFFSET, |