diff options
26 files changed, 1013 insertions, 485 deletions
diff --git a/runtime/instrumentation.h b/runtime/instrumentation.h index 56aeefc2f5..e3cbf53873 100644 --- a/runtime/instrumentation.h +++ b/runtime/instrumentation.h @@ -290,6 +290,14 @@ class Instrumentation { bool IsActive() const SHARED_REQUIRES(Locks::mutator_lock_) { return have_dex_pc_listeners_ || have_method_entry_listeners_ || have_method_exit_listeners_ || have_field_read_listeners_ || have_field_write_listeners_ || + have_exception_caught_listeners_ || have_method_unwind_listeners_ || + have_branch_listeners_ || have_invoke_virtual_or_interface_listeners_; + } + + // Any instrumentation *other* than what is needed for Jit profiling active? + bool NonJitProfilingActive() const SHARED_REQUIRES(Locks::mutator_lock_) { + return have_dex_pc_listeners_ || have_method_exit_listeners_ || + have_field_read_listeners_ || have_field_write_listeners_ || have_exception_caught_listeners_ || have_method_unwind_listeners_; } diff --git a/runtime/interpreter/interpreter.cc b/runtime/interpreter/interpreter.cc index 4fd3c78f44..a595d33f04 100644 --- a/runtime/interpreter/interpreter.cc +++ b/runtime/interpreter/interpreter.cc @@ -320,12 +320,13 @@ static inline JValue Execute(Thread* self, const DexFile::CodeItem* code_item, // No Mterp variant - just use the switch interpreter. return ExecuteSwitchImpl<false, true>(self, code_item, shadow_frame, result_register, false); + } else if (UNLIKELY(!Runtime::Current()->IsStarted())) { + return ExecuteSwitchImpl<false, false>(self, code_item, shadow_frame, result_register, + false); } else { - const instrumentation::Instrumentation* const instrumentation = - Runtime::Current()->GetInstrumentation(); while (true) { - if (instrumentation->IsActive() || !Runtime::Current()->IsStarted()) { - // TODO: allow JIT profiling instrumentation. Now, just punt on all instrumentation. + // Mterp does not support all instrumentation/debugging. + if (MterpShouldSwitchInterpreters()) { #if !defined(__clang__) return ExecuteGotoImpl<false, false>(self, code_item, shadow_frame, result_register); #else diff --git a/runtime/interpreter/mterp/arm/bincmp.S b/runtime/interpreter/mterp/arm/bincmp.S index 474bc3c276..774e1676b7 100644 --- a/runtime/interpreter/mterp/arm/bincmp.S +++ b/runtime/interpreter/mterp/arm/bincmp.S @@ -6,17 +6,29 @@ * For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le */ /* if-cmp vA, vB, +CCCC */ -#if MTERP_SUSPEND +#if MTERP_PROFILE_BRANCHES mov r1, rINST, lsr #12 @ r1<- B ubfx r0, rINST, #8, #4 @ r0<- A GET_VREG r3, r1 @ r3<- vB GET_VREG r2, r0 @ r2<- vA - FETCH_S r1, 1 @ r1<- branch offset, in code units + FETCH_S rINST, 1 @ rINST<- branch offset, in code units cmp r2, r3 @ compare (vA, vB) - mov${revcmp} r1, #2 @ r1<- BYTE branch dist for not-taken - adds r2, r1, r1 @ convert to bytes, check sign + b${revcmp} .L_${opcode}_not_taken + EXPORT_PC + mov r0, rSELF + add r1, rFP, #OFF_FP_SHADOWFRAME + mov r2, rINST + bl MterpProfileBranch @ (self, shadow_frame, offset) + cmp r0, #0 + bne MterpOnStackReplacement @ Note: offset must be in rINST + adds r2, rINST, rINST @ convert to bytes, check sign + ldr lr, [rSELF, #THREAD_FLAGS_OFFSET] FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST - ldrmi rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh rIBASE + bmi MterpCheckSuspendAndContinue + GET_INST_OPCODE ip @ extract opcode from rINST + GOTO_OPCODE ip @ jump to next instruction +.L_${opcode}_not_taken: + FETCH_ADVANCE_INST 2 @ update rPC, load rINST GET_INST_OPCODE ip @ extract opcode from rINST GOTO_OPCODE ip @ jump to next instruction #else @@ -25,10 +37,10 @@ GET_VREG r3, r1 @ r3<- vB GET_VREG r2, r0 @ r2<- vA ldr lr, [rSELF, #THREAD_FLAGS_OFFSET] - FETCH_S r1, 1 @ r1<- branch offset, in code units + FETCH_S rINST, 1 @ rINST<- branch offset, in code units cmp r2, r3 @ compare (vA, vB) - mov${revcmp} r1, #2 @ r1<- BYTE branch dist for not-taken - adds r2, r1, r1 @ convert to bytes, check sign + mov${revcmp} rINST, #2 @ rINST<- BYTE branch dist for not-taken + adds r2, rINST, rINST @ convert to bytes, check sign FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST bmi MterpCheckSuspendAndContinue GET_INST_OPCODE ip @ extract opcode from rINST diff --git a/runtime/interpreter/mterp/arm/footer.S b/runtime/interpreter/mterp/arm/footer.S index 1dba856ecb..3456a7559b 100644 --- a/runtime/interpreter/mterp/arm/footer.S +++ b/runtime/interpreter/mterp/arm/footer.S @@ -12,7 +12,6 @@ * has not yet been thrown. Just bail out to the reference interpreter to deal with it. * TUNING: for consistency, we may want to just go ahead and handle these here. */ -#define MTERP_LOGGING 0 common_errDivideByZero: EXPORT_PC #if MTERP_LOGGING @@ -103,8 +102,12 @@ MterpException: ldr rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] add rPC, r0, #CODEITEM_INSNS_OFFSET add rPC, rPC, r1, lsl #1 @ generate new dex_pc_ptr - str rPC, [rFP, #OFF_FP_DEX_PC_PTR] + /* Do we need to switch interpreters? */ + bl MterpShouldSwitchInterpreters + cmp r0, #0 + bne MterpFallback /* resume execution at catch block */ + EXPORT_PC FETCH_INST GET_INST_OPCODE ip GOTO_OPCODE ip @@ -116,14 +119,33 @@ MterpException: */ MterpCheckSuspendAndContinue: ldr rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh rIBASE + ands lr, #(THREAD_SUSPEND_REQUEST | THREAD_CHECKPOINT_REQUEST) + bne 1f + GET_INST_OPCODE ip @ extract opcode from rINST + GOTO_OPCODE ip @ jump to next instruction +1: EXPORT_PC mov r0, rSELF - ands lr, #(THREAD_SUSPEND_REQUEST | THREAD_CHECKPOINT_REQUEST) - blne MterpSuspendCheck @ (self) + bl MterpSuspendCheck @ (self) + cmp r0, #0 + bne MterpFallback GET_INST_OPCODE ip @ extract opcode from rINST GOTO_OPCODE ip @ jump to next instruction /* + * On-stack replacement has happened, and now we've returned from the compiled method. + */ +MterpOnStackReplacement: +#if MTERP_LOGGING + mov r0, rSELF + add r1, rFP, #OFF_FP_SHADOWFRAME + mov r2, rINST + bl MterpLogOSR +#endif + mov r0, #1 @ Signal normal return + b MterpDone + +/* * Bail out to reference interpreter. */ MterpFallback: diff --git a/runtime/interpreter/mterp/arm/header.S b/runtime/interpreter/mterp/arm/header.S index b2370bffb4..298af8a57e 100644 --- a/runtime/interpreter/mterp/arm/header.S +++ b/runtime/interpreter/mterp/arm/header.S @@ -85,6 +85,9 @@ unspecified registers or condition codes. */ #include "asm_support.h" +#define MTERP_PROFILE_BRANCHES 1 +#define MTERP_LOGGING 0 + /* During bringup, we'll use the shadow frame model instead of rFP */ /* single-purpose registers, given names for clarity */ #define rPC r4 @@ -109,14 +112,6 @@ unspecified registers or condition codes. #define OFF_FP_SHADOWFRAME (-SHADOWFRAME_VREGS_OFFSET) /* - * - * The reference interpreter performs explicit suspect checks, which is somewhat wasteful. - * Dalvik's interpreter folded suspend checks into the jump table mechanism, and eventually - * mterp should do so as well. - */ -#define MTERP_SUSPEND 0 - -/* * "export" the PC to dex_pc field in the shadow frame, f/b/o future exception objects. Must * be done *before* something throws. * diff --git a/runtime/interpreter/mterp/arm/invoke.S b/runtime/interpreter/mterp/arm/invoke.S index 7575865f1b..e47dd1b3ca 100644 --- a/runtime/interpreter/mterp/arm/invoke.S +++ b/runtime/interpreter/mterp/arm/invoke.S @@ -14,6 +14,9 @@ cmp r0, #0 beq MterpException FETCH_ADVANCE_INST 3 + bl MterpShouldSwitchInterpreters + cmp r0, #0 + bne MterpFallback GET_INST_OPCODE ip GOTO_OPCODE ip diff --git a/runtime/interpreter/mterp/arm/op_goto.S b/runtime/interpreter/mterp/arm/op_goto.S index 9b3632aba2..eb1d42992b 100644 --- a/runtime/interpreter/mterp/arm/op_goto.S +++ b/runtime/interpreter/mterp/arm/op_goto.S @@ -6,20 +6,28 @@ */ /* goto +AA */ /* tuning: use sbfx for 6t2+ targets */ -#if MTERP_SUSPEND +#if MTERP_PROFILE_BRANCHES mov r0, rINST, lsl #16 @ r0<- AAxx0000 - movs r1, r0, asr #24 @ r1<- ssssssAA (sign-extended) - add r2, r1, r1 @ r2<- byte offset, set flags - @ If backwards branch refresh rIBASE - ldrmi rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh handler base + movs rINST, r0, asr #24 @ rINST<- ssssssAA (sign-extended) + EXPORT_PC + mov r0, rSELF + add r1, rFP, #OFF_FP_SHADOWFRAME + mov r2, rINST + bl MterpProfileBranch @ (self, shadow_frame, offset) + cmp r0, #0 + bne MterpOnStackReplacement @ Note: offset must be in rINST + ldr lr, [rSELF, #THREAD_FLAGS_OFFSET] + add r2, rINST, rINST @ r2<- byte offset, set flags FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST + @ If backwards branch refresh rIBASE + bmi MterpCheckSuspendAndContinue GET_INST_OPCODE ip @ extract opcode from rINST GOTO_OPCODE ip @ jump to next instruction #else - ldr lr, [rSELF, #THREAD_FLAGS_OFFSET] mov r0, rINST, lsl #16 @ r0<- AAxx0000 - movs r1, r0, asr #24 @ r1<- ssssssAA (sign-extended) - add r2, r1, r1 @ r2<- byte offset, set flags + movs rINST, r0, asr #24 @ rINST<- ssssssAA (sign-extended) + ldr lr, [rSELF, #THREAD_FLAGS_OFFSET] + add r2, rINST, rINST @ r2<- byte offset, set flags FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST @ If backwards branch refresh rIBASE bmi MterpCheckSuspendAndContinue diff --git a/runtime/interpreter/mterp/arm/op_goto_16.S b/runtime/interpreter/mterp/arm/op_goto_16.S index 2231acdb9e..91639ca796 100644 --- a/runtime/interpreter/mterp/arm/op_goto_16.S +++ b/runtime/interpreter/mterp/arm/op_goto_16.S @@ -5,17 +5,25 @@ * double to get a byte offset. */ /* goto/16 +AAAA */ -#if MTERP_SUSPEND - FETCH_S r0, 1 @ r0<- ssssAAAA (sign-extended) - adds r1, r0, r0 @ r1<- byte offset, flags set +#if MTERP_PROFILE_BRANCHES + FETCH_S rINST, 1 @ rINST<- ssssAAAA (sign-extended) + EXPORT_PC + mov r0, rSELF + add r1, rFP, #OFF_FP_SHADOWFRAME + mov r2, rINST + bl MterpProfileBranch @ (self, shadow_frame, offset) + cmp r0, #0 + bne MterpOnStackReplacement @ Note: offset must be in rINST + ldr lr, [rSELF, #THREAD_FLAGS_OFFSET] + adds r1, rINST, rINST @ r1<- byte offset, flags set FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST - ldrmi rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh handler base + bmi MterpCheckSuspendAndContinue GET_INST_OPCODE ip @ extract opcode from rINST GOTO_OPCODE ip @ jump to next instruction #else - FETCH_S r0, 1 @ r0<- ssssAAAA (sign-extended) + FETCH_S rINST, 1 @ rINST<- ssssAAAA (sign-extended) ldr lr, [rSELF, #THREAD_FLAGS_OFFSET] - adds r1, r0, r0 @ r1<- byte offset, flags set + adds r1, rINST, rINST @ r1<- byte offset, flags set FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST bmi MterpCheckSuspendAndContinue GET_INST_OPCODE ip @ extract opcode from rINST diff --git a/runtime/interpreter/mterp/arm/op_goto_32.S b/runtime/interpreter/mterp/arm/op_goto_32.S index 6b72ff5ce2..e730b527ec 100644 --- a/runtime/interpreter/mterp/arm/op_goto_32.S +++ b/runtime/interpreter/mterp/arm/op_goto_32.S @@ -10,21 +10,29 @@ * offset to byte offset. */ /* goto/32 +AAAAAAAA */ -#if MTERP_SUSPEND +#if MTERP_PROFILE_BRANCHES FETCH r0, 1 @ r0<- aaaa (lo) FETCH r1, 2 @ r1<- AAAA (hi) - orr r0, r0, r1, lsl #16 @ r0<- AAAAaaaa - adds r1, r0, r0 @ r1<- byte offset + orr rINST, r0, r1, lsl #16 @ rINST<- AAAAaaaa + EXPORT_PC + mov r0, rSELF + add r1, rFP, #OFF_FP_SHADOWFRAME + mov r2, rINST + bl MterpProfileBranch @ (self, shadow_frame, offset) + cmp r0, #0 + bne MterpOnStackReplacement @ Note: offset must be in rINST + ldr lr, [rSELF, #THREAD_FLAGS_OFFSET] + adds r1, rINST, rINST @ r1<- byte offset FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST - ldrle rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh handler base + ble MterpCheckSuspendAndContinue GET_INST_OPCODE ip @ extract opcode from rINST GOTO_OPCODE ip @ jump to next instruction #else FETCH r0, 1 @ r0<- aaaa (lo) FETCH r1, 2 @ r1<- AAAA (hi) + orr rINST, r0, r1, lsl #16 @ rINST<- AAAAaaaa ldr lr, [rSELF, #THREAD_FLAGS_OFFSET] - orr r0, r0, r1, lsl #16 @ r0<- AAAAaaaa - adds r1, r0, r0 @ r1<- byte offset + adds r1, rINST, rINST @ r1<- byte offset FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST ble MterpCheckSuspendAndContinue GET_INST_OPCODE ip @ extract opcode from rINST diff --git a/runtime/interpreter/mterp/arm/op_packed_switch.S b/runtime/interpreter/mterp/arm/op_packed_switch.S index 1e3370ea6a..4c369cb136 100644 --- a/runtime/interpreter/mterp/arm/op_packed_switch.S +++ b/runtime/interpreter/mterp/arm/op_packed_switch.S @@ -9,7 +9,7 @@ * for: packed-switch, sparse-switch */ /* op vAA, +BBBB */ -#if MTERP_SUSPEND +#if MTERP_PROFILE_BRANCHES FETCH r0, 1 @ r0<- bbbb (lo) FETCH r1, 2 @ r1<- BBBB (hi) mov r3, rINST, lsr #8 @ r3<- AA @@ -17,9 +17,18 @@ GET_VREG r1, r3 @ r1<- vAA add r0, rPC, r0, lsl #1 @ r0<- PC + BBBBbbbb*2 bl $func @ r0<- code-unit branch offset - adds r1, r0, r0 @ r1<- byte offset; clear V - ldrle rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh handler base + mov rINST, r0 + EXPORT_PC + mov r0, rSELF + add r1, rFP, #OFF_FP_SHADOWFRAME + mov r2, rINST + bl MterpProfileBranch @ (self, shadow_frame, offset) + cmp r0, #0 + bne MterpOnStackReplacement @ Note: offset must be in rINST + ldr lr, [rSELF, #THREAD_FLAGS_OFFSET] + adds r1, rINST, rINST @ r1<- byte offset; clear V FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST + ble MterpCheckSuspendAndContinue GET_INST_OPCODE ip @ extract opcode from rINST GOTO_OPCODE ip @ jump to next instruction #else @@ -30,8 +39,9 @@ GET_VREG r1, r3 @ r1<- vAA add r0, rPC, r0, lsl #1 @ r0<- PC + BBBBbbbb*2 bl $func @ r0<- code-unit branch offset + mov rINST, r0 ldr lr, [rSELF, #THREAD_FLAGS_OFFSET] - adds r1, r0, r0 @ r1<- byte offset; clear V + adds r1, rINST, rINST @ r1<- byte offset; clear V FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST ble MterpCheckSuspendAndContinue GET_INST_OPCODE ip @ extract opcode from rINST diff --git a/runtime/interpreter/mterp/arm/zcmp.S b/runtime/interpreter/mterp/arm/zcmp.S index 6e9ef55104..800804d95e 100644 --- a/runtime/interpreter/mterp/arm/zcmp.S +++ b/runtime/interpreter/mterp/arm/zcmp.S @@ -6,25 +6,37 @@ * for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez */ /* if-cmp vAA, +BBBB */ -#if MTERP_SUSPEND +#if MTERP_PROFILE_BRANCHES mov r0, rINST, lsr #8 @ r0<- AA GET_VREG r2, r0 @ r2<- vAA - FETCH_S r1, 1 @ r1<- branch offset, in code units + FETCH_S rINST, 1 @ rINST<- branch offset, in code units + ldr lr, [rSELF, #THREAD_FLAGS_OFFSET] cmp r2, #0 @ compare (vA, 0) - mov${revcmp} r1, #2 @ r1<- inst branch dist for not-taken - adds r1, r1, r1 @ convert to bytes & set flags + b${revcmp} .L_${opcode}_not_taken + EXPORT_PC + mov r0, rSELF + add r1, rFP, #OFF_FP_SHADOWFRAME + mov r2, rINST + bl MterpProfileBranch @ (self, shadow_frame, offset) + cmp r0, #0 + bne MterpOnStackReplacement @ Note: offset must be in rINST + adds r1, rINST, rINST @ convert to bytes & set flags FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST - ldrmi rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh table base + bmi MterpCheckSuspendAndContinue + GET_INST_OPCODE ip @ extract opcode from rINST + GOTO_OPCODE ip @ jump to next instruction +.L_${opcode}_not_taken: + FETCH_ADVANCE_INST 2 @ update rPC, load rINST GET_INST_OPCODE ip @ extract opcode from rINST GOTO_OPCODE ip @ jump to next instruction #else mov r0, rINST, lsr #8 @ r0<- AA GET_VREG r2, r0 @ r2<- vAA - FETCH_S r1, 1 @ r1<- branch offset, in code units + FETCH_S rINST, 1 @ rINST<- branch offset, in code units ldr lr, [rSELF, #THREAD_FLAGS_OFFSET] cmp r2, #0 @ compare (vA, 0) - mov${revcmp} r1, #2 @ r1<- inst branch dist for not-taken - adds r1, r1, r1 @ convert to bytes & set flags + mov${revcmp} rINST, #2 @ rINST<- inst branch dist for not-taken + adds r1, rINST, rINST @ convert to bytes & set flags FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST bmi MterpCheckSuspendAndContinue GET_INST_OPCODE ip @ extract opcode from rINST diff --git a/runtime/interpreter/mterp/arm64/bincmp.S b/runtime/interpreter/mterp/arm64/bincmp.S index ecab2ceba7..ed850fc49d 100644 --- a/runtime/interpreter/mterp/arm64/bincmp.S +++ b/runtime/interpreter/mterp/arm64/bincmp.S @@ -6,17 +6,28 @@ * For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le */ /* if-cmp vA, vB, +CCCC */ -#if MTERP_SUSPEND - mov w1, wINST, lsr #12 // w1<- B +#if MTERP_PROFILE_BRANCHES + lsr w1, wINST, #12 // w1<- B ubfx w0, wINST, #8, #4 // w0<- A GET_VREG w3, w1 // w3<- vB GET_VREG w2, w0 // w2<- vA - FETCH_S w1, 1 // w1<- branch offset, in code units + FETCH_S wINST, 1 // wINST<- branch offset, in code units cmp w2, w3 // compare (vA, vB) - mov${condition} w1, #2 // w1<- BYTE branch dist for not-taken - adds w2, w1, w1 // convert to bytes, check sign + b.${condition} .L_${opcode}_taken + FETCH_ADVANCE_INST 2 // update rPC, load wINST + GET_INST_OPCODE ip // extract opcode from wINST + GOTO_OPCODE ip // jump to next instruction +.L_${opcode}_taken: + EXPORT_PC + mov x0, xSELF + add x1, xFP, #OFF_FP_SHADOWFRAME + sbfm x2, xINST, 0, 31 // Sign extend branch offset + bl MterpProfileBranch // (self, shadow_frame, offset) + cbnz w0, MterpOnStackReplacement // Note: offset must be in xINST + ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] + adds w2, wINST, wINST // convert to bytes, check sign FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST - ldrmi rIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET] // refresh rIBASE + b.mi MterpCheckSuspendAndContinue GET_INST_OPCODE ip // extract opcode from wINST GOTO_OPCODE ip // jump to next instruction #else @@ -25,11 +36,11 @@ GET_VREG w3, w1 // w3<- vB GET_VREG w2, w0 // w2<- vA FETCH_S w1, 1 // w1<- branch offset, in code units - ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] mov w0, #2 // Offset if branch not taken cmp w2, w3 // compare (vA, vB) - csel w1, w1, w0, ${condition} // Branch if true - adds w2, w1, w1 // convert to bytes, check sign + csel wINST, w1, w0, ${condition} // Branch if true, stashing result in callee save reg. + ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] + adds w2, wINST, wINST // convert to bytes, check sign FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST b.mi MterpCheckSuspendAndContinue GET_INST_OPCODE ip // extract opcode from wINST diff --git a/runtime/interpreter/mterp/arm64/footer.S b/runtime/interpreter/mterp/arm64/footer.S index b360539a8c..aae78de1b3 100644 --- a/runtime/interpreter/mterp/arm64/footer.S +++ b/runtime/interpreter/mterp/arm64/footer.S @@ -10,7 +10,6 @@ * has not yet been thrown. Just bail out to the reference interpreter to deal with it. * TUNING: for consistency, we may want to just go ahead and handle these here. */ -#define MTERP_LOGGING 0 common_errDivideByZero: EXPORT_PC #if MTERP_LOGGING @@ -99,8 +98,11 @@ MterpException: ldr xIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET] add xPC, x0, #CODEITEM_INSNS_OFFSET add xPC, xPC, x1, lsl #1 // generate new dex_pc_ptr - str xPC, [xFP, #OFF_FP_DEX_PC_PTR] + /* Do we need to switch interpreters? */ + bl MterpShouldSwitchInterpreters + cbnz w0, MterpFallback /* resume execution at catch block */ + EXPORT_PC FETCH_INST GET_INST_OPCODE ip GOTO_OPCODE ip @@ -120,10 +122,24 @@ check1: EXPORT_PC mov x0, xSELF bl MterpSuspendCheck // (self) + cbnz x0, MterpFallback // Something in the environment changed, switch interpreters GET_INST_OPCODE ip // extract opcode from wINST GOTO_OPCODE ip // jump to next instruction /* + * On-stack replacement has happened, and now we've returned from the compiled method. + */ +MterpOnStackReplacement: +#if MTERP_LOGGING + mov x0, xSELF + add x1, xFP, #OFF_FP_SHADOWFRAME + sbfm x2, xINST, 0, 31 + bl MterpLogOSR +#endif + mov x0, #1 // Signal normal return + b MterpDone + +/* * Bail out to reference interpreter. */ MterpFallback: diff --git a/runtime/interpreter/mterp/arm64/header.S b/runtime/interpreter/mterp/arm64/header.S index 351a6075cb..722375002b 100644 --- a/runtime/interpreter/mterp/arm64/header.S +++ b/runtime/interpreter/mterp/arm64/header.S @@ -87,6 +87,9 @@ codes. */ #include "asm_support.h" +#define MTERP_PROFILE_BRANCHES 1 +#define MTERP_LOGGING 0 + /* During bringup, we'll use the shadow frame model instead of xFP */ /* single-purpose registers, given names for clarity */ #define xPC x20 @@ -114,14 +117,6 @@ codes. #define OFF_FP_SHADOWFRAME (-SHADOWFRAME_VREGS_OFFSET) /* - * - * The reference interpreter performs explicit suspect checks, which is somewhat wasteful. - * Dalvik's interpreter folded suspend checks into the jump table mechanism, and eventually - * mterp should do so as well. - */ -#define MTERP_SUSPEND 0 - -/* * "export" the PC to dex_pc field in the shadow frame, f/b/o future exception objects. Must * be done *before* something throws. * diff --git a/runtime/interpreter/mterp/arm64/invoke.S b/runtime/interpreter/mterp/arm64/invoke.S index ff1974c51d..7a32df7bca 100644 --- a/runtime/interpreter/mterp/arm64/invoke.S +++ b/runtime/interpreter/mterp/arm64/invoke.S @@ -9,11 +9,12 @@ mov x0, xSELF add x1, xFP, #OFF_FP_SHADOWFRAME mov x2, xPC - // and x3, xINST, 0xFFFF mov x3, xINST bl $helper cbz w0, MterpException FETCH_ADVANCE_INST 3 + bl MterpShouldSwitchInterpreters + cbnz w0, MterpFallback GET_INST_OPCODE ip GOTO_OPCODE ip diff --git a/runtime/interpreter/mterp/arm64/op_goto.S b/runtime/interpreter/mterp/arm64/op_goto.S index db98a45fae..7e2f6a9c11 100644 --- a/runtime/interpreter/mterp/arm64/op_goto.S +++ b/runtime/interpreter/mterp/arm64/op_goto.S @@ -6,23 +6,20 @@ */ /* goto +AA */ /* tuning: use sbfx for 6t2+ targets */ -#if MTERP_SUSPEND - mov w0, wINST, lsl #16 // w0<- AAxx0000 - movs w1, w0, asr #24 // w1<- ssssssAA (sign-extended) - add w2, w1, w1 // w2<- byte offset, set flags - // If backwards branch refresh rIBASE - ldrmi rIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET] // refresh handler base - FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST - GET_INST_OPCODE ip // extract opcode from wINST - GOTO_OPCODE ip // jump to next instruction -#else - ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] // Preload flags for MterpCheckSuspendAndContinue lsl w0, wINST, #16 // w0<- AAxx0000 - asr w0, w0, #24 // w0<- ssssssAA (sign-extended) - adds w1, w0, w0 // Convert dalvik offset to byte offset, setting flags + asr wINST, w0, #24 // wINST<- ssssssAA (sign-extended) +#if MTERP_PROFILE_BRANCHES + EXPORT_PC + mov x0, xSELF + add x1, xFP, #OFF_FP_SHADOWFRAME + sbfm x2, xINST, 0, 31 + bl MterpProfileBranch // (self, shadow_frame, offset) + cbnz w0, MterpOnStackReplacement // Note: offset must be in wINST +#endif + ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] // Preload flags for MterpCheckSuspendAndContinue + adds w1, wINST, wINST // Convert dalvik offset to byte offset, setting flags FETCH_ADVANCE_INST_RB w1 // load wINST and advance xPC // If backwards branch refresh rIBASE b.mi MterpCheckSuspendAndContinue GET_INST_OPCODE ip // extract opcode from wINST GOTO_OPCODE ip // jump to next instruction -#endif diff --git a/runtime/interpreter/mterp/arm64/op_goto_16.S b/runtime/interpreter/mterp/arm64/op_goto_16.S index ff66a23c4e..b2b9924409 100644 --- a/runtime/interpreter/mterp/arm64/op_goto_16.S +++ b/runtime/interpreter/mterp/arm64/op_goto_16.S @@ -5,19 +5,18 @@ * double to get a byte offset. */ /* goto/16 +AAAA */ -#if MTERP_SUSPEND - FETCH_S w0, 1 // w0<- ssssAAAA (sign-extended) - adds w1, w0, w0 // w1<- byte offset, flags set - FETCH_ADVANCE_INST_RB w1 // update rPC, load rINST - ldrmi xIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET] // refresh handler base - GET_INST_OPCODE ip // extract opcode from rINST - GOTO_OPCODE ip // jump to next instruction -#else - FETCH_S w0, 1 // w0<- ssssAAAA (sign-extended) + FETCH_S wINST, 1 // wINST<- ssssAAAA (sign-extended) +#if MTERP_PROFILE_BRANCHES + EXPORT_PC + mov x0, xSELF + add x1, xFP, #OFF_FP_SHADOWFRAME + sbfm x2, xINST, 0, 31 + bl MterpProfileBranch // (self, shadow_frame, offset) + cbnz w0, MterpOnStackReplacement // Note: offset must be in xINST +#endif ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] - adds w1, w0, w0 // w1<- byte offset, flags set + adds w1, wINST, wINST // w1<- byte offset, flags set FETCH_ADVANCE_INST_RB w1 // update rPC, load rINST b.mi MterpCheckSuspendAndContinue GET_INST_OPCODE ip // extract opcode from rINST GOTO_OPCODE ip // jump to next instruction -#endif diff --git a/runtime/interpreter/mterp/arm64/op_goto_32.S b/runtime/interpreter/mterp/arm64/op_goto_32.S index 8a6980ecea..b785857b9b 100644 --- a/runtime/interpreter/mterp/arm64/op_goto_32.S +++ b/runtime/interpreter/mterp/arm64/op_goto_32.S @@ -10,23 +10,20 @@ * offset to byte offset. */ /* goto/32 +AAAAAAAA */ -#if MTERP_SUSPEND - FETCH w0, 1 // w0<- aaaa (lo) - FETCH w1, 2 // w1<- AAAA (hi) - orr w0, w0, w1, lsl #16 // w0<- AAAAaaaa - adds w1, w0, w0 // w1<- byte offset - FETCH_ADVANCE_INST_RB w1 // update rPC, load xINST - ldrle xIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET] // refresh handler base - GET_INST_OPCODE ip // extract opcode from xINST - GOTO_OPCODE ip // jump to next instruction -#else FETCH w0, 1 // w0<- aaaa (lo) FETCH w1, 2 // w1<- AAAA (hi) + orr wINST, w0, w1, lsl #16 // wINST<- AAAAaaaa +#if MTERP_PROFILE_BRANCHES + EXPORT_PC + mov x0, xSELF + add x1, xFP, #OFF_FP_SHADOWFRAME + sbfm x2, xINST, 0, 31 + bl MterpProfileBranch // (self, shadow_frame, offset) + cbnz w0, MterpOnStackReplacement // Note: offset must be in xINST +#endif ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] - orr w0, w0, w1, lsl #16 // w0<- AAAAaaaa - adds w1, w0, w0 // w1<- byte offset + adds w1, wINST, wINST // w1<- byte offset FETCH_ADVANCE_INST_RB w1 // update rPC, load xINST b.le MterpCheckSuspendAndContinue GET_INST_OPCODE ip // extract opcode from xINST GOTO_OPCODE ip // jump to next instruction -#endif diff --git a/runtime/interpreter/mterp/arm64/op_iget.S b/runtime/interpreter/mterp/arm64/op_iget.S index 165c7308e1..88533bd33d 100644 --- a/runtime/interpreter/mterp/arm64/op_iget.S +++ b/runtime/interpreter/mterp/arm64/op_iget.S @@ -1,4 +1,4 @@ -%default { "is_object":"0", "helper":"artGet32InstanceFromCode"} +%default { "extend":"", "is_object":"0", "helper":"artGet32InstanceFromCode"} /* * General instance field get. * @@ -12,6 +12,7 @@ mov x3, xSELF // w3<- self bl $helper ldr x3, [xSELF, #THREAD_EXCEPTION_OFFSET] + $extend ubfx w2, wINST, #8, #4 // w2<- A PREFETCH_INST 2 cbnz x3, MterpPossibleException // bail out diff --git a/runtime/interpreter/mterp/arm64/op_packed_switch.S b/runtime/interpreter/mterp/arm64/op_packed_switch.S index f087d23c0a..e8b4f04dfe 100644 --- a/runtime/interpreter/mterp/arm64/op_packed_switch.S +++ b/runtime/interpreter/mterp/arm64/op_packed_switch.S @@ -9,20 +9,6 @@ * for: packed-switch, sparse-switch */ /* op vAA, +BBBB */ -#if MTERP_SUSPEND - FETCH w0, 1 // w0<- bbbb (lo) - FETCH w1, 2 // w1<- BBBB (hi) - mov w3, wINST, lsr #8 // w3<- AA - orr w0, w0, w1, lsl #16 // w0<- BBBBbbbb - GET_VREG w1, w3 // w1<- vAA - add w0, rPC, w0, lsl #1 // w0<- PC + BBBBbbbb*2 - bl $func // w0<- code-unit branch offset - adds w1, w0, w0 // w1<- byte offset; clear V - ldrle rIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET] // refresh handler base - FETCH_ADVANCE_INST_RB w1 // update rPC, load wINST - GET_INST_OPCODE ip // extract opcode from wINST - GOTO_OPCODE ip // jump to next instruction -#else FETCH w0, 1 // w0<- bbbb (lo) FETCH w1, 2 // w1<- BBBB (hi) lsr w3, wINST, #8 // w3<- AA @@ -30,10 +16,18 @@ GET_VREG w1, w3 // w1<- vAA add x0, xPC, w0, lsl #1 // w0<- PC + BBBBbbbb*2 bl $func // w0<- code-unit branch offset + sbfm xINST, x0, 0, 31 +#if MTERP_PROFILE_BRANCHES + EXPORT_PC + mov x0, xSELF + add x1, xFP, #OFF_FP_SHADOWFRAME + mov x2, xINST + bl MterpProfileBranch // (self, shadow_frame, offset) + cbnz w0, MterpOnStackReplacement +#endif ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] - adds w1, w0, w0 // w1<- byte offset; clear V + adds w1, wINST, wINST // w1<- byte offset; clear V FETCH_ADVANCE_INST_RB w1 // update rPC, load wINST b.le MterpCheckSuspendAndContinue GET_INST_OPCODE ip // extract opcode from wINST GOTO_OPCODE ip // jump to next instruction -#endif diff --git a/runtime/interpreter/mterp/arm64/zcmp.S b/runtime/interpreter/mterp/arm64/zcmp.S index d4856d2668..e528d9f030 100644 --- a/runtime/interpreter/mterp/arm64/zcmp.S +++ b/runtime/interpreter/mterp/arm64/zcmp.S @@ -6,26 +6,37 @@ * for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez */ /* if-cmp vAA, +BBBB */ -#if MTERP_SUSPEND - mov w0, wINST, lsr #8 // w0<- AA +#if MTERP_PROFILE_BRANCHES + lsr w0, wINST, #8 // w0<- AA GET_VREG w2, w0 // w2<- vAA - FETCH_S w1, 1 // w1<- branch offset, in code units + FETCH_S wINST, 1 // w1<- branch offset, in code units cmp w2, #0 // compare (vA, 0) - mov${condition} w1, #2 // w1<- inst branch dist for not-taken - adds w1, w1, w1 // convert to bytes & set flags - FETCH_ADVANCE_INST_RB w1 // update rPC, load wINST - ldrmi rIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET] // refresh table base + b.${condition} .L_${opcode}_taken + FETCH_ADVANCE_INST 2 // update rPC, load wINST + GET_INST_OPCODE ip // extract opcode from wINST + GOTO_OPCODE ip // jump to next instruction +.L_${opcode}_taken: + EXPORT_PC + mov x0, xSELF + add x1, xFP, #OFF_FP_SHADOWFRAME + sbfm x2, xINST, 0, 31 + bl MterpProfileBranch // (self, shadow_frame, offset) + cbnz w0, MterpOnStackReplacement // Note: offset must be in wINST + ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] + adds w2, wINST, wINST // convert to bytes & set flags + FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST + b.mi MterpCheckSuspendAndContinue GET_INST_OPCODE ip // extract opcode from wINST GOTO_OPCODE ip // jump to next instruction #else lsr w0, wINST, #8 // w0<- AA GET_VREG w2, w0 // w2<- vAA FETCH_S w1, 1 // w1<- branch offset, in code units - ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] mov w0, #2 // Branch offset if not taken cmp w2, #0 // compare (vA, 0) - csel w1, w1, w0, ${condition} // Branch if true - adds w2, w1, w1 // convert to bytes & set flags + csel wINST, w1, w0, ${condition} // Branch if true, stashing result in callee save reg + ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] + adds w2, wINST, wINST // convert to bytes & set flags FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST b.mi MterpCheckSuspendAndContinue GET_INST_OPCODE ip // extract opcode from wINST diff --git a/runtime/interpreter/mterp/mterp.cc b/runtime/interpreter/mterp/mterp.cc index 0afd2765db..15745d2731 100644 --- a/runtime/interpreter/mterp/mterp.cc +++ b/runtime/interpreter/mterp/mterp.cc @@ -20,6 +20,8 @@ #include "interpreter/interpreter_common.h" #include "entrypoints/entrypoint_utils-inl.h" #include "mterp.h" +#include "jit/jit.h" +#include "debugger.h" namespace art { namespace interpreter { @@ -139,6 +141,20 @@ extern "C" int32_t MterpDoPackedSwitch(const uint16_t* switchData, int32_t testV return entries[index]; } +extern "C" bool MterpShouldSwitchInterpreters() + SHARED_REQUIRES(Locks::mutator_lock_) { + const instrumentation::Instrumentation* const instrumentation = + Runtime::Current()->GetInstrumentation(); + bool unhandled_instrumentation; + // TODO: enable for other targets after more extensive testing. + if (kRuntimeISA == kArm64) { + unhandled_instrumentation = instrumentation->NonJitProfilingActive(); + } else { + unhandled_instrumentation = instrumentation->IsActive(); + } + return unhandled_instrumentation || Dbg::IsDebuggerActive(); +} + extern "C" bool MterpInvokeVirtual(Thread* self, ShadowFrame* shadow_frame, uint16_t* dex_pc_ptr, uint16_t inst_data ) @@ -488,6 +504,14 @@ extern "C" void MterpLogFallback(Thread* self, ShadowFrame* shadow_frame) << self->IsExceptionPending(); } +extern "C" void MterpLogOSR(Thread* self, ShadowFrame* shadow_frame, int32_t offset) + SHARED_REQUIRES(Locks::mutator_lock_) { + UNUSED(self); + const Instruction* inst = Instruction::At(shadow_frame->GetDexPCPtr()); + uint16_t inst_data = inst->Fetch16(0); + LOG(INFO) << "OSR: " << inst->Opcode(inst_data) << ", offset = " << offset; +} + extern "C" void MterpLogSuspendFallback(Thread* self, ShadowFrame* shadow_frame, uint32_t flags) SHARED_REQUIRES(Locks::mutator_lock_) { UNUSED(self); @@ -500,9 +524,10 @@ extern "C" void MterpLogSuspendFallback(Thread* self, ShadowFrame* shadow_frame, } } -extern "C" void MterpSuspendCheck(Thread* self) +extern "C" bool MterpSuspendCheck(Thread* self) SHARED_REQUIRES(Locks::mutator_lock_) { self->AllowThreadSuspension(); + return MterpShouldSwitchInterpreters(); } extern "C" int artSet64IndirectStaticFromMterp(uint32_t field_idx, ArtMethod* referrer, @@ -618,5 +643,15 @@ extern "C" mirror::Object* artIGetObjectFromMterp(mirror::Object* obj, uint32_t return obj->GetFieldObject<mirror::Object>(MemberOffset(field_offset)); } +extern "C" bool MterpProfileBranch(Thread* self, ShadowFrame* shadow_frame, int32_t offset) + SHARED_REQUIRES(Locks::mutator_lock_) { + ArtMethod* method = shadow_frame->GetMethod(); + JValue* result = shadow_frame->GetResultRegister(); + uint32_t dex_pc = shadow_frame->GetDexPC(); + const auto* const instrumentation = Runtime::Current()->GetInstrumentation(); + instrumentation->Branch(self, method, dex_pc, offset); + return jit::Jit::MaybeDoOnStackReplacement(self, method, dex_pc, offset, result); +} + } // namespace interpreter } // namespace art diff --git a/runtime/interpreter/mterp/mterp.h b/runtime/interpreter/mterp/mterp.h index 90d21e9c89..8d24641ce4 100644 --- a/runtime/interpreter/mterp/mterp.h +++ b/runtime/interpreter/mterp/mterp.h @@ -30,6 +30,7 @@ namespace interpreter { void InitMterpTls(Thread* self); void CheckMterpAsmConstants(); +extern "C" bool MterpShouldSwitchInterpreters(); } // namespace interpreter } // namespace art diff --git a/runtime/interpreter/mterp/out/mterp_arm.S b/runtime/interpreter/mterp/out/mterp_arm.S index ee195598db..511c35b4c6 100644 --- a/runtime/interpreter/mterp/out/mterp_arm.S +++ b/runtime/interpreter/mterp/out/mterp_arm.S @@ -92,6 +92,9 @@ unspecified registers or condition codes. */ #include "asm_support.h" +#define MTERP_PROFILE_BRANCHES 1 +#define MTERP_LOGGING 0 + /* During bringup, we'll use the shadow frame model instead of rFP */ /* single-purpose registers, given names for clarity */ #define rPC r4 @@ -116,14 +119,6 @@ unspecified registers or condition codes. #define OFF_FP_SHADOWFRAME (-SHADOWFRAME_VREGS_OFFSET) /* - * - * The reference interpreter performs explicit suspect checks, which is somewhat wasteful. - * Dalvik's interpreter folded suspend checks into the jump table mechanism, and eventually - * mterp should do so as well. - */ -#define MTERP_SUSPEND 0 - -/* * "export" the PC to dex_pc field in the shadow frame, f/b/o future exception objects. Must * be done *before* something throws. * @@ -1111,20 +1106,28 @@ artMterpAsmInstructionStart = .L_op_nop */ /* goto +AA */ /* tuning: use sbfx for 6t2+ targets */ -#if MTERP_SUSPEND +#if MTERP_PROFILE_BRANCHES mov r0, rINST, lsl #16 @ r0<- AAxx0000 - movs r1, r0, asr #24 @ r1<- ssssssAA (sign-extended) - add r2, r1, r1 @ r2<- byte offset, set flags - @ If backwards branch refresh rIBASE - ldrmi rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh handler base + movs rINST, r0, asr #24 @ rINST<- ssssssAA (sign-extended) + EXPORT_PC + mov r0, rSELF + add r1, rFP, #OFF_FP_SHADOWFRAME + mov r2, rINST + bl MterpProfileBranch @ (self, shadow_frame, offset) + cmp r0, #0 + bne MterpOnStackReplacement @ Note: offset must be in rINST + ldr lr, [rSELF, #THREAD_FLAGS_OFFSET] + add r2, rINST, rINST @ r2<- byte offset, set flags FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST + @ If backwards branch refresh rIBASE + bmi MterpCheckSuspendAndContinue GET_INST_OPCODE ip @ extract opcode from rINST GOTO_OPCODE ip @ jump to next instruction #else - ldr lr, [rSELF, #THREAD_FLAGS_OFFSET] mov r0, rINST, lsl #16 @ r0<- AAxx0000 - movs r1, r0, asr #24 @ r1<- ssssssAA (sign-extended) - add r2, r1, r1 @ r2<- byte offset, set flags + movs rINST, r0, asr #24 @ rINST<- ssssssAA (sign-extended) + ldr lr, [rSELF, #THREAD_FLAGS_OFFSET] + add r2, rINST, rINST @ r2<- byte offset, set flags FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST @ If backwards branch refresh rIBASE bmi MterpCheckSuspendAndContinue @@ -1143,17 +1146,25 @@ artMterpAsmInstructionStart = .L_op_nop * double to get a byte offset. */ /* goto/16 +AAAA */ -#if MTERP_SUSPEND - FETCH_S r0, 1 @ r0<- ssssAAAA (sign-extended) - adds r1, r0, r0 @ r1<- byte offset, flags set +#if MTERP_PROFILE_BRANCHES + FETCH_S rINST, 1 @ rINST<- ssssAAAA (sign-extended) + EXPORT_PC + mov r0, rSELF + add r1, rFP, #OFF_FP_SHADOWFRAME + mov r2, rINST + bl MterpProfileBranch @ (self, shadow_frame, offset) + cmp r0, #0 + bne MterpOnStackReplacement @ Note: offset must be in rINST + ldr lr, [rSELF, #THREAD_FLAGS_OFFSET] + adds r1, rINST, rINST @ r1<- byte offset, flags set FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST - ldrmi rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh handler base + bmi MterpCheckSuspendAndContinue GET_INST_OPCODE ip @ extract opcode from rINST GOTO_OPCODE ip @ jump to next instruction #else - FETCH_S r0, 1 @ r0<- ssssAAAA (sign-extended) + FETCH_S rINST, 1 @ rINST<- ssssAAAA (sign-extended) ldr lr, [rSELF, #THREAD_FLAGS_OFFSET] - adds r1, r0, r0 @ r1<- byte offset, flags set + adds r1, rINST, rINST @ r1<- byte offset, flags set FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST bmi MterpCheckSuspendAndContinue GET_INST_OPCODE ip @ extract opcode from rINST @@ -1176,21 +1187,29 @@ artMterpAsmInstructionStart = .L_op_nop * offset to byte offset. */ /* goto/32 +AAAAAAAA */ -#if MTERP_SUSPEND +#if MTERP_PROFILE_BRANCHES FETCH r0, 1 @ r0<- aaaa (lo) FETCH r1, 2 @ r1<- AAAA (hi) - orr r0, r0, r1, lsl #16 @ r0<- AAAAaaaa - adds r1, r0, r0 @ r1<- byte offset + orr rINST, r0, r1, lsl #16 @ rINST<- AAAAaaaa + EXPORT_PC + mov r0, rSELF + add r1, rFP, #OFF_FP_SHADOWFRAME + mov r2, rINST + bl MterpProfileBranch @ (self, shadow_frame, offset) + cmp r0, #0 + bne MterpOnStackReplacement @ Note: offset must be in rINST + ldr lr, [rSELF, #THREAD_FLAGS_OFFSET] + adds r1, rINST, rINST @ r1<- byte offset FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST - ldrle rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh handler base + ble MterpCheckSuspendAndContinue GET_INST_OPCODE ip @ extract opcode from rINST GOTO_OPCODE ip @ jump to next instruction #else FETCH r0, 1 @ r0<- aaaa (lo) FETCH r1, 2 @ r1<- AAAA (hi) + orr rINST, r0, r1, lsl #16 @ rINST<- AAAAaaaa ldr lr, [rSELF, #THREAD_FLAGS_OFFSET] - orr r0, r0, r1, lsl #16 @ r0<- AAAAaaaa - adds r1, r0, r0 @ r1<- byte offset + adds r1, rINST, rINST @ r1<- byte offset FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST ble MterpCheckSuspendAndContinue GET_INST_OPCODE ip @ extract opcode from rINST @@ -1211,7 +1230,7 @@ artMterpAsmInstructionStart = .L_op_nop * for: packed-switch, sparse-switch */ /* op vAA, +BBBB */ -#if MTERP_SUSPEND +#if MTERP_PROFILE_BRANCHES FETCH r0, 1 @ r0<- bbbb (lo) FETCH r1, 2 @ r1<- BBBB (hi) mov r3, rINST, lsr #8 @ r3<- AA @@ -1219,9 +1238,18 @@ artMterpAsmInstructionStart = .L_op_nop GET_VREG r1, r3 @ r1<- vAA add r0, rPC, r0, lsl #1 @ r0<- PC + BBBBbbbb*2 bl MterpDoPackedSwitch @ r0<- code-unit branch offset - adds r1, r0, r0 @ r1<- byte offset; clear V - ldrle rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh handler base + mov rINST, r0 + EXPORT_PC + mov r0, rSELF + add r1, rFP, #OFF_FP_SHADOWFRAME + mov r2, rINST + bl MterpProfileBranch @ (self, shadow_frame, offset) + cmp r0, #0 + bne MterpOnStackReplacement @ Note: offset must be in rINST + ldr lr, [rSELF, #THREAD_FLAGS_OFFSET] + adds r1, rINST, rINST @ r1<- byte offset; clear V FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST + ble MterpCheckSuspendAndContinue GET_INST_OPCODE ip @ extract opcode from rINST GOTO_OPCODE ip @ jump to next instruction #else @@ -1232,8 +1260,9 @@ artMterpAsmInstructionStart = .L_op_nop GET_VREG r1, r3 @ r1<- vAA add r0, rPC, r0, lsl #1 @ r0<- PC + BBBBbbbb*2 bl MterpDoPackedSwitch @ r0<- code-unit branch offset + mov rINST, r0 ldr lr, [rSELF, #THREAD_FLAGS_OFFSET] - adds r1, r0, r0 @ r1<- byte offset; clear V + adds r1, rINST, rINST @ r1<- byte offset; clear V FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST ble MterpCheckSuspendAndContinue GET_INST_OPCODE ip @ extract opcode from rINST @@ -1255,7 +1284,7 @@ artMterpAsmInstructionStart = .L_op_nop * for: packed-switch, sparse-switch */ /* op vAA, +BBBB */ -#if MTERP_SUSPEND +#if MTERP_PROFILE_BRANCHES FETCH r0, 1 @ r0<- bbbb (lo) FETCH r1, 2 @ r1<- BBBB (hi) mov r3, rINST, lsr #8 @ r3<- AA @@ -1263,9 +1292,18 @@ artMterpAsmInstructionStart = .L_op_nop GET_VREG r1, r3 @ r1<- vAA add r0, rPC, r0, lsl #1 @ r0<- PC + BBBBbbbb*2 bl MterpDoSparseSwitch @ r0<- code-unit branch offset - adds r1, r0, r0 @ r1<- byte offset; clear V - ldrle rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh handler base + mov rINST, r0 + EXPORT_PC + mov r0, rSELF + add r1, rFP, #OFF_FP_SHADOWFRAME + mov r2, rINST + bl MterpProfileBranch @ (self, shadow_frame, offset) + cmp r0, #0 + bne MterpOnStackReplacement @ Note: offset must be in rINST + ldr lr, [rSELF, #THREAD_FLAGS_OFFSET] + adds r1, rINST, rINST @ r1<- byte offset; clear V FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST + ble MterpCheckSuspendAndContinue GET_INST_OPCODE ip @ extract opcode from rINST GOTO_OPCODE ip @ jump to next instruction #else @@ -1276,8 +1314,9 @@ artMterpAsmInstructionStart = .L_op_nop GET_VREG r1, r3 @ r1<- vAA add r0, rPC, r0, lsl #1 @ r0<- PC + BBBBbbbb*2 bl MterpDoSparseSwitch @ r0<- code-unit branch offset + mov rINST, r0 ldr lr, [rSELF, #THREAD_FLAGS_OFFSET] - adds r1, r0, r0 @ r1<- byte offset; clear V + adds r1, rINST, rINST @ r1<- byte offset; clear V FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST ble MterpCheckSuspendAndContinue GET_INST_OPCODE ip @ extract opcode from rINST @@ -1495,17 +1534,29 @@ artMterpAsmInstructionStart = .L_op_nop * For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le */ /* if-cmp vA, vB, +CCCC */ -#if MTERP_SUSPEND +#if MTERP_PROFILE_BRANCHES mov r1, rINST, lsr #12 @ r1<- B ubfx r0, rINST, #8, #4 @ r0<- A GET_VREG r3, r1 @ r3<- vB GET_VREG r2, r0 @ r2<- vA - FETCH_S r1, 1 @ r1<- branch offset, in code units + FETCH_S rINST, 1 @ rINST<- branch offset, in code units cmp r2, r3 @ compare (vA, vB) - movne r1, #2 @ r1<- BYTE branch dist for not-taken - adds r2, r1, r1 @ convert to bytes, check sign + bne .L_op_if_eq_not_taken + EXPORT_PC + mov r0, rSELF + add r1, rFP, #OFF_FP_SHADOWFRAME + mov r2, rINST + bl MterpProfileBranch @ (self, shadow_frame, offset) + cmp r0, #0 + bne MterpOnStackReplacement @ Note: offset must be in rINST + adds r2, rINST, rINST @ convert to bytes, check sign + ldr lr, [rSELF, #THREAD_FLAGS_OFFSET] FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST - ldrmi rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh rIBASE + bmi MterpCheckSuspendAndContinue + GET_INST_OPCODE ip @ extract opcode from rINST + GOTO_OPCODE ip @ jump to next instruction +.L_op_if_eq_not_taken: + FETCH_ADVANCE_INST 2 @ update rPC, load rINST GET_INST_OPCODE ip @ extract opcode from rINST GOTO_OPCODE ip @ jump to next instruction #else @@ -1514,10 +1565,10 @@ artMterpAsmInstructionStart = .L_op_nop GET_VREG r3, r1 @ r3<- vB GET_VREG r2, r0 @ r2<- vA ldr lr, [rSELF, #THREAD_FLAGS_OFFSET] - FETCH_S r1, 1 @ r1<- branch offset, in code units + FETCH_S rINST, 1 @ rINST<- branch offset, in code units cmp r2, r3 @ compare (vA, vB) - movne r1, #2 @ r1<- BYTE branch dist for not-taken - adds r2, r1, r1 @ convert to bytes, check sign + movne rINST, #2 @ rINST<- BYTE branch dist for not-taken + adds r2, rINST, rINST @ convert to bytes, check sign FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST bmi MterpCheckSuspendAndContinue GET_INST_OPCODE ip @ extract opcode from rINST @@ -1538,17 +1589,29 @@ artMterpAsmInstructionStart = .L_op_nop * For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le */ /* if-cmp vA, vB, +CCCC */ -#if MTERP_SUSPEND +#if MTERP_PROFILE_BRANCHES mov r1, rINST, lsr #12 @ r1<- B ubfx r0, rINST, #8, #4 @ r0<- A GET_VREG r3, r1 @ r3<- vB GET_VREG r2, r0 @ r2<- vA - FETCH_S r1, 1 @ r1<- branch offset, in code units + FETCH_S rINST, 1 @ rINST<- branch offset, in code units cmp r2, r3 @ compare (vA, vB) - moveq r1, #2 @ r1<- BYTE branch dist for not-taken - adds r2, r1, r1 @ convert to bytes, check sign + beq .L_op_if_ne_not_taken + EXPORT_PC + mov r0, rSELF + add r1, rFP, #OFF_FP_SHADOWFRAME + mov r2, rINST + bl MterpProfileBranch @ (self, shadow_frame, offset) + cmp r0, #0 + bne MterpOnStackReplacement @ Note: offset must be in rINST + adds r2, rINST, rINST @ convert to bytes, check sign + ldr lr, [rSELF, #THREAD_FLAGS_OFFSET] FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST - ldrmi rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh rIBASE + bmi MterpCheckSuspendAndContinue + GET_INST_OPCODE ip @ extract opcode from rINST + GOTO_OPCODE ip @ jump to next instruction +.L_op_if_ne_not_taken: + FETCH_ADVANCE_INST 2 @ update rPC, load rINST GET_INST_OPCODE ip @ extract opcode from rINST GOTO_OPCODE ip @ jump to next instruction #else @@ -1557,10 +1620,10 @@ artMterpAsmInstructionStart = .L_op_nop GET_VREG r3, r1 @ r3<- vB GET_VREG r2, r0 @ r2<- vA ldr lr, [rSELF, #THREAD_FLAGS_OFFSET] - FETCH_S r1, 1 @ r1<- branch offset, in code units + FETCH_S rINST, 1 @ rINST<- branch offset, in code units cmp r2, r3 @ compare (vA, vB) - moveq r1, #2 @ r1<- BYTE branch dist for not-taken - adds r2, r1, r1 @ convert to bytes, check sign + moveq rINST, #2 @ rINST<- BYTE branch dist for not-taken + adds r2, rINST, rINST @ convert to bytes, check sign FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST bmi MterpCheckSuspendAndContinue GET_INST_OPCODE ip @ extract opcode from rINST @@ -1581,17 +1644,29 @@ artMterpAsmInstructionStart = .L_op_nop * For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le */ /* if-cmp vA, vB, +CCCC */ -#if MTERP_SUSPEND +#if MTERP_PROFILE_BRANCHES mov r1, rINST, lsr #12 @ r1<- B ubfx r0, rINST, #8, #4 @ r0<- A GET_VREG r3, r1 @ r3<- vB GET_VREG r2, r0 @ r2<- vA - FETCH_S r1, 1 @ r1<- branch offset, in code units + FETCH_S rINST, 1 @ rINST<- branch offset, in code units cmp r2, r3 @ compare (vA, vB) - movge r1, #2 @ r1<- BYTE branch dist for not-taken - adds r2, r1, r1 @ convert to bytes, check sign + bge .L_op_if_lt_not_taken + EXPORT_PC + mov r0, rSELF + add r1, rFP, #OFF_FP_SHADOWFRAME + mov r2, rINST + bl MterpProfileBranch @ (self, shadow_frame, offset) + cmp r0, #0 + bne MterpOnStackReplacement @ Note: offset must be in rINST + adds r2, rINST, rINST @ convert to bytes, check sign + ldr lr, [rSELF, #THREAD_FLAGS_OFFSET] FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST - ldrmi rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh rIBASE + bmi MterpCheckSuspendAndContinue + GET_INST_OPCODE ip @ extract opcode from rINST + GOTO_OPCODE ip @ jump to next instruction +.L_op_if_lt_not_taken: + FETCH_ADVANCE_INST 2 @ update rPC, load rINST GET_INST_OPCODE ip @ extract opcode from rINST GOTO_OPCODE ip @ jump to next instruction #else @@ -1600,10 +1675,10 @@ artMterpAsmInstructionStart = .L_op_nop GET_VREG r3, r1 @ r3<- vB GET_VREG r2, r0 @ r2<- vA ldr lr, [rSELF, #THREAD_FLAGS_OFFSET] - FETCH_S r1, 1 @ r1<- branch offset, in code units + FETCH_S rINST, 1 @ rINST<- branch offset, in code units cmp r2, r3 @ compare (vA, vB) - movge r1, #2 @ r1<- BYTE branch dist for not-taken - adds r2, r1, r1 @ convert to bytes, check sign + movge rINST, #2 @ rINST<- BYTE branch dist for not-taken + adds r2, rINST, rINST @ convert to bytes, check sign FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST bmi MterpCheckSuspendAndContinue GET_INST_OPCODE ip @ extract opcode from rINST @@ -1624,17 +1699,29 @@ artMterpAsmInstructionStart = .L_op_nop * For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le */ /* if-cmp vA, vB, +CCCC */ -#if MTERP_SUSPEND +#if MTERP_PROFILE_BRANCHES mov r1, rINST, lsr #12 @ r1<- B ubfx r0, rINST, #8, #4 @ r0<- A GET_VREG r3, r1 @ r3<- vB GET_VREG r2, r0 @ r2<- vA - FETCH_S r1, 1 @ r1<- branch offset, in code units + FETCH_S rINST, 1 @ rINST<- branch offset, in code units cmp r2, r3 @ compare (vA, vB) - movlt r1, #2 @ r1<- BYTE branch dist for not-taken - adds r2, r1, r1 @ convert to bytes, check sign + blt .L_op_if_ge_not_taken + EXPORT_PC + mov r0, rSELF + add r1, rFP, #OFF_FP_SHADOWFRAME + mov r2, rINST + bl MterpProfileBranch @ (self, shadow_frame, offset) + cmp r0, #0 + bne MterpOnStackReplacement @ Note: offset must be in rINST + adds r2, rINST, rINST @ convert to bytes, check sign + ldr lr, [rSELF, #THREAD_FLAGS_OFFSET] FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST - ldrmi rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh rIBASE + bmi MterpCheckSuspendAndContinue + GET_INST_OPCODE ip @ extract opcode from rINST + GOTO_OPCODE ip @ jump to next instruction +.L_op_if_ge_not_taken: + FETCH_ADVANCE_INST 2 @ update rPC, load rINST GET_INST_OPCODE ip @ extract opcode from rINST GOTO_OPCODE ip @ jump to next instruction #else @@ -1643,10 +1730,10 @@ artMterpAsmInstructionStart = .L_op_nop GET_VREG r3, r1 @ r3<- vB GET_VREG r2, r0 @ r2<- vA ldr lr, [rSELF, #THREAD_FLAGS_OFFSET] - FETCH_S r1, 1 @ r1<- branch offset, in code units + FETCH_S rINST, 1 @ rINST<- branch offset, in code units cmp r2, r3 @ compare (vA, vB) - movlt r1, #2 @ r1<- BYTE branch dist for not-taken - adds r2, r1, r1 @ convert to bytes, check sign + movlt rINST, #2 @ rINST<- BYTE branch dist for not-taken + adds r2, rINST, rINST @ convert to bytes, check sign FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST bmi MterpCheckSuspendAndContinue GET_INST_OPCODE ip @ extract opcode from rINST @@ -1667,17 +1754,29 @@ artMterpAsmInstructionStart = .L_op_nop * For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le */ /* if-cmp vA, vB, +CCCC */ -#if MTERP_SUSPEND +#if MTERP_PROFILE_BRANCHES mov r1, rINST, lsr #12 @ r1<- B ubfx r0, rINST, #8, #4 @ r0<- A GET_VREG r3, r1 @ r3<- vB GET_VREG r2, r0 @ r2<- vA - FETCH_S r1, 1 @ r1<- branch offset, in code units + FETCH_S rINST, 1 @ rINST<- branch offset, in code units cmp r2, r3 @ compare (vA, vB) - movle r1, #2 @ r1<- BYTE branch dist for not-taken - adds r2, r1, r1 @ convert to bytes, check sign + ble .L_op_if_gt_not_taken + EXPORT_PC + mov r0, rSELF + add r1, rFP, #OFF_FP_SHADOWFRAME + mov r2, rINST + bl MterpProfileBranch @ (self, shadow_frame, offset) + cmp r0, #0 + bne MterpOnStackReplacement @ Note: offset must be in rINST + adds r2, rINST, rINST @ convert to bytes, check sign + ldr lr, [rSELF, #THREAD_FLAGS_OFFSET] FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST - ldrmi rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh rIBASE + bmi MterpCheckSuspendAndContinue + GET_INST_OPCODE ip @ extract opcode from rINST + GOTO_OPCODE ip @ jump to next instruction +.L_op_if_gt_not_taken: + FETCH_ADVANCE_INST 2 @ update rPC, load rINST GET_INST_OPCODE ip @ extract opcode from rINST GOTO_OPCODE ip @ jump to next instruction #else @@ -1686,10 +1785,10 @@ artMterpAsmInstructionStart = .L_op_nop GET_VREG r3, r1 @ r3<- vB GET_VREG r2, r0 @ r2<- vA ldr lr, [rSELF, #THREAD_FLAGS_OFFSET] - FETCH_S r1, 1 @ r1<- branch offset, in code units + FETCH_S rINST, 1 @ rINST<- branch offset, in code units cmp r2, r3 @ compare (vA, vB) - movle r1, #2 @ r1<- BYTE branch dist for not-taken - adds r2, r1, r1 @ convert to bytes, check sign + movle rINST, #2 @ rINST<- BYTE branch dist for not-taken + adds r2, rINST, rINST @ convert to bytes, check sign FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST bmi MterpCheckSuspendAndContinue GET_INST_OPCODE ip @ extract opcode from rINST @@ -1710,17 +1809,29 @@ artMterpAsmInstructionStart = .L_op_nop * For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le */ /* if-cmp vA, vB, +CCCC */ -#if MTERP_SUSPEND +#if MTERP_PROFILE_BRANCHES mov r1, rINST, lsr #12 @ r1<- B ubfx r0, rINST, #8, #4 @ r0<- A GET_VREG r3, r1 @ r3<- vB GET_VREG r2, r0 @ r2<- vA - FETCH_S r1, 1 @ r1<- branch offset, in code units + FETCH_S rINST, 1 @ rINST<- branch offset, in code units cmp r2, r3 @ compare (vA, vB) - movgt r1, #2 @ r1<- BYTE branch dist for not-taken - adds r2, r1, r1 @ convert to bytes, check sign + bgt .L_op_if_le_not_taken + EXPORT_PC + mov r0, rSELF + add r1, rFP, #OFF_FP_SHADOWFRAME + mov r2, rINST + bl MterpProfileBranch @ (self, shadow_frame, offset) + cmp r0, #0 + bne MterpOnStackReplacement @ Note: offset must be in rINST + adds r2, rINST, rINST @ convert to bytes, check sign + ldr lr, [rSELF, #THREAD_FLAGS_OFFSET] FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST - ldrmi rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh rIBASE + bmi MterpCheckSuspendAndContinue + GET_INST_OPCODE ip @ extract opcode from rINST + GOTO_OPCODE ip @ jump to next instruction +.L_op_if_le_not_taken: + FETCH_ADVANCE_INST 2 @ update rPC, load rINST GET_INST_OPCODE ip @ extract opcode from rINST GOTO_OPCODE ip @ jump to next instruction #else @@ -1729,10 +1840,10 @@ artMterpAsmInstructionStart = .L_op_nop GET_VREG r3, r1 @ r3<- vB GET_VREG r2, r0 @ r2<- vA ldr lr, [rSELF, #THREAD_FLAGS_OFFSET] - FETCH_S r1, 1 @ r1<- branch offset, in code units + FETCH_S rINST, 1 @ rINST<- branch offset, in code units cmp r2, r3 @ compare (vA, vB) - movgt r1, #2 @ r1<- BYTE branch dist for not-taken - adds r2, r1, r1 @ convert to bytes, check sign + movgt rINST, #2 @ rINST<- BYTE branch dist for not-taken + adds r2, rINST, rINST @ convert to bytes, check sign FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST bmi MterpCheckSuspendAndContinue GET_INST_OPCODE ip @ extract opcode from rINST @@ -1753,25 +1864,37 @@ artMterpAsmInstructionStart = .L_op_nop * for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez */ /* if-cmp vAA, +BBBB */ -#if MTERP_SUSPEND +#if MTERP_PROFILE_BRANCHES mov r0, rINST, lsr #8 @ r0<- AA GET_VREG r2, r0 @ r2<- vAA - FETCH_S r1, 1 @ r1<- branch offset, in code units + FETCH_S rINST, 1 @ rINST<- branch offset, in code units + ldr lr, [rSELF, #THREAD_FLAGS_OFFSET] cmp r2, #0 @ compare (vA, 0) - movne r1, #2 @ r1<- inst branch dist for not-taken - adds r1, r1, r1 @ convert to bytes & set flags + bne .L_op_if_eqz_not_taken + EXPORT_PC + mov r0, rSELF + add r1, rFP, #OFF_FP_SHADOWFRAME + mov r2, rINST + bl MterpProfileBranch @ (self, shadow_frame, offset) + cmp r0, #0 + bne MterpOnStackReplacement @ Note: offset must be in rINST + adds r1, rINST, rINST @ convert to bytes & set flags FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST - ldrmi rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh table base + bmi MterpCheckSuspendAndContinue + GET_INST_OPCODE ip @ extract opcode from rINST + GOTO_OPCODE ip @ jump to next instruction +.L_op_if_eqz_not_taken: + FETCH_ADVANCE_INST 2 @ update rPC, load rINST GET_INST_OPCODE ip @ extract opcode from rINST GOTO_OPCODE ip @ jump to next instruction #else mov r0, rINST, lsr #8 @ r0<- AA GET_VREG r2, r0 @ r2<- vAA - FETCH_S r1, 1 @ r1<- branch offset, in code units + FETCH_S rINST, 1 @ rINST<- branch offset, in code units ldr lr, [rSELF, #THREAD_FLAGS_OFFSET] cmp r2, #0 @ compare (vA, 0) - movne r1, #2 @ r1<- inst branch dist for not-taken - adds r1, r1, r1 @ convert to bytes & set flags + movne rINST, #2 @ rINST<- inst branch dist for not-taken + adds r1, rINST, rINST @ convert to bytes & set flags FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST bmi MterpCheckSuspendAndContinue GET_INST_OPCODE ip @ extract opcode from rINST @@ -1792,25 +1915,37 @@ artMterpAsmInstructionStart = .L_op_nop * for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez */ /* if-cmp vAA, +BBBB */ -#if MTERP_SUSPEND +#if MTERP_PROFILE_BRANCHES mov r0, rINST, lsr #8 @ r0<- AA GET_VREG r2, r0 @ r2<- vAA - FETCH_S r1, 1 @ r1<- branch offset, in code units + FETCH_S rINST, 1 @ rINST<- branch offset, in code units + ldr lr, [rSELF, #THREAD_FLAGS_OFFSET] cmp r2, #0 @ compare (vA, 0) - moveq r1, #2 @ r1<- inst branch dist for not-taken - adds r1, r1, r1 @ convert to bytes & set flags + beq .L_op_if_nez_not_taken + EXPORT_PC + mov r0, rSELF + add r1, rFP, #OFF_FP_SHADOWFRAME + mov r2, rINST + bl MterpProfileBranch @ (self, shadow_frame, offset) + cmp r0, #0 + bne MterpOnStackReplacement @ Note: offset must be in rINST + adds r1, rINST, rINST @ convert to bytes & set flags FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST - ldrmi rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh table base + bmi MterpCheckSuspendAndContinue + GET_INST_OPCODE ip @ extract opcode from rINST + GOTO_OPCODE ip @ jump to next instruction +.L_op_if_nez_not_taken: + FETCH_ADVANCE_INST 2 @ update rPC, load rINST GET_INST_OPCODE ip @ extract opcode from rINST GOTO_OPCODE ip @ jump to next instruction #else mov r0, rINST, lsr #8 @ r0<- AA GET_VREG r2, r0 @ r2<- vAA - FETCH_S r1, 1 @ r1<- branch offset, in code units + FETCH_S rINST, 1 @ rINST<- branch offset, in code units ldr lr, [rSELF, #THREAD_FLAGS_OFFSET] cmp r2, #0 @ compare (vA, 0) - moveq r1, #2 @ r1<- inst branch dist for not-taken - adds r1, r1, r1 @ convert to bytes & set flags + moveq rINST, #2 @ rINST<- inst branch dist for not-taken + adds r1, rINST, rINST @ convert to bytes & set flags FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST bmi MterpCheckSuspendAndContinue GET_INST_OPCODE ip @ extract opcode from rINST @@ -1831,25 +1966,37 @@ artMterpAsmInstructionStart = .L_op_nop * for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez */ /* if-cmp vAA, +BBBB */ -#if MTERP_SUSPEND +#if MTERP_PROFILE_BRANCHES mov r0, rINST, lsr #8 @ r0<- AA GET_VREG r2, r0 @ r2<- vAA - FETCH_S r1, 1 @ r1<- branch offset, in code units + FETCH_S rINST, 1 @ rINST<- branch offset, in code units + ldr lr, [rSELF, #THREAD_FLAGS_OFFSET] cmp r2, #0 @ compare (vA, 0) - movge r1, #2 @ r1<- inst branch dist for not-taken - adds r1, r1, r1 @ convert to bytes & set flags + bge .L_op_if_ltz_not_taken + EXPORT_PC + mov r0, rSELF + add r1, rFP, #OFF_FP_SHADOWFRAME + mov r2, rINST + bl MterpProfileBranch @ (self, shadow_frame, offset) + cmp r0, #0 + bne MterpOnStackReplacement @ Note: offset must be in rINST + adds r1, rINST, rINST @ convert to bytes & set flags FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST - ldrmi rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh table base + bmi MterpCheckSuspendAndContinue + GET_INST_OPCODE ip @ extract opcode from rINST + GOTO_OPCODE ip @ jump to next instruction +.L_op_if_ltz_not_taken: + FETCH_ADVANCE_INST 2 @ update rPC, load rINST GET_INST_OPCODE ip @ extract opcode from rINST GOTO_OPCODE ip @ jump to next instruction #else mov r0, rINST, lsr #8 @ r0<- AA GET_VREG r2, r0 @ r2<- vAA - FETCH_S r1, 1 @ r1<- branch offset, in code units + FETCH_S rINST, 1 @ rINST<- branch offset, in code units ldr lr, [rSELF, #THREAD_FLAGS_OFFSET] cmp r2, #0 @ compare (vA, 0) - movge r1, #2 @ r1<- inst branch dist for not-taken - adds r1, r1, r1 @ convert to bytes & set flags + movge rINST, #2 @ rINST<- inst branch dist for not-taken + adds r1, rINST, rINST @ convert to bytes & set flags FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST bmi MterpCheckSuspendAndContinue GET_INST_OPCODE ip @ extract opcode from rINST @@ -1870,25 +2017,37 @@ artMterpAsmInstructionStart = .L_op_nop * for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez */ /* if-cmp vAA, +BBBB */ -#if MTERP_SUSPEND +#if MTERP_PROFILE_BRANCHES mov r0, rINST, lsr #8 @ r0<- AA GET_VREG r2, r0 @ r2<- vAA - FETCH_S r1, 1 @ r1<- branch offset, in code units + FETCH_S rINST, 1 @ rINST<- branch offset, in code units + ldr lr, [rSELF, #THREAD_FLAGS_OFFSET] cmp r2, #0 @ compare (vA, 0) - movlt r1, #2 @ r1<- inst branch dist for not-taken - adds r1, r1, r1 @ convert to bytes & set flags + blt .L_op_if_gez_not_taken + EXPORT_PC + mov r0, rSELF + add r1, rFP, #OFF_FP_SHADOWFRAME + mov r2, rINST + bl MterpProfileBranch @ (self, shadow_frame, offset) + cmp r0, #0 + bne MterpOnStackReplacement @ Note: offset must be in rINST + adds r1, rINST, rINST @ convert to bytes & set flags FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST - ldrmi rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh table base + bmi MterpCheckSuspendAndContinue + GET_INST_OPCODE ip @ extract opcode from rINST + GOTO_OPCODE ip @ jump to next instruction +.L_op_if_gez_not_taken: + FETCH_ADVANCE_INST 2 @ update rPC, load rINST GET_INST_OPCODE ip @ extract opcode from rINST GOTO_OPCODE ip @ jump to next instruction #else mov r0, rINST, lsr #8 @ r0<- AA GET_VREG r2, r0 @ r2<- vAA - FETCH_S r1, 1 @ r1<- branch offset, in code units + FETCH_S rINST, 1 @ rINST<- branch offset, in code units ldr lr, [rSELF, #THREAD_FLAGS_OFFSET] cmp r2, #0 @ compare (vA, 0) - movlt r1, #2 @ r1<- inst branch dist for not-taken - adds r1, r1, r1 @ convert to bytes & set flags + movlt rINST, #2 @ rINST<- inst branch dist for not-taken + adds r1, rINST, rINST @ convert to bytes & set flags FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST bmi MterpCheckSuspendAndContinue GET_INST_OPCODE ip @ extract opcode from rINST @@ -1909,25 +2068,37 @@ artMterpAsmInstructionStart = .L_op_nop * for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez */ /* if-cmp vAA, +BBBB */ -#if MTERP_SUSPEND +#if MTERP_PROFILE_BRANCHES mov r0, rINST, lsr #8 @ r0<- AA GET_VREG r2, r0 @ r2<- vAA - FETCH_S r1, 1 @ r1<- branch offset, in code units + FETCH_S rINST, 1 @ rINST<- branch offset, in code units + ldr lr, [rSELF, #THREAD_FLAGS_OFFSET] cmp r2, #0 @ compare (vA, 0) - movle r1, #2 @ r1<- inst branch dist for not-taken - adds r1, r1, r1 @ convert to bytes & set flags + ble .L_op_if_gtz_not_taken + EXPORT_PC + mov r0, rSELF + add r1, rFP, #OFF_FP_SHADOWFRAME + mov r2, rINST + bl MterpProfileBranch @ (self, shadow_frame, offset) + cmp r0, #0 + bne MterpOnStackReplacement @ Note: offset must be in rINST + adds r1, rINST, rINST @ convert to bytes & set flags FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST - ldrmi rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh table base + bmi MterpCheckSuspendAndContinue + GET_INST_OPCODE ip @ extract opcode from rINST + GOTO_OPCODE ip @ jump to next instruction +.L_op_if_gtz_not_taken: + FETCH_ADVANCE_INST 2 @ update rPC, load rINST GET_INST_OPCODE ip @ extract opcode from rINST GOTO_OPCODE ip @ jump to next instruction #else mov r0, rINST, lsr #8 @ r0<- AA GET_VREG r2, r0 @ r2<- vAA - FETCH_S r1, 1 @ r1<- branch offset, in code units + FETCH_S rINST, 1 @ rINST<- branch offset, in code units ldr lr, [rSELF, #THREAD_FLAGS_OFFSET] cmp r2, #0 @ compare (vA, 0) - movle r1, #2 @ r1<- inst branch dist for not-taken - adds r1, r1, r1 @ convert to bytes & set flags + movle rINST, #2 @ rINST<- inst branch dist for not-taken + adds r1, rINST, rINST @ convert to bytes & set flags FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST bmi MterpCheckSuspendAndContinue GET_INST_OPCODE ip @ extract opcode from rINST @@ -1948,25 +2119,37 @@ artMterpAsmInstructionStart = .L_op_nop * for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez */ /* if-cmp vAA, +BBBB */ -#if MTERP_SUSPEND +#if MTERP_PROFILE_BRANCHES mov r0, rINST, lsr #8 @ r0<- AA GET_VREG r2, r0 @ r2<- vAA - FETCH_S r1, 1 @ r1<- branch offset, in code units + FETCH_S rINST, 1 @ rINST<- branch offset, in code units + ldr lr, [rSELF, #THREAD_FLAGS_OFFSET] cmp r2, #0 @ compare (vA, 0) - movgt r1, #2 @ r1<- inst branch dist for not-taken - adds r1, r1, r1 @ convert to bytes & set flags + bgt .L_op_if_lez_not_taken + EXPORT_PC + mov r0, rSELF + add r1, rFP, #OFF_FP_SHADOWFRAME + mov r2, rINST + bl MterpProfileBranch @ (self, shadow_frame, offset) + cmp r0, #0 + bne MterpOnStackReplacement @ Note: offset must be in rINST + adds r1, rINST, rINST @ convert to bytes & set flags FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST - ldrmi rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh table base + bmi MterpCheckSuspendAndContinue + GET_INST_OPCODE ip @ extract opcode from rINST + GOTO_OPCODE ip @ jump to next instruction +.L_op_if_lez_not_taken: + FETCH_ADVANCE_INST 2 @ update rPC, load rINST GET_INST_OPCODE ip @ extract opcode from rINST GOTO_OPCODE ip @ jump to next instruction #else mov r0, rINST, lsr #8 @ r0<- AA GET_VREG r2, r0 @ r2<- vAA - FETCH_S r1, 1 @ r1<- branch offset, in code units + FETCH_S rINST, 1 @ rINST<- branch offset, in code units ldr lr, [rSELF, #THREAD_FLAGS_OFFSET] cmp r2, #0 @ compare (vA, 0) - movgt r1, #2 @ r1<- inst branch dist for not-taken - adds r1, r1, r1 @ convert to bytes & set flags + movgt rINST, #2 @ rINST<- inst branch dist for not-taken + adds r1, rINST, rINST @ convert to bytes & set flags FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST bmi MterpCheckSuspendAndContinue GET_INST_OPCODE ip @ extract opcode from rINST @@ -3294,6 +3477,9 @@ artMterpAsmInstructionStart = .L_op_nop cmp r0, #0 beq MterpException FETCH_ADVANCE_INST 3 + bl MterpShouldSwitchInterpreters + cmp r0, #0 + bne MterpFallback GET_INST_OPCODE ip GOTO_OPCODE ip @@ -3326,6 +3512,9 @@ artMterpAsmInstructionStart = .L_op_nop cmp r0, #0 beq MterpException FETCH_ADVANCE_INST 3 + bl MterpShouldSwitchInterpreters + cmp r0, #0 + bne MterpFallback GET_INST_OPCODE ip GOTO_OPCODE ip @@ -3358,6 +3547,9 @@ artMterpAsmInstructionStart = .L_op_nop cmp r0, #0 beq MterpException FETCH_ADVANCE_INST 3 + bl MterpShouldSwitchInterpreters + cmp r0, #0 + bne MterpFallback GET_INST_OPCODE ip GOTO_OPCODE ip @@ -3383,6 +3575,9 @@ artMterpAsmInstructionStart = .L_op_nop cmp r0, #0 beq MterpException FETCH_ADVANCE_INST 3 + bl MterpShouldSwitchInterpreters + cmp r0, #0 + bne MterpFallback GET_INST_OPCODE ip GOTO_OPCODE ip @@ -3409,6 +3604,9 @@ artMterpAsmInstructionStart = .L_op_nop cmp r0, #0 beq MterpException FETCH_ADVANCE_INST 3 + bl MterpShouldSwitchInterpreters + cmp r0, #0 + bne MterpFallback GET_INST_OPCODE ip GOTO_OPCODE ip @@ -3453,6 +3651,9 @@ artMterpAsmInstructionStart = .L_op_nop cmp r0, #0 beq MterpException FETCH_ADVANCE_INST 3 + bl MterpShouldSwitchInterpreters + cmp r0, #0 + bne MterpFallback GET_INST_OPCODE ip GOTO_OPCODE ip @@ -3478,6 +3679,9 @@ artMterpAsmInstructionStart = .L_op_nop cmp r0, #0 beq MterpException FETCH_ADVANCE_INST 3 + bl MterpShouldSwitchInterpreters + cmp r0, #0 + bne MterpFallback GET_INST_OPCODE ip GOTO_OPCODE ip @@ -3503,6 +3707,9 @@ artMterpAsmInstructionStart = .L_op_nop cmp r0, #0 beq MterpException FETCH_ADVANCE_INST 3 + bl MterpShouldSwitchInterpreters + cmp r0, #0 + bne MterpFallback GET_INST_OPCODE ip GOTO_OPCODE ip @@ -3528,6 +3735,9 @@ artMterpAsmInstructionStart = .L_op_nop cmp r0, #0 beq MterpException FETCH_ADVANCE_INST 3 + bl MterpShouldSwitchInterpreters + cmp r0, #0 + bne MterpFallback GET_INST_OPCODE ip GOTO_OPCODE ip @@ -3553,6 +3763,9 @@ artMterpAsmInstructionStart = .L_op_nop cmp r0, #0 beq MterpException FETCH_ADVANCE_INST 3 + bl MterpShouldSwitchInterpreters + cmp r0, #0 + bne MterpFallback GET_INST_OPCODE ip GOTO_OPCODE ip @@ -7284,6 +7497,9 @@ constvalop_long_to_double: cmp r0, #0 beq MterpException FETCH_ADVANCE_INST 3 + bl MterpShouldSwitchInterpreters + cmp r0, #0 + bne MterpFallback GET_INST_OPCODE ip GOTO_OPCODE ip @@ -7309,6 +7525,9 @@ constvalop_long_to_double: cmp r0, #0 beq MterpException FETCH_ADVANCE_INST 3 + bl MterpShouldSwitchInterpreters + cmp r0, #0 + bne MterpFallback GET_INST_OPCODE ip GOTO_OPCODE ip @@ -12098,7 +12317,6 @@ artMterpAsmAltInstructionEnd: * has not yet been thrown. Just bail out to the reference interpreter to deal with it. * TUNING: for consistency, we may want to just go ahead and handle these here. */ -#define MTERP_LOGGING 0 common_errDivideByZero: EXPORT_PC #if MTERP_LOGGING @@ -12189,8 +12407,12 @@ MterpException: ldr rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] add rPC, r0, #CODEITEM_INSNS_OFFSET add rPC, rPC, r1, lsl #1 @ generate new dex_pc_ptr - str rPC, [rFP, #OFF_FP_DEX_PC_PTR] + /* Do we need to switch interpreters? */ + bl MterpShouldSwitchInterpreters + cmp r0, #0 + bne MterpFallback /* resume execution at catch block */ + EXPORT_PC FETCH_INST GET_INST_OPCODE ip GOTO_OPCODE ip @@ -12202,14 +12424,33 @@ MterpException: */ MterpCheckSuspendAndContinue: ldr rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh rIBASE + ands lr, #(THREAD_SUSPEND_REQUEST | THREAD_CHECKPOINT_REQUEST) + bne 1f + GET_INST_OPCODE ip @ extract opcode from rINST + GOTO_OPCODE ip @ jump to next instruction +1: EXPORT_PC mov r0, rSELF - ands lr, #(THREAD_SUSPEND_REQUEST | THREAD_CHECKPOINT_REQUEST) - blne MterpSuspendCheck @ (self) + bl MterpSuspendCheck @ (self) + cmp r0, #0 + bne MterpFallback GET_INST_OPCODE ip @ extract opcode from rINST GOTO_OPCODE ip @ jump to next instruction /* + * On-stack replacement has happened, and now we've returned from the compiled method. + */ +MterpOnStackReplacement: +#if MTERP_LOGGING + mov r0, rSELF + add r1, rFP, #OFF_FP_SHADOWFRAME + mov r2, rINST + bl MterpLogOSR +#endif + mov r0, #1 @ Signal normal return + b MterpDone + +/* * Bail out to reference interpreter. */ MterpFallback: diff --git a/runtime/interpreter/mterp/out/mterp_arm64.S b/runtime/interpreter/mterp/out/mterp_arm64.S index e9d28abf8b..e4825f0489 100644 --- a/runtime/interpreter/mterp/out/mterp_arm64.S +++ b/runtime/interpreter/mterp/out/mterp_arm64.S @@ -94,6 +94,9 @@ codes. */ #include "asm_support.h" +#define MTERP_PROFILE_BRANCHES 1 +#define MTERP_LOGGING 0 + /* During bringup, we'll use the shadow frame model instead of xFP */ /* single-purpose registers, given names for clarity */ #define xPC x20 @@ -121,14 +124,6 @@ codes. #define OFF_FP_SHADOWFRAME (-SHADOWFRAME_VREGS_OFFSET) /* - * - * The reference interpreter performs explicit suspect checks, which is somewhat wasteful. - * Dalvik's interpreter folded suspend checks into the jump table mechanism, and eventually - * mterp should do so as well. - */ -#define MTERP_SUSPEND 0 - -/* * "export" the PC to dex_pc field in the shadow frame, f/b/o future exception objects. Must * be done *before* something throws. * @@ -1087,26 +1082,23 @@ artMterpAsmInstructionStart = .L_op_nop */ /* goto +AA */ /* tuning: use sbfx for 6t2+ targets */ -#if MTERP_SUSPEND - mov w0, wINST, lsl #16 // w0<- AAxx0000 - movs w1, w0, asr #24 // w1<- ssssssAA (sign-extended) - add w2, w1, w1 // w2<- byte offset, set flags - // If backwards branch refresh rIBASE - ldrmi rIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET] // refresh handler base - FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST - GET_INST_OPCODE ip // extract opcode from wINST - GOTO_OPCODE ip // jump to next instruction -#else - ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] // Preload flags for MterpCheckSuspendAndContinue lsl w0, wINST, #16 // w0<- AAxx0000 - asr w0, w0, #24 // w0<- ssssssAA (sign-extended) - adds w1, w0, w0 // Convert dalvik offset to byte offset, setting flags + asr wINST, w0, #24 // wINST<- ssssssAA (sign-extended) +#if MTERP_PROFILE_BRANCHES + EXPORT_PC + mov x0, xSELF + add x1, xFP, #OFF_FP_SHADOWFRAME + sbfm x2, xINST, 0, 31 + bl MterpProfileBranch // (self, shadow_frame, offset) + cbnz w0, MterpOnStackReplacement // Note: offset must be in wINST +#endif + ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] // Preload flags for MterpCheckSuspendAndContinue + adds w1, wINST, wINST // Convert dalvik offset to byte offset, setting flags FETCH_ADVANCE_INST_RB w1 // load wINST and advance xPC // If backwards branch refresh rIBASE b.mi MterpCheckSuspendAndContinue GET_INST_OPCODE ip // extract opcode from wINST GOTO_OPCODE ip // jump to next instruction -#endif /* ------------------------------ */ .balign 128 @@ -1119,22 +1111,21 @@ artMterpAsmInstructionStart = .L_op_nop * double to get a byte offset. */ /* goto/16 +AAAA */ -#if MTERP_SUSPEND - FETCH_S w0, 1 // w0<- ssssAAAA (sign-extended) - adds w1, w0, w0 // w1<- byte offset, flags set - FETCH_ADVANCE_INST_RB w1 // update rPC, load rINST - ldrmi xIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET] // refresh handler base - GET_INST_OPCODE ip // extract opcode from rINST - GOTO_OPCODE ip // jump to next instruction -#else - FETCH_S w0, 1 // w0<- ssssAAAA (sign-extended) + FETCH_S wINST, 1 // wINST<- ssssAAAA (sign-extended) +#if MTERP_PROFILE_BRANCHES + EXPORT_PC + mov x0, xSELF + add x1, xFP, #OFF_FP_SHADOWFRAME + sbfm x2, xINST, 0, 31 + bl MterpProfileBranch // (self, shadow_frame, offset) + cbnz w0, MterpOnStackReplacement // Note: offset must be in xINST +#endif ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] - adds w1, w0, w0 // w1<- byte offset, flags set + adds w1, wINST, wINST // w1<- byte offset, flags set FETCH_ADVANCE_INST_RB w1 // update rPC, load rINST b.mi MterpCheckSuspendAndContinue GET_INST_OPCODE ip // extract opcode from rINST GOTO_OPCODE ip // jump to next instruction -#endif /* ------------------------------ */ .balign 128 @@ -1152,26 +1143,23 @@ artMterpAsmInstructionStart = .L_op_nop * offset to byte offset. */ /* goto/32 +AAAAAAAA */ -#if MTERP_SUSPEND - FETCH w0, 1 // w0<- aaaa (lo) - FETCH w1, 2 // w1<- AAAA (hi) - orr w0, w0, w1, lsl #16 // w0<- AAAAaaaa - adds w1, w0, w0 // w1<- byte offset - FETCH_ADVANCE_INST_RB w1 // update rPC, load xINST - ldrle xIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET] // refresh handler base - GET_INST_OPCODE ip // extract opcode from xINST - GOTO_OPCODE ip // jump to next instruction -#else FETCH w0, 1 // w0<- aaaa (lo) FETCH w1, 2 // w1<- AAAA (hi) + orr wINST, w0, w1, lsl #16 // wINST<- AAAAaaaa +#if MTERP_PROFILE_BRANCHES + EXPORT_PC + mov x0, xSELF + add x1, xFP, #OFF_FP_SHADOWFRAME + sbfm x2, xINST, 0, 31 + bl MterpProfileBranch // (self, shadow_frame, offset) + cbnz w0, MterpOnStackReplacement // Note: offset must be in xINST +#endif ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] - orr w0, w0, w1, lsl #16 // w0<- AAAAaaaa - adds w1, w0, w0 // w1<- byte offset + adds w1, wINST, wINST // w1<- byte offset FETCH_ADVANCE_INST_RB w1 // update rPC, load xINST b.le MterpCheckSuspendAndContinue GET_INST_OPCODE ip // extract opcode from xINST GOTO_OPCODE ip // jump to next instruction -#endif /* ------------------------------ */ .balign 128 @@ -1187,20 +1175,6 @@ artMterpAsmInstructionStart = .L_op_nop * for: packed-switch, sparse-switch */ /* op vAA, +BBBB */ -#if MTERP_SUSPEND - FETCH w0, 1 // w0<- bbbb (lo) - FETCH w1, 2 // w1<- BBBB (hi) - mov w3, wINST, lsr #8 // w3<- AA - orr w0, w0, w1, lsl #16 // w0<- BBBBbbbb - GET_VREG w1, w3 // w1<- vAA - add w0, rPC, w0, lsl #1 // w0<- PC + BBBBbbbb*2 - bl MterpDoPackedSwitch // w0<- code-unit branch offset - adds w1, w0, w0 // w1<- byte offset; clear V - ldrle rIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET] // refresh handler base - FETCH_ADVANCE_INST_RB w1 // update rPC, load wINST - GET_INST_OPCODE ip // extract opcode from wINST - GOTO_OPCODE ip // jump to next instruction -#else FETCH w0, 1 // w0<- bbbb (lo) FETCH w1, 2 // w1<- BBBB (hi) lsr w3, wINST, #8 // w3<- AA @@ -1208,13 +1182,21 @@ artMterpAsmInstructionStart = .L_op_nop GET_VREG w1, w3 // w1<- vAA add x0, xPC, w0, lsl #1 // w0<- PC + BBBBbbbb*2 bl MterpDoPackedSwitch // w0<- code-unit branch offset + sbfm xINST, x0, 0, 31 +#if MTERP_PROFILE_BRANCHES + EXPORT_PC + mov x0, xSELF + add x1, xFP, #OFF_FP_SHADOWFRAME + mov x2, xINST + bl MterpProfileBranch // (self, shadow_frame, offset) + cbnz w0, MterpOnStackReplacement +#endif ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] - adds w1, w0, w0 // w1<- byte offset; clear V + adds w1, wINST, wINST // w1<- byte offset; clear V FETCH_ADVANCE_INST_RB w1 // update rPC, load wINST b.le MterpCheckSuspendAndContinue GET_INST_OPCODE ip // extract opcode from wINST GOTO_OPCODE ip // jump to next instruction -#endif /* ------------------------------ */ .balign 128 @@ -1231,20 +1213,6 @@ artMterpAsmInstructionStart = .L_op_nop * for: packed-switch, sparse-switch */ /* op vAA, +BBBB */ -#if MTERP_SUSPEND - FETCH w0, 1 // w0<- bbbb (lo) - FETCH w1, 2 // w1<- BBBB (hi) - mov w3, wINST, lsr #8 // w3<- AA - orr w0, w0, w1, lsl #16 // w0<- BBBBbbbb - GET_VREG w1, w3 // w1<- vAA - add w0, rPC, w0, lsl #1 // w0<- PC + BBBBbbbb*2 - bl MterpDoSparseSwitch // w0<- code-unit branch offset - adds w1, w0, w0 // w1<- byte offset; clear V - ldrle rIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET] // refresh handler base - FETCH_ADVANCE_INST_RB w1 // update rPC, load wINST - GET_INST_OPCODE ip // extract opcode from wINST - GOTO_OPCODE ip // jump to next instruction -#else FETCH w0, 1 // w0<- bbbb (lo) FETCH w1, 2 // w1<- BBBB (hi) lsr w3, wINST, #8 // w3<- AA @@ -1252,13 +1220,21 @@ artMterpAsmInstructionStart = .L_op_nop GET_VREG w1, w3 // w1<- vAA add x0, xPC, w0, lsl #1 // w0<- PC + BBBBbbbb*2 bl MterpDoSparseSwitch // w0<- code-unit branch offset + sbfm xINST, x0, 0, 31 +#if MTERP_PROFILE_BRANCHES + EXPORT_PC + mov x0, xSELF + add x1, xFP, #OFF_FP_SHADOWFRAME + mov x2, xINST + bl MterpProfileBranch // (self, shadow_frame, offset) + cbnz w0, MterpOnStackReplacement +#endif ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] - adds w1, w0, w0 // w1<- byte offset; clear V + adds w1, wINST, wINST // w1<- byte offset; clear V FETCH_ADVANCE_INST_RB w1 // update rPC, load wINST b.le MterpCheckSuspendAndContinue GET_INST_OPCODE ip // extract opcode from wINST GOTO_OPCODE ip // jump to next instruction -#endif /* ------------------------------ */ @@ -1396,17 +1372,28 @@ artMterpAsmInstructionStart = .L_op_nop * For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le */ /* if-cmp vA, vB, +CCCC */ -#if MTERP_SUSPEND - mov w1, wINST, lsr #12 // w1<- B +#if MTERP_PROFILE_BRANCHES + lsr w1, wINST, #12 // w1<- B ubfx w0, wINST, #8, #4 // w0<- A GET_VREG w3, w1 // w3<- vB GET_VREG w2, w0 // w2<- vA - FETCH_S w1, 1 // w1<- branch offset, in code units + FETCH_S wINST, 1 // wINST<- branch offset, in code units cmp w2, w3 // compare (vA, vB) - moveq w1, #2 // w1<- BYTE branch dist for not-taken - adds w2, w1, w1 // convert to bytes, check sign + b.eq .L_op_if_eq_taken + FETCH_ADVANCE_INST 2 // update rPC, load wINST + GET_INST_OPCODE ip // extract opcode from wINST + GOTO_OPCODE ip // jump to next instruction +.L_op_if_eq_taken: + EXPORT_PC + mov x0, xSELF + add x1, xFP, #OFF_FP_SHADOWFRAME + sbfm x2, xINST, 0, 31 // Sign extend branch offset + bl MterpProfileBranch // (self, shadow_frame, offset) + cbnz w0, MterpOnStackReplacement // Note: offset must be in xINST + ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] + adds w2, wINST, wINST // convert to bytes, check sign FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST - ldrmi rIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET] // refresh rIBASE + b.mi MterpCheckSuspendAndContinue GET_INST_OPCODE ip // extract opcode from wINST GOTO_OPCODE ip // jump to next instruction #else @@ -1415,11 +1402,11 @@ artMterpAsmInstructionStart = .L_op_nop GET_VREG w3, w1 // w3<- vB GET_VREG w2, w0 // w2<- vA FETCH_S w1, 1 // w1<- branch offset, in code units - ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] mov w0, #2 // Offset if branch not taken cmp w2, w3 // compare (vA, vB) - csel w1, w1, w0, eq // Branch if true - adds w2, w1, w1 // convert to bytes, check sign + csel wINST, w1, w0, eq // Branch if true, stashing result in callee save reg. + ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] + adds w2, wINST, wINST // convert to bytes, check sign FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST b.mi MterpCheckSuspendAndContinue GET_INST_OPCODE ip // extract opcode from wINST @@ -1440,17 +1427,28 @@ artMterpAsmInstructionStart = .L_op_nop * For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le */ /* if-cmp vA, vB, +CCCC */ -#if MTERP_SUSPEND - mov w1, wINST, lsr #12 // w1<- B +#if MTERP_PROFILE_BRANCHES + lsr w1, wINST, #12 // w1<- B ubfx w0, wINST, #8, #4 // w0<- A GET_VREG w3, w1 // w3<- vB GET_VREG w2, w0 // w2<- vA - FETCH_S w1, 1 // w1<- branch offset, in code units + FETCH_S wINST, 1 // wINST<- branch offset, in code units cmp w2, w3 // compare (vA, vB) - movne w1, #2 // w1<- BYTE branch dist for not-taken - adds w2, w1, w1 // convert to bytes, check sign + b.ne .L_op_if_ne_taken + FETCH_ADVANCE_INST 2 // update rPC, load wINST + GET_INST_OPCODE ip // extract opcode from wINST + GOTO_OPCODE ip // jump to next instruction +.L_op_if_ne_taken: + EXPORT_PC + mov x0, xSELF + add x1, xFP, #OFF_FP_SHADOWFRAME + sbfm x2, xINST, 0, 31 // Sign extend branch offset + bl MterpProfileBranch // (self, shadow_frame, offset) + cbnz w0, MterpOnStackReplacement // Note: offset must be in xINST + ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] + adds w2, wINST, wINST // convert to bytes, check sign FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST - ldrmi rIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET] // refresh rIBASE + b.mi MterpCheckSuspendAndContinue GET_INST_OPCODE ip // extract opcode from wINST GOTO_OPCODE ip // jump to next instruction #else @@ -1459,11 +1457,11 @@ artMterpAsmInstructionStart = .L_op_nop GET_VREG w3, w1 // w3<- vB GET_VREG w2, w0 // w2<- vA FETCH_S w1, 1 // w1<- branch offset, in code units - ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] mov w0, #2 // Offset if branch not taken cmp w2, w3 // compare (vA, vB) - csel w1, w1, w0, ne // Branch if true - adds w2, w1, w1 // convert to bytes, check sign + csel wINST, w1, w0, ne // Branch if true, stashing result in callee save reg. + ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] + adds w2, wINST, wINST // convert to bytes, check sign FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST b.mi MterpCheckSuspendAndContinue GET_INST_OPCODE ip // extract opcode from wINST @@ -1484,17 +1482,28 @@ artMterpAsmInstructionStart = .L_op_nop * For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le */ /* if-cmp vA, vB, +CCCC */ -#if MTERP_SUSPEND - mov w1, wINST, lsr #12 // w1<- B +#if MTERP_PROFILE_BRANCHES + lsr w1, wINST, #12 // w1<- B ubfx w0, wINST, #8, #4 // w0<- A GET_VREG w3, w1 // w3<- vB GET_VREG w2, w0 // w2<- vA - FETCH_S w1, 1 // w1<- branch offset, in code units + FETCH_S wINST, 1 // wINST<- branch offset, in code units cmp w2, w3 // compare (vA, vB) - movlt w1, #2 // w1<- BYTE branch dist for not-taken - adds w2, w1, w1 // convert to bytes, check sign + b.lt .L_op_if_lt_taken + FETCH_ADVANCE_INST 2 // update rPC, load wINST + GET_INST_OPCODE ip // extract opcode from wINST + GOTO_OPCODE ip // jump to next instruction +.L_op_if_lt_taken: + EXPORT_PC + mov x0, xSELF + add x1, xFP, #OFF_FP_SHADOWFRAME + sbfm x2, xINST, 0, 31 // Sign extend branch offset + bl MterpProfileBranch // (self, shadow_frame, offset) + cbnz w0, MterpOnStackReplacement // Note: offset must be in xINST + ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] + adds w2, wINST, wINST // convert to bytes, check sign FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST - ldrmi rIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET] // refresh rIBASE + b.mi MterpCheckSuspendAndContinue GET_INST_OPCODE ip // extract opcode from wINST GOTO_OPCODE ip // jump to next instruction #else @@ -1503,11 +1512,11 @@ artMterpAsmInstructionStart = .L_op_nop GET_VREG w3, w1 // w3<- vB GET_VREG w2, w0 // w2<- vA FETCH_S w1, 1 // w1<- branch offset, in code units - ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] mov w0, #2 // Offset if branch not taken cmp w2, w3 // compare (vA, vB) - csel w1, w1, w0, lt // Branch if true - adds w2, w1, w1 // convert to bytes, check sign + csel wINST, w1, w0, lt // Branch if true, stashing result in callee save reg. + ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] + adds w2, wINST, wINST // convert to bytes, check sign FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST b.mi MterpCheckSuspendAndContinue GET_INST_OPCODE ip // extract opcode from wINST @@ -1528,17 +1537,28 @@ artMterpAsmInstructionStart = .L_op_nop * For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le */ /* if-cmp vA, vB, +CCCC */ -#if MTERP_SUSPEND - mov w1, wINST, lsr #12 // w1<- B +#if MTERP_PROFILE_BRANCHES + lsr w1, wINST, #12 // w1<- B ubfx w0, wINST, #8, #4 // w0<- A GET_VREG w3, w1 // w3<- vB GET_VREG w2, w0 // w2<- vA - FETCH_S w1, 1 // w1<- branch offset, in code units + FETCH_S wINST, 1 // wINST<- branch offset, in code units cmp w2, w3 // compare (vA, vB) - movge w1, #2 // w1<- BYTE branch dist for not-taken - adds w2, w1, w1 // convert to bytes, check sign + b.ge .L_op_if_ge_taken + FETCH_ADVANCE_INST 2 // update rPC, load wINST + GET_INST_OPCODE ip // extract opcode from wINST + GOTO_OPCODE ip // jump to next instruction +.L_op_if_ge_taken: + EXPORT_PC + mov x0, xSELF + add x1, xFP, #OFF_FP_SHADOWFRAME + sbfm x2, xINST, 0, 31 // Sign extend branch offset + bl MterpProfileBranch // (self, shadow_frame, offset) + cbnz w0, MterpOnStackReplacement // Note: offset must be in xINST + ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] + adds w2, wINST, wINST // convert to bytes, check sign FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST - ldrmi rIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET] // refresh rIBASE + b.mi MterpCheckSuspendAndContinue GET_INST_OPCODE ip // extract opcode from wINST GOTO_OPCODE ip // jump to next instruction #else @@ -1547,11 +1567,11 @@ artMterpAsmInstructionStart = .L_op_nop GET_VREG w3, w1 // w3<- vB GET_VREG w2, w0 // w2<- vA FETCH_S w1, 1 // w1<- branch offset, in code units - ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] mov w0, #2 // Offset if branch not taken cmp w2, w3 // compare (vA, vB) - csel w1, w1, w0, ge // Branch if true - adds w2, w1, w1 // convert to bytes, check sign + csel wINST, w1, w0, ge // Branch if true, stashing result in callee save reg. + ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] + adds w2, wINST, wINST // convert to bytes, check sign FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST b.mi MterpCheckSuspendAndContinue GET_INST_OPCODE ip // extract opcode from wINST @@ -1572,17 +1592,28 @@ artMterpAsmInstructionStart = .L_op_nop * For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le */ /* if-cmp vA, vB, +CCCC */ -#if MTERP_SUSPEND - mov w1, wINST, lsr #12 // w1<- B +#if MTERP_PROFILE_BRANCHES + lsr w1, wINST, #12 // w1<- B ubfx w0, wINST, #8, #4 // w0<- A GET_VREG w3, w1 // w3<- vB GET_VREG w2, w0 // w2<- vA - FETCH_S w1, 1 // w1<- branch offset, in code units + FETCH_S wINST, 1 // wINST<- branch offset, in code units cmp w2, w3 // compare (vA, vB) - movgt w1, #2 // w1<- BYTE branch dist for not-taken - adds w2, w1, w1 // convert to bytes, check sign + b.gt .L_op_if_gt_taken + FETCH_ADVANCE_INST 2 // update rPC, load wINST + GET_INST_OPCODE ip // extract opcode from wINST + GOTO_OPCODE ip // jump to next instruction +.L_op_if_gt_taken: + EXPORT_PC + mov x0, xSELF + add x1, xFP, #OFF_FP_SHADOWFRAME + sbfm x2, xINST, 0, 31 // Sign extend branch offset + bl MterpProfileBranch // (self, shadow_frame, offset) + cbnz w0, MterpOnStackReplacement // Note: offset must be in xINST + ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] + adds w2, wINST, wINST // convert to bytes, check sign FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST - ldrmi rIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET] // refresh rIBASE + b.mi MterpCheckSuspendAndContinue GET_INST_OPCODE ip // extract opcode from wINST GOTO_OPCODE ip // jump to next instruction #else @@ -1591,11 +1622,11 @@ artMterpAsmInstructionStart = .L_op_nop GET_VREG w3, w1 // w3<- vB GET_VREG w2, w0 // w2<- vA FETCH_S w1, 1 // w1<- branch offset, in code units - ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] mov w0, #2 // Offset if branch not taken cmp w2, w3 // compare (vA, vB) - csel w1, w1, w0, gt // Branch if true - adds w2, w1, w1 // convert to bytes, check sign + csel wINST, w1, w0, gt // Branch if true, stashing result in callee save reg. + ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] + adds w2, wINST, wINST // convert to bytes, check sign FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST b.mi MterpCheckSuspendAndContinue GET_INST_OPCODE ip // extract opcode from wINST @@ -1616,17 +1647,28 @@ artMterpAsmInstructionStart = .L_op_nop * For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le */ /* if-cmp vA, vB, +CCCC */ -#if MTERP_SUSPEND - mov w1, wINST, lsr #12 // w1<- B +#if MTERP_PROFILE_BRANCHES + lsr w1, wINST, #12 // w1<- B ubfx w0, wINST, #8, #4 // w0<- A GET_VREG w3, w1 // w3<- vB GET_VREG w2, w0 // w2<- vA - FETCH_S w1, 1 // w1<- branch offset, in code units + FETCH_S wINST, 1 // wINST<- branch offset, in code units cmp w2, w3 // compare (vA, vB) - movle w1, #2 // w1<- BYTE branch dist for not-taken - adds w2, w1, w1 // convert to bytes, check sign + b.le .L_op_if_le_taken + FETCH_ADVANCE_INST 2 // update rPC, load wINST + GET_INST_OPCODE ip // extract opcode from wINST + GOTO_OPCODE ip // jump to next instruction +.L_op_if_le_taken: + EXPORT_PC + mov x0, xSELF + add x1, xFP, #OFF_FP_SHADOWFRAME + sbfm x2, xINST, 0, 31 // Sign extend branch offset + bl MterpProfileBranch // (self, shadow_frame, offset) + cbnz w0, MterpOnStackReplacement // Note: offset must be in xINST + ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] + adds w2, wINST, wINST // convert to bytes, check sign FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST - ldrmi rIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET] // refresh rIBASE + b.mi MterpCheckSuspendAndContinue GET_INST_OPCODE ip // extract opcode from wINST GOTO_OPCODE ip // jump to next instruction #else @@ -1635,11 +1677,11 @@ artMterpAsmInstructionStart = .L_op_nop GET_VREG w3, w1 // w3<- vB GET_VREG w2, w0 // w2<- vA FETCH_S w1, 1 // w1<- branch offset, in code units - ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] mov w0, #2 // Offset if branch not taken cmp w2, w3 // compare (vA, vB) - csel w1, w1, w0, le // Branch if true - adds w2, w1, w1 // convert to bytes, check sign + csel wINST, w1, w0, le // Branch if true, stashing result in callee save reg. + ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] + adds w2, wINST, wINST // convert to bytes, check sign FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST b.mi MterpCheckSuspendAndContinue GET_INST_OPCODE ip // extract opcode from wINST @@ -1660,26 +1702,37 @@ artMterpAsmInstructionStart = .L_op_nop * for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez */ /* if-cmp vAA, +BBBB */ -#if MTERP_SUSPEND - mov w0, wINST, lsr #8 // w0<- AA +#if MTERP_PROFILE_BRANCHES + lsr w0, wINST, #8 // w0<- AA GET_VREG w2, w0 // w2<- vAA - FETCH_S w1, 1 // w1<- branch offset, in code units + FETCH_S wINST, 1 // w1<- branch offset, in code units cmp w2, #0 // compare (vA, 0) - moveq w1, #2 // w1<- inst branch dist for not-taken - adds w1, w1, w1 // convert to bytes & set flags - FETCH_ADVANCE_INST_RB w1 // update rPC, load wINST - ldrmi rIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET] // refresh table base + b.eq .L_op_if_eqz_taken + FETCH_ADVANCE_INST 2 // update rPC, load wINST + GET_INST_OPCODE ip // extract opcode from wINST + GOTO_OPCODE ip // jump to next instruction +.L_op_if_eqz_taken: + EXPORT_PC + mov x0, xSELF + add x1, xFP, #OFF_FP_SHADOWFRAME + sbfm x2, xINST, 0, 31 + bl MterpProfileBranch // (self, shadow_frame, offset) + cbnz w0, MterpOnStackReplacement // Note: offset must be in wINST + ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] + adds w2, wINST, wINST // convert to bytes & set flags + FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST + b.mi MterpCheckSuspendAndContinue GET_INST_OPCODE ip // extract opcode from wINST GOTO_OPCODE ip // jump to next instruction #else lsr w0, wINST, #8 // w0<- AA GET_VREG w2, w0 // w2<- vAA FETCH_S w1, 1 // w1<- branch offset, in code units - ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] mov w0, #2 // Branch offset if not taken cmp w2, #0 // compare (vA, 0) - csel w1, w1, w0, eq // Branch if true - adds w2, w1, w1 // convert to bytes & set flags + csel wINST, w1, w0, eq // Branch if true, stashing result in callee save reg + ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] + adds w2, wINST, wINST // convert to bytes & set flags FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST b.mi MterpCheckSuspendAndContinue GET_INST_OPCODE ip // extract opcode from wINST @@ -1700,26 +1753,37 @@ artMterpAsmInstructionStart = .L_op_nop * for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez */ /* if-cmp vAA, +BBBB */ -#if MTERP_SUSPEND - mov w0, wINST, lsr #8 // w0<- AA +#if MTERP_PROFILE_BRANCHES + lsr w0, wINST, #8 // w0<- AA GET_VREG w2, w0 // w2<- vAA - FETCH_S w1, 1 // w1<- branch offset, in code units + FETCH_S wINST, 1 // w1<- branch offset, in code units cmp w2, #0 // compare (vA, 0) - movne w1, #2 // w1<- inst branch dist for not-taken - adds w1, w1, w1 // convert to bytes & set flags - FETCH_ADVANCE_INST_RB w1 // update rPC, load wINST - ldrmi rIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET] // refresh table base + b.ne .L_op_if_nez_taken + FETCH_ADVANCE_INST 2 // update rPC, load wINST + GET_INST_OPCODE ip // extract opcode from wINST + GOTO_OPCODE ip // jump to next instruction +.L_op_if_nez_taken: + EXPORT_PC + mov x0, xSELF + add x1, xFP, #OFF_FP_SHADOWFRAME + sbfm x2, xINST, 0, 31 + bl MterpProfileBranch // (self, shadow_frame, offset) + cbnz w0, MterpOnStackReplacement // Note: offset must be in wINST + ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] + adds w2, wINST, wINST // convert to bytes & set flags + FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST + b.mi MterpCheckSuspendAndContinue GET_INST_OPCODE ip // extract opcode from wINST GOTO_OPCODE ip // jump to next instruction #else lsr w0, wINST, #8 // w0<- AA GET_VREG w2, w0 // w2<- vAA FETCH_S w1, 1 // w1<- branch offset, in code units - ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] mov w0, #2 // Branch offset if not taken cmp w2, #0 // compare (vA, 0) - csel w1, w1, w0, ne // Branch if true - adds w2, w1, w1 // convert to bytes & set flags + csel wINST, w1, w0, ne // Branch if true, stashing result in callee save reg + ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] + adds w2, wINST, wINST // convert to bytes & set flags FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST b.mi MterpCheckSuspendAndContinue GET_INST_OPCODE ip // extract opcode from wINST @@ -1740,26 +1804,37 @@ artMterpAsmInstructionStart = .L_op_nop * for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez */ /* if-cmp vAA, +BBBB */ -#if MTERP_SUSPEND - mov w0, wINST, lsr #8 // w0<- AA +#if MTERP_PROFILE_BRANCHES + lsr w0, wINST, #8 // w0<- AA GET_VREG w2, w0 // w2<- vAA - FETCH_S w1, 1 // w1<- branch offset, in code units + FETCH_S wINST, 1 // w1<- branch offset, in code units cmp w2, #0 // compare (vA, 0) - movlt w1, #2 // w1<- inst branch dist for not-taken - adds w1, w1, w1 // convert to bytes & set flags - FETCH_ADVANCE_INST_RB w1 // update rPC, load wINST - ldrmi rIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET] // refresh table base + b.lt .L_op_if_ltz_taken + FETCH_ADVANCE_INST 2 // update rPC, load wINST + GET_INST_OPCODE ip // extract opcode from wINST + GOTO_OPCODE ip // jump to next instruction +.L_op_if_ltz_taken: + EXPORT_PC + mov x0, xSELF + add x1, xFP, #OFF_FP_SHADOWFRAME + sbfm x2, xINST, 0, 31 + bl MterpProfileBranch // (self, shadow_frame, offset) + cbnz w0, MterpOnStackReplacement // Note: offset must be in wINST + ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] + adds w2, wINST, wINST // convert to bytes & set flags + FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST + b.mi MterpCheckSuspendAndContinue GET_INST_OPCODE ip // extract opcode from wINST GOTO_OPCODE ip // jump to next instruction #else lsr w0, wINST, #8 // w0<- AA GET_VREG w2, w0 // w2<- vAA FETCH_S w1, 1 // w1<- branch offset, in code units - ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] mov w0, #2 // Branch offset if not taken cmp w2, #0 // compare (vA, 0) - csel w1, w1, w0, lt // Branch if true - adds w2, w1, w1 // convert to bytes & set flags + csel wINST, w1, w0, lt // Branch if true, stashing result in callee save reg + ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] + adds w2, wINST, wINST // convert to bytes & set flags FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST b.mi MterpCheckSuspendAndContinue GET_INST_OPCODE ip // extract opcode from wINST @@ -1780,26 +1855,37 @@ artMterpAsmInstructionStart = .L_op_nop * for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez */ /* if-cmp vAA, +BBBB */ -#if MTERP_SUSPEND - mov w0, wINST, lsr #8 // w0<- AA +#if MTERP_PROFILE_BRANCHES + lsr w0, wINST, #8 // w0<- AA GET_VREG w2, w0 // w2<- vAA - FETCH_S w1, 1 // w1<- branch offset, in code units + FETCH_S wINST, 1 // w1<- branch offset, in code units cmp w2, #0 // compare (vA, 0) - movge w1, #2 // w1<- inst branch dist for not-taken - adds w1, w1, w1 // convert to bytes & set flags - FETCH_ADVANCE_INST_RB w1 // update rPC, load wINST - ldrmi rIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET] // refresh table base + b.ge .L_op_if_gez_taken + FETCH_ADVANCE_INST 2 // update rPC, load wINST + GET_INST_OPCODE ip // extract opcode from wINST + GOTO_OPCODE ip // jump to next instruction +.L_op_if_gez_taken: + EXPORT_PC + mov x0, xSELF + add x1, xFP, #OFF_FP_SHADOWFRAME + sbfm x2, xINST, 0, 31 + bl MterpProfileBranch // (self, shadow_frame, offset) + cbnz w0, MterpOnStackReplacement // Note: offset must be in wINST + ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] + adds w2, wINST, wINST // convert to bytes & set flags + FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST + b.mi MterpCheckSuspendAndContinue GET_INST_OPCODE ip // extract opcode from wINST GOTO_OPCODE ip // jump to next instruction #else lsr w0, wINST, #8 // w0<- AA GET_VREG w2, w0 // w2<- vAA FETCH_S w1, 1 // w1<- branch offset, in code units - ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] mov w0, #2 // Branch offset if not taken cmp w2, #0 // compare (vA, 0) - csel w1, w1, w0, ge // Branch if true - adds w2, w1, w1 // convert to bytes & set flags + csel wINST, w1, w0, ge // Branch if true, stashing result in callee save reg + ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] + adds w2, wINST, wINST // convert to bytes & set flags FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST b.mi MterpCheckSuspendAndContinue GET_INST_OPCODE ip // extract opcode from wINST @@ -1820,26 +1906,37 @@ artMterpAsmInstructionStart = .L_op_nop * for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez */ /* if-cmp vAA, +BBBB */ -#if MTERP_SUSPEND - mov w0, wINST, lsr #8 // w0<- AA +#if MTERP_PROFILE_BRANCHES + lsr w0, wINST, #8 // w0<- AA GET_VREG w2, w0 // w2<- vAA - FETCH_S w1, 1 // w1<- branch offset, in code units + FETCH_S wINST, 1 // w1<- branch offset, in code units cmp w2, #0 // compare (vA, 0) - movgt w1, #2 // w1<- inst branch dist for not-taken - adds w1, w1, w1 // convert to bytes & set flags - FETCH_ADVANCE_INST_RB w1 // update rPC, load wINST - ldrmi rIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET] // refresh table base + b.gt .L_op_if_gtz_taken + FETCH_ADVANCE_INST 2 // update rPC, load wINST + GET_INST_OPCODE ip // extract opcode from wINST + GOTO_OPCODE ip // jump to next instruction +.L_op_if_gtz_taken: + EXPORT_PC + mov x0, xSELF + add x1, xFP, #OFF_FP_SHADOWFRAME + sbfm x2, xINST, 0, 31 + bl MterpProfileBranch // (self, shadow_frame, offset) + cbnz w0, MterpOnStackReplacement // Note: offset must be in wINST + ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] + adds w2, wINST, wINST // convert to bytes & set flags + FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST + b.mi MterpCheckSuspendAndContinue GET_INST_OPCODE ip // extract opcode from wINST GOTO_OPCODE ip // jump to next instruction #else lsr w0, wINST, #8 // w0<- AA GET_VREG w2, w0 // w2<- vAA FETCH_S w1, 1 // w1<- branch offset, in code units - ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] mov w0, #2 // Branch offset if not taken cmp w2, #0 // compare (vA, 0) - csel w1, w1, w0, gt // Branch if true - adds w2, w1, w1 // convert to bytes & set flags + csel wINST, w1, w0, gt // Branch if true, stashing result in callee save reg + ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] + adds w2, wINST, wINST // convert to bytes & set flags FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST b.mi MterpCheckSuspendAndContinue GET_INST_OPCODE ip // extract opcode from wINST @@ -1860,26 +1957,37 @@ artMterpAsmInstructionStart = .L_op_nop * for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez */ /* if-cmp vAA, +BBBB */ -#if MTERP_SUSPEND - mov w0, wINST, lsr #8 // w0<- AA +#if MTERP_PROFILE_BRANCHES + lsr w0, wINST, #8 // w0<- AA GET_VREG w2, w0 // w2<- vAA - FETCH_S w1, 1 // w1<- branch offset, in code units + FETCH_S wINST, 1 // w1<- branch offset, in code units cmp w2, #0 // compare (vA, 0) - movle w1, #2 // w1<- inst branch dist for not-taken - adds w1, w1, w1 // convert to bytes & set flags - FETCH_ADVANCE_INST_RB w1 // update rPC, load wINST - ldrmi rIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET] // refresh table base + b.le .L_op_if_lez_taken + FETCH_ADVANCE_INST 2 // update rPC, load wINST + GET_INST_OPCODE ip // extract opcode from wINST + GOTO_OPCODE ip // jump to next instruction +.L_op_if_lez_taken: + EXPORT_PC + mov x0, xSELF + add x1, xFP, #OFF_FP_SHADOWFRAME + sbfm x2, xINST, 0, 31 + bl MterpProfileBranch // (self, shadow_frame, offset) + cbnz w0, MterpOnStackReplacement // Note: offset must be in wINST + ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] + adds w2, wINST, wINST // convert to bytes & set flags + FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST + b.mi MterpCheckSuspendAndContinue GET_INST_OPCODE ip // extract opcode from wINST GOTO_OPCODE ip // jump to next instruction #else lsr w0, wINST, #8 // w0<- AA GET_VREG w2, w0 // w2<- vAA FETCH_S w1, 1 // w1<- branch offset, in code units - ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] mov w0, #2 // Branch offset if not taken cmp w2, #0 // compare (vA, 0) - csel w1, w1, w0, le // Branch if true - adds w2, w1, w1 // convert to bytes & set flags + csel wINST, w1, w0, le // Branch if true, stashing result in callee save reg + ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] + adds w2, wINST, wINST // convert to bytes & set flags FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST b.mi MterpCheckSuspendAndContinue GET_INST_OPCODE ip // extract opcode from wINST @@ -2401,6 +2509,7 @@ artMterpAsmInstructionStart = .L_op_nop mov x3, xSELF // w3<- self bl artGet32InstanceFromCode ldr x3, [xSELF, #THREAD_EXCEPTION_OFFSET] + ubfx w2, wINST, #8, #4 // w2<- A PREFETCH_INST 2 cbnz x3, MterpPossibleException // bail out @@ -2457,6 +2566,7 @@ artMterpAsmInstructionStart = .L_op_nop mov x3, xSELF // w3<- self bl artGetObjInstanceFromCode ldr x3, [xSELF, #THREAD_EXCEPTION_OFFSET] + ubfx w2, wINST, #8, #4 // w2<- A PREFETCH_INST 2 cbnz x3, MterpPossibleException // bail out @@ -2488,6 +2598,7 @@ artMterpAsmInstructionStart = .L_op_nop mov x3, xSELF // w3<- self bl artGetBooleanInstanceFromCode ldr x3, [xSELF, #THREAD_EXCEPTION_OFFSET] + uxtb w0, w0 ubfx w2, wINST, #8, #4 // w2<- A PREFETCH_INST 2 cbnz x3, MterpPossibleException // bail out @@ -2519,6 +2630,7 @@ artMterpAsmInstructionStart = .L_op_nop mov x3, xSELF // w3<- self bl artGetByteInstanceFromCode ldr x3, [xSELF, #THREAD_EXCEPTION_OFFSET] + sxtb w0, w0 ubfx w2, wINST, #8, #4 // w2<- A PREFETCH_INST 2 cbnz x3, MterpPossibleException // bail out @@ -2550,6 +2662,7 @@ artMterpAsmInstructionStart = .L_op_nop mov x3, xSELF // w3<- self bl artGetCharInstanceFromCode ldr x3, [xSELF, #THREAD_EXCEPTION_OFFSET] + uxth w0, w0 ubfx w2, wINST, #8, #4 // w2<- A PREFETCH_INST 2 cbnz x3, MterpPossibleException // bail out @@ -2581,6 +2694,7 @@ artMterpAsmInstructionStart = .L_op_nop mov x3, xSELF // w3<- self bl artGetShortInstanceFromCode ldr x3, [xSELF, #THREAD_EXCEPTION_OFFSET] + sxth w0, w0 ubfx w2, wINST, #8, #4 // w2<- A PREFETCH_INST 2 cbnz x3, MterpPossibleException // bail out @@ -3158,11 +3272,12 @@ artMterpAsmInstructionStart = .L_op_nop mov x0, xSELF add x1, xFP, #OFF_FP_SHADOWFRAME mov x2, xPC - // and x3, xINST, 0xFFFF mov x3, xINST bl MterpInvokeVirtual cbz w0, MterpException FETCH_ADVANCE_INST 3 + bl MterpShouldSwitchInterpreters + cbnz w0, MterpFallback GET_INST_OPCODE ip GOTO_OPCODE ip @@ -3190,11 +3305,12 @@ artMterpAsmInstructionStart = .L_op_nop mov x0, xSELF add x1, xFP, #OFF_FP_SHADOWFRAME mov x2, xPC - // and x3, xINST, 0xFFFF mov x3, xINST bl MterpInvokeSuper cbz w0, MterpException FETCH_ADVANCE_INST 3 + bl MterpShouldSwitchInterpreters + cbnz w0, MterpFallback GET_INST_OPCODE ip GOTO_OPCODE ip @@ -3222,11 +3338,12 @@ artMterpAsmInstructionStart = .L_op_nop mov x0, xSELF add x1, xFP, #OFF_FP_SHADOWFRAME mov x2, xPC - // and x3, xINST, 0xFFFF mov x3, xINST bl MterpInvokeDirect cbz w0, MterpException FETCH_ADVANCE_INST 3 + bl MterpShouldSwitchInterpreters + cbnz w0, MterpFallback GET_INST_OPCODE ip GOTO_OPCODE ip @@ -3247,11 +3364,12 @@ artMterpAsmInstructionStart = .L_op_nop mov x0, xSELF add x1, xFP, #OFF_FP_SHADOWFRAME mov x2, xPC - // and x3, xINST, 0xFFFF mov x3, xINST bl MterpInvokeStatic cbz w0, MterpException FETCH_ADVANCE_INST 3 + bl MterpShouldSwitchInterpreters + cbnz w0, MterpFallback GET_INST_OPCODE ip GOTO_OPCODE ip @@ -3273,11 +3391,12 @@ artMterpAsmInstructionStart = .L_op_nop mov x0, xSELF add x1, xFP, #OFF_FP_SHADOWFRAME mov x2, xPC - // and x3, xINST, 0xFFFF mov x3, xINST bl MterpInvokeInterface cbz w0, MterpException FETCH_ADVANCE_INST 3 + bl MterpShouldSwitchInterpreters + cbnz w0, MterpFallback GET_INST_OPCODE ip GOTO_OPCODE ip @@ -3320,11 +3439,12 @@ artMterpAsmInstructionStart = .L_op_nop mov x0, xSELF add x1, xFP, #OFF_FP_SHADOWFRAME mov x2, xPC - // and x3, xINST, 0xFFFF mov x3, xINST bl MterpInvokeVirtualRange cbz w0, MterpException FETCH_ADVANCE_INST 3 + bl MterpShouldSwitchInterpreters + cbnz w0, MterpFallback GET_INST_OPCODE ip GOTO_OPCODE ip @@ -3345,11 +3465,12 @@ artMterpAsmInstructionStart = .L_op_nop mov x0, xSELF add x1, xFP, #OFF_FP_SHADOWFRAME mov x2, xPC - // and x3, xINST, 0xFFFF mov x3, xINST bl MterpInvokeSuperRange cbz w0, MterpException FETCH_ADVANCE_INST 3 + bl MterpShouldSwitchInterpreters + cbnz w0, MterpFallback GET_INST_OPCODE ip GOTO_OPCODE ip @@ -3370,11 +3491,12 @@ artMterpAsmInstructionStart = .L_op_nop mov x0, xSELF add x1, xFP, #OFF_FP_SHADOWFRAME mov x2, xPC - // and x3, xINST, 0xFFFF mov x3, xINST bl MterpInvokeDirectRange cbz w0, MterpException FETCH_ADVANCE_INST 3 + bl MterpShouldSwitchInterpreters + cbnz w0, MterpFallback GET_INST_OPCODE ip GOTO_OPCODE ip @@ -3395,11 +3517,12 @@ artMterpAsmInstructionStart = .L_op_nop mov x0, xSELF add x1, xFP, #OFF_FP_SHADOWFRAME mov x2, xPC - // and x3, xINST, 0xFFFF mov x3, xINST bl MterpInvokeStaticRange cbz w0, MterpException FETCH_ADVANCE_INST 3 + bl MterpShouldSwitchInterpreters + cbnz w0, MterpFallback GET_INST_OPCODE ip GOTO_OPCODE ip @@ -3420,11 +3543,12 @@ artMterpAsmInstructionStart = .L_op_nop mov x0, xSELF add x1, xFP, #OFF_FP_SHADOWFRAME mov x2, xPC - // and x3, xINST, 0xFFFF mov x3, xINST bl MterpInvokeInterfaceRange cbz w0, MterpException FETCH_ADVANCE_INST 3 + bl MterpShouldSwitchInterpreters + cbnz w0, MterpFallback GET_INST_OPCODE ip GOTO_OPCODE ip @@ -6852,11 +6976,12 @@ artMterpAsmInstructionStart = .L_op_nop mov x0, xSELF add x1, xFP, #OFF_FP_SHADOWFRAME mov x2, xPC - // and x3, xINST, 0xFFFF mov x3, xINST bl MterpInvokeVirtualQuick cbz w0, MterpException FETCH_ADVANCE_INST 3 + bl MterpShouldSwitchInterpreters + cbnz w0, MterpFallback GET_INST_OPCODE ip GOTO_OPCODE ip @@ -6877,11 +7002,12 @@ artMterpAsmInstructionStart = .L_op_nop mov x0, xSELF add x1, xFP, #OFF_FP_SHADOWFRAME mov x2, xPC - // and x3, xINST, 0xFFFF mov x3, xINST bl MterpInvokeVirtualQuickRange cbz w0, MterpException FETCH_ADVANCE_INST 3 + bl MterpShouldSwitchInterpreters + cbnz w0, MterpFallback GET_INST_OPCODE ip GOTO_OPCODE ip @@ -11565,7 +11691,6 @@ artMterpAsmAltInstructionEnd: * has not yet been thrown. Just bail out to the reference interpreter to deal with it. * TUNING: for consistency, we may want to just go ahead and handle these here. */ -#define MTERP_LOGGING 0 common_errDivideByZero: EXPORT_PC #if MTERP_LOGGING @@ -11654,8 +11779,11 @@ MterpException: ldr xIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET] add xPC, x0, #CODEITEM_INSNS_OFFSET add xPC, xPC, x1, lsl #1 // generate new dex_pc_ptr - str xPC, [xFP, #OFF_FP_DEX_PC_PTR] + /* Do we need to switch interpreters? */ + bl MterpShouldSwitchInterpreters + cbnz w0, MterpFallback /* resume execution at catch block */ + EXPORT_PC FETCH_INST GET_INST_OPCODE ip GOTO_OPCODE ip @@ -11675,10 +11803,24 @@ check1: EXPORT_PC mov x0, xSELF bl MterpSuspendCheck // (self) + cbnz x0, MterpFallback // Something in the environment changed, switch interpreters GET_INST_OPCODE ip // extract opcode from wINST GOTO_OPCODE ip // jump to next instruction /* + * On-stack replacement has happened, and now we've returned from the compiled method. + */ +MterpOnStackReplacement: +#if MTERP_LOGGING + mov x0, xSELF + add x1, xFP, #OFF_FP_SHADOWFRAME + sbfm x2, xINST, 0, 31 + bl MterpLogOSR +#endif + mov x0, #1 // Signal normal return + b MterpDone + +/* * Bail out to reference interpreter. */ MterpFallback: diff --git a/tools/run-jdwp-tests.sh b/tools/run-jdwp-tests.sh index e4af9fa0d7..fda630fe5d 100755 --- a/tools/run-jdwp-tests.sh +++ b/tools/run-jdwp-tests.sh @@ -133,7 +133,7 @@ vogar $vm_command \ --vm-arg -Djpda.settings.transportAddress=127.0.0.1:55107 \ --vm-arg -Djpda.settings.debuggeeJavaPath="$art_debugee $image $debuggee_args" \ --classpath $test_jack \ - --toolchain jack --language JN \ + --toolchain jack \ --vm-arg -Xcompiler-option --vm-arg --debuggable \ $test |