diff options
32 files changed, 764 insertions, 1082 deletions
diff --git a/runtime/interpreter/mterp/mips/bincmp.S b/runtime/interpreter/mterp/mips/bincmp.S index 70057f6792..68df5c3ff0 100644 --- a/runtime/interpreter/mterp/mips/bincmp.S +++ b/runtime/interpreter/mterp/mips/bincmp.S @@ -1,7 +1,6 @@ /* - * Generic two-operand compare-and-branch operation. Provide a "revcmp" - * fragment that specifies the *reverse* comparison to perform, e.g. - * for "if-le" you would use "gt". + * Generic two-operand compare-and-branch operation. Provide a "condition" + * fragment that specifies the comparison to perform. * * For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le */ @@ -9,29 +8,11 @@ GET_OPA4(a0) # a0 <- A+ GET_OPB(a1) # a1 <- B GET_VREG(a3, a1) # a3 <- vB - GET_VREG(a2, a0) # a2 <- vA - b${revcmp} a2, a3, 1f # branch to 1 if comparison failed + GET_VREG(a0, a0) # a0 <- vA FETCH_S(rINST, 1) # rINST<- branch offset, in code units - b 2f -1: - li rINST, 2 # rINST- BYTE branch dist for not-taken -2: -#if MTERP_PROFILE_BRANCHES - EXPORT_PC() - move a0, rSELF - addu a1, rFP, OFF_FP_SHADOWFRAME - move a2, rINST - JAL(MterpProfileBranch) # (self, shadow_frame, offset) - bnez v0, MterpOnStackReplacement # Note: offset must be in rINST -#endif - addu a2, rINST, rINST # convert to bytes - FETCH_ADVANCE_INST_RB(a2) # update rPC, load rINST - bgez a2, .L_${opcode}_finish - lw ra, THREAD_FLAGS_OFFSET(rSELF) - b MterpCheckSuspendAndContinue - -%break - -.L_${opcode}_finish: + b${condition} a0, a3, MterpCommonTakenBranchNoFlags # compare (vA, vB) + li t0, JIT_CHECK_OSR + beq rPROFILE, t0, .L_check_not_taken_osr + FETCH_ADVANCE_INST(2) # advance rPC, load rINST GET_INST_OPCODE(t0) # extract opcode from rINST GOTO_OPCODE(t0) # jump to next instruction diff --git a/runtime/interpreter/mterp/mips/entry.S b/runtime/interpreter/mterp/mips/entry.S index 5771a4f402..c806a679b9 100644 --- a/runtime/interpreter/mterp/mips/entry.S +++ b/runtime/interpreter/mterp/mips/entry.S @@ -60,6 +60,12 @@ ExecuteMterpImpl: /* Starting ibase */ lw rIBASE, THREAD_CURRENT_IBASE_OFFSET(rSELF) + /* Set up for backwards branches & osr profiling */ + lw a0, OFF_FP_METHOD(rFP) + addu a1, rFP, OFF_FP_SHADOWFRAME + JAL(MterpSetUpHotnessCountdown) # (method, shadow_frame) + move rPROFILE, v0 # Starting hotness countdown to rPROFILE + /* start executing the instruction at rPC */ FETCH_INST() # load rINST from rPC GET_INST_OPCODE(t0) # extract opcode from rINST diff --git a/runtime/interpreter/mterp/mips/footer.S b/runtime/interpreter/mterp/mips/footer.S index 083dc15205..1363751566 100644 --- a/runtime/interpreter/mterp/mips/footer.S +++ b/runtime/interpreter/mterp/mips/footer.S @@ -112,20 +112,110 @@ MterpException: /* NOTE: no fallthrough */ /* - * Check for suspend check request. Assumes rINST already loaded, rPC advanced and - * still needs to get the opcode and branch to it, and flags are in lr. + * Common handling for branches with support for Jit profiling. + * On entry: + * rINST <= signed offset + * rPROFILE <= signed hotness countdown (expanded to 32 bits) + * + * We have quite a few different cases for branch profiling, OSR detection and + * suspend check support here. + * + * Taken backward branches: + * If profiling active, do hotness countdown and report if we hit zero. + * If in osr check mode, see if our target is a compiled loop header entry and do OSR if so. + * Is there a pending suspend request? If so, suspend. + * + * Taken forward branches and not-taken backward branches: + * If in osr check mode, see if our target is a compiled loop header entry and do OSR if so. + * + * Our most common case is expected to be a taken backward branch with active jit profiling, + * but no full OSR check and no pending suspend request. + * Next most common case is not-taken branch with no full OSR check. */ -MterpCheckSuspendAndContinue: - lw rIBASE, THREAD_CURRENT_IBASE_OFFSET(rSELF) # refresh rIBASE +MterpCommonTakenBranchNoFlags: + bgtz rINST, .L_forward_branch # don't add forward branches to hotness +/* + * We need to subtract 1 from positive values and we should not see 0 here, + * so we may use the result of the comparison with -1. + */ +#if JIT_CHECK_OSR != -1 +# error "JIT_CHECK_OSR must be -1." +#endif + li t0, JIT_CHECK_OSR + beq rPROFILE, t0, .L_osr_check + blt rPROFILE, t0, .L_resume_backward_branch + subu rPROFILE, 1 + beqz rPROFILE, .L_add_batch # counted down to zero - report +.L_resume_backward_branch: + lw ra, THREAD_FLAGS_OFFSET(rSELF) + REFRESH_IBASE() + addu a2, rINST, rINST # a2<- byte offset + FETCH_ADVANCE_INST_RB(a2) # update rPC, load rINST and ra, (THREAD_SUSPEND_REQUEST | THREAD_CHECKPOINT_REQUEST) - bnez ra, 1f + bnez ra, .L_suspend_request_pending GET_INST_OPCODE(t0) # extract opcode from rINST GOTO_OPCODE(t0) # jump to next instruction -1: + +.L_suspend_request_pending: EXPORT_PC() move a0, rSELF JAL(MterpSuspendCheck) # (self) bnez v0, MterpFallback + REFRESH_IBASE() # might have changed during suspend + GET_INST_OPCODE(t0) # extract opcode from rINST + GOTO_OPCODE(t0) # jump to next instruction + +.L_no_count_backwards: + li t0, JIT_CHECK_OSR # check for possible OSR re-entry + bne rPROFILE, t0, .L_resume_backward_branch +.L_osr_check: + move a0, rSELF + addu a1, rFP, OFF_FP_SHADOWFRAME + move a2, rINST + EXPORT_PC() + JAL(MterpMaybeDoOnStackReplacement) # (self, shadow_frame, offset) + bnez v0, MterpOnStackReplacement + b .L_resume_backward_branch + +.L_forward_branch: + li t0, JIT_CHECK_OSR # check for possible OSR re-entry + beq rPROFILE, t0, .L_check_osr_forward +.L_resume_forward_branch: + add a2, rINST, rINST # a2<- byte offset + FETCH_ADVANCE_INST_RB(a2) # update rPC, load rINST + GET_INST_OPCODE(t0) # extract opcode from rINST + GOTO_OPCODE(t0) # jump to next instruction + +.L_check_osr_forward: + move a0, rSELF + addu a1, rFP, OFF_FP_SHADOWFRAME + move a2, rINST + EXPORT_PC() + JAL(MterpMaybeDoOnStackReplacement) # (self, shadow_frame, offset) + bnez v0, MterpOnStackReplacement + b .L_resume_forward_branch + +.L_add_batch: + addu a1, rFP, OFF_FP_SHADOWFRAME + sh rPROFILE, SHADOWFRAME_HOTNESS_COUNTDOWN_OFFSET(a1) + lw a0, OFF_FP_METHOD(rFP) + move a2, rSELF + JAL(MterpAddHotnessBatch) # (method, shadow_frame, self) + move rPROFILE, v0 # restore new hotness countdown to rPROFILE + b .L_no_count_backwards + +/* + * Entered from the conditional branch handlers when OSR check request active on + * not-taken path. All Dalvik not-taken conditional branch offsets are 2. + */ +.L_check_not_taken_osr: + move a0, rSELF + addu a1, rFP, OFF_FP_SHADOWFRAME + li a2, 2 + EXPORT_PC() + JAL(MterpMaybeDoOnStackReplacement) # (self, shadow_frame, offset) + bnez v0, MterpOnStackReplacement + FETCH_ADVANCE_INST(2) GET_INST_OPCODE(t0) # extract opcode from rINST GOTO_OPCODE(t0) # jump to next instruction @@ -172,6 +262,26 @@ MterpReturn: sw v1, 4(a2) li v0, 1 # signal return to caller. MterpDone: +/* + * At this point, we expect rPROFILE to be non-zero. If negative, hotness is disabled or we're + * checking for OSR. If greater than zero, we might have unreported hotness to register + * (the difference between the ending rPROFILE and the cached hotness counter). rPROFILE + * should only reach zero immediately after a hotness decrement, and is then reset to either + * a negative special state or the new non-zero countdown value. + */ + blez rPROFILE, .L_pop_and_return # if > 0, we may have some counts to report. + +MterpProfileActive: + move rINST, v0 # stash return value + /* Report cached hotness counts */ + lw a0, OFF_FP_METHOD(rFP) + addu a1, rFP, OFF_FP_SHADOWFRAME + move a2, rSELF + sh rPROFILE, SHADOWFRAME_HOTNESS_COUNTDOWN_OFFSET(a1) + JAL(MterpAddHotnessBatch) # (method, shadow_frame, self) + move v0, rINST # restore return value + +.L_pop_and_return: /* Restore from the stack and return. Frame size = STACK_SIZE */ STACK_LOAD_FULL() jalr zero, ra diff --git a/runtime/interpreter/mterp/mips/header.S b/runtime/interpreter/mterp/mips/header.S index 37ab21de5c..a3a67444ab 100644 --- a/runtime/interpreter/mterp/mips/header.S +++ b/runtime/interpreter/mterp/mips/header.S @@ -51,7 +51,11 @@ s2 rSELF self (Thread) pointer s3 rIBASE interpreted instruction base pointer, used for computed goto s4 rINST first 16-bit code unit of current instruction + s5 rOBJ object pointer s6 rREFS base of object references in shadow frame (ideally, we'll get rid of this later). + s7 rTEMP used as temp storage that can survive a function call + s8 rPROFILE branch profiling countdown + */ /* single-purpose registers, given names for clarity */ @@ -63,6 +67,7 @@ #define rOBJ s5 #define rREFS s6 #define rTEMP s7 +#define rPROFILE s8 #define rARG0 a0 #define rARG1 a1 @@ -160,7 +165,7 @@ #define OFF_FP_RESULT_REGISTER OFF_FP(SHADOWFRAME_RESULT_REGISTER_OFFSET) #define OFF_FP_DEX_PC_PTR OFF_FP(SHADOWFRAME_DEX_PC_PTR_OFFSET) #define OFF_FP_CODE_ITEM OFF_FP(SHADOWFRAME_CODE_ITEM_OFFSET) -#define OFF_FP_SHADOWFRAME (-SHADOWFRAME_VREGS_OFFSET) +#define OFF_FP_SHADOWFRAME OFF_FP(0) #define MTERP_PROFILE_BRANCHES 1 #define MTERP_LOGGING 0 @@ -482,3 +487,6 @@ STACK_LOAD(s8, 120); \ STACK_LOAD(ra, 124); \ DELETE_STACK(STACK_SIZE) + +#define REFRESH_IBASE() \ + lw rIBASE, THREAD_CURRENT_IBASE_OFFSET(rSELF) diff --git a/runtime/interpreter/mterp/mips/op_goto.S b/runtime/interpreter/mterp/mips/op_goto.S index d6f21c9b2c..57182a5b59 100644 --- a/runtime/interpreter/mterp/mips/op_goto.S +++ b/runtime/interpreter/mterp/mips/op_goto.S @@ -5,34 +5,6 @@ * double to get a byte offset. */ /* goto +AA */ -#if MTERP_PROFILE_BRANCHES sll a0, rINST, 16 # a0 <- AAxx0000 sra rINST, a0, 24 # rINST <- ssssssAA (sign-extended) - EXPORT_PC() - move a0, rSELF - addu a1, rFP, OFF_FP_SHADOWFRAME - move a2, rINST - JAL(MterpProfileBranch) # (self, shadow_frame, offset) - bnez v0, MterpOnStackReplacement # Note: offset must be in rINST - addu a2, rINST, rINST # a2 <- byte offset - FETCH_ADVANCE_INST_RB(a2) # update rPC, load rINST - /* If backwards branch refresh rIBASE */ - bgez a2, 1f - lw ra, THREAD_FLAGS_OFFSET(rSELF) - b MterpCheckSuspendAndContinue -1: - GET_INST_OPCODE(t0) # extract opcode from rINST - GOTO_OPCODE(t0) # jump to next instruction -#else - sll a0, rINST, 16 # a0 <- AAxx0000 - sra rINST, a0, 24 # rINST <- ssssssAA (sign-extended) - addu a2, rINST, rINST # a2 <- byte offset - FETCH_ADVANCE_INST_RB(a2) # update rPC, load rINST - /* If backwards branch refresh rIBASE */ - bgez a1, 1f - lw ra, THREAD_FLAGS_OFFSET(rSELF) - b MterpCheckSuspendAndContinue -1: - GET_INST_OPCODE(t0) # extract opcode from rINST - GOTO_OPCODE(t0) # jump to next instruction -#endif + b MterpCommonTakenBranchNoFlags diff --git a/runtime/interpreter/mterp/mips/op_goto_16.S b/runtime/interpreter/mterp/mips/op_goto_16.S index cec4432599..06c96cd545 100644 --- a/runtime/interpreter/mterp/mips/op_goto_16.S +++ b/runtime/interpreter/mterp/mips/op_goto_16.S @@ -5,30 +5,5 @@ * double to get a byte offset. */ /* goto/16 +AAAA */ -#if MTERP_PROFILE_BRANCHES FETCH_S(rINST, 1) # rINST <- ssssAAAA (sign-extended) - EXPORT_PC() - move a0, rSELF - addu a1, rFP, OFF_FP_SHADOWFRAME - move a2, rINST - JAL(MterpProfileBranch) # (self, shadow_frame, offset) - bnez v0, MterpOnStackReplacement # Note: offset must be in rINST - addu a1, rINST, rINST # a1 <- byte offset, flags set - FETCH_ADVANCE_INST_RB(a1) # update rPC, load rINST - bgez a1, 1f - lw ra, THREAD_FLAGS_OFFSET(rSELF) - b MterpCheckSuspendAndContinue -1: - GET_INST_OPCODE(t0) # extract opcode from rINST - GOTO_OPCODE(t0) # jump to next instruction -#else - FETCH_S(rINST, 1) # rINST <- ssssAAAA (sign-extended) - addu a1, rINST, rINST # a1 <- byte offset, flags set - FETCH_ADVANCE_INST_RB(a1) # update rPC, load rINST - bgez a1, 1f - lw ra, THREAD_FLAGS_OFFSET(rSELF) - b MterpCheckSuspendAndContinue -1: - GET_INST_OPCODE(t0) # extract opcode from rINST - GOTO_OPCODE(t0) # jump to next instruction -#endif + b MterpCommonTakenBranchNoFlags diff --git a/runtime/interpreter/mterp/mips/op_goto_32.S b/runtime/interpreter/mterp/mips/op_goto_32.S index 083acd1ef9..67f52e9e48 100644 --- a/runtime/interpreter/mterp/mips/op_goto_32.S +++ b/runtime/interpreter/mterp/mips/op_goto_32.S @@ -8,36 +8,8 @@ * our "backward branch" test must be "<=0" instead of "<0". */ /* goto/32 +AAAAAAAA */ -#if MTERP_PROFILE_BRANCHES FETCH(a0, 1) # a0 <- aaaa (lo) FETCH(a1, 2) # a1 <- AAAA (hi) sll a1, a1, 16 or rINST, a0, a1 # rINST <- AAAAaaaa - EXPORT_PC() - move a0, rSELF - addu a1, rFP, OFF_FP_SHADOWFRAME - move a2, rINST - JAL(MterpProfileBranch) # (self, shadow_frame, offset) - bnez v0, MterpOnStackReplacement # Note: offset must be in rINST - addu a1, rINST, rINST # a1 <- byte offset - FETCH_ADVANCE_INST_RB(a1) # update rPC, load rINST - bgtz a1, 1f - lw ra, THREAD_FLAGS_OFFSET(rSELF) - b MterpCheckSuspendAndContinue -1: - GET_INST_OPCODE(t0) # extract opcode from rINST - GOTO_OPCODE(t0) # jump to next instruction -#else - FETCH(a0, 1) # a0 <- aaaa (lo) - FETCH(a1, 2) # a1 <- AAAA (hi) - sll a1, a1, 16 - or rINST, a0, a1 # rINST <- AAAAaaaa - addu a1, rINST, rINST # a1 <- byte offset - FETCH_ADVANCE_INST_RB(a1) # update rPC, load rINST - bgtz a1, 1f - lw ra, THREAD_FLAGS_OFFSET(rSELF) - b MterpCheckSuspendAndContinue -1: - GET_INST_OPCODE(t0) # extract opcode from rINST - GOTO_OPCODE(t0) # jump to next instruction -#endif + b MterpCommonTakenBranchNoFlags diff --git a/runtime/interpreter/mterp/mips/op_if_eq.S b/runtime/interpreter/mterp/mips/op_if_eq.S index e7190d8197..d6f9987186 100644 --- a/runtime/interpreter/mterp/mips/op_if_eq.S +++ b/runtime/interpreter/mterp/mips/op_if_eq.S @@ -1 +1 @@ -%include "mips/bincmp.S" { "revcmp":"ne" } +%include "mips/bincmp.S" { "condition":"eq" } diff --git a/runtime/interpreter/mterp/mips/op_if_eqz.S b/runtime/interpreter/mterp/mips/op_if_eqz.S index 0a78fd98ac..c52b76a755 100644 --- a/runtime/interpreter/mterp/mips/op_if_eqz.S +++ b/runtime/interpreter/mterp/mips/op_if_eqz.S @@ -1 +1 @@ -%include "mips/zcmp.S" { "revcmp":"ne" } +%include "mips/zcmp.S" { "condition":"eq" } diff --git a/runtime/interpreter/mterp/mips/op_if_ge.S b/runtime/interpreter/mterp/mips/op_if_ge.S index b2629ba4e9..bd06ff5ad4 100644 --- a/runtime/interpreter/mterp/mips/op_if_ge.S +++ b/runtime/interpreter/mterp/mips/op_if_ge.S @@ -1 +1 @@ -%include "mips/bincmp.S" { "revcmp":"lt" } +%include "mips/bincmp.S" { "condition":"ge" } diff --git a/runtime/interpreter/mterp/mips/op_if_gez.S b/runtime/interpreter/mterp/mips/op_if_gez.S index b02f67709f..549231a15f 100644 --- a/runtime/interpreter/mterp/mips/op_if_gez.S +++ b/runtime/interpreter/mterp/mips/op_if_gez.S @@ -1 +1 @@ -%include "mips/zcmp.S" { "revcmp":"lt" } +%include "mips/zcmp.S" { "condition":"ge" } diff --git a/runtime/interpreter/mterp/mips/op_if_gt.S b/runtime/interpreter/mterp/mips/op_if_gt.S index f620d4a1fd..0be30912ed 100644 --- a/runtime/interpreter/mterp/mips/op_if_gt.S +++ b/runtime/interpreter/mterp/mips/op_if_gt.S @@ -1 +1 @@ -%include "mips/bincmp.S" { "revcmp":"le" } +%include "mips/bincmp.S" { "condition":"gt" } diff --git a/runtime/interpreter/mterp/mips/op_if_gtz.S b/runtime/interpreter/mterp/mips/op_if_gtz.S index 5e5dd708fa..5c7bcc48b5 100644 --- a/runtime/interpreter/mterp/mips/op_if_gtz.S +++ b/runtime/interpreter/mterp/mips/op_if_gtz.S @@ -1 +1 @@ -%include "mips/zcmp.S" { "revcmp":"le" } +%include "mips/zcmp.S" { "condition":"gt" } diff --git a/runtime/interpreter/mterp/mips/op_if_le.S b/runtime/interpreter/mterp/mips/op_if_le.S index a4e8b1ad51..c35c1a24b7 100644 --- a/runtime/interpreter/mterp/mips/op_if_le.S +++ b/runtime/interpreter/mterp/mips/op_if_le.S @@ -1 +1 @@ -%include "mips/bincmp.S" { "revcmp":"gt" } +%include "mips/bincmp.S" { "condition":"le" } diff --git a/runtime/interpreter/mterp/mips/op_if_lez.S b/runtime/interpreter/mterp/mips/op_if_lez.S index af551a62fd..3dc6543d90 100644 --- a/runtime/interpreter/mterp/mips/op_if_lez.S +++ b/runtime/interpreter/mterp/mips/op_if_lez.S @@ -1 +1 @@ -%include "mips/zcmp.S" { "revcmp":"gt" } +%include "mips/zcmp.S" { "condition":"le" } diff --git a/runtime/interpreter/mterp/mips/op_if_lt.S b/runtime/interpreter/mterp/mips/op_if_lt.S index f33b9a4c05..3f3386c9d2 100644 --- a/runtime/interpreter/mterp/mips/op_if_lt.S +++ b/runtime/interpreter/mterp/mips/op_if_lt.S @@ -1 +1 @@ -%include "mips/bincmp.S" { "revcmp":"ge" } +%include "mips/bincmp.S" { "condition":"lt" } diff --git a/runtime/interpreter/mterp/mips/op_if_ltz.S b/runtime/interpreter/mterp/mips/op_if_ltz.S index 18fcb1d477..e6d6ed6aa6 100644 --- a/runtime/interpreter/mterp/mips/op_if_ltz.S +++ b/runtime/interpreter/mterp/mips/op_if_ltz.S @@ -1 +1 @@ -%include "mips/zcmp.S" { "revcmp":"ge" } +%include "mips/zcmp.S" { "condition":"lt" } diff --git a/runtime/interpreter/mterp/mips/op_if_ne.S b/runtime/interpreter/mterp/mips/op_if_ne.S index e0a102b443..3d7bf350f1 100644 --- a/runtime/interpreter/mterp/mips/op_if_ne.S +++ b/runtime/interpreter/mterp/mips/op_if_ne.S @@ -1 +1 @@ -%include "mips/bincmp.S" { "revcmp":"eq" } +%include "mips/bincmp.S" { "condition":"ne" } diff --git a/runtime/interpreter/mterp/mips/op_if_nez.S b/runtime/interpreter/mterp/mips/op_if_nez.S index d1866a0a02..d121eae930 100644 --- a/runtime/interpreter/mterp/mips/op_if_nez.S +++ b/runtime/interpreter/mterp/mips/op_if_nez.S @@ -1 +1 @@ -%include "mips/zcmp.S" { "revcmp":"eq" } +%include "mips/zcmp.S" { "condition":"ne" } diff --git a/runtime/interpreter/mterp/mips/op_packed_switch.S b/runtime/interpreter/mterp/mips/op_packed_switch.S index 93fae973e5..ffa4f472ef 100644 --- a/runtime/interpreter/mterp/mips/op_packed_switch.S +++ b/runtime/interpreter/mterp/mips/op_packed_switch.S @@ -9,7 +9,6 @@ * for: packed-switch, sparse-switch */ /* op vAA, +BBBB */ -#if MTERP_PROFILE_BRANCHES FETCH(a0, 1) # a0 <- bbbb (lo) FETCH(a1, 2) # a1 <- BBBB (hi) GET_OPA(a3) # a3 <- AA @@ -19,39 +18,4 @@ EAS1(a0, rPC, a0) # a0 <- PC + BBBBbbbb*2 JAL($func) # a0 <- code-unit branch offset move rINST, v0 - EXPORT_PC() - move a0, rSELF - addu a1, rFP, OFF_FP_SHADOWFRAME - move a2, rINST - JAL(MterpProfileBranch) # (self, shadow_frame, offset) - bnez v0, MterpOnStackReplacement # Note: offset must be in rINST - addu a1, rINST, rINST # a1 <- byte offset - FETCH_ADVANCE_INST_RB(a1) # update rPC, load rINST - bgtz a1, .L${opcode}_finish - lw ra, THREAD_FLAGS_OFFSET(rSELF) - b MterpCheckSuspendAndContinue -#else - FETCH(a0, 1) # a0 <- bbbb (lo) - FETCH(a1, 2) # a1 <- BBBB (hi) - GET_OPA(a3) # a3 <- AA - sll t0, a1, 16 - or a0, a0, t0 # a0 <- BBBBbbbb - GET_VREG(a1, a3) # a1 <- vAA - EAS1(a0, rPC, a0) # a0 <- PC + BBBBbbbb*2 - JAL($func) # a0 <- code-unit branch offset - move rINST, v0 - addu a1, rINST, rINST # a1 <- byte offset - FETCH_ADVANCE_INST_RB(a1) # update rPC, load rINST - bgtz a1, 1f - lw ra, THREAD_FLAGS_OFFSET(rSELF) - b MterpCheckSuspendAndContinue -1: - GET_INST_OPCODE(t0) # extract opcode from rINST - GOTO_OPCODE(t0) # jump to next instruction -#endif - -%break - -.L${opcode}_finish: - GET_INST_OPCODE(t0) # extract opcode from rINST - GOTO_OPCODE(t0) # jump to next instruction + b MterpCommonTakenBranchNoFlags diff --git a/runtime/interpreter/mterp/mips/zcmp.S b/runtime/interpreter/mterp/mips/zcmp.S index 1fa13851c7..8d3a198891 100644 --- a/runtime/interpreter/mterp/mips/zcmp.S +++ b/runtime/interpreter/mterp/mips/zcmp.S @@ -1,32 +1,16 @@ /* - * Generic one-operand compare-and-branch operation. Provide a "revcmp" - * fragment that specifies the *reverse* comparison to perform, e.g. - * for "if-le" you would use "gt". + * Generic one-operand compare-and-branch operation. Provide a "condition" + * fragment that specifies the comparison to perform. * * for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez */ /* if-cmp vAA, +BBBB */ GET_OPA(a0) # a0 <- AA - GET_VREG(a2, a0) # a2 <- vAA + GET_VREG(a0, a0) # a0 <- vAA FETCH_S(rINST, 1) # rINST <- branch offset, in code units - b${revcmp} a2, zero, 1f # branch to 1 if comparison failed - b 2f -1: - li rINST, 2 # rINST- BYTE branch dist for not-taken -2: -#if MTERP_PROFILE_BRANCHES - EXPORT_PC() - move a0, rSELF - addu a1, rFP, OFF_FP_SHADOWFRAME - move a2, rINST - JAL(MterpProfileBranch) # (self, shadow_frame, offset) - bnez v0, MterpOnStackReplacement # Note: offset must be in rINST -#endif - addu a1, rINST, rINST # convert to bytes - FETCH_ADVANCE_INST_RB(a1) # update rPC, load rINST - bgez a1, 3f - lw ra, THREAD_FLAGS_OFFSET(rSELF) - b MterpCheckSuspendAndContinue -3: + b${condition} a0, zero, MterpCommonTakenBranchNoFlags + li t0, JIT_CHECK_OSR # possible OSR re-entry? + beq rPROFILE, t0, .L_check_not_taken_osr + FETCH_ADVANCE_INST(2) # advance rPC, load rINST GET_INST_OPCODE(t0) # extract opcode from rINST GOTO_OPCODE(t0) # jump to next instruction diff --git a/runtime/interpreter/mterp/mips64/bincmp.S b/runtime/interpreter/mterp/mips64/bincmp.S index aa5e74b3de..07b12100fd 100644 --- a/runtime/interpreter/mterp/mips64/bincmp.S +++ b/runtime/interpreter/mterp/mips64/bincmp.S @@ -12,21 +12,9 @@ lh rINST, 2(rPC) # rINST <- offset (sign-extended CCCC) GET_VREG a0, a2 # a0 <- vA GET_VREG a1, a3 # a1 <- vB - b${condition}c a0, a1, 1f - li rINST, 2 # offset if branch not taken -1: -#if MTERP_PROFILE_BRANCHES - EXPORT_PC - move a0, rSELF - daddu a1, rFP, OFF_FP_SHADOWFRAME - move a2, rINST - jal MterpProfileBranch # (self, shadow_frame, offset) - bnezc v0, MterpOnStackReplacement # Note: offset must be in rINST -#endif - dlsa rPC, rINST, rPC, 1 # rPC <- rPC + offset * 2 - lw ra, THREAD_FLAGS_OFFSET(rSELF) # Preload flags for MterpCheckSuspendAndContinue - move a0, rINST # a0 <- offset - FETCH_INST # load rINST - bltz a0, MterpCheckSuspendAndContinue # suspend check if backwards branch + b${condition}c a0, a1, MterpCommonTakenBranchNoFlags + li v0, JIT_CHECK_OSR # possible OSR re-entry? + beqc rPROFILE, v0, .L_check_not_taken_osr + FETCH_ADVANCE_INST 2 # advance rPC, load rINST GET_INST_OPCODE v0 # extract opcode from rINST GOTO_OPCODE v0 # jump to next instruction diff --git a/runtime/interpreter/mterp/mips64/entry.S b/runtime/interpreter/mterp/mips64/entry.S index ae6c26b706..cc48d45298 100644 --- a/runtime/interpreter/mterp/mips64/entry.S +++ b/runtime/interpreter/mterp/mips64/entry.S @@ -57,6 +57,8 @@ ExecuteMterpImpl: .cfi_rel_offset 20, STACK_OFFSET_S4 sd s5, STACK_OFFSET_S5(sp) .cfi_rel_offset 21, STACK_OFFSET_S5 + sd s6, STACK_OFFSET_S6(sp) + .cfi_rel_offset 22, STACK_OFFSET_S6 /* Remember the return register */ sd a3, SHADOWFRAME_RESULT_REGISTER_OFFSET(a2) @@ -77,6 +79,12 @@ ExecuteMterpImpl: /* Starting ibase */ REFRESH_IBASE + /* Set up for backwards branches & osr profiling */ + ld a0, OFF_FP_METHOD(rFP) + daddu a1, rFP, OFF_FP_SHADOWFRAME + jal MterpSetUpHotnessCountdown + move rPROFILE, v0 # Starting hotness countdown to rPROFILE + /* start executing the instruction at rPC */ FETCH_INST GET_INST_OPCODE v0 diff --git a/runtime/interpreter/mterp/mips64/footer.S b/runtime/interpreter/mterp/mips64/footer.S index 14d5fe01f5..9994169950 100644 --- a/runtime/interpreter/mterp/mips64/footer.S +++ b/runtime/interpreter/mterp/mips64/footer.S @@ -71,23 +71,110 @@ MterpException: /* NOTE: no fallthrough */ /* - * Check for suspend check request. Assumes rINST already loaded, rPC advanced and - * still needs to get the opcode and branch to it, and flags are in ra. + * Common handling for branches with support for Jit profiling. + * On entry: + * rINST <= signed offset + * rPROFILE <= signed hotness countdown (expanded to 64 bits) + * + * We have quite a few different cases for branch profiling, OSR detection and + * suspend check support here. + * + * Taken backward branches: + * If profiling active, do hotness countdown and report if we hit zero. + * If in osr check mode, see if our target is a compiled loop header entry and do OSR if so. + * Is there a pending suspend request? If so, suspend. + * + * Taken forward branches and not-taken backward branches: + * If in osr check mode, see if our target is a compiled loop header entry and do OSR if so. + * + * Our most common case is expected to be a taken backward branch with active jit profiling, + * but no full OSR check and no pending suspend request. + * Next most common case is not-taken branch with no full OSR check. + * + */ +MterpCommonTakenBranchNoFlags: + bgtzc rINST, .L_forward_branch # don't add forward branches to hotness +/* + * We need to subtract 1 from positive values and we should not see 0 here, + * so we may use the result of the comparison with -1. */ - .extern MterpSuspendCheck -MterpCheckSuspendAndContinue: + li v0, JIT_CHECK_OSR + beqc rPROFILE, v0, .L_osr_check + bltc rPROFILE, v0, .L_resume_backward_branch + dsubu rPROFILE, 1 + beqzc rPROFILE, .L_add_batch # counted down to zero - report +.L_resume_backward_branch: + lw ra, THREAD_FLAGS_OFFSET(rSELF) REFRESH_IBASE - and ra, ra, (THREAD_SUSPEND_REQUEST | THREAD_CHECKPOINT_REQUEST) - bnez ra, check1 - GET_INST_OPCODE v0 # extract opcode from rINST - GOTO_OPCODE v0 # jump to next instruction -check1: + daddu a2, rINST, rINST # a2<- byte offset + FETCH_ADVANCE_INST_RB a2 # update rPC, load rINST + and ra, (THREAD_SUSPEND_REQUEST | THREAD_CHECKPOINT_REQUEST) + bnezc ra, .L_suspend_request_pending + GET_INST_OPCODE v0 # extract opcode from rINST + GOTO_OPCODE v0 # jump to next instruction + +.L_suspend_request_pending: EXPORT_PC move a0, rSELF - jal MterpSuspendCheck # (self) - bnezc v0, MterpFallback # Something in the environment changed, switch interpreters - GET_INST_OPCODE v0 # extract opcode from rINST - GOTO_OPCODE v0 # jump to next instruction + jal MterpSuspendCheck # (self) + bnezc v0, MterpFallback + REFRESH_IBASE # might have changed during suspend + GET_INST_OPCODE v0 # extract opcode from rINST + GOTO_OPCODE v0 # jump to next instruction + +.L_no_count_backwards: + li v0, JIT_CHECK_OSR # check for possible OSR re-entry + bnec rPROFILE, v0, .L_resume_backward_branch +.L_osr_check: + move a0, rSELF + daddu a1, rFP, OFF_FP_SHADOWFRAME + move a2, rINST + EXPORT_PC + jal MterpMaybeDoOnStackReplacement # (self, shadow_frame, offset) + bnezc v0, MterpOnStackReplacement + b .L_resume_backward_branch + +.L_forward_branch: + li v0, JIT_CHECK_OSR # check for possible OSR re-entry + beqc rPROFILE, v0, .L_check_osr_forward +.L_resume_forward_branch: + daddu a2, rINST, rINST # a2<- byte offset + FETCH_ADVANCE_INST_RB a2 # update rPC, load rINST + GET_INST_OPCODE v0 # extract opcode from rINST + GOTO_OPCODE v0 # jump to next instruction + +.L_check_osr_forward: + move a0, rSELF + daddu a1, rFP, OFF_FP_SHADOWFRAME + move a2, rINST + EXPORT_PC + jal MterpMaybeDoOnStackReplacement # (self, shadow_frame, offset) + bnezc v0, MterpOnStackReplacement + b .L_resume_forward_branch + +.L_add_batch: + daddu a1, rFP, OFF_FP_SHADOWFRAME + sh rPROFILE, SHADOWFRAME_HOTNESS_COUNTDOWN_OFFSET(a1) + ld a0, OFF_FP_METHOD(rFP) + move a2, rSELF + jal MterpAddHotnessBatch # (method, shadow_frame, self) + move rPROFILE, v0 # restore new hotness countdown to rPROFILE + b .L_no_count_backwards + +/* + * Entered from the conditional branch handlers when OSR check request active on + * not-taken path. All Dalvik not-taken conditional branch offsets are 2. + */ +.L_check_not_taken_osr: + move a0, rSELF + daddu a1, rFP, OFF_FP_SHADOWFRAME + li a2, 2 + EXPORT_PC + jal MterpMaybeDoOnStackReplacement # (self, shadow_frame, offset) + bnezc v0, MterpOnStackReplacement + FETCH_ADVANCE_INST 2 + GET_INST_OPCODE v0 # extract opcode from rINST + GOTO_OPCODE v0 # jump to next instruction /* * On-stack replacement has happened, and now we've returned from the compiled method. @@ -143,6 +230,28 @@ MterpReturn: check2: li v0, 1 # signal return to caller. MterpDone: +/* + * At this point, we expect rPROFILE to be non-zero. If negative, hotness is disabled or we're + * checking for OSR. If greater than zero, we might have unreported hotness to register + * (the difference between the ending rPROFILE and the cached hotness counter). rPROFILE + * should only reach zero immediately after a hotness decrement, and is then reset to either + * a negative special state or the new non-zero countdown value. + */ + blez rPROFILE, .L_pop_and_return # if > 0, we may have some counts to report. + +MterpProfileActive: + move rINST, v0 # stash return value + /* Report cached hotness counts */ + ld a0, OFF_FP_METHOD(rFP) + daddu a1, rFP, OFF_FP_SHADOWFRAME + move a2, rSELF + sh rPROFILE, SHADOWFRAME_HOTNESS_COUNTDOWN_OFFSET(a1) + jal MterpAddHotnessBatch # (method, shadow_frame, self) + move v0, rINST # restore return value + +.L_pop_and_return: + ld s6, STACK_OFFSET_S6(sp) + .cfi_restore 22 ld s5, STACK_OFFSET_S5(sp) .cfi_restore 21 ld s4, STACK_OFFSET_S4(sp) @@ -169,4 +278,5 @@ MterpDone: .cfi_adjust_cfa_offset -STACK_SIZE .cfi_endproc + .set reorder .size ExecuteMterpImpl, .-ExecuteMterpImpl diff --git a/runtime/interpreter/mterp/mips64/header.S b/runtime/interpreter/mterp/mips64/header.S index dd0fbe0057..b67df20a1f 100644 --- a/runtime/interpreter/mterp/mips64/header.S +++ b/runtime/interpreter/mterp/mips64/header.S @@ -51,16 +51,18 @@ The following registers have fixed assignments: s3 rINST first 16-bit code unit of current instruction s4 rIBASE interpreted instruction base pointer, used for computed goto s5 rREFS base of object references in shadow frame (ideally, we'll get rid of this later). + s6 rPROFILE jit profile hotness countdown */ /* During bringup, we'll use the shadow frame model instead of rFP */ /* single-purpose registers, given names for clarity */ -#define rPC s0 -#define rFP s1 -#define rSELF s2 -#define rINST s3 -#define rIBASE s4 -#define rREFS s5 +#define rPC s0 +#define rFP s1 +#define rSELF s2 +#define rINST s3 +#define rIBASE s4 +#define rREFS s5 +#define rPROFILE s6 /* * This is a #include, not a %include, because we want the C pre-processor @@ -80,7 +82,7 @@ The following registers have fixed assignments: #define OFF_FP_RESULT_REGISTER OFF_FP(SHADOWFRAME_RESULT_REGISTER_OFFSET) #define OFF_FP_DEX_PC_PTR OFF_FP(SHADOWFRAME_DEX_PC_PTR_OFFSET) #define OFF_FP_CODE_ITEM OFF_FP(SHADOWFRAME_CODE_ITEM_OFFSET) -#define OFF_FP_SHADOWFRAME (-SHADOWFRAME_VREGS_OFFSET) +#define OFF_FP_SHADOWFRAME OFF_FP(0) #define MTERP_PROFILE_BRANCHES 1 #define MTERP_LOGGING 0 @@ -121,6 +123,17 @@ The following registers have fixed assignments: .endm /* + * Fetch the next instruction from an offset specified by _reg and advance xPC. + * xPC to point to the next instruction. "_reg" must specify the distance + * in bytes, *not* 16-bit code units, and may be a signed value. Must not set flags. + * + */ +.macro FETCH_ADVANCE_INST_RB reg + daddu rPC, rPC, \reg + FETCH_INST +.endm + +/* * Fetch the next instruction from the specified offset. Advances rPC * to point to the next instruction. * @@ -267,7 +280,8 @@ The following registers have fixed assignments: #define STACK_OFFSET_S3 40 #define STACK_OFFSET_S4 48 #define STACK_OFFSET_S5 56 -#define STACK_SIZE 64 +#define STACK_OFFSET_S6 64 +#define STACK_SIZE 80 /* needs 16 byte alignment */ /* Constants for float/double_to_int/long conversions */ #define INT_MIN 0x80000000 diff --git a/runtime/interpreter/mterp/mips64/op_goto.S b/runtime/interpreter/mterp/mips64/op_goto.S index 7c7d0ecf5a..68fc83d0ca 100644 --- a/runtime/interpreter/mterp/mips64/op_goto.S +++ b/runtime/interpreter/mterp/mips64/op_goto.S @@ -5,21 +5,6 @@ * double to get a byte offset. */ /* goto +AA */ - .extern MterpProfileBranch srl rINST, rINST, 8 seb rINST, rINST # rINST <- offset (sign-extended AA) -#if MTERP_PROFILE_BRANCHES - EXPORT_PC - move a0, rSELF - daddu a1, rFP, OFF_FP_SHADOWFRAME - move a2, rINST - jal MterpProfileBranch # (self, shadow_frame, offset) - bnezc v0, MterpOnStackReplacement # Note: offset must be in rINST -#endif - dlsa rPC, rINST, rPC, 1 # rPC <- rPC + offset * 2 - lw ra, THREAD_FLAGS_OFFSET(rSELF) # Preload flags for MterpCheckSuspendAndContinue - move a0, rINST # a0 <- offset - FETCH_INST # load rINST - bltz a0, MterpCheckSuspendAndContinue # suspend check if backwards branch - GET_INST_OPCODE v0 # extract opcode from rINST - GOTO_OPCODE v0 # jump to next instruction + b MterpCommonTakenBranchNoFlags diff --git a/runtime/interpreter/mterp/mips64/op_goto_16.S b/runtime/interpreter/mterp/mips64/op_goto_16.S index 566e3a78f0..ae56066352 100644 --- a/runtime/interpreter/mterp/mips64/op_goto_16.S +++ b/runtime/interpreter/mterp/mips64/op_goto_16.S @@ -5,20 +5,5 @@ * double to get a byte offset. */ /* goto/16 +AAAA */ - .extern MterpProfileBranch lh rINST, 2(rPC) # rINST <- offset (sign-extended AAAA) -#if MTERP_PROFILE_BRANCHES - EXPORT_PC - move a0, rSELF - daddu a1, rFP, OFF_FP_SHADOWFRAME - move a2, rINST - jal MterpProfileBranch # (self, shadow_frame, offset) - bnezc v0, MterpOnStackReplacement # Note: offset must be in rINST -#endif - dlsa rPC, rINST, rPC, 1 # rPC <- rPC + offset * 2 - lw ra, THREAD_FLAGS_OFFSET(rSELF) # Preload flags for MterpCheckSuspendAndContinue - move a0, rINST # a0 <- offset - FETCH_INST # load rINST - bltz a0, MterpCheckSuspendAndContinue # suspend check if backwards branch - GET_INST_OPCODE v0 # extract opcode from rINST - GOTO_OPCODE v0 # jump to next instruction + b MterpCommonTakenBranchNoFlags diff --git a/runtime/interpreter/mterp/mips64/op_goto_32.S b/runtime/interpreter/mterp/mips64/op_goto_32.S index b260083ae8..498b6d60ae 100644 --- a/runtime/interpreter/mterp/mips64/op_goto_32.S +++ b/runtime/interpreter/mterp/mips64/op_goto_32.S @@ -8,22 +8,7 @@ * our "backward branch" test must be "<=0" instead of "<0". */ /* goto/32 +AAAAAAAA */ - .extern MterpProfileBranch lh rINST, 2(rPC) # rINST <- aaaa (low) lh a1, 4(rPC) # a1 <- AAAA (high) ins rINST, a1, 16, 16 # rINST <- offset (sign-extended AAAAaaaa) -#if MTERP_PROFILE_BRANCHES - EXPORT_PC - move a0, rSELF - daddu a1, rFP, OFF_FP_SHADOWFRAME - move a2, rINST - jal MterpProfileBranch # (self, shadow_frame, offset) - bnezc v0, MterpOnStackReplacement # Note: offset must be in rINST -#endif - dlsa rPC, rINST, rPC, 1 # rPC <- rPC + offset * 2 - lw ra, THREAD_FLAGS_OFFSET(rSELF) # Preload flags for MterpCheckSuspendAndContinue - move a0, rINST # a0 <- offset - FETCH_INST # load rINST - blez a0, MterpCheckSuspendAndContinue # suspend check if backwards branch - GET_INST_OPCODE v0 # extract opcode from rINST - GOTO_OPCODE v0 # jump to next instruction + b MterpCommonTakenBranchNoFlags diff --git a/runtime/interpreter/mterp/mips64/op_packed_switch.S b/runtime/interpreter/mterp/mips64/op_packed_switch.S index 2c6eb2f3ca..27ce580642 100644 --- a/runtime/interpreter/mterp/mips64/op_packed_switch.S +++ b/runtime/interpreter/mterp/mips64/op_packed_switch.S @@ -19,18 +19,4 @@ dlsa a0, a0, rPC, 1 # a0 <- PC + BBBBbbbb*2 jal $func # v0 <- code-unit branch offset move rINST, v0 -#if MTERP_PROFILE_BRANCHES - EXPORT_PC - move a0, rSELF - daddu a1, rFP, OFF_FP_SHADOWFRAME - move a2, rINST - jal MterpProfileBranch # (self, shadow_frame, offset) - bnezc v0, MterpOnStackReplacement # Note: offset must be in rINST -#endif - dlsa rPC, rINST, rPC, 1 # rPC <- rPC + offset * 2 - lw ra, THREAD_FLAGS_OFFSET(rSELF) # Preload flags for MterpCheckSuspendAndContinue - move a0, rINST # a0 <- offset - FETCH_INST # load rINST - blez a0, MterpCheckSuspendAndContinue # suspend check if backwards branch - GET_INST_OPCODE v0 # extract opcode from rINST - GOTO_OPCODE v0 # jump to next instruction + b MterpCommonTakenBranchNoFlags diff --git a/runtime/interpreter/mterp/mips64/zcmp.S b/runtime/interpreter/mterp/mips64/zcmp.S index 0e0477fadf..75db49edd4 100644 --- a/runtime/interpreter/mterp/mips64/zcmp.S +++ b/runtime/interpreter/mterp/mips64/zcmp.S @@ -6,25 +6,12 @@ * For: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez */ /* if-cmp vAA, +BBBB */ - .extern MterpProfileBranch srl a2, rINST, 8 # a2 <- AA lh rINST, 2(rPC) # rINST <- offset (sign-extended BBBB) GET_VREG a0, a2 # a0 <- vAA - b${condition}zc a0, 1f - li rINST, 2 # offset if branch not taken -1: -#if MTERP_PROFILE_BRANCHES - EXPORT_PC - move a0, rSELF - daddu a1, rFP, OFF_FP_SHADOWFRAME - move a2, rINST - jal MterpProfileBranch # (self, shadow_frame, offset) - bnezc v0, MterpOnStackReplacement # Note: offset must be in rINST -#endif - dlsa rPC, rINST, rPC, 1 # rPC <- rPC + offset * 2 - lw ra, THREAD_FLAGS_OFFSET(rSELF) # Preload flags for MterpCheckSuspendAndContinue - move a0, rINST # a0 <- offset - FETCH_INST # load rINST - bltz a0, MterpCheckSuspendAndContinue # suspend check if backwards branch + b${condition}zc a0, MterpCommonTakenBranchNoFlags + li v0, JIT_CHECK_OSR # possible OSR re-entry? + beqc rPROFILE, v0, .L_check_not_taken_osr + FETCH_ADVANCE_INST 2 # advance rPC, load rINST GET_INST_OPCODE v0 # extract opcode from rINST GOTO_OPCODE v0 # jump to next instruction diff --git a/runtime/interpreter/mterp/out/mterp_mips.S b/runtime/interpreter/mterp/out/mterp_mips.S index b134129a5a..daa6f2a0eb 100644 --- a/runtime/interpreter/mterp/out/mterp_mips.S +++ b/runtime/interpreter/mterp/out/mterp_mips.S @@ -58,7 +58,11 @@ s2 rSELF self (Thread) pointer s3 rIBASE interpreted instruction base pointer, used for computed goto s4 rINST first 16-bit code unit of current instruction + s5 rOBJ object pointer s6 rREFS base of object references in shadow frame (ideally, we'll get rid of this later). + s7 rTEMP used as temp storage that can survive a function call + s8 rPROFILE branch profiling countdown + */ /* single-purpose registers, given names for clarity */ @@ -70,6 +74,7 @@ #define rOBJ s5 #define rREFS s6 #define rTEMP s7 +#define rPROFILE s8 #define rARG0 a0 #define rARG1 a1 @@ -167,7 +172,7 @@ #define OFF_FP_RESULT_REGISTER OFF_FP(SHADOWFRAME_RESULT_REGISTER_OFFSET) #define OFF_FP_DEX_PC_PTR OFF_FP(SHADOWFRAME_DEX_PC_PTR_OFFSET) #define OFF_FP_CODE_ITEM OFF_FP(SHADOWFRAME_CODE_ITEM_OFFSET) -#define OFF_FP_SHADOWFRAME (-SHADOWFRAME_VREGS_OFFSET) +#define OFF_FP_SHADOWFRAME OFF_FP(0) #define MTERP_PROFILE_BRANCHES 1 #define MTERP_LOGGING 0 @@ -490,6 +495,9 @@ STACK_LOAD(ra, 124); \ DELETE_STACK(STACK_SIZE) +#define REFRESH_IBASE() \ + lw rIBASE, THREAD_CURRENT_IBASE_OFFSET(rSELF) + /* File: mips/entry.S */ /* * Copyright (C) 2016 The Android Open Source Project @@ -553,6 +561,12 @@ ExecuteMterpImpl: /* Starting ibase */ lw rIBASE, THREAD_CURRENT_IBASE_OFFSET(rSELF) + /* Set up for backwards branches & osr profiling */ + lw a0, OFF_FP_METHOD(rFP) + addu a1, rFP, OFF_FP_SHADOWFRAME + JAL(MterpSetUpHotnessCountdown) # (method, shadow_frame) + move rPROFILE, v0 # Starting hotness countdown to rPROFILE + /* start executing the instruction at rPC */ FETCH_INST() # load rINST from rPC GET_INST_OPCODE(t0) # extract opcode from rINST @@ -1284,37 +1298,9 @@ artMterpAsmInstructionStart = .L_op_nop * double to get a byte offset. */ /* goto +AA */ -#if MTERP_PROFILE_BRANCHES - sll a0, rINST, 16 # a0 <- AAxx0000 - sra rINST, a0, 24 # rINST <- ssssssAA (sign-extended) - EXPORT_PC() - move a0, rSELF - addu a1, rFP, OFF_FP_SHADOWFRAME - move a2, rINST - JAL(MterpProfileBranch) # (self, shadow_frame, offset) - bnez v0, MterpOnStackReplacement # Note: offset must be in rINST - addu a2, rINST, rINST # a2 <- byte offset - FETCH_ADVANCE_INST_RB(a2) # update rPC, load rINST - /* If backwards branch refresh rIBASE */ - bgez a2, 1f - lw ra, THREAD_FLAGS_OFFSET(rSELF) - b MterpCheckSuspendAndContinue -1: - GET_INST_OPCODE(t0) # extract opcode from rINST - GOTO_OPCODE(t0) # jump to next instruction -#else sll a0, rINST, 16 # a0 <- AAxx0000 sra rINST, a0, 24 # rINST <- ssssssAA (sign-extended) - addu a2, rINST, rINST # a2 <- byte offset - FETCH_ADVANCE_INST_RB(a2) # update rPC, load rINST - /* If backwards branch refresh rIBASE */ - bgez a1, 1f - lw ra, THREAD_FLAGS_OFFSET(rSELF) - b MterpCheckSuspendAndContinue -1: - GET_INST_OPCODE(t0) # extract opcode from rINST - GOTO_OPCODE(t0) # jump to next instruction -#endif + b MterpCommonTakenBranchNoFlags /* ------------------------------ */ .balign 128 @@ -1327,33 +1313,8 @@ artMterpAsmInstructionStart = .L_op_nop * double to get a byte offset. */ /* goto/16 +AAAA */ -#if MTERP_PROFILE_BRANCHES FETCH_S(rINST, 1) # rINST <- ssssAAAA (sign-extended) - EXPORT_PC() - move a0, rSELF - addu a1, rFP, OFF_FP_SHADOWFRAME - move a2, rINST - JAL(MterpProfileBranch) # (self, shadow_frame, offset) - bnez v0, MterpOnStackReplacement # Note: offset must be in rINST - addu a1, rINST, rINST # a1 <- byte offset, flags set - FETCH_ADVANCE_INST_RB(a1) # update rPC, load rINST - bgez a1, 1f - lw ra, THREAD_FLAGS_OFFSET(rSELF) - b MterpCheckSuspendAndContinue -1: - GET_INST_OPCODE(t0) # extract opcode from rINST - GOTO_OPCODE(t0) # jump to next instruction -#else - FETCH_S(rINST, 1) # rINST <- ssssAAAA (sign-extended) - addu a1, rINST, rINST # a1 <- byte offset, flags set - FETCH_ADVANCE_INST_RB(a1) # update rPC, load rINST - bgez a1, 1f - lw ra, THREAD_FLAGS_OFFSET(rSELF) - b MterpCheckSuspendAndContinue -1: - GET_INST_OPCODE(t0) # extract opcode from rINST - GOTO_OPCODE(t0) # jump to next instruction -#endif + b MterpCommonTakenBranchNoFlags /* ------------------------------ */ .balign 128 @@ -1369,39 +1330,11 @@ artMterpAsmInstructionStart = .L_op_nop * our "backward branch" test must be "<=0" instead of "<0". */ /* goto/32 +AAAAAAAA */ -#if MTERP_PROFILE_BRANCHES FETCH(a0, 1) # a0 <- aaaa (lo) FETCH(a1, 2) # a1 <- AAAA (hi) sll a1, a1, 16 or rINST, a0, a1 # rINST <- AAAAaaaa - EXPORT_PC() - move a0, rSELF - addu a1, rFP, OFF_FP_SHADOWFRAME - move a2, rINST - JAL(MterpProfileBranch) # (self, shadow_frame, offset) - bnez v0, MterpOnStackReplacement # Note: offset must be in rINST - addu a1, rINST, rINST # a1 <- byte offset - FETCH_ADVANCE_INST_RB(a1) # update rPC, load rINST - bgtz a1, 1f - lw ra, THREAD_FLAGS_OFFSET(rSELF) - b MterpCheckSuspendAndContinue -1: - GET_INST_OPCODE(t0) # extract opcode from rINST - GOTO_OPCODE(t0) # jump to next instruction -#else - FETCH(a0, 1) # a0 <- aaaa (lo) - FETCH(a1, 2) # a1 <- AAAA (hi) - sll a1, a1, 16 - or rINST, a0, a1 # rINST <- AAAAaaaa - addu a1, rINST, rINST # a1 <- byte offset - FETCH_ADVANCE_INST_RB(a1) # update rPC, load rINST - bgtz a1, 1f - lw ra, THREAD_FLAGS_OFFSET(rSELF) - b MterpCheckSuspendAndContinue -1: - GET_INST_OPCODE(t0) # extract opcode from rINST - GOTO_OPCODE(t0) # jump to next instruction -#endif + b MterpCommonTakenBranchNoFlags /* ------------------------------ */ .balign 128 @@ -1417,7 +1350,6 @@ artMterpAsmInstructionStart = .L_op_nop * for: packed-switch, sparse-switch */ /* op vAA, +BBBB */ -#if MTERP_PROFILE_BRANCHES FETCH(a0, 1) # a0 <- bbbb (lo) FETCH(a1, 2) # a1 <- BBBB (hi) GET_OPA(a3) # a3 <- AA @@ -1427,37 +1359,7 @@ artMterpAsmInstructionStart = .L_op_nop EAS1(a0, rPC, a0) # a0 <- PC + BBBBbbbb*2 JAL(MterpDoPackedSwitch) # a0 <- code-unit branch offset move rINST, v0 - EXPORT_PC() - move a0, rSELF - addu a1, rFP, OFF_FP_SHADOWFRAME - move a2, rINST - JAL(MterpProfileBranch) # (self, shadow_frame, offset) - bnez v0, MterpOnStackReplacement # Note: offset must be in rINST - addu a1, rINST, rINST # a1 <- byte offset - FETCH_ADVANCE_INST_RB(a1) # update rPC, load rINST - bgtz a1, .Lop_packed_switch_finish - lw ra, THREAD_FLAGS_OFFSET(rSELF) - b MterpCheckSuspendAndContinue -#else - FETCH(a0, 1) # a0 <- bbbb (lo) - FETCH(a1, 2) # a1 <- BBBB (hi) - GET_OPA(a3) # a3 <- AA - sll t0, a1, 16 - or a0, a0, t0 # a0 <- BBBBbbbb - GET_VREG(a1, a3) # a1 <- vAA - EAS1(a0, rPC, a0) # a0 <- PC + BBBBbbbb*2 - JAL(MterpDoPackedSwitch) # a0 <- code-unit branch offset - move rINST, v0 - addu a1, rINST, rINST # a1 <- byte offset - FETCH_ADVANCE_INST_RB(a1) # update rPC, load rINST - bgtz a1, 1f - lw ra, THREAD_FLAGS_OFFSET(rSELF) - b MterpCheckSuspendAndContinue -1: - GET_INST_OPCODE(t0) # extract opcode from rINST - GOTO_OPCODE(t0) # jump to next instruction -#endif - + b MterpCommonTakenBranchNoFlags /* ------------------------------ */ .balign 128 @@ -1474,7 +1376,6 @@ artMterpAsmInstructionStart = .L_op_nop * for: packed-switch, sparse-switch */ /* op vAA, +BBBB */ -#if MTERP_PROFILE_BRANCHES FETCH(a0, 1) # a0 <- bbbb (lo) FETCH(a1, 2) # a1 <- BBBB (hi) GET_OPA(a3) # a3 <- AA @@ -1484,37 +1385,7 @@ artMterpAsmInstructionStart = .L_op_nop EAS1(a0, rPC, a0) # a0 <- PC + BBBBbbbb*2 JAL(MterpDoSparseSwitch) # a0 <- code-unit branch offset move rINST, v0 - EXPORT_PC() - move a0, rSELF - addu a1, rFP, OFF_FP_SHADOWFRAME - move a2, rINST - JAL(MterpProfileBranch) # (self, shadow_frame, offset) - bnez v0, MterpOnStackReplacement # Note: offset must be in rINST - addu a1, rINST, rINST # a1 <- byte offset - FETCH_ADVANCE_INST_RB(a1) # update rPC, load rINST - bgtz a1, .Lop_sparse_switch_finish - lw ra, THREAD_FLAGS_OFFSET(rSELF) - b MterpCheckSuspendAndContinue -#else - FETCH(a0, 1) # a0 <- bbbb (lo) - FETCH(a1, 2) # a1 <- BBBB (hi) - GET_OPA(a3) # a3 <- AA - sll t0, a1, 16 - or a0, a0, t0 # a0 <- BBBBbbbb - GET_VREG(a1, a3) # a1 <- vAA - EAS1(a0, rPC, a0) # a0 <- PC + BBBBbbbb*2 - JAL(MterpDoSparseSwitch) # a0 <- code-unit branch offset - move rINST, v0 - addu a1, rINST, rINST # a1 <- byte offset - FETCH_ADVANCE_INST_RB(a1) # update rPC, load rINST - bgtz a1, 1f - lw ra, THREAD_FLAGS_OFFSET(rSELF) - b MterpCheckSuspendAndContinue -1: - GET_INST_OPCODE(t0) # extract opcode from rINST - GOTO_OPCODE(t0) # jump to next instruction -#endif - + b MterpCommonTakenBranchNoFlags /* ------------------------------ */ @@ -1772,9 +1643,8 @@ artMterpAsmInstructionStart = .L_op_nop /* File: mips/op_if_eq.S */ /* File: mips/bincmp.S */ /* - * Generic two-operand compare-and-branch operation. Provide a "revcmp" - * fragment that specifies the *reverse* comparison to perform, e.g. - * for "if-le" you would use "gt". + * Generic two-operand compare-and-branch operation. Provide a "condition" + * fragment that specifies the comparison to perform. * * For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le */ @@ -1782,27 +1652,14 @@ artMterpAsmInstructionStart = .L_op_nop GET_OPA4(a0) # a0 <- A+ GET_OPB(a1) # a1 <- B GET_VREG(a3, a1) # a3 <- vB - GET_VREG(a2, a0) # a2 <- vA - bne a2, a3, 1f # branch to 1 if comparison failed + GET_VREG(a0, a0) # a0 <- vA FETCH_S(rINST, 1) # rINST<- branch offset, in code units - b 2f -1: - li rINST, 2 # rINST- BYTE branch dist for not-taken -2: -#if MTERP_PROFILE_BRANCHES - EXPORT_PC() - move a0, rSELF - addu a1, rFP, OFF_FP_SHADOWFRAME - move a2, rINST - JAL(MterpProfileBranch) # (self, shadow_frame, offset) - bnez v0, MterpOnStackReplacement # Note: offset must be in rINST -#endif - addu a2, rINST, rINST # convert to bytes - FETCH_ADVANCE_INST_RB(a2) # update rPC, load rINST - bgez a2, .L_op_if_eq_finish - lw ra, THREAD_FLAGS_OFFSET(rSELF) - b MterpCheckSuspendAndContinue - + beq a0, a3, MterpCommonTakenBranchNoFlags # compare (vA, vB) + li t0, JIT_CHECK_OSR + beq rPROFILE, t0, .L_check_not_taken_osr + FETCH_ADVANCE_INST(2) # advance rPC, load rINST + GET_INST_OPCODE(t0) # extract opcode from rINST + GOTO_OPCODE(t0) # jump to next instruction /* ------------------------------ */ @@ -1811,9 +1668,8 @@ artMterpAsmInstructionStart = .L_op_nop /* File: mips/op_if_ne.S */ /* File: mips/bincmp.S */ /* - * Generic two-operand compare-and-branch operation. Provide a "revcmp" - * fragment that specifies the *reverse* comparison to perform, e.g. - * for "if-le" you would use "gt". + * Generic two-operand compare-and-branch operation. Provide a "condition" + * fragment that specifies the comparison to perform. * * For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le */ @@ -1821,27 +1677,14 @@ artMterpAsmInstructionStart = .L_op_nop GET_OPA4(a0) # a0 <- A+ GET_OPB(a1) # a1 <- B GET_VREG(a3, a1) # a3 <- vB - GET_VREG(a2, a0) # a2 <- vA - beq a2, a3, 1f # branch to 1 if comparison failed + GET_VREG(a0, a0) # a0 <- vA FETCH_S(rINST, 1) # rINST<- branch offset, in code units - b 2f -1: - li rINST, 2 # rINST- BYTE branch dist for not-taken -2: -#if MTERP_PROFILE_BRANCHES - EXPORT_PC() - move a0, rSELF - addu a1, rFP, OFF_FP_SHADOWFRAME - move a2, rINST - JAL(MterpProfileBranch) # (self, shadow_frame, offset) - bnez v0, MterpOnStackReplacement # Note: offset must be in rINST -#endif - addu a2, rINST, rINST # convert to bytes - FETCH_ADVANCE_INST_RB(a2) # update rPC, load rINST - bgez a2, .L_op_if_ne_finish - lw ra, THREAD_FLAGS_OFFSET(rSELF) - b MterpCheckSuspendAndContinue - + bne a0, a3, MterpCommonTakenBranchNoFlags # compare (vA, vB) + li t0, JIT_CHECK_OSR + beq rPROFILE, t0, .L_check_not_taken_osr + FETCH_ADVANCE_INST(2) # advance rPC, load rINST + GET_INST_OPCODE(t0) # extract opcode from rINST + GOTO_OPCODE(t0) # jump to next instruction /* ------------------------------ */ @@ -1850,9 +1693,8 @@ artMterpAsmInstructionStart = .L_op_nop /* File: mips/op_if_lt.S */ /* File: mips/bincmp.S */ /* - * Generic two-operand compare-and-branch operation. Provide a "revcmp" - * fragment that specifies the *reverse* comparison to perform, e.g. - * for "if-le" you would use "gt". + * Generic two-operand compare-and-branch operation. Provide a "condition" + * fragment that specifies the comparison to perform. * * For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le */ @@ -1860,27 +1702,14 @@ artMterpAsmInstructionStart = .L_op_nop GET_OPA4(a0) # a0 <- A+ GET_OPB(a1) # a1 <- B GET_VREG(a3, a1) # a3 <- vB - GET_VREG(a2, a0) # a2 <- vA - bge a2, a3, 1f # branch to 1 if comparison failed + GET_VREG(a0, a0) # a0 <- vA FETCH_S(rINST, 1) # rINST<- branch offset, in code units - b 2f -1: - li rINST, 2 # rINST- BYTE branch dist for not-taken -2: -#if MTERP_PROFILE_BRANCHES - EXPORT_PC() - move a0, rSELF - addu a1, rFP, OFF_FP_SHADOWFRAME - move a2, rINST - JAL(MterpProfileBranch) # (self, shadow_frame, offset) - bnez v0, MterpOnStackReplacement # Note: offset must be in rINST -#endif - addu a2, rINST, rINST # convert to bytes - FETCH_ADVANCE_INST_RB(a2) # update rPC, load rINST - bgez a2, .L_op_if_lt_finish - lw ra, THREAD_FLAGS_OFFSET(rSELF) - b MterpCheckSuspendAndContinue - + blt a0, a3, MterpCommonTakenBranchNoFlags # compare (vA, vB) + li t0, JIT_CHECK_OSR + beq rPROFILE, t0, .L_check_not_taken_osr + FETCH_ADVANCE_INST(2) # advance rPC, load rINST + GET_INST_OPCODE(t0) # extract opcode from rINST + GOTO_OPCODE(t0) # jump to next instruction /* ------------------------------ */ @@ -1889,9 +1718,8 @@ artMterpAsmInstructionStart = .L_op_nop /* File: mips/op_if_ge.S */ /* File: mips/bincmp.S */ /* - * Generic two-operand compare-and-branch operation. Provide a "revcmp" - * fragment that specifies the *reverse* comparison to perform, e.g. - * for "if-le" you would use "gt". + * Generic two-operand compare-and-branch operation. Provide a "condition" + * fragment that specifies the comparison to perform. * * For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le */ @@ -1899,27 +1727,14 @@ artMterpAsmInstructionStart = .L_op_nop GET_OPA4(a0) # a0 <- A+ GET_OPB(a1) # a1 <- B GET_VREG(a3, a1) # a3 <- vB - GET_VREG(a2, a0) # a2 <- vA - blt a2, a3, 1f # branch to 1 if comparison failed + GET_VREG(a0, a0) # a0 <- vA FETCH_S(rINST, 1) # rINST<- branch offset, in code units - b 2f -1: - li rINST, 2 # rINST- BYTE branch dist for not-taken -2: -#if MTERP_PROFILE_BRANCHES - EXPORT_PC() - move a0, rSELF - addu a1, rFP, OFF_FP_SHADOWFRAME - move a2, rINST - JAL(MterpProfileBranch) # (self, shadow_frame, offset) - bnez v0, MterpOnStackReplacement # Note: offset must be in rINST -#endif - addu a2, rINST, rINST # convert to bytes - FETCH_ADVANCE_INST_RB(a2) # update rPC, load rINST - bgez a2, .L_op_if_ge_finish - lw ra, THREAD_FLAGS_OFFSET(rSELF) - b MterpCheckSuspendAndContinue - + bge a0, a3, MterpCommonTakenBranchNoFlags # compare (vA, vB) + li t0, JIT_CHECK_OSR + beq rPROFILE, t0, .L_check_not_taken_osr + FETCH_ADVANCE_INST(2) # advance rPC, load rINST + GET_INST_OPCODE(t0) # extract opcode from rINST + GOTO_OPCODE(t0) # jump to next instruction /* ------------------------------ */ @@ -1928,9 +1743,8 @@ artMterpAsmInstructionStart = .L_op_nop /* File: mips/op_if_gt.S */ /* File: mips/bincmp.S */ /* - * Generic two-operand compare-and-branch operation. Provide a "revcmp" - * fragment that specifies the *reverse* comparison to perform, e.g. - * for "if-le" you would use "gt". + * Generic two-operand compare-and-branch operation. Provide a "condition" + * fragment that specifies the comparison to perform. * * For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le */ @@ -1938,27 +1752,14 @@ artMterpAsmInstructionStart = .L_op_nop GET_OPA4(a0) # a0 <- A+ GET_OPB(a1) # a1 <- B GET_VREG(a3, a1) # a3 <- vB - GET_VREG(a2, a0) # a2 <- vA - ble a2, a3, 1f # branch to 1 if comparison failed + GET_VREG(a0, a0) # a0 <- vA FETCH_S(rINST, 1) # rINST<- branch offset, in code units - b 2f -1: - li rINST, 2 # rINST- BYTE branch dist for not-taken -2: -#if MTERP_PROFILE_BRANCHES - EXPORT_PC() - move a0, rSELF - addu a1, rFP, OFF_FP_SHADOWFRAME - move a2, rINST - JAL(MterpProfileBranch) # (self, shadow_frame, offset) - bnez v0, MterpOnStackReplacement # Note: offset must be in rINST -#endif - addu a2, rINST, rINST # convert to bytes - FETCH_ADVANCE_INST_RB(a2) # update rPC, load rINST - bgez a2, .L_op_if_gt_finish - lw ra, THREAD_FLAGS_OFFSET(rSELF) - b MterpCheckSuspendAndContinue - + bgt a0, a3, MterpCommonTakenBranchNoFlags # compare (vA, vB) + li t0, JIT_CHECK_OSR + beq rPROFILE, t0, .L_check_not_taken_osr + FETCH_ADVANCE_INST(2) # advance rPC, load rINST + GET_INST_OPCODE(t0) # extract opcode from rINST + GOTO_OPCODE(t0) # jump to next instruction /* ------------------------------ */ @@ -1967,9 +1768,8 @@ artMterpAsmInstructionStart = .L_op_nop /* File: mips/op_if_le.S */ /* File: mips/bincmp.S */ /* - * Generic two-operand compare-and-branch operation. Provide a "revcmp" - * fragment that specifies the *reverse* comparison to perform, e.g. - * for "if-le" you would use "gt". + * Generic two-operand compare-and-branch operation. Provide a "condition" + * fragment that specifies the comparison to perform. * * For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le */ @@ -1977,27 +1777,14 @@ artMterpAsmInstructionStart = .L_op_nop GET_OPA4(a0) # a0 <- A+ GET_OPB(a1) # a1 <- B GET_VREG(a3, a1) # a3 <- vB - GET_VREG(a2, a0) # a2 <- vA - bgt a2, a3, 1f # branch to 1 if comparison failed + GET_VREG(a0, a0) # a0 <- vA FETCH_S(rINST, 1) # rINST<- branch offset, in code units - b 2f -1: - li rINST, 2 # rINST- BYTE branch dist for not-taken -2: -#if MTERP_PROFILE_BRANCHES - EXPORT_PC() - move a0, rSELF - addu a1, rFP, OFF_FP_SHADOWFRAME - move a2, rINST - JAL(MterpProfileBranch) # (self, shadow_frame, offset) - bnez v0, MterpOnStackReplacement # Note: offset must be in rINST -#endif - addu a2, rINST, rINST # convert to bytes - FETCH_ADVANCE_INST_RB(a2) # update rPC, load rINST - bgez a2, .L_op_if_le_finish - lw ra, THREAD_FLAGS_OFFSET(rSELF) - b MterpCheckSuspendAndContinue - + ble a0, a3, MterpCommonTakenBranchNoFlags # compare (vA, vB) + li t0, JIT_CHECK_OSR + beq rPROFILE, t0, .L_check_not_taken_osr + FETCH_ADVANCE_INST(2) # advance rPC, load rINST + GET_INST_OPCODE(t0) # extract opcode from rINST + GOTO_OPCODE(t0) # jump to next instruction /* ------------------------------ */ @@ -2006,35 +1793,19 @@ artMterpAsmInstructionStart = .L_op_nop /* File: mips/op_if_eqz.S */ /* File: mips/zcmp.S */ /* - * Generic one-operand compare-and-branch operation. Provide a "revcmp" - * fragment that specifies the *reverse* comparison to perform, e.g. - * for "if-le" you would use "gt". + * Generic one-operand compare-and-branch operation. Provide a "condition" + * fragment that specifies the comparison to perform. * * for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez */ /* if-cmp vAA, +BBBB */ GET_OPA(a0) # a0 <- AA - GET_VREG(a2, a0) # a2 <- vAA + GET_VREG(a0, a0) # a0 <- vAA FETCH_S(rINST, 1) # rINST <- branch offset, in code units - bne a2, zero, 1f # branch to 1 if comparison failed - b 2f -1: - li rINST, 2 # rINST- BYTE branch dist for not-taken -2: -#if MTERP_PROFILE_BRANCHES - EXPORT_PC() - move a0, rSELF - addu a1, rFP, OFF_FP_SHADOWFRAME - move a2, rINST - JAL(MterpProfileBranch) # (self, shadow_frame, offset) - bnez v0, MterpOnStackReplacement # Note: offset must be in rINST -#endif - addu a1, rINST, rINST # convert to bytes - FETCH_ADVANCE_INST_RB(a1) # update rPC, load rINST - bgez a1, 3f - lw ra, THREAD_FLAGS_OFFSET(rSELF) - b MterpCheckSuspendAndContinue -3: + beq a0, zero, MterpCommonTakenBranchNoFlags + li t0, JIT_CHECK_OSR # possible OSR re-entry? + beq rPROFILE, t0, .L_check_not_taken_osr + FETCH_ADVANCE_INST(2) # advance rPC, load rINST GET_INST_OPCODE(t0) # extract opcode from rINST GOTO_OPCODE(t0) # jump to next instruction @@ -2045,35 +1816,19 @@ artMterpAsmInstructionStart = .L_op_nop /* File: mips/op_if_nez.S */ /* File: mips/zcmp.S */ /* - * Generic one-operand compare-and-branch operation. Provide a "revcmp" - * fragment that specifies the *reverse* comparison to perform, e.g. - * for "if-le" you would use "gt". + * Generic one-operand compare-and-branch operation. Provide a "condition" + * fragment that specifies the comparison to perform. * * for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez */ /* if-cmp vAA, +BBBB */ GET_OPA(a0) # a0 <- AA - GET_VREG(a2, a0) # a2 <- vAA + GET_VREG(a0, a0) # a0 <- vAA FETCH_S(rINST, 1) # rINST <- branch offset, in code units - beq a2, zero, 1f # branch to 1 if comparison failed - b 2f -1: - li rINST, 2 # rINST- BYTE branch dist for not-taken -2: -#if MTERP_PROFILE_BRANCHES - EXPORT_PC() - move a0, rSELF - addu a1, rFP, OFF_FP_SHADOWFRAME - move a2, rINST - JAL(MterpProfileBranch) # (self, shadow_frame, offset) - bnez v0, MterpOnStackReplacement # Note: offset must be in rINST -#endif - addu a1, rINST, rINST # convert to bytes - FETCH_ADVANCE_INST_RB(a1) # update rPC, load rINST - bgez a1, 3f - lw ra, THREAD_FLAGS_OFFSET(rSELF) - b MterpCheckSuspendAndContinue -3: + bne a0, zero, MterpCommonTakenBranchNoFlags + li t0, JIT_CHECK_OSR # possible OSR re-entry? + beq rPROFILE, t0, .L_check_not_taken_osr + FETCH_ADVANCE_INST(2) # advance rPC, load rINST GET_INST_OPCODE(t0) # extract opcode from rINST GOTO_OPCODE(t0) # jump to next instruction @@ -2084,35 +1839,19 @@ artMterpAsmInstructionStart = .L_op_nop /* File: mips/op_if_ltz.S */ /* File: mips/zcmp.S */ /* - * Generic one-operand compare-and-branch operation. Provide a "revcmp" - * fragment that specifies the *reverse* comparison to perform, e.g. - * for "if-le" you would use "gt". + * Generic one-operand compare-and-branch operation. Provide a "condition" + * fragment that specifies the comparison to perform. * * for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez */ /* if-cmp vAA, +BBBB */ GET_OPA(a0) # a0 <- AA - GET_VREG(a2, a0) # a2 <- vAA + GET_VREG(a0, a0) # a0 <- vAA FETCH_S(rINST, 1) # rINST <- branch offset, in code units - bge a2, zero, 1f # branch to 1 if comparison failed - b 2f -1: - li rINST, 2 # rINST- BYTE branch dist for not-taken -2: -#if MTERP_PROFILE_BRANCHES - EXPORT_PC() - move a0, rSELF - addu a1, rFP, OFF_FP_SHADOWFRAME - move a2, rINST - JAL(MterpProfileBranch) # (self, shadow_frame, offset) - bnez v0, MterpOnStackReplacement # Note: offset must be in rINST -#endif - addu a1, rINST, rINST # convert to bytes - FETCH_ADVANCE_INST_RB(a1) # update rPC, load rINST - bgez a1, 3f - lw ra, THREAD_FLAGS_OFFSET(rSELF) - b MterpCheckSuspendAndContinue -3: + blt a0, zero, MterpCommonTakenBranchNoFlags + li t0, JIT_CHECK_OSR # possible OSR re-entry? + beq rPROFILE, t0, .L_check_not_taken_osr + FETCH_ADVANCE_INST(2) # advance rPC, load rINST GET_INST_OPCODE(t0) # extract opcode from rINST GOTO_OPCODE(t0) # jump to next instruction @@ -2123,35 +1862,19 @@ artMterpAsmInstructionStart = .L_op_nop /* File: mips/op_if_gez.S */ /* File: mips/zcmp.S */ /* - * Generic one-operand compare-and-branch operation. Provide a "revcmp" - * fragment that specifies the *reverse* comparison to perform, e.g. - * for "if-le" you would use "gt". + * Generic one-operand compare-and-branch operation. Provide a "condition" + * fragment that specifies the comparison to perform. * * for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez */ /* if-cmp vAA, +BBBB */ GET_OPA(a0) # a0 <- AA - GET_VREG(a2, a0) # a2 <- vAA + GET_VREG(a0, a0) # a0 <- vAA FETCH_S(rINST, 1) # rINST <- branch offset, in code units - blt a2, zero, 1f # branch to 1 if comparison failed - b 2f -1: - li rINST, 2 # rINST- BYTE branch dist for not-taken -2: -#if MTERP_PROFILE_BRANCHES - EXPORT_PC() - move a0, rSELF - addu a1, rFP, OFF_FP_SHADOWFRAME - move a2, rINST - JAL(MterpProfileBranch) # (self, shadow_frame, offset) - bnez v0, MterpOnStackReplacement # Note: offset must be in rINST -#endif - addu a1, rINST, rINST # convert to bytes - FETCH_ADVANCE_INST_RB(a1) # update rPC, load rINST - bgez a1, 3f - lw ra, THREAD_FLAGS_OFFSET(rSELF) - b MterpCheckSuspendAndContinue -3: + bge a0, zero, MterpCommonTakenBranchNoFlags + li t0, JIT_CHECK_OSR # possible OSR re-entry? + beq rPROFILE, t0, .L_check_not_taken_osr + FETCH_ADVANCE_INST(2) # advance rPC, load rINST GET_INST_OPCODE(t0) # extract opcode from rINST GOTO_OPCODE(t0) # jump to next instruction @@ -2162,35 +1885,19 @@ artMterpAsmInstructionStart = .L_op_nop /* File: mips/op_if_gtz.S */ /* File: mips/zcmp.S */ /* - * Generic one-operand compare-and-branch operation. Provide a "revcmp" - * fragment that specifies the *reverse* comparison to perform, e.g. - * for "if-le" you would use "gt". + * Generic one-operand compare-and-branch operation. Provide a "condition" + * fragment that specifies the comparison to perform. * * for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez */ /* if-cmp vAA, +BBBB */ GET_OPA(a0) # a0 <- AA - GET_VREG(a2, a0) # a2 <- vAA + GET_VREG(a0, a0) # a0 <- vAA FETCH_S(rINST, 1) # rINST <- branch offset, in code units - ble a2, zero, 1f # branch to 1 if comparison failed - b 2f -1: - li rINST, 2 # rINST- BYTE branch dist for not-taken -2: -#if MTERP_PROFILE_BRANCHES - EXPORT_PC() - move a0, rSELF - addu a1, rFP, OFF_FP_SHADOWFRAME - move a2, rINST - JAL(MterpProfileBranch) # (self, shadow_frame, offset) - bnez v0, MterpOnStackReplacement # Note: offset must be in rINST -#endif - addu a1, rINST, rINST # convert to bytes - FETCH_ADVANCE_INST_RB(a1) # update rPC, load rINST - bgez a1, 3f - lw ra, THREAD_FLAGS_OFFSET(rSELF) - b MterpCheckSuspendAndContinue -3: + bgt a0, zero, MterpCommonTakenBranchNoFlags + li t0, JIT_CHECK_OSR # possible OSR re-entry? + beq rPROFILE, t0, .L_check_not_taken_osr + FETCH_ADVANCE_INST(2) # advance rPC, load rINST GET_INST_OPCODE(t0) # extract opcode from rINST GOTO_OPCODE(t0) # jump to next instruction @@ -2201,35 +1908,19 @@ artMterpAsmInstructionStart = .L_op_nop /* File: mips/op_if_lez.S */ /* File: mips/zcmp.S */ /* - * Generic one-operand compare-and-branch operation. Provide a "revcmp" - * fragment that specifies the *reverse* comparison to perform, e.g. - * for "if-le" you would use "gt". + * Generic one-operand compare-and-branch operation. Provide a "condition" + * fragment that specifies the comparison to perform. * * for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez */ /* if-cmp vAA, +BBBB */ GET_OPA(a0) # a0 <- AA - GET_VREG(a2, a0) # a2 <- vAA + GET_VREG(a0, a0) # a0 <- vAA FETCH_S(rINST, 1) # rINST <- branch offset, in code units - bgt a2, zero, 1f # branch to 1 if comparison failed - b 2f -1: - li rINST, 2 # rINST- BYTE branch dist for not-taken -2: -#if MTERP_PROFILE_BRANCHES - EXPORT_PC() - move a0, rSELF - addu a1, rFP, OFF_FP_SHADOWFRAME - move a2, rINST - JAL(MterpProfileBranch) # (self, shadow_frame, offset) - bnez v0, MterpOnStackReplacement # Note: offset must be in rINST -#endif - addu a1, rINST, rINST # convert to bytes - FETCH_ADVANCE_INST_RB(a1) # update rPC, load rINST - bgez a1, 3f - lw ra, THREAD_FLAGS_OFFSET(rSELF) - b MterpCheckSuspendAndContinue -3: + ble a0, zero, MterpCommonTakenBranchNoFlags + li t0, JIT_CHECK_OSR # possible OSR re-entry? + beq rPROFILE, t0, .L_check_not_taken_osr + FETCH_ADVANCE_INST(2) # advance rPC, load rINST GET_INST_OPCODE(t0) # extract opcode from rINST GOTO_OPCODE(t0) # jump to next instruction @@ -7983,18 +7674,6 @@ artMterpAsmInstructionEnd: .balign 4 artMterpAsmSisterStart: -/* continuation for op_packed_switch */ - -.Lop_packed_switch_finish: - GET_INST_OPCODE(t0) # extract opcode from rINST - GOTO_OPCODE(t0) # jump to next instruction - -/* continuation for op_sparse_switch */ - -.Lop_sparse_switch_finish: - GET_INST_OPCODE(t0) # extract opcode from rINST - GOTO_OPCODE(t0) # jump to next instruction - /* continuation for op_cmpl_float */ .Lop_cmpl_float_nan: @@ -8039,42 +7718,6 @@ artMterpAsmSisterStart: GET_INST_OPCODE(t0) # extract opcode from rINST SET_VREG_GOTO(rTEMP, rOBJ, t0) # vAA <- rTEMP -/* continuation for op_if_eq */ - -.L_op_if_eq_finish: - GET_INST_OPCODE(t0) # extract opcode from rINST - GOTO_OPCODE(t0) # jump to next instruction - -/* continuation for op_if_ne */ - -.L_op_if_ne_finish: - GET_INST_OPCODE(t0) # extract opcode from rINST - GOTO_OPCODE(t0) # jump to next instruction - -/* continuation for op_if_lt */ - -.L_op_if_lt_finish: - GET_INST_OPCODE(t0) # extract opcode from rINST - GOTO_OPCODE(t0) # jump to next instruction - -/* continuation for op_if_ge */ - -.L_op_if_ge_finish: - GET_INST_OPCODE(t0) # extract opcode from rINST - GOTO_OPCODE(t0) # jump to next instruction - -/* continuation for op_if_gt */ - -.L_op_if_gt_finish: - GET_INST_OPCODE(t0) # extract opcode from rINST - GOTO_OPCODE(t0) # jump to next instruction - -/* continuation for op_if_le */ - -.L_op_if_le_finish: - GET_INST_OPCODE(t0) # extract opcode from rINST - GOTO_OPCODE(t0) # jump to next instruction - /* continuation for op_float_to_int */ /* @@ -13089,20 +12732,110 @@ MterpException: /* NOTE: no fallthrough */ /* - * Check for suspend check request. Assumes rINST already loaded, rPC advanced and - * still needs to get the opcode and branch to it, and flags are in lr. + * Common handling for branches with support for Jit profiling. + * On entry: + * rINST <= signed offset + * rPROFILE <= signed hotness countdown (expanded to 32 bits) + * + * We have quite a few different cases for branch profiling, OSR detection and + * suspend check support here. + * + * Taken backward branches: + * If profiling active, do hotness countdown and report if we hit zero. + * If in osr check mode, see if our target is a compiled loop header entry and do OSR if so. + * Is there a pending suspend request? If so, suspend. + * + * Taken forward branches and not-taken backward branches: + * If in osr check mode, see if our target is a compiled loop header entry and do OSR if so. + * + * Our most common case is expected to be a taken backward branch with active jit profiling, + * but no full OSR check and no pending suspend request. + * Next most common case is not-taken branch with no full OSR check. */ -MterpCheckSuspendAndContinue: - lw rIBASE, THREAD_CURRENT_IBASE_OFFSET(rSELF) # refresh rIBASE +MterpCommonTakenBranchNoFlags: + bgtz rINST, .L_forward_branch # don't add forward branches to hotness +/* + * We need to subtract 1 from positive values and we should not see 0 here, + * so we may use the result of the comparison with -1. + */ +#if JIT_CHECK_OSR != -1 +# error "JIT_CHECK_OSR must be -1." +#endif + li t0, JIT_CHECK_OSR + beq rPROFILE, t0, .L_osr_check + blt rPROFILE, t0, .L_resume_backward_branch + subu rPROFILE, 1 + beqz rPROFILE, .L_add_batch # counted down to zero - report +.L_resume_backward_branch: + lw ra, THREAD_FLAGS_OFFSET(rSELF) + REFRESH_IBASE() + addu a2, rINST, rINST # a2<- byte offset + FETCH_ADVANCE_INST_RB(a2) # update rPC, load rINST and ra, (THREAD_SUSPEND_REQUEST | THREAD_CHECKPOINT_REQUEST) - bnez ra, 1f + bnez ra, .L_suspend_request_pending GET_INST_OPCODE(t0) # extract opcode from rINST GOTO_OPCODE(t0) # jump to next instruction -1: + +.L_suspend_request_pending: EXPORT_PC() move a0, rSELF JAL(MterpSuspendCheck) # (self) bnez v0, MterpFallback + REFRESH_IBASE() # might have changed during suspend + GET_INST_OPCODE(t0) # extract opcode from rINST + GOTO_OPCODE(t0) # jump to next instruction + +.L_no_count_backwards: + li t0, JIT_CHECK_OSR # check for possible OSR re-entry + bne rPROFILE, t0, .L_resume_backward_branch +.L_osr_check: + move a0, rSELF + addu a1, rFP, OFF_FP_SHADOWFRAME + move a2, rINST + EXPORT_PC() + JAL(MterpMaybeDoOnStackReplacement) # (self, shadow_frame, offset) + bnez v0, MterpOnStackReplacement + b .L_resume_backward_branch + +.L_forward_branch: + li t0, JIT_CHECK_OSR # check for possible OSR re-entry + beq rPROFILE, t0, .L_check_osr_forward +.L_resume_forward_branch: + add a2, rINST, rINST # a2<- byte offset + FETCH_ADVANCE_INST_RB(a2) # update rPC, load rINST + GET_INST_OPCODE(t0) # extract opcode from rINST + GOTO_OPCODE(t0) # jump to next instruction + +.L_check_osr_forward: + move a0, rSELF + addu a1, rFP, OFF_FP_SHADOWFRAME + move a2, rINST + EXPORT_PC() + JAL(MterpMaybeDoOnStackReplacement) # (self, shadow_frame, offset) + bnez v0, MterpOnStackReplacement + b .L_resume_forward_branch + +.L_add_batch: + addu a1, rFP, OFF_FP_SHADOWFRAME + sh rPROFILE, SHADOWFRAME_HOTNESS_COUNTDOWN_OFFSET(a1) + lw a0, OFF_FP_METHOD(rFP) + move a2, rSELF + JAL(MterpAddHotnessBatch) # (method, shadow_frame, self) + move rPROFILE, v0 # restore new hotness countdown to rPROFILE + b .L_no_count_backwards + +/* + * Entered from the conditional branch handlers when OSR check request active on + * not-taken path. All Dalvik not-taken conditional branch offsets are 2. + */ +.L_check_not_taken_osr: + move a0, rSELF + addu a1, rFP, OFF_FP_SHADOWFRAME + li a2, 2 + EXPORT_PC() + JAL(MterpMaybeDoOnStackReplacement) # (self, shadow_frame, offset) + bnez v0, MterpOnStackReplacement + FETCH_ADVANCE_INST(2) GET_INST_OPCODE(t0) # extract opcode from rINST GOTO_OPCODE(t0) # jump to next instruction @@ -13149,6 +12882,26 @@ MterpReturn: sw v1, 4(a2) li v0, 1 # signal return to caller. MterpDone: +/* + * At this point, we expect rPROFILE to be non-zero. If negative, hotness is disabled or we're + * checking for OSR. If greater than zero, we might have unreported hotness to register + * (the difference between the ending rPROFILE and the cached hotness counter). rPROFILE + * should only reach zero immediately after a hotness decrement, and is then reset to either + * a negative special state or the new non-zero countdown value. + */ + blez rPROFILE, .L_pop_and_return # if > 0, we may have some counts to report. + +MterpProfileActive: + move rINST, v0 # stash return value + /* Report cached hotness counts */ + lw a0, OFF_FP_METHOD(rFP) + addu a1, rFP, OFF_FP_SHADOWFRAME + move a2, rSELF + sh rPROFILE, SHADOWFRAME_HOTNESS_COUNTDOWN_OFFSET(a1) + JAL(MterpAddHotnessBatch) # (method, shadow_frame, self) + move v0, rINST # restore return value + +.L_pop_and_return: /* Restore from the stack and return. Frame size = STACK_SIZE */ STACK_LOAD_FULL() jalr zero, ra diff --git a/runtime/interpreter/mterp/out/mterp_mips64.S b/runtime/interpreter/mterp/out/mterp_mips64.S index a17252b2f8..29a12bfd31 100644 --- a/runtime/interpreter/mterp/out/mterp_mips64.S +++ b/runtime/interpreter/mterp/out/mterp_mips64.S @@ -58,16 +58,18 @@ The following registers have fixed assignments: s3 rINST first 16-bit code unit of current instruction s4 rIBASE interpreted instruction base pointer, used for computed goto s5 rREFS base of object references in shadow frame (ideally, we'll get rid of this later). + s6 rPROFILE jit profile hotness countdown */ /* During bringup, we'll use the shadow frame model instead of rFP */ /* single-purpose registers, given names for clarity */ -#define rPC s0 -#define rFP s1 -#define rSELF s2 -#define rINST s3 -#define rIBASE s4 -#define rREFS s5 +#define rPC s0 +#define rFP s1 +#define rSELF s2 +#define rINST s3 +#define rIBASE s4 +#define rREFS s5 +#define rPROFILE s6 /* * This is a #include, not a %include, because we want the C pre-processor @@ -87,7 +89,7 @@ The following registers have fixed assignments: #define OFF_FP_RESULT_REGISTER OFF_FP(SHADOWFRAME_RESULT_REGISTER_OFFSET) #define OFF_FP_DEX_PC_PTR OFF_FP(SHADOWFRAME_DEX_PC_PTR_OFFSET) #define OFF_FP_CODE_ITEM OFF_FP(SHADOWFRAME_CODE_ITEM_OFFSET) -#define OFF_FP_SHADOWFRAME (-SHADOWFRAME_VREGS_OFFSET) +#define OFF_FP_SHADOWFRAME OFF_FP(0) #define MTERP_PROFILE_BRANCHES 1 #define MTERP_LOGGING 0 @@ -128,6 +130,17 @@ The following registers have fixed assignments: .endm /* + * Fetch the next instruction from an offset specified by _reg and advance xPC. + * xPC to point to the next instruction. "_reg" must specify the distance + * in bytes, *not* 16-bit code units, and may be a signed value. Must not set flags. + * + */ +.macro FETCH_ADVANCE_INST_RB reg + daddu rPC, rPC, \reg + FETCH_INST +.endm + +/* * Fetch the next instruction from the specified offset. Advances rPC * to point to the next instruction. * @@ -274,7 +287,8 @@ The following registers have fixed assignments: #define STACK_OFFSET_S3 40 #define STACK_OFFSET_S4 48 #define STACK_OFFSET_S5 56 -#define STACK_SIZE 64 +#define STACK_OFFSET_S6 64 +#define STACK_SIZE 80 /* needs 16 byte alignment */ /* Constants for float/double_to_int/long conversions */ #define INT_MIN 0x80000000 @@ -344,6 +358,8 @@ ExecuteMterpImpl: .cfi_rel_offset 20, STACK_OFFSET_S4 sd s5, STACK_OFFSET_S5(sp) .cfi_rel_offset 21, STACK_OFFSET_S5 + sd s6, STACK_OFFSET_S6(sp) + .cfi_rel_offset 22, STACK_OFFSET_S6 /* Remember the return register */ sd a3, SHADOWFRAME_RESULT_REGISTER_OFFSET(a2) @@ -364,6 +380,12 @@ ExecuteMterpImpl: /* Starting ibase */ REFRESH_IBASE + /* Set up for backwards branches & osr profiling */ + ld a0, OFF_FP_METHOD(rFP) + daddu a1, rFP, OFF_FP_SHADOWFRAME + jal MterpSetUpHotnessCountdown + move rPROFILE, v0 # Starting hotness countdown to rPROFILE + /* start executing the instruction at rPC */ FETCH_INST GET_INST_OPCODE v0 @@ -1100,24 +1122,9 @@ artMterpAsmInstructionStart = .L_op_nop * double to get a byte offset. */ /* goto +AA */ - .extern MterpProfileBranch srl rINST, rINST, 8 seb rINST, rINST # rINST <- offset (sign-extended AA) -#if MTERP_PROFILE_BRANCHES - EXPORT_PC - move a0, rSELF - daddu a1, rFP, OFF_FP_SHADOWFRAME - move a2, rINST - jal MterpProfileBranch # (self, shadow_frame, offset) - bnezc v0, MterpOnStackReplacement # Note: offset must be in rINST -#endif - dlsa rPC, rINST, rPC, 1 # rPC <- rPC + offset * 2 - lw ra, THREAD_FLAGS_OFFSET(rSELF) # Preload flags for MterpCheckSuspendAndContinue - move a0, rINST # a0 <- offset - FETCH_INST # load rINST - bltz a0, MterpCheckSuspendAndContinue # suspend check if backwards branch - GET_INST_OPCODE v0 # extract opcode from rINST - GOTO_OPCODE v0 # jump to next instruction + b MterpCommonTakenBranchNoFlags /* ------------------------------ */ .balign 128 @@ -1130,23 +1137,8 @@ artMterpAsmInstructionStart = .L_op_nop * double to get a byte offset. */ /* goto/16 +AAAA */ - .extern MterpProfileBranch lh rINST, 2(rPC) # rINST <- offset (sign-extended AAAA) -#if MTERP_PROFILE_BRANCHES - EXPORT_PC - move a0, rSELF - daddu a1, rFP, OFF_FP_SHADOWFRAME - move a2, rINST - jal MterpProfileBranch # (self, shadow_frame, offset) - bnezc v0, MterpOnStackReplacement # Note: offset must be in rINST -#endif - dlsa rPC, rINST, rPC, 1 # rPC <- rPC + offset * 2 - lw ra, THREAD_FLAGS_OFFSET(rSELF) # Preload flags for MterpCheckSuspendAndContinue - move a0, rINST # a0 <- offset - FETCH_INST # load rINST - bltz a0, MterpCheckSuspendAndContinue # suspend check if backwards branch - GET_INST_OPCODE v0 # extract opcode from rINST - GOTO_OPCODE v0 # jump to next instruction + b MterpCommonTakenBranchNoFlags /* ------------------------------ */ .balign 128 @@ -1162,25 +1154,10 @@ artMterpAsmInstructionStart = .L_op_nop * our "backward branch" test must be "<=0" instead of "<0". */ /* goto/32 +AAAAAAAA */ - .extern MterpProfileBranch lh rINST, 2(rPC) # rINST <- aaaa (low) lh a1, 4(rPC) # a1 <- AAAA (high) ins rINST, a1, 16, 16 # rINST <- offset (sign-extended AAAAaaaa) -#if MTERP_PROFILE_BRANCHES - EXPORT_PC - move a0, rSELF - daddu a1, rFP, OFF_FP_SHADOWFRAME - move a2, rINST - jal MterpProfileBranch # (self, shadow_frame, offset) - bnezc v0, MterpOnStackReplacement # Note: offset must be in rINST -#endif - dlsa rPC, rINST, rPC, 1 # rPC <- rPC + offset * 2 - lw ra, THREAD_FLAGS_OFFSET(rSELF) # Preload flags for MterpCheckSuspendAndContinue - move a0, rINST # a0 <- offset - FETCH_INST # load rINST - blez a0, MterpCheckSuspendAndContinue # suspend check if backwards branch - GET_INST_OPCODE v0 # extract opcode from rINST - GOTO_OPCODE v0 # jump to next instruction + b MterpCommonTakenBranchNoFlags /* ------------------------------ */ .balign 128 @@ -1206,21 +1183,7 @@ artMterpAsmInstructionStart = .L_op_nop dlsa a0, a0, rPC, 1 # a0 <- PC + BBBBbbbb*2 jal MterpDoPackedSwitch # v0 <- code-unit branch offset move rINST, v0 -#if MTERP_PROFILE_BRANCHES - EXPORT_PC - move a0, rSELF - daddu a1, rFP, OFF_FP_SHADOWFRAME - move a2, rINST - jal MterpProfileBranch # (self, shadow_frame, offset) - bnezc v0, MterpOnStackReplacement # Note: offset must be in rINST -#endif - dlsa rPC, rINST, rPC, 1 # rPC <- rPC + offset * 2 - lw ra, THREAD_FLAGS_OFFSET(rSELF) # Preload flags for MterpCheckSuspendAndContinue - move a0, rINST # a0 <- offset - FETCH_INST # load rINST - blez a0, MterpCheckSuspendAndContinue # suspend check if backwards branch - GET_INST_OPCODE v0 # extract opcode from rINST - GOTO_OPCODE v0 # jump to next instruction + b MterpCommonTakenBranchNoFlags /* ------------------------------ */ .balign 128 @@ -1247,21 +1210,7 @@ artMterpAsmInstructionStart = .L_op_nop dlsa a0, a0, rPC, 1 # a0 <- PC + BBBBbbbb*2 jal MterpDoSparseSwitch # v0 <- code-unit branch offset move rINST, v0 -#if MTERP_PROFILE_BRANCHES - EXPORT_PC - move a0, rSELF - daddu a1, rFP, OFF_FP_SHADOWFRAME - move a2, rINST - jal MterpProfileBranch # (self, shadow_frame, offset) - bnezc v0, MterpOnStackReplacement # Note: offset must be in rINST -#endif - dlsa rPC, rINST, rPC, 1 # rPC <- rPC + offset * 2 - lw ra, THREAD_FLAGS_OFFSET(rSELF) # Preload flags for MterpCheckSuspendAndContinue - move a0, rINST # a0 <- offset - FETCH_INST # load rINST - blez a0, MterpCheckSuspendAndContinue # suspend check if backwards branch - GET_INST_OPCODE v0 # extract opcode from rINST - GOTO_OPCODE v0 # jump to next instruction + b MterpCommonTakenBranchNoFlags /* ------------------------------ */ @@ -1453,22 +1402,10 @@ artMterpAsmInstructionStart = .L_op_nop lh rINST, 2(rPC) # rINST <- offset (sign-extended CCCC) GET_VREG a0, a2 # a0 <- vA GET_VREG a1, a3 # a1 <- vB - beqc a0, a1, 1f - li rINST, 2 # offset if branch not taken -1: -#if MTERP_PROFILE_BRANCHES - EXPORT_PC - move a0, rSELF - daddu a1, rFP, OFF_FP_SHADOWFRAME - move a2, rINST - jal MterpProfileBranch # (self, shadow_frame, offset) - bnezc v0, MterpOnStackReplacement # Note: offset must be in rINST -#endif - dlsa rPC, rINST, rPC, 1 # rPC <- rPC + offset * 2 - lw ra, THREAD_FLAGS_OFFSET(rSELF) # Preload flags for MterpCheckSuspendAndContinue - move a0, rINST # a0 <- offset - FETCH_INST # load rINST - bltz a0, MterpCheckSuspendAndContinue # suspend check if backwards branch + beqc a0, a1, MterpCommonTakenBranchNoFlags + li v0, JIT_CHECK_OSR # possible OSR re-entry? + beqc rPROFILE, v0, .L_check_not_taken_osr + FETCH_ADVANCE_INST 2 # advance rPC, load rINST GET_INST_OPCODE v0 # extract opcode from rINST GOTO_OPCODE v0 # jump to next instruction @@ -1492,22 +1429,10 @@ artMterpAsmInstructionStart = .L_op_nop lh rINST, 2(rPC) # rINST <- offset (sign-extended CCCC) GET_VREG a0, a2 # a0 <- vA GET_VREG a1, a3 # a1 <- vB - bnec a0, a1, 1f - li rINST, 2 # offset if branch not taken -1: -#if MTERP_PROFILE_BRANCHES - EXPORT_PC - move a0, rSELF - daddu a1, rFP, OFF_FP_SHADOWFRAME - move a2, rINST - jal MterpProfileBranch # (self, shadow_frame, offset) - bnezc v0, MterpOnStackReplacement # Note: offset must be in rINST -#endif - dlsa rPC, rINST, rPC, 1 # rPC <- rPC + offset * 2 - lw ra, THREAD_FLAGS_OFFSET(rSELF) # Preload flags for MterpCheckSuspendAndContinue - move a0, rINST # a0 <- offset - FETCH_INST # load rINST - bltz a0, MterpCheckSuspendAndContinue # suspend check if backwards branch + bnec a0, a1, MterpCommonTakenBranchNoFlags + li v0, JIT_CHECK_OSR # possible OSR re-entry? + beqc rPROFILE, v0, .L_check_not_taken_osr + FETCH_ADVANCE_INST 2 # advance rPC, load rINST GET_INST_OPCODE v0 # extract opcode from rINST GOTO_OPCODE v0 # jump to next instruction @@ -1531,22 +1456,10 @@ artMterpAsmInstructionStart = .L_op_nop lh rINST, 2(rPC) # rINST <- offset (sign-extended CCCC) GET_VREG a0, a2 # a0 <- vA GET_VREG a1, a3 # a1 <- vB - bltc a0, a1, 1f - li rINST, 2 # offset if branch not taken -1: -#if MTERP_PROFILE_BRANCHES - EXPORT_PC - move a0, rSELF - daddu a1, rFP, OFF_FP_SHADOWFRAME - move a2, rINST - jal MterpProfileBranch # (self, shadow_frame, offset) - bnezc v0, MterpOnStackReplacement # Note: offset must be in rINST -#endif - dlsa rPC, rINST, rPC, 1 # rPC <- rPC + offset * 2 - lw ra, THREAD_FLAGS_OFFSET(rSELF) # Preload flags for MterpCheckSuspendAndContinue - move a0, rINST # a0 <- offset - FETCH_INST # load rINST - bltz a0, MterpCheckSuspendAndContinue # suspend check if backwards branch + bltc a0, a1, MterpCommonTakenBranchNoFlags + li v0, JIT_CHECK_OSR # possible OSR re-entry? + beqc rPROFILE, v0, .L_check_not_taken_osr + FETCH_ADVANCE_INST 2 # advance rPC, load rINST GET_INST_OPCODE v0 # extract opcode from rINST GOTO_OPCODE v0 # jump to next instruction @@ -1570,22 +1483,10 @@ artMterpAsmInstructionStart = .L_op_nop lh rINST, 2(rPC) # rINST <- offset (sign-extended CCCC) GET_VREG a0, a2 # a0 <- vA GET_VREG a1, a3 # a1 <- vB - bgec a0, a1, 1f - li rINST, 2 # offset if branch not taken -1: -#if MTERP_PROFILE_BRANCHES - EXPORT_PC - move a0, rSELF - daddu a1, rFP, OFF_FP_SHADOWFRAME - move a2, rINST - jal MterpProfileBranch # (self, shadow_frame, offset) - bnezc v0, MterpOnStackReplacement # Note: offset must be in rINST -#endif - dlsa rPC, rINST, rPC, 1 # rPC <- rPC + offset * 2 - lw ra, THREAD_FLAGS_OFFSET(rSELF) # Preload flags for MterpCheckSuspendAndContinue - move a0, rINST # a0 <- offset - FETCH_INST # load rINST - bltz a0, MterpCheckSuspendAndContinue # suspend check if backwards branch + bgec a0, a1, MterpCommonTakenBranchNoFlags + li v0, JIT_CHECK_OSR # possible OSR re-entry? + beqc rPROFILE, v0, .L_check_not_taken_osr + FETCH_ADVANCE_INST 2 # advance rPC, load rINST GET_INST_OPCODE v0 # extract opcode from rINST GOTO_OPCODE v0 # jump to next instruction @@ -1609,22 +1510,10 @@ artMterpAsmInstructionStart = .L_op_nop lh rINST, 2(rPC) # rINST <- offset (sign-extended CCCC) GET_VREG a0, a2 # a0 <- vA GET_VREG a1, a3 # a1 <- vB - bgtc a0, a1, 1f - li rINST, 2 # offset if branch not taken -1: -#if MTERP_PROFILE_BRANCHES - EXPORT_PC - move a0, rSELF - daddu a1, rFP, OFF_FP_SHADOWFRAME - move a2, rINST - jal MterpProfileBranch # (self, shadow_frame, offset) - bnezc v0, MterpOnStackReplacement # Note: offset must be in rINST -#endif - dlsa rPC, rINST, rPC, 1 # rPC <- rPC + offset * 2 - lw ra, THREAD_FLAGS_OFFSET(rSELF) # Preload flags for MterpCheckSuspendAndContinue - move a0, rINST # a0 <- offset - FETCH_INST # load rINST - bltz a0, MterpCheckSuspendAndContinue # suspend check if backwards branch + bgtc a0, a1, MterpCommonTakenBranchNoFlags + li v0, JIT_CHECK_OSR # possible OSR re-entry? + beqc rPROFILE, v0, .L_check_not_taken_osr + FETCH_ADVANCE_INST 2 # advance rPC, load rINST GET_INST_OPCODE v0 # extract opcode from rINST GOTO_OPCODE v0 # jump to next instruction @@ -1648,22 +1537,10 @@ artMterpAsmInstructionStart = .L_op_nop lh rINST, 2(rPC) # rINST <- offset (sign-extended CCCC) GET_VREG a0, a2 # a0 <- vA GET_VREG a1, a3 # a1 <- vB - blec a0, a1, 1f - li rINST, 2 # offset if branch not taken -1: -#if MTERP_PROFILE_BRANCHES - EXPORT_PC - move a0, rSELF - daddu a1, rFP, OFF_FP_SHADOWFRAME - move a2, rINST - jal MterpProfileBranch # (self, shadow_frame, offset) - bnezc v0, MterpOnStackReplacement # Note: offset must be in rINST -#endif - dlsa rPC, rINST, rPC, 1 # rPC <- rPC + offset * 2 - lw ra, THREAD_FLAGS_OFFSET(rSELF) # Preload flags for MterpCheckSuspendAndContinue - move a0, rINST # a0 <- offset - FETCH_INST # load rINST - bltz a0, MterpCheckSuspendAndContinue # suspend check if backwards branch + blec a0, a1, MterpCommonTakenBranchNoFlags + li v0, JIT_CHECK_OSR # possible OSR re-entry? + beqc rPROFILE, v0, .L_check_not_taken_osr + FETCH_ADVANCE_INST 2 # advance rPC, load rINST GET_INST_OPCODE v0 # extract opcode from rINST GOTO_OPCODE v0 # jump to next instruction @@ -1681,26 +1558,13 @@ artMterpAsmInstructionStart = .L_op_nop * For: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez */ /* if-cmp vAA, +BBBB */ - .extern MterpProfileBranch srl a2, rINST, 8 # a2 <- AA lh rINST, 2(rPC) # rINST <- offset (sign-extended BBBB) GET_VREG a0, a2 # a0 <- vAA - beqzc a0, 1f - li rINST, 2 # offset if branch not taken -1: -#if MTERP_PROFILE_BRANCHES - EXPORT_PC - move a0, rSELF - daddu a1, rFP, OFF_FP_SHADOWFRAME - move a2, rINST - jal MterpProfileBranch # (self, shadow_frame, offset) - bnezc v0, MterpOnStackReplacement # Note: offset must be in rINST -#endif - dlsa rPC, rINST, rPC, 1 # rPC <- rPC + offset * 2 - lw ra, THREAD_FLAGS_OFFSET(rSELF) # Preload flags for MterpCheckSuspendAndContinue - move a0, rINST # a0 <- offset - FETCH_INST # load rINST - bltz a0, MterpCheckSuspendAndContinue # suspend check if backwards branch + beqzc a0, MterpCommonTakenBranchNoFlags + li v0, JIT_CHECK_OSR # possible OSR re-entry? + beqc rPROFILE, v0, .L_check_not_taken_osr + FETCH_ADVANCE_INST 2 # advance rPC, load rINST GET_INST_OPCODE v0 # extract opcode from rINST GOTO_OPCODE v0 # jump to next instruction @@ -1718,26 +1582,13 @@ artMterpAsmInstructionStart = .L_op_nop * For: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez */ /* if-cmp vAA, +BBBB */ - .extern MterpProfileBranch srl a2, rINST, 8 # a2 <- AA lh rINST, 2(rPC) # rINST <- offset (sign-extended BBBB) GET_VREG a0, a2 # a0 <- vAA - bnezc a0, 1f - li rINST, 2 # offset if branch not taken -1: -#if MTERP_PROFILE_BRANCHES - EXPORT_PC - move a0, rSELF - daddu a1, rFP, OFF_FP_SHADOWFRAME - move a2, rINST - jal MterpProfileBranch # (self, shadow_frame, offset) - bnezc v0, MterpOnStackReplacement # Note: offset must be in rINST -#endif - dlsa rPC, rINST, rPC, 1 # rPC <- rPC + offset * 2 - lw ra, THREAD_FLAGS_OFFSET(rSELF) # Preload flags for MterpCheckSuspendAndContinue - move a0, rINST # a0 <- offset - FETCH_INST # load rINST - bltz a0, MterpCheckSuspendAndContinue # suspend check if backwards branch + bnezc a0, MterpCommonTakenBranchNoFlags + li v0, JIT_CHECK_OSR # possible OSR re-entry? + beqc rPROFILE, v0, .L_check_not_taken_osr + FETCH_ADVANCE_INST 2 # advance rPC, load rINST GET_INST_OPCODE v0 # extract opcode from rINST GOTO_OPCODE v0 # jump to next instruction @@ -1755,26 +1606,13 @@ artMterpAsmInstructionStart = .L_op_nop * For: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez */ /* if-cmp vAA, +BBBB */ - .extern MterpProfileBranch srl a2, rINST, 8 # a2 <- AA lh rINST, 2(rPC) # rINST <- offset (sign-extended BBBB) GET_VREG a0, a2 # a0 <- vAA - bltzc a0, 1f - li rINST, 2 # offset if branch not taken -1: -#if MTERP_PROFILE_BRANCHES - EXPORT_PC - move a0, rSELF - daddu a1, rFP, OFF_FP_SHADOWFRAME - move a2, rINST - jal MterpProfileBranch # (self, shadow_frame, offset) - bnezc v0, MterpOnStackReplacement # Note: offset must be in rINST -#endif - dlsa rPC, rINST, rPC, 1 # rPC <- rPC + offset * 2 - lw ra, THREAD_FLAGS_OFFSET(rSELF) # Preload flags for MterpCheckSuspendAndContinue - move a0, rINST # a0 <- offset - FETCH_INST # load rINST - bltz a0, MterpCheckSuspendAndContinue # suspend check if backwards branch + bltzc a0, MterpCommonTakenBranchNoFlags + li v0, JIT_CHECK_OSR # possible OSR re-entry? + beqc rPROFILE, v0, .L_check_not_taken_osr + FETCH_ADVANCE_INST 2 # advance rPC, load rINST GET_INST_OPCODE v0 # extract opcode from rINST GOTO_OPCODE v0 # jump to next instruction @@ -1792,26 +1630,13 @@ artMterpAsmInstructionStart = .L_op_nop * For: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez */ /* if-cmp vAA, +BBBB */ - .extern MterpProfileBranch srl a2, rINST, 8 # a2 <- AA lh rINST, 2(rPC) # rINST <- offset (sign-extended BBBB) GET_VREG a0, a2 # a0 <- vAA - bgezc a0, 1f - li rINST, 2 # offset if branch not taken -1: -#if MTERP_PROFILE_BRANCHES - EXPORT_PC - move a0, rSELF - daddu a1, rFP, OFF_FP_SHADOWFRAME - move a2, rINST - jal MterpProfileBranch # (self, shadow_frame, offset) - bnezc v0, MterpOnStackReplacement # Note: offset must be in rINST -#endif - dlsa rPC, rINST, rPC, 1 # rPC <- rPC + offset * 2 - lw ra, THREAD_FLAGS_OFFSET(rSELF) # Preload flags for MterpCheckSuspendAndContinue - move a0, rINST # a0 <- offset - FETCH_INST # load rINST - bltz a0, MterpCheckSuspendAndContinue # suspend check if backwards branch + bgezc a0, MterpCommonTakenBranchNoFlags + li v0, JIT_CHECK_OSR # possible OSR re-entry? + beqc rPROFILE, v0, .L_check_not_taken_osr + FETCH_ADVANCE_INST 2 # advance rPC, load rINST GET_INST_OPCODE v0 # extract opcode from rINST GOTO_OPCODE v0 # jump to next instruction @@ -1829,26 +1654,13 @@ artMterpAsmInstructionStart = .L_op_nop * For: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez */ /* if-cmp vAA, +BBBB */ - .extern MterpProfileBranch srl a2, rINST, 8 # a2 <- AA lh rINST, 2(rPC) # rINST <- offset (sign-extended BBBB) GET_VREG a0, a2 # a0 <- vAA - bgtzc a0, 1f - li rINST, 2 # offset if branch not taken -1: -#if MTERP_PROFILE_BRANCHES - EXPORT_PC - move a0, rSELF - daddu a1, rFP, OFF_FP_SHADOWFRAME - move a2, rINST - jal MterpProfileBranch # (self, shadow_frame, offset) - bnezc v0, MterpOnStackReplacement # Note: offset must be in rINST -#endif - dlsa rPC, rINST, rPC, 1 # rPC <- rPC + offset * 2 - lw ra, THREAD_FLAGS_OFFSET(rSELF) # Preload flags for MterpCheckSuspendAndContinue - move a0, rINST # a0 <- offset - FETCH_INST # load rINST - bltz a0, MterpCheckSuspendAndContinue # suspend check if backwards branch + bgtzc a0, MterpCommonTakenBranchNoFlags + li v0, JIT_CHECK_OSR # possible OSR re-entry? + beqc rPROFILE, v0, .L_check_not_taken_osr + FETCH_ADVANCE_INST 2 # advance rPC, load rINST GET_INST_OPCODE v0 # extract opcode from rINST GOTO_OPCODE v0 # jump to next instruction @@ -1866,26 +1678,13 @@ artMterpAsmInstructionStart = .L_op_nop * For: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez */ /* if-cmp vAA, +BBBB */ - .extern MterpProfileBranch srl a2, rINST, 8 # a2 <- AA lh rINST, 2(rPC) # rINST <- offset (sign-extended BBBB) GET_VREG a0, a2 # a0 <- vAA - blezc a0, 1f - li rINST, 2 # offset if branch not taken -1: -#if MTERP_PROFILE_BRANCHES - EXPORT_PC - move a0, rSELF - daddu a1, rFP, OFF_FP_SHADOWFRAME - move a2, rINST - jal MterpProfileBranch # (self, shadow_frame, offset) - bnezc v0, MterpOnStackReplacement # Note: offset must be in rINST -#endif - dlsa rPC, rINST, rPC, 1 # rPC <- rPC + offset * 2 - lw ra, THREAD_FLAGS_OFFSET(rSELF) # Preload flags for MterpCheckSuspendAndContinue - move a0, rINST # a0 <- offset - FETCH_INST # load rINST - bltz a0, MterpCheckSuspendAndContinue # suspend check if backwards branch + blezc a0, MterpCommonTakenBranchNoFlags + li v0, JIT_CHECK_OSR # possible OSR re-entry? + beqc rPROFILE, v0, .L_check_not_taken_osr + FETCH_ADVANCE_INST 2 # advance rPC, load rINST GET_INST_OPCODE v0 # extract opcode from rINST GOTO_OPCODE v0 # jump to next instruction @@ -12323,23 +12122,110 @@ MterpException: /* NOTE: no fallthrough */ /* - * Check for suspend check request. Assumes rINST already loaded, rPC advanced and - * still needs to get the opcode and branch to it, and flags are in ra. + * Common handling for branches with support for Jit profiling. + * On entry: + * rINST <= signed offset + * rPROFILE <= signed hotness countdown (expanded to 64 bits) + * + * We have quite a few different cases for branch profiling, OSR detection and + * suspend check support here. + * + * Taken backward branches: + * If profiling active, do hotness countdown and report if we hit zero. + * If in osr check mode, see if our target is a compiled loop header entry and do OSR if so. + * Is there a pending suspend request? If so, suspend. + * + * Taken forward branches and not-taken backward branches: + * If in osr check mode, see if our target is a compiled loop header entry and do OSR if so. + * + * Our most common case is expected to be a taken backward branch with active jit profiling, + * but no full OSR check and no pending suspend request. + * Next most common case is not-taken branch with no full OSR check. + * */ - .extern MterpSuspendCheck -MterpCheckSuspendAndContinue: +MterpCommonTakenBranchNoFlags: + bgtzc rINST, .L_forward_branch # don't add forward branches to hotness +/* + * We need to subtract 1 from positive values and we should not see 0 here, + * so we may use the result of the comparison with -1. + */ + li v0, JIT_CHECK_OSR + beqc rPROFILE, v0, .L_osr_check + bltc rPROFILE, v0, .L_resume_backward_branch + dsubu rPROFILE, 1 + beqzc rPROFILE, .L_add_batch # counted down to zero - report +.L_resume_backward_branch: + lw ra, THREAD_FLAGS_OFFSET(rSELF) REFRESH_IBASE - and ra, ra, (THREAD_SUSPEND_REQUEST | THREAD_CHECKPOINT_REQUEST) - bnez ra, check1 - GET_INST_OPCODE v0 # extract opcode from rINST - GOTO_OPCODE v0 # jump to next instruction -check1: + daddu a2, rINST, rINST # a2<- byte offset + FETCH_ADVANCE_INST_RB a2 # update rPC, load rINST + and ra, (THREAD_SUSPEND_REQUEST | THREAD_CHECKPOINT_REQUEST) + bnezc ra, .L_suspend_request_pending + GET_INST_OPCODE v0 # extract opcode from rINST + GOTO_OPCODE v0 # jump to next instruction + +.L_suspend_request_pending: EXPORT_PC move a0, rSELF - jal MterpSuspendCheck # (self) - bnezc v0, MterpFallback # Something in the environment changed, switch interpreters - GET_INST_OPCODE v0 # extract opcode from rINST - GOTO_OPCODE v0 # jump to next instruction + jal MterpSuspendCheck # (self) + bnezc v0, MterpFallback + REFRESH_IBASE # might have changed during suspend + GET_INST_OPCODE v0 # extract opcode from rINST + GOTO_OPCODE v0 # jump to next instruction + +.L_no_count_backwards: + li v0, JIT_CHECK_OSR # check for possible OSR re-entry + bnec rPROFILE, v0, .L_resume_backward_branch +.L_osr_check: + move a0, rSELF + daddu a1, rFP, OFF_FP_SHADOWFRAME + move a2, rINST + EXPORT_PC + jal MterpMaybeDoOnStackReplacement # (self, shadow_frame, offset) + bnezc v0, MterpOnStackReplacement + b .L_resume_backward_branch + +.L_forward_branch: + li v0, JIT_CHECK_OSR # check for possible OSR re-entry + beqc rPROFILE, v0, .L_check_osr_forward +.L_resume_forward_branch: + daddu a2, rINST, rINST # a2<- byte offset + FETCH_ADVANCE_INST_RB a2 # update rPC, load rINST + GET_INST_OPCODE v0 # extract opcode from rINST + GOTO_OPCODE v0 # jump to next instruction + +.L_check_osr_forward: + move a0, rSELF + daddu a1, rFP, OFF_FP_SHADOWFRAME + move a2, rINST + EXPORT_PC + jal MterpMaybeDoOnStackReplacement # (self, shadow_frame, offset) + bnezc v0, MterpOnStackReplacement + b .L_resume_forward_branch + +.L_add_batch: + daddu a1, rFP, OFF_FP_SHADOWFRAME + sh rPROFILE, SHADOWFRAME_HOTNESS_COUNTDOWN_OFFSET(a1) + ld a0, OFF_FP_METHOD(rFP) + move a2, rSELF + jal MterpAddHotnessBatch # (method, shadow_frame, self) + move rPROFILE, v0 # restore new hotness countdown to rPROFILE + b .L_no_count_backwards + +/* + * Entered from the conditional branch handlers when OSR check request active on + * not-taken path. All Dalvik not-taken conditional branch offsets are 2. + */ +.L_check_not_taken_osr: + move a0, rSELF + daddu a1, rFP, OFF_FP_SHADOWFRAME + li a2, 2 + EXPORT_PC + jal MterpMaybeDoOnStackReplacement # (self, shadow_frame, offset) + bnezc v0, MterpOnStackReplacement + FETCH_ADVANCE_INST 2 + GET_INST_OPCODE v0 # extract opcode from rINST + GOTO_OPCODE v0 # jump to next instruction /* * On-stack replacement has happened, and now we've returned from the compiled method. @@ -12395,6 +12281,28 @@ MterpReturn: check2: li v0, 1 # signal return to caller. MterpDone: +/* + * At this point, we expect rPROFILE to be non-zero. If negative, hotness is disabled or we're + * checking for OSR. If greater than zero, we might have unreported hotness to register + * (the difference between the ending rPROFILE and the cached hotness counter). rPROFILE + * should only reach zero immediately after a hotness decrement, and is then reset to either + * a negative special state or the new non-zero countdown value. + */ + blez rPROFILE, .L_pop_and_return # if > 0, we may have some counts to report. + +MterpProfileActive: + move rINST, v0 # stash return value + /* Report cached hotness counts */ + ld a0, OFF_FP_METHOD(rFP) + daddu a1, rFP, OFF_FP_SHADOWFRAME + move a2, rSELF + sh rPROFILE, SHADOWFRAME_HOTNESS_COUNTDOWN_OFFSET(a1) + jal MterpAddHotnessBatch # (method, shadow_frame, self) + move v0, rINST # restore return value + +.L_pop_and_return: + ld s6, STACK_OFFSET_S6(sp) + .cfi_restore 22 ld s5, STACK_OFFSET_S5(sp) .cfi_restore 21 ld s4, STACK_OFFSET_S4(sp) @@ -12421,5 +12329,6 @@ MterpDone: .cfi_adjust_cfa_offset -STACK_SIZE .cfi_endproc + .set reorder .size ExecuteMterpImpl, .-ExecuteMterpImpl |