ART: Profile all branches for on-stack replacement
Change the switch, goto and mterp interpreters to profile
not-taken as well as taken branches. This allows for on-stack
replacement when the cfg has been rearranged such that the loop
header was originally the fallthrough of a Dalvik byte-code branch.
Note that this increases the already-heavy cost of branch profiling.
Measuring on a Nexus 6 using a very branchy benchmark (logic subtest
from Caffeinemark), we see:
No profiling Taken only Taken & not-taken
mterp 9728 3434 2384
C++ goto 3914 2422 2037
C++ switch 2986 2411 2112
As measured, the cost of branch profiling is dominating execution
time. This will be addressed in follow-up CLs.
Change-Id: Ibc858f317398dd991ed8e4f3c3d72bd4c9a60594
diff --git a/runtime/interpreter/interpreter_goto_table_impl.cc b/runtime/interpreter/interpreter_goto_table_impl.cc
index ca8598e..12d6fdc 100644
--- a/runtime/interpreter/interpreter_goto_table_impl.cc
+++ b/runtime/interpreter/interpreter_goto_table_impl.cc
@@ -792,6 +792,7 @@
}
ADVANCE(offset);
} else {
+ BRANCH_INSTRUMENTATION(2);
ADVANCE(2);
}
}
@@ -810,6 +811,7 @@
}
ADVANCE(offset);
} else {
+ BRANCH_INSTRUMENTATION(2);
ADVANCE(2);
}
}
@@ -828,6 +830,7 @@
}
ADVANCE(offset);
} else {
+ BRANCH_INSTRUMENTATION(2);
ADVANCE(2);
}
}
@@ -846,6 +849,7 @@
}
ADVANCE(offset);
} else {
+ BRANCH_INSTRUMENTATION(2);
ADVANCE(2);
}
}
@@ -864,6 +868,7 @@
}
ADVANCE(offset);
} else {
+ BRANCH_INSTRUMENTATION(2);
ADVANCE(2);
}
}
@@ -882,6 +887,7 @@
}
ADVANCE(offset);
} else {
+ BRANCH_INSTRUMENTATION(2);
ADVANCE(2);
}
}
@@ -899,6 +905,7 @@
}
ADVANCE(offset);
} else {
+ BRANCH_INSTRUMENTATION(2);
ADVANCE(2);
}
}
@@ -916,6 +923,7 @@
}
ADVANCE(offset);
} else {
+ BRANCH_INSTRUMENTATION(2);
ADVANCE(2);
}
}
@@ -933,6 +941,7 @@
}
ADVANCE(offset);
} else {
+ BRANCH_INSTRUMENTATION(2);
ADVANCE(2);
}
}
@@ -950,6 +959,7 @@
}
ADVANCE(offset);
} else {
+ BRANCH_INSTRUMENTATION(2);
ADVANCE(2);
}
}
@@ -967,6 +977,7 @@
}
ADVANCE(offset);
} else {
+ BRANCH_INSTRUMENTATION(2);
ADVANCE(2);
}
}
@@ -984,6 +995,7 @@
}
ADVANCE(offset);
} else {
+ BRANCH_INSTRUMENTATION(2);
ADVANCE(2);
}
}
diff --git a/runtime/interpreter/interpreter_switch_impl.cc b/runtime/interpreter/interpreter_switch_impl.cc
index 25dbab2..0488dbf 100644
--- a/runtime/interpreter/interpreter_switch_impl.cc
+++ b/runtime/interpreter/interpreter_switch_impl.cc
@@ -712,6 +712,7 @@
}
inst = inst->RelativeAt(offset);
} else {
+ BRANCH_INSTRUMENTATION(2);
inst = inst->Next_2xx();
}
break;
@@ -727,6 +728,7 @@
}
inst = inst->RelativeAt(offset);
} else {
+ BRANCH_INSTRUMENTATION(2);
inst = inst->Next_2xx();
}
break;
@@ -742,6 +744,7 @@
}
inst = inst->RelativeAt(offset);
} else {
+ BRANCH_INSTRUMENTATION(2);
inst = inst->Next_2xx();
}
break;
@@ -757,6 +760,7 @@
}
inst = inst->RelativeAt(offset);
} else {
+ BRANCH_INSTRUMENTATION(2);
inst = inst->Next_2xx();
}
break;
@@ -772,6 +776,7 @@
}
inst = inst->RelativeAt(offset);
} else {
+ BRANCH_INSTRUMENTATION(2);
inst = inst->Next_2xx();
}
break;
@@ -787,6 +792,7 @@
}
inst = inst->RelativeAt(offset);
} else {
+ BRANCH_INSTRUMENTATION(2);
inst = inst->Next_2xx();
}
break;
@@ -801,6 +807,7 @@
}
inst = inst->RelativeAt(offset);
} else {
+ BRANCH_INSTRUMENTATION(2);
inst = inst->Next_2xx();
}
break;
@@ -815,6 +822,7 @@
}
inst = inst->RelativeAt(offset);
} else {
+ BRANCH_INSTRUMENTATION(2);
inst = inst->Next_2xx();
}
break;
@@ -829,6 +837,7 @@
}
inst = inst->RelativeAt(offset);
} else {
+ BRANCH_INSTRUMENTATION(2);
inst = inst->Next_2xx();
}
break;
@@ -843,6 +852,7 @@
}
inst = inst->RelativeAt(offset);
} else {
+ BRANCH_INSTRUMENTATION(2);
inst = inst->Next_2xx();
}
break;
@@ -857,6 +867,7 @@
}
inst = inst->RelativeAt(offset);
} else {
+ BRANCH_INSTRUMENTATION(2);
inst = inst->Next_2xx();
}
break;
@@ -871,6 +882,7 @@
}
inst = inst->RelativeAt(offset);
} else {
+ BRANCH_INSTRUMENTATION(2);
inst = inst->Next_2xx();
}
break;
diff --git a/runtime/interpreter/mterp/arm/bincmp.S b/runtime/interpreter/mterp/arm/bincmp.S
index 774e167..cfad714 100644
--- a/runtime/interpreter/mterp/arm/bincmp.S
+++ b/runtime/interpreter/mterp/arm/bincmp.S
@@ -6,14 +6,15 @@
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
/* if-cmp vA, vB, +CCCC */
-#if MTERP_PROFILE_BRANCHES
mov r1, rINST, lsr #12 @ r1<- B
ubfx r0, rINST, #8, #4 @ r0<- A
GET_VREG r3, r1 @ r3<- vB
GET_VREG r2, r0 @ r2<- vA
FETCH_S rINST, 1 @ rINST<- branch offset, in code units
cmp r2, r3 @ compare (vA, vB)
- b${revcmp} .L_${opcode}_not_taken
+ mov${revcmp} rINST, #2
+#if MTERP_PROFILE_BRANCHES
+ @ TUNING: once measurements are complete, remove #if and hand-schedule.
EXPORT_PC
mov r0, rSELF
add r1, rFP, #OFF_FP_SHADOWFRAME
@@ -21,28 +22,10 @@
bl MterpProfileBranch @ (self, shadow_frame, offset)
cmp r0, #0
bne MterpOnStackReplacement @ Note: offset must be in rINST
- adds r2, rINST, rINST @ convert to bytes, check sign
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
- bmi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-.L_${opcode}_not_taken:
- FETCH_ADVANCE_INST 2 @ update rPC, load rINST
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-#else
- mov r1, rINST, lsr #12 @ r1<- B
- ubfx r0, rINST, #8, #4 @ r0<- A
- GET_VREG r3, r1 @ r3<- vB
- GET_VREG r2, r0 @ r2<- vA
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- FETCH_S rINST, 1 @ rINST<- branch offset, in code units
- cmp r2, r3 @ compare (vA, vB)
- mov${revcmp} rINST, #2 @ rINST<- BYTE branch dist for not-taken
- adds r2, rINST, rINST @ convert to bytes, check sign
- FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
- bmi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
#endif
+ adds r2, rINST, rINST @ convert to bytes, check sign
+ ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
+ FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
+ bmi MterpCheckSuspendAndContinue
+ GET_INST_OPCODE ip @ extract opcode from rINST
+ GOTO_OPCODE ip @ jump to next instruction
diff --git a/runtime/interpreter/mterp/arm/zcmp.S b/runtime/interpreter/mterp/arm/zcmp.S
index 800804d..3d7dec0 100644
--- a/runtime/interpreter/mterp/arm/zcmp.S
+++ b/runtime/interpreter/mterp/arm/zcmp.S
@@ -6,13 +6,14 @@
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
-#if MTERP_PROFILE_BRANCHES
mov r0, rINST, lsr #8 @ r0<- AA
GET_VREG r2, r0 @ r2<- vAA
FETCH_S rINST, 1 @ rINST<- branch offset, in code units
ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
cmp r2, #0 @ compare (vA, 0)
- b${revcmp} .L_${opcode}_not_taken
+ mov${revcmp} rINST, #2
+#if MTERP_PROFILE_BRANCHES
+ @ TUNING: once measurements are complete, remove #if and hand-schedule.
EXPORT_PC
mov r0, rSELF
add r1, rFP, #OFF_FP_SHADOWFRAME
@@ -20,25 +21,9 @@
bl MterpProfileBranch @ (self, shadow_frame, offset)
cmp r0, #0
bne MterpOnStackReplacement @ Note: offset must be in rINST
- adds r1, rINST, rINST @ convert to bytes & set flags
- FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
- bmi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-.L_${opcode}_not_taken:
- FETCH_ADVANCE_INST 2 @ update rPC, load rINST
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-#else
- mov r0, rINST, lsr #8 @ r0<- AA
- GET_VREG r2, r0 @ r2<- vAA
- FETCH_S rINST, 1 @ rINST<- branch offset, in code units
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- cmp r2, #0 @ compare (vA, 0)
- mov${revcmp} rINST, #2 @ rINST<- inst branch dist for not-taken
- adds r1, rINST, rINST @ convert to bytes & set flags
- FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
- bmi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
#endif
+ adds r1, rINST, rINST @ convert to bytes & set flags
+ FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
+ bmi MterpCheckSuspendAndContinue
+ GET_INST_OPCODE ip @ extract opcode from rINST
+ GOTO_OPCODE ip @ jump to next instruction
diff --git a/runtime/interpreter/mterp/arm64/bincmp.S b/runtime/interpreter/mterp/arm64/bincmp.S
index ed850fc..2356ecb 100644
--- a/runtime/interpreter/mterp/arm64/bincmp.S
+++ b/runtime/interpreter/mterp/arm64/bincmp.S
@@ -6,31 +6,6 @@
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
/* if-cmp vA, vB, +CCCC */
-#if MTERP_PROFILE_BRANCHES
- lsr w1, wINST, #12 // w1<- B
- ubfx w0, wINST, #8, #4 // w0<- A
- GET_VREG w3, w1 // w3<- vB
- GET_VREG w2, w0 // w2<- vA
- FETCH_S wINST, 1 // wINST<- branch offset, in code units
- cmp w2, w3 // compare (vA, vB)
- b.${condition} .L_${opcode}_taken
- FETCH_ADVANCE_INST 2 // update rPC, load wINST
- GET_INST_OPCODE ip // extract opcode from wINST
- GOTO_OPCODE ip // jump to next instruction
-.L_${opcode}_taken:
- EXPORT_PC
- mov x0, xSELF
- add x1, xFP, #OFF_FP_SHADOWFRAME
- sbfm x2, xINST, 0, 31 // Sign extend branch offset
- bl MterpProfileBranch // (self, shadow_frame, offset)
- cbnz w0, MterpOnStackReplacement // Note: offset must be in xINST
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
- adds w2, wINST, wINST // convert to bytes, check sign
- FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
- b.mi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip // extract opcode from wINST
- GOTO_OPCODE ip // jump to next instruction
-#else
lsr w1, wINST, #12 // w1<- B
ubfx w0, wINST, #8, #4 // w0<- A
GET_VREG w3, w1 // w3<- vB
@@ -39,10 +14,18 @@
mov w0, #2 // Offset if branch not taken
cmp w2, w3 // compare (vA, vB)
csel wINST, w1, w0, ${condition} // Branch if true, stashing result in callee save reg.
+#if MTERP_PROFILE_BRANCHES
+ // TUINING: once measurements are complete, remove #if and hand-schedule.
+ EXPORT_PC
+ mov x0, xSELF
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ sbfm x2, xINST, 0, 31 // Sign extend branch offset
+ bl MterpProfileBranch // (self, shadow_frame, offset)
+ cbnz w0, MterpOnStackReplacement // Note: offset must be in xINST
+#endif
ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
adds w2, wINST, wINST // convert to bytes, check sign
FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
b.mi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
-#endif
diff --git a/runtime/interpreter/mterp/arm64/zcmp.S b/runtime/interpreter/mterp/arm64/zcmp.S
index e528d9f..3f1e1b1 100644
--- a/runtime/interpreter/mterp/arm64/zcmp.S
+++ b/runtime/interpreter/mterp/arm64/zcmp.S
@@ -6,39 +6,24 @@
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
-#if MTERP_PROFILE_BRANCHES
- lsr w0, wINST, #8 // w0<- AA
- GET_VREG w2, w0 // w2<- vAA
- FETCH_S wINST, 1 // w1<- branch offset, in code units
- cmp w2, #0 // compare (vA, 0)
- b.${condition} .L_${opcode}_taken
- FETCH_ADVANCE_INST 2 // update rPC, load wINST
- GET_INST_OPCODE ip // extract opcode from wINST
- GOTO_OPCODE ip // jump to next instruction
-.L_${opcode}_taken:
- EXPORT_PC
- mov x0, xSELF
- add x1, xFP, #OFF_FP_SHADOWFRAME
- sbfm x2, xINST, 0, 31
- bl MterpProfileBranch // (self, shadow_frame, offset)
- cbnz w0, MterpOnStackReplacement // Note: offset must be in wINST
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
- adds w2, wINST, wINST // convert to bytes & set flags
- FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
- b.mi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip // extract opcode from wINST
- GOTO_OPCODE ip // jump to next instruction
-#else
lsr w0, wINST, #8 // w0<- AA
GET_VREG w2, w0 // w2<- vAA
FETCH_S w1, 1 // w1<- branch offset, in code units
mov w0, #2 // Branch offset if not taken
cmp w2, #0 // compare (vA, 0)
csel wINST, w1, w0, ${condition} // Branch if true, stashing result in callee save reg
+#if MTERP_PROFILE_BRANCHES
+ // TUNING: once measurements are complete, remove #if and hand-schedule.
+ EXPORT_PC
+ mov x0, xSELF
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ sbfm x2, xINST, 0, 31
+ bl MterpProfileBranch // (self, shadow_frame, offset)
+ cbnz w0, MterpOnStackReplacement // Note: offset must be in wINST
+#endif
ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
adds w2, wINST, wINST // convert to bytes & set flags
FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
b.mi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
-#endif
diff --git a/runtime/interpreter/mterp/out/mterp_arm.S b/runtime/interpreter/mterp/out/mterp_arm.S
index 94cbd2d..2b74d4c 100644
--- a/runtime/interpreter/mterp/out/mterp_arm.S
+++ b/runtime/interpreter/mterp/out/mterp_arm.S
@@ -1534,14 +1534,15 @@
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
/* if-cmp vA, vB, +CCCC */
-#if MTERP_PROFILE_BRANCHES
mov r1, rINST, lsr #12 @ r1<- B
ubfx r0, rINST, #8, #4 @ r0<- A
GET_VREG r3, r1 @ r3<- vB
GET_VREG r2, r0 @ r2<- vA
FETCH_S rINST, 1 @ rINST<- branch offset, in code units
cmp r2, r3 @ compare (vA, vB)
- bne .L_op_if_eq_not_taken
+ movne rINST, #2
+#if MTERP_PROFILE_BRANCHES
+ @ TUNING: once measurements are complete, remove #if and hand-schedule.
EXPORT_PC
mov r0, rSELF
add r1, rFP, #OFF_FP_SHADOWFRAME
@@ -1549,31 +1550,13 @@
bl MterpProfileBranch @ (self, shadow_frame, offset)
cmp r0, #0
bne MterpOnStackReplacement @ Note: offset must be in rINST
- adds r2, rINST, rINST @ convert to bytes, check sign
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
- bmi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-.L_op_if_eq_not_taken:
- FETCH_ADVANCE_INST 2 @ update rPC, load rINST
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-#else
- mov r1, rINST, lsr #12 @ r1<- B
- ubfx r0, rINST, #8, #4 @ r0<- A
- GET_VREG r3, r1 @ r3<- vB
- GET_VREG r2, r0 @ r2<- vA
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- FETCH_S rINST, 1 @ rINST<- branch offset, in code units
- cmp r2, r3 @ compare (vA, vB)
- movne rINST, #2 @ rINST<- BYTE branch dist for not-taken
- adds r2, rINST, rINST @ convert to bytes, check sign
- FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
- bmi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
#endif
+ adds r2, rINST, rINST @ convert to bytes, check sign
+ ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
+ FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
+ bmi MterpCheckSuspendAndContinue
+ GET_INST_OPCODE ip @ extract opcode from rINST
+ GOTO_OPCODE ip @ jump to next instruction
/* ------------------------------ */
@@ -1589,14 +1572,15 @@
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
/* if-cmp vA, vB, +CCCC */
-#if MTERP_PROFILE_BRANCHES
mov r1, rINST, lsr #12 @ r1<- B
ubfx r0, rINST, #8, #4 @ r0<- A
GET_VREG r3, r1 @ r3<- vB
GET_VREG r2, r0 @ r2<- vA
FETCH_S rINST, 1 @ rINST<- branch offset, in code units
cmp r2, r3 @ compare (vA, vB)
- beq .L_op_if_ne_not_taken
+ moveq rINST, #2
+#if MTERP_PROFILE_BRANCHES
+ @ TUNING: once measurements are complete, remove #if and hand-schedule.
EXPORT_PC
mov r0, rSELF
add r1, rFP, #OFF_FP_SHADOWFRAME
@@ -1604,31 +1588,13 @@
bl MterpProfileBranch @ (self, shadow_frame, offset)
cmp r0, #0
bne MterpOnStackReplacement @ Note: offset must be in rINST
- adds r2, rINST, rINST @ convert to bytes, check sign
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
- bmi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-.L_op_if_ne_not_taken:
- FETCH_ADVANCE_INST 2 @ update rPC, load rINST
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-#else
- mov r1, rINST, lsr #12 @ r1<- B
- ubfx r0, rINST, #8, #4 @ r0<- A
- GET_VREG r3, r1 @ r3<- vB
- GET_VREG r2, r0 @ r2<- vA
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- FETCH_S rINST, 1 @ rINST<- branch offset, in code units
- cmp r2, r3 @ compare (vA, vB)
- moveq rINST, #2 @ rINST<- BYTE branch dist for not-taken
- adds r2, rINST, rINST @ convert to bytes, check sign
- FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
- bmi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
#endif
+ adds r2, rINST, rINST @ convert to bytes, check sign
+ ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
+ FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
+ bmi MterpCheckSuspendAndContinue
+ GET_INST_OPCODE ip @ extract opcode from rINST
+ GOTO_OPCODE ip @ jump to next instruction
/* ------------------------------ */
@@ -1644,14 +1610,15 @@
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
/* if-cmp vA, vB, +CCCC */
-#if MTERP_PROFILE_BRANCHES
mov r1, rINST, lsr #12 @ r1<- B
ubfx r0, rINST, #8, #4 @ r0<- A
GET_VREG r3, r1 @ r3<- vB
GET_VREG r2, r0 @ r2<- vA
FETCH_S rINST, 1 @ rINST<- branch offset, in code units
cmp r2, r3 @ compare (vA, vB)
- bge .L_op_if_lt_not_taken
+ movge rINST, #2
+#if MTERP_PROFILE_BRANCHES
+ @ TUNING: once measurements are complete, remove #if and hand-schedule.
EXPORT_PC
mov r0, rSELF
add r1, rFP, #OFF_FP_SHADOWFRAME
@@ -1659,31 +1626,13 @@
bl MterpProfileBranch @ (self, shadow_frame, offset)
cmp r0, #0
bne MterpOnStackReplacement @ Note: offset must be in rINST
- adds r2, rINST, rINST @ convert to bytes, check sign
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
- bmi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-.L_op_if_lt_not_taken:
- FETCH_ADVANCE_INST 2 @ update rPC, load rINST
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-#else
- mov r1, rINST, lsr #12 @ r1<- B
- ubfx r0, rINST, #8, #4 @ r0<- A
- GET_VREG r3, r1 @ r3<- vB
- GET_VREG r2, r0 @ r2<- vA
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- FETCH_S rINST, 1 @ rINST<- branch offset, in code units
- cmp r2, r3 @ compare (vA, vB)
- movge rINST, #2 @ rINST<- BYTE branch dist for not-taken
- adds r2, rINST, rINST @ convert to bytes, check sign
- FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
- bmi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
#endif
+ adds r2, rINST, rINST @ convert to bytes, check sign
+ ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
+ FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
+ bmi MterpCheckSuspendAndContinue
+ GET_INST_OPCODE ip @ extract opcode from rINST
+ GOTO_OPCODE ip @ jump to next instruction
/* ------------------------------ */
@@ -1699,14 +1648,15 @@
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
/* if-cmp vA, vB, +CCCC */
-#if MTERP_PROFILE_BRANCHES
mov r1, rINST, lsr #12 @ r1<- B
ubfx r0, rINST, #8, #4 @ r0<- A
GET_VREG r3, r1 @ r3<- vB
GET_VREG r2, r0 @ r2<- vA
FETCH_S rINST, 1 @ rINST<- branch offset, in code units
cmp r2, r3 @ compare (vA, vB)
- blt .L_op_if_ge_not_taken
+ movlt rINST, #2
+#if MTERP_PROFILE_BRANCHES
+ @ TUNING: once measurements are complete, remove #if and hand-schedule.
EXPORT_PC
mov r0, rSELF
add r1, rFP, #OFF_FP_SHADOWFRAME
@@ -1714,31 +1664,13 @@
bl MterpProfileBranch @ (self, shadow_frame, offset)
cmp r0, #0
bne MterpOnStackReplacement @ Note: offset must be in rINST
- adds r2, rINST, rINST @ convert to bytes, check sign
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
- bmi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-.L_op_if_ge_not_taken:
- FETCH_ADVANCE_INST 2 @ update rPC, load rINST
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-#else
- mov r1, rINST, lsr #12 @ r1<- B
- ubfx r0, rINST, #8, #4 @ r0<- A
- GET_VREG r3, r1 @ r3<- vB
- GET_VREG r2, r0 @ r2<- vA
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- FETCH_S rINST, 1 @ rINST<- branch offset, in code units
- cmp r2, r3 @ compare (vA, vB)
- movlt rINST, #2 @ rINST<- BYTE branch dist for not-taken
- adds r2, rINST, rINST @ convert to bytes, check sign
- FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
- bmi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
#endif
+ adds r2, rINST, rINST @ convert to bytes, check sign
+ ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
+ FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
+ bmi MterpCheckSuspendAndContinue
+ GET_INST_OPCODE ip @ extract opcode from rINST
+ GOTO_OPCODE ip @ jump to next instruction
/* ------------------------------ */
@@ -1754,14 +1686,15 @@
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
/* if-cmp vA, vB, +CCCC */
-#if MTERP_PROFILE_BRANCHES
mov r1, rINST, lsr #12 @ r1<- B
ubfx r0, rINST, #8, #4 @ r0<- A
GET_VREG r3, r1 @ r3<- vB
GET_VREG r2, r0 @ r2<- vA
FETCH_S rINST, 1 @ rINST<- branch offset, in code units
cmp r2, r3 @ compare (vA, vB)
- ble .L_op_if_gt_not_taken
+ movle rINST, #2
+#if MTERP_PROFILE_BRANCHES
+ @ TUNING: once measurements are complete, remove #if and hand-schedule.
EXPORT_PC
mov r0, rSELF
add r1, rFP, #OFF_FP_SHADOWFRAME
@@ -1769,31 +1702,13 @@
bl MterpProfileBranch @ (self, shadow_frame, offset)
cmp r0, #0
bne MterpOnStackReplacement @ Note: offset must be in rINST
- adds r2, rINST, rINST @ convert to bytes, check sign
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
- bmi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-.L_op_if_gt_not_taken:
- FETCH_ADVANCE_INST 2 @ update rPC, load rINST
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-#else
- mov r1, rINST, lsr #12 @ r1<- B
- ubfx r0, rINST, #8, #4 @ r0<- A
- GET_VREG r3, r1 @ r3<- vB
- GET_VREG r2, r0 @ r2<- vA
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- FETCH_S rINST, 1 @ rINST<- branch offset, in code units
- cmp r2, r3 @ compare (vA, vB)
- movle rINST, #2 @ rINST<- BYTE branch dist for not-taken
- adds r2, rINST, rINST @ convert to bytes, check sign
- FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
- bmi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
#endif
+ adds r2, rINST, rINST @ convert to bytes, check sign
+ ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
+ FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
+ bmi MterpCheckSuspendAndContinue
+ GET_INST_OPCODE ip @ extract opcode from rINST
+ GOTO_OPCODE ip @ jump to next instruction
/* ------------------------------ */
@@ -1809,14 +1724,15 @@
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
/* if-cmp vA, vB, +CCCC */
-#if MTERP_PROFILE_BRANCHES
mov r1, rINST, lsr #12 @ r1<- B
ubfx r0, rINST, #8, #4 @ r0<- A
GET_VREG r3, r1 @ r3<- vB
GET_VREG r2, r0 @ r2<- vA
FETCH_S rINST, 1 @ rINST<- branch offset, in code units
cmp r2, r3 @ compare (vA, vB)
- bgt .L_op_if_le_not_taken
+ movgt rINST, #2
+#if MTERP_PROFILE_BRANCHES
+ @ TUNING: once measurements are complete, remove #if and hand-schedule.
EXPORT_PC
mov r0, rSELF
add r1, rFP, #OFF_FP_SHADOWFRAME
@@ -1824,31 +1740,13 @@
bl MterpProfileBranch @ (self, shadow_frame, offset)
cmp r0, #0
bne MterpOnStackReplacement @ Note: offset must be in rINST
- adds r2, rINST, rINST @ convert to bytes, check sign
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
- bmi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-.L_op_if_le_not_taken:
- FETCH_ADVANCE_INST 2 @ update rPC, load rINST
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-#else
- mov r1, rINST, lsr #12 @ r1<- B
- ubfx r0, rINST, #8, #4 @ r0<- A
- GET_VREG r3, r1 @ r3<- vB
- GET_VREG r2, r0 @ r2<- vA
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- FETCH_S rINST, 1 @ rINST<- branch offset, in code units
- cmp r2, r3 @ compare (vA, vB)
- movgt rINST, #2 @ rINST<- BYTE branch dist for not-taken
- adds r2, rINST, rINST @ convert to bytes, check sign
- FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
- bmi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
#endif
+ adds r2, rINST, rINST @ convert to bytes, check sign
+ ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
+ FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
+ bmi MterpCheckSuspendAndContinue
+ GET_INST_OPCODE ip @ extract opcode from rINST
+ GOTO_OPCODE ip @ jump to next instruction
/* ------------------------------ */
@@ -1864,13 +1762,14 @@
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
-#if MTERP_PROFILE_BRANCHES
mov r0, rINST, lsr #8 @ r0<- AA
GET_VREG r2, r0 @ r2<- vAA
FETCH_S rINST, 1 @ rINST<- branch offset, in code units
ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
cmp r2, #0 @ compare (vA, 0)
- bne .L_op_if_eqz_not_taken
+ movne rINST, #2
+#if MTERP_PROFILE_BRANCHES
+ @ TUNING: once measurements are complete, remove #if and hand-schedule.
EXPORT_PC
mov r0, rSELF
add r1, rFP, #OFF_FP_SHADOWFRAME
@@ -1878,28 +1777,12 @@
bl MterpProfileBranch @ (self, shadow_frame, offset)
cmp r0, #0
bne MterpOnStackReplacement @ Note: offset must be in rINST
- adds r1, rINST, rINST @ convert to bytes & set flags
- FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
- bmi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-.L_op_if_eqz_not_taken:
- FETCH_ADVANCE_INST 2 @ update rPC, load rINST
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-#else
- mov r0, rINST, lsr #8 @ r0<- AA
- GET_VREG r2, r0 @ r2<- vAA
- FETCH_S rINST, 1 @ rINST<- branch offset, in code units
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- cmp r2, #0 @ compare (vA, 0)
- movne rINST, #2 @ rINST<- inst branch dist for not-taken
- adds r1, rINST, rINST @ convert to bytes & set flags
- FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
- bmi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
#endif
+ adds r1, rINST, rINST @ convert to bytes & set flags
+ FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
+ bmi MterpCheckSuspendAndContinue
+ GET_INST_OPCODE ip @ extract opcode from rINST
+ GOTO_OPCODE ip @ jump to next instruction
/* ------------------------------ */
@@ -1915,13 +1798,14 @@
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
-#if MTERP_PROFILE_BRANCHES
mov r0, rINST, lsr #8 @ r0<- AA
GET_VREG r2, r0 @ r2<- vAA
FETCH_S rINST, 1 @ rINST<- branch offset, in code units
ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
cmp r2, #0 @ compare (vA, 0)
- beq .L_op_if_nez_not_taken
+ moveq rINST, #2
+#if MTERP_PROFILE_BRANCHES
+ @ TUNING: once measurements are complete, remove #if and hand-schedule.
EXPORT_PC
mov r0, rSELF
add r1, rFP, #OFF_FP_SHADOWFRAME
@@ -1929,28 +1813,12 @@
bl MterpProfileBranch @ (self, shadow_frame, offset)
cmp r0, #0
bne MterpOnStackReplacement @ Note: offset must be in rINST
- adds r1, rINST, rINST @ convert to bytes & set flags
- FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
- bmi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-.L_op_if_nez_not_taken:
- FETCH_ADVANCE_INST 2 @ update rPC, load rINST
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-#else
- mov r0, rINST, lsr #8 @ r0<- AA
- GET_VREG r2, r0 @ r2<- vAA
- FETCH_S rINST, 1 @ rINST<- branch offset, in code units
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- cmp r2, #0 @ compare (vA, 0)
- moveq rINST, #2 @ rINST<- inst branch dist for not-taken
- adds r1, rINST, rINST @ convert to bytes & set flags
- FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
- bmi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
#endif
+ adds r1, rINST, rINST @ convert to bytes & set flags
+ FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
+ bmi MterpCheckSuspendAndContinue
+ GET_INST_OPCODE ip @ extract opcode from rINST
+ GOTO_OPCODE ip @ jump to next instruction
/* ------------------------------ */
@@ -1966,13 +1834,14 @@
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
-#if MTERP_PROFILE_BRANCHES
mov r0, rINST, lsr #8 @ r0<- AA
GET_VREG r2, r0 @ r2<- vAA
FETCH_S rINST, 1 @ rINST<- branch offset, in code units
ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
cmp r2, #0 @ compare (vA, 0)
- bge .L_op_if_ltz_not_taken
+ movge rINST, #2
+#if MTERP_PROFILE_BRANCHES
+ @ TUNING: once measurements are complete, remove #if and hand-schedule.
EXPORT_PC
mov r0, rSELF
add r1, rFP, #OFF_FP_SHADOWFRAME
@@ -1980,28 +1849,12 @@
bl MterpProfileBranch @ (self, shadow_frame, offset)
cmp r0, #0
bne MterpOnStackReplacement @ Note: offset must be in rINST
- adds r1, rINST, rINST @ convert to bytes & set flags
- FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
- bmi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-.L_op_if_ltz_not_taken:
- FETCH_ADVANCE_INST 2 @ update rPC, load rINST
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-#else
- mov r0, rINST, lsr #8 @ r0<- AA
- GET_VREG r2, r0 @ r2<- vAA
- FETCH_S rINST, 1 @ rINST<- branch offset, in code units
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- cmp r2, #0 @ compare (vA, 0)
- movge rINST, #2 @ rINST<- inst branch dist for not-taken
- adds r1, rINST, rINST @ convert to bytes & set flags
- FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
- bmi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
#endif
+ adds r1, rINST, rINST @ convert to bytes & set flags
+ FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
+ bmi MterpCheckSuspendAndContinue
+ GET_INST_OPCODE ip @ extract opcode from rINST
+ GOTO_OPCODE ip @ jump to next instruction
/* ------------------------------ */
@@ -2017,13 +1870,14 @@
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
-#if MTERP_PROFILE_BRANCHES
mov r0, rINST, lsr #8 @ r0<- AA
GET_VREG r2, r0 @ r2<- vAA
FETCH_S rINST, 1 @ rINST<- branch offset, in code units
ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
cmp r2, #0 @ compare (vA, 0)
- blt .L_op_if_gez_not_taken
+ movlt rINST, #2
+#if MTERP_PROFILE_BRANCHES
+ @ TUNING: once measurements are complete, remove #if and hand-schedule.
EXPORT_PC
mov r0, rSELF
add r1, rFP, #OFF_FP_SHADOWFRAME
@@ -2031,28 +1885,12 @@
bl MterpProfileBranch @ (self, shadow_frame, offset)
cmp r0, #0
bne MterpOnStackReplacement @ Note: offset must be in rINST
- adds r1, rINST, rINST @ convert to bytes & set flags
- FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
- bmi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-.L_op_if_gez_not_taken:
- FETCH_ADVANCE_INST 2 @ update rPC, load rINST
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-#else
- mov r0, rINST, lsr #8 @ r0<- AA
- GET_VREG r2, r0 @ r2<- vAA
- FETCH_S rINST, 1 @ rINST<- branch offset, in code units
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- cmp r2, #0 @ compare (vA, 0)
- movlt rINST, #2 @ rINST<- inst branch dist for not-taken
- adds r1, rINST, rINST @ convert to bytes & set flags
- FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
- bmi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
#endif
+ adds r1, rINST, rINST @ convert to bytes & set flags
+ FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
+ bmi MterpCheckSuspendAndContinue
+ GET_INST_OPCODE ip @ extract opcode from rINST
+ GOTO_OPCODE ip @ jump to next instruction
/* ------------------------------ */
@@ -2068,13 +1906,14 @@
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
-#if MTERP_PROFILE_BRANCHES
mov r0, rINST, lsr #8 @ r0<- AA
GET_VREG r2, r0 @ r2<- vAA
FETCH_S rINST, 1 @ rINST<- branch offset, in code units
ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
cmp r2, #0 @ compare (vA, 0)
- ble .L_op_if_gtz_not_taken
+ movle rINST, #2
+#if MTERP_PROFILE_BRANCHES
+ @ TUNING: once measurements are complete, remove #if and hand-schedule.
EXPORT_PC
mov r0, rSELF
add r1, rFP, #OFF_FP_SHADOWFRAME
@@ -2082,28 +1921,12 @@
bl MterpProfileBranch @ (self, shadow_frame, offset)
cmp r0, #0
bne MterpOnStackReplacement @ Note: offset must be in rINST
- adds r1, rINST, rINST @ convert to bytes & set flags
- FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
- bmi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-.L_op_if_gtz_not_taken:
- FETCH_ADVANCE_INST 2 @ update rPC, load rINST
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-#else
- mov r0, rINST, lsr #8 @ r0<- AA
- GET_VREG r2, r0 @ r2<- vAA
- FETCH_S rINST, 1 @ rINST<- branch offset, in code units
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- cmp r2, #0 @ compare (vA, 0)
- movle rINST, #2 @ rINST<- inst branch dist for not-taken
- adds r1, rINST, rINST @ convert to bytes & set flags
- FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
- bmi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
#endif
+ adds r1, rINST, rINST @ convert to bytes & set flags
+ FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
+ bmi MterpCheckSuspendAndContinue
+ GET_INST_OPCODE ip @ extract opcode from rINST
+ GOTO_OPCODE ip @ jump to next instruction
/* ------------------------------ */
@@ -2119,13 +1942,14 @@
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
-#if MTERP_PROFILE_BRANCHES
mov r0, rINST, lsr #8 @ r0<- AA
GET_VREG r2, r0 @ r2<- vAA
FETCH_S rINST, 1 @ rINST<- branch offset, in code units
ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
cmp r2, #0 @ compare (vA, 0)
- bgt .L_op_if_lez_not_taken
+ movgt rINST, #2
+#if MTERP_PROFILE_BRANCHES
+ @ TUNING: once measurements are complete, remove #if and hand-schedule.
EXPORT_PC
mov r0, rSELF
add r1, rFP, #OFF_FP_SHADOWFRAME
@@ -2133,28 +1957,12 @@
bl MterpProfileBranch @ (self, shadow_frame, offset)
cmp r0, #0
bne MterpOnStackReplacement @ Note: offset must be in rINST
- adds r1, rINST, rINST @ convert to bytes & set flags
- FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
- bmi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-.L_op_if_lez_not_taken:
- FETCH_ADVANCE_INST 2 @ update rPC, load rINST
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-#else
- mov r0, rINST, lsr #8 @ r0<- AA
- GET_VREG r2, r0 @ r2<- vAA
- FETCH_S rINST, 1 @ rINST<- branch offset, in code units
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- cmp r2, #0 @ compare (vA, 0)
- movgt rINST, #2 @ rINST<- inst branch dist for not-taken
- adds r1, rINST, rINST @ convert to bytes & set flags
- FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
- bmi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
#endif
+ adds r1, rINST, rINST @ convert to bytes & set flags
+ FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
+ bmi MterpCheckSuspendAndContinue
+ GET_INST_OPCODE ip @ extract opcode from rINST
+ GOTO_OPCODE ip @ jump to next instruction
/* ------------------------------ */
diff --git a/runtime/interpreter/mterp/out/mterp_arm64.S b/runtime/interpreter/mterp/out/mterp_arm64.S
index e4825f0..c7c0fb5 100644
--- a/runtime/interpreter/mterp/out/mterp_arm64.S
+++ b/runtime/interpreter/mterp/out/mterp_arm64.S
@@ -1372,31 +1372,6 @@
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
/* if-cmp vA, vB, +CCCC */
-#if MTERP_PROFILE_BRANCHES
- lsr w1, wINST, #12 // w1<- B
- ubfx w0, wINST, #8, #4 // w0<- A
- GET_VREG w3, w1 // w3<- vB
- GET_VREG w2, w0 // w2<- vA
- FETCH_S wINST, 1 // wINST<- branch offset, in code units
- cmp w2, w3 // compare (vA, vB)
- b.eq .L_op_if_eq_taken
- FETCH_ADVANCE_INST 2 // update rPC, load wINST
- GET_INST_OPCODE ip // extract opcode from wINST
- GOTO_OPCODE ip // jump to next instruction
-.L_op_if_eq_taken:
- EXPORT_PC
- mov x0, xSELF
- add x1, xFP, #OFF_FP_SHADOWFRAME
- sbfm x2, xINST, 0, 31 // Sign extend branch offset
- bl MterpProfileBranch // (self, shadow_frame, offset)
- cbnz w0, MterpOnStackReplacement // Note: offset must be in xINST
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
- adds w2, wINST, wINST // convert to bytes, check sign
- FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
- b.mi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip // extract opcode from wINST
- GOTO_OPCODE ip // jump to next instruction
-#else
lsr w1, wINST, #12 // w1<- B
ubfx w0, wINST, #8, #4 // w0<- A
GET_VREG w3, w1 // w3<- vB
@@ -1405,13 +1380,21 @@
mov w0, #2 // Offset if branch not taken
cmp w2, w3 // compare (vA, vB)
csel wINST, w1, w0, eq // Branch if true, stashing result in callee save reg.
+#if MTERP_PROFILE_BRANCHES
+ // TUINING: once measurements are complete, remove #if and hand-schedule.
+ EXPORT_PC
+ mov x0, xSELF
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ sbfm x2, xINST, 0, 31 // Sign extend branch offset
+ bl MterpProfileBranch // (self, shadow_frame, offset)
+ cbnz w0, MterpOnStackReplacement // Note: offset must be in xINST
+#endif
ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
adds w2, wINST, wINST // convert to bytes, check sign
FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
b.mi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
-#endif
/* ------------------------------ */
@@ -1427,31 +1410,6 @@
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
/* if-cmp vA, vB, +CCCC */
-#if MTERP_PROFILE_BRANCHES
- lsr w1, wINST, #12 // w1<- B
- ubfx w0, wINST, #8, #4 // w0<- A
- GET_VREG w3, w1 // w3<- vB
- GET_VREG w2, w0 // w2<- vA
- FETCH_S wINST, 1 // wINST<- branch offset, in code units
- cmp w2, w3 // compare (vA, vB)
- b.ne .L_op_if_ne_taken
- FETCH_ADVANCE_INST 2 // update rPC, load wINST
- GET_INST_OPCODE ip // extract opcode from wINST
- GOTO_OPCODE ip // jump to next instruction
-.L_op_if_ne_taken:
- EXPORT_PC
- mov x0, xSELF
- add x1, xFP, #OFF_FP_SHADOWFRAME
- sbfm x2, xINST, 0, 31 // Sign extend branch offset
- bl MterpProfileBranch // (self, shadow_frame, offset)
- cbnz w0, MterpOnStackReplacement // Note: offset must be in xINST
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
- adds w2, wINST, wINST // convert to bytes, check sign
- FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
- b.mi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip // extract opcode from wINST
- GOTO_OPCODE ip // jump to next instruction
-#else
lsr w1, wINST, #12 // w1<- B
ubfx w0, wINST, #8, #4 // w0<- A
GET_VREG w3, w1 // w3<- vB
@@ -1460,13 +1418,21 @@
mov w0, #2 // Offset if branch not taken
cmp w2, w3 // compare (vA, vB)
csel wINST, w1, w0, ne // Branch if true, stashing result in callee save reg.
+#if MTERP_PROFILE_BRANCHES
+ // TUINING: once measurements are complete, remove #if and hand-schedule.
+ EXPORT_PC
+ mov x0, xSELF
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ sbfm x2, xINST, 0, 31 // Sign extend branch offset
+ bl MterpProfileBranch // (self, shadow_frame, offset)
+ cbnz w0, MterpOnStackReplacement // Note: offset must be in xINST
+#endif
ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
adds w2, wINST, wINST // convert to bytes, check sign
FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
b.mi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
-#endif
/* ------------------------------ */
@@ -1482,31 +1448,6 @@
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
/* if-cmp vA, vB, +CCCC */
-#if MTERP_PROFILE_BRANCHES
- lsr w1, wINST, #12 // w1<- B
- ubfx w0, wINST, #8, #4 // w0<- A
- GET_VREG w3, w1 // w3<- vB
- GET_VREG w2, w0 // w2<- vA
- FETCH_S wINST, 1 // wINST<- branch offset, in code units
- cmp w2, w3 // compare (vA, vB)
- b.lt .L_op_if_lt_taken
- FETCH_ADVANCE_INST 2 // update rPC, load wINST
- GET_INST_OPCODE ip // extract opcode from wINST
- GOTO_OPCODE ip // jump to next instruction
-.L_op_if_lt_taken:
- EXPORT_PC
- mov x0, xSELF
- add x1, xFP, #OFF_FP_SHADOWFRAME
- sbfm x2, xINST, 0, 31 // Sign extend branch offset
- bl MterpProfileBranch // (self, shadow_frame, offset)
- cbnz w0, MterpOnStackReplacement // Note: offset must be in xINST
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
- adds w2, wINST, wINST // convert to bytes, check sign
- FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
- b.mi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip // extract opcode from wINST
- GOTO_OPCODE ip // jump to next instruction
-#else
lsr w1, wINST, #12 // w1<- B
ubfx w0, wINST, #8, #4 // w0<- A
GET_VREG w3, w1 // w3<- vB
@@ -1515,13 +1456,21 @@
mov w0, #2 // Offset if branch not taken
cmp w2, w3 // compare (vA, vB)
csel wINST, w1, w0, lt // Branch if true, stashing result in callee save reg.
+#if MTERP_PROFILE_BRANCHES
+ // TUINING: once measurements are complete, remove #if and hand-schedule.
+ EXPORT_PC
+ mov x0, xSELF
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ sbfm x2, xINST, 0, 31 // Sign extend branch offset
+ bl MterpProfileBranch // (self, shadow_frame, offset)
+ cbnz w0, MterpOnStackReplacement // Note: offset must be in xINST
+#endif
ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
adds w2, wINST, wINST // convert to bytes, check sign
FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
b.mi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
-#endif
/* ------------------------------ */
@@ -1537,31 +1486,6 @@
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
/* if-cmp vA, vB, +CCCC */
-#if MTERP_PROFILE_BRANCHES
- lsr w1, wINST, #12 // w1<- B
- ubfx w0, wINST, #8, #4 // w0<- A
- GET_VREG w3, w1 // w3<- vB
- GET_VREG w2, w0 // w2<- vA
- FETCH_S wINST, 1 // wINST<- branch offset, in code units
- cmp w2, w3 // compare (vA, vB)
- b.ge .L_op_if_ge_taken
- FETCH_ADVANCE_INST 2 // update rPC, load wINST
- GET_INST_OPCODE ip // extract opcode from wINST
- GOTO_OPCODE ip // jump to next instruction
-.L_op_if_ge_taken:
- EXPORT_PC
- mov x0, xSELF
- add x1, xFP, #OFF_FP_SHADOWFRAME
- sbfm x2, xINST, 0, 31 // Sign extend branch offset
- bl MterpProfileBranch // (self, shadow_frame, offset)
- cbnz w0, MterpOnStackReplacement // Note: offset must be in xINST
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
- adds w2, wINST, wINST // convert to bytes, check sign
- FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
- b.mi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip // extract opcode from wINST
- GOTO_OPCODE ip // jump to next instruction
-#else
lsr w1, wINST, #12 // w1<- B
ubfx w0, wINST, #8, #4 // w0<- A
GET_VREG w3, w1 // w3<- vB
@@ -1570,13 +1494,21 @@
mov w0, #2 // Offset if branch not taken
cmp w2, w3 // compare (vA, vB)
csel wINST, w1, w0, ge // Branch if true, stashing result in callee save reg.
+#if MTERP_PROFILE_BRANCHES
+ // TUINING: once measurements are complete, remove #if and hand-schedule.
+ EXPORT_PC
+ mov x0, xSELF
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ sbfm x2, xINST, 0, 31 // Sign extend branch offset
+ bl MterpProfileBranch // (self, shadow_frame, offset)
+ cbnz w0, MterpOnStackReplacement // Note: offset must be in xINST
+#endif
ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
adds w2, wINST, wINST // convert to bytes, check sign
FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
b.mi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
-#endif
/* ------------------------------ */
@@ -1592,31 +1524,6 @@
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
/* if-cmp vA, vB, +CCCC */
-#if MTERP_PROFILE_BRANCHES
- lsr w1, wINST, #12 // w1<- B
- ubfx w0, wINST, #8, #4 // w0<- A
- GET_VREG w3, w1 // w3<- vB
- GET_VREG w2, w0 // w2<- vA
- FETCH_S wINST, 1 // wINST<- branch offset, in code units
- cmp w2, w3 // compare (vA, vB)
- b.gt .L_op_if_gt_taken
- FETCH_ADVANCE_INST 2 // update rPC, load wINST
- GET_INST_OPCODE ip // extract opcode from wINST
- GOTO_OPCODE ip // jump to next instruction
-.L_op_if_gt_taken:
- EXPORT_PC
- mov x0, xSELF
- add x1, xFP, #OFF_FP_SHADOWFRAME
- sbfm x2, xINST, 0, 31 // Sign extend branch offset
- bl MterpProfileBranch // (self, shadow_frame, offset)
- cbnz w0, MterpOnStackReplacement // Note: offset must be in xINST
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
- adds w2, wINST, wINST // convert to bytes, check sign
- FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
- b.mi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip // extract opcode from wINST
- GOTO_OPCODE ip // jump to next instruction
-#else
lsr w1, wINST, #12 // w1<- B
ubfx w0, wINST, #8, #4 // w0<- A
GET_VREG w3, w1 // w3<- vB
@@ -1625,13 +1532,21 @@
mov w0, #2 // Offset if branch not taken
cmp w2, w3 // compare (vA, vB)
csel wINST, w1, w0, gt // Branch if true, stashing result in callee save reg.
+#if MTERP_PROFILE_BRANCHES
+ // TUINING: once measurements are complete, remove #if and hand-schedule.
+ EXPORT_PC
+ mov x0, xSELF
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ sbfm x2, xINST, 0, 31 // Sign extend branch offset
+ bl MterpProfileBranch // (self, shadow_frame, offset)
+ cbnz w0, MterpOnStackReplacement // Note: offset must be in xINST
+#endif
ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
adds w2, wINST, wINST // convert to bytes, check sign
FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
b.mi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
-#endif
/* ------------------------------ */
@@ -1647,31 +1562,6 @@
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
/* if-cmp vA, vB, +CCCC */
-#if MTERP_PROFILE_BRANCHES
- lsr w1, wINST, #12 // w1<- B
- ubfx w0, wINST, #8, #4 // w0<- A
- GET_VREG w3, w1 // w3<- vB
- GET_VREG w2, w0 // w2<- vA
- FETCH_S wINST, 1 // wINST<- branch offset, in code units
- cmp w2, w3 // compare (vA, vB)
- b.le .L_op_if_le_taken
- FETCH_ADVANCE_INST 2 // update rPC, load wINST
- GET_INST_OPCODE ip // extract opcode from wINST
- GOTO_OPCODE ip // jump to next instruction
-.L_op_if_le_taken:
- EXPORT_PC
- mov x0, xSELF
- add x1, xFP, #OFF_FP_SHADOWFRAME
- sbfm x2, xINST, 0, 31 // Sign extend branch offset
- bl MterpProfileBranch // (self, shadow_frame, offset)
- cbnz w0, MterpOnStackReplacement // Note: offset must be in xINST
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
- adds w2, wINST, wINST // convert to bytes, check sign
- FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
- b.mi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip // extract opcode from wINST
- GOTO_OPCODE ip // jump to next instruction
-#else
lsr w1, wINST, #12 // w1<- B
ubfx w0, wINST, #8, #4 // w0<- A
GET_VREG w3, w1 // w3<- vB
@@ -1680,13 +1570,21 @@
mov w0, #2 // Offset if branch not taken
cmp w2, w3 // compare (vA, vB)
csel wINST, w1, w0, le // Branch if true, stashing result in callee save reg.
+#if MTERP_PROFILE_BRANCHES
+ // TUINING: once measurements are complete, remove #if and hand-schedule.
+ EXPORT_PC
+ mov x0, xSELF
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ sbfm x2, xINST, 0, 31 // Sign extend branch offset
+ bl MterpProfileBranch // (self, shadow_frame, offset)
+ cbnz w0, MterpOnStackReplacement // Note: offset must be in xINST
+#endif
ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
adds w2, wINST, wINST // convert to bytes, check sign
FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
b.mi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
-#endif
/* ------------------------------ */
@@ -1702,42 +1600,27 @@
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
-#if MTERP_PROFILE_BRANCHES
- lsr w0, wINST, #8 // w0<- AA
- GET_VREG w2, w0 // w2<- vAA
- FETCH_S wINST, 1 // w1<- branch offset, in code units
- cmp w2, #0 // compare (vA, 0)
- b.eq .L_op_if_eqz_taken
- FETCH_ADVANCE_INST 2 // update rPC, load wINST
- GET_INST_OPCODE ip // extract opcode from wINST
- GOTO_OPCODE ip // jump to next instruction
-.L_op_if_eqz_taken:
- EXPORT_PC
- mov x0, xSELF
- add x1, xFP, #OFF_FP_SHADOWFRAME
- sbfm x2, xINST, 0, 31
- bl MterpProfileBranch // (self, shadow_frame, offset)
- cbnz w0, MterpOnStackReplacement // Note: offset must be in wINST
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
- adds w2, wINST, wINST // convert to bytes & set flags
- FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
- b.mi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip // extract opcode from wINST
- GOTO_OPCODE ip // jump to next instruction
-#else
lsr w0, wINST, #8 // w0<- AA
GET_VREG w2, w0 // w2<- vAA
FETCH_S w1, 1 // w1<- branch offset, in code units
mov w0, #2 // Branch offset if not taken
cmp w2, #0 // compare (vA, 0)
csel wINST, w1, w0, eq // Branch if true, stashing result in callee save reg
+#if MTERP_PROFILE_BRANCHES
+ // TUNING: once measurements are complete, remove #if and hand-schedule.
+ EXPORT_PC
+ mov x0, xSELF
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ sbfm x2, xINST, 0, 31
+ bl MterpProfileBranch // (self, shadow_frame, offset)
+ cbnz w0, MterpOnStackReplacement // Note: offset must be in wINST
+#endif
ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
adds w2, wINST, wINST // convert to bytes & set flags
FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
b.mi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
-#endif
/* ------------------------------ */
@@ -1753,42 +1636,27 @@
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
-#if MTERP_PROFILE_BRANCHES
- lsr w0, wINST, #8 // w0<- AA
- GET_VREG w2, w0 // w2<- vAA
- FETCH_S wINST, 1 // w1<- branch offset, in code units
- cmp w2, #0 // compare (vA, 0)
- b.ne .L_op_if_nez_taken
- FETCH_ADVANCE_INST 2 // update rPC, load wINST
- GET_INST_OPCODE ip // extract opcode from wINST
- GOTO_OPCODE ip // jump to next instruction
-.L_op_if_nez_taken:
- EXPORT_PC
- mov x0, xSELF
- add x1, xFP, #OFF_FP_SHADOWFRAME
- sbfm x2, xINST, 0, 31
- bl MterpProfileBranch // (self, shadow_frame, offset)
- cbnz w0, MterpOnStackReplacement // Note: offset must be in wINST
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
- adds w2, wINST, wINST // convert to bytes & set flags
- FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
- b.mi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip // extract opcode from wINST
- GOTO_OPCODE ip // jump to next instruction
-#else
lsr w0, wINST, #8 // w0<- AA
GET_VREG w2, w0 // w2<- vAA
FETCH_S w1, 1 // w1<- branch offset, in code units
mov w0, #2 // Branch offset if not taken
cmp w2, #0 // compare (vA, 0)
csel wINST, w1, w0, ne // Branch if true, stashing result in callee save reg
+#if MTERP_PROFILE_BRANCHES
+ // TUNING: once measurements are complete, remove #if and hand-schedule.
+ EXPORT_PC
+ mov x0, xSELF
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ sbfm x2, xINST, 0, 31
+ bl MterpProfileBranch // (self, shadow_frame, offset)
+ cbnz w0, MterpOnStackReplacement // Note: offset must be in wINST
+#endif
ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
adds w2, wINST, wINST // convert to bytes & set flags
FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
b.mi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
-#endif
/* ------------------------------ */
@@ -1804,42 +1672,27 @@
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
-#if MTERP_PROFILE_BRANCHES
- lsr w0, wINST, #8 // w0<- AA
- GET_VREG w2, w0 // w2<- vAA
- FETCH_S wINST, 1 // w1<- branch offset, in code units
- cmp w2, #0 // compare (vA, 0)
- b.lt .L_op_if_ltz_taken
- FETCH_ADVANCE_INST 2 // update rPC, load wINST
- GET_INST_OPCODE ip // extract opcode from wINST
- GOTO_OPCODE ip // jump to next instruction
-.L_op_if_ltz_taken:
- EXPORT_PC
- mov x0, xSELF
- add x1, xFP, #OFF_FP_SHADOWFRAME
- sbfm x2, xINST, 0, 31
- bl MterpProfileBranch // (self, shadow_frame, offset)
- cbnz w0, MterpOnStackReplacement // Note: offset must be in wINST
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
- adds w2, wINST, wINST // convert to bytes & set flags
- FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
- b.mi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip // extract opcode from wINST
- GOTO_OPCODE ip // jump to next instruction
-#else
lsr w0, wINST, #8 // w0<- AA
GET_VREG w2, w0 // w2<- vAA
FETCH_S w1, 1 // w1<- branch offset, in code units
mov w0, #2 // Branch offset if not taken
cmp w2, #0 // compare (vA, 0)
csel wINST, w1, w0, lt // Branch if true, stashing result in callee save reg
+#if MTERP_PROFILE_BRANCHES
+ // TUNING: once measurements are complete, remove #if and hand-schedule.
+ EXPORT_PC
+ mov x0, xSELF
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ sbfm x2, xINST, 0, 31
+ bl MterpProfileBranch // (self, shadow_frame, offset)
+ cbnz w0, MterpOnStackReplacement // Note: offset must be in wINST
+#endif
ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
adds w2, wINST, wINST // convert to bytes & set flags
FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
b.mi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
-#endif
/* ------------------------------ */
@@ -1855,42 +1708,27 @@
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
-#if MTERP_PROFILE_BRANCHES
- lsr w0, wINST, #8 // w0<- AA
- GET_VREG w2, w0 // w2<- vAA
- FETCH_S wINST, 1 // w1<- branch offset, in code units
- cmp w2, #0 // compare (vA, 0)
- b.ge .L_op_if_gez_taken
- FETCH_ADVANCE_INST 2 // update rPC, load wINST
- GET_INST_OPCODE ip // extract opcode from wINST
- GOTO_OPCODE ip // jump to next instruction
-.L_op_if_gez_taken:
- EXPORT_PC
- mov x0, xSELF
- add x1, xFP, #OFF_FP_SHADOWFRAME
- sbfm x2, xINST, 0, 31
- bl MterpProfileBranch // (self, shadow_frame, offset)
- cbnz w0, MterpOnStackReplacement // Note: offset must be in wINST
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
- adds w2, wINST, wINST // convert to bytes & set flags
- FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
- b.mi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip // extract opcode from wINST
- GOTO_OPCODE ip // jump to next instruction
-#else
lsr w0, wINST, #8 // w0<- AA
GET_VREG w2, w0 // w2<- vAA
FETCH_S w1, 1 // w1<- branch offset, in code units
mov w0, #2 // Branch offset if not taken
cmp w2, #0 // compare (vA, 0)
csel wINST, w1, w0, ge // Branch if true, stashing result in callee save reg
+#if MTERP_PROFILE_BRANCHES
+ // TUNING: once measurements are complete, remove #if and hand-schedule.
+ EXPORT_PC
+ mov x0, xSELF
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ sbfm x2, xINST, 0, 31
+ bl MterpProfileBranch // (self, shadow_frame, offset)
+ cbnz w0, MterpOnStackReplacement // Note: offset must be in wINST
+#endif
ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
adds w2, wINST, wINST // convert to bytes & set flags
FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
b.mi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
-#endif
/* ------------------------------ */
@@ -1906,42 +1744,27 @@
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
-#if MTERP_PROFILE_BRANCHES
- lsr w0, wINST, #8 // w0<- AA
- GET_VREG w2, w0 // w2<- vAA
- FETCH_S wINST, 1 // w1<- branch offset, in code units
- cmp w2, #0 // compare (vA, 0)
- b.gt .L_op_if_gtz_taken
- FETCH_ADVANCE_INST 2 // update rPC, load wINST
- GET_INST_OPCODE ip // extract opcode from wINST
- GOTO_OPCODE ip // jump to next instruction
-.L_op_if_gtz_taken:
- EXPORT_PC
- mov x0, xSELF
- add x1, xFP, #OFF_FP_SHADOWFRAME
- sbfm x2, xINST, 0, 31
- bl MterpProfileBranch // (self, shadow_frame, offset)
- cbnz w0, MterpOnStackReplacement // Note: offset must be in wINST
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
- adds w2, wINST, wINST // convert to bytes & set flags
- FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
- b.mi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip // extract opcode from wINST
- GOTO_OPCODE ip // jump to next instruction
-#else
lsr w0, wINST, #8 // w0<- AA
GET_VREG w2, w0 // w2<- vAA
FETCH_S w1, 1 // w1<- branch offset, in code units
mov w0, #2 // Branch offset if not taken
cmp w2, #0 // compare (vA, 0)
csel wINST, w1, w0, gt // Branch if true, stashing result in callee save reg
+#if MTERP_PROFILE_BRANCHES
+ // TUNING: once measurements are complete, remove #if and hand-schedule.
+ EXPORT_PC
+ mov x0, xSELF
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ sbfm x2, xINST, 0, 31
+ bl MterpProfileBranch // (self, shadow_frame, offset)
+ cbnz w0, MterpOnStackReplacement // Note: offset must be in wINST
+#endif
ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
adds w2, wINST, wINST // convert to bytes & set flags
FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
b.mi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
-#endif
/* ------------------------------ */
@@ -1957,42 +1780,27 @@
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
-#if MTERP_PROFILE_BRANCHES
- lsr w0, wINST, #8 // w0<- AA
- GET_VREG w2, w0 // w2<- vAA
- FETCH_S wINST, 1 // w1<- branch offset, in code units
- cmp w2, #0 // compare (vA, 0)
- b.le .L_op_if_lez_taken
- FETCH_ADVANCE_INST 2 // update rPC, load wINST
- GET_INST_OPCODE ip // extract opcode from wINST
- GOTO_OPCODE ip // jump to next instruction
-.L_op_if_lez_taken:
- EXPORT_PC
- mov x0, xSELF
- add x1, xFP, #OFF_FP_SHADOWFRAME
- sbfm x2, xINST, 0, 31
- bl MterpProfileBranch // (self, shadow_frame, offset)
- cbnz w0, MterpOnStackReplacement // Note: offset must be in wINST
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
- adds w2, wINST, wINST // convert to bytes & set flags
- FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
- b.mi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip // extract opcode from wINST
- GOTO_OPCODE ip // jump to next instruction
-#else
lsr w0, wINST, #8 // w0<- AA
GET_VREG w2, w0 // w2<- vAA
FETCH_S w1, 1 // w1<- branch offset, in code units
mov w0, #2 // Branch offset if not taken
cmp w2, #0 // compare (vA, 0)
csel wINST, w1, w0, le // Branch if true, stashing result in callee save reg
+#if MTERP_PROFILE_BRANCHES
+ // TUNING: once measurements are complete, remove #if and hand-schedule.
+ EXPORT_PC
+ mov x0, xSELF
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ sbfm x2, xINST, 0, 31
+ bl MterpProfileBranch // (self, shadow_frame, offset)
+ cbnz w0, MterpOnStackReplacement // Note: offset must be in wINST
+#endif
ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
adds w2, wINST, wINST // convert to bytes & set flags
FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
b.mi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
-#endif
/* ------------------------------ */