ART: Enable JitProfiling for x86_64 Mterp
Adds branch profiling and enables for x86_64.
Support interpreter switching in x86_64 mterp.
Change-Id: I0cb9fcf3e2a01e411d84efc78449e86c10e6bcac
Signed-off-by: Serguei Katkov <serguei.i.katkov@intel.com>
diff --git a/runtime/interpreter/mterp/mterp.cc b/runtime/interpreter/mterp/mterp.cc
index e1bde1b..ca727f4 100644
--- a/runtime/interpreter/mterp/mterp.cc
+++ b/runtime/interpreter/mterp/mterp.cc
@@ -150,7 +150,8 @@
bool unhandled_instrumentation;
// TODO: enable for other targets after more extensive testing.
if ((kRuntimeISA == kArm64) || (kRuntimeISA == kArm) ||
- (kRuntimeISA == kX86) || (kRuntimeISA == kMips)) {
+ (kRuntimeISA == kX86_64) || (kRuntimeISA == kX86) ||
+ (kRuntimeISA == kMips)) {
unhandled_instrumentation = instrumentation->NonJitProfilingActive();
} else {
unhandled_instrumentation = instrumentation->IsActive();
diff --git a/runtime/interpreter/mterp/out/mterp_x86_64.S b/runtime/interpreter/mterp/out/mterp_x86_64.S
index 53fa50f..a1360e0 100644
--- a/runtime/interpreter/mterp/out/mterp_x86_64.S
+++ b/runtime/interpreter/mterp/out/mterp_x86_64.S
@@ -169,13 +169,23 @@
#define OFF_FP_CODE_ITEM OFF_FP(SHADOWFRAME_CODE_ITEM_OFFSET)
#define OFF_FP_SHADOWFRAME (-SHADOWFRAME_VREGS_OFFSET)
+#define MTERP_PROFILE_BRANCHES 1
+#define MTERP_LOGGING 0
+
/*
- *
- * The reference interpreter performs explicit suspect checks, which is somewhat wasteful.
- * Dalvik's interpreter folded suspend checks into the jump table mechanism, and eventually
- * mterp should do so as well.
+ * Profile branch. rINST should contain the offset. %eax is scratch.
*/
-#define MTERP_SUSPEND 0
+.macro MTERP_PROFILE_BRANCH
+#ifdef MTERP_PROFILE_BRANCHES
+ EXPORT_PC
+ movq rSELF, OUT_ARG0
+ leaq OFF_FP_SHADOWFRAME(rFP), OUT_ARG1
+ movl rINST, OUT_32_ARG2
+ call SYMBOL(MterpProfileBranch)
+ testb %al, %al
+ jnz MterpOnStackReplacement
+#endif
+.endm
/*
* "export" the PC to dex_pc field in the shadow frame, f/b/o future exception objects. Must
@@ -992,17 +1002,12 @@
* double to get a byte offset.
*/
/* goto +AA */
- movsbq rINSTbl, %rax # rax <- ssssssAA
- addq %rax, %rax # rax <- AA * 2
- leaq (rPC, %rax), rPC
+ movsbq rINSTbl, rINSTq # rINSTq <- ssssssAA
+ MTERP_PROFILE_BRANCH
+ addq rINSTq, rINSTq # rINSTq <- AA * 2
+ leaq (rPC, rINSTq), rPC
FETCH_INST
- jg 1f # AA * 2 > 0 => no suspend check
-#if MTERP_SUSPEND
- REFRESH_IBASE
-#else
- jmp MterpCheckSuspendAndContinue
-#endif
-1:
+ jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
GOTO_NEXT
/* ------------------------------ */
@@ -1016,17 +1021,12 @@
* double to get a byte offset.
*/
/* goto/16 +AAAA */
- movswq 2(rPC), %rax # rax <- ssssAAAA
- addq %rax, %rax # rax <- AA * 2
- leaq (rPC, %rax), rPC
+ movswq 2(rPC), rINSTq # rINSTq <- ssssAAAA
+ MTERP_PROFILE_BRANCH
+ addq rINSTq, rINSTq # rINSTq <- AA * 2
+ leaq (rPC, rINSTq), rPC
FETCH_INST
- jg 1f # AA * 2 > 0 => no suspend check
-#if MTERP_SUSPEND
- REFRESH_IBASE
-#else
- jmp MterpCheckSuspendAndContinue
-#endif
-1:
+ jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
GOTO_NEXT
/* ------------------------------ */
@@ -1043,17 +1043,12 @@
* to convert from Dalvik offset to byte offset.
*/
/* goto/32 +AAAAAAAA */
- movslq 2(rPC), %rax # rax <- AAAAAAAA
- addq %rax, %rax # rax <- AA * 2
- leaq (rPC, %rax), rPC
+ movslq 2(rPC), rINSTq # rINSTq <- AAAAAAAA
+ MTERP_PROFILE_BRANCH
+ addq rINSTq, rINSTq # rINSTq <- AA * 2
+ leaq (rPC, rINSTq), rPC
FETCH_INST
- jg 1f # AA * 2 > 0 => no suspend check
-#if MTERP_SUSPEND
- REFRESH_IBASE
-#else
- jmp MterpCheckSuspendAndContinue
-#endif
-1:
+ jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
GOTO_NEXT
/* ------------------------------ */
@@ -1074,17 +1069,12 @@
leaq (rPC,OUT_ARG0,2), OUT_ARG0 # rcx <- PC + BBBBbbbb*2
GET_VREG OUT_32_ARG1, rINSTq # eax <- vAA
call SYMBOL(MterpDoPackedSwitch)
- addl %eax, %eax
- movslq %eax, %rax
- leaq (rPC, %rax), rPC
+ movslq %eax, rINSTq
+ MTERP_PROFILE_BRANCH
+ addq rINSTq, rINSTq
+ leaq (rPC, rINSTq), rPC
FETCH_INST
- jg 1f
-#if MTERP_SUSPEND
- REFRESH_IBASE
-#else
- jmp MterpCheckSuspendAndContinue
-#endif
-1:
+ jle MterpCheckSuspendAndContinue
GOTO_NEXT
/* ------------------------------ */
@@ -1106,17 +1096,12 @@
leaq (rPC,OUT_ARG0,2), OUT_ARG0 # rcx <- PC + BBBBbbbb*2
GET_VREG OUT_32_ARG1, rINSTq # eax <- vAA
call SYMBOL(MterpDoSparseSwitch)
- addl %eax, %eax
- movslq %eax, %rax
- leaq (rPC, %rax), rPC
+ movslq %eax, rINSTq
+ MTERP_PROFILE_BRANCH
+ addq rINSTq, rINSTq
+ leaq (rPC, rINSTq), rPC
FETCH_INST
- jg 1f
-#if MTERP_SUSPEND
- REFRESH_IBASE
-#else
- jmp MterpCheckSuspendAndContinue
-#endif
-1:
+ jle MterpCheckSuspendAndContinue
GOTO_NEXT
@@ -1324,20 +1309,15 @@
andb $0xf, %cl # rcx <- A
GET_VREG %eax, %rcx # eax <- vA
cmpl VREG_ADDRESS(rINSTq), %eax # compare (vA, vB)
- movl $2, %eax # assume not taken
+ movl $2, rINST # assume not taken
jne 1f
- movswq 2(rPC),%rax # Get signed branch offset
+ movswq 2(rPC), rINSTq # Get signed branch offset
1:
- addq %rax, %rax # rax <- AA * 2
- leaq (rPC, %rax), rPC
+ MTERP_PROFILE_BRANCH
+ addq rINSTq, rINSTq # rax <- AA * 2
+ leaq (rPC, rINSTq), rPC
FETCH_INST
- jg 2f # AA * 2 > 0 => no suspend check
-#if MTERP_SUSPEND
- REFRESH_IBASE
-#else
- jmp MterpCheckSuspendAndContinue
-#endif
-2:
+ jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
GOTO_NEXT
@@ -1359,20 +1339,15 @@
andb $0xf, %cl # rcx <- A
GET_VREG %eax, %rcx # eax <- vA
cmpl VREG_ADDRESS(rINSTq), %eax # compare (vA, vB)
- movl $2, %eax # assume not taken
+ movl $2, rINST # assume not taken
je 1f
- movswq 2(rPC),%rax # Get signed branch offset
+ movswq 2(rPC), rINSTq # Get signed branch offset
1:
- addq %rax, %rax # rax <- AA * 2
- leaq (rPC, %rax), rPC
+ MTERP_PROFILE_BRANCH
+ addq rINSTq, rINSTq # rax <- AA * 2
+ leaq (rPC, rINSTq), rPC
FETCH_INST
- jg 2f # AA * 2 > 0 => no suspend check
-#if MTERP_SUSPEND
- REFRESH_IBASE
-#else
- jmp MterpCheckSuspendAndContinue
-#endif
-2:
+ jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
GOTO_NEXT
@@ -1394,20 +1369,15 @@
andb $0xf, %cl # rcx <- A
GET_VREG %eax, %rcx # eax <- vA
cmpl VREG_ADDRESS(rINSTq), %eax # compare (vA, vB)
- movl $2, %eax # assume not taken
+ movl $2, rINST # assume not taken
jge 1f
- movswq 2(rPC),%rax # Get signed branch offset
+ movswq 2(rPC), rINSTq # Get signed branch offset
1:
- addq %rax, %rax # rax <- AA * 2
- leaq (rPC, %rax), rPC
+ MTERP_PROFILE_BRANCH
+ addq rINSTq, rINSTq # rax <- AA * 2
+ leaq (rPC, rINSTq), rPC
FETCH_INST
- jg 2f # AA * 2 > 0 => no suspend check
-#if MTERP_SUSPEND
- REFRESH_IBASE
-#else
- jmp MterpCheckSuspendAndContinue
-#endif
-2:
+ jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
GOTO_NEXT
@@ -1429,20 +1399,15 @@
andb $0xf, %cl # rcx <- A
GET_VREG %eax, %rcx # eax <- vA
cmpl VREG_ADDRESS(rINSTq), %eax # compare (vA, vB)
- movl $2, %eax # assume not taken
+ movl $2, rINST # assume not taken
jl 1f
- movswq 2(rPC),%rax # Get signed branch offset
+ movswq 2(rPC), rINSTq # Get signed branch offset
1:
- addq %rax, %rax # rax <- AA * 2
- leaq (rPC, %rax), rPC
+ MTERP_PROFILE_BRANCH
+ addq rINSTq, rINSTq # rax <- AA * 2
+ leaq (rPC, rINSTq), rPC
FETCH_INST
- jg 2f # AA * 2 > 0 => no suspend check
-#if MTERP_SUSPEND
- REFRESH_IBASE
-#else
- jmp MterpCheckSuspendAndContinue
-#endif
-2:
+ jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
GOTO_NEXT
@@ -1464,20 +1429,15 @@
andb $0xf, %cl # rcx <- A
GET_VREG %eax, %rcx # eax <- vA
cmpl VREG_ADDRESS(rINSTq), %eax # compare (vA, vB)
- movl $2, %eax # assume not taken
+ movl $2, rINST # assume not taken
jle 1f
- movswq 2(rPC),%rax # Get signed branch offset
+ movswq 2(rPC), rINSTq # Get signed branch offset
1:
- addq %rax, %rax # rax <- AA * 2
- leaq (rPC, %rax), rPC
+ MTERP_PROFILE_BRANCH
+ addq rINSTq, rINSTq # rax <- AA * 2
+ leaq (rPC, rINSTq), rPC
FETCH_INST
- jg 2f # AA * 2 > 0 => no suspend check
-#if MTERP_SUSPEND
- REFRESH_IBASE
-#else
- jmp MterpCheckSuspendAndContinue
-#endif
-2:
+ jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
GOTO_NEXT
@@ -1499,20 +1459,15 @@
andb $0xf, %cl # rcx <- A
GET_VREG %eax, %rcx # eax <- vA
cmpl VREG_ADDRESS(rINSTq), %eax # compare (vA, vB)
- movl $2, %eax # assume not taken
+ movl $2, rINST # assume not taken
jg 1f
- movswq 2(rPC),%rax # Get signed branch offset
+ movswq 2(rPC), rINSTq # Get signed branch offset
1:
- addq %rax, %rax # rax <- AA * 2
- leaq (rPC, %rax), rPC
+ MTERP_PROFILE_BRANCH
+ addq rINSTq, rINSTq # rax <- AA * 2
+ leaq (rPC, rINSTq), rPC
FETCH_INST
- jg 2f # AA * 2 > 0 => no suspend check
-#if MTERP_SUSPEND
- REFRESH_IBASE
-#else
- jmp MterpCheckSuspendAndContinue
-#endif
-2:
+ jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
GOTO_NEXT
@@ -1530,20 +1485,15 @@
*/
/* if-cmp vAA, +BBBB */
cmpl $0, VREG_ADDRESS(rINSTq) # compare (vA, 0)
- movl $2, %eax # assume branch not taken
+ movl $2, rINST # assume branch not taken
jne 1f
- movswq 2(rPC),%rax # fetch signed displacement
+ movswq 2(rPC), rINSTq # fetch signed displacement
1:
- addq %rax, %rax # eax <- AA * 2
- leaq (rPC, %rax), rPC
+ MTERP_PROFILE_BRANCH
+ addq rINSTq, rINSTq # rINSTq <- AA * 2
+ leaq (rPC, rINSTq), rPC
FETCH_INST
- jg 2f # AA * 2 > 0 => no suspend check
-#if MTERP_SUSPEND
- REFRESH_IBASE
-#else
- jmp MterpCheckSuspendAndContinue
-#endif
-2:
+ jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
GOTO_NEXT
@@ -1561,20 +1511,15 @@
*/
/* if-cmp vAA, +BBBB */
cmpl $0, VREG_ADDRESS(rINSTq) # compare (vA, 0)
- movl $2, %eax # assume branch not taken
+ movl $2, rINST # assume branch not taken
je 1f
- movswq 2(rPC),%rax # fetch signed displacement
+ movswq 2(rPC), rINSTq # fetch signed displacement
1:
- addq %rax, %rax # eax <- AA * 2
- leaq (rPC, %rax), rPC
+ MTERP_PROFILE_BRANCH
+ addq rINSTq, rINSTq # rINSTq <- AA * 2
+ leaq (rPC, rINSTq), rPC
FETCH_INST
- jg 2f # AA * 2 > 0 => no suspend check
-#if MTERP_SUSPEND
- REFRESH_IBASE
-#else
- jmp MterpCheckSuspendAndContinue
-#endif
-2:
+ jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
GOTO_NEXT
@@ -1592,20 +1537,15 @@
*/
/* if-cmp vAA, +BBBB */
cmpl $0, VREG_ADDRESS(rINSTq) # compare (vA, 0)
- movl $2, %eax # assume branch not taken
+ movl $2, rINST # assume branch not taken
jge 1f
- movswq 2(rPC),%rax # fetch signed displacement
+ movswq 2(rPC), rINSTq # fetch signed displacement
1:
- addq %rax, %rax # eax <- AA * 2
- leaq (rPC, %rax), rPC
+ MTERP_PROFILE_BRANCH
+ addq rINSTq, rINSTq # rINSTq <- AA * 2
+ leaq (rPC, rINSTq), rPC
FETCH_INST
- jg 2f # AA * 2 > 0 => no suspend check
-#if MTERP_SUSPEND
- REFRESH_IBASE
-#else
- jmp MterpCheckSuspendAndContinue
-#endif
-2:
+ jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
GOTO_NEXT
@@ -1623,20 +1563,15 @@
*/
/* if-cmp vAA, +BBBB */
cmpl $0, VREG_ADDRESS(rINSTq) # compare (vA, 0)
- movl $2, %eax # assume branch not taken
+ movl $2, rINST # assume branch not taken
jl 1f
- movswq 2(rPC),%rax # fetch signed displacement
+ movswq 2(rPC), rINSTq # fetch signed displacement
1:
- addq %rax, %rax # eax <- AA * 2
- leaq (rPC, %rax), rPC
+ MTERP_PROFILE_BRANCH
+ addq rINSTq, rINSTq # rINSTq <- AA * 2
+ leaq (rPC, rINSTq), rPC
FETCH_INST
- jg 2f # AA * 2 > 0 => no suspend check
-#if MTERP_SUSPEND
- REFRESH_IBASE
-#else
- jmp MterpCheckSuspendAndContinue
-#endif
-2:
+ jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
GOTO_NEXT
@@ -1654,20 +1589,15 @@
*/
/* if-cmp vAA, +BBBB */
cmpl $0, VREG_ADDRESS(rINSTq) # compare (vA, 0)
- movl $2, %eax # assume branch not taken
+ movl $2, rINST # assume branch not taken
jle 1f
- movswq 2(rPC),%rax # fetch signed displacement
+ movswq 2(rPC), rINSTq # fetch signed displacement
1:
- addq %rax, %rax # eax <- AA * 2
- leaq (rPC, %rax), rPC
+ MTERP_PROFILE_BRANCH
+ addq rINSTq, rINSTq # rINSTq <- AA * 2
+ leaq (rPC, rINSTq), rPC
FETCH_INST
- jg 2f # AA * 2 > 0 => no suspend check
-#if MTERP_SUSPEND
- REFRESH_IBASE
-#else
- jmp MterpCheckSuspendAndContinue
-#endif
-2:
+ jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
GOTO_NEXT
@@ -1685,20 +1615,15 @@
*/
/* if-cmp vAA, +BBBB */
cmpl $0, VREG_ADDRESS(rINSTq) # compare (vA, 0)
- movl $2, %eax # assume branch not taken
+ movl $2, rINST # assume branch not taken
jg 1f
- movswq 2(rPC),%rax # fetch signed displacement
+ movswq 2(rPC), rINSTq # fetch signed displacement
1:
- addq %rax, %rax # eax <- AA * 2
- leaq (rPC, %rax), rPC
+ MTERP_PROFILE_BRANCH
+ addq rINSTq, rINSTq # rINSTq <- AA * 2
+ leaq (rPC, rINSTq), rPC
FETCH_INST
- jg 2f # AA * 2 > 0 => no suspend check
-#if MTERP_SUSPEND
- REFRESH_IBASE
-#else
- jmp MterpCheckSuspendAndContinue
-#endif
-2:
+ jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
GOTO_NEXT
@@ -2931,7 +2856,12 @@
call SYMBOL(MterpInvokeVirtual)
testb %al, %al
jz MterpException
- ADVANCE_PC_FETCH_AND_GOTO_NEXT 3
+ ADVANCE_PC 3
+ call SYMBOL(MterpShouldSwitchInterpreters)
+ testb %al, %al
+ jnz MterpFallback
+ FETCH_INST
+ GOTO_NEXT
/*
* Handle a virtual method call.
@@ -2961,7 +2891,12 @@
call SYMBOL(MterpInvokeSuper)
testb %al, %al
jz MterpException
- ADVANCE_PC_FETCH_AND_GOTO_NEXT 3
+ ADVANCE_PC 3
+ call SYMBOL(MterpShouldSwitchInterpreters)
+ testb %al, %al
+ jnz MterpFallback
+ FETCH_INST
+ GOTO_NEXT
/*
* Handle a "super" method call.
@@ -2991,7 +2926,12 @@
call SYMBOL(MterpInvokeDirect)
testb %al, %al
jz MterpException
- ADVANCE_PC_FETCH_AND_GOTO_NEXT 3
+ ADVANCE_PC 3
+ call SYMBOL(MterpShouldSwitchInterpreters)
+ testb %al, %al
+ jnz MterpFallback
+ FETCH_INST
+ GOTO_NEXT
/* ------------------------------ */
@@ -3014,7 +2954,12 @@
call SYMBOL(MterpInvokeStatic)
testb %al, %al
jz MterpException
- ADVANCE_PC_FETCH_AND_GOTO_NEXT 3
+ ADVANCE_PC 3
+ call SYMBOL(MterpShouldSwitchInterpreters)
+ testb %al, %al
+ jnz MterpFallback
+ FETCH_INST
+ GOTO_NEXT
@@ -3038,7 +2983,12 @@
call SYMBOL(MterpInvokeInterface)
testb %al, %al
jz MterpException
- ADVANCE_PC_FETCH_AND_GOTO_NEXT 3
+ ADVANCE_PC 3
+ call SYMBOL(MterpShouldSwitchInterpreters)
+ testb %al, %al
+ jnz MterpFallback
+ FETCH_INST
+ GOTO_NEXT
/*
* Handle an interface method call.
@@ -3080,7 +3030,12 @@
call SYMBOL(MterpInvokeVirtualRange)
testb %al, %al
jz MterpException
- ADVANCE_PC_FETCH_AND_GOTO_NEXT 3
+ ADVANCE_PC 3
+ call SYMBOL(MterpShouldSwitchInterpreters)
+ testb %al, %al
+ jnz MterpFallback
+ FETCH_INST
+ GOTO_NEXT
/* ------------------------------ */
@@ -3103,7 +3058,12 @@
call SYMBOL(MterpInvokeSuperRange)
testb %al, %al
jz MterpException
- ADVANCE_PC_FETCH_AND_GOTO_NEXT 3
+ ADVANCE_PC 3
+ call SYMBOL(MterpShouldSwitchInterpreters)
+ testb %al, %al
+ jnz MterpFallback
+ FETCH_INST
+ GOTO_NEXT
/* ------------------------------ */
@@ -3126,7 +3086,12 @@
call SYMBOL(MterpInvokeDirectRange)
testb %al, %al
jz MterpException
- ADVANCE_PC_FETCH_AND_GOTO_NEXT 3
+ ADVANCE_PC 3
+ call SYMBOL(MterpShouldSwitchInterpreters)
+ testb %al, %al
+ jnz MterpFallback
+ FETCH_INST
+ GOTO_NEXT
/* ------------------------------ */
@@ -3149,7 +3114,12 @@
call SYMBOL(MterpInvokeStaticRange)
testb %al, %al
jz MterpException
- ADVANCE_PC_FETCH_AND_GOTO_NEXT 3
+ ADVANCE_PC 3
+ call SYMBOL(MterpShouldSwitchInterpreters)
+ testb %al, %al
+ jnz MterpFallback
+ FETCH_INST
+ GOTO_NEXT
/* ------------------------------ */
@@ -3172,7 +3142,12 @@
call SYMBOL(MterpInvokeInterfaceRange)
testb %al, %al
jz MterpException
- ADVANCE_PC_FETCH_AND_GOTO_NEXT 3
+ ADVANCE_PC 3
+ call SYMBOL(MterpShouldSwitchInterpreters)
+ testb %al, %al
+ jnz MterpFallback
+ FETCH_INST
+ GOTO_NEXT
/* ------------------------------ */
@@ -5811,7 +5786,12 @@
call SYMBOL(MterpInvokeVirtualQuick)
testb %al, %al
jz MterpException
- ADVANCE_PC_FETCH_AND_GOTO_NEXT 3
+ ADVANCE_PC 3
+ call SYMBOL(MterpShouldSwitchInterpreters)
+ testb %al, %al
+ jnz MterpFallback
+ FETCH_INST
+ GOTO_NEXT
/* ------------------------------ */
@@ -5834,7 +5814,12 @@
call SYMBOL(MterpInvokeVirtualQuickRange)
testb %al, %al
jz MterpException
- ADVANCE_PC_FETCH_AND_GOTO_NEXT 3
+ ADVANCE_PC 3
+ call SYMBOL(MterpShouldSwitchInterpreters)
+ testb %al, %al
+ jnz MterpFallback
+ FETCH_INST
+ GOTO_NEXT
/* ------------------------------ */
@@ -11805,7 +11790,6 @@
* has not yet been thrown. Just bail out to the reference interpreter to deal with it.
* TUNING: for consistency, we may want to just go ahead and handle these here.
*/
-#define MTERP_LOGGING 0
common_errDivideByZero:
EXPORT_PC
#if MTERP_LOGGING
@@ -11891,13 +11875,17 @@
call SYMBOL(MterpHandleException)
testb %al, %al
jz MterpExceptionReturn
- REFRESH_IBASE
movq OFF_FP_CODE_ITEM(rFP), %rax
mov OFF_FP_DEX_PC(rFP), %ecx
leaq CODEITEM_INSNS_OFFSET(%rax), rPC
leaq (rPC, %rcx, 2), rPC
movq rPC, OFF_FP_DEX_PC_PTR(rFP)
+ /* Do we need to switch interpreters? */
+ call SYMBOL(MterpShouldSwitchInterpreters)
+ testb %al, %al
+ jnz MterpFallback
/* resume execution at catch block */
+ REFRESH_IBASE
FETCH_INST
GOTO_NEXT
/* NOTE: no fallthrough */
@@ -11917,6 +11905,19 @@
GOTO_NEXT
/*
+ * On-stack replacement has happened, and now we've returned from the compiled method.
+ */
+MterpOnStackReplacement:
+#if MTERP_LOGGING
+ movq rSELF, OUT_ARG0
+ leaq OFF_FP_SHADOWFRAME(rFP), OUT_ARG1
+ movl rINST, OUT_32_ARG2
+ call SYMBOL(MterpLogOSR)
+#endif
+ movl $1, %eax
+ jmp MterpDone
+
+/*
* Bail out to reference interpreter.
*/
MterpFallback:
diff --git a/runtime/interpreter/mterp/x86_64/bincmp.S b/runtime/interpreter/mterp/x86_64/bincmp.S
index 5e4225f..a16050b 100644
--- a/runtime/interpreter/mterp/x86_64/bincmp.S
+++ b/runtime/interpreter/mterp/x86_64/bincmp.S
@@ -11,18 +11,13 @@
andb $$0xf, %cl # rcx <- A
GET_VREG %eax, %rcx # eax <- vA
cmpl VREG_ADDRESS(rINSTq), %eax # compare (vA, vB)
- movl $$2, %eax # assume not taken
+ movl $$2, rINST # assume not taken
j${revcmp} 1f
- movswq 2(rPC),%rax # Get signed branch offset
+ movswq 2(rPC), rINSTq # Get signed branch offset
1:
- addq %rax, %rax # rax <- AA * 2
- leaq (rPC, %rax), rPC
+ MTERP_PROFILE_BRANCH
+ addq rINSTq, rINSTq # rax <- AA * 2
+ leaq (rPC, rINSTq), rPC
FETCH_INST
- jg 2f # AA * 2 > 0 => no suspend check
-#if MTERP_SUSPEND
- REFRESH_IBASE
-#else
- jmp MterpCheckSuspendAndContinue
-#endif
-2:
+ jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
GOTO_NEXT
diff --git a/runtime/interpreter/mterp/x86_64/footer.S b/runtime/interpreter/mterp/x86_64/footer.S
index cb60c01..573256b 100644
--- a/runtime/interpreter/mterp/x86_64/footer.S
+++ b/runtime/interpreter/mterp/x86_64/footer.S
@@ -12,7 +12,6 @@
* has not yet been thrown. Just bail out to the reference interpreter to deal with it.
* TUNING: for consistency, we may want to just go ahead and handle these here.
*/
-#define MTERP_LOGGING 0
common_errDivideByZero:
EXPORT_PC
#if MTERP_LOGGING
@@ -98,13 +97,17 @@
call SYMBOL(MterpHandleException)
testb %al, %al
jz MterpExceptionReturn
- REFRESH_IBASE
movq OFF_FP_CODE_ITEM(rFP), %rax
mov OFF_FP_DEX_PC(rFP), %ecx
leaq CODEITEM_INSNS_OFFSET(%rax), rPC
leaq (rPC, %rcx, 2), rPC
movq rPC, OFF_FP_DEX_PC_PTR(rFP)
+ /* Do we need to switch interpreters? */
+ call SYMBOL(MterpShouldSwitchInterpreters)
+ testb %al, %al
+ jnz MterpFallback
/* resume execution at catch block */
+ REFRESH_IBASE
FETCH_INST
GOTO_NEXT
/* NOTE: no fallthrough */
@@ -124,6 +127,19 @@
GOTO_NEXT
/*
+ * On-stack replacement has happened, and now we've returned from the compiled method.
+ */
+MterpOnStackReplacement:
+#if MTERP_LOGGING
+ movq rSELF, OUT_ARG0
+ leaq OFF_FP_SHADOWFRAME(rFP), OUT_ARG1
+ movl rINST, OUT_32_ARG2
+ call SYMBOL(MterpLogOSR)
+#endif
+ movl $$1, %eax
+ jmp MterpDone
+
+/*
* Bail out to reference interpreter.
*/
MterpFallback:
diff --git a/runtime/interpreter/mterp/x86_64/header.S b/runtime/interpreter/mterp/x86_64/header.S
index dfc7b53..eb84ea1 100644
--- a/runtime/interpreter/mterp/x86_64/header.S
+++ b/runtime/interpreter/mterp/x86_64/header.S
@@ -162,13 +162,23 @@
#define OFF_FP_CODE_ITEM OFF_FP(SHADOWFRAME_CODE_ITEM_OFFSET)
#define OFF_FP_SHADOWFRAME (-SHADOWFRAME_VREGS_OFFSET)
+#define MTERP_PROFILE_BRANCHES 1
+#define MTERP_LOGGING 0
+
/*
- *
- * The reference interpreter performs explicit suspect checks, which is somewhat wasteful.
- * Dalvik's interpreter folded suspend checks into the jump table mechanism, and eventually
- * mterp should do so as well.
+ * Profile branch. rINST should contain the offset. %eax is scratch.
*/
-#define MTERP_SUSPEND 0
+.macro MTERP_PROFILE_BRANCH
+#ifdef MTERP_PROFILE_BRANCHES
+ EXPORT_PC
+ movq rSELF, OUT_ARG0
+ leaq OFF_FP_SHADOWFRAME(rFP), OUT_ARG1
+ movl rINST, OUT_32_ARG2
+ call SYMBOL(MterpProfileBranch)
+ testb %al, %al
+ jnz MterpOnStackReplacement
+#endif
+.endm
/*
* "export" the PC to dex_pc field in the shadow frame, f/b/o future exception objects. Must
diff --git a/runtime/interpreter/mterp/x86_64/invoke.S b/runtime/interpreter/mterp/x86_64/invoke.S
index 86eccdb..f7e6155 100644
--- a/runtime/interpreter/mterp/x86_64/invoke.S
+++ b/runtime/interpreter/mterp/x86_64/invoke.S
@@ -14,4 +14,9 @@
call SYMBOL($helper)
testb %al, %al
jz MterpException
- ADVANCE_PC_FETCH_AND_GOTO_NEXT 3
+ ADVANCE_PC 3
+ call SYMBOL(MterpShouldSwitchInterpreters)
+ testb %al, %al
+ jnz MterpFallback
+ FETCH_INST
+ GOTO_NEXT
diff --git a/runtime/interpreter/mterp/x86_64/op_goto.S b/runtime/interpreter/mterp/x86_64/op_goto.S
index 05a2dda..c4fc976 100644
--- a/runtime/interpreter/mterp/x86_64/op_goto.S
+++ b/runtime/interpreter/mterp/x86_64/op_goto.S
@@ -5,15 +5,10 @@
* double to get a byte offset.
*/
/* goto +AA */
- movsbq rINSTbl, %rax # rax <- ssssssAA
- addq %rax, %rax # rax <- AA * 2
- leaq (rPC, %rax), rPC
+ movsbq rINSTbl, rINSTq # rINSTq <- ssssssAA
+ MTERP_PROFILE_BRANCH
+ addq rINSTq, rINSTq # rINSTq <- AA * 2
+ leaq (rPC, rINSTq), rPC
FETCH_INST
- jg 1f # AA * 2 > 0 => no suspend check
-#if MTERP_SUSPEND
- REFRESH_IBASE
-#else
- jmp MterpCheckSuspendAndContinue
-#endif
-1:
+ jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
GOTO_NEXT
diff --git a/runtime/interpreter/mterp/x86_64/op_goto_16.S b/runtime/interpreter/mterp/x86_64/op_goto_16.S
index 029749c..8cb9a5c 100644
--- a/runtime/interpreter/mterp/x86_64/op_goto_16.S
+++ b/runtime/interpreter/mterp/x86_64/op_goto_16.S
@@ -5,15 +5,10 @@
* double to get a byte offset.
*/
/* goto/16 +AAAA */
- movswq 2(rPC), %rax # rax <- ssssAAAA
- addq %rax, %rax # rax <- AA * 2
- leaq (rPC, %rax), rPC
+ movswq 2(rPC), rINSTq # rINSTq <- ssssAAAA
+ MTERP_PROFILE_BRANCH
+ addq rINSTq, rINSTq # rINSTq <- AA * 2
+ leaq (rPC, rINSTq), rPC
FETCH_INST
- jg 1f # AA * 2 > 0 => no suspend check
-#if MTERP_SUSPEND
- REFRESH_IBASE
-#else
- jmp MterpCheckSuspendAndContinue
-#endif
-1:
+ jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
GOTO_NEXT
diff --git a/runtime/interpreter/mterp/x86_64/op_goto_32.S b/runtime/interpreter/mterp/x86_64/op_goto_32.S
index 2823310..4ecdacd 100644
--- a/runtime/interpreter/mterp/x86_64/op_goto_32.S
+++ b/runtime/interpreter/mterp/x86_64/op_goto_32.S
@@ -8,15 +8,10 @@
* to convert from Dalvik offset to byte offset.
*/
/* goto/32 +AAAAAAAA */
- movslq 2(rPC), %rax # rax <- AAAAAAAA
- addq %rax, %rax # rax <- AA * 2
- leaq (rPC, %rax), rPC
+ movslq 2(rPC), rINSTq # rINSTq <- AAAAAAAA
+ MTERP_PROFILE_BRANCH
+ addq rINSTq, rINSTq # rINSTq <- AA * 2
+ leaq (rPC, rINSTq), rPC
FETCH_INST
- jg 1f # AA * 2 > 0 => no suspend check
-#if MTERP_SUSPEND
- REFRESH_IBASE
-#else
- jmp MterpCheckSuspendAndContinue
-#endif
-1:
+ jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
GOTO_NEXT
diff --git a/runtime/interpreter/mterp/x86_64/op_packed_switch.S b/runtime/interpreter/mterp/x86_64/op_packed_switch.S
index 0400ca4..cb0acb7 100644
--- a/runtime/interpreter/mterp/x86_64/op_packed_switch.S
+++ b/runtime/interpreter/mterp/x86_64/op_packed_switch.S
@@ -13,15 +13,10 @@
leaq (rPC,OUT_ARG0,2), OUT_ARG0 # rcx <- PC + BBBBbbbb*2
GET_VREG OUT_32_ARG1, rINSTq # eax <- vAA
call SYMBOL($func)
- addl %eax, %eax
- movslq %eax, %rax
- leaq (rPC, %rax), rPC
+ movslq %eax, rINSTq
+ MTERP_PROFILE_BRANCH
+ addq rINSTq, rINSTq
+ leaq (rPC, rINSTq), rPC
FETCH_INST
- jg 1f
-#if MTERP_SUSPEND
- REFRESH_IBASE
-#else
- jmp MterpCheckSuspendAndContinue
-#endif
-1:
+ jle MterpCheckSuspendAndContinue
GOTO_NEXT
diff --git a/runtime/interpreter/mterp/x86_64/zcmp.S b/runtime/interpreter/mterp/x86_64/zcmp.S
index e503ec1..0051407 100644
--- a/runtime/interpreter/mterp/x86_64/zcmp.S
+++ b/runtime/interpreter/mterp/x86_64/zcmp.S
@@ -7,18 +7,13 @@
*/
/* if-cmp vAA, +BBBB */
cmpl $$0, VREG_ADDRESS(rINSTq) # compare (vA, 0)
- movl $$2, %eax # assume branch not taken
+ movl $$2, rINST # assume branch not taken
j${revcmp} 1f
- movswq 2(rPC),%rax # fetch signed displacement
+ movswq 2(rPC), rINSTq # fetch signed displacement
1:
- addq %rax, %rax # eax <- AA * 2
- leaq (rPC, %rax), rPC
+ MTERP_PROFILE_BRANCH
+ addq rINSTq, rINSTq # rINSTq <- AA * 2
+ leaq (rPC, rINSTq), rPC
FETCH_INST
- jg 2f # AA * 2 > 0 => no suspend check
-#if MTERP_SUSPEND
- REFRESH_IBASE
-#else
- jmp MterpCheckSuspendAndContinue
-#endif
-2:
+ jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
GOTO_NEXT