Wu Zhangjin | 538f195 | 2009-11-20 20:34:32 +0800 | [diff] [blame] | 1 | /* |
| 2 | * Code for replacing ftrace calls with jumps. |
| 3 | * |
| 4 | * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> |
| 5 | * Copyright (C) 2009 DSLab, Lanzhou University, China |
Wu Zhangjin | f7a904d | 2010-01-04 17:16:51 +0800 | [diff] [blame] | 6 | * Author: Wu Zhangjin <wuzhangjin@gmail.com> |
Wu Zhangjin | 538f195 | 2009-11-20 20:34:32 +0800 | [diff] [blame] | 7 | * |
| 8 | * Thanks goes to Steven Rostedt for writing the original x86 version. |
| 9 | */ |
| 10 | |
| 11 | #include <linux/uaccess.h> |
| 12 | #include <linux/init.h> |
| 13 | #include <linux/ftrace.h> |
| 14 | |
| 15 | #include <asm/cacheflush.h> |
Wu Zhangjin | 29c5d34 | 2009-11-20 20:34:34 +0800 | [diff] [blame] | 16 | #include <asm/asm.h> |
| 17 | #include <asm/asm-offsets.h> |
Wu Zhangjin | 538f195 | 2009-11-20 20:34:32 +0800 | [diff] [blame] | 18 | |
| 19 | #ifdef CONFIG_DYNAMIC_FTRACE |
| 20 | |
| 21 | #define JAL 0x0c000000 /* jump & link: ip --> ra, jump to target */ |
| 22 | #define ADDR_MASK 0x03ffffff /* op_code|addr : 31...26|25 ....0 */ |
| 23 | #define jump_insn_encode(op_code, addr) \ |
| 24 | ((unsigned int)((op_code) | (((addr) >> 2) & ADDR_MASK))) |
| 25 | |
| 26 | static unsigned int ftrace_nop = 0x00000000; |
| 27 | |
| 28 | static int ftrace_modify_code(unsigned long ip, unsigned int new_code) |
| 29 | { |
Wu Zhangjin | 046199c | 2009-11-20 20:34:36 +0800 | [diff] [blame] | 30 | int faulted; |
| 31 | |
| 32 | /* *(unsigned int *)ip = new_code; */ |
| 33 | safe_store_code(new_code, ip, faulted); |
| 34 | |
| 35 | if (unlikely(faulted)) |
| 36 | return -EFAULT; |
Wu Zhangjin | 538f195 | 2009-11-20 20:34:32 +0800 | [diff] [blame] | 37 | |
| 38 | flush_icache_range(ip, ip + 8); |
| 39 | |
| 40 | return 0; |
| 41 | } |
| 42 | |
| 43 | static int lui_v1; |
| 44 | static int jal_mcount; |
| 45 | |
| 46 | int ftrace_make_nop(struct module *mod, |
| 47 | struct dyn_ftrace *rec, unsigned long addr) |
| 48 | { |
| 49 | unsigned int new; |
Wu Zhangjin | 046199c | 2009-11-20 20:34:36 +0800 | [diff] [blame] | 50 | int faulted; |
Wu Zhangjin | 538f195 | 2009-11-20 20:34:32 +0800 | [diff] [blame] | 51 | unsigned long ip = rec->ip; |
| 52 | |
| 53 | /* We have compiled module with -mlong-calls, but compiled the kernel |
| 54 | * without it, we need to cope with them respectively. */ |
| 55 | if (ip & 0x40000000) { |
| 56 | /* record it for ftrace_make_call */ |
Wu Zhangjin | 046199c | 2009-11-20 20:34:36 +0800 | [diff] [blame] | 57 | if (lui_v1 == 0) { |
| 58 | /* lui_v1 = *(unsigned int *)ip; */ |
| 59 | safe_load_code(lui_v1, ip, faulted); |
| 60 | |
| 61 | if (unlikely(faulted)) |
| 62 | return -EFAULT; |
| 63 | } |
Wu Zhangjin | 538f195 | 2009-11-20 20:34:32 +0800 | [diff] [blame] | 64 | |
| 65 | /* lui v1, hi_16bit_of_mcount --> b 1f (0x10000004) |
| 66 | * addiu v1, v1, low_16bit_of_mcount |
| 67 | * move at, ra |
| 68 | * jalr v1 |
| 69 | * nop |
| 70 | * 1f: (ip + 12) |
| 71 | */ |
| 72 | new = 0x10000004; |
| 73 | } else { |
| 74 | /* record/calculate it for ftrace_make_call */ |
| 75 | if (jal_mcount == 0) { |
| 76 | /* We can record it directly like this: |
| 77 | * jal_mcount = *(unsigned int *)ip; |
| 78 | * Herein, jump over the first two nop instructions */ |
| 79 | jal_mcount = jump_insn_encode(JAL, (MCOUNT_ADDR + 8)); |
| 80 | } |
| 81 | |
| 82 | /* move at, ra |
| 83 | * jalr v1 --> nop |
| 84 | */ |
| 85 | new = ftrace_nop; |
| 86 | } |
| 87 | return ftrace_modify_code(ip, new); |
| 88 | } |
| 89 | |
| 90 | static int modified; /* initialized as 0 by default */ |
| 91 | |
| 92 | int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) |
| 93 | { |
| 94 | unsigned int new; |
| 95 | unsigned long ip = rec->ip; |
| 96 | |
| 97 | /* We just need to remove the "b ftrace_stub" at the fist time! */ |
| 98 | if (modified == 0) { |
| 99 | modified = 1; |
| 100 | ftrace_modify_code(addr, ftrace_nop); |
| 101 | } |
| 102 | /* ip, module: 0xc0000000, kernel: 0x80000000 */ |
| 103 | new = (ip & 0x40000000) ? lui_v1 : jal_mcount; |
| 104 | |
| 105 | return ftrace_modify_code(ip, new); |
| 106 | } |
| 107 | |
| 108 | #define FTRACE_CALL_IP ((unsigned long)(&ftrace_call)) |
| 109 | |
| 110 | int ftrace_update_ftrace_func(ftrace_func_t func) |
| 111 | { |
| 112 | unsigned int new; |
| 113 | |
| 114 | new = jump_insn_encode(JAL, (unsigned long)func); |
| 115 | |
| 116 | return ftrace_modify_code(FTRACE_CALL_IP, new); |
| 117 | } |
| 118 | |
| 119 | int __init ftrace_dyn_arch_init(void *data) |
| 120 | { |
| 121 | /* The return code is retured via data */ |
| 122 | *(unsigned long *)data = 0; |
| 123 | |
| 124 | return 0; |
| 125 | } |
| 126 | #endif /* CONFIG_DYNAMIC_FTRACE */ |
Wu Zhangjin | 29c5d34 | 2009-11-20 20:34:34 +0800 | [diff] [blame] | 127 | |
| 128 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
| 129 | |
Wu Zhangjin | e17ff5f | 2009-11-20 20:34:35 +0800 | [diff] [blame] | 130 | #ifdef CONFIG_DYNAMIC_FTRACE |
| 131 | |
| 132 | extern void ftrace_graph_call(void); |
| 133 | #define JMP 0x08000000 /* jump to target directly */ |
| 134 | #define CALL_FTRACE_GRAPH_CALLER \ |
| 135 | jump_insn_encode(JMP, (unsigned long)(&ftrace_graph_caller)) |
| 136 | #define FTRACE_GRAPH_CALL_IP ((unsigned long)(&ftrace_graph_call)) |
| 137 | |
| 138 | int ftrace_enable_ftrace_graph_caller(void) |
| 139 | { |
| 140 | return ftrace_modify_code(FTRACE_GRAPH_CALL_IP, |
| 141 | CALL_FTRACE_GRAPH_CALLER); |
| 142 | } |
| 143 | |
| 144 | int ftrace_disable_ftrace_graph_caller(void) |
| 145 | { |
| 146 | return ftrace_modify_code(FTRACE_GRAPH_CALL_IP, ftrace_nop); |
| 147 | } |
| 148 | |
| 149 | #endif /* !CONFIG_DYNAMIC_FTRACE */ |
| 150 | |
Wu Zhangjin | 7326c4e | 2009-11-20 20:34:38 +0800 | [diff] [blame] | 151 | #ifndef KBUILD_MCOUNT_RA_ADDRESS |
Wu Zhangjin | 29c5d34 | 2009-11-20 20:34:34 +0800 | [diff] [blame] | 152 | #define S_RA_SP (0xafbf << 16) /* s{d,w} ra, offset(sp) */ |
| 153 | #define S_R_SP (0xafb0 << 16) /* s{d,w} R, offset(sp) */ |
| 154 | #define OFFSET_MASK 0xffff /* stack offset range: 0 ~ PT_SIZE */ |
| 155 | |
| 156 | unsigned long ftrace_get_parent_addr(unsigned long self_addr, |
| 157 | unsigned long parent, |
| 158 | unsigned long parent_addr, |
| 159 | unsigned long fp) |
| 160 | { |
| 161 | unsigned long sp, ip, ra; |
| 162 | unsigned int code; |
Wu Zhangjin | 046199c | 2009-11-20 20:34:36 +0800 | [diff] [blame] | 163 | int faulted; |
Wu Zhangjin | 29c5d34 | 2009-11-20 20:34:34 +0800 | [diff] [blame] | 164 | |
| 165 | /* in module or kernel? */ |
| 166 | if (self_addr & 0x40000000) { |
| 167 | /* module: move to the instruction "lui v1, HI_16BIT_OF_MCOUNT" */ |
| 168 | ip = self_addr - 20; |
| 169 | } else { |
| 170 | /* kernel: move to the instruction "move ra, at" */ |
| 171 | ip = self_addr - 12; |
| 172 | } |
| 173 | |
| 174 | /* search the text until finding the non-store instruction or "s{d,w} |
| 175 | * ra, offset(sp)" instruction */ |
| 176 | do { |
| 177 | ip -= 4; |
| 178 | |
Wu Zhangjin | 046199c | 2009-11-20 20:34:36 +0800 | [diff] [blame] | 179 | /* get the code at "ip": code = *(unsigned int *)ip; */ |
| 180 | safe_load_code(code, ip, faulted); |
| 181 | |
| 182 | if (unlikely(faulted)) |
| 183 | return 0; |
Wu Zhangjin | 29c5d34 | 2009-11-20 20:34:34 +0800 | [diff] [blame] | 184 | |
| 185 | /* If we hit the non-store instruction before finding where the |
| 186 | * ra is stored, then this is a leaf function and it does not |
| 187 | * store the ra on the stack. */ |
| 188 | if ((code & S_R_SP) != S_R_SP) |
| 189 | return parent_addr; |
| 190 | |
| 191 | } while (((code & S_RA_SP) != S_RA_SP)); |
| 192 | |
| 193 | sp = fp + (code & OFFSET_MASK); |
Wu Zhangjin | 046199c | 2009-11-20 20:34:36 +0800 | [diff] [blame] | 194 | |
| 195 | /* ra = *(unsigned long *)sp; */ |
| 196 | safe_load_stack(ra, sp, faulted); |
| 197 | if (unlikely(faulted)) |
| 198 | return 0; |
Wu Zhangjin | 29c5d34 | 2009-11-20 20:34:34 +0800 | [diff] [blame] | 199 | |
| 200 | if (ra == parent) |
| 201 | return sp; |
Wu Zhangjin | 29c5d34 | 2009-11-20 20:34:34 +0800 | [diff] [blame] | 202 | return 0; |
| 203 | } |
| 204 | |
Wu Zhangjin | 7326c4e | 2009-11-20 20:34:38 +0800 | [diff] [blame] | 205 | #endif |
| 206 | |
Wu Zhangjin | 29c5d34 | 2009-11-20 20:34:34 +0800 | [diff] [blame] | 207 | /* |
| 208 | * Hook the return address and push it in the stack of return addrs |
| 209 | * in current thread info. |
| 210 | */ |
| 211 | void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, |
| 212 | unsigned long fp) |
| 213 | { |
| 214 | unsigned long old; |
| 215 | struct ftrace_graph_ent trace; |
| 216 | unsigned long return_hooker = (unsigned long) |
| 217 | &return_to_handler; |
Wu Zhangjin | 046199c | 2009-11-20 20:34:36 +0800 | [diff] [blame] | 218 | int faulted; |
Wu Zhangjin | 29c5d34 | 2009-11-20 20:34:34 +0800 | [diff] [blame] | 219 | |
| 220 | if (unlikely(atomic_read(¤t->tracing_graph_pause))) |
| 221 | return; |
| 222 | |
| 223 | /* "parent" is the stack address saved the return address of the caller |
Wu Zhangjin | 7326c4e | 2009-11-20 20:34:38 +0800 | [diff] [blame] | 224 | * of _mcount. |
| 225 | * |
| 226 | * if the gcc < 4.5, a leaf function does not save the return address |
| 227 | * in the stack address, so, we "emulate" one in _mcount's stack space, |
| 228 | * and hijack it directly, but for a non-leaf function, it save the |
| 229 | * return address to the its own stack space, we can not hijack it |
| 230 | * directly, but need to find the real stack address, |
Wu Zhangjin | 29c5d34 | 2009-11-20 20:34:34 +0800 | [diff] [blame] | 231 | * ftrace_get_parent_addr() does it! |
Wu Zhangjin | 7326c4e | 2009-11-20 20:34:38 +0800 | [diff] [blame] | 232 | * |
| 233 | * if gcc>= 4.5, with the new -mmcount-ra-address option, for a |
| 234 | * non-leaf function, the location of the return address will be saved |
| 235 | * to $12 for us, and for a leaf function, only put a zero into $12. we |
| 236 | * do it in ftrace_graph_caller of mcount.S. |
Wu Zhangjin | 29c5d34 | 2009-11-20 20:34:34 +0800 | [diff] [blame] | 237 | */ |
| 238 | |
Wu Zhangjin | 046199c | 2009-11-20 20:34:36 +0800 | [diff] [blame] | 239 | /* old = *parent; */ |
| 240 | safe_load_stack(old, parent, faulted); |
| 241 | if (unlikely(faulted)) |
| 242 | goto out; |
Wu Zhangjin | 7326c4e | 2009-11-20 20:34:38 +0800 | [diff] [blame] | 243 | #ifndef KBUILD_MCOUNT_RA_ADDRESS |
Wu Zhangjin | 29c5d34 | 2009-11-20 20:34:34 +0800 | [diff] [blame] | 244 | parent = (unsigned long *)ftrace_get_parent_addr(self_addr, old, |
| 245 | (unsigned long)parent, |
| 246 | fp); |
Wu Zhangjin | 29c5d34 | 2009-11-20 20:34:34 +0800 | [diff] [blame] | 247 | /* If fails when getting the stack address of the non-leaf function's |
| 248 | * ra, stop function graph tracer and return */ |
Wu Zhangjin | 046199c | 2009-11-20 20:34:36 +0800 | [diff] [blame] | 249 | if (parent == 0) |
| 250 | goto out; |
Wu Zhangjin | 7326c4e | 2009-11-20 20:34:38 +0800 | [diff] [blame] | 251 | #endif |
Wu Zhangjin | 046199c | 2009-11-20 20:34:36 +0800 | [diff] [blame] | 252 | /* *parent = return_hooker; */ |
| 253 | safe_store_stack(return_hooker, parent, faulted); |
| 254 | if (unlikely(faulted)) |
| 255 | goto out; |
Wu Zhangjin | 29c5d34 | 2009-11-20 20:34:34 +0800 | [diff] [blame] | 256 | |
| 257 | if (ftrace_push_return_trace(old, self_addr, &trace.depth, fp) == |
| 258 | -EBUSY) { |
| 259 | *parent = old; |
| 260 | return; |
| 261 | } |
| 262 | |
| 263 | trace.func = self_addr; |
| 264 | |
| 265 | /* Only trace if the calling function expects to */ |
| 266 | if (!ftrace_graph_entry(&trace)) { |
| 267 | current->curr_ret_stack--; |
| 268 | *parent = old; |
| 269 | } |
Wu Zhangjin | 046199c | 2009-11-20 20:34:36 +0800 | [diff] [blame] | 270 | return; |
| 271 | out: |
| 272 | ftrace_graph_stop(); |
| 273 | WARN_ON(1); |
Wu Zhangjin | 29c5d34 | 2009-11-20 20:34:34 +0800 | [diff] [blame] | 274 | } |
| 275 | #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ |