blob: ce1f3e44c24fabf07408fd3e0ed70919a3828cb3 [file] [log] [blame]
Steven Rostedt4e491d12008-05-14 23:49:44 -04001/*
2 * Code for replacing ftrace calls with jumps.
3 *
4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5 *
6 * Thanks goes out to P.A. Semi, Inc for supplying me with a PPC64 box.
7 *
Steven Rostedt6794c782009-02-09 21:10:27 -08008 * Added function graph tracer code, taken from x86 that was written
9 * by Frederic Weisbecker, and ported to PPC by Steven Rostedt.
10 *
Steven Rostedt4e491d12008-05-14 23:49:44 -040011 */
12
13#include <linux/spinlock.h>
14#include <linux/hardirq.h>
Steven Rostedte4486fe2008-11-14 16:21:20 -080015#include <linux/uaccess.h>
Steven Rostedtf48cb8b2008-11-14 20:47:03 -080016#include <linux/module.h>
Steven Rostedt4e491d12008-05-14 23:49:44 -040017#include <linux/ftrace.h>
18#include <linux/percpu.h>
19#include <linux/init.h>
20#include <linux/list.h>
21
22#include <asm/cacheflush.h>
Steven Rostedtf48cb8b2008-11-14 20:47:03 -080023#include <asm/code-patching.h>
Abhishek Sagar395a59d2008-06-21 23:47:27 +053024#include <asm/ftrace.h>
Steven Rostedt4e491d12008-05-14 23:49:44 -040025
Steven Rostedt4e491d12008-05-14 23:49:44 -040026
Steven Rostedt6794c782009-02-09 21:10:27 -080027#ifdef CONFIG_DYNAMIC_FTRACE
Steven Rostedtb54dcfe2009-02-13 06:31:39 -080028static unsigned int
Steven Rostedt46542882009-02-10 22:19:54 -080029ftrace_call_replace(unsigned long ip, unsigned long addr, int link)
Steven Rostedt4e491d12008-05-14 23:49:44 -040030{
Steven Rostedtb54dcfe2009-02-13 06:31:39 -080031 unsigned int op;
Steven Rostedt4e491d12008-05-14 23:49:44 -040032
Michael Ellerman4a9e3f82009-05-28 19:33:34 +000033 addr = ppc_function_entry((void *)addr);
Steven Rostedt4e491d12008-05-14 23:49:44 -040034
Steven Rostedt46542882009-02-10 22:19:54 -080035 /* if (link) set op to 'bl' else 'b' */
Steven Rostedtbb9b9032009-02-13 06:45:27 -080036 op = create_branch((unsigned int *)ip, addr, link ? 1 : 0);
Steven Rostedt4e491d12008-05-14 23:49:44 -040037
Steven Rostedtb54dcfe2009-02-13 06:31:39 -080038 return op;
Steven Rostedt4e491d12008-05-14 23:49:44 -040039}
40
Steven Rostedt8fd6e5a2008-11-14 16:21:19 -080041static int
Steven Rostedtb54dcfe2009-02-13 06:31:39 -080042ftrace_modify_code(unsigned long ip, unsigned int old, unsigned int new)
Steven Rostedt4e491d12008-05-14 23:49:44 -040043{
Steven Rostedtb54dcfe2009-02-13 06:31:39 -080044 unsigned int replaced;
Steven Rostedt4e491d12008-05-14 23:49:44 -040045
Steven Rostedt4e491d12008-05-14 23:49:44 -040046 /*
47 * Note: Due to modules and __init, code can
48 * disappear and change, we need to protect against faulting
Steven Rostedte4486fe2008-11-14 16:21:20 -080049 * as well as code changing. We do this by using the
50 * probe_kernel_* functions.
Steven Rostedt4e491d12008-05-14 23:49:44 -040051 *
52 * No real locking needed, this code is run through
Steven Rostedte4486fe2008-11-14 16:21:20 -080053 * kstop_machine, or before SMP starts.
Steven Rostedt4e491d12008-05-14 23:49:44 -040054 */
Steven Rostedt4e491d12008-05-14 23:49:44 -040055
Steven Rostedte4486fe2008-11-14 16:21:20 -080056 /* read the text we want to modify */
Steven Rostedtb54dcfe2009-02-13 06:31:39 -080057 if (probe_kernel_read(&replaced, (void *)ip, MCOUNT_INSN_SIZE))
Steven Rostedte4486fe2008-11-14 16:21:20 -080058 return -EFAULT;
Steven Rostedt4e491d12008-05-14 23:49:44 -040059
Steven Rostedte4486fe2008-11-14 16:21:20 -080060 /* Make sure it is what we expect it to be */
Steven Rostedtb54dcfe2009-02-13 06:31:39 -080061 if (replaced != old)
Steven Rostedte4486fe2008-11-14 16:21:20 -080062 return -EINVAL;
Steven Rostedt4e491d12008-05-14 23:49:44 -040063
Steven Rostedte4486fe2008-11-14 16:21:20 -080064 /* replace the text with the new text */
Steven Rostedtb54dcfe2009-02-13 06:31:39 -080065 if (probe_kernel_write((void *)ip, &new, MCOUNT_INSN_SIZE))
Steven Rostedte4486fe2008-11-14 16:21:20 -080066 return -EPERM;
67
68 flush_icache_range(ip, ip + 8);
69
70 return 0;
Steven Rostedt4e491d12008-05-14 23:49:44 -040071}
72
Steven Rostedtf48cb8b2008-11-14 20:47:03 -080073/*
74 * Helper functions that are the same for both PPC64 and PPC32.
75 */
Steven Rostedt8fd6e5a2008-11-14 16:21:19 -080076static int test_24bit_addr(unsigned long ip, unsigned long addr)
77{
Steven Rostedt8fd6e5a2008-11-14 16:21:19 -080078
Steven Rostedt0029ff82008-11-25 14:06:19 -080079 /* use the create_branch to verify that this offset can be branched */
80 return create_branch((unsigned int *)ip, addr, 0);
Steven Rostedt8fd6e5a2008-11-14 16:21:19 -080081}
82
Steven Rostedt17be5b32009-02-05 21:33:09 -080083#ifdef CONFIG_MODULES
84
Steven Rostedtf48cb8b2008-11-14 20:47:03 -080085static int is_bl_op(unsigned int op)
Steven Rostedt8fd6e5a2008-11-14 16:21:19 -080086{
Steven Rostedtf48cb8b2008-11-14 20:47:03 -080087 return (op & 0xfc000003) == 0x48000001;
88}
89
Steven Rostedtf48cb8b2008-11-14 20:47:03 -080090static unsigned long find_bl_target(unsigned long ip, unsigned int op)
91{
92 static int offset;
93
94 offset = (op & 0x03fffffc);
95 /* make it signed */
96 if (offset & 0x02000000)
97 offset |= 0xfe000000;
98
99 return ip + (long)offset;
100}
101
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800102#ifdef CONFIG_PPC64
103static int
104__ftrace_make_nop(struct module *mod,
105 struct dyn_ftrace *rec, unsigned long addr)
106{
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800107 unsigned int op;
108 unsigned int jmp[5];
109 unsigned long ptr;
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800110 unsigned long ip = rec->ip;
111 unsigned long tramp;
112 int offset;
113
114 /* read where this goes */
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800115 if (probe_kernel_read(&op, (void *)ip, sizeof(int)))
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800116 return -EFAULT;
117
118 /* Make sure that that this is still a 24bit jump */
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800119 if (!is_bl_op(op)) {
120 printk(KERN_ERR "Not expected bl: opcode is %x\n", op);
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800121 return -EINVAL;
122 }
123
124 /* lets find where the pointer goes */
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800125 tramp = find_bl_target(ip, op);
Steven Rostedt8fd6e5a2008-11-14 16:21:19 -0800126
127 /*
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800128 * On PPC64 the trampoline looks like:
129 * 0x3d, 0x82, 0x00, 0x00, addis r12,r2, <high>
130 * 0x39, 0x8c, 0x00, 0x00, addi r12,r12, <low>
131 * Where the bytes 2,3,6 and 7 make up the 32bit offset
132 * to the TOC that holds the pointer.
133 * to jump to.
134 * 0xf8, 0x41, 0x00, 0x28, std r2,40(r1)
135 * 0xe9, 0x6c, 0x00, 0x20, ld r11,32(r12)
136 * The actually address is 32 bytes from the offset
137 * into the TOC.
138 * 0xe8, 0x4c, 0x00, 0x28, ld r2,40(r12)
Steven Rostedt8fd6e5a2008-11-14 16:21:19 -0800139 */
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800140
Michael Ellerman021376a2009-05-13 20:30:24 +0000141 pr_devel("ip:%lx jumps to %lx r2: %lx", ip, tramp, mod->arch.toc);
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800142
143 /* Find where the trampoline jumps to */
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800144 if (probe_kernel_read(jmp, (void *)tramp, sizeof(jmp))) {
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800145 printk(KERN_ERR "Failed to read %lx\n", tramp);
146 return -EFAULT;
Steven Rostedt8fd6e5a2008-11-14 16:21:19 -0800147 }
148
Michael Ellerman021376a2009-05-13 20:30:24 +0000149 pr_devel(" %08x %08x", jmp[0], jmp[1]);
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800150
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800151 /* verify that this is what we expect it to be */
152 if (((jmp[0] & 0xffff0000) != 0x3d820000) ||
153 ((jmp[1] & 0xffff0000) != 0x398c0000) ||
154 (jmp[2] != 0xf8410028) ||
155 (jmp[3] != 0xe96c0020) ||
156 (jmp[4] != 0xe84c0028)) {
157 printk(KERN_ERR "Not a trampoline\n");
158 return -EINVAL;
159 }
160
Steven Rostedtf25f9072009-02-07 20:22:40 +0000161 /* The bottom half is signed extended */
162 offset = ((unsigned)((unsigned short)jmp[0]) << 16) +
163 (int)((short)jmp[1]);
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800164
Michael Ellerman021376a2009-05-13 20:30:24 +0000165 pr_devel(" %x ", offset);
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800166
167 /* get the address this jumps too */
168 tramp = mod->arch.toc + offset + 32;
Michael Ellerman021376a2009-05-13 20:30:24 +0000169 pr_devel("toc: %lx", tramp);
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800170
171 if (probe_kernel_read(jmp, (void *)tramp, 8)) {
172 printk(KERN_ERR "Failed to read %lx\n", tramp);
173 return -EFAULT;
174 }
175
Michael Ellerman021376a2009-05-13 20:30:24 +0000176 pr_devel(" %08x %08x\n", jmp[0], jmp[1]);
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800177
178 ptr = ((unsigned long)jmp[0] << 32) + jmp[1];
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800179
180 /* This should match what was called */
Michael Ellerman4a9e3f82009-05-28 19:33:34 +0000181 if (ptr != ppc_function_entry((void *)addr)) {
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800182 printk(KERN_ERR "addr does not match %lx\n", ptr);
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800183 return -EINVAL;
184 }
185
186 /*
187 * We want to nop the line, but the next line is
188 * 0xe8, 0x41, 0x00, 0x28 ld r2,40(r1)
189 * This needs to be turned to a nop too.
190 */
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800191 if (probe_kernel_read(&op, (void *)(ip+4), MCOUNT_INSN_SIZE))
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800192 return -EFAULT;
193
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800194 if (op != 0xe8410028) {
195 printk(KERN_ERR "Next line is not ld! (%08x)\n", op);
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800196 return -EINVAL;
197 }
198
199 /*
200 * Milton Miller pointed out that we can not blindly do nops.
201 * If a task was preempted when calling a trace function,
202 * the nops will remove the way to restore the TOC in r2
203 * and the r2 TOC will get corrupted.
204 */
205
206 /*
207 * Replace:
208 * bl <tramp> <==== will be replaced with "b 1f"
209 * ld r2,40(r1)
210 * 1:
211 */
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800212 op = 0x48000008; /* b +8 */
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800213
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800214 if (probe_kernel_write((void *)ip, &op, MCOUNT_INSN_SIZE))
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800215 return -EPERM;
216
Steven Rostedtec682ce2008-11-25 10:22:48 -0800217
218 flush_icache_range(ip, ip + 8);
219
Steven Rostedt8fd6e5a2008-11-14 16:21:19 -0800220 return 0;
221}
222
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800223#else /* !PPC64 */
224static int
225__ftrace_make_nop(struct module *mod,
226 struct dyn_ftrace *rec, unsigned long addr)
227{
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800228 unsigned int op;
229 unsigned int jmp[4];
Steven Rostedt7cc45e62008-11-15 02:39:05 -0500230 unsigned long ip = rec->ip;
231 unsigned long tramp;
Steven Rostedt7cc45e62008-11-15 02:39:05 -0500232
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800233 if (probe_kernel_read(&op, (void *)ip, MCOUNT_INSN_SIZE))
Steven Rostedt7cc45e62008-11-15 02:39:05 -0500234 return -EFAULT;
235
236 /* Make sure that that this is still a 24bit jump */
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800237 if (!is_bl_op(op)) {
238 printk(KERN_ERR "Not expected bl: opcode is %x\n", op);
Steven Rostedt7cc45e62008-11-15 02:39:05 -0500239 return -EINVAL;
240 }
241
242 /* lets find where the pointer goes */
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800243 tramp = find_bl_target(ip, op);
Steven Rostedt7cc45e62008-11-15 02:39:05 -0500244
245 /*
246 * On PPC32 the trampoline looks like:
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800247 * 0x3d, 0x60, 0x00, 0x00 lis r11,sym@ha
248 * 0x39, 0x6b, 0x00, 0x00 addi r11,r11,sym@l
249 * 0x7d, 0x69, 0x03, 0xa6 mtctr r11
250 * 0x4e, 0x80, 0x04, 0x20 bctr
Steven Rostedt7cc45e62008-11-15 02:39:05 -0500251 */
252
Michael Ellerman021376a2009-05-13 20:30:24 +0000253 pr_devel("ip:%lx jumps to %lx", ip, tramp);
Steven Rostedt7cc45e62008-11-15 02:39:05 -0500254
255 /* Find where the trampoline jumps to */
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800256 if (probe_kernel_read(jmp, (void *)tramp, sizeof(jmp))) {
Steven Rostedt7cc45e62008-11-15 02:39:05 -0500257 printk(KERN_ERR "Failed to read %lx\n", tramp);
258 return -EFAULT;
259 }
260
Michael Ellerman021376a2009-05-13 20:30:24 +0000261 pr_devel(" %08x %08x ", jmp[0], jmp[1]);
Steven Rostedt7cc45e62008-11-15 02:39:05 -0500262
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800263 /* verify that this is what we expect it to be */
264 if (((jmp[0] & 0xffff0000) != 0x3d600000) ||
265 ((jmp[1] & 0xffff0000) != 0x396b0000) ||
266 (jmp[2] != 0x7d6903a6) ||
267 (jmp[3] != 0x4e800420)) {
268 printk(KERN_ERR "Not a trampoline\n");
269 return -EINVAL;
270 }
271
272 tramp = (jmp[1] & 0xffff) |
273 ((jmp[0] & 0xffff) << 16);
Steven Rostedt7cc45e62008-11-15 02:39:05 -0500274 if (tramp & 0x8000)
275 tramp -= 0x10000;
276
Michael Ellerman021376a2009-05-13 20:30:24 +0000277 pr_devel(" %lx ", tramp);
Steven Rostedt7cc45e62008-11-15 02:39:05 -0500278
279 if (tramp != addr) {
280 printk(KERN_ERR
281 "Trampoline location %08lx does not match addr\n",
282 tramp);
283 return -EINVAL;
284 }
285
Kumar Gala16c57b32009-02-10 20:10:44 +0000286 op = PPC_INST_NOP;
Steven Rostedt7cc45e62008-11-15 02:39:05 -0500287
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800288 if (probe_kernel_write((void *)ip, &op, MCOUNT_INSN_SIZE))
Steven Rostedt7cc45e62008-11-15 02:39:05 -0500289 return -EPERM;
290
Steven Rostedtec682ce2008-11-25 10:22:48 -0800291 flush_icache_range(ip, ip + 8);
292
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800293 return 0;
294}
295#endif /* PPC64 */
Steven Rostedt17be5b32009-02-05 21:33:09 -0800296#endif /* CONFIG_MODULES */
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800297
298int ftrace_make_nop(struct module *mod,
299 struct dyn_ftrace *rec, unsigned long addr)
Steven Rostedt8fd6e5a2008-11-14 16:21:19 -0800300{
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800301 unsigned long ip = rec->ip;
Steven Rostedtb54dcfe2009-02-13 06:31:39 -0800302 unsigned int old, new;
Steven Rostedt8fd6e5a2008-11-14 16:21:19 -0800303
304 /*
305 * If the calling address is more that 24 bits away,
306 * then we had to use a trampoline to make the call.
307 * Otherwise just update the call site.
308 */
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800309 if (test_24bit_addr(ip, addr)) {
Steven Rostedt8fd6e5a2008-11-14 16:21:19 -0800310 /* within range */
Steven Rostedt46542882009-02-10 22:19:54 -0800311 old = ftrace_call_replace(ip, addr, 1);
Michael Ellerman92e02a52009-05-28 19:33:36 +0000312 new = PPC_INST_NOP;
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800313 return ftrace_modify_code(ip, old, new);
Steven Rostedt8fd6e5a2008-11-14 16:21:19 -0800314 }
315
Steven Rostedt17be5b32009-02-05 21:33:09 -0800316#ifdef CONFIG_MODULES
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800317 /*
318 * Out of range jumps are called from modules.
319 * We should either already have a pointer to the module
320 * or it has been passed in.
321 */
322 if (!rec->arch.mod) {
323 if (!mod) {
324 printk(KERN_ERR "No module loaded addr=%lx\n",
325 addr);
326 return -EFAULT;
327 }
328 rec->arch.mod = mod;
329 } else if (mod) {
330 if (mod != rec->arch.mod) {
331 printk(KERN_ERR
332 "Record mod %p not equal to passed in mod %p\n",
333 rec->arch.mod, mod);
334 return -EINVAL;
335 }
336 /* nothing to do if mod == rec->arch.mod */
337 } else
338 mod = rec->arch.mod;
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800339
340 return __ftrace_make_nop(mod, rec, addr);
Steven Rostedt17be5b32009-02-05 21:33:09 -0800341#else
342 /* We should not get here without modules */
343 return -EINVAL;
344#endif /* CONFIG_MODULES */
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800345}
346
Steven Rostedt17be5b32009-02-05 21:33:09 -0800347#ifdef CONFIG_MODULES
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800348#ifdef CONFIG_PPC64
349static int
350__ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
351{
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800352 unsigned int op[2];
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800353 unsigned long ip = rec->ip;
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800354
355 /* read where this goes */
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800356 if (probe_kernel_read(op, (void *)ip, MCOUNT_INSN_SIZE * 2))
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800357 return -EFAULT;
358
359 /*
360 * It should be pointing to two nops or
361 * b +8; ld r2,40(r1)
362 */
363 if (((op[0] != 0x48000008) || (op[1] != 0xe8410028)) &&
Kumar Gala16c57b32009-02-10 20:10:44 +0000364 ((op[0] != PPC_INST_NOP) || (op[1] != PPC_INST_NOP))) {
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800365 printk(KERN_ERR "Expected NOPs but have %x %x\n", op[0], op[1]);
366 return -EINVAL;
367 }
368
369 /* If we never set up a trampoline to ftrace_caller, then bail */
370 if (!rec->arch.mod->arch.tramp) {
371 printk(KERN_ERR "No ftrace trampoline\n");
372 return -EINVAL;
373 }
374
Steven Rostedt0029ff82008-11-25 14:06:19 -0800375 /* create the branch to the trampoline */
376 op[0] = create_branch((unsigned int *)ip,
377 rec->arch.mod->arch.tramp, BRANCH_SET_LINK);
378 if (!op[0]) {
379 printk(KERN_ERR "REL24 out of range!\n");
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800380 return -EINVAL;
381 }
382
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800383 /* ld r2,40(r1) */
384 op[1] = 0xe8410028;
385
Michael Ellerman021376a2009-05-13 20:30:24 +0000386 pr_devel("write to %lx\n", rec->ip);
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800387
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800388 if (probe_kernel_write((void *)ip, op, MCOUNT_INSN_SIZE * 2))
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800389 return -EPERM;
390
Steven Rostedtec682ce2008-11-25 10:22:48 -0800391 flush_icache_range(ip, ip + 8);
392
Steven Rostedt8fd6e5a2008-11-14 16:21:19 -0800393 return 0;
394}
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800395#else
396static int
397__ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
398{
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800399 unsigned int op;
Steven Rostedt7cc45e62008-11-15 02:39:05 -0500400 unsigned long ip = rec->ip;
Steven Rostedt7cc45e62008-11-15 02:39:05 -0500401
402 /* read where this goes */
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800403 if (probe_kernel_read(&op, (void *)ip, MCOUNT_INSN_SIZE))
Steven Rostedt7cc45e62008-11-15 02:39:05 -0500404 return -EFAULT;
405
406 /* It should be pointing to a nop */
Kumar Gala16c57b32009-02-10 20:10:44 +0000407 if (op != PPC_INST_NOP) {
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800408 printk(KERN_ERR "Expected NOP but have %x\n", op);
Steven Rostedt7cc45e62008-11-15 02:39:05 -0500409 return -EINVAL;
410 }
411
412 /* If we never set up a trampoline to ftrace_caller, then bail */
413 if (!rec->arch.mod->arch.tramp) {
414 printk(KERN_ERR "No ftrace trampoline\n");
415 return -EINVAL;
416 }
417
Steven Rostedt0029ff82008-11-25 14:06:19 -0800418 /* create the branch to the trampoline */
419 op = create_branch((unsigned int *)ip,
420 rec->arch.mod->arch.tramp, BRANCH_SET_LINK);
421 if (!op) {
422 printk(KERN_ERR "REL24 out of range!\n");
Steven Rostedt7cc45e62008-11-15 02:39:05 -0500423 return -EINVAL;
424 }
425
Michael Ellerman021376a2009-05-13 20:30:24 +0000426 pr_devel("write to %lx\n", rec->ip);
Steven Rostedt7cc45e62008-11-15 02:39:05 -0500427
Steven Rostedtd9af12b72008-11-25 06:39:18 -0800428 if (probe_kernel_write((void *)ip, &op, MCOUNT_INSN_SIZE))
Steven Rostedt7cc45e62008-11-15 02:39:05 -0500429 return -EPERM;
430
Steven Rostedtec682ce2008-11-25 10:22:48 -0800431 flush_icache_range(ip, ip + 8);
432
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800433 return 0;
434}
435#endif /* CONFIG_PPC64 */
Steven Rostedt17be5b32009-02-05 21:33:09 -0800436#endif /* CONFIG_MODULES */
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800437
438int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
439{
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800440 unsigned long ip = rec->ip;
Steven Rostedtb54dcfe2009-02-13 06:31:39 -0800441 unsigned int old, new;
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800442
443 /*
444 * If the calling address is more that 24 bits away,
445 * then we had to use a trampoline to make the call.
446 * Otherwise just update the call site.
447 */
448 if (test_24bit_addr(ip, addr)) {
449 /* within range */
Michael Ellerman92e02a52009-05-28 19:33:36 +0000450 old = PPC_INST_NOP;
Steven Rostedt46542882009-02-10 22:19:54 -0800451 new = ftrace_call_replace(ip, addr, 1);
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800452 return ftrace_modify_code(ip, old, new);
453 }
454
Steven Rostedt17be5b32009-02-05 21:33:09 -0800455#ifdef CONFIG_MODULES
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800456 /*
457 * Out of range jumps are called from modules.
458 * Being that we are converting from nop, it had better
459 * already have a module defined.
460 */
461 if (!rec->arch.mod) {
462 printk(KERN_ERR "No module loaded\n");
463 return -EINVAL;
464 }
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800465
466 return __ftrace_make_call(rec, addr);
Steven Rostedt17be5b32009-02-05 21:33:09 -0800467#else
468 /* We should not get here without modules */
469 return -EINVAL;
470#endif /* CONFIG_MODULES */
Steven Rostedtf48cb8b2008-11-14 20:47:03 -0800471}
Steven Rostedt8fd6e5a2008-11-14 16:21:19 -0800472
Steven Rostedt15adc042008-10-23 09:33:08 -0400473int ftrace_update_ftrace_func(ftrace_func_t func)
Steven Rostedt4e491d12008-05-14 23:49:44 -0400474{
475 unsigned long ip = (unsigned long)(&ftrace_call);
Steven Rostedtb54dcfe2009-02-13 06:31:39 -0800476 unsigned int old, new;
Steven Rostedt4e491d12008-05-14 23:49:44 -0400477 int ret;
478
Steven Rostedtb54dcfe2009-02-13 06:31:39 -0800479 old = *(unsigned int *)&ftrace_call;
Steven Rostedt46542882009-02-10 22:19:54 -0800480 new = ftrace_call_replace(ip, (unsigned long)func, 1);
Steven Rostedt4e491d12008-05-14 23:49:44 -0400481 ret = ftrace_modify_code(ip, old, new);
482
483 return ret;
484}
485
Steven Rostedt4e491d12008-05-14 23:49:44 -0400486int __init ftrace_dyn_arch_init(void *data)
487{
Steven Rostedt8fd6e5a2008-11-14 16:21:19 -0800488 /* caller expects data to be zero */
489 unsigned long *p = data;
Steven Rostedt4e491d12008-05-14 23:49:44 -0400490
Steven Rostedt8fd6e5a2008-11-14 16:21:19 -0800491 *p = 0;
Steven Rostedt4e491d12008-05-14 23:49:44 -0400492
493 return 0;
494}
Steven Rostedt6794c782009-02-09 21:10:27 -0800495#endif /* CONFIG_DYNAMIC_FTRACE */
496
497#ifdef CONFIG_FUNCTION_GRAPH_TRACER
498
Steven Rostedt46542882009-02-10 22:19:54 -0800499#ifdef CONFIG_DYNAMIC_FTRACE
500extern void ftrace_graph_call(void);
501extern void ftrace_graph_stub(void);
502
503int ftrace_enable_ftrace_graph_caller(void)
504{
505 unsigned long ip = (unsigned long)(&ftrace_graph_call);
506 unsigned long addr = (unsigned long)(&ftrace_graph_caller);
507 unsigned long stub = (unsigned long)(&ftrace_graph_stub);
Steven Rostedtb54dcfe2009-02-13 06:31:39 -0800508 unsigned int old, new;
Steven Rostedt46542882009-02-10 22:19:54 -0800509
Steven Rostedtb54dcfe2009-02-13 06:31:39 -0800510 old = ftrace_call_replace(ip, stub, 0);
Steven Rostedt46542882009-02-10 22:19:54 -0800511 new = ftrace_call_replace(ip, addr, 0);
512
513 return ftrace_modify_code(ip, old, new);
514}
515
516int ftrace_disable_ftrace_graph_caller(void)
517{
518 unsigned long ip = (unsigned long)(&ftrace_graph_call);
519 unsigned long addr = (unsigned long)(&ftrace_graph_caller);
520 unsigned long stub = (unsigned long)(&ftrace_graph_stub);
Steven Rostedtb54dcfe2009-02-13 06:31:39 -0800521 unsigned int old, new;
Steven Rostedt46542882009-02-10 22:19:54 -0800522
Steven Rostedtb54dcfe2009-02-13 06:31:39 -0800523 old = ftrace_call_replace(ip, addr, 0);
Steven Rostedt46542882009-02-10 22:19:54 -0800524 new = ftrace_call_replace(ip, stub, 0);
525
526 return ftrace_modify_code(ip, old, new);
527}
528#endif /* CONFIG_DYNAMIC_FTRACE */
529
Steven Rostedtbb725342009-02-11 12:45:49 -0800530#ifdef CONFIG_PPC64
531extern void mod_return_to_handler(void);
532#endif
533
Steven Rostedt6794c782009-02-09 21:10:27 -0800534/*
535 * Hook the return address and push it in the stack of return addrs
536 * in current thread info.
537 */
538void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr)
539{
540 unsigned long old;
Steven Rostedt6794c782009-02-09 21:10:27 -0800541 int faulted;
542 struct ftrace_graph_ent trace;
Steven Rostedtbb725342009-02-11 12:45:49 -0800543 unsigned long return_hooker = (unsigned long)&return_to_handler;
Steven Rostedt6794c782009-02-09 21:10:27 -0800544
545 if (unlikely(atomic_read(&current->tracing_graph_pause)))
546 return;
547
Michael Ellermanf4952f62009-04-06 04:40:45 +0000548#ifdef CONFIG_PPC64
Steven Rostedtbb725342009-02-11 12:45:49 -0800549 /* non core kernel code needs to save and restore the TOC */
550 if (REGION_ID(self_addr) != KERNEL_REGION_ID)
551 return_hooker = (unsigned long)&mod_return_to_handler;
552#endif
553
Michael Ellerman4a9e3f82009-05-28 19:33:34 +0000554 return_hooker = ppc_function_entry((void *)return_hooker);
Steven Rostedt6794c782009-02-09 21:10:27 -0800555
556 /*
557 * Protect against fault, even if it shouldn't
558 * happen. This tool is too much intrusive to
559 * ignore such a protection.
560 */
561 asm volatile(
562 "1: " PPC_LL "%[old], 0(%[parent])\n"
563 "2: " PPC_STL "%[return_hooker], 0(%[parent])\n"
564 " li %[faulted], 0\n"
Steven Rostedtfad4f472009-02-11 19:10:57 -0500565 "3:\n"
Steven Rostedt6794c782009-02-09 21:10:27 -0800566
567 ".section .fixup, \"ax\"\n"
568 "4: li %[faulted], 1\n"
569 " b 3b\n"
570 ".previous\n"
571
572 ".section __ex_table,\"a\"\n"
573 PPC_LONG_ALIGN "\n"
574 PPC_LONG "1b,4b\n"
575 PPC_LONG "2b,4b\n"
576 ".previous"
577
Steven Rostedtc3cf8662009-05-15 04:33:54 +0000578 : [old] "=&r" (old), [faulted] "=r" (faulted)
Steven Rostedt6794c782009-02-09 21:10:27 -0800579 : [parent] "r" (parent), [return_hooker] "r" (return_hooker)
580 : "memory"
581 );
582
583 if (unlikely(faulted)) {
584 ftrace_graph_stop();
585 WARN_ON(1);
586 return;
587 }
588
Steven Rostedt71e308a2009-06-18 12:45:08 -0400589 if (ftrace_push_return_trace(old, self_addr, &trace.depth, 0) == -EBUSY) {
Steven Rostedt6794c782009-02-09 21:10:27 -0800590 *parent = old;
591 return;
592 }
593
594 trace.func = self_addr;
595
596 /* Only trace if the calling function expects to */
597 if (!ftrace_graph_entry(&trace)) {
598 current->curr_ret_stack--;
599 *parent = old;
600 }
601}
602#endif /* CONFIG_FUNCTION_GRAPH_TRACER */